diff --git a/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl b/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl new file mode 100644 index 0000000000..2d40543e47 --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl @@ -0,0 +1,28 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema_b: + catalog_name: main + name: test-schema-b-$UNIQUE_NAME + grants: + - principal: account users + privileges: + - USE_SCHEMA + - principal: admins + privileges: + - CREATE_TABLE + - USE_SCHEMA + + schema_a: + catalog_name: main + name: test-schema-a-$UNIQUE_NAME + grants: + # Reference principal and privileges from schema_b by index + - principal: ${resources.schemas.schema_b.grants[0].principal} + privileges: + - USE_SCHEMA + - principal: ${resources.schemas.schema_b.grants[1].principal} + privileges: + - CREATE_TABLE diff --git a/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl b/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl new file mode 100644 index 0000000000..d9aebda0ce --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + foo: + catalog_name: main + name: test-schema-$UNIQUE_NAME + grants: + - principal: account users + privileges: + - USE_SCHEMA diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index a83f46df49..ba32b1f599 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 48ed3d4977..7bee328d23 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,7 +1,7 @@ Cloud = false Slow = true -# Cross-resource permission references (${resources.jobs.X.permissions[N].field}) require -# permissions to be part of the job schema, which was added after v0.293.0. +# $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] +EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 17f14550ca..725019505d 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 781987f7ca..a29f9dbb5a 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -13,3 +13,6 @@ EnvMatrixExclude.no_secret_scope = ["INPUT_CONFIG=secret_scope.yml.tmpl"] # ends up as the permission level value. EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] + +# Grant cross-references require the EmbeddedSlice pattern not present in terraform mode. +EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 17f14550ca..725019505d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index b87f1ec7e5..36badf07e5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -45,6 +45,8 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl", diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 2046d78c9d..7a24ba0f3c 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -24,7 +24,7 @@ "state": { "securable_type": "function", "full_name": "main.schema_grants.mymodel", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -52,7 +52,7 @@ "state": { "securable_type": "schema", "full_name": "main.schema_grants", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -89,7 +89,7 @@ "state": { "securable_type": "volume", "full_name": "main.schema_grants.volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index b02419eb75..97dfa44414 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -253,6 +253,10 @@ resources.catalogs.*.updated_by string REMOTE resources.catalogs.*.url string INPUT resources.catalogs.*.grants.full_name string ALL resources.catalogs.*.grants.securable_type string ALL +resources.catalogs.*.grants[*] catalog.PrivilegeAssignment ALL +resources.catalogs.*.grants[*].principal string ALL +resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL +resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL resources.clusters.*.apply_policy_default_values bool INPUT STATE resources.clusters.*.autoscale *compute.AutoScale ALL resources.clusters.*.autoscale.max_workers int ALL @@ -683,6 +687,10 @@ resources.external_locations.*.updated_by string REMOTE resources.external_locations.*.url string ALL resources.external_locations.*.grants.full_name string ALL resources.external_locations.*.grants.securable_type string ALL +resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL +resources.external_locations.*.grants[*].principal string ALL +resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL +resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL @@ -2680,6 +2688,10 @@ resources.registered_models.*.updated_by string ALL resources.registered_models.*.url string INPUT resources.registered_models.*.grants.full_name string ALL resources.registered_models.*.grants.securable_type string ALL +resources.registered_models.*.grants[*] catalog.PrivilegeAssignment ALL +resources.registered_models.*.grants[*].principal string ALL +resources.registered_models.*.grants[*].privileges []catalog.Privilege ALL +resources.registered_models.*.grants[*].privileges[*] catalog.Privilege ALL resources.schemas.*.browse_only bool REMOTE resources.schemas.*.catalog_name string ALL resources.schemas.*.catalog_type catalog.CatalogType REMOTE @@ -2709,6 +2721,10 @@ resources.schemas.*.updated_by string REMOTE resources.schemas.*.url string INPUT resources.schemas.*.grants.full_name string ALL resources.schemas.*.grants.securable_type string ALL +resources.schemas.*.grants[*] catalog.PrivilegeAssignment ALL +resources.schemas.*.grants[*].principal string ALL +resources.schemas.*.grants[*].privileges []catalog.Privilege ALL +resources.schemas.*.grants[*].privileges[*] catalog.Privilege ALL resources.secret_scopes.*.backend_azure_keyvault *workspace.AzureKeyVaultSecretScopeMetadata STATE resources.secret_scopes.*.backend_azure_keyvault.dns_name string STATE resources.secret_scopes.*.backend_azure_keyvault.resource_id string STATE @@ -2871,3 +2887,7 @@ resources.volumes.*.volume_id string REMOTE resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL resources.volumes.*.grants.securable_type string ALL +resources.volumes.*.grants[*] catalog.PrivilegeAssignment ALL +resources.volumes.*.grants[*].principal string ALL +resources.volumes.*.grants[*].privileges []catalog.Privilege ALL +resources.volumes.*.grants[*].privileges[*] catalog.Privilege ALL diff --git a/acceptance/bundle/resource_deps/grant_ref/databricks.yml b/acceptance/bundle/resource_deps/grant_ref/databricks.yml new file mode 100644 index 0000000000..4b7a4bd961 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: test-bundle + +resources: + schemas: + # schema_b has grants that schema_a references + schema_b: + catalog_name: main + name: schema B + grants: + - principal: viewers + privileges: + - USE_SCHEMA + - principal: editors + privileges: + - CREATE_TABLE + - USE_SCHEMA + + # schema_a references schema_b's grant principals + schema_a: + catalog_name: main + name: schema A + grants: + # Reference by integer index + - principal: ${resources.schemas.schema_b.grants[0].principal} + privileges: + - USE_SCHEMA + # Reference by integer index (second entry) + - principal: ${resources.schemas.schema_b.grants[1].principal} + privileges: + - CREATE_TABLE diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json new file mode 100644 index 0000000000..970fab66ae --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -0,0 +1,97 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.schemas.schema_a": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "schema A" + } + } + }, + "resources.schemas.schema_a.grants": { + "depends_on": [ + { + "node": "resources.schemas.schema_a", + "label": "${resources.schemas.schema_a.id}" + }, + { + "node": "resources.schemas.schema_b.grants", + "label": "${resources.schemas.schema_b.grants[0].principal}" + }, + { + "node": "resources.schemas.schema_b.grants", + "label": "${resources.schemas.schema_b.grants[1].principal}" + } + ], + "action": "create", + "new_state": { + "value": { + "securable_type": "schema", + "full_name": "", + "__embed__": [ + { + "principal": "viewers", + "privileges": [ + "USE_SCHEMA" + ] + }, + { + "principal": "editors", + "privileges": [ + "CREATE_TABLE" + ] + } + ] + }, + "vars": { + "full_name": "${resources.schemas.schema_a.id}" + } + } + }, + "resources.schemas.schema_b": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "schema B" + } + } + }, + "resources.schemas.schema_b.grants": { + "depends_on": [ + { + "node": "resources.schemas.schema_b", + "label": "${resources.schemas.schema_b.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "securable_type": "schema", + "full_name": "", + "__embed__": [ + { + "principal": "viewers", + "privileges": [ + "USE_SCHEMA" + ] + }, + { + "principal": "editors", + "privileges": [ + "CREATE_TABLE", + "USE_SCHEMA" + ] + } + ] + }, + "vars": { + "full_name": "${resources.schemas.schema_b.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/output.txt b/acceptance/bundle/resource_deps/grant_ref/output.txt new file mode 100644 index 0000000000..47cfaccbcb --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle plan +create schemas.schema_a +create schemas.schema_a.grants +create schemas.schema_b +create schemas.schema_b.grants + +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script new file mode 100644 index 0000000000..82800b1654 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -0,0 +1,4 @@ + +trace $CLI bundle plan +trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/grant_ref/test.toml b/acceptance/bundle/resource_deps/grant_ref/test.toml new file mode 100644 index 0000000000..790c13e6dc --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/test.toml @@ -0,0 +1,4 @@ +RecordRequests = false + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 6f50fd2b5f..333e2928d5 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -22,7 +22,7 @@ "new_state": { "value": { "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 6996fb49e5..a63b52c327 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -25,7 +25,7 @@ "new_state": { "value": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -38,7 +38,7 @@ }, "remote_state": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -49,13 +49,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "CREATE_SCHEMA", "new": "USE_CATALOG", "remote": "CREATE_SCHEMA" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "USE_CATALOG", "new": "USE_SCHEMA", diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index b5ab8d0b9d..482faa449f 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "function", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 01d2d581b2..c8d1428519 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index 7b08263765..7232642adc 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -29,7 +29,7 @@ "value": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -43,7 +43,7 @@ "remote_state": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -54,13 +54,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "CREATE_TABLE", "new": "APPLY_TAG", "remote": "CREATE_TABLE" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "USE_SCHEMA", "new": "CREATE_TABLE", diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt index a259b5603f..2a48dc2405 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com" } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index 2f9e4d51a1..daae7c36ac 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -40,7 +40,7 @@ "value": { "securable_type": "volume", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index a6709a074f..4de57d5ed6 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -56,7 +56,7 @@ "value": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -70,7 +70,7 @@ "remote_state": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -81,13 +81,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "READ_VOLUME", "new": "MANAGE", "remote": "READ_VOLUME" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "WRITE_VOLUME", "new": "READ_VOLUME", diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 538bc05d47..da59fedbe0 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -508,7 +508,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "catalog", FullName: "mycatalog", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeUseCatalog}, Principal: "user@example.com", }}, @@ -519,7 +519,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "external_location", FullName: "myexternallocation", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeReadFiles}, Principal: "user@example.com", }}, @@ -530,7 +530,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "schema", FullName: "main.myschema", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, @@ -541,7 +541,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "volume", FullName: "main.myschema.myvolume", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, @@ -552,7 +552,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "registered-model", FullName: "modelid", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 8c0989aa99..7a1f169938 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -23,7 +23,7 @@ var grantResourceToSecurableType = map[string]string{ type GrantsState struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` - Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` + EmbeddedSlice []catalog.PrivilegeAssignment `json:"__embed__,omitempty"` } func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVar, error) { @@ -56,7 +56,7 @@ func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVa Value: &GrantsState{ SecurableType: securableType, FullName: "", - Grants: *grantsPtr, + EmbeddedSlice: *grantsPtr, }, Refs: map[string]string{ "full_name": "${" + baseNode + ".id}", @@ -90,7 +90,7 @@ func (r *ResourceGrants) DoRead(ctx context.Context, id string) (*GrantsState, e return &GrantsState{ SecurableType: securableType, FullName: fullName, - Grants: assignments, + EmbeddedSlice: assignments, }, nil } @@ -121,7 +121,7 @@ func (r *ResourceGrants) applyGrants(ctx context.Context, state *GrantsState) er var changes []catalog.PermissionsChange // For each principal in the config, add their grants and remove everything else - for _, grantAssignment := range state.Grants { + for _, grantAssignment := range state.EmbeddedSlice { changes = append(changes, catalog.PermissionsChange{ Principal: grantAssignment.Principal, Add: grantAssignment.Privileges, diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 02590794d4..7a64f53d4b 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -48,7 +48,8 @@ type StatePermission struct { // by unrelated types and it's harder to evaluate that fixed string because it's non-local. type PermissionsState struct { - ObjectID string `json:"object_id"` + ObjectID string `json:"object_id"` + // By convention, EmbedSlice fields should have __embed__ json tag, see permissions.go for details EmbeddedSlice []StatePermission `json:"__embed__,omitempty"` } diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 41bf90fc2d..381d63a12e 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,9 +3,10 @@ package dstate import ( "encoding/json" "fmt" - "strings" "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -40,18 +41,38 @@ var migrations = map[int]func(*Database) error{ 1: migrateV1ToV2, } -// migrateV1ToV2 migrates permissions entries from the old format -// (iam.AccessControlRequest with "permissions" key and "permission_level" field) -// to the new format (StatePermission with "_" key and "level" field). +// migrateV1ToV2 migrates permissions and grants entries from the old format +// to the new format (__embed__ keys, permission_level -> level). func migrateV1ToV2(db *Database) error { for key, entry := range db.State { - if !strings.HasSuffix(key, ".permissions") { + if len(entry.State) == 0 { continue } - if len(entry.State) == 0 { + + path, pathErr := structpath.ParsePath(key) + if pathErr != nil || path == nil { + continue + } + // path points to the last node; read its key directly. + lastKey, ok := path.StringKey() + if !ok { continue } - migrated, err := migratePermissionsEntry(entry.State) + + var ( + migrated json.RawMessage + err error + ) + + switch lastKey { + case "permissions": + migrated, err = migratePermissionsEntry(entry.State) + case "grants": + migrated, err = migrateGrantsEntry(entry.State) + default: + continue + } + if err != nil { return fmt.Errorf("migrating %s: %w", key, err) } @@ -80,11 +101,6 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return nil, err } - // If old format had no permissions, try parsing as new format (might already be migrated). - if len(old.Permissions) == 0 { - return raw, nil - } - newState := dresources.PermissionsState{ ObjectID: old.ObjectID, } @@ -97,5 +113,27 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { }) } - return json.MarshalIndent(newState, " ", " ") + return json.Marshal(newState) +} + +// oldGrantsStateV1 is the grants state format before v2. +type oldGrantsStateV1 struct { + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` +} + +func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { + var old oldGrantsStateV1 + if err := json.Unmarshal(raw, &old); err != nil { + return nil, err + } + + newState := dresources.GrantsState{ + SecurableType: old.SecurableType, + FullName: old.FullName, + EmbeddedSlice: old.Grants, + } + + return json.Marshal(newState) }