direct: support references to/from grants

acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl

@@ -0,0 +1,28 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    schema_b:
+      catalog_name: main
+      name: test-schema-b-$UNIQUE_NAME
+      grants:
+        - principal: account users
+          privileges:
+            - USE_SCHEMA
+        - principal: admins
+          privileges:
+            - CREATE_TABLE
+            - USE_SCHEMA
+
+    schema_a:
+      catalog_name: main
+      name: test-schema-a-$UNIQUE_NAME
+      grants:
+        # Reference principal and privileges from schema_b by index
+        - principal: ${resources.schemas.schema_b.grants[0].principal}
+          privileges:
+            - USE_SCHEMA
+        - principal: ${resources.schemas.schema_b.grants[1].principal}
+          privileges:
+            - CREATE_TABLE

acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    foo:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME
+      grants:
+        - principal: account users
+          privileges:
+            - USE_SCHEMA

acceptance/bundle/invariant/continue_293/test.toml

@@ -1,7 +1,7 @@
 Cloud = false
 Slow = true
 
-# Cross-resource permission references (${resources.jobs.X.permissions[N].field}) require
-# permissions to be part of the job schema, which was added after v0.293.0.
+# $resources references to permissions and grants are not supported on v0.293.0
 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"]
 EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"]
+EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"]

acceptance/bundle/invariant/migrate/test.toml

@@ -13,3 +13,6 @@ EnvMatrixExclude.no_secret_scope = ["INPUT_CONFIG=secret_scope.yml.tmpl"]
 # ends up as the permission level value.
 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"]
 EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"]
+
+# Grant cross-references require the EmbeddedSlice pattern not present in terraform mode.
+EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -45,6 +45,8 @@ EnvMatrix.INPUT_CONFIG = [
     "postgres_project.yml.tmpl",
     "registered_model.yml.tmpl",
     "schema.yml.tmpl",
+    "schema_grant_ref.yml.tmpl",
+    "schema_with_grants.yml.tmpl",
     "secret_scope.yml.tmpl",
     "synced_database_table.yml.tmpl",
     "volume.yml.tmpl",

acceptance/bundle/migrate/grants/out.new_state.json

@@ -24,7 +24,7 @@
    "state": {
     "securable_type": "function",
     "full_name": "main.schema_grants.mymodel",
-    "grants": [
+    "__embed__": [
      {
       "principal": "deco-test-user@databricks.com",
       "privileges": [
@@ -52,7 +52,7 @@
    "state": {
     "securable_type": "schema",
     "full_name": "main.schema_grants",
-    "grants": [
+    "__embed__": [
      {
       "principal": "deco-test-user@databricks.com",
       "privileges": [
@@ -89,7 +89,7 @@
    "state": {
     "securable_type": "volume",
     "full_name": "main.schema_grants.volume_name",
-    "grants": [
+    "__embed__": [
      {
       "principal": "deco-test-user@databricks.com",
       "privileges": [

acceptance/bundle/refschema/out.fields.txt

@@ -253,6 +253,10 @@ resources.catalogs.*.updated_by	string	REMOTE
 resources.catalogs.*.url	string	INPUT
 resources.catalogs.*.grants.full_name	string	ALL
 resources.catalogs.*.grants.securable_type	string	ALL
+resources.catalogs.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.catalogs.*.grants[*].principal	string	ALL
+resources.catalogs.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.catalogs.*.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.clusters.*.apply_policy_default_values	bool	INPUT	STATE
 resources.clusters.*.autoscale	*compute.AutoScale	ALL
 resources.clusters.*.autoscale.max_workers	int	ALL
@@ -683,6 +687,10 @@ resources.external_locations.*.updated_by	string	REMOTE
 resources.external_locations.*.url	string	ALL
 resources.external_locations.*.grants.full_name	string	ALL
 resources.external_locations.*.grants.securable_type	string	ALL
+resources.external_locations.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.external_locations.*.grants[*].principal	string	ALL
+resources.external_locations.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.external_locations.*.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.jobs.*.budget_policy_id	string	ALL
 resources.jobs.*.continuous	*jobs.Continuous	ALL
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	ALL
@@ -2680,6 +2688,10 @@ resources.registered_models.*.updated_by	string	ALL
 resources.registered_models.*.url	string	INPUT
 resources.registered_models.*.grants.full_name	string	ALL
 resources.registered_models.*.grants.securable_type	string	ALL
+resources.registered_models.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.registered_models.*.grants[*].principal	string	ALL
+resources.registered_models.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.registered_models.*.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.schemas.*.browse_only	bool	REMOTE
 resources.schemas.*.catalog_name	string	ALL
 resources.schemas.*.catalog_type	catalog.CatalogType	REMOTE
@@ -2709,6 +2721,10 @@ resources.schemas.*.updated_by	string	REMOTE
 resources.schemas.*.url	string	INPUT
 resources.schemas.*.grants.full_name	string	ALL
 resources.schemas.*.grants.securable_type	string	ALL
+resources.schemas.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.schemas.*.grants[*].principal	string	ALL
+resources.schemas.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.schemas.*.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.secret_scopes.*.backend_azure_keyvault	*workspace.AzureKeyVaultSecretScopeMetadata	STATE
 resources.secret_scopes.*.backend_azure_keyvault.dns_name	string	STATE
 resources.secret_scopes.*.backend_azure_keyvault.resource_id	string	STATE
@@ -2871,3 +2887,7 @@ resources.volumes.*.volume_id	string	REMOTE
 resources.volumes.*.volume_type	catalog.VolumeType	ALL
 resources.volumes.*.grants.full_name	string	ALL
 resources.volumes.*.grants.securable_type	string	ALL
+resources.volumes.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.volumes.*.grants[*].principal	string	ALL
+resources.volumes.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.volumes.*.grants[*].privileges[*]	catalog.Privilege	ALL

acceptance/bundle/resource_deps/grant_ref/databricks.yml

@@ -0,0 +1,31 @@
+bundle:
+  name: test-bundle
+
+resources:
+  schemas:
+    # schema_b has grants that schema_a references
+    schema_b:
+      catalog_name: main
+      name: schema B
+      grants:
+        - principal: viewers
+          privileges:
+            - USE_SCHEMA
+        - principal: editors
+          privileges:
+            - CREATE_TABLE
+            - USE_SCHEMA
+
+    # schema_a references schema_b's grant principals
+    schema_a:
+      catalog_name: main
+      name: schema A
+      grants:
+        # Reference by integer index
+        - principal: ${resources.schemas.schema_b.grants[0].principal}
+          privileges:
+            - USE_SCHEMA
+        # Reference by integer index (second entry)
+        - principal: ${resources.schemas.schema_b.grants[1].principal}
+          privileges:
+            - CREATE_TABLE

acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json

@@ -0,0 +1,97 @@
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.schemas.schema_a": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "catalog_name": "main",
+          "name": "schema A"
+        }
+      }
+    },
+    "resources.schemas.schema_a.grants": {
+      "depends_on": [
+        {
+          "node": "resources.schemas.schema_a",
+          "label": "${resources.schemas.schema_a.id}"
+        },
+        {
+          "node": "resources.schemas.schema_b.grants",
+          "label": "${resources.schemas.schema_b.grants[0].principal}"
+        },
+        {
+          "node": "resources.schemas.schema_b.grants",
+          "label": "${resources.schemas.schema_b.grants[1].principal}"
+        }
+      ],
+      "action": "create",
+      "new_state": {
+        "value": {
+          "securable_type": "schema",
+          "full_name": "",
+          "__embed__": [
+            {
+              "principal": "viewers",
+              "privileges": [
+                "USE_SCHEMA"
+              ]
+            },
+            {
+              "principal": "editors",
+              "privileges": [
+                "CREATE_TABLE"
+              ]
+            }
+          ]
+        },
+        "vars": {
+          "full_name": "${resources.schemas.schema_a.id}"
+        }
+      }
+    },
+    "resources.schemas.schema_b": {
+      "action": "create",
+      "new_state": {
+        "value": {
+          "catalog_name": "main",
+          "name": "schema B"
+        }
+      }
+    },
+    "resources.schemas.schema_b.grants": {
+      "depends_on": [
+        {
+          "node": "resources.schemas.schema_b",
+          "label": "${resources.schemas.schema_b.id}"
+        }
+      ],
+      "action": "create",
+      "new_state": {
+        "value": {
+          "securable_type": "schema",
+          "full_name": "",
+          "__embed__": [
+            {
+              "principal": "viewers",
+              "privileges": [
+                "USE_SCHEMA"
+              ]
+            },
+            {
+              "principal": "editors",
+              "privileges": [
+                "CREATE_TABLE",
+                "USE_SCHEMA"
+              ]
+            }
+          ]
+        },
+        "vars": {
+          "full_name": "${resources.schemas.schema_b.id}"
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resource_deps/grant_ref/output.txt

@@ -0,0 +1,16 @@
+
+>>> [CLI] bundle plan
+create schemas.schema_a
+create schemas.schema_a.grants
+create schemas.schema_b
+create schemas.schema_b.grants
+
+Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle plan -o json
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!

acceptance/bundle/resource_deps/grant_ref/script

@@ -0,0 +1,4 @@
+
+trace $CLI bundle plan
+trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle deploy

acceptance/bundle/resource_deps/grant_ref/test.toml

@@ -0,0 +1,4 @@
+RecordRequests = false
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json

@@ -22,7 +22,7 @@
       "new_state": {
         "value": {
           "full_name": "",
-          "grants": [
+          "__embed__": [
             {
               "principal": "deco-test-user@databricks.com",
               "privileges": [

acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json

@@ -25,7 +25,7 @@
       "new_state": {
         "value": {
           "full_name": "catalog_grants_[UNIQUE_NAME]",
-          "grants": [
+          "__embed__": [
             {
               "principal": "deco-test-user@databricks.com",
               "privileges": [
@@ -38,7 +38,7 @@
       },
       "remote_state": {
         "full_name": "catalog_grants_[UNIQUE_NAME]",
-        "grants": [
+        "__embed__": [
           {
             "principal": "deco-test-user@databricks.com",
             "privileges": [
@@ -49,13 +49,13 @@
         ]
       },
       "changes": {
-        "grants[0].privileges[0]": {
+        "[0].privileges[0]": {
           "action": "update",
           "old": "CREATE_SCHEMA",
           "new": "USE_CATALOG",
           "remote": "CREATE_SCHEMA"
         },
-        "grants[0].privileges[1]": {
+        "[0].privileges[1]": {
           "action": "update",
           "old": "USE_CATALOG",
           "new": "USE_SCHEMA",

acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json

@@ -31,7 +31,7 @@
         "value": {
           "securable_type": "function",
           "full_name": "",
-          "grants": [
+          "__embed__": [
             {
               "principal": "deco-test-user@databricks.com",
               "privileges": [

acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json

@@ -23,7 +23,7 @@
         "value": {
           "securable_type": "schema",
           "full_name": "",
-          "grants": [
+          "__embed__": [
             {
               "principal": "deco-test-user@databricks.com",
               "privileges": [