direct: grants: revoke removed principals + pass PlanEntry to DoUpdate

NEXT_CHANGELOG.md

@@ -11,6 +11,7 @@
 * engine/direct: Fix 400 error when deploying grants with ALL_PRIVILEGES ([#4801](https://github.com/databricks/cli/pull/4801))
 * Deduplicate grant entries with duplicate principals or privileges during initialization ([#4801](https://github.com/databricks/cli/pull/4801))
 * engine/direct: Fix unwanted recreation of secret scopes when scope_backend_type is not set ([#4834](https://github.com/databricks/cli/pull/4834))
+* engine/direct: Fix deploying removed principals ([#4824](https://github.com/databricks/cli/pull/4824))
 
 ### Dependency updates
 

acceptance/bin/gron.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import argparse
 import json
 import sys
 from pathlib import Path
@@ -7,7 +8,7 @@
 from print_requests import read_json_many
 
 
-def gron(obj, path="json"):
+def gron(obj, path="json", noindex=False):
     """Flatten JSON into greppable assignments.
 
     The path parameter defaults to "json" to match the original gron tool,
@@ -26,6 +27,10 @@ def gron(obj, path="json"):
     json.items[0] = "apple";
     json.items[1] = "banana";
 
+    >>> gron({"items": ["apple", "banana"]}, noindex=True)
+    json.items[] = "apple";
+    json.items[] = "banana";
+
     >>> gron({"tasks": [{"libraries": [{"whl": "file.whl"}]}]})
     json.tasks[0].libraries[0].whl = "file.whl";
 
@@ -38,31 +43,34 @@ def gron(obj, path="json"):
             print(f"{path} = {{}};")
         else:
             for key in obj:
-                gron(obj[key], f"{path}.{key}")
+                gron(obj[key], f"{path}.{key}", noindex=noindex)
     elif isinstance(obj, list):
         if not obj:
             print(f"{path} = [];")
         else:
             for i, item in enumerate(obj):
-                gron(item, f"{path}[{i}]")
+                index = "[]" if noindex else f"[{i}]"
+                gron(item, f"{path}{index}", noindex=noindex)
     else:
         print(f"{path} = {json.dumps(obj)};")
 
 
 def main():
-    if len(sys.argv) > 1:
-        with open(sys.argv[1]) as f:
-            content = f.read()
-        data = read_json_many(content)
-        if len(data) == 1:
-            data = data[0]
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--noindex", action="store_true")
+    parser.add_argument("file", nargs="?")
+    args = parser.parse_args()
+
+    if args.file:
+        content = Path(args.file).read_text()
     else:
         content = sys.stdin.read()
-        data = read_json_many(content)
-        if len(data) == 1:
-            data = data[0]
 
-    gron(data)
+    data = read_json_many(content)
+    if len(data) == 1:
+        data = data[0]
+
+    gron(data, noindex=args.noindex)
 
 
 if __name__ == "__main__":

acceptance/bundle/migrate/grants/out.original_state.json

@@ -180,15 +180,15 @@
           "attributes": {
             "catalog_name": "main",
             "comment": null,
-            "enable_predictive_optimization": null,
+            "enable_predictive_optimization": "INHERIT",
             "force_destroy": true,
             "id": "main.schema_grants",
-            "metastore_id": null,
+            "metastore_id": "[UUID]",
             "name": "schema_grants",
             "owner": "[USERNAME]",
             "properties": null,
             "provider_config": [],
-            "schema_id": "",
+            "schema_id": "[UUID]",
             "storage_root": null
           },
           "sensitive_attributes": [],

acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json

@@ -12,10 +12,18 @@
         "catalog_type": "MANAGED_CATALOG",
         "created_at": [UNIX_TIME_MILLIS][0],
         "created_by": "[USERNAME]",
+        "effective_predictive_optimization_flag": {
+          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_type": "METASTORE",
+          "value": "ENABLE"
+        },
+        "enable_predictive_optimization": "INHERIT",
         "full_name": "main.schema_dup_grants_[UNIQUE_NAME]",
+        "metastore_id": "[UUID]",
         "name": "schema_dup_grants_[UNIQUE_NAME]",
         "owner": "[USERNAME]",
-        "updated_at": [UNIX_TIME_MILLIS][0],
+        "schema_id": "[UUID]",
+        "updated_at": [UNIX_TIME_MILLIS][1],
         "updated_by": "[USERNAME]"
       }
     },

acceptance/bundle/resources/grants/schemas/out_of_band_principal/databricks.yml.tmpl

@@ -0,0 +1,12 @@
+bundle:
+  name: schema-grants-out-of-band-principal-$UNIQUE_NAME
+
+resources:
+  schemas:
+    grants_schema:
+      name: schema_out_of_band_principal_$UNIQUE_NAME
+      catalog_name: main
+      grants:
+        - principal: $CURRENT_USER_NAME
+          privileges:
+            - CREATE_TABLE

acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json

@@ -0,0 +1,83 @@
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 1,
+  "plan": {
+    "resources.schemas.grants_schema": {
+      "action": "skip",
+      "remote_state": {
+        "browse_only": false,
+        "catalog_name": "main",
+        "catalog_type": "MANAGED_CATALOG",
+        "created_at": [UNIX_TIME_MILLIS][0],
+        "created_by": "[USERNAME]",
+        "effective_predictive_optimization_flag": {
+          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_type": "METASTORE",
+          "value": "ENABLE"
+        },
+        "enable_predictive_optimization": "INHERIT",
+        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
+        "metastore_id": "[UUID]",
+        "name": "schema_out_of_band_principal_[UNIQUE_NAME]",
+        "owner": "[USERNAME]",
+        "schema_id": "[UUID]",
+        "updated_at": [UNIX_TIME_MILLIS][1],
+        "updated_by": "[USERNAME]"
+      }
+    },
+    "resources.schemas.grants_schema.grants": {
+      "depends_on": [
+        {
+          "node": "resources.schemas.grants_schema",
+          "label": "${resources.schemas.grants_schema.id}"
+        }
+      ],
+      "action": "update",
+      "new_state": {
+        "value": {
+          "securable_type": "schema",
+          "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
+          "__embed__": [
+            {
+              "principal": "[USERNAME]",
+              "privileges": [
+                "CREATE_TABLE"
+              ]
+            }
+          ]
+        }
+      },
+      "remote_state": {
+        "securable_type": "schema",
+        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
+        "__embed__": [
+          {
+            "principal": "[USERNAME]",
+            "privileges": [
+              "CREATE_TABLE"
+            ]
+          },
+          {
+            "principal": "deco-test-user@databricks.com",
+            "privileges": [
+              "USE_SCHEMA"
+            ]
+          }
+        ]
+      },
+      "changes": {
+        "[principal='deco-test-user@databricks.com']": {
+          "action": "update",
+          "remote": {
+            "principal": "deco-test-user@databricks.com",
+            "privileges": [
+              "USE_SCHEMA"
+            ]
+          }
+        }
+      }
+    }
+  }
+}

acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.terraform.json

@@ -0,0 +1,11 @@
+{
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.schemas.grants_schema": {
+      "action": "skip"
+    },
+    "resources.schemas.grants_schema.grants": {
+      "action": "update"
+    }
+  }
+}

acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json

@@ -0,0 +1,52 @@
+{
+  "plan_version": 2,
+  "cli_version": "[DEV_VERSION]",
+  "lineage": "[UUID]",
+  "serial": 2,
+  "plan": {
+    "resources.schemas.grants_schema": {
+      "action": "skip",
+      "remote_state": {
+        "browse_only": false,
+        "catalog_name": "main",
+        "catalog_type": "MANAGED_CATALOG",
+        "created_at": [UNIX_TIME_MILLIS][0],
+        "created_by": "[USERNAME]",
+        "effective_predictive_optimization_flag": {
+          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_type": "METASTORE",
+          "value": "ENABLE"
+        },
+        "enable_predictive_optimization": "INHERIT",
+        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
+        "metastore_id": "[UUID]",
+        "name": "schema_out_of_band_principal_[UNIQUE_NAME]",
+        "owner": "[USERNAME]",
+        "schema_id": "[UUID]",
+        "updated_at": [UNIX_TIME_MILLIS][1],
+        "updated_by": "[USERNAME]"
+      }
+    },
+    "resources.schemas.grants_schema.grants": {
+      "depends_on": [
+        {
+          "node": "resources.schemas.grants_schema",
+          "label": "${resources.schemas.grants_schema.id}"
+        }
+      ],
+      "action": "skip",
+      "remote_state": {
+        "securable_type": "schema",
+        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
+        "__embed__": [
+          {
+            "principal": "[USERNAME]",
+            "privileges": [
+              "CREATE_TABLE"
+            ]
+          }
+        ]
+      }
+    }
+  }
+}

acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.terraform.json

@@ -0,0 +1,11 @@
+{
+  "cli_version": "[DEV_VERSION]",
+  "plan": {
+    "resources.schemas.grants_schema": {
+      "action": "skip"
+    },
+    "resources.schemas.grants_schema.grants": {
+      "action": "skip"
+    }
+  }
+}

acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt

@@ -0,0 +1,30 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-out-of-band-principal-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] grants update schema main.schema_out_of_band_principal_[UNIQUE_NAME] --json @update.json
+
+>>> [CLI] bundle plan -o json
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-out-of-band-principal-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle plan -o json
+
+>>> errcode [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.schemas.grants_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.grants_schema
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/schema-grants-out-of-band-principal-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/grants/schemas/out_of_band_principal/script

@@ -0,0 +1,17 @@
+SCHEMA_NAME=schema_out_of_band_principal_$UNIQUE_NAME
+SCHEMA_FULL_NAME=main.$SCHEMA_NAME
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace errcode $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null
+$CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null
+trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
+trace $CLI bundle deploy
+trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json

acceptance/bundle/resources/grants/schemas/out_of_band_principal/update.json

@@ -0,0 +1,8 @@
+{
+  "changes": [
+    {
+      "principal": "deco-test-user@databricks.com",
+      "add": ["USE_SCHEMA"]
+    }
+  ]
+}

acceptance/bundle/resources/grants/schemas/remove_principal/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: schema-grants-remove-principal-$UNIQUE_NAME
+
+resources:
+  schemas:
+    grants_schema:
+      name: schema_remove_principal_$UNIQUE_NAME
+      catalog_name: main
+      grants:
+        - principal: $CURRENT_USER_NAME
+          privileges:
+            - CREATE_TABLE
+        - { principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA] } # TO_REMOVE

acceptance/bundle/resources/grants/schemas/remove_principal/out.grants.direct.txt

@@ -0,0 +1,2 @@
+json.privilege_assignments[].principal = "[USERNAME]";
+json.privilege_assignments[].privileges[] = "CREATE_TABLE";

acceptance/bundle/resources/grants/schemas/remove_principal/out.grants.terraform.txt

@@ -0,0 +1,2 @@
+json.privilege_assignments[].principal = "[USERNAME]";
+json.privilege_assignments[].privileges[] = "CREATE_TABLE";