Fix issues with grants tests (#4868)

denik · web-flow · commit 518b42837515 · 2026-03-30T14:52:56.000+02:00
## Changes
- Fix non-deterministic order of grants in the recorded plan
- Add METASTORE_NAME replacement to capture AWS/Azure differences.
- Add --repl option to sort_lines.py to sort according to replaced
values to prevent UUID from affecting the sort order (`aaa...` vs
`deco...` vs `fff...`).
diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py
@@ -1,10 +1,35 @@
 #!/usr/bin/env python3
 """
 Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH.
+With --repl, applies TEST_TMP_DIR/repls.json replacements as sort key for stable output across different environments.
 """
 
 import sys
+import os
+import json
+import re
+from pathlib import Path
+
+use_repl = "--repl" in sys.argv[1:]
 
 lines = sys.stdin.readlines()
-lines.sort()
+
+if use_repl:
+    repls = json.loads((Path(os.environ["TEST_TMP_DIR"]) / "repls.json").read_text())
+    patterns = []
+    for r in repls:
+        try:
+            patterns.append((re.compile(r["Old"]), r["New"]))
+        except re.error as e:
+            print(f"Regex error for pattern {r}: {e}", file=sys.stderr)
+
+    def sort_key(line):
+        for comp, new in patterns:
+            line = comp.sub(new, line)
+        return line
+
+    lines.sort(key=sort_key)
+else:
+    lines.sort()
+
 sys.stdout.write("".join(lines))
diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json
@@ -51,7 +51,7 @@
      "myjob_timeout": "0",
      "myvolume_id": "mycat.myschema.myvol",
      "volume_catalog_name": "mycat",
-     "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+     "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
     }
    },
    "depends_on": [
diff --git a/acceptance/bundle/migrate/basic/out.original_state.json b/acceptance/bundle/migrate/basic/out.original_state.json
@@ -222,7 +222,7 @@
               "myjob_timeout": "0",
               "myvolume_id": "mycat.myschema.myvol",
               "volume_catalog_name": "mycat",
-              "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+              "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
             },
             "target": null,
             "timeouts": null,
@@ -255,7 +255,7 @@
             "owner": "[USERNAME]",
             "provider_config": [],
             "schema_name": "myschema",
-            "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+            "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
             "volume_path": "/Volumes/mycat/myschema/myvol",
             "volume_type": "MANAGED"
           },
diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json
@@ -155,7 +155,7 @@
             "myjob_timeout": "${resources.jobs.test_job.timeout_seconds}",
             "myvolume_id": "mycat.myschema.myvol",
             "volume_catalog_name": "mycat",
-            "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+            "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
           }
         },
         "vars": {
@@ -191,7 +191,7 @@
           "myjob_timeout": "0",
           "myvolume_id": "mycat.myschema.myvol",
           "volume_catalog_name": "mycat",
-          "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+          "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
         }
       },
       "changes": {
@@ -230,7 +230,7 @@
         "name": "myvol",
         "owner": "[USERNAME]",
         "schema_name": "myschema",
-        "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+        "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
         "updated_at": [UNIX_TIME_MILLIS][2],
         "volume_id": "[UUID]",
         "volume_type": "MANAGED"
@@ -239,7 +239,7 @@
         "storage_location": {
           "action": "skip",
           "reason": "backend_default",
-          "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+          "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
         }
       }
     }
diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt
@@ -25,7 +25,7 @@ To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/dev/resources.js
 >>> diff out.original_state.json out.backup_state.json
 
 >>> jq .state."resources.pipelines.test_pipeline".state.tags.volume_storage_location out.new_state.json
-"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan
 Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
diff --git a/acceptance/bundle/migrate/grants/out.original_state.json b/acceptance/bundle/migrate/grants/out.original_state.json
@@ -212,7 +212,7 @@
             "owner": "[USERNAME]",
             "provider_config": [],
             "schema_name": "schema_grants",
-            "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+            "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
             "volume_path": "/Volumes/main/schema_grants/volume_name",
             "volume_type": "MANAGED"
           },
diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json
@@ -21,7 +21,7 @@
   "path": "/api/2.1/unity-catalog/volumes",
   "body": {
     "catalog_name": "main",
-    "comment": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+    "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
     "name": "volumefoo-[UNIQUE_NAME]",
     "schema_name": "myschema-[UNIQUE_NAME]",
     "volume_type": "MANAGED"
diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt
@@ -10,7 +10,7 @@ When expanding the plan for databricks_volume.foo to include new values
 learned so far during apply, provider
 "registry.terraform.io/databricks/databricks" produced an invalid new value
 for .comment: was null, but now
-cty.StringVal("s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]").
+cty.StringVal("s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]").
 
 This is a bug in the provider, which should be reported in the provider's own
 issue tracker.
diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json
@@ -13,7 +13,7 @@
         "created_at": [UNIX_TIME_MILLIS][0],
         "created_by": "[USERNAME]",
         "effective_predictive_optimization_flag": {
-          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_name": "[METASTORE_NAME]",
           "inherited_from_type": "METASTORE",
           "value": "ENABLE"
         },
diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt
@@ -0,0 +1,4 @@
+json[].principal = "[USERNAME]";
+json[].principal = "deco-test-user@databricks.com";
+json[].privileges[] = "CREATE_TABLE";
+json[].privileges[] = "USE_SCHEMA";
diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt
@@ -0,0 +1 @@
+json = null;
diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json
@@ -13,7 +13,7 @@
         "created_at": [UNIX_TIME_MILLIS][0],
         "created_by": "[USERNAME]",
         "effective_predictive_optimization_flag": {
-          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_name": "[METASTORE_NAME]",
           "inherited_from_type": "METASTORE",
           "value": "ENABLE"
         },
@@ -51,21 +51,7 @@
       },
       "remote_state": {
         "securable_type": "schema",
-        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]",
-        "__embed__": [
-          {
-            "principal": "[USERNAME]",
-            "privileges": [
-              "CREATE_TABLE"
-            ]
-          },
-          {
-            "principal": "deco-test-user@databricks.com",
-            "privileges": [
-              "USE_SCHEMA"
-            ]
-          }
-        ]
+        "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]"
       },
       "changes": {
         "[principal='deco-test-user@databricks.com']": {
diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json
@@ -13,7 +13,7 @@
         "created_at": [UNIX_TIME_MILLIS][0],
         "created_by": "[USERNAME]",
         "effective_predictive_optimization_flag": {
-          "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1",
+          "inherited_from_name": "[METASTORE_NAME]",
           "inherited_from_type": "METASTORE",
           "value": "ENABLE"
         },
diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script
@@ -13,5 +13,9 @@ trace $CLI bundle deploy
 trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null
 $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null
 trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json
+# remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file
+jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt
+tmp=$(mktemp)
+jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json
diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt b/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt
@@ -6,8 +6,8 @@ Updating deployment state...
 Deployment complete!
 
 >>> [CLI] grants get schema main.schema_remove_principal_[UNIQUE_NAME]
-json.privilege_assignments[].principal = "deco-test-user@databricks.com";
 json.privilege_assignments[].principal = "[USERNAME]";
+json.privilege_assignments[].principal = "deco-test-user@databricks.com";
 json.privilege_assignments[].privileges[] = "CREATE_TABLE";
 json.privilege_assignments[].privileges[] = "USE_SCHEMA";
 
diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/script b/acceptance/bundle/resources/grants/schemas/remove_principal/script
@@ -7,12 +7,12 @@ cleanup() {
 trap cleanup EXIT
 
 trace $CLI bundle deploy
-trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com'
+trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort_lines.py --repl | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com'
 
 grep -v 'TO_REMOVE' databricks.yml > tmp.yml && mv tmp.yml databricks.yml
 
 trace $CLI bundle deploy
-trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" '!deco-test-user@databricks.com' &> out.grants.$DATABRICKS_BUNDLE_ENGINE.txt
+trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort_lines.py --repl | contains.py "$CURRENT_USER_NAME" '!deco-test-user@databricks.com' &> out.grants.$DATABRICKS_BUNDLE_ENGINE.txt
 trace $CLI bundle plan
 trace $CLI bundle deploy
 trace $CLI bundle plan
diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json
@@ -33,14 +33,14 @@
         "name": "volume_name",
         "owner": "[USERNAME]",
         "schema_name": "schema_grants_[UNIQUE_NAME]",
-        "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+        "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
         "volume_type": "MANAGED"
       },
       "changes": {
         "storage_location": {
           "action": "skip",
           "reason": "backend_default",
-          "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+          "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
         }
       }
     },
diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt
@@ -74,7 +74,7 @@ Error: Resource catalog.SchemaInfo not found: main.myschema
   "created_at":[UNIX_TIME_MILLIS][0],
   "created_by":"[USERNAME]",
   "effective_predictive_optimization_flag": {
-    "inherited_from_name":"deco-uc-prod-isolated-aws-us-east-1",
+    "inherited_from_name":"[METASTORE_NAME]",
     "inherited_from_type":"METASTORE",
     "value":"ENABLE"
   },
diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt
@@ -48,7 +48,7 @@ Deployment complete!
   "name":"myvolume",
   "owner":"[USERNAME]",
   "schema_name":"myschema",
-  "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+  "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
   "updated_at":[UNIX_TIME_MILLIS][0],
   "volume_id":"[UUID]",
   "volume_type":"MANAGED"
diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json
@@ -24,7 +24,7 @@
         "name": "myvolume",
         "owner": "[USERNAME]",
         "schema_name": "myschema",
-        "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+        "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
         "updated_at": [UNIX_TIME_MILLIS][0],
         "volume_id": "[UUID]",
         "volume_type": "MANAGED"
@@ -40,7 +40,7 @@
         "storage_location": {
           "action": "skip",
           "reason": "backend_default",
-          "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
+          "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"
         }
       }
     }
diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt
@@ -67,7 +67,7 @@ Deployment complete!
   "name":"mynewvolume",
   "owner":"[USERNAME]",
   "schema_name":"myschema",
-  "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+  "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
   "updated_at":[UNIX_TIME_MILLIS][1],
   "updated_by":"[USERNAME]",
   "volume_id":"[UUID]",
diff --git a/acceptance/bundle/resources/volumes/remote-change-name/output.txt b/acceptance/bundle/resources/volumes/remote-change-name/output.txt
@@ -14,7 +14,7 @@ Deployment complete!
   "name":"my_new_name",
   "owner":"[USERNAME]",
   "schema_name":"myschema",
-  "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+  "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
   "updated_at":[UNIX_TIME_MILLIS][1],
   "updated_by":"[USERNAME]",
   "volume_id":"[UUID]",
@@ -30,7 +30,7 @@ Deployment complete!
   "name":"my_new_name",
   "owner":"[USERNAME]",
   "schema_name":"myschema",
-  "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]",
+  "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]",
   "updated_at":[UNIX_TIME_MILLIS][1],
   "updated_by":"[USERNAME]",
   "volume_id":"[UUID]",
diff --git a/acceptance/test.toml b/acceptance/test.toml
@@ -95,3 +95,7 @@ Order = 9
 Old = '2\d\d\d-\d\d-\d\d(T| )\d\d:\d\d:\d\dZ?'
 New = "[TIMESTAMP]"
 Order = 9
+
+[[Repls]]
+Old = "deco-uc-prod-isolated-aws-us-east-1|metastore_azure_eastus2"
+New = "[METASTORE_NAME]"
diff --git a/libs/testserver/grants.go b/libs/testserver/grants.go
@@ -84,6 +84,9 @@ func (s *FakeWorkspace) GrantsUpdate(req Request, securableType, fullName string
 	}
 
 	// Convert back to assignments with sorted privileges
+	// Note order of assignments is randomized due to map. This is intentional, azure backend behaves the same way
+	// (deco env run -i -n azure-prod-ucws -- go test ./acceptance -run ^TestAccept$/^bundle$/^resources$/^grants$/^schemas$/^out_of_band_principal$/direct -count=10 -failfast -timeout=1h)
+
 	var assignments []catalog.PrivilegeAssignment
 	for principal, privs := range principalPrivs {
 		if len(privs) == 0 {
diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go
@@ -31,7 +31,7 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response {
 		}
 	}
 	volume.VolumeId = nextUUID()
-	volume.StorageLocation = fmt.Sprintf("s3://deco-uc-prod-isolated-aws-us-east-1/metastore/%s/volumes/%s", TestMetastore.MetastoreId, volume.VolumeId)
+	volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId)
 
 	volume.CreatedAt = nowMilli()
 	volume.UpdatedAt = volume.CreatedAt

Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@`
`51`	`51`	`"myjob_timeout": "0",`
`52`	`52`	`"myvolume_id": "mycat.myschema.myvol",`
`53`	`53`	`"volume_catalog_name": "mycat",`
`54`		`- "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"`
	`54`	`+ "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"`
`55`	`55`	`}`
`56`	`56`	`},`
`57`	`57`	`"depends_on": [`