From cb9654f97c37eb9218ecd7f5d3cbe5295993d0b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 13:24:18 +0200 Subject: [PATCH 1/7] Fix non-deterministic sort of __embed__ grants in acceptance test Add --repl option to sort_lines.py that applies test replacements as sort key before sorting, ensuring stable order across environments. Co-authored-by: Isaac --- acceptance/bin/sort_lines.py | 27 ++++++++++++++++++- .../out.plan.direct.json | 18 ++----------- .../schemas/out_of_band_principal/output.txt | 8 ++++++ .../schemas/out_of_band_principal/script | 4 +++ 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py index 9ac87feee9..fde64ba5c7 100755 --- a/acceptance/bin/sort_lines.py +++ b/acceptance/bin/sort_lines.py @@ -1,10 +1,35 @@ #!/usr/bin/env python3 """ Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH. +With --repl, applies TEST_TMP_DIR/repls.json replacements as sort key for stable output across different environments. """ import sys +import os +import json +import re +from pathlib import Path + +use_repl = "--repl" in sys.argv[1:] lines = sys.stdin.readlines() -lines.sort() + +if use_repl: + repls = json.loads((Path(os.environ["TEST_TMP_DIR"]) / "repls.json").read_text()) + patterns = [] + for r in repls: + try: + patterns.append((re.compile(r["Old"]), r["New"])) + except re.error as e: + print(f"Regex error for pattern {r}: {e}", file=sys.stderr) + + def sort_key(line): + for comp, new in patterns: + line = comp.sub(new, line) + return line + + lines.sort(key=sort_key) +else: + lines.sort() + sys.stdout.write("".join(lines)) diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json index 12bfde2659..16cbf5fc15 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json @@ -13,7 +13,7 @@ "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "effective_predictive_optimization_flag": { - "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1", + "inherited_from_name": "[METASTORE_NAME]", "inherited_from_type": "METASTORE", "value": "ENABLE" }, @@ -51,21 +51,7 @@ }, "remote_state": { "securable_type": "schema", - "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]", - "__embed__": [ - { - "principal": "[USERNAME]", - "privileges": [ - "CREATE_TABLE" - ] - }, - { - "principal": "deco-test-user@databricks.com", - "privileges": [ - "USE_SCHEMA" - ] - } - ] + "full_name": "main.schema_out_of_band_principal_[UNIQUE_NAME]" }, "changes": { "[principal='deco-test-user@databricks.com']": { diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt index 269907e2fd..9ca7fef6ac 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt @@ -9,6 +9,14 @@ Deployment complete! >>> [CLI] bundle plan -o json +>>> jq .plan["resources.schemas.grants_schema.grants"].remote_state.__embed__ out.plan.direct.json +json[].principal = "[USERNAME]"; +json[].principal = "deco-test-user@databricks.com"; +json[].privileges[] = "CREATE_TABLE"; +json[].privileges[] = "USE_SCHEMA"; + +>>> jq del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__) out.plan.direct.json + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-out-of-band-principal-[UNIQUE_NAME]/default/files... Deploying resources... diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index d3b2d17ed3..7c0499caff 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -13,5 +13,9 @@ trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +# remote_state.__embed__ order is non-deterministic; extract and sort separately +trace jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl +tmp=$(mktemp) +trace jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json From 050fd72b7380f0d36486bf5a1cd98b48b6d844bc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 13:26:22 +0200 Subject: [PATCH 2/7] Add METASTORE_NAME replacement and update acceptance test outputs Co-authored-by: Isaac --- acceptance/bundle/migrate/basic/out.new_state.json | 2 +- .../bundle/migrate/basic/out.original_state.json | 4 ++-- acceptance/bundle/migrate/basic/out.plan_update.json | 8 ++++---- acceptance/bundle/migrate/basic/output.txt | 2 +- .../bundle/migrate/grants/out.original_state.json | 2 +- .../out.deploy.requests.direct.json | 2 +- .../out.deploy.terraform.txt | 2 +- .../duplicate_principals/out.plan.direct.json | 2 +- .../out_of_band_principal/out.plan2.direct.json | 2 +- .../resources/grants/volumes/out.plan2.direct.json | 4 ++-- .../bundle/resources/schemas/recreate/output.txt | 2 +- .../resources/volumes/change-comment/output.txt | 2 +- .../volumes/change-name/out.plan.direct.json | 4 ++-- .../bundle/resources/volumes/change-name/output.txt | 2 +- .../resources/volumes/remote-change-name/output.txt | 4 ++-- .../cmd/workspace/apps/run-local-node/output.txt | 12 +----------- acceptance/test.toml | 4 ++++ libs/testserver/grants.go | 3 +++ 18 files changed, 30 insertions(+), 33 deletions(-) diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index f6bdf06f62..c5e2050d92 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -51,7 +51,7 @@ "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", "volume_catalog_name": "mycat", - "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } }, "depends_on": [ diff --git a/acceptance/bundle/migrate/basic/out.original_state.json b/acceptance/bundle/migrate/basic/out.original_state.json index 4752f733b0..d790032e2d 100644 --- a/acceptance/bundle/migrate/basic/out.original_state.json +++ b/acceptance/bundle/migrate/basic/out.original_state.json @@ -222,7 +222,7 @@ "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", "volume_catalog_name": "mycat", - "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" }, "target": null, "timeouts": null, @@ -255,7 +255,7 @@ "owner": "[USERNAME]", "provider_config": [], "schema_name": "myschema", - "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_path": "/Volumes/mycat/myschema/myvol", "volume_type": "MANAGED" }, diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 310b029594..44ba986a2f 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -155,7 +155,7 @@ "myjob_timeout": "${resources.jobs.test_job.timeout_seconds}", "myvolume_id": "mycat.myschema.myvol", "volume_catalog_name": "mycat", - "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } }, "vars": { @@ -191,7 +191,7 @@ "myjob_timeout": "0", "myvolume_id": "mycat.myschema.myvol", "volume_catalog_name": "mycat", - "volume_storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "volume_storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } }, "changes": { @@ -230,7 +230,7 @@ "name": "myvol", "owner": "[USERNAME]", "schema_name": "myschema", - "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at": [UNIX_TIME_MILLIS][2], "volume_id": "[UUID]", "volume_type": "MANAGED" @@ -239,7 +239,7 @@ "storage_location": { "action": "skip", "reason": "backend_default", - "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt index 9a3887c2ad..0d31bbd682 100644 --- a/acceptance/bundle/migrate/basic/output.txt +++ b/acceptance/bundle/migrate/basic/output.txt @@ -25,7 +25,7 @@ To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/dev/resources.js >>> diff out.original_state.json out.backup_state.json >>> jq .state."resources.pipelines.test_pipeline".state.tags.volume_storage_location out.new_state.json -"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" +"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged diff --git a/acceptance/bundle/migrate/grants/out.original_state.json b/acceptance/bundle/migrate/grants/out.original_state.json index 742398134b..a410fff585 100644 --- a/acceptance/bundle/migrate/grants/out.original_state.json +++ b/acceptance/bundle/migrate/grants/out.original_state.json @@ -212,7 +212,7 @@ "owner": "[USERNAME]", "provider_config": [], "schema_name": "schema_grants", - "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_path": "/Volumes/main/schema_grants/volume_name", "volume_type": "MANAGED" }, diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json index 2fd43725ab..5ce661fb12 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.requests.direct.json @@ -21,7 +21,7 @@ "path": "/api/2.1/unity-catalog/volumes", "body": { "catalog_name": "main", - "comment": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "comment": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "name": "volumefoo-[UNIQUE_NAME]", "schema_name": "myschema-[UNIQUE_NAME]", "volume_type": "MANAGED" diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt index bf2c81e61b..ae9fe8af8e 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.deploy.terraform.txt @@ -10,7 +10,7 @@ When expanding the plan for databricks_volume.foo to include new values learned so far during apply, provider "registry.terraform.io/databricks/databricks" produced an invalid new value for .comment: was null, but now -cty.StringVal("s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"). +cty.StringVal("s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]"). This is a bug in the provider, which should be reported in the provider's own issue tracker. diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json index a9a5727032..85d713a72e 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json @@ -13,7 +13,7 @@ "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "effective_predictive_optimization_flag": { - "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1", + "inherited_from_name": "[METASTORE_NAME]", "inherited_from_type": "METASTORE", "value": "ENABLE" }, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json index 38758d490a..4a4faa5d9e 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json @@ -13,7 +13,7 @@ "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "effective_predictive_optimization_flag": { - "inherited_from_name": "deco-uc-prod-isolated-aws-us-east-1", + "inherited_from_name": "[METASTORE_NAME]", "inherited_from_type": "METASTORE", "value": "ENABLE" }, diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index a431a20960..01b0321ccd 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -33,14 +33,14 @@ "name": "volume_name", "owner": "[USERNAME]", "schema_name": "schema_grants_[UNIQUE_NAME]", - "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "volume_type": "MANAGED" }, "changes": { "storage_location": { "action": "skip", "reason": "backend_default", - "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } }, diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 135db4876c..4241fe2b1d 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -74,7 +74,7 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "created_at":[UNIX_TIME_MILLIS][0], "created_by":"[USERNAME]", "effective_predictive_optimization_flag": { - "inherited_from_name":"deco-uc-prod-isolated-aws-us-east-1", + "inherited_from_name":"[METASTORE_NAME]", "inherited_from_type":"METASTORE", "value":"ENABLE" }, diff --git a/acceptance/bundle/resources/volumes/change-comment/output.txt b/acceptance/bundle/resources/volumes/change-comment/output.txt index e68c981548..5277d987e0 100644 --- a/acceptance/bundle/resources/volumes/change-comment/output.txt +++ b/acceptance/bundle/resources/volumes/change-comment/output.txt @@ -48,7 +48,7 @@ Deployment complete! "name":"myvolume", "owner":"[USERNAME]", "schema_name":"myschema", - "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at":[UNIX_TIME_MILLIS][0], "volume_id":"[UUID]", "volume_type":"MANAGED" diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index f6bfbfbc03..b130d77646 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -24,7 +24,7 @@ "name": "myvolume", "owner": "[USERNAME]", "schema_name": "myschema", - "storage_location": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "volume_id": "[UUID]", "volume_type": "MANAGED" @@ -40,7 +40,7 @@ "storage_location": { "action": "skip", "reason": "backend_default", - "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" + "remote": "s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/resources/volumes/change-name/output.txt b/acceptance/bundle/resources/volumes/change-name/output.txt index 1f1b78cab8..b3ad13b947 100644 --- a/acceptance/bundle/resources/volumes/change-name/output.txt +++ b/acceptance/bundle/resources/volumes/change-name/output.txt @@ -67,7 +67,7 @@ Deployment complete! "name":"mynewvolume", "owner":"[USERNAME]", "schema_name":"myschema", - "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at":[UNIX_TIME_MILLIS][1], "updated_by":"[USERNAME]", "volume_id":"[UUID]", diff --git a/acceptance/bundle/resources/volumes/remote-change-name/output.txt b/acceptance/bundle/resources/volumes/remote-change-name/output.txt index b76ce9edec..b008fe3336 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/output.txt +++ b/acceptance/bundle/resources/volumes/remote-change-name/output.txt @@ -14,7 +14,7 @@ Deployment complete! "name":"my_new_name", "owner":"[USERNAME]", "schema_name":"myschema", - "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at":[UNIX_TIME_MILLIS][1], "updated_by":"[USERNAME]", "volume_id":"[UUID]", @@ -30,7 +30,7 @@ Deployment complete! "name":"my_new_name", "owner":"[USERNAME]", "schema_name":"myschema", - "storage_location":"s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]", + "storage_location":"s3://[METASTORE_NAME]/metastore/[UUID]/volumes/[UUID]", "updated_at":[UNIX_TIME_MILLIS][1], "updated_by":"[USERNAME]", "volume_id":"[UUID]", diff --git a/acceptance/cmd/workspace/apps/run-local-node/output.txt b/acceptance/cmd/workspace/apps/run-local-node/output.txt index 0185dbe523..4de672232f 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/output.txt +++ b/acceptance/cmd/workspace/apps/run-local-node/output.txt @@ -1,12 +1,2 @@ -Running command: node -e console.log('Hello, world') -Hello, world -=== Starting the app in background... -=== Waiting -=== Checking app is running... ->>> curl -s -o - http://127.0.0.1:$(port) -{"message":"Hello From App","timestamp":"[TIMESTAMP]","status":"running"} - -=== Sending shutdown request... ->>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown -Process terminated +Exit code: 1 diff --git a/acceptance/test.toml b/acceptance/test.toml index 9065ec4ba5..3881887cf4 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -95,3 +95,7 @@ Order = 9 Old = '2\d\d\d-\d\d-\d\d(T| )\d\d:\d\d:\d\dZ?' New = "[TIMESTAMP]" Order = 9 + +[[Repls]] +Old = "deco-uc-prod-isolated-aws-us-east-1|metastore_azure_eastus2" +New = "[METASTORE_NAME]" diff --git a/libs/testserver/grants.go b/libs/testserver/grants.go index 4710a49d29..8a5be42ed9 100644 --- a/libs/testserver/grants.go +++ b/libs/testserver/grants.go @@ -84,6 +84,9 @@ func (s *FakeWorkspace) GrantsUpdate(req Request, securableType, fullName string } // Convert back to assignments with sorted privileges + // Note order of assignments is randomized due to map. This is intentional, azure backend behaves the same way + // (deco env run -i -n azure-prod-ucws -- go test ./acceptance -run ^TestAccept$/^bundle$/^resources$/^grants$/^schemas$/^out_of_band_principal$/direct -count=10 -failfast -timeout=1h) + var assignments []catalog.PrivilegeAssignment for principal, privs := range principalPrivs { if len(privs) == 0 { From d32ba0160b1a13b8e89fccda1e2ab34140b9ce3d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 13:31:32 +0200 Subject: [PATCH 3/7] Use testMetastoreName constant in volumes.go Co-authored-by: Isaac --- libs/testserver/volumes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/testserver/volumes.go b/libs/testserver/volumes.go index 9eb29fe4ee..66e5d047ab 100644 --- a/libs/testserver/volumes.go +++ b/libs/testserver/volumes.go @@ -31,7 +31,7 @@ func (s *FakeWorkspace) VolumesCreate(req Request) Response { } } volume.VolumeId = nextUUID() - volume.StorageLocation = fmt.Sprintf("s3://deco-uc-prod-isolated-aws-us-east-1/metastore/%s/volumes/%s", TestMetastore.MetastoreId, volume.VolumeId) + volume.StorageLocation = fmt.Sprintf("s3://%s/metastore/%s/volumes/%s", testMetastoreName, TestMetastore.MetastoreId, volume.VolumeId) volume.CreatedAt = nowMilli() volume.UpdatedAt = volume.CreatedAt From 4c6f18046cc8819364b1114f475375c9e6cbfdfa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 13:44:22 +0200 Subject: [PATCH 4/7] Use per-engine output files for non-deterministic __embed__ grants Co-authored-by: Isaac --- .../schemas/out_of_band_principal/out.embed.direct.txt | 4 ++++ .../schemas/out_of_band_principal/out.embed.terraform.txt | 1 + .../grants/schemas/out_of_band_principal/output.txt | 8 -------- .../resources/grants/schemas/out_of_band_principal/script | 6 +++--- 4 files changed, 8 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt create mode 100644 acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt new file mode 100644 index 0000000000..231a0192b7 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.direct.txt @@ -0,0 +1,4 @@ +json[].principal = "[USERNAME]"; +json[].principal = "deco-test-user@databricks.com"; +json[].privileges[] = "CREATE_TABLE"; +json[].privileges[] = "USE_SCHEMA"; diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt new file mode 100644 index 0000000000..c5b4eb6863 --- /dev/null +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.embed.terraform.txt @@ -0,0 +1 @@ +json = null; diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt index 9ca7fef6ac..269907e2fd 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/output.txt @@ -9,14 +9,6 @@ Deployment complete! >>> [CLI] bundle plan -o json ->>> jq .plan["resources.schemas.grants_schema.grants"].remote_state.__embed__ out.plan.direct.json -json[].principal = "[USERNAME]"; -json[].principal = "deco-test-user@databricks.com"; -json[].privileges[] = "CREATE_TABLE"; -json[].privileges[] = "USE_SCHEMA"; - ->>> jq del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__) out.plan.direct.json - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/schema-grants-out-of-band-principal-[UNIQUE_NAME]/default/files... Deploying resources... diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index 7c0499caff..3562267303 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -13,9 +13,9 @@ trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -# remote_state.__embed__ order is non-deterministic; extract and sort separately -trace jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl +# remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file +jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) -trace jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json +jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan -o json > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json From 9b85b5f76ce7273ff43bb2bcfadf9679706f3cb2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 13:47:34 +0200 Subject: [PATCH 5/7] Revert accidental change to run-local-node output.txt Co-authored-by: Isaac --- .../cmd/workspace/apps/run-local-node/output.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/acceptance/cmd/workspace/apps/run-local-node/output.txt b/acceptance/cmd/workspace/apps/run-local-node/output.txt index 4de672232f..0185dbe523 100644 --- a/acceptance/cmd/workspace/apps/run-local-node/output.txt +++ b/acceptance/cmd/workspace/apps/run-local-node/output.txt @@ -1,2 +1,12 @@ +Running command: node -e console.log('Hello, world') +Hello, world -Exit code: 1 +=== Starting the app in background... +=== Waiting +=== Checking app is running... +>>> curl -s -o - http://127.0.0.1:$(port) +{"message":"Hello From App","timestamp":"[TIMESTAMP]","status":"running"} + +=== Sending shutdown request... +>>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown +Process terminated From 4ab79200d8cf9e9e34320630096a51e77bea7339 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 14:26:08 +0200 Subject: [PATCH 6/7] Fix sort order in remove_principal output.txt Co-authored-by: Isaac --- .../bundle/resources/grants/schemas/remove_principal/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt b/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt index 30e139d4fe..83a506e161 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/output.txt @@ -6,8 +6,8 @@ Updating deployment state... Deployment complete! >>> [CLI] grants get schema main.schema_remove_principal_[UNIQUE_NAME] -json.privilege_assignments[].principal = "deco-test-user@databricks.com"; json.privilege_assignments[].principal = "[USERNAME]"; +json.privilege_assignments[].principal = "deco-test-user@databricks.com"; json.privilege_assignments[].privileges[] = "CREATE_TABLE"; json.privilege_assignments[].privileges[] = "USE_SCHEMA"; From 06f639fdc6f05dd2fa12c1a40b3541c32f75d464 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 30 Mar 2026 14:29:06 +0200 Subject: [PATCH 7/7] Use sort_lines.py --repl in remove_principal script Co-authored-by: Isaac --- .../bundle/resources/grants/schemas/remove_principal/script | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/script b/acceptance/bundle/resources/grants/schemas/remove_principal/script index 9196653c47..2730c8ba97 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/script +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/script @@ -7,12 +7,12 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' +trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort_lines.py --repl | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' grep -v 'TO_REMOVE' databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle deploy -trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" '!deco-test-user@databricks.com' &> out.grants.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI grants get schema main.schema_remove_principal_$UNIQUE_NAME | gron.py --noindex | sort_lines.py --repl | contains.py "$CURRENT_USER_NAME" '!deco-test-user@databricks.com' &> out.grants.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan trace $CLI bundle deploy trace $CLI bundle plan