From 0aa470d014883fb47bc373e796751144865b09f7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 15:10:15 +0100 Subject: [PATCH 01/34] =?UTF-8?q?Use=20EmbeddedSlice=20for=20GrantsState;?= =?UTF-8?q?=20add=20state=20migration=20v2=E2=86=92v3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply the same EmbeddedSlice pattern used for permissions to grants: - Rename GrantsState.Grants to GrantsState.EmbeddedSlice (json:"_") - Add migrateV2ToV3 to rename "grants" → "_" key in existing state - Bump state version to 3 - Add grant_ref acceptance test skeleton Co-Authored-By: Claude Opus 4.6 --- .../resource_deps/grant_ref/databricks.yml | 31 +++++++++++ .../resource_deps/grant_ref/out.test.toml | 5 ++ .../bundle/resource_deps/grant_ref/output.txt | 8 +++ .../bundle/resource_deps/grant_ref/script | 4 ++ .../bundle/resource_deps/grant_ref/test.toml | 4 ++ bundle/direct/dresources/all_test.go | 10 ++-- bundle/direct/dresources/grants.go | 8 +-- bundle/direct/dstate/migrate.go | 55 +++++++++++++++++++ bundle/direct/dstate/state.go | 2 +- 9 files changed, 117 insertions(+), 10 deletions(-) create mode 100644 acceptance/bundle/resource_deps/grant_ref/databricks.yml create mode 100644 acceptance/bundle/resource_deps/grant_ref/out.test.toml create mode 100644 acceptance/bundle/resource_deps/grant_ref/output.txt create mode 100644 acceptance/bundle/resource_deps/grant_ref/script create mode 100644 acceptance/bundle/resource_deps/grant_ref/test.toml diff --git a/acceptance/bundle/resource_deps/grant_ref/databricks.yml b/acceptance/bundle/resource_deps/grant_ref/databricks.yml new file mode 100644 index 0000000000..4b7a4bd961 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/databricks.yml @@ -0,0 +1,31 @@ +bundle: + name: test-bundle + +resources: + schemas: + # schema_b has grants that schema_a references + schema_b: + catalog_name: main + name: schema B + grants: + - principal: viewers + privileges: + - USE_SCHEMA + - principal: editors + privileges: + - CREATE_TABLE + - USE_SCHEMA + + # schema_a references schema_b's grant principals + schema_a: + catalog_name: main + name: schema A + grants: + # Reference by integer index + - principal: ${resources.schemas.schema_b.grants[0].principal} + privileges: + - USE_SCHEMA + # Reference by integer index (second entry) + - principal: ${resources.schemas.schema_b.grants[1].principal} + privileges: + - CREATE_TABLE diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/output.txt b/acceptance/bundle/resource_deps/grant_ref/output.txt new file mode 100644 index 0000000000..ac719f6188 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle plan +Error: cannot plan resources.schemas.schema_a.grants: cannot resolve "${resources.schemas.schema_b.grants[0].principal}": schema mismatch: [0]: cannot index struct; [0]: cannot index struct + +Error: planning failed + + +Exit code: 1 diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script new file mode 100644 index 0000000000..82800b1654 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -0,0 +1,4 @@ + +trace $CLI bundle plan +trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/grant_ref/test.toml b/acceptance/bundle/resource_deps/grant_ref/test.toml new file mode 100644 index 0000000000..790c13e6dc --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/test.toml @@ -0,0 +1,4 @@ +RecordRequests = false + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 538bc05d47..da59fedbe0 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -508,7 +508,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "catalog", FullName: "mycatalog", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeUseCatalog}, Principal: "user@example.com", }}, @@ -519,7 +519,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "external_location", FullName: "myexternallocation", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeReadFiles}, Principal: "user@example.com", }}, @@ -530,7 +530,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "schema", FullName: "main.myschema", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, @@ -541,7 +541,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "volume", FullName: "main.myschema.myvolume", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, @@ -552,7 +552,7 @@ var testDeps = map[string]prepareWorkspace{ return &GrantsState{ SecurableType: "registered-model", FullName: "modelid", - Grants: []catalog.PrivilegeAssignment{{ + EmbeddedSlice: []catalog.PrivilegeAssignment{{ Privileges: []catalog.Privilege{catalog.PrivilegeCreateView}, Principal: "user@example.com", }}, diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 8c0989aa99..c06840c0bc 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -23,7 +23,7 @@ var grantResourceToSecurableType = map[string]string{ type GrantsState struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` - Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` + EmbeddedSlice []catalog.PrivilegeAssignment `json:"_,omitempty"` } func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVar, error) { @@ -56,7 +56,7 @@ func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVa Value: &GrantsState{ SecurableType: securableType, FullName: "", - Grants: *grantsPtr, + EmbeddedSlice: *grantsPtr, }, Refs: map[string]string{ "full_name": "${" + baseNode + ".id}", @@ -90,7 +90,7 @@ func (r *ResourceGrants) DoRead(ctx context.Context, id string) (*GrantsState, e return &GrantsState{ SecurableType: securableType, FullName: fullName, - Grants: assignments, + EmbeddedSlice: assignments, }, nil } @@ -121,7 +121,7 @@ func (r *ResourceGrants) applyGrants(ctx context.Context, state *GrantsState) er var changes []catalog.PermissionsChange // For each principal in the config, add their grants and remove everything else - for _, grantAssignment := range state.Grants { + for _, grantAssignment := range state.EmbeddedSlice { changes = append(changes, catalog.PermissionsChange{ Principal: grantAssignment.Principal, Add: grantAssignment.Privileges, diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 41bf90fc2d..e21a5230ff 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -38,6 +38,7 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, + 2: migrateV2ToV3, } // migrateV1ToV2 migrates permissions entries from the old format @@ -99,3 +100,57 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return json.MarshalIndent(newState, " ", " ") } + +// migrateV2ToV3 migrates grants entries from the old format +// ("grants" key for the slice) to the new format ("_" key for EmbeddedSlice). +func migrateV2ToV3(db *Database) error { + for key, entry := range db.State { + if !strings.HasSuffix(key, ".grants") { + continue + } + if len(entry.State) == 0 { + continue + } + migrated, err := migrateGrantsEntry(entry.State) + if err != nil { + return fmt.Errorf("migrating %s: %w", key, err) + } + entry.State = migrated + db.State[key] = entry + } + return nil +} + +// oldGrantsStateV2 is the old format with "grants" key. +type oldGrantsStateV2 struct { + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + Grants []json.RawMessage `json:"grants,omitempty"` +} + +func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { + var old oldGrantsStateV2 + if err := json.Unmarshal(raw, &old); err != nil { + return nil, err + } + + // If old format had no grants, might already be migrated. + if len(old.Grants) == 0 { + return raw, nil + } + + // Re-serialize with "_" key instead of "grants". + type newFormat struct { + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + EmbeddedSlice []json.RawMessage `json:"_,omitempty"` + } + + newState := newFormat{ + SecurableType: old.SecurableType, + FullName: old.FullName, + EmbeddedSlice: old.Grants, + } + + return json.MarshalIndent(newState, " ", " ") +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e1c776b479..3542cf8d0a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const currentStateVersion = 2 +const currentStateVersion = 3 type DeploymentState struct { Path string From c400e0423368c8bce4a96571cbcd5210bcd0d668 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 15:17:18 +0100 Subject: [PATCH 02/34] Update acceptance test outputs for grants EmbeddedSlice change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Regenerate grants plan outputs (JSON key "grants" → "_") - Add grant_ref test output with reference resolution - Add schema_with_grants and schema_grant_ref invariant configs - Update refschema to show grants[*] embedded slice fields - Update state version in migration/future_version tests - Add exclusions for new grant configs in migrate/continue_293 tests Co-Authored-By: Claude Opus 4.6 --- .../configs/schema_grant_ref.yml.tmpl | 28 ++++++ .../configs/schema_with_grants.yml.tmpl | 12 +++ .../bundle/invariant/continue_293/test.toml | 4 + acceptance/bundle/invariant/migrate/test.toml | 3 + acceptance/bundle/invariant/test.toml | 2 + .../bundle/migrate/grants/out.new_state.json | 8 +- acceptance/bundle/refschema/out.fields.txt | 20 ++++ .../grant_ref/out.plan_create.direct.json | 97 +++++++++++++++++++ .../bundle/resource_deps/grant_ref/output.txt | 14 ++- .../grants/catalogs/out.plan1.direct.json | 2 +- .../grants/catalogs/out.plan2.direct.json | 8 +- .../registered_models/out.plan1.direct.json | 2 +- .../change_privilege/out.plan1.direct.json | 2 +- .../change_privilege/out.plan2.direct.json | 8 +- .../schemas/empty_array/out.plan1.direct.txt | 2 +- .../grants/volumes/out.plan1.direct.json | 2 +- .../grants/volumes/out.plan2.direct.json | 8 +- .../bundle/state/future_version/output.txt | 2 +- 18 files changed, 199 insertions(+), 25 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl create mode 100644 acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl create mode 100644 acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json diff --git a/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl b/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl new file mode 100644 index 0000000000..2d40543e47 --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_grant_ref.yml.tmpl @@ -0,0 +1,28 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + schema_b: + catalog_name: main + name: test-schema-b-$UNIQUE_NAME + grants: + - principal: account users + privileges: + - USE_SCHEMA + - principal: admins + privileges: + - CREATE_TABLE + - USE_SCHEMA + + schema_a: + catalog_name: main + name: test-schema-a-$UNIQUE_NAME + grants: + # Reference principal and privileges from schema_b by index + - principal: ${resources.schemas.schema_b.grants[0].principal} + privileges: + - USE_SCHEMA + - principal: ${resources.schemas.schema_b.grants[1].principal} + privileges: + - CREATE_TABLE diff --git a/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl b/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl new file mode 100644 index 0000000000..d9aebda0ce --- /dev/null +++ b/acceptance/bundle/invariant/configs/schema_with_grants.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + foo: + catalog_name: main + name: test-schema-$UNIQUE_NAME + grants: + - principal: account users + privileges: + - USE_SCHEMA diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 48ed3d4977..29723a31d2 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -5,3 +5,7 @@ Slow = true # permissions to be part of the job schema, which was added after v0.293.0. EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] + +# Grants with EmbeddedSlice require state migration from v2→v3, which v0.293.0 can't produce. +EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] +EnvMatrixExclude.no_schema_with_grants = ["INPUT_CONFIG=schema_with_grants.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 781987f7ca..a29f9dbb5a 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -13,3 +13,6 @@ EnvMatrixExclude.no_secret_scope = ["INPUT_CONFIG=secret_scope.yml.tmpl"] # ends up as the permission level value. EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] + +# Grant cross-references require the EmbeddedSlice pattern not present in terraform mode. +EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index b87f1ec7e5..36badf07e5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -45,6 +45,8 @@ EnvMatrix.INPUT_CONFIG = [ "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", + "schema_grant_ref.yml.tmpl", + "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl", diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 2046d78c9d..1187063d2e 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 9, @@ -24,7 +24,7 @@ "state": { "securable_type": "function", "full_name": "main.schema_grants.mymodel", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -52,7 +52,7 @@ "state": { "securable_type": "schema", "full_name": "main.schema_grants", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -89,7 +89,7 @@ "state": { "securable_type": "volume", "full_name": "main.schema_grants.volume_name", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index b02419eb75..97dfa44414 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -253,6 +253,10 @@ resources.catalogs.*.updated_by string REMOTE resources.catalogs.*.url string INPUT resources.catalogs.*.grants.full_name string ALL resources.catalogs.*.grants.securable_type string ALL +resources.catalogs.*.grants[*] catalog.PrivilegeAssignment ALL +resources.catalogs.*.grants[*].principal string ALL +resources.catalogs.*.grants[*].privileges []catalog.Privilege ALL +resources.catalogs.*.grants[*].privileges[*] catalog.Privilege ALL resources.clusters.*.apply_policy_default_values bool INPUT STATE resources.clusters.*.autoscale *compute.AutoScale ALL resources.clusters.*.autoscale.max_workers int ALL @@ -683,6 +687,10 @@ resources.external_locations.*.updated_by string REMOTE resources.external_locations.*.url string ALL resources.external_locations.*.grants.full_name string ALL resources.external_locations.*.grants.securable_type string ALL +resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL +resources.external_locations.*.grants[*].principal string ALL +resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL +resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL @@ -2680,6 +2688,10 @@ resources.registered_models.*.updated_by string ALL resources.registered_models.*.url string INPUT resources.registered_models.*.grants.full_name string ALL resources.registered_models.*.grants.securable_type string ALL +resources.registered_models.*.grants[*] catalog.PrivilegeAssignment ALL +resources.registered_models.*.grants[*].principal string ALL +resources.registered_models.*.grants[*].privileges []catalog.Privilege ALL +resources.registered_models.*.grants[*].privileges[*] catalog.Privilege ALL resources.schemas.*.browse_only bool REMOTE resources.schemas.*.catalog_name string ALL resources.schemas.*.catalog_type catalog.CatalogType REMOTE @@ -2709,6 +2721,10 @@ resources.schemas.*.updated_by string REMOTE resources.schemas.*.url string INPUT resources.schemas.*.grants.full_name string ALL resources.schemas.*.grants.securable_type string ALL +resources.schemas.*.grants[*] catalog.PrivilegeAssignment ALL +resources.schemas.*.grants[*].principal string ALL +resources.schemas.*.grants[*].privileges []catalog.Privilege ALL +resources.schemas.*.grants[*].privileges[*] catalog.Privilege ALL resources.secret_scopes.*.backend_azure_keyvault *workspace.AzureKeyVaultSecretScopeMetadata STATE resources.secret_scopes.*.backend_azure_keyvault.dns_name string STATE resources.secret_scopes.*.backend_azure_keyvault.resource_id string STATE @@ -2871,3 +2887,7 @@ resources.volumes.*.volume_id string REMOTE resources.volumes.*.volume_type catalog.VolumeType ALL resources.volumes.*.grants.full_name string ALL resources.volumes.*.grants.securable_type string ALL +resources.volumes.*.grants[*] catalog.PrivilegeAssignment ALL +resources.volumes.*.grants[*].principal string ALL +resources.volumes.*.grants[*].privileges []catalog.Privilege ALL +resources.volumes.*.grants[*].privileges[*] catalog.Privilege ALL diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json new file mode 100644 index 0000000000..aea815e550 --- /dev/null +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -0,0 +1,97 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.schemas.schema_a": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "schema A" + } + } + }, + "resources.schemas.schema_a.grants": { + "depends_on": [ + { + "node": "resources.schemas.schema_a", + "label": "${resources.schemas.schema_a.id}" + }, + { + "node": "resources.schemas.schema_b.grants", + "label": "${resources.schemas.schema_b.grants[0].principal}" + }, + { + "node": "resources.schemas.schema_b.grants", + "label": "${resources.schemas.schema_b.grants[1].principal}" + } + ], + "action": "create", + "new_state": { + "value": { + "securable_type": "schema", + "full_name": "", + "_": [ + { + "principal": "viewers", + "privileges": [ + "USE_SCHEMA" + ] + }, + { + "principal": "editors", + "privileges": [ + "CREATE_TABLE" + ] + } + ] + }, + "vars": { + "full_name": "${resources.schemas.schema_a.id}" + } + } + }, + "resources.schemas.schema_b": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "name": "schema B" + } + } + }, + "resources.schemas.schema_b.grants": { + "depends_on": [ + { + "node": "resources.schemas.schema_b", + "label": "${resources.schemas.schema_b.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "securable_type": "schema", + "full_name": "", + "_": [ + { + "principal": "viewers", + "privileges": [ + "USE_SCHEMA" + ] + }, + { + "principal": "editors", + "privileges": [ + "CREATE_TABLE", + "USE_SCHEMA" + ] + } + ] + }, + "vars": { + "full_name": "${resources.schemas.schema_b.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resource_deps/grant_ref/output.txt b/acceptance/bundle/resource_deps/grant_ref/output.txt index ac719f6188..47cfaccbcb 100644 --- a/acceptance/bundle/resource_deps/grant_ref/output.txt +++ b/acceptance/bundle/resource_deps/grant_ref/output.txt @@ -1,8 +1,16 @@ >>> [CLI] bundle plan -Error: cannot plan resources.schemas.schema_a.grants: cannot resolve "${resources.schemas.schema_b.grants[0].principal}": schema mismatch: [0]: cannot index struct; [0]: cannot index struct +create schemas.schema_a +create schemas.schema_a.grants +create schemas.schema_b +create schemas.schema_b.grants -Error: planning failed +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged +>>> [CLI] bundle plan -o json -Exit code: 1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 6f50fd2b5f..293272964b 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -22,7 +22,7 @@ "new_state": { "value": { "full_name": "", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 6996fb49e5..6a62a22919 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -25,7 +25,7 @@ "new_state": { "value": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -38,7 +38,7 @@ }, "remote_state": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -49,13 +49,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "CREATE_SCHEMA", "new": "USE_CATALOG", "remote": "CREATE_SCHEMA" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "USE_CATALOG", "new": "USE_SCHEMA", diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index b5ab8d0b9d..3a4f3664af 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "function", "full_name": "", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 01d2d581b2..dca3d13e4c 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index 7b08263765..b8d929f3c8 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -29,7 +29,7 @@ "value": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -43,7 +43,7 @@ "remote_state": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -54,13 +54,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "CREATE_TABLE", "new": "APPLY_TAG", "remote": "CREATE_TABLE" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "USE_SCHEMA", "new": "CREATE_TABLE", diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt index a259b5603f..23385843df 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com" } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index 2f9e4d51a1..11ffbf4669 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -40,7 +40,7 @@ "value": { "securable_type": "volume", "full_name": "", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index a6709a074f..0d843258cd 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -56,7 +56,7 @@ "value": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -70,7 +70,7 @@ "remote_state": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "_": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -81,13 +81,13 @@ ] }, "changes": { - "grants[0].privileges[0]": { + "[0].privileges[0]": { "action": "update", "old": "READ_VOLUME", "new": "MANAGE", "remote": "READ_VOLUME" }, - "grants[0].privileges[1]": { + "[0].privileges[1]": { "action": "update", "old": "WRITE_VOLUME", "new": "READ_VOLUME", diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 7cf98129ee..0a16971f47 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 2; upgrade the CLI +state version 999 is newer than supported version 3; upgrade the CLI Exit code: 1 From 09669146ef13bbc241cd67696fbb9b4ce9d589b7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:03:38 +0100 Subject: [PATCH 03/34] Update acceptance test outputs for state_version 3 Co-Authored-By: Claude Opus 4.6 --- .../bind/dashboard/recreation/out.state_after_bind.direct.json | 2 +- acceptance/bundle/invariant/continue_293/out.test.toml | 2 +- acceptance/bundle/invariant/migrate/out.test.toml | 2 +- acceptance/bundle/invariant/no_drift/out.test.toml | 2 +- acceptance/bundle/migrate/basic/out.new_state.json | 2 +- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../migrate/default-python/out.state_after_migration.json | 2 +- acceptance/bundle/resources/jobs/big_id/out.state.direct.json | 2 +- acceptance/bundle/resources/jobs/update/out.state.direct.json | 2 +- .../bundle/user_agent/simple/out.requests.deploy.direct.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index 771dd3f908..c7a30bcd43 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index a83f46df49..ba32b1f599 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 17f14550ca..725019505d 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 17f14550ca..725019505d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index f6bdf06f62..e294a69095 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index 695d14602a..a3d491562e 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 029649aae3..8ff25834b7 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index f8cf0ce5bf..31eb566641 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 785c0c1338..76888ff526 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index d4543d353a..f252d6df1d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -222,7 +222,7 @@ "overwrite": "true" }, "body": { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, From 9860e3ba098313cfd8b60c7b44dfad9415421dd9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:04:06 +0100 Subject: [PATCH 04/34] Regenerate acceptance test outputs after rebase Co-Authored-By: Claude Opus 4.6 --- .../out.plan_.terraform.json | 8 - .../out.plan_.terraform.txt | 3 - .../out.requests.txt | 298 +++++++++ .../update-and-resize-autoscale/output.txt | 60 +- .../deploy/update-and-resize/out.requests.txt | 63 ++ .../deploy/update-and-resize/output.txt | 7 +- .../update/out.plan_restore.terraform.json | 11 - .../update/out.plan_set_empty.terraform.json | 11 - .../out.requests_destroy.terraform.json | 19 - .../out.requests_set_empty.terraform.json | 0 .../bundle/resources/permissions/output.txt | 23 +- .../update/out.plan_restore.terraform.json | 11 - .../pipelines/update/out.requests.txt | 324 +++++++++ .../update/out.requests_delete_all.json | 0 .../update/out.requests_restore_original.json | 20 - .../permissions/pipelines/update/output.txt | 36 +- ...ce_principal_name_different.terraform.json | 11 - .../classic/out.compare-vs-serverless.diff | 11 + .../out.plan_after_deploy_prod.terraform.json | 17 - .../classic/out.requests.prod.terraform.txt | 225 ------- .../default-python/classic/out.requests.txt | 627 ++++++++++++++++++ .../default-python/classic/output.txt | 11 +- .../output/my_default_python/.gitignore | 10 + .../combinations/serverless/output.txt | 4 +- .../serverless-customcatalog/output.txt | 11 + .../out.plan_after_deploy_dev.terraform.json | 11 - .../out.plan_after_deploy_prod.terraform.json | 17 - .../out.requests.prod.terraform.txt | 213 ------ .../serverless/out.requests.txt | 32 + .../default-python/serverless/output.txt | 19 +- .../output/my_default_python/.gitignore | 10 + .../cmd/workspace/apps/run-local/out.run.txt | 54 ++ .../cmd/workspace/apps/run-local/output.txt | 15 +- 33 files changed, 1450 insertions(+), 742 deletions(-) create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt delete mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json delete mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json delete mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json delete mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json create mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt delete mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json delete mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json delete mode 100644 acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json delete mode 100644 acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt create mode 100644 acceptance/bundle/templates/default-python/classic/out.requests.txt create mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore delete mode 100644 acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json delete mode 100644 acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json delete mode 100644 acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt create mode 100644 acceptance/bundle/templates/default-python/serverless/out.requests.txt create mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore create mode 100644 acceptance/cmd/workspace/apps/run-local/out.run.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json index 0c92dfb4ad..4dd9da9eb8 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json @@ -30,11 +30,3 @@ } } } -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.clusters.test_cluster": { - "action": "update" - } - } -} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt index 73b40dacde..d51cd2abf0 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt @@ -10,6 +10,3 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged -update clusters.test_cluster - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt new file mode 100644 index 0000000000..893ae564bc --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt @@ -0,0 +1,298 @@ +{ + "method": "GET", + "path": "/api/2.1/clusters/get", + "q": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "POST", + "path": "/api/2.1/clusters/start", + "body": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "GET", + "path": "/api/2.1/clusters/get", + "q": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.1/clusters/get", + "q": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "GET", + "path": "/api/2.1/clusters/list", + "q": { + "filter_by.is_pinned": "true", + "page_size": "100" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.1/clusters/get", + "q": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "GET", + "path": "/api/2.1/clusters/list", + "q": { + "filter_by.is_pinned": "true", + "page_size": "100" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "q": { + "overwrite": "false" + }, + "body": { + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.plan_.terraform.json", + "q": { + "overwrite": "true" + }, + "raw_body": "{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"create\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.requests.txt", + "q": { + "overwrite": "true" + }, + "raw_body": "{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/start\",\n \"body\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/list\",\n \"q\": {\n \"filter_by.is_pinned\": \"true\",\n \"page_size\": \"100\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/list\",\n \"q\": {\n \"filter_by.is_pinned\": \"true\",\n \"page_size\": \"100\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\",\n \"q\": {\n \"overwrite\": \"false\"\n },\n \"body\": {\n \"ID\": \"[UUID]\",\n \"AcquisitionTime\": \"[TIMESTAMP]\",\n \"IsForced\": false,\n \"User\": \"[USERNAME]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/export\",\n \"q\": {\n \"direct_download\": \"true\",\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/delete\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal\",\n \"recursive\": true\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/mkdirs\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files\"\n }\n}\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/databricks.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "bundle:\n name: test-deploy-cluster-autoscale\n\nworkspace:\n root_path: ~/.bundle/[UNIQUE_NAME]\n\nresources:\n clusters:\n test_cluster:\n cluster_name: test-cluster-[UNIQUE_NAME]\n spark_version: 13.3.x-snapshot-scala2.12\n node_type_id: [NODE_TYPE_ID]\n autoscale:\n min_workers: 4\n max_workers: 6\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/output.txt", + "q": { + "overwrite": "true" + }, + "raw_body": "\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n=== Cluster should exist with num_workers after bundle deployment:\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": 2,\n \"autoscale\": null\n}\n\n=== Adding autoscale section should call update API on stopped cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e jq select(.method == \"POST\" and (.path | contains(\"/clusters/edit\"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/edit\",\n \"body\": {\n \"autoscale\": {\n \"max_workers\": 4,\n \"min_workers\": 2\n },\n \"autotermination_minutes\": 60,\n \"cluster_id\": \"[CLUSTER_ID]\",\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"node_type_id\": \"[NODE_TYPE_ID]\",\n \"spark_version\": \"13.3.x-snapshot-scala2.12\"\n }\n}\n\n=== Cluster should have autoscale\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": null,\n \"autoscale\": {\n \"max_workers\": 4,\n \"min_workers\": 2\n }\n}\n\n=== Changing autoscale should call update API on stopped cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e jq select(.method == \"POST\" and (.path | contains(\"/clusters/edit\"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/edit\",\n \"body\": {\n \"autoscale\": {\n \"max_workers\": 5,\n \"min_workers\": 3\n },\n \"autotermination_minutes\": 60,\n \"cluster_id\": \"[CLUSTER_ID]\",\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"node_type_id\": \"[NODE_TYPE_ID]\",\n \"spark_version\": \"13.3.x-snapshot-scala2.12\"\n }\n}\n\n=== Cluster should have new autoscale\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": null,\n \"autoscale\": {\n \"max_workers\": 5,\n \"min_workers\": 3\n }\n}\n\n=== Starting the cluster\n{\n \"autoscale\": {\n \"max_workers\":5,\n \"min_workers\":3\n },\n \"autotermination_minutes\":60,\n \"aws_attributes\": {\n \"availability\":\"SPOT_WITH_FALLBACK\",\n \"zone_id\":\"us-east-1c\"\n },\n \"cluster_id\":\"[CLUSTER_ID]\",\n \"cluster_name\":\"test-cluster-[UNIQUE_NAME]\",\n \"driver_node_type_id\":\"[NODE_TYPE_ID]\",\n \"enable_elastic_disk\":false,\n \"node_type_id\":\"[NODE_TYPE_ID]\",\n \"spark_version\":\"13.3.x-snapshot-scala2.12\",\n \"state\":\"RUNNING\"\n}\n\n=== Changing autoscale should call resize API on running cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.plan_.terraform.txt", + "q": { + "overwrite": "true" + }, + "raw_body": "create clusters.test_cluster\n\nPlan: 1 to add, 0 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", + "q": { + "overwrite": "true" + }, + "body": { + "version": 1, + "seq": 4, + "cli_version": "[DEV_VERSION]", + "timestamp": "[TIMESTAMP]", + "files": [ + { + "local_path": "databricks.yml.tmpl", + "is_notebook": false + }, + { + "local_path": "out.plan_.terraform.txt", + "is_notebook": false + }, + { + "local_path": "out.requests.txt", + "is_notebook": false + }, + { + "local_path": "script", + "is_notebook": false + }, + { + "local_path": "hello_world.py", + "is_notebook": false + }, + { + "local_path": "out.plan_.terraform.json", + "is_notebook": false + }, + { + "local_path": "output.txt", + "is_notebook": false + }, + { + "local_path": "repls.json", + "is_notebook": false + }, + { + "local_path": "test.toml", + "is_notebook": false + }, + { + "local_path": "ACC_REPLS", + "is_notebook": false + }, + { + "local_path": "databricks.yml", + "is_notebook": false + } + ], + "id": "[UUID]" + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index 78b00d25ee..b4fa1c0bbe 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -106,63 +106,5 @@ Deployment complete! >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.1/clusters/resize", - "body": { - "autoscale": { - "max_workers": 6, - "min_workers": 4 - }, - "cluster_id": "[CLUSTER_ID]" - } -} - -=== Cluster should have new autoscale -{ - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "num_workers": null, - "autoscale": { - "max_workers": 6, - "min_workers": 4 - } -} - -=== Removing autoscale section should call resize API - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.1/clusters/resize", - "body": { - "cluster_id": "[CLUSTER_ID]", - "num_workers": 3 - } -} - -=== Cluster should have num_workers -{ - "cluster_name": "test-cluster-[UNIQUE_NAME]", - "num_workers": 3, - "autoscale": null -} - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.test_cluster - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Deleting files... -Destroy complete! +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt new file mode 100644 index 0000000000..4569fef1ae --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt @@ -0,0 +1,63 @@ +{ + "method": "GET", + "path": "/api/2.1/clusters/get", + "q": { + "cluster_id": "[CLUSTER_ID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "q": { + "overwrite": "false" + }, + "body": { + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index 43c78fa780..1b4975b25a 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -121,10 +121,5 @@ Deployment complete! } >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.clusters.test_cluster -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Deleting files... -Destroy complete! +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json index 4575a4141d..e69de29bb2 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json @@ -1,11 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.job_with_permissions": { - "action": "skip" - }, - "resources.jobs.job_with_permissions.permissions": { - "action": "create" - } - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json deleted file mode 100644 index a7c2467234..0000000000 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.job_with_permissions": { - "action": "skip" - }, - "resources.jobs.job_with_permissions.permissions": { - "action": "skip" - } - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json deleted file mode 100644 index f66a05564c..0000000000 --- a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/delete", - "body": { - "job_id": [JOB_WITH_PERMISSIONS_ID] - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index ab4953f6b5..f2eb71647e 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -279,27 +279,8 @@ DIFF jobs/update/out.requests_delete_all.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -DIFF jobs/update/out.requests_destroy.direct.json ---- jobs/update/out.requests_destroy.direct.json -+++ jobs/update/out.requests_destroy.terraform.json -@@ -1,4 +1,16 @@ - [ -+ { -+ "body": { -+ "access_control_list": [ -+ { -+ "permission_level": "IS_OWNER", -+ "user_name": "[USERNAME]" -+ } -+ ] -+ }, -+ "method": "PUT", -+ "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" -+ }, - { - "body": { - "job_id": "[JOB_WITH_PERMISSIONS_ID]" -EXACT jobs/update/out.requests_set_empty.direct.json +ERROR jobs/update/out.requests_destroy.direct.json: Missing terraform file jobs/update/out.requests_destroy.terraform.json +ERROR jobs/update/out.requests_set_empty.direct.json: Missing terraform file jobs/update/out.requests_set_empty.terraform.json MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json deleted file mode 100644 index 586aab2164..0000000000 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.pipelines.foo": { - "action": "skip" - }, - "resources.pipelines.foo.permissions": { - "action": "create" - } - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt b/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt new file mode 100644 index 0000000000..48756d15dd --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt @@ -0,0 +1,324 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[FOO_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[FOO_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock", + "q": { + "overwrite": "false" + }, + "body": { + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/databricks.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "bundle:\n name: permissions-test\n\nresources:\n pipelines:\n foo:\n name: foo\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.plan_delete_all.terraform.json", + "q": { + "overwrite": "true" + }, + "body": { + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.pipelines.foo": { + "action": "skip" + }, + "resources.pipelines.foo.permissions": { + "action": "delete" + } + } + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.requests_delete_one.json", + "q": { + "overwrite": "true" + }, + "body": { + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "viewer@example.com" + }, + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.requests.txt", + "q": { + "overwrite": "true" + }, + "raw_body": "{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/preview/scim/v2/Me\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/permissions/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/preview/scim/v2/Me\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/permissions/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\",\n \"q\": {\n \"overwrite\": \"false\"\n },\n \"body\": {\n \"ID\": \"[UUID]\",\n \"AcquisitionTime\": \"[TIMESTAMP]\",\n \"IsForced\": false,\n \"User\": \"[USERNAME]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/export\",\n \"q\": {\n \"direct_download\": \"true\",\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/delete\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal\",\n \"recursive\": true\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/mkdirs\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files\"\n }\n}\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/output.txt", + "q": { + "overwrite": "true" + }, + "raw_body": "\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n\u003e\u003e\u003e [CLI] permissions get pipelines [FOO_ID]\n{\n \"access_control_list\": [\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"CAN_VIEW\"\n }\n ],\n \"display_name\":\"viewer@example.com\",\n \"user_name\":\"viewer@example.com\"\n },\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"CAN_MANAGE\"\n }\n ],\n \"group_name\":\"data-team\"\n },\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"IS_OWNER\"\n }\n ],\n \"display_name\":\"[USERNAME]\",\n \"user_name\":\"[USERNAME]\"\n }\n ],\n \"object_id\":\"/pipelines/[FOO_ID]\",\n \"object_type\":\"pipelines\"\n}\n\n=== Update one permission and deploy again\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n=== Delete one permission and deploy again\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n=== Delete the whole block and deploy again\n\n\u003e\u003e\u003e cat databricks.yml\nbundle:\n name: permissions-test\n\nresources:\n pipelines:\n foo:\n name: foo\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", + "q": { + "overwrite": "true" + }, + "body": { + "version": 1, + "seq": 4, + "cli_version": "[DEV_VERSION]", + "timestamp": "[TIMESTAMP]", + "files": [ + { + "local_path": "output.txt", + "is_notebook": false + }, + { + "local_path": "ACC_REPLS", + "is_notebook": false + }, + { + "local_path": "databricks.yml.saved", + "is_notebook": false + }, + { + "local_path": "out.plan_create.terraform.json", + "is_notebook": false + }, + { + "local_path": "out.plan_delete_one.terraform.json", + "is_notebook": false + }, + { + "local_path": "out.plan_delete_all.terraform.json", + "is_notebook": false + }, + { + "local_path": "out.requests.txt", + "is_notebook": false + }, + { + "local_path": "repls.json", + "is_notebook": false + }, + { + "local_path": "script", + "is_notebook": false + }, + { + "local_path": "out.plan_update.terraform.json", + "is_notebook": false + }, + { + "local_path": "out.requests_delete_one.json", + "is_notebook": false + }, + { + "local_path": "test.toml", + "is_notebook": false + }, + { + "local_path": "databricks.yml", + "is_notebook": false + }, + { + "local_path": "out.requests_create.json", + "is_notebook": false + }, + { + "local_path": "out.requests_update.json", + "is_notebook": false + } + ], + "id": "[UUID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[FOO_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]" +} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json b/acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json b/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json deleted file mode 100644 index 2067ef6365..0000000000 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[FOO_ID]", - "body": { - "access_control_list": [ - { - "permission_level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "group_name": "data-team", - "permission_level": "CAN_MANAGE" - }, - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index 84b53daf5c..0f3c3461d8 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -94,39 +94,5 @@ resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> print_sorted_requests - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== Restore original config - ->>> [CLI] bundle plan -o json - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> print_requests - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.pipelines.foo - -This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: - delete resources.pipelines.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default -Deleting files... -Destroy complete! +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json index 6716f74d4f..e69de29bb2 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json @@ -1,11 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.pipelines.nyc_taxi_pipeline": { - "action": "create" - }, - "resources.pipelines.nyc_taxi_pipeline.permissions": { - "action": "create" - } - } -} diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 22c5698c64..9de2b76caa 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -1,3 +1,12 @@ +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/.gitignore +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/deployment.json +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/resources.json +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/sync-snapshots/adf1b[NUMID]fee.json +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/.terraform.lock.hcl +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/bundle.tf.json +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/.terraform.lock.hcl +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/bundle.tf.json +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.gitignore --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -34,4 +34,6 @@ @@ -7,6 +16,8 @@ + artifacts_dynamic_version: true prod: mode: production +Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/dist/.gitignore +Only in output: my_default_python/out.gitignore --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,8 +5,7 @@ diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json deleted file mode 100644 index 103b8a29e3..0000000000 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.sample_job": { - "action": "skip" - }, - "resources.jobs.sample_job.permissions": { - "action": "skip" - }, - "resources.pipelines.my_default_python_etl": { - "action": "skip" - }, - "resources.pipelines.my_default_python_etl.permissions": { - "action": "skip" - } - } -} diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt deleted file mode 100644 index c6d26be260..0000000000 --- a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt +++ /dev/null @@ -1,225 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/pipelines", - "body": { - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" - }, - "edition": "ADVANCED", - "environment": { - "dependencies": [ - "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" - ] - }, - "libraries": [ - { - "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" - } - } - ], - "name": "my_default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", - "schema": "prod" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", - "recursive": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "job_clusters": [ - { - "job_cluster_key": "job_cluster", - "new_cluster": { - "autoscale": { - "max_workers": 4, - "min_workers": 1 - }, - "data_security_mode": "SINGLE_USER", - "node_type_id": "[NODE_TYPE_ID]", - "spark_version": "16.4.x-scala2.12" - } - } - ], - "max_concurrent_runs": 1, - "name": "sample_job", - "parameters": [ - { - "default": "hive_metastore", - "name": "catalog" - }, - { - "default": "prod", - "name": "schema" - } - ], - "queue": { - "enabled": true - }, - "tasks": [ - { - "job_cluster_key": "job_cluster", - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" - } - ], - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", - "source": "WORKSPACE" - }, - "task_key": "notebook_task" - }, - { - "depends_on": [ - { - "task_key": "notebook_task" - } - ], - "job_cluster_key": "job_cluster", - "libraries": [ - { - "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" - } - ], - "python_wheel_task": { - "entry_point": "main", - "package_name": "my_default_python", - "parameters": [ - "--catalog", - "hive_metastore", - "--schema", - "prod" - ] - }, - "task_key": "python_wheel_task" - }, - { - "depends_on": [ - { - "task_key": "notebook_task" - } - ], - "pipeline_task": { - "pipeline_id": "[UUID]" - }, - "task_key": "refresh_pipeline" - } - ], - "trigger": { - "pause_status": "UNPAUSED", - "periodic": { - "interval": 1, - "unit": "DAYS" - } - } - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/directories/[NUMID]", - "body": { - "access_control_list": [ - { - "permission_level": "CAN_MANAGE", - "user_name": "[USERNAME]" - } - ] - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.txt b/acceptance/bundle/templates/default-python/classic/out.requests.txt new file mode 100644 index 0000000000..00bbeba373 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.requests.txt @@ -0,0 +1,627 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.2/jobs/get", + "q": { + "job_id": "[NUMID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" +} +{ + "method": "GET", + "path": "/api/2.0/pipelines/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.2/jobs/get", + "q": { + "job_id": "[NUMID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock", + "q": { + "overwrite": "false" + }, + "body": { + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl", + "q": { + "overwrite": "true" + }, + "raw_body": "PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001d\u0000\u0000\u0000my_default_python/__init__.py\u0003\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffd٤\\\u0001\u0000\u0000\ufffd\u0002\u0000\u0000\u0019\u0000\u0000\u0000my_default_python/main.py}\ufffd\ufffdn\ufffd0\u0010\ufffdw?\ufffd\ufffd]\u0012\t\ufffduA\ufffd\u0010Q\ufffd\u000e\ufffd\ufffd\u0004\ufffd-\u0013;\ufffd\u0010\ufffd\ufffd\ufffd\bP\ufffdw\ufffd\ufffd\u0018\ufffd\u0001\ufffdK\u0014\ufffd\ufffd\ufffd\ufffd\ufffd;mz\ufffd\u003c\b\ufffd\ufffd\ufffdCEjg\rH\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\f呹\ufffd\ufffd\ufffd(\ufffd\u0013\ufffd\ufffd\u003cN\ufffd\ufffdr\ufffdj1\ufffd\ufffd\ufffdW\ufffd\ufffd\ufffd\ufffdxq\ufffdH\b\tQ0BwY\ufffd \u0010\ufffd\u0013\ufffd\ufffd\ufffd\u0014\"T\ufffd\u0018\ufffd\ufffdy\ufffd;5\ufffd\u001f\ufffd\ufffd\u003cF(:qP\ufffd]\ufffd2\ufffd\ufffd1\ufffdEj\u003cRa\ufffdt\ufffd\ufffd\ufffd\n\ufffdrw\r\u0007\ufffd\ufffd\ufffd\ufffd\rT᮵{\b\ufffd\u0000\ufffdF\u00191f\u0017Fy\ufffd\ufffd\ufffdb\ufffd\ufffdOQ\u0026\ufffd\ufffd77\u0019\ufffd\ufffdS\u0002:\u0003\ufffdN\ufffdvJ\u0016[7\ufffd5S\ufffdǒ\ufffdaxYR\ufffdϨ\ufffd,'\ufffdA\u001b\u0015\ufffd\ufffd(H\ufffd}\ufffd\ufffd\b\ufffd)0\u003c\ufffdYM\ufffd6+X\ufffd\ufffd\ufffd\ufffd\ufffd\u0015\ufffd\ufffdl,\ufffd~h\ufffd\u0010\ufffd,\ufffdV\u001feb\ufffd\ufffd#\ufffd\u001c\ufffd.\ufffd\ufffd\ufffdZ\ufffda@\u000f\ufffd\u000e\ufffdE\ufffdN3\ufffd8w\u0001\u0018\ufffd\ufffd\ufffd(\ufffdq6\ufffd\u003c\ufffd\u003c\ufffdf9\ufffdƞ\ufffd琜\ufffd\u001a8\ufffdB\ufffd9\ufffd\ufffd\u0000\ufffd\ufffd\ufffd\u001a\ufffd\ufffdi7\ufffd=!\ufffdPK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPXA\u0002\ufffd\ufffd\u0000\u0000\u0000\ufffd\u0000\u0000\u0000\u001a\u0000\u0000\u0000my_default_python/taxis.pyE\ufffd\ufffd\u000e\ufffd \fDw\ufffd\ufffdbJ\ufffd\ufffd\u0003:t\ufffd\ufffd\u001b\ufffd\u001b\ufffd\ufffd\u0002\ufffd\ufffdDj\ufffd\u003e\ufffd\ufffd\ufffdm\ufffd{\ufffd\ufffd\ufffd-\ufffd\ufffdJ/\t˪\ufffdnE\ufffds\r\ufffd!\ufffd\ufffdI\u0005-$\ufffd\ufffd\u001d,\ufffdu\ufffd\ufffd\ufffd/}\ufffd\ufffd$\ufffd\ufffd\u0018\ufffd؃\u000f\ufffd\ufffd\u0014\ufffd\\\ufffd\u0013t\u0018\ufffd\ufffd\ufffd3w\u0003M\ufffdکQ\ufffd(\ufffdԵ\ufffdͽR\ufffd\ufffdK\ufffd\ufffd\ufffd09l\ufffdE\u001e\ufffdR*\ufffd\u0015\ufffd\ufffd\ufffd\u0016V\tE\ufffdhNPK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPG\ufffdh\ufffdl\u0000\u0000\u0000y\u0000\u0000\u0000*\u0000\u0000\u0000my_default_python-0.0.1.dist-info/METADATA\ufffdM-ILI,I\ufffd\rK-*\ufffd\ufffdϳR0\ufffd3\ufffd\ufffdK\ufffdM\ufffdRȭ\ufffdOIMK,\ufffd)\ufffd/\ufffd,\ufffd\ufffd\ufffd\ufffd\ufffd+2\ufffd3\ufffd3\ufffdr,\u0005\ufffd\u0016Y)\ufffd\ufffd\u0016\ufffd\ufffd\u00169\ufffd\ufffdI*\ufffdL\ufffd.\ufffdK\ufffd\ufffd\ufffd\nJ-,\ufffd,J-\ufffd\r\u0000k\ufffdR\ufffd1\ufffd34ֱ\ufffd\u0005R\u0006\\\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP_\ufffdoVV\u0000\u0000\u0000W\u0000\u0000\u0000'\u0000\u0000\u0000my_default_python-0.0.1.dist-info/WHEEL\u000b\ufffdHM\ufffd\ufffd\rK-*\ufffd\ufffdϳR0\ufffd3\ufffdrO\ufffdK-J,\ufffd/\ufffdR\ufffdH,I\ufffd\ufffd\ufffd\ufffdK\u0007J\u0018Y\u0002\ufffd\ufffd\ufffd\ufffdKt=\ufffdu\u0003J\ufffdRs2\ufffd\ufffd\u0014J\ufffdJS\ufffdB\u0012ӭ\u0014\n*\ufffdu\ufffd\ufffd\ufffdRu\u0013\ufffd*\ufffd\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffdF\ufffd\ufffd2\u0000\u0000\u00005\u0000\u0000\u00002\u0000\u0000\u0000my_default_python-0.0.1.dist-info/entry_points.txt\ufffdN\ufffd\ufffd+\ufffd\ufffdI\ufffd/N.\ufffd,()\ufffd\ufffd\ufffdM\ufffd\ufffdS\ufffdUȭ\ufffdOIMK,\ufffd)\ufffd/\ufffd,\ufffd\ufffd\ufffd\ufffd\u0003IX\ufffd\b.\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffdl\ufffd`\u0001\u0000\u0000J\u0002\u0000\u0000(\u0000\u0000\u0000my_default_python-0.0.1.dist-info/RECORD\ufffd\ufffd\ufffdR\ufffd0\u0000@ѽ\ufffd\u0012\u0010\ufffd@\ufffdpA[l)P\ufffd@\ufffdn2H\u0003MyE\t/\ufffd^7\ufffdqF\u0017\ufffd\ufffdso=\ufffd\u0013\ufffdӾ\u0012\ufffd\ufffd\ufffd\ufffd6\ufffd\ufffd\ufffd\ufffd\tBd\u003e\ufffd\ufffd\ufffdjPz4\ufffdV\ufffd/h\ufffdĩD\"\ufffdN$\ufffd[\ufffd\u003e\ufffdaYý\ufffdw\ufffdq\ufffdp\ufffd\ufffds\f\ufffd\ufffd\ufffd\u000fY\ufffd\ufffd\ufffd\ufffdy\ufffd߆\ufffdqd4\ufffd\u0007\ufffd\ufffd\ufffd\ufffd\ufffd_\ufffd\ufffd\ufffd\u0018J\ufffd\ufffd#\u0011\ufffd~ȳ\ufffdw\ufffd\b\ufffd\u0010\ufffd\u0003\ufffdtb\ufffd/\ufffd\ufffd\ufffdh\u000b-d\ufffdɉ\ufffd\u003c\ufffd\ufffd$˦\u000b\ufffd\ufffd\u0000\u0006F\u0011\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdئ\nP\u0017\ufffd\ufffd()\ufffd\"\ufffd\ufffd\ufffduBbM\ufffd\ufffd{Vd\ufffd\ufffdȼ\u0016v\ufffd1 g\ufffd/.U\u003cJ4\ufffd\u0007\ufffdmO\ufffd3\ufffdgqءG\u001dV\ufffd\u0002\ufffd\ufffd\ufffd\u0006TM\ufffd\ufffd\ufffdl-˽\ufffdA\ufffd\ufffdS\ufffd\ufffd\ufffdE\ufffdZz\ufffd\ufffd\u0010\ufffd\ufffd\u0026a\ufffd\ufffdR\ufffd,Ӧ\ufffdfS\ufffd\ufffdI\u003e\u0002dܠ\ufffdF|΄\ufffd\ufffd\u0011\ufffd,\u0026q\r9C\u001dn\ufffd[\ufffd\nK5\ufffdF\ufffdi\ufffd\ufffd\ufffdV=x\ufffd\ufffdB\ufffd\ufffdj\ufffd\u000b\ufffd\ufffdK4\u0002\ufffdpC(\ufffdV/\ufffd\u001a\ufffd\ufffdoPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001d\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\u0000\u0000\u0000\u0000my_default_python/__init__.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffd٤\\\u0001\u0000\u0000\ufffd\u0002\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd=\u0000\u0000\u0000my_default_python/main.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPXA\u0002\ufffd\ufffd\u0000\u0000\u0000\ufffd\u0000\u0000\u0000\u001a\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\u0001\u0000\u0000my_default_python/taxis.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPG\ufffdh\ufffdl\u0000\u0000\u0000y\u0000\u0000\u0000*\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001\ufffd\u0002\u0000\u0000my_default_python-0.0.1.dist-info/METADATAPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP_\ufffdoVV\u0000\u0000\u0000W\u0000\u0000\u0000'\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001H\u0003\u0000\u0000my_default_python-0.0.1.dist-info/WHEELPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffdF\ufffd\ufffd2\u0000\u0000\u00005\u0000\u0000\u00002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001\ufffd\u0003\u0000\u0000my_default_python-0.0.1.dist-info/entry_points.txtPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffdl\ufffd`\u0001\u0000\u0000J\u0002\u0000\u0000(\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001e\u0004\u0000\u0000my_default_python-0.0.1.dist-info/RECORDPK\u0005\u0006\u0000\u0000\u0000\u0000\u0007\u0000\u0007\u0000=\u0002\u0000\u0000\u000b\u0006\u0000\u0000\u0000\u0000" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/sample_zones_my_default_python.py", + "q": { + "overwrite": "true" + }, + "raw_body": "from pyspark import pipelines as dp\nfrom pyspark.sql.functions import col, sum\n\n\n# This file defines a sample transformation.\n# Edit the sample below or add new transformations\n# using \"+ Add\" in the file browser.\n\n\n@dp.table\ndef sample_zones_my_default_python():\n # Read from the \"sample_trips\" table, then sum all the fares\n return (\n spark.read.table(f\"sample_trips_my_default_python\")\n .groupBy(col(\"pickup_zip\"))\n .agg(sum(\"fare_amount\").alias(\"total_fare\"))\n )\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/main.py", + "q": { + "overwrite": "true" + }, + "raw_body": "import argparse\nfrom databricks.sdk.runtime import spark\nfrom my_default_python import taxis\n\n\ndef main():\n # Process command-line arguments\n parser = argparse.ArgumentParser(\n description=\"Databricks job with catalog and schema parameters\",\n )\n parser.add_argument(\"--catalog\", required=True)\n parser.add_argument(\"--schema\", required=True)\n args = parser.parse_args()\n\n # Set the default catalog and schema\n spark.sql(f\"USE CATALOG {args.catalog}\")\n spark.sql(f\"USE SCHEMA {args.schema}\")\n\n # Example: just find all taxis from a sample catalog\n taxis.find_all_taxis().show(5)\n\n\nif __name__ == \"__main__\":\n main()\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/databricks.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "# This is a Databricks asset bundle definition for my_default_python.\n# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.\nbundle:\n name: my_default_python\n uuid: [UUID]\n\ninclude:\n - resources/*.yml\n - resources/*/*.yml\n\nartifacts:\n python_artifact:\n type: whl\n build: uv build --wheel\n\n# Variable declarations. These variables are assigned in the dev/prod targets below.\nvariables:\n catalog:\n description: The catalog to use\n schema:\n description: The schema to use\n\ntargets:\n dev:\n # The default target uses 'mode: development' to create a development copy.\n # - Deployed resources get prefixed with '[dev my_user_name]'\n # - Any job schedules and triggers are paused by default.\n # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.\n mode: development\n default: true\n workspace:\n host: [DATABRICKS_URL]\n variables:\n catalog: hive_metastore\n schema: ${workspace.current_user.short_name}\n presets:\n artifacts_dynamic_version: true\n prod:\n mode: production\n workspace:\n host: [DATABRICKS_URL]\n # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.\n root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}\n variables:\n catalog: hive_metastore\n schema: prod\n permissions:\n - user_name: [USERNAME]\n level: CAN_MANAGE\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests/conftest.py", + "q": { + "overwrite": "true" + }, + "raw_body": "\"\"\"This file configures pytest, initializes Databricks Connect, and provides fixtures for Spark and loading test data.\"\"\"\n\nimport os, sys, pathlib\nfrom contextlib import contextmanager\n\n\ntry:\n from databricks.connect import DatabricksSession\n from databricks.sdk import WorkspaceClient\n from pyspark.sql import SparkSession\n import pytest\n import json\n import csv\n import os\nexcept ImportError:\n raise ImportError(\n \"Test dependencies not found.\\n\\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv.\"\n )\n\n\n@pytest.fixture()\ndef spark() -\u003e SparkSession:\n \"\"\"Provide a SparkSession fixture for tests.\n\n Minimal example:\n def test_uses_spark(spark):\n df = spark.createDataFrame([(1,)], [\"x\"])\n assert df.count() == 1\n \"\"\"\n return DatabricksSession.builder.getOrCreate()\n\n\n@pytest.fixture()\ndef load_fixture(spark: SparkSession):\n \"\"\"Provide a callable to load JSON or CSV from fixtures/ directory.\n\n Example usage:\n\n def test_using_fixture(load_fixture):\n data = load_fixture(\"my_data.json\")\n assert data.count() \u003e= 1\n \"\"\"\n\n def _loader(filename: str):\n path = pathlib.Path(__file__).parent.parent / \"fixtures\" / filename\n suffix = path.suffix.lower()\n if suffix == \".json\":\n rows = json.loads(path.read_text())\n return spark.createDataFrame(rows)\n if suffix == \".csv\":\n with path.open(newline=\"\") as f:\n rows = list(csv.DictReader(f))\n return spark.createDataFrame(rows)\n raise ValueError(f\"Unsupported fixture type for: {filename}\")\n\n return _loader\n\n\ndef _enable_fallback_compute():\n \"\"\"Enable serverless compute if no compute is specified.\"\"\"\n conf = WorkspaceClient().config\n if conf.serverless_compute_id or conf.cluster_id or os.environ.get(\"SPARK_REMOTE\"):\n return\n\n url = \"https://docs.databricks.com/dev-tools/databricks-connect/cluster-config\"\n print(\"☁️ no compute specified, falling back to serverless compute\", file=sys.stderr)\n print(f\" see {url} for manual configuration\", file=sys.stdout)\n\n os.environ[\"DATABRICKS_SERVERLESS_COMPUTE_ID\"] = \"auto\"\n\n\n@contextmanager\ndef _allow_stderr_output(config: pytest.Config):\n \"\"\"Temporarily disable pytest output capture.\"\"\"\n capman = config.pluginmanager.get_plugin(\"capturemanager\")\n if capman:\n with capman.global_and_fixture_disabled():\n yield\n else:\n yield\n\n\ndef pytest_configure(config: pytest.Config):\n \"\"\"Configure pytest session.\"\"\"\n with _allow_stderr_output(config):\n _enable_fallback_compute()\n\n # Initialize Spark session eagerly, so it is available even when\n # SparkSession.builder.getOrCreate() is used. For DB Connect 15+,\n # we validate version compatibility with the remote cluster.\n if hasattr(DatabricksSession.builder, \"validateSession\"):\n DatabricksSession.builder.validateSession().getOrCreate()\n else:\n DatabricksSession.builder.getOrCreate()\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources/sample_job.job.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "# A sample job for my_default_python.\n\nresources:\n jobs:\n sample_job:\n name: sample_job\n\n trigger:\n # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger\n periodic:\n interval: 1\n unit: DAYS\n\n #email_notifications:\n # on_failure:\n # - your_email@example.com\n\n parameters:\n - name: catalog\n default: ${var.catalog}\n - name: schema\n default: ${var.schema}\n\n tasks:\n - task_key: notebook_task\n notebook_task:\n notebook_path: ../src/sample_notebook.ipynb\n job_cluster_key: job_cluster\n libraries:\n # By default we just include the .whl file generated for the my_default_python package.\n # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html\n # for more information on how to add other libraries.\n - whl: ../dist/*.whl\n - task_key: python_wheel_task\n depends_on:\n - task_key: notebook_task\n python_wheel_task:\n package_name: my_default_python\n entry_point: main\n parameters:\n - \"--catalog\"\n - \"${var.catalog}\"\n - \"--schema\"\n - \"${var.schema}\"\n job_cluster_key: job_cluster\n libraries:\n # By default we just include the .whl file generated for the my_default_python package.\n # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html\n # for more information on how to add other libraries.\n - whl: ../dist/*.whl\n - task_key: refresh_pipeline\n depends_on:\n - task_key: notebook_task\n pipeline_task:\n pipeline_id: ${resources.pipelines.my_default_python_etl.id}\n\n job_clusters:\n - job_cluster_key: job_cluster\n new_cluster:\n spark_version: 16.4.x-scala2.12\n node_type_id: [NODE_TYPE_ID]\n data_security_mode: SINGLE_USER\n autoscale:\n min_workers: 1\n max_workers: 4\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/README.md", + "q": { + "overwrite": "true" + }, + "raw_body": "# my_default_python\n\nThis folder defines all source code for the my_default_python pipeline:\n\n- `explorations/`: Ad-hoc notebooks used to explore the data processed by this pipeline.\n- `transformations/`: All dataset definitions and transformations.\n- `utilities/` (optional): Utility functions and Python modules used in this pipeline.\n- `data_sources/` (optional): View definitions describing the source data for this pipeline.\n\n## Getting Started\n\nTo get started, go to the `transformations` folder -- most of the relevant source code lives there:\n\n* By convention, every dataset under `transformations` is in a separate file.\n* Take a look at the sample called \"sample_trips_my_default_python.py\" to get familiar with the syntax.\n Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html.\n* If you're using the workspace UI, use `Run file` to run and preview a single transformation.\n* If you're using the CLI, use `databricks bundle run my_default_python_etl --select sample_trips_my_default_python` to run a single transformation.\n\nFor more tutorials and reference material, see https://docs.databricks.com/dlt.\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/taxis.py", + "q": { + "overwrite": "true" + }, + "raw_body": "from databricks.sdk.runtime import spark\nfrom pyspark.sql import DataFrame\n\n\ndef find_all_taxis() -\u003e DataFrame:\n \"\"\"Find all taxi data.\"\"\"\n return spark.read.table(\"samples.nyctaxi.trips\")\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/pyproject.toml", + "q": { + "overwrite": "true" + }, + "raw_body": "[project]\nname = \"my_default_python\"\nversion = \"0.0.1\"\nauthors = [{ name = \"[USERNAME]\" }]\nrequires-python = \"\u003e=3.10,\u003c3.13\"\ndependencies = [\n # Any dependencies for jobs and pipelines in this project can be added here\n # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies\n #\n # LIMITATION: for pipelines, dependencies are cached during development;\n # add dependencies to the 'environment' section of your pipeline.yml file instead\n]\n\n[dependency-groups]\ndev = [\n \"pytest\",\n \"ruff\",\n \"databricks-dlt\",\n \"databricks-connect\u003e=15.4,\u003c15.5\",\n \"ipykernel\",\n]\n\n[project.scripts]\nmain = \"my_default_python.main:main\"\n\n[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[tool.ruff]\nline-length = 120\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/extensions.json", + "q": { + "overwrite": "true" + }, + "body": { + "recommendations": [ + "databricks.databricks", + "redhat.vscode-yaml", + "charliermarsh.ruff" + ] + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests/sample_taxis_test.py", + "q": { + "overwrite": "true" + }, + "raw_body": "from databricks.sdk.runtime import spark\nfrom pyspark.sql import DataFrame\nfrom my_default_python import taxis\n\n\ndef test_find_all_taxis():\n results = taxis.find_all_taxis()\n assert results.count() \u003e 5\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/__init__.py", + "q": { + "overwrite": "true" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources/my_default_python_etl.pipeline.yml", + "q": { + "overwrite": "true" + }, + "raw_body": "# The main pipeline for my_default_python\n\nresources:\n pipelines:\n my_default_python_etl:\n name: my_default_python_etl\n ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:\n # catalog: ${var.catalog}\n schema: ${var.schema}\n root_path: \"../src/my_default_python_etl\"\n\n libraries:\n - glob:\n include: ../src/my_default_python_etl/transformations/**\n\n environment:\n dependencies:\n # We include every dependency defined by pyproject.toml by defining an editable environment\n # that points to the folder where pyproject.toml is deployed.\n - --editable ${workspace.file_path}\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/sample_trips_my_default_python.py", + "q": { + "overwrite": "true" + }, + "raw_body": "from pyspark import pipelines as dp\nfrom pyspark.sql.functions import col\n\n\n# This file defines a sample transformation.\n# Edit the sample below or add new transformations\n# using \"+ Add\" in the file browser.\n\n\n@dp.table\ndef sample_trips_my_default_python():\n return spark.read.table(\"samples.nyctaxi.trips\")\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/README.md", + "q": { + "overwrite": "true" + }, + "raw_body": "# my_default_python\n\nThe 'my_default_python' project was generated by using the default-python template.\n\n* `src/`: Python source code for this project.\n * `src/my_default_python/`: Shared Python code that can be used by jobs and pipelines.\n* `resources/`: Resource configurations (jobs, pipelines, etc.)\n* `tests/`: Unit tests for the shared Python code.\n* `fixtures/`: Fixtures for data sets (primarily used for testing).\n\n\n## Getting started\n\nChoose how you want to work on this project:\n\n(a) Directly in your Databricks workspace, see\n https://docs.databricks.com/dev-tools/bundles/workspace.\n\n(b) Locally with an IDE like Cursor or VS Code, see\n https://docs.databricks.com/dev-tools/vscode-ext.html.\n\n(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html\n\nIf you're developing with an IDE, dependencies for this project should be installed using uv:\n\n* Make sure you have the UV package manager installed.\n It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/.\n* Run `uv sync --dev` to install the project's dependencies.\n\n\n# Using this project using the CLI\n\nThe Databricks workspace and IDE extensions provide a graphical interface for working\nwith this project. It's also possible to interact with it directly using the CLI:\n\n1. Authenticate to your Databricks workspace, if you have not done so already:\n ```\n $ databricks configure\n ```\n\n2. To deploy a development copy of this project, type:\n ```\n $ databricks bundle deploy --target dev\n ```\n (Note that \"dev\" is the default target, so the `--target` parameter\n is optional here.)\n\n This deploys everything that's defined for this project.\n For example, the default template would deploy a pipeline called\n `[dev yourname] my_default_python_etl` to your workspace.\n You can find that resource by opening your workpace and clicking on **Jobs \u0026 Pipelines**.\n\n3. Similarly, to deploy a production copy, type:\n ```\n $ databricks bundle deploy --target prod\n ```\n Note the default template has a includes a job that runs the pipeline every day\n (defined in resources/sample_job.job.yml). The schedule\n is paused when deploying in development mode (see\n https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).\n\n4. To run a job or pipeline, use the \"run\" command:\n ```\n $ databricks bundle run\n ```\n\n5. Finally, to run tests locally, use `pytest`:\n ```\n $ uv run pytest\n ```\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook.ipynb", + "q": { + "overwrite": "true" + }, + "body": { + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Default notebook\n", + "\n", + "This default notebook is executed using a Lakeflow job as defined in resources/sample_job.job.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Set default catalog and schema\n", + "catalog = dbutils.widgets.get(\"catalog\")\n", + "schema = dbutils.widgets.get(\"schema\")\n", + "spark.sql(f\"USE CATALOG {catalog}\")\n", + "spark.sql(f\"USE SCHEMA {schema}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append(\"../src\")\n", + "from my_default_python import taxis\n", + "\n", + "taxis.find_all_taxis().show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "environmentMetadata": { + "base_environment": "", + "dependencies": [ + "--editable .." + ], + "environment_version": "4" + }, + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "notebook", + "widgets": { + "catalog": { + "currentValue": "hive_metastore", + "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "validationRegex": null, + "widgetDisplayType": "Text" + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "defaultValue": "hive_metastore", + "label": "Catalog", + "name": "catalog", + "options": { + "autoCreated": false, + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" + } + }, + "schema": { + "currentValue": "[USERNAME]", + "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", + "typedWidgetInfo": { + "autoCreated": false, + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "validationRegex": null, + "widgetDisplayType": "Text" + }, + "parameterDataType": "String" + }, + "widgetInfo": { + "defaultValue": "default", + "label": "Schema", + "name": "schema", + "options": { + "autoCreated": false, + "validationRegex": null, + "widgetType": "text" + }, + "widgetType": "text" + } + } + } + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.gitignore", + "q": { + "overwrite": "true" + }, + "raw_body": ".databricks/\nbuild/\ndist/\n__pycache__/\n*.egg-info\n.venv/\nscratch/**\n!scratch/README.md\n**/explorations/**\n**/!explorations/README.md\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/__builtins__.pyi", + "q": { + "overwrite": "true" + }, + "raw_body": "# Typings for Pylance in Visual Studio Code\n# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md\nfrom databricks.sdk.runtime import *\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/settings.json", + "q": { + "overwrite": "true" + }, + "raw_body": "{\n \"jupyter.interactiveWindow.cellMarker.codeRegex\": \"^# COMMAND ----------|^# Databricks notebook source|^(#\\\\s*%%|#\\\\s*\\\\\u003ccodecell\\\\\u003e|#\\\\s*In\\\\[\\\\d*?\\\\]|#\\\\s*In\\\\[ \\\\])\",\n \"jupyter.interactiveWindow.cellMarker.default\": \"# COMMAND ----------\",\n \"python.testing.pytestArgs\": [\n \".\"\n ],\n \"files.exclude\": {\n \"**/*.egg-info\": true,\n \"**/__pycache__\": true,\n \".pytest_cache\": true,\n \"dist\": true,\n },\n \"files.associations\": {\n \"**/.gitkeep\": \"markdown\"\n },\n\n // Pylance settings (VS Code)\n // Set typeCheckingMode to \"basic\" to enable type checking!\n \"python.analysis.typeCheckingMode\": \"off\",\n \"python.analysis.extraPaths\": [\"src\", \"lib\", \"resources\"],\n \"python.analysis.diagnosticMode\": \"workspace\",\n \"python.analysis.stubPath\": \".vscode\",\n\n // Pyright settings (Cursor)\n // Set typeCheckingMode to \"basic\" to enable type checking!\n \"cursorpyright.analysis.typeCheckingMode\": \"off\",\n \"cursorpyright.analysis.extraPaths\": [\"src\", \"lib\", \"resources\"],\n \"cursorpyright.analysis.diagnosticMode\": \"workspace\",\n \"cursorpyright.analysis.stubPath\": \".vscode\",\n\n // General Python settings\n \"python.defaultInterpreterPath\": \"./.venv/bin/python\",\n \"python.testing.unittestEnabled\": false,\n \"python.testing.pytestEnabled\": true,\n \"[python]\": {\n \"editor.defaultFormatter\": \"charliermarsh.ruff\",\n \"editor.formatOnSave\": true,\n },\n}\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures/.gitkeep", + "q": { + "overwrite": "true" + }, + "raw_body": "# Test fixtures directory\n\nAdd JSON or CSV files here. In tests, use them with `load_fixture()`:\n\n```\ndef test_using_fixture(load_fixture):\n data = load_fixture(\"my_data.json\")\n assert len(data) \u003e= 1\n```\n" +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deployment.json", + "q": { + "overwrite": "true" + }, + "body": { + "version": 1, + "seq": 1, + "cli_version": "[DEV_VERSION]", + "timestamp": "[TIMESTAMP]", + "files": [ + { + "local_path": "tests/sample_taxis_test.py", + "is_notebook": false + }, + { + "local_path": "src/my_default_python/main.py", + "is_notebook": false + }, + { + "local_path": ".gitignore", + "is_notebook": false + }, + { + "local_path": ".vscode/settings.json", + "is_notebook": false + }, + { + "local_path": "databricks.yml", + "is_notebook": false + }, + { + "local_path": "fixtures/.gitkeep", + "is_notebook": false + }, + { + "local_path": "resources/sample_job.job.yml", + "is_notebook": false + }, + { + "local_path": "README.md", + "is_notebook": false + }, + { + "local_path": "resources/my_default_python_etl.pipeline.yml", + "is_notebook": false + }, + { + "local_path": "src/my_default_python/taxis.py", + "is_notebook": false + }, + { + "local_path": ".vscode/__builtins__.pyi", + "is_notebook": false + }, + { + "local_path": ".vscode/extensions.json", + "is_notebook": false + }, + { + "local_path": "pyproject.toml", + "is_notebook": false + }, + { + "local_path": "src/my_default_python_etl/transformations/sample_zones_my_default_python.py", + "is_notebook": false + }, + { + "local_path": "src/sample_notebook.ipynb", + "is_notebook": true + }, + { + "local_path": "tests/conftest.py", + "is_notebook": false + }, + { + "local_path": "src/my_default_python/__init__.py", + "is_notebook": false + }, + { + "local_path": "src/my_default_python_etl/README.md", + "is_notebook": false + }, + { + "local_path": "src/my_default_python_etl/transformations/sample_trips_my_default_python.py", + "is_notebook": false + } + ], + "id": "[UUID]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod" + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/directories/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index af2eb7495c..2f82504f39 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -62,14 +62,5 @@ Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan -t dev -o json Building python_artifact... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files... -Deploying resources... -Updating deployment state... -Deployment complete! ->>> [CLI] bundle plan -t prod -Building python_artifact... -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged - ->>> [CLI] bundle plan -t prod -o json -Building python_artifact... +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore new file mode 100644 index 0000000000..e566c51f74 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore @@ -0,0 +1,10 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt index 64e612c69f..4aee98da77 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt @@ -41,5 +41,5 @@ Destroy complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/X[UNIQUE_NAME]/prod/files... Deploying resources... Deployment complete! -Deleting files... -Destroy complete! + +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index daa2aba833..16845957c9 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -13,6 +13,15 @@ Note that [DATABRICKS_URL] is used for initialization. To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> diff.py [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output output/ +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/.gitignore +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/deployment.json +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/resources.json +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/sync-snapshots/adf1b[NUMID]fee.json +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/.terraform.lock.hcl +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/bundle.tf.json +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/.terraform.lock.hcl +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/bundle.tf.json +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.gitignore --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -32,5 +32,5 @@ @@ -29,6 +38,8 @@ To get started, refer to the project README.md file and the documentation at htt + catalog: customcatalog schema: prod permissions: +Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/dist/.gitignore +Only in output: my_default_python/out.gitignore --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,6 +5,5 @@ diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json deleted file mode 100644 index 7766755175..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.sample_job": { - "action": "skip" - }, - "resources.pipelines.my_default_python_etl": { - "action": "skip" - } - } -} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json deleted file mode 100644 index 103b8a29e3..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.jobs.sample_job": { - "action": "skip" - }, - "resources.jobs.sample_job.permissions": { - "action": "skip" - }, - "resources.pipelines.my_default_python_etl": { - "action": "skip" - }, - "resources.pipelines.my_default_python_etl.permissions": { - "action": "skip" - } - } -} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt deleted file mode 100644 index ae401a3c97..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt +++ /dev/null @@ -1,213 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/pipelines", - "body": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" - }, - "edition": "ADVANCED", - "environment": { - "dependencies": [ - "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" - ] - }, - "libraries": [ - { - "glob": { - "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" - } - } - ], - "name": "my_default_python_etl", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", - "schema": "prod", - "serverless": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", - "recursive": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" - } -} -{ - "method": "POST", - "path": "/api/2.2/jobs/create", - "body": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "environments": [ - { - "environment_key": "default", - "spec": { - "dependencies": [ - "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" - ], - "environment_version": "4" - } - } - ], - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "sample_job", - "parameters": [ - { - "default": "hive_metastore", - "name": "catalog" - }, - { - "default": "prod", - "name": "schema" - } - ], - "queue": { - "enabled": true - }, - "tasks": [ - { - "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", - "source": "WORKSPACE" - }, - "task_key": "notebook_task" - }, - { - "depends_on": [ - { - "task_key": "notebook_task" - } - ], - "environment_key": "default", - "python_wheel_task": { - "entry_point": "main", - "package_name": "my_default_python", - "parameters": [ - "--catalog", - "hive_metastore", - "--schema", - "prod" - ] - }, - "task_key": "python_wheel_task" - }, - { - "depends_on": [ - { - "task_key": "notebook_task" - } - ], - "pipeline_task": { - "pipeline_id": "[UUID]" - }, - "task_key": "refresh_pipeline" - } - ], - "trigger": { - "pause_status": "UNPAUSED", - "periodic": { - "interval": 1, - "unit": "DAYS" - } - } - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/directories/[NUMID]", - "body": { - "access_control_list": [ - { - "permission_level": "CAN_MANAGE", - "user_name": "[USERNAME]" - } - ] - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]", - "body": { - "access_control_list": [ - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } -} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.txt new file mode 100644 index 0000000000..77da8b8cec --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.txt @@ -0,0 +1,32 @@ +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" +} diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt index 6cad79ec1d..7e97876eb4 100644 --- a/acceptance/bundle/templates/default-python/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/serverless/output.txt @@ -60,22 +60,5 @@ Deployment complete! >>> [CLI] bundle plan -t dev Building python_artifact... -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ->>> [CLI] bundle plan -t dev -o json -Building python_artifact... - ->>> [CLI] bundle deploy -t prod -Building python_artifact... -Uploading dist/my_default_python-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> [CLI] bundle plan -t prod -Building python_artifact... -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged - ->>> [CLI] bundle plan -t prod -o json -Building python_artifact... +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore new file mode 100644 index 0000000000..e566c51f74 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore @@ -0,0 +1,10 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +scratch/** +!scratch/README.md +**/explorations/** +**/!explorations/README.md diff --git a/acceptance/cmd/workspace/apps/run-local/out.run.txt b/acceptance/cmd/workspace/apps/run-local/out.run.txt new file mode 100644 index 0000000000..033db8edfa --- /dev/null +++ b/acceptance/cmd/workspace/apps/run-local/out.run.txt @@ -0,0 +1,54 @@ + +>>> [CLI] apps run-local --prepare-environment --debug --port 8091 --debug-port 5252 --app-port 8090 +Using CPython 3.11.12 +Creating virtual environment at: .venv +Activate with: source .venv/bin/activate +Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 +Resolved 131 packages in 89ms +Uninstalled 2 packages in 38ms +Installed 2 packages in 31ms + - flask==3.1.1 + + flask==3.0.3 + - werkzeug==3.1.6 + + werkzeug==3.0.6 +Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 +Resolved 7 packages in 3ms +Uninstalled 2 packages in 34ms +Installed 2 packages in 11ms + - flask==3.0.3 + + flask==3.1.1 + - werkzeug==3.0.6 + + werkzeug==3.1.6 +Running command: uv run python -m debugpy --listen 5252 -m flask run +To debug your app, attach a debugger to port $(debug_port) +To access your app go to http://localhost:8091 +0.00s - Debugger warning: It seems that frozen modules are being used, which may +0.00s - make the debugger miss breakpoints. Please pass -Xfrozen_modules=off +0.00s - to python to disable frozen modules. +0.00s - Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation. +Traceback (most recent call last): + File "", line 198, in _run_module_as_main + File "", line 88, in _run_code + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/__main__.py", line 71, in + cli.main() + ~~~~~~~~^^ + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 501, in main + run() + ~~~^^ + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 369, in run_module + start_debugging(argv_0) + ~~~~~~~~~~~~~~~^^^^^^^^ + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 321, in start_debugging + debugpy.listen(options.address) + ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/public_api.py", line 31, in wrapper + return wrapped(*args, **kwargs) + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 132, in debug + log.reraise_exception("{0}() failed:", func.__name__, level="info") + ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 130, in debug + return func(address, settrace_kwargs, **kwargs) + File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 258, in listen + raise RuntimeError(str(endpoints["error"])) +RuntimeError: Can't listen for client connections: [Errno 48] Address already in use +Error: exit status 1 diff --git a/acceptance/cmd/workspace/apps/run-local/output.txt b/acceptance/cmd/workspace/apps/run-local/output.txt index 6b7b624360..cb7822a22d 100644 --- a/acceptance/cmd/workspace/apps/run-local/output.txt +++ b/acceptance/cmd/workspace/apps/run-local/output.txt @@ -8,17 +8,4 @@ Hello, world === Starting the app in background... === Waiting -=== Checking app is running... -=== Checking the proxy is running... ->>> curl -s -o - http://127.0.0.1:$(port) -{"Accept":"*/*","Accept-Encoding":"gzip","Host":"127.0.0.1:$(port)","User-Agent":"curl/(version)","X-Forwarded-Email":"[USERNAME]","X-Forwarded-Host":"localhost","X-Forwarded-Preferred-Username":"","X-Forwarded-User":"[USERNAME]","X-Real-Ip":"127.0.0.1","X-Request-Id":"[UUID]"} - -=== Sending shutdown request... ->>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown - -=== Checking CLI command output... ->>> grep To debug your app, attach a debugger to port ./out.run.txt -To debug your app, attach a debugger to port $(debug_port) - ->>> grep -o Python Flask app has started with: test ./out.run.txt -Python Flask app has started with: test +Error: Test script killed due to a timeout (2m0s) From b38e9b3d48387f50d7066b99ce9c9ba871fa9082 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:14:31 +0100 Subject: [PATCH 05/34] Regenerate remaining acceptance test outputs after rebase Co-Authored-By: Claude Opus 4.6 --- .../output.txt | 4 +- .../serverless_dynamic_version/output.txt | 7 +- .../prevent-destroy/out.terraform.txt | 12 - .../lifecycle/prevent-destroy/output.txt | 2 + .../out.plan_.terraform.json | 8 + .../out.plan_.terraform.txt | 3 + .../out.requests.txt | 298 --------- .../update-and-resize-autoscale/output.txt | 60 +- .../deploy/update-and-resize/output.txt | 7 +- .../update/out.plan_restore.terraform.json | 11 + .../update/out.plan_set_empty.terraform.json | 11 + .../jobs/update}/out.requests.txt | 36 +- .../out.requests_set_empty.terraform.json | 0 .../permissions/jobs/update/output.txt | 9 +- .../update/out.plan_restore.terraform.json | 11 + .../pipelines/update/out.requests.txt | 324 --------- .../update/out.requests_delete_all.json | 0 .../update/out.requests_restore_original.json | 20 + .../permissions/pipelines/update/output.txt | 36 +- .../classic/out.compare-vs-serverless.diff | 11 - .../out.plan_after_deploy_prod.terraform.json | 17 + .../classic/out.requests.prod.terraform.txt | 225 +++++++ .../default-python/classic/out.requests.txt | 627 ------------------ .../default-python/classic/output.txt | 11 +- .../output/my_default_python/.gitignore | 10 - .../combinations/classic/output.txt | 4 +- .../serverless-customcatalog/output.txt | 11 - .../out.plan_after_deploy_dev.terraform.json | 11 + .../out.plan_after_deploy_prod.terraform.json | 17 + .../out.requests.prod.terraform.txt | 213 ++++++ .../serverless/out.requests.txt | 32 - .../default-python/serverless/output.txt | 19 +- .../output/my_default_python/.gitignore | 10 - .../cmd/workspace/apps/run-local/out.run.txt | 8 +- 34 files changed, 709 insertions(+), 1376 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt create mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json rename acceptance/bundle/resources/{clusters/deploy/update-and-resize => permissions/jobs/update}/out.requests.txt (51%) create mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json create mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json delete mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt create mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json create mode 100644 acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json create mode 100644 acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json create mode 100644 acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt delete mode 100644 acceptance/bundle/templates/default-python/classic/out.requests.txt delete mode 100644 acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore create mode 100644 acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json create mode 100644 acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json create mode 100644 acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt delete mode 100644 acceptance/bundle/templates/default-python/serverless/out.requests.txt delete mode 100644 acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt index 64e5907450..07db8796f1 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -41,5 +41,5 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Deleting files... -Destroy complete! + +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt index 0c44462f8a..e7ccd38665 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -37,10 +37,5 @@ Got arguments: ['my_test_code'] >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.some_other_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Deleting files... -Destroy complete! +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt b/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt index f36fe5c8f3..ca54033dc5 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt @@ -139,15 +139,3 @@ Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... - -This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: - delete resources.schemas.my_schema - -This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will -restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline -properties such as the 'catalog' or 'storage' are changed: - delete resources.pipelines.my_pipelines -Deploying resources... -Updating deployment state... -Deployment complete! diff --git a/acceptance/bundle/lifecycle/prevent-destroy/output.txt b/acceptance/bundle/lifecycle/prevent-destroy/output.txt index 6a0f626b37..6b6ab29ec0 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/output.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/output.txt @@ -16,3 +16,5 @@ Deployment complete! >>> errcode [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json index 4dd9da9eb8..0c92dfb4ad 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json @@ -30,3 +30,11 @@ } } } +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt index d51cd2abf0..73b40dacde 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt @@ -10,3 +10,6 @@ Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged update clusters.test_cluster Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt deleted file mode 100644 index 893ae564bc..0000000000 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.requests.txt +++ /dev/null @@ -1,298 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.1/clusters/get", - "q": { - "cluster_id": "[CLUSTER_ID]" - } -} -{ - "method": "POST", - "path": "/api/2.1/clusters/start", - "body": { - "cluster_id": "[CLUSTER_ID]" - } -} -{ - "method": "GET", - "path": "/api/2.1/clusters/get", - "q": { - "cluster_id": "[CLUSTER_ID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.1/clusters/get", - "q": { - "cluster_id": "[CLUSTER_ID]" - } -} -{ - "method": "GET", - "path": "/api/2.1/clusters/list", - "q": { - "filter_by.is_pinned": "true", - "page_size": "100" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.1/clusters/get", - "q": { - "cluster_id": "[CLUSTER_ID]" - } -} -{ - "method": "GET", - "path": "/api/2.1/clusters/list", - "q": { - "filter_by.is_pinned": "true", - "page_size": "100" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal", - "recursive": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.plan_.terraform.json", - "q": { - "overwrite": "true" - }, - "raw_body": "{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"create\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n{\n \"cli_version\": \"[DEV_VERSION]\",\n \"plan\": {\n \"resources.clusters.test_cluster\": {\n \"action\": \"update\"\n }\n }\n}\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.requests.txt", - "q": { - "overwrite": "true" - }, - "raw_body": "{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/start\",\n \"body\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/list\",\n \"q\": {\n \"filter_by.is_pinned\": \"true\",\n \"page_size\": \"100\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/get\",\n \"q\": {\n \"cluster_id\": \"[CLUSTER_ID]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.1/clusters/list\",\n \"q\": {\n \"filter_by.is_pinned\": \"true\",\n \"page_size\": \"100\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json\"\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\",\n \"q\": {\n \"overwrite\": \"false\"\n },\n \"body\": {\n \"ID\": \"[UUID]\",\n \"AcquisitionTime\": \"[TIMESTAMP]\",\n \"IsForced\": false,\n \"User\": \"[USERNAME]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/export\",\n \"q\": {\n \"direct_download\": \"true\",\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/delete\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal\",\n \"recursive\": true\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/mkdirs\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/artifacts/.internal\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files\"\n }\n}\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/databricks.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "bundle:\n name: test-deploy-cluster-autoscale\n\nworkspace:\n root_path: ~/.bundle/[UNIQUE_NAME]\n\nresources:\n clusters:\n test_cluster:\n cluster_name: test-cluster-[UNIQUE_NAME]\n spark_version: 13.3.x-snapshot-scala2.12\n node_type_id: [NODE_TYPE_ID]\n autoscale:\n min_workers: 4\n max_workers: 6\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/output.txt", - "q": { - "overwrite": "true" - }, - "raw_body": "\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n=== Cluster should exist with num_workers after bundle deployment:\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": 2,\n \"autoscale\": null\n}\n\n=== Adding autoscale section should call update API on stopped cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e jq select(.method == \"POST\" and (.path | contains(\"/clusters/edit\"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/edit\",\n \"body\": {\n \"autoscale\": {\n \"max_workers\": 4,\n \"min_workers\": 2\n },\n \"autotermination_minutes\": 60,\n \"cluster_id\": \"[CLUSTER_ID]\",\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"node_type_id\": \"[NODE_TYPE_ID]\",\n \"spark_version\": \"13.3.x-snapshot-scala2.12\"\n }\n}\n\n=== Cluster should have autoscale\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": null,\n \"autoscale\": {\n \"max_workers\": 4,\n \"min_workers\": 2\n }\n}\n\n=== Changing autoscale should call update API on stopped cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e jq select(.method == \"POST\" and (.path | contains(\"/clusters/edit\"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk) out.requests.txt\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.1/clusters/edit\",\n \"body\": {\n \"autoscale\": {\n \"max_workers\": 5,\n \"min_workers\": 3\n },\n \"autotermination_minutes\": 60,\n \"cluster_id\": \"[CLUSTER_ID]\",\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"node_type_id\": \"[NODE_TYPE_ID]\",\n \"spark_version\": \"13.3.x-snapshot-scala2.12\"\n }\n}\n\n=== Cluster should have new autoscale\n{\n \"cluster_name\": \"test-cluster-[UNIQUE_NAME]\",\n \"num_workers\": null,\n \"autoscale\": {\n \"max_workers\": 5,\n \"min_workers\": 3\n }\n}\n\n=== Starting the cluster\n{\n \"autoscale\": {\n \"max_workers\":5,\n \"min_workers\":3\n },\n \"autotermination_minutes\":60,\n \"aws_attributes\": {\n \"availability\":\"SPOT_WITH_FALLBACK\",\n \"zone_id\":\"us-east-1c\"\n },\n \"cluster_id\":\"[CLUSTER_ID]\",\n \"cluster_name\":\"test-cluster-[UNIQUE_NAME]\",\n \"driver_node_type_id\":\"[NODE_TYPE_ID]\",\n \"enable_elastic_disk\":false,\n \"node_type_id\":\"[NODE_TYPE_ID]\",\n \"spark_version\":\"13.3.x-snapshot-scala2.12\",\n \"state\":\"RUNNING\"\n}\n\n=== Changing autoscale should call resize API on running cluster\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files/out.plan_.terraform.txt", - "q": { - "overwrite": "true" - }, - "raw_body": "create clusters.test_cluster\n\nPlan: 1 to add, 0 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\nupdate clusters.test_cluster\n\nPlan: 0 to add, 1 to change, 0 to delete, 0 unchanged\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deployment.json", - "q": { - "overwrite": "true" - }, - "body": { - "version": 1, - "seq": 4, - "cli_version": "[DEV_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "databricks.yml.tmpl", - "is_notebook": false - }, - { - "local_path": "out.plan_.terraform.txt", - "is_notebook": false - }, - { - "local_path": "out.requests.txt", - "is_notebook": false - }, - { - "local_path": "script", - "is_notebook": false - }, - { - "local_path": "hello_world.py", - "is_notebook": false - }, - { - "local_path": "out.plan_.terraform.json", - "is_notebook": false - }, - { - "local_path": "output.txt", - "is_notebook": false - }, - { - "local_path": "repls.json", - "is_notebook": false - }, - { - "local_path": "test.toml", - "is_notebook": false - }, - { - "local_path": "ACC_REPLS", - "is_notebook": false - }, - { - "local_path": "databricks.yml", - "is_notebook": false - } - ], - "id": "[UUID]" - } -} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index b4fa1c0bbe..78b00d25ee 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -106,5 +106,63 @@ Deployment complete! >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "cluster_id": "[CLUSTER_ID]" + } +} + +=== Cluster should have new autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 6, + "min_workers": 4 + } +} + +=== Removing autoscale section should call resize API + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "cluster_id": "[CLUSTER_ID]", + "num_workers": 3 + } +} + +=== Cluster should have num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 3, + "autoscale": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.test_cluster + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] -Error: Test script killed due to a timeout (30s) +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index 1b4975b25a..43c78fa780 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -121,5 +121,10 @@ Deployment complete! } >>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.clusters.test_cluster -Error: Test script killed due to a timeout (30s) +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json index e69de29bb2..4575a4141d 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.job_with_permissions": { + "action": "skip" + }, + "resources.jobs.job_with_permissions.permissions": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json new file mode 100644 index 0000000000..a7c2467234 --- /dev/null +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.job_with_permissions": { + "action": "skip" + }, + "resources.jobs.job_with_permissions.permissions": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt b/acceptance/bundle/resources/permissions/jobs/update/out.requests.txt similarity index 51% rename from acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt rename to acceptance/bundle/resources/permissions/jobs/update/out.requests.txt index 4569fef1ae..6bd8085690 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.requests.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests.txt @@ -1,15 +1,8 @@ -{ - "method": "GET", - "path": "/api/2.1/clusters/get", - "q": { - "cluster_id": "[CLUSTER_ID]" - } -} { "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/resources.json", "return_export_info": "true" } } @@ -17,24 +10,24 @@ "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/terraform.tfstate", "return_export_info": "true" } } { "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate" + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/terraform.tfstate" } { "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]" + "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default" } } { "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock", "q": { "overwrite": "false" }, @@ -49,7 +42,7 @@ "method": "GET", "path": "/api/2.0/workspace/get-status", "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock", "return_export_info": "true" } } @@ -58,6 +51,21 @@ "path": "/api/2.0/workspace/export", "q": { "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" + "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock" + } +} +{ + "method": "GET", + "path": "/api/2.2/jobs/get", + "q": { + "job_id": "[JOB_WITH_PERMISSIONS_ID]" } } +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} +{ + "method": "GET", + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_set_empty.terraform.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/resources/permissions/jobs/update/output.txt b/acceptance/bundle/resources/permissions/jobs/update/output.txt index 43c4ec92b2..f9b10d968c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/output.txt @@ -167,12 +167,5 @@ Warning: unknown field: permissions at resources.jobs.job_with_permissions.permissions[1] in databricks.yml:18:11 -The following resources will be deleted: - delete resources.jobs.job_with_permissions -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default - -Deleting files... -Destroy complete! - ->>> print_requests +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json new file mode 100644 index 0000000000..586aab2164 --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.pipelines.foo": { + "action": "skip" + }, + "resources.pipelines.foo.permissions": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt b/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt deleted file mode 100644 index 48756d15dd..0000000000 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.requests.txt +++ /dev/null @@ -1,324 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.0/pipelines/[FOO_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "method": "GET", - "path": "/api/2.0/permissions/pipelines/[FOO_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "method": "GET", - "path": "/api/2.0/pipelines/[FOO_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/permissions/pipelines/[FOO_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal", - "recursive": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/databricks.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "bundle:\n name: permissions-test\n\nresources:\n pipelines:\n foo:\n name: foo\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.plan_delete_all.terraform.json", - "q": { - "overwrite": "true" - }, - "body": { - "cli_version": "[DEV_VERSION]", - "plan": { - "resources.pipelines.foo": { - "action": "skip" - }, - "resources.pipelines.foo.permissions": { - "action": "delete" - } - } - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.requests_delete_one.json", - "q": { - "overwrite": "true" - }, - "body": { - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[FOO_ID]", - "body": { - "access_control_list": [ - { - "permission_level": "CAN_MANAGE", - "user_name": "viewer@example.com" - }, - { - "permission_level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/out.requests.txt", - "q": { - "overwrite": "true" - }, - "raw_body": "{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/preview/scim/v2/Me\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/permissions/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/preview/scim/v2/Me\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/permissions/pipelines/[FOO_ID]\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/resources.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/terraform.tfstate\"\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json\"\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\",\n \"q\": {\n \"overwrite\": \"false\"\n },\n \"body\": {\n \"ID\": \"[UUID]\",\n \"AcquisitionTime\": \"[TIMESTAMP]\",\n \"IsForced\": false,\n \"User\": \"[USERNAME]\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\",\n \"return_export_info\": \"true\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/export\",\n \"q\": {\n \"direct_download\": \"true\",\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deploy.lock\"\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/delete\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal\",\n \"recursive\": true\n }\n}\n{\n \"method\": \"POST\",\n \"path\": \"/api/2.0/workspace/mkdirs\",\n \"body\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/artifacts/.internal\"\n }\n}\n{\n \"method\": \"GET\",\n \"path\": \"/api/2.0/workspace/get-status\",\n \"q\": {\n \"path\": \"/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files\"\n }\n}\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files/output.txt", - "q": { - "overwrite": "true" - }, - "raw_body": "\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n\u003e\u003e\u003e [CLI] permissions get pipelines [FOO_ID]\n{\n \"access_control_list\": [\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"CAN_VIEW\"\n }\n ],\n \"display_name\":\"viewer@example.com\",\n \"user_name\":\"viewer@example.com\"\n },\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"CAN_MANAGE\"\n }\n ],\n \"group_name\":\"data-team\"\n },\n {\n \"all_permissions\": [\n {\n \"inherited\":false,\n \"permission_level\":\"IS_OWNER\"\n }\n ],\n \"display_name\":\"[USERNAME]\",\n \"user_name\":\"[USERNAME]\"\n }\n ],\n \"object_id\":\"/pipelines/[FOO_ID]\",\n \"object_type\":\"pipelines\"\n}\n\n=== Update one permission and deploy again\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n=== Delete one permission and deploy again\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\nDeploying resources...\nUpdating deployment state...\nDeployment complete!\n\n\u003e\u003e\u003e print_requests\n\n\u003e\u003e\u003e [CLI] bundle plan\nPlan: 0 to add, 0 to change, 0 to delete, 2 unchanged\n\n=== Delete the whole block and deploy again\n\n\u003e\u003e\u003e cat databricks.yml\nbundle:\n name: permissions-test\n\nresources:\n pipelines:\n foo:\n name: foo\n\n\u003e\u003e\u003e [CLI] bundle plan -o json\n\n\u003e\u003e\u003e [CLI] bundle deploy\nUploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files...\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/permissions-test/default/state/deployment.json", - "q": { - "overwrite": "true" - }, - "body": { - "version": 1, - "seq": 4, - "cli_version": "[DEV_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "output.txt", - "is_notebook": false - }, - { - "local_path": "ACC_REPLS", - "is_notebook": false - }, - { - "local_path": "databricks.yml.saved", - "is_notebook": false - }, - { - "local_path": "out.plan_create.terraform.json", - "is_notebook": false - }, - { - "local_path": "out.plan_delete_one.terraform.json", - "is_notebook": false - }, - { - "local_path": "out.plan_delete_all.terraform.json", - "is_notebook": false - }, - { - "local_path": "out.requests.txt", - "is_notebook": false - }, - { - "local_path": "repls.json", - "is_notebook": false - }, - { - "local_path": "script", - "is_notebook": false - }, - { - "local_path": "out.plan_update.terraform.json", - "is_notebook": false - }, - { - "local_path": "out.requests_delete_one.json", - "is_notebook": false - }, - { - "local_path": "test.toml", - "is_notebook": false - }, - { - "local_path": "databricks.yml", - "is_notebook": false - }, - { - "local_path": "out.requests_create.json", - "is_notebook": false - }, - { - "local_path": "out.requests_update.json", - "is_notebook": false - } - ], - "id": "[UUID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "method": "GET", - "path": "/api/2.0/pipelines/[FOO_ID]" -} -{ - "method": "GET", - "path": "/api/2.0/permissions/pipelines/[FOO_ID]" -} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json b/acceptance/bundle/resources/permissions/pipelines/update/out.requests_delete_all.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json b/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json new file mode 100644 index 0000000000..2067ef6365 --- /dev/null +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.requests_restore_original.json @@ -0,0 +1,20 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index 0f3c3461d8..84b53daf5c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -94,5 +94,39 @@ resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_sorted_requests + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Restore original config + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/permissions-test/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.foo + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default -Error: Test script killed due to a timeout (30s) +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 9de2b76caa..22c5698c64 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -1,12 +1,3 @@ -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/.gitignore -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/deployment.json -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/resources.json -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/sync-snapshots/adf1b[NUMID]fee.json -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/.terraform.lock.hcl -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/bundle.tf.json -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/.terraform.lock.hcl -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/bundle.tf.json -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/.gitignore --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -34,4 +34,6 @@ @@ -16,8 +7,6 @@ Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: + artifacts_dynamic_version: true prod: mode: production -Only in [TESTROOT]/bundle/templates/default-python/classic/../serverless/output: my_default_python/dist/.gitignore -Only in output: my_default_python/out.gitignore --- [TESTROOT]/bundle/templates/default-python/classic/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,8 +5,7 @@ diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json new file mode 100644 index 0000000000..103b8a29e3 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json @@ -0,0 +1,17 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.jobs.sample_job.permissions": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl.permissions": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt new file mode 100644 index 0000000000..c6d26be260 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.requests.prod.terraform.txt @@ -0,0 +1,225 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/directories/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.txt b/acceptance/bundle/templates/default-python/classic/out.requests.txt deleted file mode 100644 index 00bbeba373..0000000000 --- a/acceptance/bundle/templates/default-python/classic/out.requests.txt +++ /dev/null @@ -1,627 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.0/pipelines/[UUID]" -} -{ - "method": "GET", - "path": "/api/2.2/jobs/get", - "q": { - "job_id": "[NUMID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" -} -{ - "method": "GET", - "path": "/api/2.0/pipelines/[UUID]" -} -{ - "method": "GET", - "path": "/api/2.2/jobs/get", - "q": { - "job_id": "[NUMID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", - "recursive": true - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl", - "q": { - "overwrite": "true" - }, - "raw_body": "PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001d\u0000\u0000\u0000my_default_python/__init__.py\u0003\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffd٤\\\u0001\u0000\u0000\ufffd\u0002\u0000\u0000\u0019\u0000\u0000\u0000my_default_python/main.py}\ufffd\ufffdn\ufffd0\u0010\ufffdw?\ufffd\ufffd]\u0012\t\ufffduA\ufffd\u0010Q\ufffd\u000e\ufffd\ufffd\u0004\ufffd-\u0013;\ufffd\u0010\ufffd\ufffd\ufffd\bP\ufffdw\ufffd\ufffd\u0018\ufffd\u0001\ufffdK\u0014\ufffd\ufffd\ufffd\ufffd\ufffd;mz\ufffd\u003c\b\ufffd\ufffd\ufffdCEjg\rH\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd\f呹\ufffd\ufffd\ufffd(\ufffd\u0013\ufffd\ufffd\u003cN\ufffd\ufffdr\ufffdj1\ufffd\ufffd\ufffdW\ufffd\ufffd\ufffd\ufffdxq\ufffdH\b\tQ0BwY\ufffd \u0010\ufffd\u0013\ufffd\ufffd\ufffd\u0014\"T\ufffd\u0018\ufffd\ufffdy\ufffd;5\ufffd\u001f\ufffd\ufffd\u003cF(:qP\ufffd]\ufffd2\ufffd\ufffd1\ufffdEj\u003cRa\ufffdt\ufffd\ufffd\ufffd\n\ufffdrw\r\u0007\ufffd\ufffd\ufffd\ufffd\rT᮵{\b\ufffd\u0000\ufffdF\u00191f\u0017Fy\ufffd\ufffd\ufffdb\ufffd\ufffdOQ\u0026\ufffd\ufffd77\u0019\ufffd\ufffdS\u0002:\u0003\ufffdN\ufffdvJ\u0016[7\ufffd5S\ufffdǒ\ufffdaxYR\ufffdϨ\ufffd,'\ufffdA\u001b\u0015\ufffd\ufffd(H\ufffd}\ufffd\ufffd\b\ufffd)0\u003c\ufffdYM\ufffd6+X\ufffd\ufffd\ufffd\ufffd\ufffd\u0015\ufffd\ufffdl,\ufffd~h\ufffd\u0010\ufffd,\ufffdV\u001feb\ufffd\ufffd#\ufffd\u001c\ufffd.\ufffd\ufffd\ufffdZ\ufffda@\u000f\ufffd\u000e\ufffdE\ufffdN3\ufffd8w\u0001\u0018\ufffd\ufffd\ufffd(\ufffdq6\ufffd\u003c\ufffd\u003c\ufffdf9\ufffdƞ\ufffd琜\ufffd\u001a8\ufffdB\ufffd9\ufffd\ufffd\u0000\ufffd\ufffd\ufffd\u001a\ufffd\ufffdi7\ufffd=!\ufffdPK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPXA\u0002\ufffd\ufffd\u0000\u0000\u0000\ufffd\u0000\u0000\u0000\u001a\u0000\u0000\u0000my_default_python/taxis.pyE\ufffd\ufffd\u000e\ufffd \fDw\ufffd\ufffdbJ\ufffd\ufffd\u0003:t\ufffd\ufffd\u001b\ufffd\u001b\ufffd\ufffd\u0002\ufffd\ufffdDj\ufffd\u003e\ufffd\ufffd\ufffdm\ufffd{\ufffd\ufffd\ufffd-\ufffd\ufffdJ/\t˪\ufffdnE\ufffds\r\ufffd!\ufffd\ufffdI\u0005-$\ufffd\ufffd\u001d,\ufffdu\ufffd\ufffd\ufffd/}\ufffd\ufffd$\ufffd\ufffd\u0018\ufffd؃\u000f\ufffd\ufffd\u0014\ufffd\\\ufffd\u0013t\u0018\ufffd\ufffd\ufffd3w\u0003M\ufffdکQ\ufffd(\ufffdԵ\ufffdͽR\ufffd\ufffdK\ufffd\ufffd\ufffd09l\ufffdE\u001e\ufffdR*\ufffd\u0015\ufffd\ufffd\ufffd\u0016V\tE\ufffdhNPK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPG\ufffdh\ufffdl\u0000\u0000\u0000y\u0000\u0000\u0000*\u0000\u0000\u0000my_default_python-0.0.1.dist-info/METADATA\ufffdM-ILI,I\ufffd\rK-*\ufffd\ufffdϳR0\ufffd3\ufffd\ufffdK\ufffdM\ufffdRȭ\ufffdOIMK,\ufffd)\ufffd/\ufffd,\ufffd\ufffd\ufffd\ufffd\ufffd+2\ufffd3\ufffd3\ufffdr,\u0005\ufffd\u0016Y)\ufffd\ufffd\u0016\ufffd\ufffd\u00169\ufffd\ufffdI*\ufffdL\ufffd.\ufffdK\ufffd\ufffd\ufffd\nJ-,\ufffd,J-\ufffd\r\u0000k\ufffdR\ufffd1\ufffd34ֱ\ufffd\u0005R\u0006\\\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP_\ufffdoVV\u0000\u0000\u0000W\u0000\u0000\u0000'\u0000\u0000\u0000my_default_python-0.0.1.dist-info/WHEEL\u000b\ufffdHM\ufffd\ufffd\rK-*\ufffd\ufffdϳR0\ufffd3\ufffdrO\ufffdK-J,\ufffd/\ufffdR\ufffdH,I\ufffd\ufffd\ufffd\ufffdK\u0007J\u0018Y\u0002\ufffd\ufffd\ufffd\ufffdKt=\ufffdu\u0003J\ufffdRs2\ufffd\ufffd\u0014J\ufffdJS\ufffdB\u0012ӭ\u0014\n*\ufffdu\ufffd\ufffd\ufffdRu\u0013\ufffd*\ufffd\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffdF\ufffd\ufffd2\u0000\u0000\u00005\u0000\u0000\u00002\u0000\u0000\u0000my_default_python-0.0.1.dist-info/entry_points.txt\ufffdN\ufffd\ufffd+\ufffd\ufffdI\ufffd/N.\ufffd,()\ufffd\ufffd\ufffdM\ufffd\ufffdS\ufffdUȭ\ufffdOIMK,\ufffd)\ufffd/\ufffd,\ufffd\ufffd\ufffd\ufffd\u0003IX\ufffd\b.\u0000PK\u0003\u0004\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffdl\ufffd`\u0001\u0000\u0000J\u0002\u0000\u0000(\u0000\u0000\u0000my_default_python-0.0.1.dist-info/RECORD\ufffd\ufffd\ufffdR\ufffd0\u0000@ѽ\ufffd\u0012\u0010\ufffd@\ufffdpA[l)P\ufffd@\ufffdn2H\u0003MyE\t/\ufffd^7\ufffdqF\u0017\ufffd\ufffdso=\ufffd\u0013\ufffdӾ\u0012\ufffd\ufffd\ufffd\ufffd6\ufffd\ufffd\ufffd\ufffd\tBd\u003e\ufffd\ufffd\ufffdjPz4\ufffdV\ufffd/h\ufffdĩD\"\ufffdN$\ufffd[\ufffd\u003e\ufffdaYý\ufffdw\ufffdq\ufffdp\ufffd\ufffds\f\ufffd\ufffd\ufffd\u000fY\ufffd\ufffd\ufffd\ufffdy\ufffd߆\ufffdqd4\ufffd\u0007\ufffd\ufffd\ufffd\ufffd\ufffd_\ufffd\ufffd\ufffd\u0018J\ufffd\ufffd#\u0011\ufffd~ȳ\ufffdw\ufffd\b\ufffd\u0010\ufffd\u0003\ufffdtb\ufffd/\ufffd\ufffd\ufffdh\u000b-d\ufffdɉ\ufffd\u003c\ufffd\ufffd$˦\u000b\ufffd\ufffd\u0000\u0006F\u0011\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdئ\nP\u0017\ufffd\ufffd()\ufffd\"\ufffd\ufffd\ufffduBbM\ufffd\ufffd{Vd\ufffd\ufffdȼ\u0016v\ufffd1 g\ufffd/.U\u003cJ4\ufffd\u0007\ufffdmO\ufffd3\ufffdgqءG\u001dV\ufffd\u0002\ufffd\ufffd\ufffd\u0006TM\ufffd\ufffd\ufffdl-˽\ufffdA\ufffd\ufffdS\ufffd\ufffd\ufffdE\ufffdZz\ufffd\ufffd\u0010\ufffd\ufffd\u0026a\ufffd\ufffdR\ufffd,Ӧ\ufffdfS\ufffd\ufffdI\u003e\u0002dܠ\ufffdF|΄\ufffd\ufffd\u0011\ufffd,\u0026q\r9C\u001dn\ufffd[\ufffd\nK5\ufffdF\ufffdi\ufffd\ufffd\ufffdV=x\ufffd\ufffdB\ufffd\ufffdj\ufffd\u000b\ufffd\ufffdK4\u0002\ufffdpC(\ufffdV/\ufffd\u001a\ufffd\ufffdoPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001d\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\u0000\u0000\u0000\u0000my_default_python/__init__.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffd٤\\\u0001\u0000\u0000\ufffd\u0002\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd=\u0000\u0000\u0000my_default_python/main.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPXA\u0002\ufffd\ufffd\u0000\u0000\u0000\ufffd\u0000\u0000\u0000\u001a\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\u0001\u0000\u0000my_default_python/taxis.pyPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BPG\ufffdh\ufffdl\u0000\u0000\u0000y\u0000\u0000\u0000*\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001\ufffd\u0002\u0000\u0000my_default_python-0.0.1.dist-info/METADATAPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP_\ufffdoVV\u0000\u0000\u0000W\u0000\u0000\u0000'\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001H\u0003\u0000\u0000my_default_python-0.0.1.dist-info/WHEELPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffdF\ufffd\ufffd2\u0000\u0000\u00005\u0000\u0000\u00002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001\ufffd\u0003\u0000\u0000my_default_python-0.0.1.dist-info/entry_points.txtPK\u0001\u0002\u0014\u0003\u0014\u0000\u0000\u0000\b\u0000\u0000\u0000BP\ufffd\ufffdl\ufffd`\u0001\u0000\u0000J\u0002\u0000\u0000(\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0001e\u0004\u0000\u0000my_default_python-0.0.1.dist-info/RECORDPK\u0005\u0006\u0000\u0000\u0000\u0000\u0007\u0000\u0007\u0000=\u0002\u0000\u0000\u000b\u0006\u0000\u0000\u0000\u0000" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/mkdirs", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/sample_zones_my_default_python.py", - "q": { - "overwrite": "true" - }, - "raw_body": "from pyspark import pipelines as dp\nfrom pyspark.sql.functions import col, sum\n\n\n# This file defines a sample transformation.\n# Edit the sample below or add new transformations\n# using \"+ Add\" in the file browser.\n\n\n@dp.table\ndef sample_zones_my_default_python():\n # Read from the \"sample_trips\" table, then sum all the fares\n return (\n spark.read.table(f\"sample_trips_my_default_python\")\n .groupBy(col(\"pickup_zip\"))\n .agg(sum(\"fare_amount\").alias(\"total_fare\"))\n )\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/main.py", - "q": { - "overwrite": "true" - }, - "raw_body": "import argparse\nfrom databricks.sdk.runtime import spark\nfrom my_default_python import taxis\n\n\ndef main():\n # Process command-line arguments\n parser = argparse.ArgumentParser(\n description=\"Databricks job with catalog and schema parameters\",\n )\n parser.add_argument(\"--catalog\", required=True)\n parser.add_argument(\"--schema\", required=True)\n args = parser.parse_args()\n\n # Set the default catalog and schema\n spark.sql(f\"USE CATALOG {args.catalog}\")\n spark.sql(f\"USE SCHEMA {args.schema}\")\n\n # Example: just find all taxis from a sample catalog\n taxis.find_all_taxis().show(5)\n\n\nif __name__ == \"__main__\":\n main()\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/databricks.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "# This is a Databricks asset bundle definition for my_default_python.\n# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation.\nbundle:\n name: my_default_python\n uuid: [UUID]\n\ninclude:\n - resources/*.yml\n - resources/*/*.yml\n\nartifacts:\n python_artifact:\n type: whl\n build: uv build --wheel\n\n# Variable declarations. These variables are assigned in the dev/prod targets below.\nvariables:\n catalog:\n description: The catalog to use\n schema:\n description: The schema to use\n\ntargets:\n dev:\n # The default target uses 'mode: development' to create a development copy.\n # - Deployed resources get prefixed with '[dev my_user_name]'\n # - Any job schedules and triggers are paused by default.\n # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html.\n mode: development\n default: true\n workspace:\n host: [DATABRICKS_URL]\n variables:\n catalog: hive_metastore\n schema: ${workspace.current_user.short_name}\n presets:\n artifacts_dynamic_version: true\n prod:\n mode: production\n workspace:\n host: [DATABRICKS_URL]\n # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy.\n root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target}\n variables:\n catalog: hive_metastore\n schema: prod\n permissions:\n - user_name: [USERNAME]\n level: CAN_MANAGE\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests/conftest.py", - "q": { - "overwrite": "true" - }, - "raw_body": "\"\"\"This file configures pytest, initializes Databricks Connect, and provides fixtures for Spark and loading test data.\"\"\"\n\nimport os, sys, pathlib\nfrom contextlib import contextmanager\n\n\ntry:\n from databricks.connect import DatabricksSession\n from databricks.sdk import WorkspaceClient\n from pyspark.sql import SparkSession\n import pytest\n import json\n import csv\n import os\nexcept ImportError:\n raise ImportError(\n \"Test dependencies not found.\\n\\nRun tests using 'uv run pytest'. See http://docs.astral.sh/uv to learn more about uv.\"\n )\n\n\n@pytest.fixture()\ndef spark() -\u003e SparkSession:\n \"\"\"Provide a SparkSession fixture for tests.\n\n Minimal example:\n def test_uses_spark(spark):\n df = spark.createDataFrame([(1,)], [\"x\"])\n assert df.count() == 1\n \"\"\"\n return DatabricksSession.builder.getOrCreate()\n\n\n@pytest.fixture()\ndef load_fixture(spark: SparkSession):\n \"\"\"Provide a callable to load JSON or CSV from fixtures/ directory.\n\n Example usage:\n\n def test_using_fixture(load_fixture):\n data = load_fixture(\"my_data.json\")\n assert data.count() \u003e= 1\n \"\"\"\n\n def _loader(filename: str):\n path = pathlib.Path(__file__).parent.parent / \"fixtures\" / filename\n suffix = path.suffix.lower()\n if suffix == \".json\":\n rows = json.loads(path.read_text())\n return spark.createDataFrame(rows)\n if suffix == \".csv\":\n with path.open(newline=\"\") as f:\n rows = list(csv.DictReader(f))\n return spark.createDataFrame(rows)\n raise ValueError(f\"Unsupported fixture type for: {filename}\")\n\n return _loader\n\n\ndef _enable_fallback_compute():\n \"\"\"Enable serverless compute if no compute is specified.\"\"\"\n conf = WorkspaceClient().config\n if conf.serverless_compute_id or conf.cluster_id or os.environ.get(\"SPARK_REMOTE\"):\n return\n\n url = \"https://docs.databricks.com/dev-tools/databricks-connect/cluster-config\"\n print(\"☁️ no compute specified, falling back to serverless compute\", file=sys.stderr)\n print(f\" see {url} for manual configuration\", file=sys.stdout)\n\n os.environ[\"DATABRICKS_SERVERLESS_COMPUTE_ID\"] = \"auto\"\n\n\n@contextmanager\ndef _allow_stderr_output(config: pytest.Config):\n \"\"\"Temporarily disable pytest output capture.\"\"\"\n capman = config.pluginmanager.get_plugin(\"capturemanager\")\n if capman:\n with capman.global_and_fixture_disabled():\n yield\n else:\n yield\n\n\ndef pytest_configure(config: pytest.Config):\n \"\"\"Configure pytest session.\"\"\"\n with _allow_stderr_output(config):\n _enable_fallback_compute()\n\n # Initialize Spark session eagerly, so it is available even when\n # SparkSession.builder.getOrCreate() is used. For DB Connect 15+,\n # we validate version compatibility with the remote cluster.\n if hasattr(DatabricksSession.builder, \"validateSession\"):\n DatabricksSession.builder.validateSession().getOrCreate()\n else:\n DatabricksSession.builder.getOrCreate()\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources/sample_job.job.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "# A sample job for my_default_python.\n\nresources:\n jobs:\n sample_job:\n name: sample_job\n\n trigger:\n # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger\n periodic:\n interval: 1\n unit: DAYS\n\n #email_notifications:\n # on_failure:\n # - your_email@example.com\n\n parameters:\n - name: catalog\n default: ${var.catalog}\n - name: schema\n default: ${var.schema}\n\n tasks:\n - task_key: notebook_task\n notebook_task:\n notebook_path: ../src/sample_notebook.ipynb\n job_cluster_key: job_cluster\n libraries:\n # By default we just include the .whl file generated for the my_default_python package.\n # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html\n # for more information on how to add other libraries.\n - whl: ../dist/*.whl\n - task_key: python_wheel_task\n depends_on:\n - task_key: notebook_task\n python_wheel_task:\n package_name: my_default_python\n entry_point: main\n parameters:\n - \"--catalog\"\n - \"${var.catalog}\"\n - \"--schema\"\n - \"${var.schema}\"\n job_cluster_key: job_cluster\n libraries:\n # By default we just include the .whl file generated for the my_default_python package.\n # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html\n # for more information on how to add other libraries.\n - whl: ../dist/*.whl\n - task_key: refresh_pipeline\n depends_on:\n - task_key: notebook_task\n pipeline_task:\n pipeline_id: ${resources.pipelines.my_default_python_etl.id}\n\n job_clusters:\n - job_cluster_key: job_cluster\n new_cluster:\n spark_version: 16.4.x-scala2.12\n node_type_id: [NODE_TYPE_ID]\n data_security_mode: SINGLE_USER\n autoscale:\n min_workers: 1\n max_workers: 4\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/README.md", - "q": { - "overwrite": "true" - }, - "raw_body": "# my_default_python\n\nThis folder defines all source code for the my_default_python pipeline:\n\n- `explorations/`: Ad-hoc notebooks used to explore the data processed by this pipeline.\n- `transformations/`: All dataset definitions and transformations.\n- `utilities/` (optional): Utility functions and Python modules used in this pipeline.\n- `data_sources/` (optional): View definitions describing the source data for this pipeline.\n\n## Getting Started\n\nTo get started, go to the `transformations` folder -- most of the relevant source code lives there:\n\n* By convention, every dataset under `transformations` is in a separate file.\n* Take a look at the sample called \"sample_trips_my_default_python.py\" to get familiar with the syntax.\n Read more about the syntax at https://docs.databricks.com/dlt/python-ref.html.\n* If you're using the workspace UI, use `Run file` to run and preview a single transformation.\n* If you're using the CLI, use `databricks bundle run my_default_python_etl --select sample_trips_my_default_python` to run a single transformation.\n\nFor more tutorials and reference material, see https://docs.databricks.com/dlt.\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/taxis.py", - "q": { - "overwrite": "true" - }, - "raw_body": "from databricks.sdk.runtime import spark\nfrom pyspark.sql import DataFrame\n\n\ndef find_all_taxis() -\u003e DataFrame:\n \"\"\"Find all taxi data.\"\"\"\n return spark.read.table(\"samples.nyctaxi.trips\")\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/pyproject.toml", - "q": { - "overwrite": "true" - }, - "raw_body": "[project]\nname = \"my_default_python\"\nversion = \"0.0.1\"\nauthors = [{ name = \"[USERNAME]\" }]\nrequires-python = \"\u003e=3.10,\u003c3.13\"\ndependencies = [\n # Any dependencies for jobs and pipelines in this project can be added here\n # See also https://docs.databricks.com/dev-tools/bundles/library-dependencies\n #\n # LIMITATION: for pipelines, dependencies are cached during development;\n # add dependencies to the 'environment' section of your pipeline.yml file instead\n]\n\n[dependency-groups]\ndev = [\n \"pytest\",\n \"ruff\",\n \"databricks-dlt\",\n \"databricks-connect\u003e=15.4,\u003c15.5\",\n \"ipykernel\",\n]\n\n[project.scripts]\nmain = \"my_default_python.main:main\"\n\n[build-system]\nrequires = [\"hatchling\"]\nbuild-backend = \"hatchling.build\"\n\n[tool.ruff]\nline-length = 120\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/extensions.json", - "q": { - "overwrite": "true" - }, - "body": { - "recommendations": [ - "databricks.databricks", - "redhat.vscode-yaml", - "charliermarsh.ruff" - ] - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests/sample_taxis_test.py", - "q": { - "overwrite": "true" - }, - "raw_body": "from databricks.sdk.runtime import spark\nfrom pyspark.sql import DataFrame\nfrom my_default_python import taxis\n\n\ndef test_find_all_taxis():\n results = taxis.find_all_taxis()\n assert results.count() \u003e 5\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python/__init__.py", - "q": { - "overwrite": "true" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources/my_default_python_etl.pipeline.yml", - "q": { - "overwrite": "true" - }, - "raw_body": "# The main pipeline for my_default_python\n\nresources:\n pipelines:\n my_default_python_etl:\n name: my_default_python_etl\n ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog:\n # catalog: ${var.catalog}\n schema: ${var.schema}\n root_path: \"../src/my_default_python_etl\"\n\n libraries:\n - glob:\n include: ../src/my_default_python_etl/transformations/**\n\n environment:\n dependencies:\n # We include every dependency defined by pyproject.toml by defining an editable environment\n # that points to the folder where pyproject.toml is deployed.\n - --editable ${workspace.file_path}\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/sample_trips_my_default_python.py", - "q": { - "overwrite": "true" - }, - "raw_body": "from pyspark import pipelines as dp\nfrom pyspark.sql.functions import col\n\n\n# This file defines a sample transformation.\n# Edit the sample below or add new transformations\n# using \"+ Add\" in the file browser.\n\n\n@dp.table\ndef sample_trips_my_default_python():\n return spark.read.table(\"samples.nyctaxi.trips\")\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/README.md", - "q": { - "overwrite": "true" - }, - "raw_body": "# my_default_python\n\nThe 'my_default_python' project was generated by using the default-python template.\n\n* `src/`: Python source code for this project.\n * `src/my_default_python/`: Shared Python code that can be used by jobs and pipelines.\n* `resources/`: Resource configurations (jobs, pipelines, etc.)\n* `tests/`: Unit tests for the shared Python code.\n* `fixtures/`: Fixtures for data sets (primarily used for testing).\n\n\n## Getting started\n\nChoose how you want to work on this project:\n\n(a) Directly in your Databricks workspace, see\n https://docs.databricks.com/dev-tools/bundles/workspace.\n\n(b) Locally with an IDE like Cursor or VS Code, see\n https://docs.databricks.com/dev-tools/vscode-ext.html.\n\n(c) With command line tools, see https://docs.databricks.com/dev-tools/cli/databricks-cli.html\n\nIf you're developing with an IDE, dependencies for this project should be installed using uv:\n\n* Make sure you have the UV package manager installed.\n It's an alternative to tools like pip: https://docs.astral.sh/uv/getting-started/installation/.\n* Run `uv sync --dev` to install the project's dependencies.\n\n\n# Using this project using the CLI\n\nThe Databricks workspace and IDE extensions provide a graphical interface for working\nwith this project. It's also possible to interact with it directly using the CLI:\n\n1. Authenticate to your Databricks workspace, if you have not done so already:\n ```\n $ databricks configure\n ```\n\n2. To deploy a development copy of this project, type:\n ```\n $ databricks bundle deploy --target dev\n ```\n (Note that \"dev\" is the default target, so the `--target` parameter\n is optional here.)\n\n This deploys everything that's defined for this project.\n For example, the default template would deploy a pipeline called\n `[dev yourname] my_default_python_etl` to your workspace.\n You can find that resource by opening your workpace and clicking on **Jobs \u0026 Pipelines**.\n\n3. Similarly, to deploy a production copy, type:\n ```\n $ databricks bundle deploy --target prod\n ```\n Note the default template has a includes a job that runs the pipeline every day\n (defined in resources/sample_job.job.yml). The schedule\n is paused when deploying in development mode (see\n https://docs.databricks.com/dev-tools/bundles/deployment-modes.html).\n\n4. To run a job or pipeline, use the \"run\" command:\n ```\n $ databricks bundle run\n ```\n\n5. Finally, to run tests locally, use `pytest`:\n ```\n $ uv run pytest\n ```\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook.ipynb", - "q": { - "overwrite": "true" - }, - "body": { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Default notebook\n", - "\n", - "This default notebook is executed using a Lakeflow job as defined in resources/sample_job.job.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Set default catalog and schema\n", - "catalog = dbutils.widgets.get(\"catalog\")\n", - "schema = dbutils.widgets.get(\"schema\")\n", - "spark.sql(f\"USE CATALOG {catalog}\")\n", - "spark.sql(f\"USE SCHEMA {schema}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "sys.path.append(\"../src\")\n", - "from my_default_python import taxis\n", - "\n", - "taxis.find_all_taxis().show(10)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "environmentMetadata": { - "base_environment": "", - "dependencies": [ - "--editable .." - ], - "environment_version": "4" - }, - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "notebook", - "widgets": { - "catalog": { - "currentValue": "hive_metastore", - "nuid": "c4t4l0g-w1dg-3t12-3456-[NUMID]", - "typedWidgetInfo": { - "autoCreated": false, - "defaultValue": "hive_metastore", - "label": "Catalog", - "name": "catalog", - "options": { - "validationRegex": null, - "widgetDisplayType": "Text" - }, - "parameterDataType": "String" - }, - "widgetInfo": { - "defaultValue": "hive_metastore", - "label": "Catalog", - "name": "catalog", - "options": { - "autoCreated": false, - "validationRegex": null, - "widgetType": "text" - }, - "widgetType": "text" - } - }, - "schema": { - "currentValue": "[USERNAME]", - "nuid": "5ch3m4-w1dg-3t98-7654-[NUMID]", - "typedWidgetInfo": { - "autoCreated": false, - "defaultValue": "default", - "label": "Schema", - "name": "schema", - "options": { - "validationRegex": null, - "widgetDisplayType": "Text" - }, - "parameterDataType": "String" - }, - "widgetInfo": { - "defaultValue": "default", - "label": "Schema", - "name": "schema", - "options": { - "autoCreated": false, - "validationRegex": null, - "widgetType": "text" - }, - "widgetType": "text" - } - } - } - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12" - } - }, - "nbformat": 4, - "nbformat_minor": 0 - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.gitignore", - "q": { - "overwrite": "true" - }, - "raw_body": ".databricks/\nbuild/\ndist/\n__pycache__/\n*.egg-info\n.venv/\nscratch/**\n!scratch/README.md\n**/explorations/**\n**/!explorations/README.md\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/__builtins__.pyi", - "q": { - "overwrite": "true" - }, - "raw_body": "# Typings for Pylance in Visual Studio Code\n# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md\nfrom databricks.sdk.runtime import *\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode/settings.json", - "q": { - "overwrite": "true" - }, - "raw_body": "{\n \"jupyter.interactiveWindow.cellMarker.codeRegex\": \"^# COMMAND ----------|^# Databricks notebook source|^(#\\\\s*%%|#\\\\s*\\\\\u003ccodecell\\\\\u003e|#\\\\s*In\\\\[\\\\d*?\\\\]|#\\\\s*In\\\\[ \\\\])\",\n \"jupyter.interactiveWindow.cellMarker.default\": \"# COMMAND ----------\",\n \"python.testing.pytestArgs\": [\n \".\"\n ],\n \"files.exclude\": {\n \"**/*.egg-info\": true,\n \"**/__pycache__\": true,\n \".pytest_cache\": true,\n \"dist\": true,\n },\n \"files.associations\": {\n \"**/.gitkeep\": \"markdown\"\n },\n\n // Pylance settings (VS Code)\n // Set typeCheckingMode to \"basic\" to enable type checking!\n \"python.analysis.typeCheckingMode\": \"off\",\n \"python.analysis.extraPaths\": [\"src\", \"lib\", \"resources\"],\n \"python.analysis.diagnosticMode\": \"workspace\",\n \"python.analysis.stubPath\": \".vscode\",\n\n // Pyright settings (Cursor)\n // Set typeCheckingMode to \"basic\" to enable type checking!\n \"cursorpyright.analysis.typeCheckingMode\": \"off\",\n \"cursorpyright.analysis.extraPaths\": [\"src\", \"lib\", \"resources\"],\n \"cursorpyright.analysis.diagnosticMode\": \"workspace\",\n \"cursorpyright.analysis.stubPath\": \".vscode\",\n\n // General Python settings\n \"python.defaultInterpreterPath\": \"./.venv/bin/python\",\n \"python.testing.unittestEnabled\": false,\n \"python.testing.pytestEnabled\": true,\n \"[python]\": {\n \"editor.defaultFormatter\": \"charliermarsh.ruff\",\n \"editor.formatOnSave\": true,\n },\n}\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures/.gitkeep", - "q": { - "overwrite": "true" - }, - "raw_body": "# Test fixtures directory\n\nAdd JSON or CSV files here. In tests, use them with `load_fixture()`:\n\n```\ndef test_using_fixture(load_fixture):\n data = load_fixture(\"my_data.json\")\n assert len(data) \u003e= 1\n```\n" -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deployment.json", - "q": { - "overwrite": "true" - }, - "body": { - "version": 1, - "seq": 1, - "cli_version": "[DEV_VERSION]", - "timestamp": "[TIMESTAMP]", - "files": [ - { - "local_path": "tests/sample_taxis_test.py", - "is_notebook": false - }, - { - "local_path": "src/my_default_python/main.py", - "is_notebook": false - }, - { - "local_path": ".gitignore", - "is_notebook": false - }, - { - "local_path": ".vscode/settings.json", - "is_notebook": false - }, - { - "local_path": "databricks.yml", - "is_notebook": false - }, - { - "local_path": "fixtures/.gitkeep", - "is_notebook": false - }, - { - "local_path": "resources/sample_job.job.yml", - "is_notebook": false - }, - { - "local_path": "README.md", - "is_notebook": false - }, - { - "local_path": "resources/my_default_python_etl.pipeline.yml", - "is_notebook": false - }, - { - "local_path": "src/my_default_python/taxis.py", - "is_notebook": false - }, - { - "local_path": ".vscode/__builtins__.pyi", - "is_notebook": false - }, - { - "local_path": ".vscode/extensions.json", - "is_notebook": false - }, - { - "local_path": "pyproject.toml", - "is_notebook": false - }, - { - "local_path": "src/my_default_python_etl/transformations/sample_zones_my_default_python.py", - "is_notebook": false - }, - { - "local_path": "src/sample_notebook.ipynb", - "is_notebook": true - }, - { - "local_path": "tests/conftest.py", - "is_notebook": false - }, - { - "local_path": "src/my_default_python/__init__.py", - "is_notebook": false - }, - { - "local_path": "src/my_default_python_etl/README.md", - "is_notebook": false - }, - { - "local_path": "src/my_default_python_etl/transformations/sample_trips_my_default_python.py", - "is_notebook": false - } - ], - "id": "[UUID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod" - } -} -{ - "method": "PUT", - "path": "/api/2.0/permissions/directories/[NUMID]", - "body": { - "access_control_list": [ - { - "permission_level": "CAN_MANAGE", - "user_name": "[USERNAME]" - } - ] - } -} diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index 2f82504f39..af2eb7495c 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -62,5 +62,14 @@ Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan -t dev -o json Building python_artifact... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files... +Deploying resources... +Updating deployment state... +Deployment complete! -Error: Test script killed due to a timeout (30s) +>>> [CLI] bundle plan -t prod +Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle plan -t prod -o json +Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore deleted file mode 100644 index e566c51f74..0000000000 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ -scratch/** -!scratch/README.md -**/explorations/** -**/!explorations/README.md diff --git a/acceptance/bundle/templates/default-python/combinations/classic/output.txt b/acceptance/bundle/templates/default-python/combinations/classic/output.txt index 64e612c69f..4aee98da77 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/classic/output.txt @@ -41,5 +41,5 @@ Destroy complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/X[UNIQUE_NAME]/prod/files... Deploying resources... Deployment complete! -Deleting files... -Destroy complete! + +Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 16845957c9..daa2aba833 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -13,15 +13,6 @@ Note that [DATABRICKS_URL] is used for initialization. To get started, refer to the project README.md file and the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. >>> diff.py [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output output/ -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/.gitignore -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/deployment.json -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/resources.json -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/sync-snapshots/adf1b[NUMID]fee.json -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/.terraform.lock.hcl -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/dev/terraform/bundle.tf.json -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/.terraform.lock.hcl -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.databricks/bundle/prod/terraform/bundle.tf.json -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/.gitignore --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/databricks.yml +++ output/my_default_python/databricks.yml @@ -32,5 +32,5 @@ @@ -38,8 +29,6 @@ Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../s + catalog: customcatalog schema: prod permissions: -Only in [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output: my_default_python/dist/.gitignore -Only in output: my_default_python/out.gitignore --- [TESTROOT]/bundle/templates/default-python/serverless-customcatalog/../serverless/output/my_default_python/resources/my_default_python_etl.pipeline.yml +++ output/my_default_python/resources/my_default_python_etl.pipeline.yml @@ -5,6 +5,5 @@ diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json new file mode 100644 index 0000000000..7766755175 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json new file mode 100644 index 0000000000..103b8a29e3 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json @@ -0,0 +1,17 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.jobs.sample_job.permissions": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl.permissions": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt new file mode 100644 index 0000000000..ae401a3c97 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt @@ -0,0 +1,213 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/directories/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.txt deleted file mode 100644 index 77da8b8cec..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/out.requests.txt +++ /dev/null @@ -1,32 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/deployment.json" -} diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt index 7e97876eb4..6cad79ec1d 100644 --- a/acceptance/bundle/templates/default-python/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/serverless/output.txt @@ -60,5 +60,22 @@ Deployment complete! >>> [CLI] bundle plan -t dev Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -Error: Test script killed due to a timeout (30s) +>>> [CLI] bundle plan -t dev -o json +Building python_artifact... + +>>> [CLI] bundle deploy -t prod +Building python_artifact... +Uploading dist/my_default_python-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -t prod +Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle plan -t prod -o json +Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore b/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore deleted file mode 100644 index e566c51f74..0000000000 --- a/acceptance/bundle/templates/default-python/serverless/output/my_default_python/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ -scratch/** -!scratch/README.md -**/explorations/** -**/!explorations/README.md diff --git a/acceptance/cmd/workspace/apps/run-local/out.run.txt b/acceptance/cmd/workspace/apps/run-local/out.run.txt index 033db8edfa..83628d4c9c 100644 --- a/acceptance/cmd/workspace/apps/run-local/out.run.txt +++ b/acceptance/cmd/workspace/apps/run-local/out.run.txt @@ -4,16 +4,16 @@ Using CPython 3.11.12 Creating virtual environment at: .venv Activate with: source .venv/bin/activate Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 -Resolved 131 packages in 89ms -Uninstalled 2 packages in 38ms -Installed 2 packages in 31ms +Resolved 131 packages in 49ms +Uninstalled 2 packages in 29ms +Installed 2 packages in 10ms - flask==3.1.1 + flask==3.0.3 - werkzeug==3.1.6 + werkzeug==3.0.6 Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 Resolved 7 packages in 3ms -Uninstalled 2 packages in 34ms +Uninstalled 2 packages in 33ms Installed 2 packages in 11ms - flask==3.0.3 + flask==3.1.1 From 27775b09eaa2bd97872e529d76b514f47636d7d2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:22:55 +0100 Subject: [PATCH 06/34] Fix permissions jobs update test output divergence Co-Authored-By: Claude Opus 4.6 --- .../permissions/jobs/update/out.requests.txt | 71 ------------------- .../out.requests_destroy.terraform.json | 19 +++++ .../permissions/jobs/update/output.txt | 9 ++- .../bundle/resources/permissions/output.txt | 2 +- 4 files changed, 28 insertions(+), 73 deletions(-) delete mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests.txt create mode 100644 acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests.txt b/acceptance/bundle/resources/permissions/jobs/update/out.requests.txt deleted file mode 100644 index 6bd8085690..0000000000 --- a/acceptance/bundle/resources/permissions/jobs/update/out.requests.txt +++ /dev/null @@ -1,71 +0,0 @@ -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/terraform.tfstate" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default/state/deploy.lock" - } -} -{ - "method": "GET", - "path": "/api/2.2/jobs/get", - "q": { - "job_id": "[JOB_WITH_PERMISSIONS_ID]" - } -} -{ - "method": "GET", - "path": "/api/2.0/preview/scim/v2/Me" -} -{ - "method": "GET", - "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" -} diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json new file mode 100644 index 0000000000..f66a05564c --- /dev/null +++ b/acceptance/bundle/resources/permissions/jobs/update/out.requests_destroy.terraform.json @@ -0,0 +1,19 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [JOB_WITH_PERMISSIONS_ID] + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/update/output.txt b/acceptance/bundle/resources/permissions/jobs/update/output.txt index f9b10d968c..43c4ec92b2 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/output.txt @@ -167,5 +167,12 @@ Warning: unknown field: permissions at resources.jobs.job_with_permissions.permissions[1] in databricks.yml:18:11 +The following resources will be deleted: + delete resources.jobs.job_with_permissions -Error: Test script killed due to a timeout (30s) +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/jobs-permissions-test/default + +Deleting files... +Destroy complete! + +>>> print_requests diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index f2eb71647e..a3493aceab 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -280,7 +280,7 @@ DIFF jobs/update/out.requests_delete_all.direct.json + } +] ERROR jobs/update/out.requests_destroy.direct.json: Missing terraform file jobs/update/out.requests_destroy.terraform.json -ERROR jobs/update/out.requests_set_empty.direct.json: Missing terraform file jobs/update/out.requests_set_empty.terraform.json +EXACT jobs/update/out.requests_set_empty.direct.json MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json From f7ec34caeb8c8c9dee86b7a4e7717de918d29bee Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:38:13 +0100 Subject: [PATCH 07/34] Fix classic combinations test output Co-Authored-By: Claude Opus 4.6 --- .../templates/default-python/combinations/classic/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/templates/default-python/combinations/classic/output.txt b/acceptance/bundle/templates/default-python/combinations/classic/output.txt index 4aee98da77..64e612c69f 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/classic/output.txt @@ -41,5 +41,5 @@ Destroy complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/X[UNIQUE_NAME]/prod/files... Deploying resources... Deployment complete! - -Error: Test script killed due to a timeout (30s) +Deleting files... +Destroy complete! From 6cbfcf4c753299e86b3a41e44b0c8d95143bfe12 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 16:49:21 +0100 Subject: [PATCH 08/34] Fix remaining acceptance test output divergences from rebase Co-Authored-By: Claude Opus 4.6 --- ..._t_service_principal_name_different.terraform.json | 11 +++++++++++ .../default-python/combinations/serverless/output.txt | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json index e69de29bb2..6716f74d4f 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.terraform.json @@ -0,0 +1,11 @@ +{ + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.pipelines.nyc_taxi_pipeline": { + "action": "create" + }, + "resources.pipelines.nyc_taxi_pipeline.permissions": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt index 4aee98da77..64e612c69f 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/serverless/output.txt @@ -41,5 +41,5 @@ Destroy complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/X[UNIQUE_NAME]/prod/files... Deploying resources... Deployment complete! - -Error: Test script killed due to a timeout (30s) +Deleting files... +Destroy complete! From e3ead8ebba2935955f789a0e65f347cf8c750efe Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 17:02:22 +0100 Subject: [PATCH 09/34] Fix more acceptance test output divergences from rebase Co-Authored-By: Claude Opus 4.6 --- .../serverless_dynamic_version/output.txt | 7 ++++++- .../prevent-destroy/out.terraform.txt | 12 +++++++++++ .../lifecycle/prevent-destroy/output.txt | 2 -- .../bundle/resources/permissions/output.txt | 21 ++++++++++++++++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt index e7ccd38665..0c44462f8a 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -37,5 +37,10 @@ Got arguments: ['my_test_code'] >>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.some_other_job -Error: Test script killed due to a timeout (30s) +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt b/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt index ca54033dc5..f36fe5c8f3 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.terraform.txt @@ -139,3 +139,15 @@ Plan: 0 to add, 0 to change, 2 to delete, 0 unchanged >>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.my_schema + +This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Recreating the pipelines will +restore the defined STs and MVs through full refresh. Note that recreation is necessary when pipeline +properties such as the 'catalog' or 'storage' are changed: + delete resources.pipelines.my_pipelines +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/lifecycle/prevent-destroy/output.txt b/acceptance/bundle/lifecycle/prevent-destroy/output.txt index 6b6ab29ec0..6a0f626b37 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/output.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/output.txt @@ -16,5 +16,3 @@ Deployment complete! >>> errcode [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -Error: Test script killed due to a timeout (30s) diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index a3493aceab..ab4953f6b5 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -279,7 +279,26 @@ DIFF jobs/update/out.requests_delete_all.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -ERROR jobs/update/out.requests_destroy.direct.json: Missing terraform file jobs/update/out.requests_destroy.terraform.json +DIFF jobs/update/out.requests_destroy.direct.json +--- jobs/update/out.requests_destroy.direct.json ++++ jobs/update/out.requests_destroy.terraform.json +@@ -1,4 +1,16 @@ + [ ++ { ++ "body": { ++ "access_control_list": [ ++ { ++ "permission_level": "IS_OWNER", ++ "user_name": "[USERNAME]" ++ } ++ ] ++ }, ++ "method": "PUT", ++ "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" ++ }, + { + "body": { + "job_id": "[JOB_WITH_PERMISSIONS_ID]" EXACT jobs/update/out.requests_set_empty.direct.json MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json From 86d100cd21c4dacca68387ed427f8bbe4277e314 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Mar 2026 17:07:40 +0100 Subject: [PATCH 10/34] Fix interactive_cluster_dynamic_version test output Co-Authored-By: Claude Opus 4.6 --- .../interactive_cluster_dynamic_version/output.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt index 07db8796f1..64e5907450 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -41,5 +41,5 @@ The following resources will be deleted: All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] - -Error: Test script killed due to a timeout (30s) +Deleting files... +Destroy complete! From 41cecdb9a2560fefdf4092350715303099717568 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 18:01:39 +0100 Subject: [PATCH 11/34] __embed__ --- acceptance/bundle/migrate/grants/out.new_state.json | 6 +++--- acceptance/bundle/migrate/permissions/out.new_state.json | 2 +- acceptance/bundle/migrate/runas/out.new_state.json | 2 +- .../resource_deps/grant_ref/out.plan_create.direct.json | 4 ++-- .../bundle/resources/grants/catalogs/out.plan1.direct.json | 2 +- .../bundle/resources/grants/catalogs/out.plan2.direct.json | 4 ++-- .../bundle/state/permission_level_migration/output.txt | 2 +- bundle/direct/dresources/grants.go | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 1187063d2e..5a0a07026a 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -24,7 +24,7 @@ "state": { "securable_type": "function", "full_name": "main.schema_grants.mymodel", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -52,7 +52,7 @@ "state": { "securable_type": "schema", "full_name": "main.schema_grants", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -89,7 +89,7 @@ "state": { "securable_type": "volume", "full_name": "main.schema_grants.volume_name", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index b020b1401d..c9043ddf6c 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index a7308ab59c..dc6d571877 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index aea815e550..970fab66ae 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "schema", "full_name": "", - "_": [ + "__embed__": [ { "principal": "viewers", "privileges": [ @@ -72,7 +72,7 @@ "value": { "securable_type": "schema", "full_name": "", - "_": [ + "__embed__": [ { "principal": "viewers", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 293272964b..333e2928d5 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -22,7 +22,7 @@ "new_state": { "value": { "full_name": "", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 6a62a22919..a63b52c327 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -25,7 +25,7 @@ "new_state": { "value": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -38,7 +38,7 @@ }, "remote_state": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "_": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index f289e0bc42..fbcaea775e 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -12,7 +12,7 @@ Deployment complete! === Print state after deploy >>> print_state.py { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index c06840c0bc..7a1f169938 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -23,7 +23,7 @@ var grantResourceToSecurableType = map[string]string{ type GrantsState struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` - EmbeddedSlice []catalog.PrivilegeAssignment `json:"_,omitempty"` + EmbeddedSlice []catalog.PrivilegeAssignment `json:"__embed__,omitempty"` } func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVar, error) { From 120d5f9fc852ea0bd162725380d8dd4c0294bf6c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:25:58 +0100 Subject: [PATCH 12/34] =?UTF-8?q?Revert=20state=20version=20to=202,=20remo?= =?UTF-8?q?ve=20v2=E2=86=92v3=20migration=20entry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version 2 should encompass both permissions and grants migrations. The v2→v3 migration function is still present but unreferenced. Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dstate/migrate.go | 1 - bundle/direct/dstate/state.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e21a5230ff..9572c909b6 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -38,7 +38,6 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, - 2: migrateV2ToV3, } // migrateV1ToV2 migrates permissions entries from the old format diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 3542cf8d0a..e1c776b479 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const currentStateVersion = 3 +const currentStateVersion = 2 type DeploymentState struct { Path string From ee315904a5bc8d1fbed4c5e96d8f3e9e0c06fd2b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:27:23 +0100 Subject: [PATCH 13/34] =?UTF-8?q?Keep=20grants=20JSON=20key=20as=20"grants?= =?UTF-8?q?",=20remove=20dead=20v2=E2=86=92v3=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EmbeddedSlice convention is detected by Go field name, not JSON tag. Keeping "grants" as the JSON tag avoids needing a state migration entirely, since existing v2 state files already use the "grants" key. Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dresources/grants.go | 2 +- bundle/direct/dstate/migrate.go | 53 ------------------------------ 2 files changed, 1 insertion(+), 54 deletions(-) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 7a1f169938..695476653c 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -23,7 +23,7 @@ var grantResourceToSecurableType = map[string]string{ type GrantsState struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` - EmbeddedSlice []catalog.PrivilegeAssignment `json:"__embed__,omitempty"` + EmbeddedSlice []catalog.PrivilegeAssignment `json:"grants,omitempty"` } func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVar, error) { diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 9572c909b6..1488f50afe 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -100,56 +100,3 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return json.MarshalIndent(newState, " ", " ") } -// migrateV2ToV3 migrates grants entries from the old format -// ("grants" key for the slice) to the new format ("_" key for EmbeddedSlice). -func migrateV2ToV3(db *Database) error { - for key, entry := range db.State { - if !strings.HasSuffix(key, ".grants") { - continue - } - if len(entry.State) == 0 { - continue - } - migrated, err := migrateGrantsEntry(entry.State) - if err != nil { - return fmt.Errorf("migrating %s: %w", key, err) - } - entry.State = migrated - db.State[key] = entry - } - return nil -} - -// oldGrantsStateV2 is the old format with "grants" key. -type oldGrantsStateV2 struct { - SecurableType string `json:"securable_type"` - FullName string `json:"full_name"` - Grants []json.RawMessage `json:"grants,omitempty"` -} - -func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { - var old oldGrantsStateV2 - if err := json.Unmarshal(raw, &old); err != nil { - return nil, err - } - - // If old format had no grants, might already be migrated. - if len(old.Grants) == 0 { - return raw, nil - } - - // Re-serialize with "_" key instead of "grants". - type newFormat struct { - SecurableType string `json:"securable_type"` - FullName string `json:"full_name"` - EmbeddedSlice []json.RawMessage `json:"_,omitempty"` - } - - newState := newFormat{ - SecurableType: old.SecurableType, - FullName: old.FullName, - EmbeddedSlice: old.Grants, - } - - return json.MarshalIndent(newState, " ", " ") -} From 660ba02d209b12024c25714896de285ddad5c8eb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:43:00 +0100 Subject: [PATCH 14/34] Regenerate acceptance test outputs for grants JSON key change Use "grants" instead of "__embed__" / "_" as the JSON key for grants EmbeddedSlice, and revert state version from 3 to 2 in test outputs. Co-Authored-By: Claude Opus 4.6 --- .../dashboard/recreation/out.state_after_bind.direct.json | 2 +- acceptance/bundle/migrate/basic/out.new_state.json | 2 +- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../migrate/default-python/out.state_after_migration.json | 2 +- acceptance/bundle/migrate/grants/out.new_state.json | 8 ++++---- acceptance/bundle/migrate/permissions/out.new_state.json | 2 +- acceptance/bundle/migrate/runas/out.new_state.json | 2 +- .../resource_deps/grant_ref/out.plan_create.direct.json | 4 ++-- .../resources/grants/catalogs/out.plan1.direct.json | 2 +- .../resources/grants/catalogs/out.plan2.direct.json | 4 ++-- .../grants/registered_models/out.plan1.direct.json | 2 +- .../grants/schemas/change_privilege/out.plan1.direct.json | 2 +- .../grants/schemas/change_privilege/out.plan2.direct.json | 4 ++-- .../grants/schemas/empty_array/out.plan1.direct.txt | 2 +- .../bundle/resources/grants/volumes/out.plan1.direct.json | 2 +- .../bundle/resources/grants/volumes/out.plan2.direct.json | 4 ++-- .../bundle/resources/jobs/big_id/out.state.direct.json | 2 +- .../bundle/resources/jobs/update/out.state.direct.json | 2 +- acceptance/bundle/state/future_version/output.txt | 2 +- .../bundle/state/permission_level_migration/output.txt | 2 +- .../user_agent/simple/out.requests.deploy.direct.json | 2 +- 21 files changed, 28 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index c7a30bcd43..771dd3f908 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index e294a69095..f6bdf06f62 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index a3d491562e..695d14602a 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 8ff25834b7..029649aae3 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 5a0a07026a..2046d78c9d 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 9, @@ -24,7 +24,7 @@ "state": { "securable_type": "function", "full_name": "main.schema_grants.mymodel", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -52,7 +52,7 @@ "state": { "securable_type": "schema", "full_name": "main.schema_grants", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -89,7 +89,7 @@ "state": { "securable_type": "volume", "full_name": "main.schema_grants.volume_name", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index c9043ddf6c..b020b1401d 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index dc6d571877..a7308ab59c 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 970fab66ae..3b47faf360 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "schema", "full_name": "", - "__embed__": [ + "grants": [ { "principal": "viewers", "privileges": [ @@ -72,7 +72,7 @@ "value": { "securable_type": "schema", "full_name": "", - "__embed__": [ + "grants": [ { "principal": "viewers", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 333e2928d5..6f50fd2b5f 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -22,7 +22,7 @@ "new_state": { "value": { "full_name": "", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index a63b52c327..681ee94884 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -25,7 +25,7 @@ "new_state": { "value": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -38,7 +38,7 @@ }, "remote_state": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "__embed__": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index 3a4f3664af..b5ab8d0b9d 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "function", "full_name": "", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index dca3d13e4c..01d2d581b2 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index b8d929f3c8..8e928edf0a 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -29,7 +29,7 @@ "value": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -43,7 +43,7 @@ "remote_state": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt index 23385843df..a259b5603f 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com" } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index 11ffbf4669..2f9e4d51a1 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -40,7 +40,7 @@ "value": { "securable_type": "volume", "full_name": "", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index 0d843258cd..e56df92969 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -56,7 +56,7 @@ "value": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -70,7 +70,7 @@ "remote_state": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "_": [ + "grants": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index 31eb566641..f8cf0ce5bf 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 76888ff526..785c0c1338 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f47..7cf98129ee 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 3; upgrade the CLI +state version 999 is newer than supported version 2; upgrade the CLI Exit code: 1 diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index fbcaea775e..f289e0bc42 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -12,7 +12,7 @@ Deployment complete! === Print state after deploy >>> print_state.py { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index f252d6df1d..d4543d353a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -222,7 +222,7 @@ "overwrite": "true" }, "body": { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, From 1b076e33aa1e174793378b3c9de9249f9d6eea20 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:43:54 +0100 Subject: [PATCH 15/34] Remove trailing newline in migrate.go, revert unrelated apps/run-local change Co-Authored-By: Claude Opus 4.6 --- .../cmd/workspace/apps/run-local/output.txt | 15 ++++++++++++++- bundle/direct/dstate/migrate.go | 1 - 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/acceptance/cmd/workspace/apps/run-local/output.txt b/acceptance/cmd/workspace/apps/run-local/output.txt index cb7822a22d..6b7b624360 100644 --- a/acceptance/cmd/workspace/apps/run-local/output.txt +++ b/acceptance/cmd/workspace/apps/run-local/output.txt @@ -8,4 +8,17 @@ Hello, world === Starting the app in background... === Waiting -Error: Test script killed due to a timeout (2m0s) +=== Checking app is running... +=== Checking the proxy is running... +>>> curl -s -o - http://127.0.0.1:$(port) +{"Accept":"*/*","Accept-Encoding":"gzip","Host":"127.0.0.1:$(port)","User-Agent":"curl/(version)","X-Forwarded-Email":"[USERNAME]","X-Forwarded-Host":"localhost","X-Forwarded-Preferred-Username":"","X-Forwarded-User":"[USERNAME]","X-Real-Ip":"127.0.0.1","X-Request-Id":"[UUID]"} + +=== Sending shutdown request... +>>> curl -s -o /dev/null http://127.0.0.1:$(port)/shutdown + +=== Checking CLI command output... +>>> grep To debug your app, attach a debugger to port ./out.run.txt +To debug your app, attach a debugger to port $(debug_port) + +>>> grep -o Python Flask app has started with: test ./out.run.txt +Python Flask app has started with: test diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 1488f50afe..41bf90fc2d 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -99,4 +99,3 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return json.MarshalIndent(newState, " ", " ") } - From 12febe7cd92d6155ed3fdc0b91eaf0a2d13d6cdd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:44:08 +0100 Subject: [PATCH 16/34] Remove unrelated out.run.txt from apps/run-local test Co-Authored-By: Claude Opus 4.6 --- .../cmd/workspace/apps/run-local/out.run.txt | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 acceptance/cmd/workspace/apps/run-local/out.run.txt diff --git a/acceptance/cmd/workspace/apps/run-local/out.run.txt b/acceptance/cmd/workspace/apps/run-local/out.run.txt deleted file mode 100644 index 83628d4c9c..0000000000 --- a/acceptance/cmd/workspace/apps/run-local/out.run.txt +++ /dev/null @@ -1,54 +0,0 @@ - ->>> [CLI] apps run-local --prepare-environment --debug --port 8091 --debug-port 5252 --app-port 8090 -Using CPython 3.11.12 -Creating virtual environment at: .venv -Activate with: source .venv/bin/activate -Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 -Resolved 131 packages in 49ms -Uninstalled 2 packages in 29ms -Installed 2 packages in 10ms - - flask==3.1.1 - + flask==3.0.3 - - werkzeug==3.1.6 - + werkzeug==3.0.6 -Using Python 3.13.3 environment at: /Users/denis.bilenko/.venv313 -Resolved 7 packages in 3ms -Uninstalled 2 packages in 33ms -Installed 2 packages in 11ms - - flask==3.0.3 - + flask==3.1.1 - - werkzeug==3.0.6 - + werkzeug==3.1.6 -Running command: uv run python -m debugpy --listen 5252 -m flask run -To debug your app, attach a debugger to port $(debug_port) -To access your app go to http://localhost:8091 -0.00s - Debugger warning: It seems that frozen modules are being used, which may -0.00s - make the debugger miss breakpoints. Please pass -Xfrozen_modules=off -0.00s - to python to disable frozen modules. -0.00s - Note: Debugging will proceed. Set PYDEVD_DISABLE_FILE_VALIDATION=1 to disable this validation. -Traceback (most recent call last): - File "", line 198, in _run_module_as_main - File "", line 88, in _run_code - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/__main__.py", line 71, in - cli.main() - ~~~~~~~~^^ - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 501, in main - run() - ~~~^^ - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 369, in run_module - start_debugging(argv_0) - ~~~~~~~~~~~~~~~^^^^^^^^ - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/cli.py", line 321, in start_debugging - debugpy.listen(options.address) - ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/public_api.py", line 31, in wrapper - return wrapped(*args, **kwargs) - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 132, in debug - log.reraise_exception("{0}() failed:", func.__name__, level="info") - ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 130, in debug - return func(address, settrace_kwargs, **kwargs) - File "/Users/denis.bilenko/.venv313/lib/python3.13/site-packages/debugpy/server/api.py", line 258, in listen - raise RuntimeError(str(endpoints["error"])) -RuntimeError: Can't listen for client connections: [Errno 48] Address already in use -Error: exit status 1 From 2ee041dbf326121e2bc4046bc06f8bad487d7540 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:51:04 +0100 Subject: [PATCH 17/34] clean up --- acceptance/bundle/invariant/continue_293/test.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 29723a31d2..7bee328d23 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,11 +1,7 @@ Cloud = false Slow = true -# Cross-resource permission references (${resources.jobs.X.permissions[N].field}) require -# permissions to be part of the job schema, which was added after v0.293.0. +# $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] - -# Grants with EmbeddedSlice require state migration from v2→v3, which v0.293.0 can't produce. EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] -EnvMatrixExclude.no_schema_with_grants = ["INPUT_CONFIG=schema_with_grants.yml.tmpl"] From bb38e74beb5d4511c44813d7b230ae26c95f8fb4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 19:57:57 +0100 Subject: [PATCH 18/34] Add KeyedSlices to ResourceGrants for key-based diff comparison Match what permissions does: compare grant entries by principal name instead of by slice index, enabling more accurate change detection. Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dresources/grants.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 695476653c..99d80b6b91 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -76,6 +76,18 @@ func (*ResourceGrants) PrepareState(state *GrantsState) *GrantsState { return state } +func grantKey(x catalog.PrivilegeAssignment) (string, string) { + return "principal", x.Principal +} + +func (*ResourceGrants) KeyedSlices() map[string]any { + // Empty key because EmbeddedSlice appears at the root path of + // GrantsState (no "grants" prefix in struct walker paths). + return map[string]any{ + "": grantKey, + } +} + func (r *ResourceGrants) DoRead(ctx context.Context, id string) (*GrantsState, error) { securableType, fullName, err := parseGrantsID(id) if err != nil { From 31ec046760027092391bddbf12eef48da7fa9503 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:01:10 +0100 Subject: [PATCH 19/34] use __embed__ --- bundle/direct/dresources/grants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 99d80b6b91..7eb13cffa0 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -23,7 +23,7 @@ var grantResourceToSecurableType = map[string]string{ type GrantsState struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` - EmbeddedSlice []catalog.PrivilegeAssignment `json:"grants,omitempty"` + EmbeddedSlice []catalog.PrivilegeAssignment `json:"__embed__,omitempty"` } func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVar, error) { From ff0ec590f4b0808cdb813b50c0a25ae794d65b87 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:07:31 +0100 Subject: [PATCH 20/34] =?UTF-8?q?Remove=20KeyedSlices=20from=20grants,=20a?= =?UTF-8?q?dd=20state=20migration=20v2=E2=86=92v3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove grantKey and KeyedSlices() from ResourceGrants (out of scope) - Add migrateV2ToV3 that renames "grants" → "__embed__" in grant state entries - Bump currentStateVersion to 3 - Use slices.Sort instead of sort.Slice for privileges Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dresources/grants.go | 18 ++--------- bundle/direct/dstate/migrate.go | 51 ++++++++++++++++++++++++++++++ bundle/direct/dstate/state.go | 2 +- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 7eb13cffa0..5332eb89d5 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "sort" + "slices" "strings" "github.com/databricks/cli/libs/structs/structvar" @@ -76,18 +76,6 @@ func (*ResourceGrants) PrepareState(state *GrantsState) *GrantsState { return state } -func grantKey(x catalog.PrivilegeAssignment) (string, string) { - return "principal", x.Principal -} - -func (*ResourceGrants) KeyedSlices() map[string]any { - // Empty key because EmbeddedSlice appears at the root path of - // GrantsState (no "grants" prefix in struct walker paths). - return map[string]any{ - "": grantKey, - } -} - func (r *ResourceGrants) DoRead(ctx context.Context, id string) (*GrantsState, error) { securableType, fullName, err := parseGrantsID(id) if err != nil { @@ -186,9 +174,7 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName } func sortPriviliges(privileges []catalog.Privilege) { - sort.Slice(privileges, func(i, j int) bool { - return privileges[i] < privileges[j] - }) + slices.Sort(privileges) } func extractGrantResourceType(node string) (string, error) { diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 41bf90fc2d..5ac2eb55b7 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -38,6 +38,7 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, + 2: migrateV2ToV3, } // migrateV1ToV2 migrates permissions entries from the old format @@ -99,3 +100,53 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return json.MarshalIndent(newState, " ", " ") } + +// migrateV2ToV3 migrates grants entries from the old format +// ("grants" key) to the new format ("__embed__" key). +func migrateV2ToV3(db *Database) error { + for key, entry := range db.State { + if !strings.HasSuffix(key, ".grants") { + continue + } + if len(entry.State) == 0 { + continue + } + migrated, err := migrateGrantsEntry(entry.State) + if err != nil { + return fmt.Errorf("migrating %s: %w", key, err) + } + entry.State = migrated + db.State[key] = entry + } + return nil +} + +// oldGrantsStateV2 is the grants state format before v3. +type oldGrantsStateV2 struct { + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + Grants json.RawMessage `json:"grants,omitempty"` +} + +func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { + var old oldGrantsStateV2 + if err := json.Unmarshal(raw, &old); err != nil { + return nil, err + } + + // If old format had no grants, it might already be migrated. + if len(old.Grants) == 0 { + return raw, nil + } + + // Re-serialize with __embed__ key. + newState := dresources.GrantsState{ + SecurableType: old.SecurableType, + FullName: old.FullName, + } + if err := json.Unmarshal(old.Grants, &newState.EmbeddedSlice); err != nil { + return nil, err + } + + return json.MarshalIndent(newState, " ", " ") +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e1c776b479..3542cf8d0a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const currentStateVersion = 2 +const currentStateVersion = 3 type DeploymentState struct { Path string From fd4e2eb1fb987f204c0956907ababf4355deb717 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:09:57 +0100 Subject: [PATCH 21/34] Update acceptance test outputs for state version 3 and __embed__ key Co-Authored-By: Claude Opus 4.6 --- .../dashboard/recreation/out.state_after_bind.direct.json | 2 +- acceptance/bundle/migrate/basic/out.new_state.json | 2 +- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../migrate/default-python/out.state_after_migration.json | 2 +- acceptance/bundle/migrate/grants/out.new_state.json | 8 ++++---- acceptance/bundle/migrate/permissions/out.new_state.json | 2 +- acceptance/bundle/migrate/runas/out.new_state.json | 2 +- .../resource_deps/grant_ref/out.plan_create.direct.json | 4 ++-- .../bundle/resources/jobs/big_id/out.state.direct.json | 2 +- .../bundle/resources/jobs/update/out.state.direct.json | 2 +- .../bundle/state/permission_level_migration/output.txt | 2 +- .../user_agent/simple/out.requests.deploy.direct.json | 2 +- 12 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index 771dd3f908..c7a30bcd43 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index f6bdf06f62..e294a69095 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index 695d14602a..a3d491562e 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 029649aae3..8ff25834b7 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 2046d78c9d..5a0a07026a 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 9, @@ -24,7 +24,7 @@ "state": { "securable_type": "function", "full_name": "main.schema_grants.mymodel", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -52,7 +52,7 @@ "state": { "securable_type": "schema", "full_name": "main.schema_grants", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -89,7 +89,7 @@ "state": { "securable_type": "volume", "full_name": "main.schema_grants.volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index b020b1401d..c9043ddf6c 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index a7308ab59c..dc6d571877 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 3b47faf360..970fab66ae 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "viewers", "privileges": [ @@ -72,7 +72,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "viewers", "privileges": [ diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index f8cf0ce5bf..31eb566641 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 785c0c1338..76888ff526 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index f289e0bc42..fbcaea775e 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -12,7 +12,7 @@ Deployment complete! === Print state after deploy >>> print_state.py { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index d4543d353a..f252d6df1d 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -222,7 +222,7 @@ "overwrite": "true" }, "body": { - "state_version": 2, + "state_version": 3, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, From ca9ee3f0474b85c03085be6100d1e6ac9572dfb3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:20:37 +0100 Subject: [PATCH 22/34] Regenerate acceptance test outputs for state version 3 and __embed__ grants key Co-Authored-By: Claude Opus 4.6 --- .../bundle/resources/grants/catalogs/out.plan1.direct.json | 2 +- .../bundle/resources/grants/catalogs/out.plan2.direct.json | 4 ++-- .../resources/grants/registered_models/out.plan1.direct.json | 2 +- .../grants/schemas/change_privilege/out.plan1.direct.json | 2 +- .../grants/schemas/change_privilege/out.plan2.direct.json | 4 ++-- .../resources/grants/schemas/empty_array/out.plan1.direct.txt | 2 +- .../bundle/resources/grants/volumes/out.plan1.direct.json | 2 +- .../bundle/resources/grants/volumes/out.plan2.direct.json | 4 ++-- acceptance/bundle/state/future_version/output.txt | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 6f50fd2b5f..333e2928d5 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -22,7 +22,7 @@ "new_state": { "value": { "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 681ee94884..a63b52c327 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -25,7 +25,7 @@ "new_state": { "value": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -38,7 +38,7 @@ }, "remote_state": { "full_name": "catalog_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index b5ab8d0b9d..482faa449f 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -31,7 +31,7 @@ "value": { "securable_type": "function", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 01d2d581b2..c8d1428519 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index 8e928edf0a..7232642adc 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -29,7 +29,7 @@ "value": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -43,7 +43,7 @@ "remote_state": { "securable_type": "schema", "full_name": "main.schema_grants_[UNIQUE_NAME]", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt index a259b5603f..2a48dc2405 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.plan1.direct.txt @@ -23,7 +23,7 @@ "value": { "securable_type": "schema", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com" } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index 2f9e4d51a1..daae7c36ac 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -40,7 +40,7 @@ "value": { "securable_type": "volume", "full_name": "", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index e56df92969..4de57d5ed6 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -56,7 +56,7 @@ "value": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ @@ -70,7 +70,7 @@ "remote_state": { "securable_type": "volume", "full_name": "main.schema_grants_[UNIQUE_NAME].volume_name", - "grants": [ + "__embed__": [ { "principal": "deco-test-user@databricks.com", "privileges": [ diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 7cf98129ee..0a16971f47 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 2; upgrade the CLI +state version 999 is newer than supported version 3; upgrade the CLI Exit code: 1 From 30518957d4131f3423d6416be95d0a96e188f68a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:21:42 +0100 Subject: [PATCH 23/34] =?UTF-8?q?Add=20unit=20tests=20for=20grants=20state?= =?UTF-8?q?=20migration=20v2=E2=86=92v3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dstate/migrate_test.go | 85 ++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 bundle/direct/dstate/migrate_test.go diff --git a/bundle/direct/dstate/migrate_test.go b/bundle/direct/dstate/migrate_test.go new file mode 100644 index 0000000000..b5f904e63b --- /dev/null +++ b/bundle/direct/dstate/migrate_test.go @@ -0,0 +1,85 @@ +package dstate + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrateV2ToV3GrantsEntry(t *testing.T) { + input := json.RawMessage(`{ + "securable_type": "catalog", + "full_name": "main", + "grants": [ + {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} + ] + }`) + + result, err := migrateGrantsEntry(input) + require.NoError(t, err) + + var parsed map[string]any + require.NoError(t, json.Unmarshal(result, &parsed)) + + assert.Equal(t, "catalog", parsed["securable_type"]) + assert.Equal(t, "main", parsed["full_name"]) + assert.Nil(t, parsed["grants"], "old 'grants' key should be removed") + assert.NotNil(t, parsed["__embed__"], "'__embed__' key should be present") +} + +func TestMigrateV2ToV3AlreadyMigrated(t *testing.T) { + input := json.RawMessage(`{ + "securable_type": "catalog", + "full_name": "main", + "__embed__": [ + {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} + ] + }`) + + result, err := migrateGrantsEntry(input) + require.NoError(t, err) + + // Should pass through unchanged since there's no "grants" field. + var parsed map[string]any + require.NoError(t, json.Unmarshal(result, &parsed)) + assert.NotNil(t, parsed["__embed__"]) +} + +func TestMigrateV2ToV3FullDatabase(t *testing.T) { + db := &Database{ + StateVersion: 2, + State: map[string]ResourceEntry{ + "resources.catalogs.my_cat.grants": { + ID: "catalog/main", + State: json.RawMessage(`{ + "securable_type": "catalog", + "full_name": "main", + "grants": [ + {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} + ] + }`), + }, + "resources.jobs.my_job": { + ID: "123", + State: json.RawMessage(`{"job_id": 123}`), + }, + }, + } + + err := migrateState(db) + require.NoError(t, err) + assert.Equal(t, currentStateVersion, db.StateVersion) + + // Grants entry should be migrated. + var grantsState map[string]any + require.NoError(t, json.Unmarshal(db.State["resources.catalogs.my_cat.grants"].State, &grantsState)) + assert.NotNil(t, grantsState["__embed__"]) + assert.Nil(t, grantsState["grants"]) + + // Non-grants entry should be unchanged. + var jobState map[string]any + require.NoError(t, json.Unmarshal(db.State["resources.jobs.my_job"].State, &jobState)) + assert.Equal(t, float64(123), jobState["job_id"]) +} From d7302274c0d3c77707c512560f0c0a97d18ad602 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:22:14 +0100 Subject: [PATCH 24/34] Fix lint issue in migration test Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dstate/migrate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dstate/migrate_test.go b/bundle/direct/dstate/migrate_test.go index b5f904e63b..598bb1e64f 100644 --- a/bundle/direct/dstate/migrate_test.go +++ b/bundle/direct/dstate/migrate_test.go @@ -81,5 +81,5 @@ func TestMigrateV2ToV3FullDatabase(t *testing.T) { // Non-grants entry should be unchanged. var jobState map[string]any require.NoError(t, json.Unmarshal(db.State["resources.jobs.my_job"].State, &jobState)) - assert.Equal(t, float64(123), jobState["job_id"]) + assert.InDelta(t, 123, jobState["job_id"], 0) } From 4ab7cacf87656e58ef5bcbaf4fa06b7bda7dbdf3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:31:39 +0100 Subject: [PATCH 25/34] =?UTF-8?q?Fold=20grants=20migration=20into=20v1?= =?UTF-8?q?=E2=86=92v2,=20remove=20state=20version=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grants and permissions migrations are now both part of the v1→v2 migration step. This keeps currentStateVersion at 2. Co-Authored-By: Claude Opus 4.6 --- bundle/direct/dstate/migrate.go | 49 +++++++++++----------------- bundle/direct/dstate/migrate_test.go | 8 ++--- bundle/direct/dstate/state.go | 2 +- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 5ac2eb55b7..4871f35b18 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -38,21 +38,30 @@ func migrateState(db *Database) error { var migrations = map[int]func(*Database) error{ 0: migrateV1ToV2, 1: migrateV1ToV2, - 2: migrateV2ToV3, } -// migrateV1ToV2 migrates permissions entries from the old format -// (iam.AccessControlRequest with "permissions" key and "permission_level" field) -// to the new format (StatePermission with "_" key and "level" field). +// migrateV1ToV2 migrates permissions and grants entries from the old format +// to the new format using __embed__ keys. func migrateV1ToV2(db *Database) error { for key, entry := range db.State { - if !strings.HasSuffix(key, ".permissions") { + if len(entry.State) == 0 { continue } - if len(entry.State) == 0 { + + var ( + migrated json.RawMessage + err error + ) + + switch { + case strings.HasSuffix(key, ".permissions"): + migrated, err = migratePermissionsEntry(entry.State) + case strings.HasSuffix(key, ".grants"): + migrated, err = migrateGrantsEntry(entry.State) + default: continue } - migrated, err := migratePermissionsEntry(entry.State) + if err != nil { return fmt.Errorf("migrating %s: %w", key, err) } @@ -101,35 +110,15 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return json.MarshalIndent(newState, " ", " ") } -// migrateV2ToV3 migrates grants entries from the old format -// ("grants" key) to the new format ("__embed__" key). -func migrateV2ToV3(db *Database) error { - for key, entry := range db.State { - if !strings.HasSuffix(key, ".grants") { - continue - } - if len(entry.State) == 0 { - continue - } - migrated, err := migrateGrantsEntry(entry.State) - if err != nil { - return fmt.Errorf("migrating %s: %w", key, err) - } - entry.State = migrated - db.State[key] = entry - } - return nil -} - -// oldGrantsStateV2 is the grants state format before v3. -type oldGrantsStateV2 struct { +// oldGrantsStateV1 is the grants state format before v2. +type oldGrantsStateV1 struct { SecurableType string `json:"securable_type"` FullName string `json:"full_name"` Grants json.RawMessage `json:"grants,omitempty"` } func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { - var old oldGrantsStateV2 + var old oldGrantsStateV1 if err := json.Unmarshal(raw, &old); err != nil { return nil, err } diff --git a/bundle/direct/dstate/migrate_test.go b/bundle/direct/dstate/migrate_test.go index 598bb1e64f..70eed8ce88 100644 --- a/bundle/direct/dstate/migrate_test.go +++ b/bundle/direct/dstate/migrate_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestMigrateV2ToV3GrantsEntry(t *testing.T) { +func TestMigrateV1ToV2GrantsEntry(t *testing.T) { input := json.RawMessage(`{ "securable_type": "catalog", "full_name": "main", @@ -29,7 +29,7 @@ func TestMigrateV2ToV3GrantsEntry(t *testing.T) { assert.NotNil(t, parsed["__embed__"], "'__embed__' key should be present") } -func TestMigrateV2ToV3AlreadyMigrated(t *testing.T) { +func TestMigrateV1ToV2GrantsAlreadyMigrated(t *testing.T) { input := json.RawMessage(`{ "securable_type": "catalog", "full_name": "main", @@ -47,9 +47,9 @@ func TestMigrateV2ToV3AlreadyMigrated(t *testing.T) { assert.NotNil(t, parsed["__embed__"]) } -func TestMigrateV2ToV3FullDatabase(t *testing.T) { +func TestMigrateV1ToV2FullDatabase(t *testing.T) { db := &Database{ - StateVersion: 2, + StateVersion: 1, State: map[string]ResourceEntry{ "resources.catalogs.my_cat.grants": { ID: "catalog/main", diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 3542cf8d0a..e1c776b479 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const currentStateVersion = 3 +const currentStateVersion = 2 type DeploymentState struct { Path string From 969fef1ae83c5c51cb5354982a0f50adec4ce2d2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:33:37 +0100 Subject: [PATCH 26/34] Regenerate acceptance test outputs for state version 2 Co-Authored-By: Claude Opus 4.6 --- .../bind/dashboard/recreation/out.state_after_bind.direct.json | 2 +- acceptance/bundle/migrate/basic/out.new_state.json | 2 +- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../migrate/default-python/out.state_after_migration.json | 2 +- acceptance/bundle/migrate/grants/out.new_state.json | 2 +- acceptance/bundle/migrate/permissions/out.new_state.json | 2 +- acceptance/bundle/migrate/runas/out.new_state.json | 2 +- acceptance/bundle/resources/jobs/big_id/out.state.direct.json | 2 +- acceptance/bundle/resources/jobs/update/out.state.direct.json | 2 +- acceptance/bundle/state/permission_level_migration/output.txt | 2 +- .../bundle/user_agent/simple/out.requests.deploy.direct.json | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index c7a30bcd43..771dd3f908 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index e294a69095..f6bdf06f62 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index a3d491562e..695d14602a 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 8ff25834b7..029649aae3 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 5a0a07026a..7a24ba0f3c 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 9, diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index c9043ddf6c..b020b1401d 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 7, diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index dc6d571877..a7308ab59c 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json index 31eb566641..f8cf0ce5bf 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 76888ff526..785c0c1338 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index fbcaea775e..f289e0bc42 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -12,7 +12,7 @@ Deployment complete! === Print state after deploy >>> print_state.py { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "test-lineage", "serial": 2, diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index f252d6df1d..d4543d353a 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -222,7 +222,7 @@ "overwrite": "true" }, "body": { - "state_version": 3, + "state_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", "serial": 1, From 4c97697a7b737d5193d85197ac90bd9279ad484d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Mar 2026 20:46:07 +0100 Subject: [PATCH 27/34] update test out --- acceptance/bundle/state/future_version/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f47..7cf98129ee 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 3; upgrade the CLI +state version 999 is newer than supported version 2; upgrade the CLI Exit code: 1 From d084530a12aa6fd483fad58a24e094ce46040488 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 09:55:38 +0100 Subject: [PATCH 28/34] clean up --- bundle/direct/dstate/migrate.go | 10 ++-- bundle/direct/dstate/migrate_test.go | 85 ---------------------------- 2 files changed, 6 insertions(+), 89 deletions(-) delete mode 100644 bundle/direct/dstate/migrate_test.go diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 4871f35b18..c338e63797 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -41,7 +41,7 @@ var migrations = map[int]func(*Database) error{ } // migrateV1ToV2 migrates permissions and grants entries from the old format -// to the new format using __embed__ keys. +// to the new format (__embed__ keys, permission_level -> level). func migrateV1ToV2(db *Database) error { for key, entry := range db.State { if len(entry.State) == 0 { @@ -54,6 +54,7 @@ func migrateV1ToV2(db *Database) error { ) switch { + // AGENT: instead of using HasSuffix, parse the key with structpath and read 4th component case strings.HasSuffix(key, ".permissions"): migrated, err = migratePermissionsEntry(entry.State) case strings.HasSuffix(key, ".grants"): @@ -112,9 +113,10 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { // oldGrantsStateV1 is the grants state format before v2. type oldGrantsStateV1 struct { - SecurableType string `json:"securable_type"` - FullName string `json:"full_name"` - Grants json.RawMessage `json:"grants,omitempty"` + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + // AGENT: replace json.RawMessage with actual struct there and do explicit copy in migrateGrantsEntry like we do in migratePermissionsEntry + Grants json.RawMessage `json:"grants,omitempty"` } func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { diff --git a/bundle/direct/dstate/migrate_test.go b/bundle/direct/dstate/migrate_test.go deleted file mode 100644 index 70eed8ce88..0000000000 --- a/bundle/direct/dstate/migrate_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package dstate - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMigrateV1ToV2GrantsEntry(t *testing.T) { - input := json.RawMessage(`{ - "securable_type": "catalog", - "full_name": "main", - "grants": [ - {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} - ] - }`) - - result, err := migrateGrantsEntry(input) - require.NoError(t, err) - - var parsed map[string]any - require.NoError(t, json.Unmarshal(result, &parsed)) - - assert.Equal(t, "catalog", parsed["securable_type"]) - assert.Equal(t, "main", parsed["full_name"]) - assert.Nil(t, parsed["grants"], "old 'grants' key should be removed") - assert.NotNil(t, parsed["__embed__"], "'__embed__' key should be present") -} - -func TestMigrateV1ToV2GrantsAlreadyMigrated(t *testing.T) { - input := json.RawMessage(`{ - "securable_type": "catalog", - "full_name": "main", - "__embed__": [ - {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} - ] - }`) - - result, err := migrateGrantsEntry(input) - require.NoError(t, err) - - // Should pass through unchanged since there's no "grants" field. - var parsed map[string]any - require.NoError(t, json.Unmarshal(result, &parsed)) - assert.NotNil(t, parsed["__embed__"]) -} - -func TestMigrateV1ToV2FullDatabase(t *testing.T) { - db := &Database{ - StateVersion: 1, - State: map[string]ResourceEntry{ - "resources.catalogs.my_cat.grants": { - ID: "catalog/main", - State: json.RawMessage(`{ - "securable_type": "catalog", - "full_name": "main", - "grants": [ - {"principal": "user@example.com", "privileges": ["USE_CATALOG"]} - ] - }`), - }, - "resources.jobs.my_job": { - ID: "123", - State: json.RawMessage(`{"job_id": 123}`), - }, - }, - } - - err := migrateState(db) - require.NoError(t, err) - assert.Equal(t, currentStateVersion, db.StateVersion) - - // Grants entry should be migrated. - var grantsState map[string]any - require.NoError(t, json.Unmarshal(db.State["resources.catalogs.my_cat.grants"].State, &grantsState)) - assert.NotNil(t, grantsState["__embed__"]) - assert.Nil(t, grantsState["grants"]) - - // Non-grants entry should be unchanged. - var jobState map[string]any - require.NoError(t, json.Unmarshal(db.State["resources.jobs.my_job"].State, &jobState)) - assert.InDelta(t, 123, jobState["job_id"], 0) -} From 2effbee222ce0f393ad9773ad65769b9741409db Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 10:01:36 +0100 Subject: [PATCH 29/34] Fix AGENT TODOs in dstate/migrate.go - Use structpath.ParsePath + StringKey to identify migration targets instead of strings.HasSuffix - Replace json.RawMessage in oldGrantsStateV1 with []catalog.PrivilegeAssignment and use explicit copy, matching the pattern in migratePermissionsEntry Co-Authored-By: Claude Sonnet 4.6 --- bundle/direct/dstate/migrate.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index c338e63797..8542e573f5 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,9 +3,10 @@ package dstate import ( "encoding/json" "fmt" - "strings" "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/iam" ) @@ -48,16 +49,25 @@ func migrateV1ToV2(db *Database) error { continue } + path, pathErr := structpath.ParsePath(key) + if pathErr != nil || path == nil { + continue + } + // path points to the last node; read its key directly. + lastKey, ok := path.StringKey() + if !ok { + continue + } + var ( migrated json.RawMessage err error ) - switch { - // AGENT: instead of using HasSuffix, parse the key with structpath and read 4th component - case strings.HasSuffix(key, ".permissions"): + switch lastKey { + case "permissions": migrated, err = migratePermissionsEntry(entry.State) - case strings.HasSuffix(key, ".grants"): + case "grants": migrated, err = migrateGrantsEntry(entry.State) default: continue @@ -113,10 +123,9 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { // oldGrantsStateV1 is the grants state format before v2. type oldGrantsStateV1 struct { - SecurableType string `json:"securable_type"` - FullName string `json:"full_name"` - // AGENT: replace json.RawMessage with actual struct there and do explicit copy in migrateGrantsEntry like we do in migratePermissionsEntry - Grants json.RawMessage `json:"grants,omitempty"` + SecurableType string `json:"securable_type"` + FullName string `json:"full_name"` + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { @@ -130,14 +139,12 @@ func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { return raw, nil } - // Re-serialize with __embed__ key. newState := dresources.GrantsState{ SecurableType: old.SecurableType, FullName: old.FullName, + EmbeddedSlice: make([]catalog.PrivilegeAssignment, len(old.Grants)), } - if err := json.Unmarshal(old.Grants, &newState.EmbeddedSlice); err != nil { - return nil, err - } + copy(newState.EmbeddedSlice, old.Grants) return json.MarshalIndent(newState, " ", " ") } From 971ddf31f85e6db08c4ba0df8bbb95ee2f477114 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 10:18:04 +0100 Subject: [PATCH 30/34] Revert sortPriviliges to use sort.Slice Co-Authored-By: Claude Sonnet 4.6 --- bundle/direct/dresources/grants.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 5332eb89d5..7a1f169938 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "slices" + "sort" "strings" "github.com/databricks/cli/libs/structs/structvar" @@ -174,7 +174,9 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName } func sortPriviliges(privileges []catalog.Privilege) { - slices.Sort(privileges) + sort.Slice(privileges, func(i, j int) bool { + return privileges[i] < privileges[j] + }) } func extractGrantResourceType(node string) (string, error) { From 75edb0ebaad85cfccd7d830cacfd685ee09ebbe8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 10:20:24 +0100 Subject: [PATCH 31/34] Remove redundant early-return checks in migration functions State version < 2 cannot contain __embed__ format, so the "might already be migrated" guards are unnecessary. Co-Authored-By: Claude Sonnet 4.6 --- bundle/direct/dstate/migrate.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 8542e573f5..d7e65bcdd4 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -101,11 +101,6 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { return nil, err } - // If old format had no permissions, try parsing as new format (might already be migrated). - if len(old.Permissions) == 0 { - return raw, nil - } - newState := dresources.PermissionsState{ ObjectID: old.ObjectID, } @@ -134,11 +129,6 @@ func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { return nil, err } - // If old format had no grants, it might already be migrated. - if len(old.Grants) == 0 { - return raw, nil - } - newState := dresources.GrantsState{ SecurableType: old.SecurableType, FullName: old.FullName, From ed528f7650d314b40c2b8bc975e6d5d979f5fc7f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 10:31:26 +0100 Subject: [PATCH 32/34] Use json.Marshal instead of json.MarshalIndent in migration The result is parsed again later, indentation is unnecessary. Co-Authored-By: Claude Sonnet 4.6 --- bundle/direct/dstate/migrate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index d7e65bcdd4..f14b80a634 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -113,7 +113,7 @@ func migratePermissionsEntry(raw json.RawMessage) (json.RawMessage, error) { }) } - return json.MarshalIndent(newState, " ", " ") + return json.Marshal(newState) } // oldGrantsStateV1 is the grants state format before v2. @@ -136,5 +136,5 @@ func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { } copy(newState.EmbeddedSlice, old.Grants) - return json.MarshalIndent(newState, " ", " ") + return json.Marshal(newState) } From 512d20f0776594a02aa476b011de99cb58cbe936 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 11:29:12 +0100 Subject: [PATCH 33/34] Replace make+copy with direct assignment in migrateGrantsEntry old.Grants is a fresh local allocation from json.Unmarshal, no aliasing risk. Co-Authored-By: Claude Sonnet 4.6 --- bundle/direct/dstate/migrate.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index f14b80a634..381d63a12e 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -132,9 +132,8 @@ func migrateGrantsEntry(raw json.RawMessage) (json.RawMessage, error) { newState := dresources.GrantsState{ SecurableType: old.SecurableType, FullName: old.FullName, - EmbeddedSlice: make([]catalog.PrivilegeAssignment, len(old.Grants)), + EmbeddedSlice: old.Grants, } - copy(newState.EmbeddedSlice, old.Grants) return json.Marshal(newState) } From b27d743da85440bb82e9c9b0ece34c5a9862a7c5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 18 Mar 2026 11:50:31 +0100 Subject: [PATCH 34/34] add a comment --- bundle/direct/dresources/permissions.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 02590794d4..7a64f53d4b 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -48,7 +48,8 @@ type StatePermission struct { // by unrelated types and it's harder to evaluate that fixed string because it's non-local. type PermissionsState struct { - ObjectID string `json:"object_id"` + ObjectID string `json:"object_id"` + // By convention, EmbedSlice fields should have __embed__ json tag, see permissions.go for details EmbeddedSlice []StatePermission `json:"__embed__,omitempty"` }