From b2e6272db4751110519ff4b7becc011d1ec6d98e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Mar 2026 11:03:19 +0100 Subject: [PATCH 01/42] Added support for lifecycle.started option --- .../bundle/lifecycle/started/databricks.yml | 9 ++ .../bundle/lifecycle/started/out.test.toml | 5 + .../bundle/lifecycle/started/output.txt | 5 + acceptance/bundle/lifecycle/started/script | 1 + acceptance/bundle/lifecycle/started/test.toml | 7 ++ .../apps/lifecycle-started/app/app.py | 1 + .../lifecycle-started/databricks.yml.tmpl | 11 ++ .../apps/lifecycle-started/out.test.toml | 5 + .../apps/lifecycle-started/output.txt | 106 +++++++++++++++++ .../resources/apps/lifecycle-started/script | 52 +++++++++ .../apps/lifecycle-started/test.toml | 8 ++ bundle/appdeploy/app.go | 110 ++++++++++++++++++ .../mutator/validate_lifecycle_started.go | 77 ++++++++++++ .../validate_lifecycle_started_test.go | 109 +++++++++++++++++ bundle/config/resources/lifecycle.go | 4 + bundle/deploy/terraform/tfdyn/convert_app.go | 2 +- .../deploy/terraform/tfdyn/convert_cluster.go | 2 +- bundle/deploy/terraform/tfdyn/lifecycle.go | 14 ++- bundle/direct/bundle_apply.go | 21 +++- bundle/direct/dresources/app.go | 74 ++++++++++-- bundle/direct/dresources/app_test.go | 4 +- bundle/direct/dresources/cluster.go | 8 ++ bundle/direct/dresources/lifecycle_ctx.go | 16 +++ bundle/direct/dresources/sql_warehouse.go | 8 ++ bundle/direct/dresources/type_test.go | 11 +- bundle/internal/schema/annotations.yml | 3 + bundle/phases/plan.go | 1 + bundle/run/app.go | 108 +---------------- bundle/run/app_test.go | 7 +- libs/testserver/apps.go | 73 ++++++++++++ libs/testserver/handlers.go | 16 +++ 31 files changed, 750 insertions(+), 128 deletions(-) create mode 100644 acceptance/bundle/lifecycle/started/databricks.yml create mode 100644 acceptance/bundle/lifecycle/started/out.test.toml create mode 100644 acceptance/bundle/lifecycle/started/output.txt create mode 100644 acceptance/bundle/lifecycle/started/script create mode 100644 acceptance/bundle/lifecycle/started/test.toml create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/app/app.py create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.test.toml create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/output.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/script create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/test.toml create mode 100644 bundle/appdeploy/app.go create mode 100644 bundle/config/mutator/validate_lifecycle_started.go create mode 100644 bundle/config/mutator/validate_lifecycle_started_test.go create mode 100644 bundle/direct/dresources/lifecycle_ctx.go diff --git a/acceptance/bundle/lifecycle/started/databricks.yml b/acceptance/bundle/lifecycle/started/databricks.yml new file mode 100644 index 0000000000..313b1cb8de --- /dev/null +++ b/acceptance/bundle/lifecycle/started/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: test_lifecycle_started + +resources: + jobs: + my_job: + name: my_job + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/lifecycle/started/output.txt b/acceptance/bundle/lifecycle/started/output.txt new file mode 100644 index 0000000000..0ffc8f1d18 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/output.txt @@ -0,0 +1,5 @@ +Error: lifecycle.started is not supported for resources.jobs.my_job; it is only supported for apps, clusters, and sql_warehouses + in databricks.yml:9:18 + + +Exit code: 1 diff --git a/acceptance/bundle/lifecycle/started/script b/acceptance/bundle/lifecycle/started/script new file mode 100644 index 0000000000..4fbc2b517c --- /dev/null +++ b/acceptance/bundle/lifecycle/started/script @@ -0,0 +1 @@ +errcode $CLI bundle plan diff --git a/acceptance/bundle/lifecycle/started/test.toml b/acceptance/bundle/lifecycle/started/test.toml new file mode 100644 index 0000000000..d1240963e0 --- /dev/null +++ b/acceptance/bundle/lifecycle/started/test.toml @@ -0,0 +1,7 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt new file mode 100644 index 0000000000..4ad8a24b60 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -0,0 +1,106 @@ + +=== Deploy app with lifecycle.started=true +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify create request: no_compute must not be set +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +=== Check app compute state: must be ACTIVE +>>> [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Update description and re-deploy +>>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify POST /deployments was called after update +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Stop the app +>>> [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +=== Change lifecycle.started to false and update +>>> update_file.py databricks.yml started: true started: false + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify POST /deployments is NOT called (lifecycle.started=false) +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +=== Check app compute state: must still be STOPPED +>>> [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Switch lifecycle.started back to true and update +>>> update_file.py databricks.yml started: false started: true + +>>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify POST /start and POST /deployments were called +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Check app compute state: must be ACTIVE again +>>> [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script new file mode 100644 index 0000000000..2b9f234a4d --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -0,0 +1,52 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy app with lifecycle.started=true" +trace $CLI bundle deploy + +title "Verify create request: no_compute must not be set" +trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must be ACTIVE" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' + +title "Update description and re-deploy" +trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +trace $CLI bundle deploy + +title "Verify POST /deployments was called after update" +trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +rm -f out.requests.txt + +title "Stop the app" +trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' + +title "Change lifecycle.started to false and update" +trace update_file.py databricks.yml "started: true" "started: false" +trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 +trace $CLI bundle deploy + +title "Verify POST /deployments is NOT called (lifecycle.started=false)" +trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must still be STOPPED" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' + +title "Switch lifecycle.started back to true and update" +trace update_file.py databricks.yml "started: false" "started: true" +trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 +trace $CLI bundle deploy + +title "Verify POST /start and POST /deployments were called" +trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt +rm -f out.requests.txt + +title "Check app compute state: must be ACTIVE again" +trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/appdeploy/app.go b/bundle/appdeploy/app.go new file mode 100644 index 0000000000..e73922d490 --- /dev/null +++ b/bundle/appdeploy/app.go @@ -0,0 +1,110 @@ +package appdeploy + +import ( + "context" + "fmt" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go" + sdkapps "github.com/databricks/databricks-sdk-go/service/apps" +) + +func logProgress(ctx context.Context, msg string) { + if msg == "" { + return + } + cmdio.LogString(ctx, "✓ "+msg) +} + +// BuildDeployment constructs an AppDeployment from the app's source code path, inline config and git source. +func BuildDeployment(sourcePath string, config *resources.AppConfig, gitSource *sdkapps.GitSource) sdkapps.AppDeployment { + deployment := sdkapps.AppDeployment{ + Mode: sdkapps.AppDeploymentModeSnapshot, + SourceCodePath: sourcePath, + } + + if gitSource != nil { + deployment.GitSource = gitSource + } + + if config != nil { + if len(config.Command) > 0 { + deployment.Command = config.Command + } + + if len(config.Env) > 0 { + deployment.EnvVars = make([]sdkapps.EnvVar, len(config.Env)) + for i, env := range config.Env { + deployment.EnvVars[i] = sdkapps.EnvVar{ + Name: env.Name, + Value: env.Value, + ValueFrom: env.ValueFrom, + } + } + } + } + + return deployment +} + +// WaitForDeploymentToComplete waits for active and pending deployments on an app to finish. +func WaitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *sdkapps.App) error { + if app.ActiveDeployment != nil && + app.ActiveDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the active deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Active deployment is completed!") + } + + if app.PendingDeployment != nil && + app.PendingDeployment.Status.State == sdkapps.AppDeploymentStateInProgress { + logProgress(ctx, "Waiting for the pending deployment to complete...") + _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) + if err != nil { + return err + } + logProgress(ctx, "Pending deployment is completed!") + } + + return nil +} + +// Deploy deploys the app using the provided deployment request. +// If another deployment is in progress, it waits for it to complete and retries. +func Deploy(ctx context.Context, w *databricks.WorkspaceClient, appName string, deployment sdkapps.AppDeployment) error { + wait, err := w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + existingApp, getErr := w.Apps.Get(ctx, sdkapps.GetAppRequest{Name: appName}) + if getErr != nil { + return fmt.Errorf("failed to get app %s: %w", appName, getErr) + } + + if waitErr := WaitForDeploymentToComplete(ctx, w, existingApp); waitErr != nil { + return waitErr + } + + wait, err = w.Apps.Deploy(ctx, sdkapps.CreateAppDeploymentRequest{ + AppName: appName, + AppDeployment: deployment, + }) + if err != nil { + return err + } + } + + _, err = wait.OnProgress(func(ad *sdkapps.AppDeployment) { + if ad.Status == nil { + return + } + logProgress(ctx, ad.Status.Message) + }).Get() + return err +} diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go new file mode 100644 index 0000000000..4a02b965d2 --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -0,0 +1,77 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +// supportedForLifecycleStarted lists resource types that support lifecycle.started. +var supportedForLifecycleStarted = map[string]bool{ + "apps": true, + "clusters": true, + "sql_warehouses": true, +} + +type validateLifecycleStarted struct { + engine engine.EngineType +} + +// ValidateLifecycleStarted returns a mutator that validates lifecycle.started +// is only used on supported resource types (apps, clusters, sql_warehouses). +// lifecycle.started is only supported in direct deployment mode. +func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { + return &validateLifecycleStarted{engine: e} +} + +func (m *validateLifecycleStarted) Name() string { + return "ValidateLifecycleStarted" +} + +func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + // lifecycle.started is a direct-mode-only feature; ignore it in other modes. + if !m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + + _, err := dyn.MapByPattern( + b.Config.Value(), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + func(path dyn.Path, v dyn.Value) (dyn.Value, error) { + resourceType := path[1].Key() + if supportedForLifecycleStarted[resourceType] { + return v, nil + } + + startedV, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("lifecycle"), dyn.Key("started"))) + if err != nil { + return v, nil + } + + started, ok := startedV.AsBool() + if !ok || !started { + return v, nil + } + + resourceKey := path.String() + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.started is not supported for %s; it is only supported for apps, clusters, and sql_warehouses", resourceKey), + Locations: []dyn.Location{startedV.Location()}, + }) + + return v, nil + }, + ) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + } + + return diags +} diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go new file mode 100644 index 0000000000..7e3e10a619 --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started_test.go @@ -0,0 +1,109 @@ +package mutator + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func boolPtr(b bool) *bool { return &b } + +func TestValidateLifecycleStarted_UnsupportedResource(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + require.Error(t, m.Error()) + assert.Contains(t, m.Error().Error(), "lifecycle.started is not supported for resources.jobs.my_job") +} + +func TestValidateLifecycleStarted_SupportedResources(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + SqlWarehouses: map[string]*resources.SqlWarehouse{ + "my_warehouse": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + CreateWarehouseRequest: sql.CreateWarehouseRequest{ + Name: "my_warehouse", + }, + }, + }, + }, + }, + } + + m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + assert.NoError(t, m.Error()) +} + +func TestValidateLifecycleStarted_StartedFalse(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(false), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + assert.NoError(t, m.Error()) +} + +func TestValidateLifecycleStarted_TerraformModeIgnored(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{ + Started: boolPtr(true), + }, + }, + JobSettings: jobs.JobSettings{Name: "my_job"}, + }, + }, + }, + }, + } + + // In TF mode, lifecycle.started is ignored — no error even for unsupported resource types. + m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineTerraform)) + assert.NoError(t, m.Error()) +} diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index c3de7ce8ea..2ab74b4e30 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -5,4 +5,8 @@ package resources type Lifecycle struct { // Lifecycle setting to prevent the resource from being destroyed. PreventDestroy bool `json:"prevent_destroy,omitempty"` + + // If set to true, the resource will be deployed in started mode. + // Supported only for apps, clusters, and sql_warehouses. + Started *bool `json:"started,omitempty"` } diff --git a/bundle/deploy/terraform/tfdyn/convert_app.go b/bundle/deploy/terraform/tfdyn/convert_app.go index b25d403766..87095eb0aa 100644 --- a/bundle/deploy/terraform/tfdyn/convert_app.go +++ b/bundle/deploy/terraform/tfdyn/convert_app.go @@ -38,7 +38,7 @@ func (appConverter) Convert(ctx context.Context, key string, vin dyn.Value, out return err } - // We always set no_compute to true as it allows DABs not to wait for app compute to be started when app is created. + // Always skip compute during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_compute", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index e53b22a38d..3a2439014a 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -29,7 +29,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, return err } - // We always set no_wait as it allows DABs not to wait for cluster to be started. + // Always skip wait during creation; lifecycle.started is only supported in direct mode. vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) if err != nil { return err diff --git a/bundle/deploy/terraform/tfdyn/lifecycle.go b/bundle/deploy/terraform/tfdyn/lifecycle.go index 1600cdef2d..669aa83040 100644 --- a/bundle/deploy/terraform/tfdyn/lifecycle.go +++ b/bundle/deploy/terraform/tfdyn/lifecycle.go @@ -11,7 +11,19 @@ func convertLifecycle(ctx context.Context, vout, vLifecycle dyn.Value) (dyn.Valu return vout, nil } - vout, err := dyn.Set(vout, "lifecycle", vLifecycle) + // Strip lifecycle.started: it is a DABs-only field not understood by Terraform. + var err error + vLifecycle, err = dyn.DropKeys(vLifecycle, []string{"started"}) + if err != nil { + return dyn.InvalidValue, err + } + + // If only lifecycle.started was set (now empty), skip setting the lifecycle block. + if m, ok := vLifecycle.AsMap(); ok && m.Len() == 0 { + return vout, nil + } + + vout, err = dyn.Set(vout, "lifecycle", vLifecycle) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 77d98a8a1f..0fededbb9b 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -5,8 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -95,7 +98,21 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // We don't keep NewState around for 'skip' nodes if action != deployplan.Skip { - if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) { + // Propagate lifecycle.started into context so resource implementations can use it. + deployCtx := ctx + if !strings.HasSuffix(resourceKey, ".permissions") && !strings.HasSuffix(resourceKey, ".grants") { + nodePath, err := dyn.NewPathFromString(resourceKey) + if err == nil { + startedV, err := dyn.GetByPath(configRoot.Value(), append(nodePath, dyn.Key("lifecycle"), dyn.Key("started"))) + if err == nil { + if started, ok := startedV.AsBool(); ok && started { + deployCtx = dresources.WithLifecycleStarted(deployCtx) + } + } + } + } + + if !b.resolveReferences(deployCtx, resourceKey, entry, errorPrefix, false) { return false } @@ -121,7 +138,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = b.StateDB.SaveState(resourceKey, dbentry.ID, sv.Value, entry.DependsOn) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry.Changes) + err = d.Deploy(deployCtx, &b.StateDB, sv.Value, action, entry.Changes) } if err != nil { diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index e753079dac..8f6735a445 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -14,6 +15,15 @@ import ( "github.com/databricks/databricks-sdk-go/service/apps" ) +// AppState is the state type for App resources. It extends apps.App with fields +// needed for app deployments (Apps.Deploy) that are not part of the remote state. +type AppState struct { + apps.App + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -22,18 +32,33 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { return &ResourceApp{client: client} } -func (*ResourceApp) PrepareState(input *resources.App) *apps.App { - return &input.App +func (*ResourceApp) PrepareState(input *resources.App) *AppState { + return &AppState{ + App: input.App, + SourceCodePath: input.SourceCodePath, + Config: input.Config, + GitSource: input.GitSource, + } +} + +// RemapState maps the remote apps.App to AppState for diff comparison. +// Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, +// so they default to zero values, which prevents false drift detection. +func (*ResourceApp) RemapState(remote *apps.App) *AppState { + return &AppState{App: *remote} } func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { return r.client.Apps.GetByName(ctx, id) } -func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, *apps.App, error) { +func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { + // With lifecycle.started=true, start the app compute (no_compute=false). + // Otherwise, skip compute startup during creation. + noCompute := !lifecycleStartedFromContext(ctx) request := apps.CreateAppRequest{ - App: *config, - NoCompute: true, + App: config.App, + NoCompute: noCompute, ForceSendFields: nil, } @@ -68,11 +93,11 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, changes Changes) (*apps.App, error) { +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*apps.App, error) { updateMask := strings.Join(collectUpdatePathsWithPrefix(changes, ""), ",") request := apps.AsyncUpdateAppRequest{ - App: config, + App: &config.App, AppName: id, UpdateMask: updateMask, } @@ -89,15 +114,48 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) } + + // With lifecycle.started=true, ensure the app compute is running and deploy the latest code. + if lifecycleStartedFromContext(ctx) { + // Start compute if it is stopped (mirrors bundle run behavior). + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get app %s: %w", id, err) + } + if isComputeStopped(app) { + startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) + if err != nil { + return nil, fmt.Errorf("failed to start app %s: %w", id, err) + } + startedApp, err := startWaiter.Get() + if err != nil { + return nil, fmt.Errorf("failed to wait for app %s to start: %w", id, err) + } + if err := appdeploy.WaitForDeploymentToComplete(ctx, r.client, startedApp); err != nil { + return nil, err + } + } + deployment := appdeploy.BuildDeployment(config.SourceCodePath, config.Config, config.GitSource) + if err := appdeploy.Deploy(ctx, r.client, id, deployment); err != nil { + return nil, err + } + } + return nil, nil } +func isComputeStopped(app *apps.App) bool { + return app.ComputeStatus == nil || + app.ComputeStatus.State == apps.ComputeStateStopped || + app.ComputeStatus.State == apps.ComputeStateError +} + func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { _, err := r.client.Apps.DeleteByName(ctx, id) return err } -func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *apps.App) (*apps.App, error) { +func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*apps.App, error) { return r.waitForApp(ctx, r.client, config.Name) } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index e0eeed5b77..ad9ca01e8a 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -57,7 +57,7 @@ func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) @@ -113,7 +113,7 @@ func TestAppDoCreate_RetriesWhenGetReturnsNotFound(t *testing.T) { r := (&ResourceApp{}).New(client) ctx := t.Context() - name, _, err := r.DoCreate(ctx, &apps.App{Name: "test-app"}) + name, _, err := r.DoCreate(ctx, &AppState{App: apps.App{Name: "test-app"}}) require.NoError(t, err) assert.Equal(t, "test-app", name) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index a8f78d12f9..ffa876666e 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -83,6 +83,14 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterS if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the cluster to reach the running state. + if lifecycleStartedFromContext(ctx) { + details, err := wait.Get() + if err != nil { + return "", nil, err + } + return details.ClusterId, details, nil + } return wait.ClusterId, nil, nil } diff --git a/bundle/direct/dresources/lifecycle_ctx.go b/bundle/direct/dresources/lifecycle_ctx.go new file mode 100644 index 0000000000..c4a8f5d681 --- /dev/null +++ b/bundle/direct/dresources/lifecycle_ctx.go @@ -0,0 +1,16 @@ +package dresources + +import "context" + +type lifecycleStartedKeyType struct{} + +// WithLifecycleStarted returns a context with lifecycle.started set to true. +func WithLifecycleStarted(ctx context.Context) context.Context { + return context.WithValue(ctx, lifecycleStartedKeyType{}, true) +} + +// lifecycleStartedFromContext returns true if lifecycle.started is set in the context. +func lifecycleStartedFromContext(ctx context.Context) bool { + v, _ := ctx.Value(lifecycleStartedKeyType{}).(bool) + return v +} diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 5d9d7793b7..6641971e7a 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -54,6 +54,14 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateW if err != nil { return "", nil, err } + // With lifecycle.started=true, wait for the warehouse to reach the running state. + if lifecycleStartedFromContext(ctx) { + warehouse, err := waiter.Get() + if err != nil { + return "", nil, err + } + return warehouse.Id, warehouse, nil + } return waiter.Id, nil, nil } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d061d4d0da..7eb73acbd7 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,6 +16,12 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ + // source_code_path, config, and git_source are bundle-specific deployment fields not present in the remote App state. + "apps": { + "config", + "git_source", + "source_code_path", + }, "clusters": { "apply_policy_default_values", }, @@ -91,11 +97,6 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, - "apps": { - "config", - "source_code_path", - "git_source", - }, "dashboards": { "file_path", }, diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8dcf37f7e8..a7a286b39f 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -658,6 +658,9 @@ github.com/databricks/cli/bundle/config/resources.Lifecycle: "prevent_destroy": "description": |- Lifecycle setting to prevent the resource from being destroyed. + "started": + "description": |- + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. github.com/databricks/cli/bundle/config/resources.MlflowExperimentPermission: "group_name": "description": |- diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 6e43f42291..ad987ceff3 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/run/app.go b/bundle/run/app.go index 1dd4484093..c3a6497f1d 100644 --- a/bundle/run/app.go +++ b/bundle/run/app.go @@ -7,10 +7,10 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/run/output" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/spf13/cobra" ) @@ -124,7 +124,7 @@ func (a *appRunner) start(ctx context.Context) error { // active and pending deployments fields (if any). If there are active or pending deployments, // we need to wait for them to complete before we can do the new deployment. // Otherwise, the new deployment will fail. - err = waitForDeploymentToComplete(ctx, w, startedApp) + err = appdeploy.WaitForDeploymentToComplete(ctx, w, startedApp) if err != nil { return err } @@ -133,108 +133,10 @@ func (a *appRunner) start(ctx context.Context) error { return nil } -func waitForDeploymentToComplete(ctx context.Context, w *databricks.WorkspaceClient, app *apps.App) error { - // We first wait for the active deployment to complete. - if app.ActiveDeployment != nil && - app.ActiveDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the active deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.ActiveDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Active deployment is completed!") - } - - // Then, we wait for the pending deployment to complete. - if app.PendingDeployment != nil && - app.PendingDeployment.Status.State == apps.AppDeploymentStateInProgress { - logProgress(ctx, "Waiting for the pending deployment to complete...") - _, err := w.Apps.WaitGetDeploymentAppSucceeded(ctx, app.Name, app.PendingDeployment.DeploymentId, 20*time.Minute, nil) - if err != nil { - return err - } - logProgress(ctx, "Pending deployment is completed!") - } - - return nil -} - -// buildAppDeployment creates an AppDeployment struct with inline config if provided -func (a *appRunner) buildAppDeployment() apps.AppDeployment { - deployment := apps.AppDeployment{ - Mode: apps.AppDeploymentModeSnapshot, - SourceCodePath: a.app.SourceCodePath, - } - - // Add git source if provided - if a.app.GitSource != nil { - deployment.GitSource = a.app.GitSource - } - - // Add inline config if provided - if a.app.Config != nil { - if len(a.app.Config.Command) > 0 { - deployment.Command = a.app.Config.Command - } - - if len(a.app.Config.Env) > 0 { - deployment.EnvVars = make([]apps.EnvVar, len(a.app.Config.Env)) - for i, env := range a.app.Config.Env { - deployment.EnvVars[i] = apps.EnvVar{ - Name: env.Name, - Value: env.Value, - ValueFrom: env.ValueFrom, - } - } - } - } - - return deployment -} - func (a *appRunner) deploy(ctx context.Context) error { - app := a.app - b := a.bundle - w := b.WorkspaceClient() - - wait, err := w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - // If deploy returns an error, then there's an active deployment in progress, wait for it to complete. - // For this we first need to get an app and its acrive and pending deployments and then wait for them. - if err != nil { - app, err := w.Apps.Get(ctx, apps.GetAppRequest{Name: app.Name}) - if err != nil { - return fmt.Errorf("failed to get app %s: %w", app.Name, err) - } - - err = waitForDeploymentToComplete(ctx, w, app) - if err != nil { - return err - } - - // Now we can try to deploy the app again - wait, err = w.Apps.Deploy(ctx, apps.CreateAppDeploymentRequest{ - AppName: app.Name, - AppDeployment: a.buildAppDeployment(), - }) - if err != nil { - return err - } - } - - _, err = wait.OnProgress(func(ad *apps.AppDeployment) { - if ad.Status == nil { - return - } - logProgress(ctx, ad.Status.Message) - }).Get() - if err != nil { - return err - } - - return nil + w := a.bundle.WorkspaceClient() + deployment := appdeploy.BuildDeployment(a.app.SourceCodePath, a.app.Config, a.app.GitSource) + return appdeploy.Deploy(ctx, w, a.app.Name, deployment) } func (a *appRunner) Cancel(ctx context.Context) error { diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 112fcea28d..1debdc813f 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" @@ -294,11 +295,7 @@ func TestBuildAppDeploymentWithValueFrom(t *testing.T) { }, } - runner := &appRunner{ - app: app, - } - - deployment := runner.buildAppDeployment() + deployment := appdeploy.BuildDeployment(app.SourceCodePath, app.Config, app.GitSource) require.Equal(t, apps.AppDeploymentModeSnapshot, deployment.Mode) require.Equal(t, "/path/to/app", deployment.SourceCodePath) diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 8abd4cd416..ef29402eef 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -89,6 +89,79 @@ func (s *FakeWorkspace) AppsGetUpdate(_ Request, name string) Response { } } +func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + var deployment apps.AppDeployment + if err := json.Unmarshal(req.Body, &deployment); err != nil { + return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} + } + + deployment.DeploymentId = "deploy-1" + deployment.Status = &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + } + + return Response{Body: deployment} +} + +func (s *FakeWorkspace) AppsGetDeployment(_ Request, name string, deploymentID string) Response { + defer s.LockUnlock()() + + _, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + return Response{Body: apps.AppDeployment{ + DeploymentId: deploymentID, + Status: &apps.AppDeploymentStatus{ + State: apps.AppDeploymentStateSucceeded, + Message: "Deployment succeeded.", + }, + }} +} + +func (s *FakeWorkspace) AppsStart(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateActive, + Message: "App compute is active.", + } + s.Apps[name] = app + + return Response{Body: app} +} + +func (s *FakeWorkspace) AppsStop(_ Request, name string) Response { + defer s.LockUnlock()() + + app, ok := s.Apps[name] + if !ok { + return Response{StatusCode: 404} + } + + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + s.Apps[name] = app + + return Response{Body: app} +} + func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { var app apps.App diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 9e30cb5f0c..b2a95b1902 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -379,6 +379,22 @@ func AddDefaultHandlers(server *Server) { // Apps: + server.Handle("POST", "/api/2.0/apps/{name}/deployments", func(req Request) any { + return req.Workspace.AppsCreateDeployment(req, req.Vars["name"]) + }) + + server.Handle("GET", "/api/2.0/apps/{name}/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.AppsGetDeployment(req, req.Vars["name"], req.Vars["deployment_id"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/start", func(req Request) any { + return req.Workspace.AppsStart(req, req.Vars["name"]) + }) + + server.Handle("POST", "/api/2.0/apps/{name}/stop", func(req Request) any { + return req.Workspace.AppsStop(req, req.Vars["name"]) + }) + server.Handle("POST", "/api/2.0/apps/{name}/update", func(req Request) any { return req.Workspace.AppsCreateUpdate(req, req.Vars["name"]) }) From 2cbff32fd5451c6f631e2cf84bbdd7aca3f6ba09 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Mar 2026 11:08:29 +0100 Subject: [PATCH 02/42] fixed sche,a --- bundle/schema/jsonschema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index abfdb55233..2944b0072d 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -895,6 +895,10 @@ "prevent_destroy": { "description": "Lifecycle setting to prevent the resource from being destroyed.", "$ref": "#/$defs/bool" + }, + "started": { + "description": "Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode.", + "$ref": "#/$defs/bool" } }, "additionalProperties": false From 5910d1c14b29c73f51aa04f75ff885f9507a0d78 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Mar 2026 14:11:27 +0100 Subject: [PATCH 03/42] some refactoring and fixed test output --- acceptance/bundle/refschema/out.fields.txt | 61 +++++++++++++------ .../validate_lifecycle_started_test.go | 9 ++- bundle/direct/bundle_apply.go | 21 +------ bundle/direct/dresources/app.go | 33 +++++++++- bundle/direct/dresources/cluster.go | 33 +++++++--- bundle/direct/dresources/lifecycle_ctx.go | 16 ----- bundle/direct/dresources/sql_warehouse.go | 37 ++++++++--- bundle/direct/dresources/type_test.go | 7 ++- bundle/run/app_test.go | 2 +- libs/testserver/apps.go | 2 +- 10 files changed, 138 insertions(+), 83 deletions(-) delete mode 100644 bundle/direct/dresources/lifecycle_ctx.go diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d032aa48f8..b8afe67cc3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -34,6 +34,7 @@ resources.alerts.*.file_path string INPUT resources.alerts.*.id string ALL resources.alerts.*.lifecycle resources.Lifecycle INPUT resources.alerts.*.lifecycle.prevent_destroy bool INPUT +resources.alerts.*.lifecycle.started *bool INPUT resources.alerts.*.lifecycle_state sql.AlertLifecycleState ALL resources.alerts.*.modified_status string INPUT resources.alerts.*.owner_user_name string ALL @@ -93,14 +94,14 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.active_instances int ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config *resources.AppConfig INPUT -resources.apps.*.config.command []string INPUT -resources.apps.*.config.command[*] string INPUT -resources.apps.*.config.env []resources.AppEnvVar INPUT -resources.apps.*.config.env[*] resources.AppEnvVar INPUT -resources.apps.*.config.env[*].name string INPUT -resources.apps.*.config.env[*].value string INPUT -resources.apps.*.config.env[*].value_from string INPUT +resources.apps.*.config *resources.AppConfig INPUT STATE +resources.apps.*.config.command []string INPUT STATE +resources.apps.*.config.command[*] string INPUT STATE +resources.apps.*.config.env []resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*] resources.AppEnvVar INPUT STATE +resources.apps.*.config.env[*].name string INPUT STATE +resources.apps.*.config.env[*].value string INPUT STATE +resources.apps.*.config.env[*].value_from string INPUT STATE resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -112,18 +113,19 @@ resources.apps.*.effective_user_api_scopes[*] string ALL resources.apps.*.git_repository *apps.GitRepository ALL resources.apps.*.git_repository.provider string ALL resources.apps.*.git_repository.url string ALL -resources.apps.*.git_source *apps.GitSource INPUT -resources.apps.*.git_source.branch string INPUT -resources.apps.*.git_source.commit string INPUT -resources.apps.*.git_source.git_repository *apps.GitRepository INPUT -resources.apps.*.git_source.git_repository.provider string INPUT -resources.apps.*.git_source.git_repository.url string INPUT -resources.apps.*.git_source.resolved_commit string INPUT -resources.apps.*.git_source.source_code_path string INPUT -resources.apps.*.git_source.tag string INPUT +resources.apps.*.git_source *apps.GitSource INPUT STATE +resources.apps.*.git_source.branch string INPUT STATE +resources.apps.*.git_source.commit string INPUT STATE +resources.apps.*.git_source.git_repository *apps.GitRepository INPUT STATE +resources.apps.*.git_source.git_repository.provider string INPUT STATE +resources.apps.*.git_source.git_repository.url string INPUT STATE +resources.apps.*.git_source.resolved_commit string INPUT STATE +resources.apps.*.git_source.source_code_path string INPUT STATE +resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT +resources.apps.*.lifecycle.started *bool INPUT resources.apps.*.modified_status string INPUT resources.apps.*.name string ALL resources.apps.*.oauth2_app_client_id string ALL @@ -197,7 +199,7 @@ resources.apps.*.resources[*].uc_securable.securable_type apps.AppResourceUcSecu resources.apps.*.service_principal_client_id string ALL resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL -resources.apps.*.source_code_path string INPUT +resources.apps.*.source_code_path string INPUT STATE resources.apps.*.space string ALL resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL @@ -233,6 +235,7 @@ resources.catalogs.*.id string INPUT resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.catalogs.*.lifecycle.started *bool INPUT resources.catalogs.*.metastore_id string REMOTE resources.catalogs.*.modified_status string INPUT resources.catalogs.*.name string ALL @@ -377,6 +380,7 @@ resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE resources.clusters.*.lifecycle resources.Lifecycle INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT +resources.clusters.*.lifecycle.started *bool INPUT resources.clusters.*.modified_status string INPUT resources.clusters.*.node_type_id string ALL resources.clusters.*.num_workers int ALL @@ -501,6 +505,7 @@ resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE resources.clusters.*.ssh_public_keys []string ALL resources.clusters.*.ssh_public_keys[*] string ALL resources.clusters.*.start_time int64 REMOTE +resources.clusters.*.started bool STATE resources.clusters.*.state compute.State REMOTE resources.clusters.*.state_message string REMOTE resources.clusters.*.terminated_time int64 REMOTE @@ -536,6 +541,7 @@ resources.dashboards.*.file_path string INPUT resources.dashboards.*.id string INPUT resources.dashboards.*.lifecycle resources.Lifecycle INPUT resources.dashboards.*.lifecycle.prevent_destroy bool INPUT +resources.dashboards.*.lifecycle.started *bool INPUT resources.dashboards.*.lifecycle_state dashboards.LifecycleState ALL resources.dashboards.*.modified_status string INPUT resources.dashboards.*.parent_path string ALL @@ -557,6 +563,7 @@ resources.database_catalogs.*.database_name string ALL resources.database_catalogs.*.id string INPUT resources.database_catalogs.*.lifecycle resources.Lifecycle INPUT resources.database_catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.database_catalogs.*.lifecycle.started *bool INPUT resources.database_catalogs.*.modified_status string INPUT resources.database_catalogs.*.name string ALL resources.database_catalogs.*.uid string ALL @@ -591,6 +598,7 @@ resources.database_instances.*.enable_readable_secondaries bool ALL resources.database_instances.*.id string INPUT resources.database_instances.*.lifecycle resources.Lifecycle INPUT resources.database_instances.*.lifecycle.prevent_destroy bool INPUT +resources.database_instances.*.lifecycle.started *bool INPUT resources.database_instances.*.modified_status string INPUT resources.database_instances.*.name string ALL resources.database_instances.*.node_count int ALL @@ -622,6 +630,7 @@ resources.experiments.*.id string INPUT resources.experiments.*.last_update_time int64 REMOTE resources.experiments.*.lifecycle resources.Lifecycle INPUT resources.experiments.*.lifecycle.prevent_destroy bool INPUT +resources.experiments.*.lifecycle.started *bool INPUT resources.experiments.*.lifecycle_stage string REMOTE resources.experiments.*.modified_status string INPUT resources.experiments.*.name string ALL @@ -676,6 +685,7 @@ resources.external_locations.*.id string INPUT resources.external_locations.*.isolation_mode catalog.IsolationMode REMOTE resources.external_locations.*.lifecycle resources.Lifecycle INPUT resources.external_locations.*.lifecycle.prevent_destroy bool INPUT +resources.external_locations.*.lifecycle.started *bool INPUT resources.external_locations.*.metastore_id string REMOTE resources.external_locations.*.modified_status string INPUT resources.external_locations.*.name string ALL @@ -864,6 +874,7 @@ resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks boo resources.jobs.*.job_id int64 REMOTE resources.jobs.*.lifecycle resources.Lifecycle INPUT resources.jobs.*.lifecycle.prevent_destroy bool INPUT +resources.jobs.*.lifecycle.started *bool INPUT resources.jobs.*.max_concurrent_runs int ALL resources.jobs.*.modified_status string INPUT resources.jobs.*.name string ALL @@ -2059,6 +2070,7 @@ resources.model_serving_endpoints.*.endpoint_id string REMOTE resources.model_serving_endpoints.*.id string INPUT resources.model_serving_endpoints.*.lifecycle resources.Lifecycle INPUT resources.model_serving_endpoints.*.lifecycle.prevent_destroy bool INPUT +resources.model_serving_endpoints.*.lifecycle.started *bool INPUT resources.model_serving_endpoints.*.modified_status string INPUT resources.model_serving_endpoints.*.name string INPUT STATE resources.model_serving_endpoints.*.rate_limits []serving.RateLimit INPUT STATE @@ -2102,6 +2114,7 @@ resources.models.*.latest_versions[*].user_id string REMOTE resources.models.*.latest_versions[*].version string REMOTE resources.models.*.lifecycle resources.Lifecycle INPUT resources.models.*.lifecycle.prevent_destroy bool INPUT +resources.models.*.lifecycle.started *bool INPUT resources.models.*.modified_status string INPUT resources.models.*.name string ALL resources.models.*.permission_level ml.PermissionLevel REMOTE @@ -2420,6 +2433,7 @@ resources.pipelines.*.libraries[*].notebook.path string ALL resources.pipelines.*.libraries[*].whl string ALL resources.pipelines.*.lifecycle resources.Lifecycle INPUT resources.pipelines.*.lifecycle.prevent_destroy bool INPUT +resources.pipelines.*.lifecycle.started *bool INPUT resources.pipelines.*.modified_status string INPUT resources.pipelines.*.name string ALL resources.pipelines.*.notifications []pipelines.Notifications ALL @@ -2467,6 +2481,7 @@ resources.postgres_branches.*.id string INPUT resources.postgres_branches.*.is_protected bool INPUT STATE resources.postgres_branches.*.lifecycle resources.Lifecycle INPUT resources.postgres_branches.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_branches.*.lifecycle.started *bool INPUT resources.postgres_branches.*.modified_status string INPUT resources.postgres_branches.*.name string REMOTE resources.postgres_branches.*.no_expiry bool INPUT STATE @@ -2510,6 +2525,7 @@ resources.postgres_endpoints.*.group.min int INPUT STATE resources.postgres_endpoints.*.id string INPUT resources.postgres_endpoints.*.lifecycle resources.Lifecycle INPUT resources.postgres_endpoints.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_endpoints.*.lifecycle.started *bool INPUT resources.postgres_endpoints.*.modified_status string INPUT resources.postgres_endpoints.*.name string REMOTE resources.postgres_endpoints.*.no_suspension bool INPUT STATE @@ -2577,6 +2593,7 @@ resources.postgres_projects.*.initial_endpoint_spec.group.max int REMOTE resources.postgres_projects.*.initial_endpoint_spec.group.min int REMOTE resources.postgres_projects.*.lifecycle resources.Lifecycle INPUT resources.postgres_projects.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_projects.*.lifecycle.started *bool INPUT resources.postgres_projects.*.modified_status string INPUT resources.postgres_projects.*.name string REMOTE resources.postgres_projects.*.pg_version int INPUT STATE @@ -2654,6 +2671,7 @@ resources.quality_monitors.*.inference_log.timestamp_col string ALL resources.quality_monitors.*.latest_monitor_failure_msg string ALL resources.quality_monitors.*.lifecycle resources.Lifecycle INPUT resources.quality_monitors.*.lifecycle.prevent_destroy bool INPUT +resources.quality_monitors.*.lifecycle.started *bool INPUT resources.quality_monitors.*.modified_status string INPUT resources.quality_monitors.*.monitor_version int64 REMOTE resources.quality_monitors.*.notifications *catalog.MonitorNotifications ALL @@ -2698,6 +2716,7 @@ resources.registered_models.*.full_name string ALL resources.registered_models.*.id string INPUT resources.registered_models.*.lifecycle resources.Lifecycle INPUT resources.registered_models.*.lifecycle.prevent_destroy bool INPUT +resources.registered_models.*.lifecycle.started *bool INPUT resources.registered_models.*.metastore_id string ALL resources.registered_models.*.modified_status string INPUT resources.registered_models.*.name string ALL @@ -2728,6 +2747,7 @@ resources.schemas.*.full_name string REMOTE resources.schemas.*.id string INPUT resources.schemas.*.lifecycle resources.Lifecycle INPUT resources.schemas.*.lifecycle.prevent_destroy bool INPUT +resources.schemas.*.lifecycle.started *bool INPUT resources.schemas.*.metastore_id string REMOTE resources.schemas.*.modified_status string INPUT resources.schemas.*.name string ALL @@ -2757,6 +2777,7 @@ resources.secret_scopes.*.keyvault_metadata.dns_name string INPUT REMOTE resources.secret_scopes.*.keyvault_metadata.resource_id string INPUT REMOTE resources.secret_scopes.*.lifecycle resources.Lifecycle INPUT resources.secret_scopes.*.lifecycle.prevent_destroy bool INPUT +resources.secret_scopes.*.lifecycle.started *bool INPUT resources.secret_scopes.*.modified_status string INPUT resources.secret_scopes.*.name string INPUT REMOTE resources.secret_scopes.*.scope string STATE @@ -2790,6 +2811,7 @@ resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT +resources.sql_warehouses.*.lifecycle.started *bool INPUT resources.sql_warehouses.*.max_num_clusters int ALL resources.sql_warehouses.*.min_num_clusters int ALL resources.sql_warehouses.*.modified_status string INPUT @@ -2802,6 +2824,7 @@ resources.sql_warehouses.*.odbc_params.path string REMOTE resources.sql_warehouses.*.odbc_params.port int REMOTE resources.sql_warehouses.*.odbc_params.protocol string REMOTE resources.sql_warehouses.*.spot_instance_policy sql.SpotInstancePolicy ALL +resources.sql_warehouses.*.started bool STATE resources.sql_warehouses.*.state sql.State REMOTE resources.sql_warehouses.*.tags *sql.EndpointTags ALL resources.sql_warehouses.*.tags.custom_tags []sql.EndpointTagPair ALL @@ -2864,6 +2887,7 @@ resources.synced_database_tables.*.effective_logical_database_name string ALL resources.synced_database_tables.*.id string INPUT resources.synced_database_tables.*.lifecycle resources.Lifecycle INPUT resources.synced_database_tables.*.lifecycle.prevent_destroy bool INPUT +resources.synced_database_tables.*.lifecycle.started *bool INPUT resources.synced_database_tables.*.logical_database_name string ALL resources.synced_database_tables.*.modified_status string INPUT resources.synced_database_tables.*.name string ALL @@ -2895,6 +2919,7 @@ resources.volumes.*.full_name string REMOTE resources.volumes.*.id string INPUT resources.volumes.*.lifecycle resources.Lifecycle INPUT resources.volumes.*.lifecycle.prevent_destroy bool INPUT +resources.volumes.*.lifecycle.started *bool INPUT resources.volumes.*.metastore_id string REMOTE resources.volumes.*.modified_status string INPUT resources.volumes.*.name string ALL diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go index 7e3e10a619..311915892d 100644 --- a/bundle/config/mutator/validate_lifecycle_started_test.go +++ b/bundle/config/mutator/validate_lifecycle_started_test.go @@ -1,7 +1,6 @@ package mutator import ( - "context" "testing" "github.com/databricks/cli/bundle" @@ -34,7 +33,7 @@ func TestValidateLifecycleStarted_UnsupportedResource(t *testing.T) { }, } - m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) require.Error(t, m.Error()) assert.Contains(t, m.Error().Error(), "lifecycle.started is not supported for resources.jobs.my_job") } @@ -59,7 +58,7 @@ func TestValidateLifecycleStarted_SupportedResources(t *testing.T) { }, } - m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) assert.NoError(t, m.Error()) } @@ -81,7 +80,7 @@ func TestValidateLifecycleStarted_StartedFalse(t *testing.T) { }, } - m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineDirect)) + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) assert.NoError(t, m.Error()) } @@ -104,6 +103,6 @@ func TestValidateLifecycleStarted_TerraformModeIgnored(t *testing.T) { } // In TF mode, lifecycle.started is ignored — no error even for unsupported resource types. - m := bundle.Apply(context.Background(), b, ValidateLifecycleStarted(engine.EngineTerraform)) + m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineTerraform)) assert.NoError(t, m.Error()) } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 0fededbb9b..77d98a8a1f 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -5,11 +5,8 @@ import ( "encoding/json" "errors" "fmt" - "strings" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -98,21 +95,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // We don't keep NewState around for 'skip' nodes if action != deployplan.Skip { - // Propagate lifecycle.started into context so resource implementations can use it. - deployCtx := ctx - if !strings.HasSuffix(resourceKey, ".permissions") && !strings.HasSuffix(resourceKey, ".grants") { - nodePath, err := dyn.NewPathFromString(resourceKey) - if err == nil { - startedV, err := dyn.GetByPath(configRoot.Value(), append(nodePath, dyn.Key("lifecycle"), dyn.Key("started"))) - if err == nil { - if started, ok := startedV.AsBool(); ok && started { - deployCtx = dresources.WithLifecycleStarted(deployCtx) - } - } - } - } - - if !b.resolveReferences(deployCtx, resourceKey, entry, errorPrefix, false) { + if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) { return false } @@ -138,7 +121,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = b.StateDB.SaveState(resourceKey, dbentry.ID, sv.Value, entry.DependsOn) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) - err = d.Deploy(deployCtx, &b.StateDB, sv.Value, action, entry.Changes) + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry.Changes) } if err != nil { diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 8f6735a445..ff6843cb50 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -9,6 +9,8 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" @@ -22,6 +24,7 @@ type AppState struct { SourceCodePath string `json:"source_code_path,omitempty"` Config *resources.AppConfig `json:"config,omitempty"` GitSource *apps.GitSource `json:"git_source,omitempty"` + Started bool `json:"started,omitempty"` } type ResourceApp struct { @@ -33,11 +36,13 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { } func (*ResourceApp) PrepareState(input *resources.App) *AppState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started return &AppState{ App: input.App, SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, + Started: started, } } @@ -45,7 +50,13 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { // Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, // so they default to zero values, which prevents false drift detection. func (*ResourceApp) RemapState(remote *apps.App) *AppState { - return &AppState{App: *remote} + return &AppState{ + App: *remote, + SourceCodePath: "", + Config: nil, + GitSource: nil, + Started: false, + } } func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { @@ -55,7 +66,7 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { // With lifecycle.started=true, start the app compute (no_compute=false). // Otherwise, skip compute startup during creation. - noCompute := !lifecycleStartedFromContext(ctx) + noCompute := !config.Started request := apps.CreateAppRequest{ App: config.App, NoCompute: noCompute, @@ -116,7 +127,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } // With lifecycle.started=true, ensure the app compute is running and deploy the latest code. - if lifecycleStartedFromContext(ctx) { + if config.Started { // Start compute if it is stopped (mirrors bundle run behavior). app, err := r.client.Apps.GetByName(ctx, id) if err != nil { @@ -144,6 +155,22 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, return nil, nil } +// localOnlyFields are AppState fields that have no counterpart in the remote state. +// They must not appear in the App update_mask. +var localOnlyFields = map[string]bool{ + "source_code_path": true, + "config": true, + "git_source": true, + "started": true, +} + +func (*ResourceApp) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *apps.App) error { + if change.Action == deployplan.Update && localOnlyFields[p.Prefix(1).String()] { + change.Action = deployplan.Skip + } + return nil +} + func isComputeStopped(app *apps.App) bool { return app.ComputeStatus == nil || app.ComputeStatus.State == apps.ComputeStateStopped || diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index ffa876666e..2eea7d517d 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -16,6 +16,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) +type ClusterState struct { + compute.ClusterSpec + Started bool `json:"started,omitempty"` +} + type ResourceCluster struct { client *databricks.WorkspaceClient } @@ -26,11 +31,15 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { } } -func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.ClusterSpec { - return &input.ClusterSpec +func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &ClusterState{ + ClusterSpec: input.ClusterSpec, + Started: started, + } } -func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { +func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterState { spec := &compute.ClusterSpec{ ApplyPolicyDefaultValues: false, Autoscale: input.Autoscale, @@ -71,20 +80,20 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.Clu if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return spec + return &ClusterState{ClusterSpec: *spec, Started: false} } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { return r.client.Clusters.GetByClusterId(ctx, id) } -func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterSpec) (string, *compute.ClusterDetails, error) { - wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(config)) +func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (string, *compute.ClusterDetails, error) { + wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(&config.ClusterSpec)) if err != nil { return "", nil, err } // With lifecycle.started=true, wait for the cluster to reach the running state. - if lifecycleStartedFromContext(ctx) { + if config.Started { details, err := wait.Get() if err != nil { return "", nil, err @@ -94,12 +103,12 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterS return wait.ClusterId, nil, nil } -func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ Changes) (*compute.ClusterDetails, error) { +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 timeout := 15 * time.Minute _, err := retries.Poll(ctx, timeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { - wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, config)) + wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, &config.ClusterSpec)) if err == nil { return wait, nil } @@ -115,7 +124,7 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compu return nil, err } -func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { +func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState) error { _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ ClusterId: id, NumWorkers: config.NumWorkers, @@ -137,6 +146,10 @@ func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath. path := p.Prefix(1).String() switch path { + case "started": + // started is lifecycle metadata, not an actual cluster property. + change.Action = deployplan.Skip + case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 diff --git a/bundle/direct/dresources/lifecycle_ctx.go b/bundle/direct/dresources/lifecycle_ctx.go deleted file mode 100644 index c4a8f5d681..0000000000 --- a/bundle/direct/dresources/lifecycle_ctx.go +++ /dev/null @@ -1,16 +0,0 @@ -package dresources - -import "context" - -type lifecycleStartedKeyType struct{} - -// WithLifecycleStarted returns a context with lifecycle.started set to true. -func WithLifecycleStarted(ctx context.Context) context.Context { - return context.WithValue(ctx, lifecycleStartedKeyType{}, true) -} - -// lifecycleStartedFromContext returns true if lifecycle.started is set in the context. -func lifecycleStartedFromContext(ctx context.Context) bool { - v, _ := ctx.Value(lifecycleStartedKeyType{}).(bool) - return v -} diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 6641971e7a..066f63e184 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -4,12 +4,19 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/sql" ) +type SqlWarehouseState struct { + sql.CreateWarehouseRequest + Started bool `json:"started,omitempty"` +} + type ResourceSqlWarehouse struct { client *databricks.WorkspaceClient } @@ -20,12 +27,16 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq } // PrepareState converts bundle config to the SDK type. -func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.CreateWarehouseRequest { - return &input.CreateWarehouseRequest +func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { + started := input.Lifecycle.Started != nil && *input.Lifecycle.Started + return &SqlWarehouseState{ + CreateWarehouseRequest: input.CreateWarehouseRequest, + Started: started, + } } -func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest { - return &sql.CreateWarehouseRequest{ +func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { + return &SqlWarehouseState{Started: false, CreateWarehouseRequest: sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -40,7 +51,7 @@ func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sq Tags: warehouse.Tags, WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), - } + }} } // DoRead reads the warehouse by id. @@ -49,13 +60,13 @@ func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*sql.GetW } // DoCreate creates the warehouse and returns its id. -func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateWarehouseRequest) (string, *sql.GetWarehouseResponse, error) { - waiter, err := r.client.Warehouses.Create(ctx, *config) +func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehouseState) (string, *sql.GetWarehouseResponse, error) { + waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest) if err != nil { return "", nil, err } // With lifecycle.started=true, wait for the warehouse to reach the running state. - if lifecycleStartedFromContext(ctx) { + if config.Started { warehouse, err := waiter.Get() if err != nil { return "", nil, err @@ -66,7 +77,7 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateW } // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ Changes) (*sql.GetWarehouseResponse, error) { +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, _ Changes) (*sql.GetWarehouseResponse, error) { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, @@ -100,3 +111,11 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string) error { return r.client.Warehouses.DeleteById(ctx, oldID) } + +func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *sql.GetWarehouseResponse) error { + if change.Action == deployplan.Update && p.Prefix(1).String() == "started" { + // started is lifecycle metadata, not an actual warehouse property. + change.Action = deployplan.Skip + } + return nil +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 7eb73acbd7..b4436b784e 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,14 +16,16 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ - // source_code_path, config, and git_source are bundle-specific deployment fields not present in the remote App state. + // source_code_path, config, git_source, and started are bundle-specific deployment fields not present in the remote App state. "apps": { "config", "git_source", "source_code_path", + "started", }, "clusters": { "apply_policy_default_values", + "started", }, "external_locations": { "skip_validation", @@ -39,6 +41,9 @@ var knownMissingInRemoteType = map[string][]string{ "route_optimized", "tags", }, + "sql_warehouses": { + "started", + }, "quality_monitors": { "skip_builtin_dashboard", "warehouse_id", diff --git a/bundle/run/app_test.go b/bundle/run/app_test.go index 1debdc813f..1c05fa63eb 100644 --- a/bundle/run/app_test.go +++ b/bundle/run/app_test.go @@ -9,10 +9,10 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/internal/bundletest" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dyn" diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index ef29402eef..8d7ac8bfc7 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -111,7 +111,7 @@ func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response return Response{Body: deployment} } -func (s *FakeWorkspace) AppsGetDeployment(_ Request, name string, deploymentID string) Response { +func (s *FakeWorkspace) AppsGetDeployment(_ Request, name, deploymentID string) Response { defer s.LockUnlock()() _, ok := s.Apps[name] From 0c9ce97000c766e34321eaf6251f467d8829122f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 6 Mar 2026 14:28:02 +0100 Subject: [PATCH 04/42] fixed python codegene --- .../databricks/bundles/catalogs/_models/lifecycle.py | 10 ++++++++++ python/databricks/bundles/jobs/_models/lifecycle.py | 10 ++++++++++ .../databricks/bundles/pipelines/_models/lifecycle.py | 10 ++++++++++ python/databricks/bundles/schemas/_models/lifecycle.py | 10 ++++++++++ python/databricks/bundles/volumes/_models/lifecycle.py | 10 ++++++++++ 5 files changed, 50 insertions(+) diff --git a/python/databricks/bundles/catalogs/_models/lifecycle.py b/python/databricks/bundles/catalogs/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/catalogs/_models/lifecycle.py +++ b/python/databricks/bundles/catalogs/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/jobs/_models/lifecycle.py b/python/databricks/bundles/jobs/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/jobs/_models/lifecycle.py +++ b/python/databricks/bundles/jobs/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/pipelines/_models/lifecycle.py b/python/databricks/bundles/pipelines/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/pipelines/_models/lifecycle.py +++ b/python/databricks/bundles/pipelines/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/schemas/_models/lifecycle.py b/python/databricks/bundles/schemas/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/schemas/_models/lifecycle.py +++ b/python/databricks/bundles/schemas/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/volumes/_models/lifecycle.py b/python/databricks/bundles/volumes/_models/lifecycle.py index c934967f37..e6f5a8d6ad 100644 --- a/python/databricks/bundles/volumes/_models/lifecycle.py +++ b/python/databricks/bundles/volumes/_models/lifecycle.py @@ -18,6 +18,11 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] = None + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -34,5 +39,10 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ + started: VariableOrOptional[bool] + """ + Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + """ + LifecycleParam = LifecycleDict | Lifecycle From 5ea3d6a21ee8065573e6cbb171b9440e520aff72 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Mar 2026 15:47:03 +0100 Subject: [PATCH 05/42] updated tests --- .../lifecycle-started/out.deploy.direct.txt | 75 ++++++++++++ .../out.deploy.terraform.txt | 52 +++++++++ .../lifecycle-started/out.destroy.direct.txt | 9 ++ .../out.destroy.terraform.txt | 3 + .../apps/lifecycle-started/out.test.toml | 2 +- .../apps/lifecycle-started/output.txt | 96 ---------------- .../resources/apps/lifecycle-started/script | 46 +++----- .../apps/lifecycle-started/test.toml | 2 +- .../mutator/validate_lifecycle_started.go | 8 +- .../validate_lifecycle_started_test.go | 108 ------------------ 10 files changed, 159 insertions(+), 242 deletions(-) create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt delete mode 100644 bundle/config/mutator/validate_lifecycle_started_test.go diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt new file mode 100644 index 0000000000..c1ed9e6fce --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt @@ -0,0 +1,75 @@ + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt new file mode 100644 index 0000000000..68729dc774 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt @@ -0,0 +1,52 @@ + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps stop [UNIQUE_NAME] +Error: Not Found + +Exit code: 1 + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 + +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt new file mode 100644 index 0000000000..8061857d58 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt new file mode 100644 index 0000000000..16da688a34 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt @@ -0,0 +1,3 @@ + +>>> [CLI] bundle destroy --auto-approve +No active deployment found to destroy! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index 19b2c349a3..e4c769f3b4 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index 4ad8a24b60..1b50a6a148 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -1,106 +1,10 @@ -=== Deploy app with lifecycle.started=true ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Verify create request: no_compute must not be set ->>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps", - "body": { - "description": "my_app_description", - "name": "[UNIQUE_NAME]" - } -} - -=== Check app compute state: must be ACTIVE ->>> [CLI] apps get [UNIQUE_NAME] -"ACTIVE" - -=== Update description and re-deploy >>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -✓ Deployment succeeded. -Updating deployment state... -Deployment complete! - -=== Verify POST /deployments was called after update ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", - "body": { - "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" - } -} - -=== Stop the app ->>> [CLI] apps stop [UNIQUE_NAME] -"STOPPED" - -=== Change lifecycle.started to false and update >>> update_file.py databricks.yml started: true started: false >>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - -=== Verify POST /deployments is NOT called (lifecycle.started=false) ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt - -=== Check app compute state: must still be STOPPED ->>> [CLI] apps get [UNIQUE_NAME] -"STOPPED" - -=== Switch lifecycle.started back to true and update >>> update_file.py databricks.yml started: false started: true >>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -✓ Deployment succeeded. -Updating deployment state... -Deployment complete! - -=== Verify POST /start and POST /deployments were called ->>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/start", - "body": {} -} -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", - "body": { - "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" - } -} - -=== Check app compute state: must be ACTIVE again ->>> [CLI] apps get [UNIQUE_NAME] -"ACTIVE" - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.apps.myapp - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script index 2b9f234a4d..5bc75f99f9 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/script +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -1,52 +1,32 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { - trace $CLI bundle destroy --auto-approve + trace $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt rm -f out.requests.txt } trap cleanup EXIT -title "Deploy app with lifecycle.started=true" -trace $CLI bundle deploy - -title "Verify create request: no_compute must not be set" -trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt +{ trace errcode $CLI bundle deploy; } &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +{ trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true -title "Check app compute state: must be ACTIVE" -trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' - -title "Update description and re-deploy" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION -trace $CLI bundle deploy - -title "Verify POST /deployments was called after update" -trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm -f out.requests.txt +{ trace errcode $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true -title "Stop the app" -trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' - -title "Change lifecycle.started to false and update" trace update_file.py databricks.yml "started: true" "started: false" trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 -trace $CLI bundle deploy - -title "Verify POST /deployments is NOT called (lifecycle.started=false)" -trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm -f out.requests.txt +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true -title "Check app compute state: must still be STOPPED" -trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' - -title "Switch lifecycle.started back to true and update" trace update_file.py databricks.yml "started: false" "started: true" trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 -trace $CLI bundle deploy - -title "Verify POST /start and POST /deployments were called" -trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt +{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +{ trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm -f out.requests.txt - -title "Check app compute state: must be ACTIVE again" -trace $CLI apps get $UNIQUE_NAME | jq '.compute_status.state' +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml index bfe2b2f2a7..79997a76d4 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -5,4 +5,4 @@ RecordRequests = true Ignore = [".databricks", "databricks.yml"] [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index 4a02b965d2..f1c40f0531 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -33,13 +33,15 @@ func (m *validateLifecycleStarted) Name() string { } func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics // lifecycle.started is a direct-mode-only feature; ignore it in other modes. if !m.engine.IsDirect() { - return nil + return diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", + }) } - var diags diag.Diagnostics - _, err := dyn.MapByPattern( b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go deleted file mode 100644 index 311915892d..0000000000 --- a/bundle/config/mutator/validate_lifecycle_started_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package mutator - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/databricks/databricks-sdk-go/service/sql" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func boolPtr(b bool) *bool { return &b } - -func TestValidateLifecycleStarted_UnsupportedResource(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "my_job": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - Started: boolPtr(true), - }, - }, - JobSettings: jobs.JobSettings{Name: "my_job"}, - }, - }, - }, - }, - } - - m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) - require.Error(t, m.Error()) - assert.Contains(t, m.Error().Error(), "lifecycle.started is not supported for resources.jobs.my_job") -} - -func TestValidateLifecycleStarted_SupportedResources(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - SqlWarehouses: map[string]*resources.SqlWarehouse{ - "my_warehouse": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - Started: boolPtr(true), - }, - }, - CreateWarehouseRequest: sql.CreateWarehouseRequest{ - Name: "my_warehouse", - }, - }, - }, - }, - }, - } - - m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) - assert.NoError(t, m.Error()) -} - -func TestValidateLifecycleStarted_StartedFalse(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "my_job": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - Started: boolPtr(false), - }, - }, - JobSettings: jobs.JobSettings{Name: "my_job"}, - }, - }, - }, - }, - } - - m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineDirect)) - assert.NoError(t, m.Error()) -} - -func TestValidateLifecycleStarted_TerraformModeIgnored(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "my_job": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - Started: boolPtr(true), - }, - }, - JobSettings: jobs.JobSettings{Name: "my_job"}, - }, - }, - }, - }, - } - - // In TF mode, lifecycle.started is ignored — no error even for unsupported resource types. - m := bundle.Apply(t.Context(), b, ValidateLifecycleStarted(engine.EngineTerraform)) - assert.NoError(t, m.Error()) -} From 4a61282d840a253c2a537a9b240448874e0ce270 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Mar 2026 16:18:44 +0100 Subject: [PATCH 06/42] fixed lifecycle check --- .../out.deploy.terraform.txt | 19 +++++++-------- .../out.destroy.terraform.txt | 8 ++++++- .../mutator/validate_lifecycle_started.go | 24 +++++++++++-------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt index 68729dc774..cdc924fe7c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt @@ -1,6 +1,7 @@ >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -14,6 +15,7 @@ Exit code: 1 >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -26,20 +28,19 @@ Error: Not Found Exit code: 1 >>> errcode [CLI] bundle deploy -Error: lifecycle.started is only supported in direct deployment mode - - -Exit code: 1 +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! >>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt >>> errcode [CLI] apps get [UNIQUE_NAME] -Error: Resource apps.App not found: [UNIQUE_NAME] - -Exit code: 1 +"ACTIVE" >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -47,6 +48,4 @@ Exit code: 1 >>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt >>> errcode [CLI] apps get [UNIQUE_NAME] -Error: Resource apps.App not found: [UNIQUE_NAME] - -Exit code: 1 +"ACTIVE" diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt index 16da688a34..8061857d58 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt @@ -1,3 +1,9 @@ >>> [CLI] bundle destroy --auto-approve -No active deployment found to destroy! +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index f1c40f0531..e432d8053c 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -34,22 +34,12 @@ func (m *validateLifecycleStarted) Name() string { func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - // lifecycle.started is a direct-mode-only feature; ignore it in other modes. - if !m.engine.IsDirect() { - return diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.started is only supported in direct deployment mode", - }) - } _, err := dyn.MapByPattern( b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(path dyn.Path, v dyn.Value) (dyn.Value, error) { resourceType := path[1].Key() - if supportedForLifecycleStarted[resourceType] { - return v, nil - } startedV, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("lifecycle"), dyn.Key("started"))) if err != nil { @@ -61,6 +51,20 @@ func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) di return v, nil } + // lifecycle.started is a direct-mode-only feature; + if !m.engine.IsDirect() { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", + Locations: []dyn.Location{startedV.Location()}, + }) + return v, nil + } + + if supportedForLifecycleStarted[resourceType] { + return v, nil + } + resourceKey := path.String() diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, From 7fc5b467f2d67df718cefd9b1d45c79201626e0c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Mar 2026 17:20:29 +0100 Subject: [PATCH 07/42] introduce lifecyclewithstarted --- acceptance/bundle/refschema/out.fields.txt | 23 +--- .../mutator/validate_lifecycle_started.go | 55 ++------ .../validate_lifecycle_started_test.go | 115 ++++++++++++++++ bundle/config/resources.go | 3 + bundle/config/resources/apps.go | 8 ++ bundle/config/resources/base.go | 5 + bundle/config/resources/clusters.go | 8 ++ bundle/config/resources/external_location.go | 5 + bundle/config/resources/lifecycle.go | 25 +++- bundle/config/resources/sql_warehouses.go | 8 ++ bundle/internal/schema/annotations.yml | 4 + bundle/phases/plan.go | 33 ++--- bundle/phases/plan_test.go | 128 ++++++++++-------- bundle/schema/jsonschema.json | 24 +++- .../bundles/catalogs/_models/lifecycle.py | 10 -- .../bundles/jobs/_models/lifecycle.py | 10 -- .../bundles/pipelines/_models/lifecycle.py | 10 -- .../bundles/schemas/_models/lifecycle.py | 10 -- .../bundles/volumes/_models/lifecycle.py | 10 -- 19 files changed, 301 insertions(+), 193 deletions(-) create mode 100644 bundle/config/mutator/validate_lifecycle_started_test.go diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index b8afe67cc3..39cae2c526 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -34,7 +34,6 @@ resources.alerts.*.file_path string INPUT resources.alerts.*.id string ALL resources.alerts.*.lifecycle resources.Lifecycle INPUT resources.alerts.*.lifecycle.prevent_destroy bool INPUT -resources.alerts.*.lifecycle.started *bool INPUT resources.alerts.*.lifecycle_state sql.AlertLifecycleState ALL resources.alerts.*.modified_status string INPUT resources.alerts.*.owner_user_name string ALL @@ -124,6 +123,7 @@ resources.apps.*.git_source.source_code_path string INPUT STATE resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL resources.apps.*.lifecycle resources.Lifecycle INPUT +resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT resources.apps.*.lifecycle.started *bool INPUT resources.apps.*.modified_status string INPUT @@ -235,7 +235,6 @@ resources.catalogs.*.id string INPUT resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.prevent_destroy bool INPUT -resources.catalogs.*.lifecycle.started *bool INPUT resources.catalogs.*.metastore_id string REMOTE resources.catalogs.*.modified_status string INPUT resources.catalogs.*.name string ALL @@ -379,6 +378,7 @@ resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE resources.clusters.*.lifecycle resources.Lifecycle INPUT +resources.clusters.*.lifecycle resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT resources.clusters.*.lifecycle.started *bool INPUT resources.clusters.*.modified_status string INPUT @@ -541,7 +541,6 @@ resources.dashboards.*.file_path string INPUT resources.dashboards.*.id string INPUT resources.dashboards.*.lifecycle resources.Lifecycle INPUT resources.dashboards.*.lifecycle.prevent_destroy bool INPUT -resources.dashboards.*.lifecycle.started *bool INPUT resources.dashboards.*.lifecycle_state dashboards.LifecycleState ALL resources.dashboards.*.modified_status string INPUT resources.dashboards.*.parent_path string ALL @@ -563,7 +562,6 @@ resources.database_catalogs.*.database_name string ALL resources.database_catalogs.*.id string INPUT resources.database_catalogs.*.lifecycle resources.Lifecycle INPUT resources.database_catalogs.*.lifecycle.prevent_destroy bool INPUT -resources.database_catalogs.*.lifecycle.started *bool INPUT resources.database_catalogs.*.modified_status string INPUT resources.database_catalogs.*.name string ALL resources.database_catalogs.*.uid string ALL @@ -598,7 +596,6 @@ resources.database_instances.*.enable_readable_secondaries bool ALL resources.database_instances.*.id string INPUT resources.database_instances.*.lifecycle resources.Lifecycle INPUT resources.database_instances.*.lifecycle.prevent_destroy bool INPUT -resources.database_instances.*.lifecycle.started *bool INPUT resources.database_instances.*.modified_status string INPUT resources.database_instances.*.name string ALL resources.database_instances.*.node_count int ALL @@ -630,7 +627,6 @@ resources.experiments.*.id string INPUT resources.experiments.*.last_update_time int64 REMOTE resources.experiments.*.lifecycle resources.Lifecycle INPUT resources.experiments.*.lifecycle.prevent_destroy bool INPUT -resources.experiments.*.lifecycle.started *bool INPUT resources.experiments.*.lifecycle_stage string REMOTE resources.experiments.*.modified_status string INPUT resources.experiments.*.name string ALL @@ -685,7 +681,6 @@ resources.external_locations.*.id string INPUT resources.external_locations.*.isolation_mode catalog.IsolationMode REMOTE resources.external_locations.*.lifecycle resources.Lifecycle INPUT resources.external_locations.*.lifecycle.prevent_destroy bool INPUT -resources.external_locations.*.lifecycle.started *bool INPUT resources.external_locations.*.metastore_id string REMOTE resources.external_locations.*.modified_status string INPUT resources.external_locations.*.name string ALL @@ -874,7 +869,6 @@ resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks boo resources.jobs.*.job_id int64 REMOTE resources.jobs.*.lifecycle resources.Lifecycle INPUT resources.jobs.*.lifecycle.prevent_destroy bool INPUT -resources.jobs.*.lifecycle.started *bool INPUT resources.jobs.*.max_concurrent_runs int ALL resources.jobs.*.modified_status string INPUT resources.jobs.*.name string ALL @@ -2070,7 +2064,6 @@ resources.model_serving_endpoints.*.endpoint_id string REMOTE resources.model_serving_endpoints.*.id string INPUT resources.model_serving_endpoints.*.lifecycle resources.Lifecycle INPUT resources.model_serving_endpoints.*.lifecycle.prevent_destroy bool INPUT -resources.model_serving_endpoints.*.lifecycle.started *bool INPUT resources.model_serving_endpoints.*.modified_status string INPUT resources.model_serving_endpoints.*.name string INPUT STATE resources.model_serving_endpoints.*.rate_limits []serving.RateLimit INPUT STATE @@ -2114,7 +2107,6 @@ resources.models.*.latest_versions[*].user_id string REMOTE resources.models.*.latest_versions[*].version string REMOTE resources.models.*.lifecycle resources.Lifecycle INPUT resources.models.*.lifecycle.prevent_destroy bool INPUT -resources.models.*.lifecycle.started *bool INPUT resources.models.*.modified_status string INPUT resources.models.*.name string ALL resources.models.*.permission_level ml.PermissionLevel REMOTE @@ -2433,7 +2425,6 @@ resources.pipelines.*.libraries[*].notebook.path string ALL resources.pipelines.*.libraries[*].whl string ALL resources.pipelines.*.lifecycle resources.Lifecycle INPUT resources.pipelines.*.lifecycle.prevent_destroy bool INPUT -resources.pipelines.*.lifecycle.started *bool INPUT resources.pipelines.*.modified_status string INPUT resources.pipelines.*.name string ALL resources.pipelines.*.notifications []pipelines.Notifications ALL @@ -2481,7 +2472,6 @@ resources.postgres_branches.*.id string INPUT resources.postgres_branches.*.is_protected bool INPUT STATE resources.postgres_branches.*.lifecycle resources.Lifecycle INPUT resources.postgres_branches.*.lifecycle.prevent_destroy bool INPUT -resources.postgres_branches.*.lifecycle.started *bool INPUT resources.postgres_branches.*.modified_status string INPUT resources.postgres_branches.*.name string REMOTE resources.postgres_branches.*.no_expiry bool INPUT STATE @@ -2525,7 +2515,6 @@ resources.postgres_endpoints.*.group.min int INPUT STATE resources.postgres_endpoints.*.id string INPUT resources.postgres_endpoints.*.lifecycle resources.Lifecycle INPUT resources.postgres_endpoints.*.lifecycle.prevent_destroy bool INPUT -resources.postgres_endpoints.*.lifecycle.started *bool INPUT resources.postgres_endpoints.*.modified_status string INPUT resources.postgres_endpoints.*.name string REMOTE resources.postgres_endpoints.*.no_suspension bool INPUT STATE @@ -2593,7 +2582,6 @@ resources.postgres_projects.*.initial_endpoint_spec.group.max int REMOTE resources.postgres_projects.*.initial_endpoint_spec.group.min int REMOTE resources.postgres_projects.*.lifecycle resources.Lifecycle INPUT resources.postgres_projects.*.lifecycle.prevent_destroy bool INPUT -resources.postgres_projects.*.lifecycle.started *bool INPUT resources.postgres_projects.*.modified_status string INPUT resources.postgres_projects.*.name string REMOTE resources.postgres_projects.*.pg_version int INPUT STATE @@ -2671,7 +2659,6 @@ resources.quality_monitors.*.inference_log.timestamp_col string ALL resources.quality_monitors.*.latest_monitor_failure_msg string ALL resources.quality_monitors.*.lifecycle resources.Lifecycle INPUT resources.quality_monitors.*.lifecycle.prevent_destroy bool INPUT -resources.quality_monitors.*.lifecycle.started *bool INPUT resources.quality_monitors.*.modified_status string INPUT resources.quality_monitors.*.monitor_version int64 REMOTE resources.quality_monitors.*.notifications *catalog.MonitorNotifications ALL @@ -2716,7 +2703,6 @@ resources.registered_models.*.full_name string ALL resources.registered_models.*.id string INPUT resources.registered_models.*.lifecycle resources.Lifecycle INPUT resources.registered_models.*.lifecycle.prevent_destroy bool INPUT -resources.registered_models.*.lifecycle.started *bool INPUT resources.registered_models.*.metastore_id string ALL resources.registered_models.*.modified_status string INPUT resources.registered_models.*.name string ALL @@ -2747,7 +2733,6 @@ resources.schemas.*.full_name string REMOTE resources.schemas.*.id string INPUT resources.schemas.*.lifecycle resources.Lifecycle INPUT resources.schemas.*.lifecycle.prevent_destroy bool INPUT -resources.schemas.*.lifecycle.started *bool INPUT resources.schemas.*.metastore_id string REMOTE resources.schemas.*.modified_status string INPUT resources.schemas.*.name string ALL @@ -2777,7 +2762,6 @@ resources.secret_scopes.*.keyvault_metadata.dns_name string INPUT REMOTE resources.secret_scopes.*.keyvault_metadata.resource_id string INPUT REMOTE resources.secret_scopes.*.lifecycle resources.Lifecycle INPUT resources.secret_scopes.*.lifecycle.prevent_destroy bool INPUT -resources.secret_scopes.*.lifecycle.started *bool INPUT resources.secret_scopes.*.modified_status string INPUT resources.secret_scopes.*.name string INPUT REMOTE resources.secret_scopes.*.scope string STATE @@ -2810,6 +2794,7 @@ resources.sql_warehouses.*.id string INPUT REMOTE resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT +resources.sql_warehouses.*.lifecycle resources.LifecycleWithStarted INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT resources.sql_warehouses.*.lifecycle.started *bool INPUT resources.sql_warehouses.*.max_num_clusters int ALL @@ -2887,7 +2872,6 @@ resources.synced_database_tables.*.effective_logical_database_name string ALL resources.synced_database_tables.*.id string INPUT resources.synced_database_tables.*.lifecycle resources.Lifecycle INPUT resources.synced_database_tables.*.lifecycle.prevent_destroy bool INPUT -resources.synced_database_tables.*.lifecycle.started *bool INPUT resources.synced_database_tables.*.logical_database_name string ALL resources.synced_database_tables.*.modified_status string INPUT resources.synced_database_tables.*.name string ALL @@ -2919,7 +2903,6 @@ resources.volumes.*.full_name string REMOTE resources.volumes.*.id string INPUT resources.volumes.*.lifecycle resources.Lifecycle INPUT resources.volumes.*.lifecycle.prevent_destroy bool INPUT -resources.volumes.*.lifecycle.started *bool INPUT resources.volumes.*.metastore_id string REMOTE resources.volumes.*.modified_status string INPUT resources.volumes.*.name string ALL diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index e432d8053c..f2b3ef317c 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -2,21 +2,13 @@ package mutator import ( "context" - "fmt" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" ) -// supportedForLifecycleStarted lists resource types that support lifecycle.started. -var supportedForLifecycleStarted = map[string]bool{ - "apps": true, - "clusters": true, - "sql_warehouses": true, -} - type validateLifecycleStarted struct { engine engine.EngineType } @@ -35,48 +27,21 @@ func (m *validateLifecycleStarted) Name() string { func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics - _, err := dyn.MapByPattern( - b.Config.Value(), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - func(path dyn.Path, v dyn.Value) (dyn.Value, error) { - resourceType := path[1].Key() - - startedV, err := dyn.GetByPath(v, dyn.NewPath(dyn.Key("lifecycle"), dyn.Key("started"))) - if err != nil { - return v, nil - } - - started, ok := startedV.AsBool() - if !ok || !started { - return v, nil + for _, group := range b.Config.Resources.AllResources() { + for _, resource := range group.Resources { + lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) + if !ok || lws.Started == nil || !*lws.Started { + continue } - // lifecycle.started is a direct-mode-only feature; + // lifecycle.started is a direct-mode-only feature. if !m.engine.IsDirect() { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.started is only supported in direct deployment mode", - Locations: []dyn.Location{startedV.Location()}, + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", }) - return v, nil } - - if supportedForLifecycleStarted[resourceType] { - return v, nil - } - - resourceKey := path.String() - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.started is not supported for %s; it is only supported for apps, clusters, and sql_warehouses", resourceKey), - Locations: []dyn.Location{startedV.Location()}, - }) - - return v, nil - }, - ) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + } } return diags diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go new file mode 100644 index 0000000000..0c9df6a6ce --- /dev/null +++ b/bundle/config/mutator/validate_lifecycle_started_test.go @@ -0,0 +1,115 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func boolPtr(v bool) *bool { + return &v +} + +func TestValidateLifecycleStartedDirectEngine(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "my_app": { + Lifecycle: resources.LifecycleWithStarted{ + Started: boolPtr(true), + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineDirect)) + require.NoError(t, diags.Error()) +} + +func TestValidateLifecycleStartedTerraformEngine(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "my_app": { + Lifecycle: resources.LifecycleWithStarted{ + Started: boolPtr(true), + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) + require.Error(t, diags.Error()) + assert.Contains(t, diags.Error().Error(), "lifecycle.started is only supported in direct deployment mode") +} + +func TestValidateLifecycleStartedFalse(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Clusters: map[string]*resources.Cluster{ + "my_cluster": { + Lifecycle: resources.LifecycleWithStarted{ + Started: boolPtr(false), + }, + }, + }, + }, + }, + } + + // started=false should not produce an error even with terraform engine + diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) + require.NoError(t, diags.Error()) +} + +func TestValidateLifecycleStartedNotSet(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + SqlWarehouses: map[string]*resources.SqlWarehouse{ + "my_warehouse": { + Lifecycle: resources.LifecycleWithStarted{}, + }, + }, + }, + }, + } + + // started not set should not produce an error + diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) + require.NoError(t, diags.Error()) +} + +func TestValidateLifecycleStartedJobNotSupported(t *testing.T) { + // Jobs don't have LifecycleWithStarted, so lifecycle.started is not accessible. + // This test verifies that resources without LifecycleWithStarted don't trigger errors. + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my_job": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{PreventDestroy: false}, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) + require.NoError(t, diags.Error()) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index aaf4687a84..977c1b04ea 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -53,6 +53,9 @@ type ConfigResource interface { // InitializeURL initializes the URL field of the resource. InitializeURL(baseURL url.URL) + + // GetLifecycle returns the lifecycle settings for the resource. + GetLifecycle() resources.ILifecycle } // ResourceGroup represents a group of resources of the same type. diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index e48f6e7dae..26faa7d3a5 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -37,6 +37,9 @@ type App struct { apps.App // nolint App struct also defines Id and URL field with the same json tag "id" and "url" // Note: apps.App already includes GitRepository field from the SDK + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + // SourceCodePath is a required field used by DABs to point to Databricks app source code // on local disk and to the corresponding workspace path during app deployment. SourceCodePath string `json:"source_code_path,omitempty"` @@ -53,6 +56,11 @@ type App struct { Permissions []AppPermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (a *App) GetLifecycle() ILifecycle { + return a.Lifecycle +} + func (a *App) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, a) } diff --git a/bundle/config/resources/base.go b/bundle/config/resources/base.go index 792db28972..5af311304b 100644 --- a/bundle/config/resources/base.go +++ b/bundle/config/resources/base.go @@ -7,3 +7,8 @@ type BaseResource struct { URL string `json:"url,omitempty" bundle:"internal"` Lifecycle Lifecycle `json:"lifecycle,omitempty"` } + +// GetLifecycle returns the lifecycle settings for the resource. +func (b *BaseResource) GetLifecycle() ILifecycle { + return b.Lifecycle +} diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index c549ac4a6b..1b1eb1a323 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -14,9 +14,17 @@ type Cluster struct { BaseResource compute.ClusterSpec + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + Permissions []ClusterPermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (s *Cluster) GetLifecycle() ILifecycle { + return s.Lifecycle +} + func (s *Cluster) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) } diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go index 29cb5469d2..36b4e37db7 100644 --- a/bundle/config/resources/external_location.go +++ b/bundle/config/resources/external_location.go @@ -62,6 +62,11 @@ func (e *ExternalLocation) GetName() string { return e.Name } +// GetLifecycle returns the lifecycle settings for the resource. +func (e *ExternalLocation) GetLifecycle() ILifecycle { + return e.Lifecycle +} + func (e *ExternalLocation) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, e) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 2ab74b4e30..9805199c74 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -1,12 +1,33 @@ package resources -// Lifecycle is a struct that contains the lifecycle settings for a resource. -// It controls the behavior of the resource when it is deployed or destroyed. +// ILifecycle is implemented by Lifecycle and LifecycleWithStarted. +type ILifecycle interface { + HasPreventDestroy() bool +} + +// Lifecycle contains base lifecycle settings supported by all resources. type Lifecycle struct { // Lifecycle setting to prevent the resource from being destroyed. PreventDestroy bool `json:"prevent_destroy,omitempty"` +} + +// HasPreventDestroy returns true if prevent_destroy is set. +func (l Lifecycle) HasPreventDestroy() bool { + return l.PreventDestroy +} + +// LifecycleWithStarted contains lifecycle settings for resources that support lifecycle.started. +// It is used by apps, clusters, and sql_warehouses. +type LifecycleWithStarted struct { + // Lifecycle setting to prevent the resource from being destroyed. + PreventDestroy bool `json:"prevent_destroy,omitempty"` // If set to true, the resource will be deployed in started mode. // Supported only for apps, clusters, and sql_warehouses. Started *bool `json:"started,omitempty"` } + +// HasPreventDestroy returns true if prevent_destroy is set. +func (l LifecycleWithStarted) HasPreventDestroy() bool { + return l.PreventDestroy +} diff --git a/bundle/config/resources/sql_warehouses.go b/bundle/config/resources/sql_warehouses.go index bed567b805..0251ac0532 100644 --- a/bundle/config/resources/sql_warehouses.go +++ b/bundle/config/resources/sql_warehouses.go @@ -14,9 +14,17 @@ type SqlWarehouse struct { BaseResource sql.CreateWarehouseRequest + // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. + Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + Permissions []SqlWarehousePermission `json:"permissions,omitempty"` } +// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. +func (sw *SqlWarehouse) GetLifecycle() ILifecycle { + return sw.Lifecycle +} + func (sw *SqlWarehouse) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, sw) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a7a286b39f..548e0d3dcb 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -658,6 +658,10 @@ github.com/databricks/cli/bundle/config/resources.Lifecycle: "prevent_destroy": "description": |- Lifecycle setting to prevent the resource from being destroyed. +github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted: + "prevent_destroy": + "description": |- + Lifecycle setting to prevent the resource from being destroyed. "started": "description": |- Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index ad987ceff3..fd692c9614 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" @@ -13,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt" - "github.com/databricks/cli/libs/dyn" ) // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". @@ -33,32 +33,27 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { - root := b.Config.Value() var errs []error for _, action := range actions { if action.ActionType != deployplan.Recreate && action.ActionType != deployplan.Delete { continue } - path, err := dyn.NewPathFromString(action.ResourceKey) - if err != nil { - return fmt.Errorf("failed to parse %q", action.ResourceKey) - } - - path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) - - // If there is no prevent_destroy, skip - preventDestroyV, err := dyn.GetByPath(root, path) - if err != nil { + // ResourceKey format: "resources.{type}.{key}" + parts := strings.SplitN(action.ResourceKey, ".", 3) + if len(parts) != 3 || parts[0] != "resources" { continue } - - preventDestroy, ok := preventDestroyV.AsBool() - if !ok { - return fmt.Errorf("internal error: prevent_destroy is not a boolean for %s", action.ResourceKey) - } - if preventDestroy { - errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) + resourceType, resourceKey := parts[1], parts[2] + + for _, group := range b.Config.Resources.AllResources() { + if group.Description.PluralName != resourceType { + continue + } + if r, ok := group.Resources[resourceKey]; ok && r.GetLifecycle().HasPreventDestroy() { + errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) + } + break } } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 28b19d0e1e..3873f19f8c 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -1,65 +1,94 @@ package phases import ( - "context" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) -func TestCheckPreventDestroyForAllResources(t *testing.T) { - supportedResources := config.SupportedResources() +func TestCheckPreventDestroyForJob(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_resource": { + BaseResource: resources.BaseResource{ + Lifecycle: resources.Lifecycle{PreventDestroy: true}, + }, + JobSettings: jobs.JobSettings{}, + }, + }, + }, + }, + } + + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, + }, + } + + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.jobs.test_resource has lifecycle.prevent_destroy set") + require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") + require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources.jobs.test_resource") +} - for resourceType := range supportedResources { - t.Run(resourceType, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", +func TestCheckPreventDestroyForApp(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Apps: map[string]*resources.App{ + "test_resource": { + Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, }, - Resources: config.Resources{}, }, - } + }, + }, + } + + actions := []deployplan.Action{ + { + ResourceKey: "resources.apps.test_resource", + ActionType: deployplan.Delete, + }, + } - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - // Use Mutate to set the configuration dynamically - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - // Set the resource with lifecycle.prevent_destroy = true - return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ - resourceType: dyn.NewValue(map[string]dyn.Value{ - "test_resource": dyn.NewValue(map[string]dyn.Value{ - "lifecycle": dyn.NewValue(map[string]dyn.Value{ - "prevent_destroy": dyn.NewValue(true, nil), - }, nil), - }, nil), - }, nil), - }, nil)) - }) - require.NoError(t, err) - }) + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_resource has lifecycle.prevent_destroy set") +} - actions := []deployplan.Action{ - { - ResourceKey: "resources." + resourceType + ".test_resource", - ActionType: deployplan.Recreate, +func TestCheckPreventDestroyNoError(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "test_resource": { + JobSettings: jobs.JobSettings{}, + }, }, - } + }, + }, + } - err := checkForPreventDestroy(b, actions) - require.Error(t, err) - require.Contains(t, err.Error(), "resources."+resourceType+".test_resource has lifecycle.prevent_destroy set") - require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") - require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources."+resourceType+".test_resource") - }) + actions := []deployplan.Action{ + { + ResourceKey: "resources.jobs.test_resource", + ActionType: deployplan.Recreate, + }, } + + err := checkForPreventDestroy(b, actions) + require.NoError(t, err) } func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { @@ -71,9 +100,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { Resources: config.Resources{ Jobs: map[string]*resources.Job{ "test_job": { - JobSettings: jobs.JobSettings{ - Tasks: []jobs.Task{}, - }, + JobSettings: jobs.JobSettings{}, }, }, Apps: map[string]*resources.App{ @@ -81,11 +108,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { App: apps.App{ Name: "Test App", }, - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{ - PreventDestroy: true, - }, - }, + Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, }, }, }, @@ -103,10 +126,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { }, } - ctx := t.Context() - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { - err := checkForPreventDestroy(b, actions) - require.Error(t, err) - require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") - }) + err := checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 2944b0072d..32c8f44518 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -166,7 +166,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "name": { "description": "The name of the app. The name must contain only lowercase alphanumeric characters and hyphens.\nIt must be unique within the workspace.", @@ -416,7 +416,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "node_type_id": { "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", @@ -888,6 +888,24 @@ ] }, "resources.Lifecycle": { + "oneOf": [ + { + "type": "object", + "properties": { + "prevent_destroy": { + "description": "Lifecycle setting to prevent the resource from being destroyed.", + "$ref": "#/$defs/bool" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "resources.LifecycleWithStarted": { "oneOf": [ { "type": "object", @@ -1810,7 +1828,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" }, "max_num_clusters": { "description": "Maximum number of clusters that the autoscaler will create to handle\nconcurrent queries.\n\nSupported values:\n- Must be \u003e= min_num_clusters\n- Must be \u003c= 40.\n\nDefaults to min_clusters if unset.", diff --git a/python/databricks/bundles/catalogs/_models/lifecycle.py b/python/databricks/bundles/catalogs/_models/lifecycle.py index e6f5a8d6ad..c934967f37 100644 --- a/python/databricks/bundles/catalogs/_models/lifecycle.py +++ b/python/databricks/bundles/catalogs/_models/lifecycle.py @@ -18,11 +18,6 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] = None - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -39,10 +34,5 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/jobs/_models/lifecycle.py b/python/databricks/bundles/jobs/_models/lifecycle.py index e6f5a8d6ad..c934967f37 100644 --- a/python/databricks/bundles/jobs/_models/lifecycle.py +++ b/python/databricks/bundles/jobs/_models/lifecycle.py @@ -18,11 +18,6 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] = None - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -39,10 +34,5 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/pipelines/_models/lifecycle.py b/python/databricks/bundles/pipelines/_models/lifecycle.py index e6f5a8d6ad..c934967f37 100644 --- a/python/databricks/bundles/pipelines/_models/lifecycle.py +++ b/python/databricks/bundles/pipelines/_models/lifecycle.py @@ -18,11 +18,6 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] = None - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -39,10 +34,5 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/schemas/_models/lifecycle.py b/python/databricks/bundles/schemas/_models/lifecycle.py index e6f5a8d6ad..c934967f37 100644 --- a/python/databricks/bundles/schemas/_models/lifecycle.py +++ b/python/databricks/bundles/schemas/_models/lifecycle.py @@ -18,11 +18,6 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] = None - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -39,10 +34,5 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/volumes/_models/lifecycle.py b/python/databricks/bundles/volumes/_models/lifecycle.py index e6f5a8d6ad..c934967f37 100644 --- a/python/databricks/bundles/volumes/_models/lifecycle.py +++ b/python/databricks/bundles/volumes/_models/lifecycle.py @@ -18,11 +18,6 @@ class Lifecycle: Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] = None - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - @classmethod def from_dict(cls, value: "LifecycleDict") -> "Self": return _transform(cls, value) @@ -39,10 +34,5 @@ class LifecycleDict(TypedDict, total=False): Lifecycle setting to prevent the resource from being destroyed. """ - started: VariableOrOptional[bool] - """ - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. - """ - LifecycleParam = LifecycleDict | Lifecycle From df48e0870d4ddce53800ef65ccf2bf736bae099f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 10 Mar 2026 17:29:53 +0100 Subject: [PATCH 08/42] fix test output --- acceptance/bundle/lifecycle/started/output.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/lifecycle/started/output.txt b/acceptance/bundle/lifecycle/started/output.txt index 0ffc8f1d18..0b4912bfe3 100644 --- a/acceptance/bundle/lifecycle/started/output.txt +++ b/acceptance/bundle/lifecycle/started/output.txt @@ -1,5 +1,7 @@ -Error: lifecycle.started is not supported for resources.jobs.my_job; it is only supported for apps, clusters, and sql_warehouses - in databricks.yml:9:18 +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 +create jobs.my_job -Exit code: 1 +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged From 464c1b83be25498e666259070eb57d53aab4d5e6 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 11 Mar 2026 11:16:07 +0100 Subject: [PATCH 09/42] fix tests --- .../out.deploy.terraform.txt | 3 - .../validate_lifecycle_started_test.go | 115 ------------------ 2 files changed, 118 deletions(-) delete mode 100644 bundle/config/mutator/validate_lifecycle_started_test.go diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt index cdc924fe7c..18744fe287 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt @@ -1,7 +1,6 @@ >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 Exit code: 1 @@ -15,7 +14,6 @@ Exit code: 1 >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 Exit code: 1 @@ -40,7 +38,6 @@ Deployment complete! >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 Exit code: 1 diff --git a/bundle/config/mutator/validate_lifecycle_started_test.go b/bundle/config/mutator/validate_lifecycle_started_test.go deleted file mode 100644 index 0c9df6a6ce..0000000000 --- a/bundle/config/mutator/validate_lifecycle_started_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func boolPtr(v bool) *bool { - return &v -} - -func TestValidateLifecycleStartedDirectEngine(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "my_app": { - Lifecycle: resources.LifecycleWithStarted{ - Started: boolPtr(true), - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineDirect)) - require.NoError(t, diags.Error()) -} - -func TestValidateLifecycleStartedTerraformEngine(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "my_app": { - Lifecycle: resources.LifecycleWithStarted{ - Started: boolPtr(true), - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) - require.Error(t, diags.Error()) - assert.Contains(t, diags.Error().Error(), "lifecycle.started is only supported in direct deployment mode") -} - -func TestValidateLifecycleStartedFalse(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Clusters: map[string]*resources.Cluster{ - "my_cluster": { - Lifecycle: resources.LifecycleWithStarted{ - Started: boolPtr(false), - }, - }, - }, - }, - }, - } - - // started=false should not produce an error even with terraform engine - diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) - require.NoError(t, diags.Error()) -} - -func TestValidateLifecycleStartedNotSet(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - SqlWarehouses: map[string]*resources.SqlWarehouse{ - "my_warehouse": { - Lifecycle: resources.LifecycleWithStarted{}, - }, - }, - }, - }, - } - - // started not set should not produce an error - diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) - require.NoError(t, diags.Error()) -} - -func TestValidateLifecycleStartedJobNotSupported(t *testing.T) { - // Jobs don't have LifecycleWithStarted, so lifecycle.started is not accessible. - // This test verifies that resources without LifecycleWithStarted don't trigger errors. - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "my_job": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{PreventDestroy: false}, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateLifecycleStarted(engine.EngineTerraform)) - require.NoError(t, diags.Error()) -} From 34589cf1c811669c4a2b295f6e72a884a93ba635 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 11:44:29 +0100 Subject: [PATCH 10/42] fixed starting clusters amnd sql warehouses --- acceptance/bundle/refschema/out.fields.txt | 3 --- bundle/config/resources.go | 2 +- bundle/config/resources/apps.go | 2 +- bundle/config/resources/base.go | 2 +- bundle/config/resources/clusters.go | 2 +- bundle/config/resources/external_location.go | 2 +- bundle/config/resources/lifecycle.go | 12 +++------- bundle/config/resources/sql_warehouses.go | 2 +- bundle/direct/dresources/cluster.go | 23 +++++++++++++++++++- bundle/direct/dresources/sql_warehouse.go | 18 +++++++++++++++ bundle/phases/plan_test.go | 4 ++-- 11 files changed, 51 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 39cae2c526..4c3e50137f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -122,7 +122,6 @@ resources.apps.*.git_source.resolved_commit string INPUT STATE resources.apps.*.git_source.source_code_path string INPUT STATE resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL -resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT resources.apps.*.lifecycle.started *bool INPUT @@ -377,7 +376,6 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE -resources.clusters.*.lifecycle resources.Lifecycle INPUT resources.clusters.*.lifecycle resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT resources.clusters.*.lifecycle.started *bool INPUT @@ -2793,7 +2791,6 @@ resources.sql_warehouses.*.health.summary string REMOTE resources.sql_warehouses.*.id string INPUT REMOTE resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE -resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT resources.sql_warehouses.*.lifecycle resources.LifecycleWithStarted INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT resources.sql_warehouses.*.lifecycle.started *bool INPUT diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 977c1b04ea..4131f68695 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -55,7 +55,7 @@ type ConfigResource interface { InitializeURL(baseURL url.URL) // GetLifecycle returns the lifecycle settings for the resource. - GetLifecycle() resources.ILifecycle + GetLifecycle() resources.LifecycleConfig } // ResourceGroup represents a group of resources of the same type. diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index 26faa7d3a5..590029b3d1 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -57,7 +57,7 @@ type App struct { } // GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. -func (a *App) GetLifecycle() ILifecycle { +func (a *App) GetLifecycle() LifecycleConfig { return a.Lifecycle } diff --git a/bundle/config/resources/base.go b/bundle/config/resources/base.go index 5af311304b..ceeb1f0b86 100644 --- a/bundle/config/resources/base.go +++ b/bundle/config/resources/base.go @@ -9,6 +9,6 @@ type BaseResource struct { } // GetLifecycle returns the lifecycle settings for the resource. -func (b *BaseResource) GetLifecycle() ILifecycle { +func (b *BaseResource) GetLifecycle() LifecycleConfig { return b.Lifecycle } diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index 1b1eb1a323..e1b37ec926 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -21,7 +21,7 @@ type Cluster struct { } // GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. -func (s *Cluster) GetLifecycle() ILifecycle { +func (s *Cluster) GetLifecycle() LifecycleConfig { return s.Lifecycle } diff --git a/bundle/config/resources/external_location.go b/bundle/config/resources/external_location.go index 36b4e37db7..cc413de84f 100644 --- a/bundle/config/resources/external_location.go +++ b/bundle/config/resources/external_location.go @@ -63,7 +63,7 @@ func (e *ExternalLocation) GetName() string { } // GetLifecycle returns the lifecycle settings for the resource. -func (e *ExternalLocation) GetLifecycle() ILifecycle { +func (e *ExternalLocation) GetLifecycle() LifecycleConfig { return e.Lifecycle } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 9805199c74..db2b130d31 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -1,7 +1,7 @@ package resources -// ILifecycle is implemented by Lifecycle and LifecycleWithStarted. -type ILifecycle interface { +// LifecycleConfig is implemented by Lifecycle and LifecycleWithStarted. +type LifecycleConfig interface { HasPreventDestroy() bool } @@ -19,15 +19,9 @@ func (l Lifecycle) HasPreventDestroy() bool { // LifecycleWithStarted contains lifecycle settings for resources that support lifecycle.started. // It is used by apps, clusters, and sql_warehouses. type LifecycleWithStarted struct { - // Lifecycle setting to prevent the resource from being destroyed. - PreventDestroy bool `json:"prevent_destroy,omitempty"` + Lifecycle // If set to true, the resource will be deployed in started mode. // Supported only for apps, clusters, and sql_warehouses. Started *bool `json:"started,omitempty"` } - -// HasPreventDestroy returns true if prevent_destroy is set. -func (l LifecycleWithStarted) HasPreventDestroy() bool { - return l.PreventDestroy -} diff --git a/bundle/config/resources/sql_warehouses.go b/bundle/config/resources/sql_warehouses.go index 0251ac0532..dcd0e451c8 100644 --- a/bundle/config/resources/sql_warehouses.go +++ b/bundle/config/resources/sql_warehouses.go @@ -21,7 +21,7 @@ type SqlWarehouse struct { } // GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. -func (sw *SqlWarehouse) GetLifecycle() ILifecycle { +func (sw *SqlWarehouse) GetLifecycle() LifecycleConfig { return sw.Lifecycle } diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 2eea7d517d..82dfff109d 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -121,7 +121,28 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust } return nil, retries.Halt(err) }) - return nil, err + if err != nil { + return nil, err + } + + // With lifecycle.started=true, ensure the cluster is running after the update. + if config.Started { + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get cluster %s: %w", id, err) + } + if details.State == compute.StateTerminated { + startWait, err := r.client.Clusters.Start(ctx, compute.StartCluster{ClusterId: id}) + if err != nil { + return nil, fmt.Errorf("failed to start cluster %s: %w", id, err) + } + if _, err = startWait.Get(); err != nil { + return nil, fmt.Errorf("failed to wait for cluster %s to start: %w", id, err) + } + } + } + + return nil, nil } func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState) error { diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 066f63e184..a75cc422ab 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "fmt" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" @@ -105,6 +106,23 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * log.Warnf(ctx, "sql_warehouses: response contains unexpected id=%#v (expected %#v)", waiter.Id, id) } + // With lifecycle.started=true, ensure the warehouse is running after the update. + if config.Started { + warehouse, err := r.client.Warehouses.GetById(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get warehouse %s: %w", id, err) + } + if warehouse.State == sql.StateStopped { + startWait, err := r.client.Warehouses.Start(ctx, sql.StartRequest{Id: id}) + if err != nil { + return nil, fmt.Errorf("failed to start warehouse %s: %w", id, err) + } + if _, err = startWait.Get(); err != nil { + return nil, fmt.Errorf("failed to wait for warehouse %s to start: %w", id, err) + } + } + } + return nil, nil } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 3873f19f8c..dd2ca77527 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -48,7 +48,7 @@ func TestCheckPreventDestroyForApp(t *testing.T) { Resources: config.Resources{ Apps: map[string]*resources.App{ "test_resource": { - Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, + Lifecycle: resources.LifecycleWithStarted{Lifecycle: resources.Lifecycle{PreventDestroy: true}}, }, }, }, @@ -108,7 +108,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { App: apps.App{ Name: "Test App", }, - Lifecycle: resources.LifecycleWithStarted{PreventDestroy: true}, + Lifecycle: resources.LifecycleWithStarted{Lifecycle: resources.Lifecycle{PreventDestroy: true}}, }, }, }, From 93af424027467937967cd995448f6e9a179759ba Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 11:52:51 +0100 Subject: [PATCH 11/42] fix required --- acceptance/bundle/refschema/out.fields.txt | 3 +++ bundle/internal/validation/required.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 4c3e50137f..39cae2c526 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -122,6 +122,7 @@ resources.apps.*.git_source.resolved_commit string INPUT STATE resources.apps.*.git_source.source_code_path string INPUT STATE resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL +resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT resources.apps.*.lifecycle.started *bool INPUT @@ -376,6 +377,7 @@ resources.clusters.*.jdbc_port int REMOTE resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.lifecycle resources.Lifecycle INPUT resources.clusters.*.lifecycle resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT resources.clusters.*.lifecycle.started *bool INPUT @@ -2791,6 +2793,7 @@ resources.sql_warehouses.*.health.summary string REMOTE resources.sql_warehouses.*.id string INPUT REMOTE resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE +resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT resources.sql_warehouses.*.lifecycle resources.LifecycleWithStarted INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT resources.sql_warehouses.*.lifecycle.started *bool INPUT diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index b08e0f4065..87cd15a50c 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -52,6 +52,11 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } + // Anonymous embedded structs are transparent in JSON; skip them as standalone fields. + if field.Anonymous { + return true + } + // Do not generate required validation code for fields that are internal or readonly. bundleTag := structtag.BundleTag(field.Tag.Get("bundle")) if bundleTag.Internal() || bundleTag.ReadOnly() { From 132ead92a0f11d54b93c778cafd4bd6aa742445a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 15:05:40 +0100 Subject: [PATCH 12/42] also stop resources --- .../started-validation/databricks.yml | 10 ++++ .../started-validation/out.direct.txt | 5 ++ .../started-validation/out.terraform.txt | 6 ++ .../started-validation/out.test.toml | 5 ++ .../lifecycle/started-validation/output.txt | 0 .../lifecycle/started-validation/script | 1 + .../lifecycle/started-validation/test.toml | 4 ++ .../mutator/validate_lifecycle_started.go | 2 +- bundle/direct/dresources/app.go | 42 +++++++++----- bundle/direct/dresources/cluster.go | 58 ++++++++++++++----- bundle/direct/dresources/sql_warehouse.go | 58 ++++++++++++++----- 11 files changed, 150 insertions(+), 41 deletions(-) create mode 100644 acceptance/bundle/lifecycle/started-validation/databricks.yml create mode 100644 acceptance/bundle/lifecycle/started-validation/out.direct.txt create mode 100644 acceptance/bundle/lifecycle/started-validation/out.terraform.txt create mode 100644 acceptance/bundle/lifecycle/started-validation/out.test.toml create mode 100644 acceptance/bundle/lifecycle/started-validation/output.txt create mode 100644 acceptance/bundle/lifecycle/started-validation/script create mode 100644 acceptance/bundle/lifecycle/started-validation/test.toml diff --git a/acceptance/bundle/lifecycle/started-validation/databricks.yml b/acceptance/bundle/lifecycle/started-validation/databricks.yml new file mode 100644 index 0000000000..60fb045e1d --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test_lifecycle_started_validation + +resources: + clusters: + my_cluster: + cluster_name: my_cluster + spark_version: 14.3.x-scala2.12 + lifecycle: + started: true diff --git a/acceptance/bundle/lifecycle/started-validation/out.direct.txt b/acceptance/bundle/lifecycle/started-validation/out.direct.txt new file mode 100644 index 0000000000..f9bd1b79cf --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.direct.txt @@ -0,0 +1,5 @@ + +>>> errcode [CLI] bundle plan +create clusters.my_cluster + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt new file mode 100644 index 0000000000..b89c46ac98 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt @@ -0,0 +1,6 @@ + +>>> errcode [CLI] bundle plan +Error: lifecycle.started is only supported in direct deployment mode + + +Exit code: 1 diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml new file mode 100644 index 0000000000..58724616cf --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started-validation/output.txt b/acceptance/bundle/lifecycle/started-validation/output.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/lifecycle/started-validation/script b/acceptance/bundle/lifecycle/started-validation/script new file mode 100644 index 0000000000..ff9d8c0454 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/script @@ -0,0 +1 @@ +trace errcode $CLI bundle plan >> out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 diff --git a/acceptance/bundle/lifecycle/started-validation/test.toml b/acceptance/bundle/lifecycle/started-validation/test.toml new file mode 100644 index 0000000000..bc76388306 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/test.toml @@ -0,0 +1,4 @@ +Ignore = [".databricks"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index f2b3ef317c..2cbc8b976e 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -30,7 +30,7 @@ func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) di for _, group := range b.Config.Resources.AllResources() { for _, resource := range group.Resources { lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) - if !ok || lws.Started == nil || !*lws.Started { + if !ok || lws.Started == nil { continue } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index ff6843cb50..280a6382d6 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -24,7 +24,7 @@ type AppState struct { SourceCodePath string `json:"source_code_path,omitempty"` Config *resources.AppConfig `json:"config,omitempty"` GitSource *apps.GitSource `json:"git_source,omitempty"` - Started bool `json:"started,omitempty"` + Started *bool `json:"started,omitempty"` } type ResourceApp struct { @@ -36,13 +36,12 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { } func (*ResourceApp) PrepareState(input *resources.App) *AppState { - started := input.Lifecycle.Started != nil && *input.Lifecycle.Started return &AppState{ App: input.App, SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, - Started: started, + Started: input.Lifecycle.Started, } } @@ -55,7 +54,7 @@ func (*ResourceApp) RemapState(remote *apps.App) *AppState { SourceCodePath: "", Config: nil, GitSource: nil, - Started: false, + Started: nil, } } @@ -64,9 +63,9 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) } func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { - // With lifecycle.started=true, start the app compute (no_compute=false). - // Otherwise, skip compute startup during creation. - noCompute := !config.Started + // Start app compute only when lifecycle.started=true is explicit. + // For nil (omitted) or false, use no_compute=true (do not start compute). + noCompute := !(config.Started != nil && *config.Started) request := apps.CreateAppRequest{ App: config.App, NoCompute: noCompute, @@ -126,13 +125,17 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) } - // With lifecycle.started=true, ensure the app compute is running and deploy the latest code. - if config.Started { - // Start compute if it is stopped (mirrors bundle run behavior). - app, err := r.client.Apps.GetByName(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get app %s: %w", id, err) - } + if config.Started == nil { + return nil, nil + } + + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get app %s: %w", id, err) + } + + if *config.Started { + // lifecycle.started=true: ensure the app compute is running and deploy the latest code. if isComputeStopped(app) { startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) if err != nil { @@ -150,6 +153,17 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, if err := appdeploy.Deploy(ctx, r.client, id, deployment); err != nil { return nil, err } + } else { + // lifecycle.started=false: ensure the app compute is stopped. + if !isComputeStopped(app) { + stopWaiter, err := r.client.Apps.Stop(ctx, apps.StopAppRequest{Name: id}) + if err != nil { + return nil, fmt.Errorf("failed to stop app %s: %w", id, err) + } + if _, err = stopWaiter.Get(); err != nil { + return nil, fmt.Errorf("failed to wait for app %s to stop: %w", id, err) + } + } } return nil, nil diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 82dfff109d..51dcfc2395 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -18,7 +18,7 @@ import ( type ClusterState struct { compute.ClusterSpec - Started bool `json:"started,omitempty"` + Started *bool `json:"started,omitempty"` } type ResourceCluster struct { @@ -32,10 +32,9 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { } func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { - started := input.Lifecycle.Started != nil && *input.Lifecycle.Started return &ClusterState{ ClusterSpec: input.ClusterSpec, - Started: started, + Started: input.Lifecycle.Started, } } @@ -80,7 +79,7 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterStat if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return &ClusterState{ClusterSpec: *spec, Started: false} + return &ClusterState{ClusterSpec: *spec, Started: nil} } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { @@ -92,15 +91,32 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (s if err != nil { return "", nil, err } - // With lifecycle.started=true, wait for the cluster to reach the running state. - if config.Started { + switch { + case config.Started != nil && *config.Started: + // lifecycle.started=true: wait for the cluster to reach the running state. details, err := wait.Get() if err != nil { return "", nil, err } return details.ClusterId, details, nil + case config.Started != nil && !*config.Started: + // lifecycle.started=false: wait for running, then terminate to reach stopped state. + details, err := wait.Get() + if err != nil { + return "", nil, err + } + terminateWait, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: details.ClusterId}) + if err != nil { + return "", nil, fmt.Errorf("failed to terminate cluster %s: %w", details.ClusterId, err) + } + if _, err = terminateWait.Get(); err != nil { + return "", nil, fmt.Errorf("failed to wait for cluster %s to terminate: %w", details.ClusterId, err) + } + return details.ClusterId, nil, nil + default: + // lifecycle.started omitted: default behaviour, return immediately without waiting. + return wait.ClusterId, nil, nil } - return wait.ClusterId, nil, nil } func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { @@ -125,12 +141,17 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust return nil, err } - // With lifecycle.started=true, ensure the cluster is running after the update. - if config.Started { - details, err := r.client.Clusters.GetByClusterId(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get cluster %s: %w", id, err) - } + if config.Started == nil { + return nil, nil + } + + details, err := r.client.Clusters.GetByClusterId(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get cluster %s: %w", id, err) + } + + if *config.Started { + // lifecycle.started=true: ensure the cluster is running. if details.State == compute.StateTerminated { startWait, err := r.client.Clusters.Start(ctx, compute.StartCluster{ClusterId: id}) if err != nil { @@ -140,6 +161,17 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust return nil, fmt.Errorf("failed to wait for cluster %s to start: %w", id, err) } } + } else { + // lifecycle.started=false: ensure the cluster is stopped. + if details.State != compute.StateTerminated && details.State != compute.StateTerminating { + terminateWait, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: id}) + if err != nil { + return nil, fmt.Errorf("failed to terminate cluster %s: %w", id, err) + } + if _, err = terminateWait.Get(); err != nil { + return nil, fmt.Errorf("failed to wait for cluster %s to terminate: %w", id, err) + } + } } return nil, nil diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index a75cc422ab..e90009aa75 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -15,7 +15,7 @@ import ( type SqlWarehouseState struct { sql.CreateWarehouseRequest - Started bool `json:"started,omitempty"` + Started *bool `json:"started,omitempty"` } type ResourceSqlWarehouse struct { @@ -29,15 +29,14 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq // PrepareState converts bundle config to the SDK type. func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { - started := input.Lifecycle.Started != nil && *input.Lifecycle.Started return &SqlWarehouseState{ CreateWarehouseRequest: input.CreateWarehouseRequest, - Started: started, + Started: input.Lifecycle.Started, } } func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { - return &SqlWarehouseState{Started: false, CreateWarehouseRequest: sql.CreateWarehouseRequest{ + return &SqlWarehouseState{Started: nil, CreateWarehouseRequest: sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -66,15 +65,32 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehous if err != nil { return "", nil, err } - // With lifecycle.started=true, wait for the warehouse to reach the running state. - if config.Started { + switch { + case config.Started != nil && *config.Started: + // lifecycle.started=true: wait for the warehouse to reach the running state. warehouse, err := waiter.Get() if err != nil { return "", nil, err } return warehouse.Id, warehouse, nil + case config.Started != nil && !*config.Started: + // lifecycle.started=false: wait for running, then stop to reach stopped state. + warehouse, err := waiter.Get() + if err != nil { + return "", nil, err + } + stopWait, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: warehouse.Id}) + if err != nil { + return "", nil, fmt.Errorf("failed to stop warehouse %s: %w", warehouse.Id, err) + } + if _, err = stopWait.Get(); err != nil { + return "", nil, fmt.Errorf("failed to wait for warehouse %s to stop: %w", warehouse.Id, err) + } + return warehouse.Id, nil, nil + default: + // lifecycle.started omitted: default behaviour, return immediately without waiting. + return waiter.Id, nil, nil } - return waiter.Id, nil, nil } // DoUpdate updates the warehouse in place. @@ -106,12 +122,17 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * log.Warnf(ctx, "sql_warehouses: response contains unexpected id=%#v (expected %#v)", waiter.Id, id) } - // With lifecycle.started=true, ensure the warehouse is running after the update. - if config.Started { - warehouse, err := r.client.Warehouses.GetById(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get warehouse %s: %w", id, err) - } + if config.Started == nil { + return nil, nil + } + + warehouse, err := r.client.Warehouses.GetById(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get warehouse %s: %w", id, err) + } + + if *config.Started { + // lifecycle.started=true: ensure the warehouse is running. if warehouse.State == sql.StateStopped { startWait, err := r.client.Warehouses.Start(ctx, sql.StartRequest{Id: id}) if err != nil { @@ -121,6 +142,17 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * return nil, fmt.Errorf("failed to wait for warehouse %s to start: %w", id, err) } } + } else { + // lifecycle.started=false: ensure the warehouse is stopped. + if warehouse.State != sql.StateStopped && warehouse.State != sql.StateStopping { + stopWait, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: id}) + if err != nil { + return nil, fmt.Errorf("failed to stop warehouse %s: %w", id, err) + } + if _, err = stopWait.Get(); err != nil { + return nil, fmt.Errorf("failed to wait for warehouse %s to stop: %w", id, err) + } + } } return nil, nil From f9aee88eebaa00eda4bb626807c90e28a75fc748 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 15:42:09 +0100 Subject: [PATCH 13/42] fixes --- .../started-validation/out.terraform.txt | 1 + .../mutator/validate_lifecycle_started.go | 8 +++-- bundle/direct/dresources/app.go | 2 +- bundle/phases/plan_test.go | 35 +++++++++++++++++++ libs/testserver/apps.go | 2 +- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt index b89c46ac98..d086f7b90f 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt +++ b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt @@ -1,6 +1,7 @@ >>> errcode [CLI] bundle plan Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:10:18 Exit code: 1 diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index 2cbc8b976e..de76a38e93 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -28,7 +28,7 @@ func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) di var diags diag.Diagnostics for _, group := range b.Config.Resources.AllResources() { - for _, resource := range group.Resources { + for key, resource := range group.Resources { lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) if !ok || lws.Started == nil { continue @@ -36,9 +36,11 @@ func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) di // lifecycle.started is a direct-mode-only feature. if !m.engine.IsDirect() { + path := "resources." + group.Description.PluralName + "." + key + ".lifecycle.started" diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.started is only supported in direct deployment mode", + Severity: diag.Error, + Summary: "lifecycle.started is only supported in direct deployment mode", + Locations: b.Config.GetLocations(path), }) } } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 280a6382d6..6e08e33d48 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -65,7 +65,7 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { // Start app compute only when lifecycle.started=true is explicit. // For nil (omitted) or false, use no_compute=true (do not start compute). - noCompute := !(config.Started != nil && *config.Started) + noCompute := config.Started == nil || !*config.Started request := apps.CreateAppRequest{ App: config.App, NoCompute: noCompute, diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index dd2ca77527..1318b63912 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -7,11 +7,46 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/apps" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) +func TestCheckPreventDestroyForAllResources(t *testing.T) { + for resourceType := range config.SupportedResources() { + t.Run(resourceType, func(t *testing.T) { + b := &bundle.Bundle{} + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + resourceType: dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) + + actions := []deployplan.Action{ + { + ResourceKey: "resources." + resourceType + ".test_resource", + ActionType: deployplan.Recreate, + }, + } + + err = checkForPreventDestroy(b, actions) + require.Error(t, err) + require.Contains(t, err.Error(), "resources."+resourceType+".test_resource has lifecycle.prevent_destroy set") + require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") + require.Contains(t, err.Error(), "disable lifecycle.prevent_destroy for resources."+resourceType+".test_resource") + }) + } +} + func TestCheckPreventDestroyForJob(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 8d7ac8bfc7..ff3c9ee047 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -102,7 +102,7 @@ func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response return Response{StatusCode: 500, Body: fmt.Sprintf("internal error: %s", err)} } - deployment.DeploymentId = "deploy-1" + deployment.DeploymentId = fmt.Sprintf("deploy-%d", nextID()) deployment.Status = &apps.AppDeploymentStatus{ State: apps.AppDeploymentStateSucceeded, Message: "Deployment succeeded.", From 6370d81c279f0601832340f879b3dd31ac382f91 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 15:44:28 +0100 Subject: [PATCH 14/42] fix schemaa test --- acceptance/bundle/refschema/out.fields.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 39cae2c526..61e3f3fe1f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -201,6 +201,7 @@ resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL resources.apps.*.source_code_path string INPUT STATE resources.apps.*.space string ALL +resources.apps.*.started *bool STATE resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*].unity_catalog *apps.UnityCatalog ALL @@ -505,7 +506,7 @@ resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE resources.clusters.*.ssh_public_keys []string ALL resources.clusters.*.ssh_public_keys[*] string ALL resources.clusters.*.start_time int64 REMOTE -resources.clusters.*.started bool STATE +resources.clusters.*.started *bool STATE resources.clusters.*.state compute.State REMOTE resources.clusters.*.state_message string REMOTE resources.clusters.*.terminated_time int64 REMOTE @@ -2809,7 +2810,7 @@ resources.sql_warehouses.*.odbc_params.path string REMOTE resources.sql_warehouses.*.odbc_params.port int REMOTE resources.sql_warehouses.*.odbc_params.protocol string REMOTE resources.sql_warehouses.*.spot_instance_policy sql.SpotInstancePolicy ALL -resources.sql_warehouses.*.started bool STATE +resources.sql_warehouses.*.started *bool STATE resources.sql_warehouses.*.state sql.State REMOTE resources.sql_warehouses.*.tags *sql.EndpointTags ALL resources.sql_warehouses.*.tags.custom_tags []sql.EndpointTagPair ALL From 67de29d54f8043bc0461ef3048c250572cdc2ea3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 16 Mar 2026 16:10:55 +0100 Subject: [PATCH 15/42] fixed tf output --- .../out.deploy.terraform.txt | 20 +++++++++++++------ .../out.destroy.terraform.txt | 8 +------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt index 18744fe287..c65dabe49b 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt @@ -1,6 +1,7 @@ >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -14,6 +15,7 @@ Exit code: 1 >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -26,18 +28,22 @@ Error: Not Found Exit code: 1 >>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 + + +Exit code: 1 >>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt >>> errcode [CLI] apps get [UNIQUE_NAME] -"ACTIVE" +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 >>> errcode [CLI] bundle deploy Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 Exit code: 1 @@ -45,4 +51,6 @@ Exit code: 1 >>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt >>> errcode [CLI] apps get [UNIQUE_NAME] -"ACTIVE" +Error: Resource apps.App not found: [UNIQUE_NAME] + +Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt index 8061857d58..16da688a34 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt @@ -1,9 +1,3 @@ >>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.apps.myapp - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! +No active deployment found to destroy! From 1e87fc5f1a12add01099daade3f3057b943662f6 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 13:52:54 +0100 Subject: [PATCH 16/42] remove clusters, sql_warehouses --- .../lifecycle/started-validation/app/app.py | 1 + .../started-validation/databricks.yml | 14 ++- .../started-validation/out.direct.txt | 9 +- .../started-validation/out.terraform.txt | 6 +- .../apps/lifecycle-started-toggle/app/app.py | 1 + .../databricks.yml.tmpl | 11 ++ .../lifecycle-started-toggle/out.test.toml | 5 + .../apps/lifecycle-started-toggle/output.txt | 81 +++++++++++++ .../apps/lifecycle-started-toggle/script | 29 +++++ .../apps/lifecycle-started-toggle/test.toml | 8 ++ .../mutator/validate_lifecycle_started.go | 2 +- bundle/config/resources/clusters.go | 8 -- bundle/config/resources/sql_warehouses.go | 8 -- bundle/direct/dresources/app.go | 108 ++++++++++++++---- bundle/direct/dresources/app_test.go | 32 ++++++ bundle/direct/dresources/cluster.go | 68 +---------- bundle/direct/dresources/resources.yml | 25 ++++ bundle/direct/dresources/sql_warehouse.go | 74 +----------- bundle/direct/dresources/type_test.go | 4 - libs/testserver/apps.go | 14 ++- 20 files changed, 320 insertions(+), 188 deletions(-) create mode 100644 acceptance/bundle/lifecycle/started-validation/app/app.py create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/script create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml diff --git a/acceptance/bundle/lifecycle/started-validation/app/app.py b/acceptance/bundle/lifecycle/started-validation/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/lifecycle/started-validation/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/lifecycle/started-validation/databricks.yml b/acceptance/bundle/lifecycle/started-validation/databricks.yml index 60fb045e1d..d6d2d81c2d 100644 --- a/acceptance/bundle/lifecycle/started-validation/databricks.yml +++ b/acceptance/bundle/lifecycle/started-validation/databricks.yml @@ -2,9 +2,15 @@ bundle: name: test_lifecycle_started_validation resources: - clusters: - my_cluster: - cluster_name: my_cluster - spark_version: 14.3.x-scala2.12 + jobs: + my_job: + name: my_job + lifecycle: + started: true + + apps: + my_app: + name: my_app + source_code_path: ./app lifecycle: started: true diff --git a/acceptance/bundle/lifecycle/started-validation/out.direct.txt b/acceptance/bundle/lifecycle/started-validation/out.direct.txt index f9bd1b79cf..12a310841c 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.direct.txt +++ b/acceptance/bundle/lifecycle/started-validation/out.direct.txt @@ -1,5 +1,10 @@ >>> errcode [CLI] bundle plan -create clusters.my_cluster +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +create apps.my_app +create jobs.my_job + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt index d086f7b90f..ef87f528c1 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.terraform.txt +++ b/acceptance/bundle/lifecycle/started-validation/out.terraform.txt @@ -1,7 +1,11 @@ >>> errcode [CLI] bundle plan +Warning: unknown field: started + at resources.jobs.my_job.lifecycle + in databricks.yml:9:9 + Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:10:18 + in databricks.yml:16:18 Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl new file mode 100644 index 0000000000..b6ecf37530 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-toggle-$UNIQUE_NAME + +resources: + apps: + mykey: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: false diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt new file mode 100644 index 0000000000..173d96d1bc --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -0,0 +1,81 @@ + +=== Deploy with started=false: app created without compute (no_compute=true) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Toggle started=false -> started=true: only Start should be called, no Update +>>> update_file.py databricks.yml started: false started: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Warn: unexpected local and remote diffs (bool, *bool); entry=&{ false true } ch={started 0x1400030b4ac} +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Toggle started=true -> started=false: only Stop should be called, no Update +>>> update_file.py databricks.yml started: true started: false + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... +Warn: unexpected local and remote diffs (bool, *bool); entry=&{ true false } ch={started 0x1400013e4a4} +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/stop" +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.mykey + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/script b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script new file mode 100644 index 0000000000..7b5a2c8c32 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_app_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps"))) | del(.body.url)' < out.requests.txt + rm out.requests.txt +} + +title "Deploy with started=false: app created without compute (no_compute=true)" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Toggle started=false -> started=true: only Start should be called, no Update" +trace update_file.py databricks.yml "started: false" "started: true" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true + +title "Toggle started=true -> started=false: only Stop should be called, no Update" +trace update_file.py databricks.yml "started: true" "started: false" +trace $CLI bundle deploy +trace print_app_requests +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index de76a38e93..c7be225f29 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -14,7 +14,7 @@ type validateLifecycleStarted struct { } // ValidateLifecycleStarted returns a mutator that validates lifecycle.started -// is only used on supported resource types (apps, clusters, sql_warehouses). +// is only used on supported resource types (apps). // lifecycle.started is only supported in direct deployment mode. func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { return &validateLifecycleStarted{engine: e} diff --git a/bundle/config/resources/clusters.go b/bundle/config/resources/clusters.go index e1b37ec926..c549ac4a6b 100644 --- a/bundle/config/resources/clusters.go +++ b/bundle/config/resources/clusters.go @@ -14,17 +14,9 @@ type Cluster struct { BaseResource compute.ClusterSpec - // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. - Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` - Permissions []ClusterPermission `json:"permissions,omitempty"` } -// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. -func (s *Cluster) GetLifecycle() LifecycleConfig { - return s.Lifecycle -} - func (s *Cluster) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) } diff --git a/bundle/config/resources/sql_warehouses.go b/bundle/config/resources/sql_warehouses.go index dcd0e451c8..bed567b805 100644 --- a/bundle/config/resources/sql_warehouses.go +++ b/bundle/config/resources/sql_warehouses.go @@ -14,17 +14,9 @@ type SqlWarehouse struct { BaseResource sql.CreateWarehouseRequest - // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. - Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` - Permissions []SqlWarehousePermission `json:"permissions,omitempty"` } -// GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. -func (sw *SqlWarehouse) GetLifecycle() LifecycleConfig { - return sw.Lifecycle -} - func (sw *SqlWarehouse) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, sw) } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 6e08e33d48..ec7b958359 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -10,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/appdeploy" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" @@ -27,6 +27,65 @@ type AppState struct { Started *bool `json:"started,omitempty"` } +// bundleOnlyFields are the AppState fields not present in apps.App. +// They must be explicitly marshaled/unmarshaled since apps.App's promoted MarshalJSON +// would otherwise shadow them. +type bundleOnlyFields struct { + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Started *bool `json:"started,omitempty"` +} + +// MarshalJSON merges apps.App fields with bundle-only fields into a single JSON object. +// apps.App promotes its own MarshalJSON to AppState, which would otherwise omit bundle-only fields. +func (s AppState) MarshalJSON() ([]byte, error) { + appJSON, err := json.Marshal(s.App) + if err != nil { + return nil, err + } + extrasJSON, err := json.Marshal(bundleOnlyFields{ + SourceCodePath: s.SourceCodePath, + Config: s.Config, + GitSource: s.GitSource, + Started: s.Started, + }) + if err != nil { + return nil, err + } + var result, extras map[string]json.RawMessage + if err := json.Unmarshal(appJSON, &result); err != nil { + return nil, err + } + if result == nil { + result = make(map[string]json.RawMessage) + } + if err := json.Unmarshal(extrasJSON, &extras); err != nil { + return nil, err + } + for k, v := range extras { + result[k] = v + } + return json.Marshal(result) +} + +// UnmarshalJSON sets apps.App fields and bundle-only fields from a single JSON object. +// apps.App promotes its own UnmarshalJSON to *AppState, which would otherwise drop bundle-only fields. +func (s *AppState) UnmarshalJSON(data []byte) error { + if err := json.Unmarshal(data, &s.App); err != nil { + return err + } + var e bundleOnlyFields + if err := json.Unmarshal(data, &e); err != nil { + return err + } + s.SourceCodePath = e.SourceCodePath + s.Config = e.Config + s.GitSource = e.GitSource + s.Started = e.Started + return nil +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -104,25 +163,34 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, * } func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*apps.App, error) { - updateMask := strings.Join(collectUpdatePathsWithPrefix(changes, ""), ",") - - request := apps.AsyncUpdateAppRequest{ - App: &config.App, - AppName: id, - UpdateMask: updateMask, - } - updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) - if err != nil { - return nil, err + // Build update mask excluding local-only fields that have no counterpart in the API. + var maskPaths []string + for path, change := range changes { + if change.Action == deployplan.Update && !localOnlyFields[path] { + maskPaths = append(maskPaths, path) + } } + updateMask := strings.Join(maskPaths, ",") - response, err := updateWaiter.Get() - if err != nil { - return nil, err - } + if updateMask != "" { + request := apps.AsyncUpdateAppRequest{ + App: &config.App, + AppName: id, + UpdateMask: updateMask, + } + updateWaiter, err := r.client.Apps.CreateUpdate(ctx, request) + if err != nil { + return nil, err + } - if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { - return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) + response, err := updateWaiter.Get() + if err != nil { + return nil, err + } + + if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { + return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) + } } if config.Started == nil { @@ -178,12 +246,6 @@ var localOnlyFields = map[string]bool{ "started": true, } -func (*ResourceApp) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *apps.App) error { - if change.Action == deployplan.Update && localOnlyFields[p.Prefix(1).String()] { - change.Action = deployplan.Skip - } - return nil -} func isComputeStopped(app *apps.App) bool { return app.ComputeStatus == nil || diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index ad9ca01e8a..35340440f5 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -1,8 +1,10 @@ package dresources import ( + "encoding/json" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" @@ -10,6 +12,36 @@ import ( "github.com/stretchr/testify/require" ) +// TestAppStateMarshalUnmarshal verifies that AppState correctly preserves bundle-only fields +// (SourceCodePath, Config, GitSource, Started) through a JSON round-trip. +// apps.App promotes its MarshalJSON/UnmarshalJSON which would otherwise drop these fields. +func TestAppStateMarshalUnmarshal(t *testing.T) { + started := true + original := AppState{ + App: apps.App{ + Name: "my-app", + Description: "test description", + }, + SourceCodePath: "/Workspace/Users/user/.bundle/app/files", + Config: &resources.AppConfig{ + Command: []string{"python", "app.py"}, + }, + Started: &started, + } + + data, err := json.Marshal(original) + require.NoError(t, err) + + var restored AppState + require.NoError(t, json.Unmarshal(data, &restored)) + + assert.Equal(t, original.App.Name, restored.App.Name) + assert.Equal(t, original.App.Description, restored.App.Description) + assert.Equal(t, original.SourceCodePath, restored.SourceCodePath) + assert.Equal(t, original.Config, restored.Config) + assert.Equal(t, original.Started, restored.Started) +} + // TestAppDoCreate_RetriesWhenAppIsDeleting verifies that DoCreate retries when // an app already exists but is in DELETING state. func TestAppDoCreate_RetriesWhenAppIsDeleting(t *testing.T) { diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 51dcfc2395..602d13c7c7 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -18,7 +18,6 @@ import ( type ClusterState struct { compute.ClusterSpec - Started *bool `json:"started,omitempty"` } type ResourceCluster struct { @@ -34,7 +33,6 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { return &ClusterState{ ClusterSpec: input.ClusterSpec, - Started: input.Lifecycle.Started, } } @@ -79,7 +77,7 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterStat if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return &ClusterState{ClusterSpec: *spec, Started: nil} + return &ClusterState{ClusterSpec: *spec} } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { @@ -91,32 +89,7 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (s if err != nil { return "", nil, err } - switch { - case config.Started != nil && *config.Started: - // lifecycle.started=true: wait for the cluster to reach the running state. - details, err := wait.Get() - if err != nil { - return "", nil, err - } - return details.ClusterId, details, nil - case config.Started != nil && !*config.Started: - // lifecycle.started=false: wait for running, then terminate to reach stopped state. - details, err := wait.Get() - if err != nil { - return "", nil, err - } - terminateWait, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: details.ClusterId}) - if err != nil { - return "", nil, fmt.Errorf("failed to terminate cluster %s: %w", details.ClusterId, err) - } - if _, err = terminateWait.Get(); err != nil { - return "", nil, fmt.Errorf("failed to wait for cluster %s to terminate: %w", details.ClusterId, err) - } - return details.ClusterId, nil, nil - default: - // lifecycle.started omitted: default behaviour, return immediately without waiting. - return wait.ClusterId, nil, nil - } + return wait.ClusterId, nil, nil } func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { @@ -141,39 +114,6 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust return nil, err } - if config.Started == nil { - return nil, nil - } - - details, err := r.client.Clusters.GetByClusterId(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get cluster %s: %w", id, err) - } - - if *config.Started { - // lifecycle.started=true: ensure the cluster is running. - if details.State == compute.StateTerminated { - startWait, err := r.client.Clusters.Start(ctx, compute.StartCluster{ClusterId: id}) - if err != nil { - return nil, fmt.Errorf("failed to start cluster %s: %w", id, err) - } - if _, err = startWait.Get(); err != nil { - return nil, fmt.Errorf("failed to wait for cluster %s to start: %w", id, err) - } - } - } else { - // lifecycle.started=false: ensure the cluster is stopped. - if details.State != compute.StateTerminated && details.State != compute.StateTerminating { - terminateWait, err := r.client.Clusters.Delete(ctx, compute.DeleteCluster{ClusterId: id}) - if err != nil { - return nil, fmt.Errorf("failed to terminate cluster %s: %w", id, err) - } - if _, err = terminateWait.Get(); err != nil { - return nil, fmt.Errorf("failed to wait for cluster %s to terminate: %w", id, err) - } - } - } - return nil, nil } @@ -199,10 +139,6 @@ func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath. path := p.Prefix(1).String() switch path { - case "started": - // started is lifecycle metadata, not an actual cluster property. - change.Action = deployplan.Skip - case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index eea56c90e8..81482a27f3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -375,10 +375,35 @@ resources: recreate_on_changes: - field: name reason: immutable + ignore_remote_changes: + # source_code_path, config, git_source, and started are bundle-local fields + # not stored in remote state. Remote always returns empty; skip the spurious diff + # when the local config hasn't changed these fields. + - field: source_code_path + reason: local_only + - field: config + reason: local_only + - field: git_source + reason: local_only + - field: started + reason: local_only backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/internal/providers/pluginfw/products/app/resource_app.go#L41 # s["compute_size"] = s["compute_size"].SetComputed() - field: compute_size + # Output-only fields assigned by the platform; never set by users in bundle config. + - field: app_status + - field: compute_status + - field: url + - field: id + - field: service_principal_client_id + - field: service_principal_id + - field: service_principal_name + - field: create_time + - field: update_time + - field: creator + - field: updater + - field: effective_budget_policy_id secret_scopes: backend_defaults: diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index e90009aa75..d1987d6100 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -2,10 +2,8 @@ package dresources import ( "context" - "fmt" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" @@ -15,7 +13,6 @@ import ( type SqlWarehouseState struct { sql.CreateWarehouseRequest - Started *bool `json:"started,omitempty"` } type ResourceSqlWarehouse struct { @@ -31,12 +28,11 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { return &SqlWarehouseState{ CreateWarehouseRequest: input.CreateWarehouseRequest, - Started: input.Lifecycle.Started, } } func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { - return &SqlWarehouseState{Started: nil, CreateWarehouseRequest: sql.CreateWarehouseRequest{ + return &SqlWarehouseState{CreateWarehouseRequest: sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -65,32 +61,11 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehous if err != nil { return "", nil, err } - switch { - case config.Started != nil && *config.Started: - // lifecycle.started=true: wait for the warehouse to reach the running state. - warehouse, err := waiter.Get() - if err != nil { - return "", nil, err - } - return warehouse.Id, warehouse, nil - case config.Started != nil && !*config.Started: - // lifecycle.started=false: wait for running, then stop to reach stopped state. - warehouse, err := waiter.Get() - if err != nil { - return "", nil, err - } - stopWait, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: warehouse.Id}) - if err != nil { - return "", nil, fmt.Errorf("failed to stop warehouse %s: %w", warehouse.Id, err) - } - if _, err = stopWait.Get(); err != nil { - return "", nil, fmt.Errorf("failed to wait for warehouse %s to stop: %w", warehouse.Id, err) - } - return warehouse.Id, nil, nil - default: - // lifecycle.started omitted: default behaviour, return immediately without waiting. - return waiter.Id, nil, nil + warehouse, err := waiter.Get() + if err != nil { + return "", nil, err } + return warehouse.Id, warehouse, nil } // DoUpdate updates the warehouse in place. @@ -122,39 +97,6 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * log.Warnf(ctx, "sql_warehouses: response contains unexpected id=%#v (expected %#v)", waiter.Id, id) } - if config.Started == nil { - return nil, nil - } - - warehouse, err := r.client.Warehouses.GetById(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get warehouse %s: %w", id, err) - } - - if *config.Started { - // lifecycle.started=true: ensure the warehouse is running. - if warehouse.State == sql.StateStopped { - startWait, err := r.client.Warehouses.Start(ctx, sql.StartRequest{Id: id}) - if err != nil { - return nil, fmt.Errorf("failed to start warehouse %s: %w", id, err) - } - if _, err = startWait.Get(); err != nil { - return nil, fmt.Errorf("failed to wait for warehouse %s to start: %w", id, err) - } - } - } else { - // lifecycle.started=false: ensure the warehouse is stopped. - if warehouse.State != sql.StateStopped && warehouse.State != sql.StateStopping { - stopWait, err := r.client.Warehouses.Stop(ctx, sql.StopRequest{Id: id}) - if err != nil { - return nil, fmt.Errorf("failed to stop warehouse %s: %w", id, err) - } - if _, err = stopWait.Get(); err != nil { - return nil, fmt.Errorf("failed to wait for warehouse %s to stop: %w", id, err) - } - } - } - return nil, nil } @@ -162,10 +104,6 @@ func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string) error return r.client.Warehouses.DeleteById(ctx, oldID) } -func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, p *structpath.PathNode, change *ChangeDesc, _ *sql.GetWarehouseResponse) error { - if change.Action == deployplan.Update && p.Prefix(1).String() == "started" { - // started is lifecycle metadata, not an actual warehouse property. - change.Action = deployplan.Skip - } +func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, _ *structpath.PathNode, _ *ChangeDesc, _ *sql.GetWarehouseResponse) error { return nil } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index b4436b784e..119a77c9e9 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -25,7 +25,6 @@ var knownMissingInRemoteType = map[string][]string{ }, "clusters": { "apply_policy_default_values", - "started", }, "external_locations": { "skip_validation", @@ -41,9 +40,6 @@ var knownMissingInRemoteType = map[string][]string{ "route_optimized", "tags", }, - "sql_warehouses": { - "started", - }, "quality_monitors": { "skip_builtin_dashboard", "warehouse_id", diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index ff3c9ee047..c2530fcb29 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -206,9 +206,17 @@ func (s *FakeWorkspace) AppsUpsert(req Request, name string) Response { Message: "Application is running.", } - app.ComputeStatus = &apps.ComputeStatus{ - State: "ACTIVE", - Message: "App compute is active.", + // Respect no_compute query param: if true, start the app in STOPPED state. + if req.URL.Query().Get("no_compute") == "true" { + app.ComputeStatus = &apps.ComputeStatus{ + State: apps.ComputeStateStopped, + Message: "App compute is stopped.", + } + } else { + app.ComputeStatus = &apps.ComputeStatus{ + State: "ACTIVE", + Message: "App compute is active.", + } } app.Url = name + "-123.cloud.databricksapps.com" From 1b5919792938893f398f49f4431868314ad8f157 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 14:04:42 +0100 Subject: [PATCH 17/42] revert clusters,sql_warehouse --- bundle/direct/dresources/cluster.go | 30 +++++++-------------- bundle/direct/dresources/sql_warehouse.go | 33 +++++++---------------- 2 files changed, 19 insertions(+), 44 deletions(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 602d13c7c7..a8f78d12f9 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -16,10 +16,6 @@ import ( "github.com/databricks/databricks-sdk-go/service/compute" ) -type ClusterState struct { - compute.ClusterSpec -} - type ResourceCluster struct { client *databricks.WorkspaceClient } @@ -30,13 +26,11 @@ func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { } } -func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { - return &ClusterState{ - ClusterSpec: input.ClusterSpec, - } +func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.ClusterSpec { + return &input.ClusterSpec } -func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterState { +func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { spec := &compute.ClusterSpec{ ApplyPolicyDefaultValues: false, Autoscale: input.Autoscale, @@ -77,27 +71,27 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *ClusterStat if input.Spec != nil { spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues } - return &ClusterState{ClusterSpec: *spec} + return spec } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*compute.ClusterDetails, error) { return r.client.Clusters.GetByClusterId(ctx, id) } -func (r *ResourceCluster) DoCreate(ctx context.Context, config *ClusterState) (string, *compute.ClusterDetails, error) { - wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(&config.ClusterSpec)) +func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterSpec) (string, *compute.ClusterDetails, error) { + wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(config)) if err != nil { return "", nil, err } return wait.ClusterId, nil, nil } -func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *ClusterState, _ Changes) (*compute.ClusterDetails, error) { +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ Changes) (*compute.ClusterDetails, error) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 timeout := 15 * time.Minute _, err := retries.Poll(ctx, timeout, func() (*compute.WaitGetClusterRunning[struct{}], *retries.Err) { - wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, &config.ClusterSpec)) + wait, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, config)) if err == nil { return wait, nil } @@ -110,14 +104,10 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *Clust } return nil, retries.Halt(err) }) - if err != nil { - return nil, err - } - - return nil, nil + return nil, err } -func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState) error { +func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ ClusterId: id, NumWorkers: config.NumWorkers, diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index d1987d6100..5d9d7793b7 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -5,16 +5,11 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/sql" ) -type SqlWarehouseState struct { - sql.CreateWarehouseRequest -} - type ResourceSqlWarehouse struct { client *databricks.WorkspaceClient } @@ -25,14 +20,12 @@ func (*ResourceSqlWarehouse) New(client *databricks.WorkspaceClient) *ResourceSq } // PrepareState converts bundle config to the SDK type. -func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWarehouseState { - return &SqlWarehouseState{ - CreateWarehouseRequest: input.CreateWarehouseRequest, - } +func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.CreateWarehouseRequest { + return &input.CreateWarehouseRequest } -func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *SqlWarehouseState { - return &SqlWarehouseState{CreateWarehouseRequest: sql.CreateWarehouseRequest{ +func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest { + return &sql.CreateWarehouseRequest{ AutoStopMins: warehouse.AutoStopMins, Channel: warehouse.Channel, ClusterSize: warehouse.ClusterSize, @@ -47,7 +40,7 @@ func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *Sq Tags: warehouse.Tags, WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), - }} + } } // DoRead reads the warehouse by id. @@ -56,20 +49,16 @@ func (r *ResourceSqlWarehouse) DoRead(ctx context.Context, id string) (*sql.GetW } // DoCreate creates the warehouse and returns its id. -func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *SqlWarehouseState) (string, *sql.GetWarehouseResponse, error) { - waiter, err := r.client.Warehouses.Create(ctx, config.CreateWarehouseRequest) +func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateWarehouseRequest) (string, *sql.GetWarehouseResponse, error) { + waiter, err := r.client.Warehouses.Create(ctx, *config) if err != nil { return "", nil, err } - warehouse, err := waiter.Get() - if err != nil { - return "", nil, err - } - return warehouse.Id, warehouse, nil + return waiter.Id, nil, nil } // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, _ Changes) (*sql.GetWarehouseResponse, error) { +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ Changes) (*sql.GetWarehouseResponse, error) { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, @@ -103,7 +92,3 @@ func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config * func (r *ResourceSqlWarehouse) DoDelete(ctx context.Context, oldID string) error { return r.client.Warehouses.DeleteById(ctx, oldID) } - -func (*ResourceSqlWarehouse) OverrideChangeDesc(_ context.Context, _ *structpath.PathNode, _ *ChangeDesc, _ *sql.GetWarehouseResponse) error { - return nil -} From cb2af59442d22cba5a537188321f851fbfaa5290 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 14:06:59 +0100 Subject: [PATCH 18/42] fix schema --- bundle/schema/jsonschema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 32c8f44518..95092d4f59 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -416,7 +416,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, "node_type_id": { "description": "This field encodes, through a single value, the resources available to each of\nthe Spark nodes in this cluster. For example, the Spark nodes can be provisioned\nand optimized for memory or compute intensive workloads. A list of available node\ntypes can be retrieved by using the :method:clusters/listNodeTypes API call.", @@ -1828,7 +1828,7 @@ }, "lifecycle": { "description": "Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.", - "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.LifecycleWithStarted" + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, "max_num_clusters": { "description": "Maximum number of clusters that the autoscaler will create to handle\nconcurrent queries.\n\nSupported values:\n- Must be \u003e= min_num_clusters\n- Must be \u003c= 40.\n\nDefaults to min_clusters if unset.", From 1b3d1a4b5331ce89cdac84303fd2d3f308215845 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 14:14:48 +0100 Subject: [PATCH 19/42] fix + fmt --- bundle/direct/dresources/app.go | 1 - bundle/direct/dresources/app_test.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index ec7b958359..d94b5e7d32 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -246,7 +246,6 @@ var localOnlyFields = map[string]bool{ "started": true, } - func isComputeStopped(app *apps.App) bool { return app.ComputeStatus == nil || app.ComputeStatus.State == apps.ComputeStateStopped || diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 35340440f5..237a596093 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -35,8 +35,8 @@ func TestAppStateMarshalUnmarshal(t *testing.T) { var restored AppState require.NoError(t, json.Unmarshal(data, &restored)) - assert.Equal(t, original.App.Name, restored.App.Name) - assert.Equal(t, original.App.Description, restored.App.Description) + assert.Equal(t, original.Name, restored.Name) + assert.Equal(t, original.Description, restored.Description) assert.Equal(t, original.SourceCodePath, restored.SourceCodePath) assert.Equal(t, original.Config, restored.Config) assert.Equal(t, original.Started, restored.Started) From 6580b5c04ba811b14695d14f0eae48b97f9b252a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 14:17:44 +0100 Subject: [PATCH 20/42] fix comment --- bundle/deploy/terraform/tfdyn/convert_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/deploy/terraform/tfdyn/convert_cluster.go b/bundle/deploy/terraform/tfdyn/convert_cluster.go index 3a2439014a..e53b22a38d 100644 --- a/bundle/deploy/terraform/tfdyn/convert_cluster.go +++ b/bundle/deploy/terraform/tfdyn/convert_cluster.go @@ -29,7 +29,7 @@ func (clusterConverter) Convert(ctx context.Context, key string, vin dyn.Value, return err } - // Always skip wait during creation; lifecycle.started is only supported in direct mode. + // We always set no_wait as it allows DABs not to wait for cluster to be started. vout, err = dyn.Set(vout, "no_wait", dyn.V(true)) if err != nil { return err From 32480a6f6c26c73351e9fe66a57662809b38f205 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 14:55:29 +0100 Subject: [PATCH 21/42] fixed output --- .../bundle/resources/apps/lifecycle-started-toggle/output.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index 173d96d1bc..d1d37d7b53 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -27,7 +27,6 @@ Deployment complete! >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... -Warn: unexpected local and remote diffs (bool, *bool); entry=&{ false true } ch={started 0x1400030b4ac} Deploying resources... ✓ Deployment succeeded. Updating deployment state... @@ -56,7 +55,6 @@ Deployment complete! >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files... -Warn: unexpected local and remote diffs (bool, *bool); entry=&{ true false } ch={started 0x1400013e4a4} Deploying resources... Updating deployment state... Deployment complete! From db4cc352f5b3a09d6e5f043c1a8e9bd0a78e4c9b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 15:10:26 +0100 Subject: [PATCH 22/42] fixed app test output --- acceptance/bundle/apps/git_source/output.txt | 5 ++++- acceptance/bundle/refschema/out.fields.txt | 6 ------ .../apps/current_can_manage/out.plan.direct.json | 3 ++- .../permissions/apps/other_can_manage/out.plan.direct.json | 3 ++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/apps/git_source/output.txt b/acceptance/bundle/apps/git_source/output.txt index 6779647f8a..dee5a96faf 100644 --- a/acceptance/bundle/apps/git_source/output.txt +++ b/acceptance/bundle/apps/git_source/output.txt @@ -44,7 +44,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> cat tmp.app-run ✓ Getting the status of the app [APP_NAME] ✓ App is in RUNNING state -✓ App compute is in ACTIVE state +✓ App compute is in STOPPED state +✓ Starting the app [APP_NAME] +✓ App is starting... +✓ App is started! ✓ Deployment succeeded You can access the app at [APP_NAME]-123.cloud.databricksapps.com diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 61e3f3fe1f..ed94995760 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -379,9 +379,7 @@ resources.clusters.*.kind compute.Kind ALL resources.clusters.*.last_restarted_time int64 REMOTE resources.clusters.*.last_state_loss_time int64 REMOTE resources.clusters.*.lifecycle resources.Lifecycle INPUT -resources.clusters.*.lifecycle resources.LifecycleWithStarted INPUT resources.clusters.*.lifecycle.prevent_destroy bool INPUT -resources.clusters.*.lifecycle.started *bool INPUT resources.clusters.*.modified_status string INPUT resources.clusters.*.node_type_id string ALL resources.clusters.*.num_workers int ALL @@ -506,7 +504,6 @@ resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE resources.clusters.*.ssh_public_keys []string ALL resources.clusters.*.ssh_public_keys[*] string ALL resources.clusters.*.start_time int64 REMOTE -resources.clusters.*.started *bool STATE resources.clusters.*.state compute.State REMOTE resources.clusters.*.state_message string REMOTE resources.clusters.*.terminated_time int64 REMOTE @@ -2795,9 +2792,7 @@ resources.sql_warehouses.*.id string INPUT REMOTE resources.sql_warehouses.*.instance_profile_arn string ALL resources.sql_warehouses.*.jdbc_url string REMOTE resources.sql_warehouses.*.lifecycle resources.Lifecycle INPUT -resources.sql_warehouses.*.lifecycle resources.LifecycleWithStarted INPUT resources.sql_warehouses.*.lifecycle.prevent_destroy bool INPUT -resources.sql_warehouses.*.lifecycle.started *bool INPUT resources.sql_warehouses.*.max_num_clusters int ALL resources.sql_warehouses.*.min_num_clusters int ALL resources.sql_warehouses.*.modified_status string INPUT @@ -2810,7 +2805,6 @@ resources.sql_warehouses.*.odbc_params.path string REMOTE resources.sql_warehouses.*.odbc_params.port int REMOTE resources.sql_warehouses.*.odbc_params.protocol string REMOTE resources.sql_warehouses.*.spot_instance_policy sql.SpotInstancePolicy ALL -resources.sql_warehouses.*.started *bool STATE resources.sql_warehouses.*.state sql.State REMOTE resources.sql_warehouses.*.tags *sql.EndpointTags ALL resources.sql_warehouses.*.tags.custom_tags []sql.EndpointTagPair ALL diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json index a970eafec7..40c12a2c54 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json @@ -7,7 +7,8 @@ "new_state": { "value": { "description": "", - "name": "foo" + "name": "foo", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json index a970eafec7..40c12a2c54 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json @@ -7,7 +7,8 @@ "new_state": { "value": { "description": "", - "name": "foo" + "name": "foo", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files" } } }, From 88b4313e26a83e720f65cce01da33bfa9be9609e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 17 Mar 2026 16:54:27 +0100 Subject: [PATCH 23/42] test app output --- acceptance/apps/deploy/bundle-no-args/output.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/apps/deploy/bundle-no-args/output.txt b/acceptance/apps/deploy/bundle-no-args/output.txt index 18509ae92f..d6c98f8535 100644 --- a/acceptance/apps/deploy/bundle-no-args/output.txt +++ b/acceptance/apps/deploy/bundle-no-args/output.txt @@ -7,7 +7,10 @@ Updating deployment state... Deployment complete! ✓ Getting the status of the app myapp ✓ App is in RUNNING state -✓ App compute is in ACTIVE state +✓ App compute is in STOPPED state +✓ Starting the app myapp +✓ App is starting... +✓ App is started! ✓ Deployment succeeded You can access the app at myapp-123.cloud.databricksapps.com ✔ Deployment complete! From b0a761aa5ce30e2e34423e6269fc7b4e89f77e99 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 20 Mar 2026 14:12:08 +0100 Subject: [PATCH 24/42] addressed feedback --- .../app/app.py | 1 + .../databricks.yml.tmpl | 11 +++ .../output.txt | 16 ++++ .../lifecycle-started-terraform-error/script | 7 ++ .../test.toml | 5 ++ .../lifecycle-started/out.deploy.direct.txt | 75 ---------------- .../out.deploy.terraform.txt | 56 ------------ .../lifecycle-started/out.destroy.direct.txt | 9 -- .../out.destroy.terraform.txt | 3 - .../apps/lifecycle-started/out.test.toml | 2 +- .../apps/lifecycle-started/output.txt | 88 +++++++++++++++++++ .../resources/apps/lifecycle-started/script | 30 ++++--- .../apps/lifecycle-started/test.toml | 2 +- .../mutator/validate_lifecycle_started.go | 4 +- bundle/direct/dresources/app.go | 84 ++++-------------- bundle/direct/dresources/app_test.go | 2 +- 16 files changed, 168 insertions(+), 227 deletions(-) create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml delete mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt delete mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt delete mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt delete mode 100644 acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl new file mode 100644 index 0000000000..ee78dfc150 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: lifecycle-started-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt new file mode 100644 index 0000000000..80395fe51a --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/output.txt @@ -0,0 +1,16 @@ + +=== bundle plan fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle plan +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 + + +Exit code: 1 + +=== bundle deploy fails with lifecycle.started on terraform engine +>>> errcode [CLI] bundle deploy +Error: lifecycle.started is only supported in direct deployment mode + in databricks.yml:11:18 + + +Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script new file mode 100644 index 0000000000..4fb2038cd2 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/script @@ -0,0 +1,7 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "bundle plan fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle plan + +title "bundle deploy fails with lifecycle.started on terraform engine" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml new file mode 100644 index 0000000000..a9f28de48a --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt deleted file mode 100644 index c1ed9e6fce..0000000000 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.direct.txt +++ /dev/null @@ -1,75 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps", - "body": { - "description": "my_app_description", - "name": "[UNIQUE_NAME]" - } -} - ->>> errcode [CLI] apps get [UNIQUE_NAME] -"ACTIVE" - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -✓ Deployment succeeded. -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", - "body": { - "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" - } -} - ->>> errcode [CLI] apps stop [UNIQUE_NAME] -"STOPPED" - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt - ->>> errcode [CLI] apps get [UNIQUE_NAME] -"STOPPED" - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... -Deploying resources... -✓ Deployment succeeded. -Updating deployment state... -Deployment complete! - ->>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/start", - "body": {} -} -{ - "method": "POST", - "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", - "body": { - "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" - } -} - ->>> errcode [CLI] apps get [UNIQUE_NAME] -"ACTIVE" diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt deleted file mode 100644 index c65dabe49b..0000000000 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.deploy.terraform.txt +++ /dev/null @@ -1,56 +0,0 @@ - ->>> errcode [CLI] bundle deploy -Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 - - -Exit code: 1 - ->>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt - ->>> errcode [CLI] apps get [UNIQUE_NAME] -Error: Resource apps.App not found: [UNIQUE_NAME] - -Exit code: 1 - ->>> errcode [CLI] bundle deploy -Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 - - -Exit code: 1 - ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt - ->>> errcode [CLI] apps stop [UNIQUE_NAME] -Error: Not Found - -Exit code: 1 - ->>> errcode [CLI] bundle deploy -Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 - - -Exit code: 1 - ->>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt - ->>> errcode [CLI] apps get [UNIQUE_NAME] -Error: Resource apps.App not found: [UNIQUE_NAME] - -Exit code: 1 - ->>> errcode [CLI] bundle deploy -Error: lifecycle.started is only supported in direct deployment mode - in databricks.yml:11:18 - - -Exit code: 1 - ->>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt - ->>> errcode [CLI] apps get [UNIQUE_NAME] -Error: Resource apps.App not found: [UNIQUE_NAME] - -Exit code: 1 diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt deleted file mode 100644 index 8061857d58..0000000000 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.direct.txt +++ /dev/null @@ -1,9 +0,0 @@ - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.apps.myapp - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default - -Deleting files... -Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt b/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt deleted file mode 100644 index 16da688a34..0000000000 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.destroy.terraform.txt +++ /dev/null @@ -1,3 +0,0 @@ - ->>> [CLI] bundle destroy --auto-approve -No active deployment found to destroy! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index e4c769f3b4..19b2c349a3 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index 1b50a6a148..1ba62d6df0 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -1,10 +1,98 @@ +=== Deploy with started=true: app created with compute running +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and .path == "/api/2.0/apps") out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps", + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +=== Re-deploy with description change: code deployed again >>> update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +=== Stop app externally, then deploy with started=false: app stays stopped +>>> errcode [CLI] apps stop [UNIQUE_NAME] +"STOPPED" + >>> update_file.py databricks.yml started: true started: false >>> update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | endswith("/deployments"))) out.requests.txt + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"STOPPED" + +=== Deploy with started=true: compute restarted and code deployed >>> update_file.py databricks.yml started: false started: true >>> update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments")))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start", + "body": {} +} +{ + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default/files/app" + } +} + +>>> errcode [CLI] apps get [UNIQUE_NAME] +"ACTIVE" + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script index 5bc75f99f9..53d2bf5773 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/script +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -1,32 +1,36 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { - trace $CLI bundle destroy --auto-approve &> out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt } trap cleanup EXIT -{ trace errcode $CLI bundle deploy; } &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -{ trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +title "Deploy with started=true: app created with compute running" +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and .path == "/api/2.0/apps")' out.requests.txt; } || true rm -f out.requests.txt -{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true +title "Re-deploy with description change: code deployed again" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION -{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } || true rm -f out.requests.txt -{ trace errcode $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true +title "Stop app externally, then deploy with started=false: app stays stopped" +{ trace errcode $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state'; } || true trace update_file.py databricks.yml "started: true" "started: false" trace update_file.py databricks.yml MY_APP_DESCRIPTION MY_APP_DESCRIPTION_2 -{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | endswith("/deployments")))' out.requests.txt; } || true rm -f out.requests.txt -{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true +title "Deploy with started=true: compute restarted and code deployed" trace update_file.py databricks.yml "started: false" "started: true" trace update_file.py databricks.yml MY_APP_DESCRIPTION_2 MY_APP_DESCRIPTION_3 -{ trace errcode $CLI bundle deploy; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -{ trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode $CLI bundle deploy +{ trace jq 'select(.method == "POST" and (.path | (endswith("/start") or endswith("/deployments"))))' out.requests.txt; } || true rm -f out.requests.txt -{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true +{ trace errcode $CLI apps get $UNIQUE_NAME | jq '.compute_status.state'; } || true diff --git a/acceptance/bundle/resources/apps/lifecycle-started/test.toml b/acceptance/bundle/resources/apps/lifecycle-started/test.toml index 79997a76d4..bfe2b2f2a7 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/test.toml @@ -5,4 +5,4 @@ RecordRequests = true Ignore = [".databricks", "databricks.yml"] [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index c7be225f29..ee88e413c0 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -13,8 +13,8 @@ type validateLifecycleStarted struct { engine engine.EngineType } -// ValidateLifecycleStarted returns a mutator that validates lifecycle.started -// is only used on supported resource types (apps). +// ValidateLifecycleStarted returns a mutator that errors when lifecycle.started +// is used with the terraform deployment engine. // lifecycle.started is only supported in direct deployment mode. func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { return &validateLifecycleStarted{engine: e} diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index d94b5e7d32..8f5934db0c 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -2,9 +2,9 @@ package dresources import ( "context" - "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -27,63 +28,14 @@ type AppState struct { Started *bool `json:"started,omitempty"` } -// bundleOnlyFields are the AppState fields not present in apps.App. -// They must be explicitly marshaled/unmarshaled since apps.App's promoted MarshalJSON -// would otherwise shadow them. -type bundleOnlyFields struct { - SourceCodePath string `json:"source_code_path,omitempty"` - Config *resources.AppConfig `json:"config,omitempty"` - GitSource *apps.GitSource `json:"git_source,omitempty"` - Started *bool `json:"started,omitempty"` +// Custom marshaler needed because embedded apps.App has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *AppState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) } -// MarshalJSON merges apps.App fields with bundle-only fields into a single JSON object. -// apps.App promotes its own MarshalJSON to AppState, which would otherwise omit bundle-only fields. func (s AppState) MarshalJSON() ([]byte, error) { - appJSON, err := json.Marshal(s.App) - if err != nil { - return nil, err - } - extrasJSON, err := json.Marshal(bundleOnlyFields{ - SourceCodePath: s.SourceCodePath, - Config: s.Config, - GitSource: s.GitSource, - Started: s.Started, - }) - if err != nil { - return nil, err - } - var result, extras map[string]json.RawMessage - if err := json.Unmarshal(appJSON, &result); err != nil { - return nil, err - } - if result == nil { - result = make(map[string]json.RawMessage) - } - if err := json.Unmarshal(extrasJSON, &extras); err != nil { - return nil, err - } - for k, v := range extras { - result[k] = v - } - return json.Marshal(result) -} - -// UnmarshalJSON sets apps.App fields and bundle-only fields from a single JSON object. -// apps.App promotes its own UnmarshalJSON to *AppState, which would otherwise drop bundle-only fields. -func (s *AppState) UnmarshalJSON(data []byte) error { - if err := json.Unmarshal(data, &s.App); err != nil { - return err - } - var e bundleOnlyFields - if err := json.Unmarshal(data, &e); err != nil { - return err - } - s.SourceCodePath = e.SourceCodePath - s.Config = e.Config - s.GitSource = e.GitSource - s.Started = e.Started - return nil + return marshal.Marshal(s) } type ResourceApp struct { @@ -107,13 +59,13 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { // RemapState maps the remote apps.App to AppState for diff comparison. // Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, // so they default to zero values, which prevents false drift detection. +// Started is computed from the remote compute status so the planner can determine +// whether the app needs to be started or stopped. func (*ResourceApp) RemapState(remote *apps.App) *AppState { + started := !isComputeStopped(remote) return &AppState{ - App: *remote, - SourceCodePath: "", - Config: nil, - GitSource: nil, - Started: nil, + App: *remote, + Started: &started, } } @@ -170,6 +122,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, maskPaths = append(maskPaths, path) } } + slices.Sort(maskPaths) updateMask := strings.Join(maskPaths, ",") if updateMask != "" { @@ -197,14 +150,13 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, return nil, nil } - app, err := r.client.Apps.GetByName(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to get app %s: %w", id, err) - } + // The planner computes the remote started value in RemapState based on compute status, + // so changes["started"].Action == Update means the compute state differs from the desired state. + startedChange := changes["started"] if *config.Started { // lifecycle.started=true: ensure the app compute is running and deploy the latest code. - if isComputeStopped(app) { + if startedChange != nil && startedChange.Action == deployplan.Update { startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) if err != nil { return nil, fmt.Errorf("failed to start app %s: %w", id, err) @@ -223,7 +175,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } } else { // lifecycle.started=false: ensure the app compute is stopped. - if !isComputeStopped(app) { + if startedChange != nil && startedChange.Action == deployplan.Update { stopWaiter, err := r.client.Apps.Stop(ctx, apps.StopAppRequest{Name: id}) if err != nil { return nil, fmt.Errorf("failed to stop app %s: %w", id, err) diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 237a596093..2ffc5a2707 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -14,7 +14,7 @@ import ( // TestAppStateMarshalUnmarshal verifies that AppState correctly preserves bundle-only fields // (SourceCodePath, Config, GitSource, Started) through a JSON round-trip. -// apps.App promotes its MarshalJSON/UnmarshalJSON which would otherwise drop these fields. +// Without the custom marshaler, apps.App's promoted MarshalJSON would drop these extra fields. func TestAppStateMarshalUnmarshal(t *testing.T) { started := true original := AppState{ From fd81054c3cfaf75590d4d8a3b239c50d0ff85b6c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 20 Mar 2026 14:34:47 +0100 Subject: [PATCH 25/42] fix lint and test --- .../apps/lifecycle-started-terraform-error/out.test.toml | 5 +++++ .../apps/lifecycle-started-terraform-error/test.toml | 1 + bundle/direct/dresources/app.go | 7 +++++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml new file mode 100644 index 0000000000..a9f28de48a --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml index a9f28de48a..34b19e94a9 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +RecordRequests = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 8f5934db0c..fe3a6634b5 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -64,8 +64,11 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { func (*ResourceApp) RemapState(remote *apps.App) *AppState { started := !isComputeStopped(remote) return &AppState{ - App: *remote, - Started: &started, + App: *remote, + Started: &started, + SourceCodePath: "", + Config: &resources.AppConfig{}, + GitSource: &apps.GitSource{}, } } From 58b5fbe5f0535d98086d73dbfe1f6183dcde61df Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 20 Mar 2026 14:39:39 +0100 Subject: [PATCH 26/42] lint --- bundle/direct/dresources/app.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index fe3a6634b5..c725568a61 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -67,8 +67,8 @@ func (*ResourceApp) RemapState(remote *apps.App) *AppState { App: *remote, Started: &started, SourceCodePath: "", - Config: &resources.AppConfig{}, - GitSource: &apps.GitSource{}, + Config: nil, + GitSource: nil, } } From ab3fdbb158ccf0f38c92547409a3eb44077dbedb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 13:20:11 +0100 Subject: [PATCH 27/42] fixed started field --- acceptance/bundle/refschema/out.fields.txt | 4 +- bundle/direct/dresources/app.go | 87 ++++++++++++++-------- bundle/direct/dresources/app_test.go | 6 +- bundle/direct/dresources/type_test.go | 6 +- 4 files changed, 65 insertions(+), 38 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index ed94995760..1d414787ea 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -122,10 +122,11 @@ resources.apps.*.git_source.resolved_commit string INPUT STATE resources.apps.*.git_source.source_code_path string INPUT STATE resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL +resources.apps.*.lifecycle dresources.AppStateLifecycle REMOTE STATE resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT -resources.apps.*.lifecycle.started *bool INPUT +resources.apps.*.lifecycle.started *bool ALL resources.apps.*.modified_status string INPUT resources.apps.*.name string ALL resources.apps.*.oauth2_app_client_id string ALL @@ -201,7 +202,6 @@ resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL resources.apps.*.source_code_path string INPUT STATE resources.apps.*.space string ALL -resources.apps.*.started *bool STATE resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*].unity_catalog *apps.UnityCatalog ALL diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index c725568a61..4ff9f638e1 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -18,17 +18,29 @@ import ( "github.com/databricks/databricks-sdk-go/service/apps" ) +// AppStateLifecycle holds lifecycle settings persisted in state. +type AppStateLifecycle struct { + Started *bool `json:"started,omitempty"` +} + // AppState is the state type for App resources. It extends apps.App with fields // needed for app deployments (Apps.Deploy) that are not part of the remote state. type AppState struct { apps.App SourceCodePath string `json:"source_code_path,omitempty"` - Config *resources.AppConfig `json:"config,omitempty"` - GitSource *apps.GitSource `json:"git_source,omitempty"` - Started *bool `json:"started,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` } -// Custom marshaler needed because embedded apps.App has its own MarshalJSON +// AppRemote extends apps.App with lifecycle.started so that it appears in +// RemoteType and can be used for $resource resolution. +type AppRemote struct { + apps.App + Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` +} + +// Custom marshalers needed because embedded apps.App has its own MarshalJSON // which would otherwise take over and ignore the additional fields. func (s *AppState) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) @@ -38,6 +50,14 @@ func (s AppState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } +func (r *AppRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, r) +} + +func (r AppRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(r) +} + type ResourceApp struct { client *databricks.WorkspaceClient } @@ -52,34 +72,33 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, - Started: input.Lifecycle.Started, + Lifecycle: AppStateLifecycle{Started: input.Lifecycle.Started}, } } -// RemapState maps the remote apps.App to AppState for diff comparison. +// RemapState maps the remote AppRemote to AppState for diff comparison. // Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, // so they default to zero values, which prevents false drift detection. -// Started is computed from the remote compute status so the planner can determine -// whether the app needs to be started or stopped. -func (*ResourceApp) RemapState(remote *apps.App) *AppState { - started := !isComputeStopped(remote) +func (*ResourceApp) RemapState(remote *AppRemote) *AppState { return &AppState{ - App: *remote, - Started: &started, - SourceCodePath: "", - Config: nil, - GitSource: nil, + App: remote.App, + Lifecycle: remote.Lifecycle, } } -func (r *ResourceApp) DoRead(ctx context.Context, id string) (*apps.App, error) { - return r.client.Apps.GetByName(ctx, id) +func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) { + app, err := r.client.Apps.GetByName(ctx, id) + if err != nil { + return nil, err + } + started := !isComputeStopped(app) + return &AppRemote{App: *app, Lifecycle: AppStateLifecycle{Started: &started}}, nil } -func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *apps.App, error) { +func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *AppRemote, error) { // Start app compute only when lifecycle.started=true is explicit. // For nil (omitted) or false, use no_compute=true (do not start compute). - noCompute := config.Started == nil || !*config.Started + noCompute := config.Lifecycle.Started == nil || !*config.Lifecycle.Started request := apps.CreateAppRequest{ App: config.App, NoCompute: noCompute, @@ -117,7 +136,7 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*apps.App, error) { +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, changes Changes) (*AppRemote, error) { // Build update mask excluding local-only fields that have no counterpart in the API. var maskPaths []string for path, change := range changes { @@ -149,15 +168,15 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } } - if config.Started == nil { + if config.Lifecycle.Started == nil { return nil, nil } // The planner computes the remote started value in RemapState based on compute status, - // so changes["started"].Action == Update means the compute state differs from the desired state. - startedChange := changes["started"] + // so changes["lifecycle.started"].Action == Update means the compute state differs from the desired state. + startedChange := changes["lifecycle.started"] - if *config.Started { + if *config.Lifecycle.Started { // lifecycle.started=true: ensure the app compute is running and deploy the latest code. if startedChange != nil && startedChange.Action == deployplan.Update { startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) @@ -195,10 +214,11 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, // localOnlyFields are AppState fields that have no counterpart in the remote state. // They must not appear in the App update_mask. var localOnlyFields = map[string]bool{ - "source_code_path": true, - "config": true, - "git_source": true, - "started": true, + "source_code_path": true, + "config": true, + "git_source": true, + "lifecycle": true, + "lifecycle.started": true, } func isComputeStopped(app *apps.App) bool { @@ -212,7 +232,7 @@ func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { return err } -func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*apps.App, error) { +func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*AppRemote, error) { return r.waitForApp(ctx, r.client, config.Name) } @@ -221,9 +241,9 @@ func (r *ResourceApp) WaitAfterCreate(ctx context.Context, config *AppState) (*a // We can't use the default waiter from SDK because it only waits on ACTIVE state but we need also STOPPED state. // Ideally this should be done in Go SDK but currently only ACTIVE is marked as terminal state // so this would need to be addressed by Apps service team first in their proto. -func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceClient, name string) (*apps.App, error) { +func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceClient, name string) (*AppRemote, error) { retrier := retries.New[apps.App](retries.WithTimeout(-1), retries.WithRetryFunc(shouldRetry)) - return retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { + app, err := retrier.Run(ctx, func(ctx context.Context) (*apps.App, error) { app, err := w.Apps.GetByName(ctx, name) if err != nil { return nil, retries.Halt(err) @@ -241,4 +261,9 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli return nil, retries.Continues(statusMessage) } }) + if err != nil { + return nil, err + } + started := !isComputeStopped(app) + return &AppRemote{App: *app, Lifecycle: AppStateLifecycle{Started: &started}}, nil } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 2ffc5a2707..6ccf2a9e5d 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -13,7 +13,7 @@ import ( ) // TestAppStateMarshalUnmarshal verifies that AppState correctly preserves bundle-only fields -// (SourceCodePath, Config, GitSource, Started) through a JSON round-trip. +// (SourceCodePath, Config, GitSource, Lifecycle) through a JSON round-trip. // Without the custom marshaler, apps.App's promoted MarshalJSON would drop these extra fields. func TestAppStateMarshalUnmarshal(t *testing.T) { started := true @@ -26,7 +26,7 @@ func TestAppStateMarshalUnmarshal(t *testing.T) { Config: &resources.AppConfig{ Command: []string{"python", "app.py"}, }, - Started: &started, + Lifecycle: AppStateLifecycle{Started: &started}, } data, err := json.Marshal(original) @@ -39,7 +39,7 @@ func TestAppStateMarshalUnmarshal(t *testing.T) { assert.Equal(t, original.Description, restored.Description) assert.Equal(t, original.SourceCodePath, restored.SourceCodePath) assert.Equal(t, original.Config, restored.Config) - assert.Equal(t, original.Started, restored.Started) + assert.Equal(t, original.Lifecycle.Started, restored.Lifecycle.Started) } // TestAppDoCreate_RetriesWhenAppIsDeleting verifies that DoCreate retries when diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 119a77c9e9..d9d13db66b 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,12 +16,11 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ - // source_code_path, config, git_source, and started are bundle-specific deployment fields not present in the remote App state. + // source_code_path, config, and git_source are bundle-specific deployment fields not present in the remote App state. "apps": { "config", "git_source", "source_code_path", - "started", }, "clusters": { "apply_policy_default_values", @@ -98,6 +97,9 @@ var knownMissingInStateType = map[string][]string{ "alerts": { "file_path", }, + "apps": { + "lifecycle.prevent_destroy", + }, "dashboards": { "file_path", }, From ecc3c7058cb8dff7f3418aa65c462e822575843b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 15:02:07 +0100 Subject: [PATCH 28/42] fix --- bundle/direct/dresources/resources.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 81482a27f3..f7a3fc92c3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -376,7 +376,7 @@ resources: - field: name reason: immutable ignore_remote_changes: - # source_code_path, config, git_source, and started are bundle-local fields + # source_code_path, config, git_source, and lifecycle are bundle-local fields # not stored in remote state. Remote always returns empty; skip the spurious diff # when the local config hasn't changed these fields. - field: source_code_path @@ -385,7 +385,9 @@ resources: reason: local_only - field: git_source reason: local_only - - field: started + - field: lifecycle + reason: local_only + - field: lifecycle.started reason: local_only backend_defaults: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/internal/providers/pluginfw/products/app/resource_app.go#L41 From 90bf2d2299aa3d8e0c7bf185ac99913dda02d064 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 15:10:27 +0100 Subject: [PATCH 29/42] fixed lint --- bundle/direct/dresources/app.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 4ff9f638e1..6b8ebcd0b7 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -28,9 +28,9 @@ type AppStateLifecycle struct { type AppState struct { apps.App SourceCodePath string `json:"source_code_path,omitempty"` - Config *resources.AppConfig `json:"config,omitempty"` - GitSource *apps.GitSource `json:"git_source,omitempty"` - Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` } // AppRemote extends apps.App with lifecycle.started so that it appears in @@ -81,8 +81,11 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { // so they default to zero values, which prevents false drift detection. func (*ResourceApp) RemapState(remote *AppRemote) *AppState { return &AppState{ - App: remote.App, - Lifecycle: remote.Lifecycle, + App: remote.App, + SourceCodePath: "", + Config: nil, + GitSource: nil, + Lifecycle: remote.Lifecycle, } } From 197b576e25c593808418e70a0b151c769530231b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 17:00:27 +0100 Subject: [PATCH 30/42] fixed drift --- acceptance/bundle/refschema/out.fields.txt | 2 +- bundle/direct/dresources/app.go | 29 +++++++++++----------- bundle/direct/dresources/app_test.go | 3 ++- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 1d414787ea..ae571612bb 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -122,7 +122,7 @@ resources.apps.*.git_source.resolved_commit string INPUT STATE resources.apps.*.git_source.source_code_path string INPUT STATE resources.apps.*.git_source.tag string INPUT STATE resources.apps.*.id string ALL -resources.apps.*.lifecycle dresources.AppStateLifecycle REMOTE STATE +resources.apps.*.lifecycle *dresources.AppStateLifecycle REMOTE STATE resources.apps.*.lifecycle resources.Lifecycle INPUT resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 6b8ebcd0b7..9f272810e6 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -30,14 +30,14 @@ type AppState struct { SourceCodePath string `json:"source_code_path,omitempty"` Config *resources.AppConfig `json:"config,omitempty"` GitSource *apps.GitSource `json:"git_source,omitempty"` - Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } // AppRemote extends apps.App with lifecycle.started so that it appears in // RemoteType and can be used for $resource resolution. type AppRemote struct { apps.App - Lifecycle AppStateLifecycle `json:"lifecycle,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } // Custom marshalers needed because embedded apps.App has its own MarshalJSON @@ -67,25 +67,27 @@ func (*ResourceApp) New(client *databricks.WorkspaceClient) *ResourceApp { } func (*ResourceApp) PrepareState(input *resources.App) *AppState { - return &AppState{ + s := &AppState{ App: input.App, SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, - Lifecycle: AppStateLifecycle{Started: input.Lifecycle.Started}, } + if input.Lifecycle.Started != nil { + s.Lifecycle = &AppStateLifecycle{Started: input.Lifecycle.Started} + } + return s } // RemapState maps the remote AppRemote to AppState for diff comparison. // Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, // so they default to zero values, which prevents false drift detection. +// Started is derived from compute status so the planner can detect start/stop changes. func (*ResourceApp) RemapState(remote *AppRemote) *AppState { + started := !isComputeStopped(&remote.App) return &AppState{ - App: remote.App, - SourceCodePath: "", - Config: nil, - GitSource: nil, - Lifecycle: remote.Lifecycle, + App: remote.App, + Lifecycle: &AppStateLifecycle{Started: &started}, } } @@ -95,13 +97,13 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) return nil, err } started := !isComputeStopped(app) - return &AppRemote{App: *app, Lifecycle: AppStateLifecycle{Started: &started}}, nil + return &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}}, nil } func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *AppRemote, error) { // Start app compute only when lifecycle.started=true is explicit. // For nil (omitted) or false, use no_compute=true (do not start compute). - noCompute := config.Lifecycle.Started == nil || !*config.Lifecycle.Started + noCompute := config.Lifecycle == nil || config.Lifecycle.Started == nil || !*config.Lifecycle.Started request := apps.CreateAppRequest{ App: config.App, NoCompute: noCompute, @@ -171,7 +173,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } } - if config.Lifecycle.Started == nil { + if config.Lifecycle == nil || config.Lifecycle.Started == nil { return nil, nil } @@ -267,6 +269,5 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli if err != nil { return nil, err } - started := !isComputeStopped(app) - return &AppRemote{App: *app, Lifecycle: AppStateLifecycle{Started: &started}}, nil + return &AppRemote{App: *app}, nil } diff --git a/bundle/direct/dresources/app_test.go b/bundle/direct/dresources/app_test.go index 6ccf2a9e5d..5fef3e45c9 100644 --- a/bundle/direct/dresources/app_test.go +++ b/bundle/direct/dresources/app_test.go @@ -26,7 +26,7 @@ func TestAppStateMarshalUnmarshal(t *testing.T) { Config: &resources.AppConfig{ Command: []string{"python", "app.py"}, }, - Lifecycle: AppStateLifecycle{Started: &started}, + Lifecycle: &AppStateLifecycle{Started: &started}, } data, err := json.Marshal(original) @@ -39,6 +39,7 @@ func TestAppStateMarshalUnmarshal(t *testing.T) { assert.Equal(t, original.Description, restored.Description) assert.Equal(t, original.SourceCodePath, restored.SourceCodePath) assert.Equal(t, original.Config, restored.Config) + require.NotNil(t, restored.Lifecycle) assert.Equal(t, original.Lifecycle.Started, restored.Lifecycle.Started) } From 2d93e5f8b56c21dd0219fa01dfb830887e3d8854 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 17:04:53 +0100 Subject: [PATCH 31/42] fixed merged conflict --- bundle/direct/dresources/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 00104a7ac9..e1b3dd9bf8 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -179,7 +179,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, // The planner computes the remote started value in RemapState based on compute status, // so changes["lifecycle.started"].Action == Update means the compute state differs from the desired state. - startedChange := changes["lifecycle.started"] + startedChange := entry.Changes["lifecycle.started"] if *config.Lifecycle.Started { // lifecycle.started=true: ensure the app compute is running and deploy the latest code. From 9bf4eae9272b54df68197221ce831feae80502d8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 26 Mar 2026 17:17:29 +0100 Subject: [PATCH 32/42] fix lint --- bundle/direct/dresources/app.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index e1b3dd9bf8..1c5edb64c9 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -72,6 +72,7 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, + Lifecycle: nil, } if input.Lifecycle.Started != nil { s.Lifecycle = &AppStateLifecycle{Started: input.Lifecycle.Started} @@ -86,8 +87,11 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { func (*ResourceApp) RemapState(remote *AppRemote) *AppState { started := !isComputeStopped(&remote.App) return &AppState{ - App: remote.App, - Lifecycle: &AppStateLifecycle{Started: &started}, + App: remote.App, + SourceCodePath: "", + Config: nil, + GitSource: nil, + Lifecycle: &AppStateLifecycle{Started: &started}, } } @@ -269,5 +273,6 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli if err != nil { return nil, err } - return &AppRemote{App: *app}, nil + started := !isComputeStopped(app) + return &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}}, nil } From 8c21f2db0c2f4212dd609a4582869429bd5145f3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 27 Mar 2026 14:47:14 +0100 Subject: [PATCH 33/42] addressed feedback --- .../configs/app_lifecycle_ref.yml.tmpl | 15 +++ .../apps/lifecycle-started-omitted/app/app.py | 1 + .../databricks.yml.tmpl | 9 ++ .../lifecycle-started-omitted/out.test.toml | 5 + .../apps/lifecycle-started-omitted/output.txt | 81 ++++++++++++ .../apps/lifecycle-started-omitted/script | 85 +++++++++++++ .../apps/lifecycle-started-omitted/test.toml | 8 ++ bundle/direct/dresources/app.go | 92 +++++++++++--- bundle/direct/dresources/resources.yml | 23 ++-- bundle/direct/dresources/type_test.go | 3 - bundle/phases/plan.go | 32 +++-- bundle/phases/plan_test.go | 120 ++++++++---------- 12 files changed, 363 insertions(+), 111 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/script create mode 100644 acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml diff --git a/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl b/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl new file mode 100644 index 0000000000..f0f1e3cbc0 --- /dev/null +++ b/acceptance/bundle/invariant/configs/app_lifecycle_ref.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + apps: + foo: + name: app-foo-$UNIQUE_NAME + source_code_path: ./app + lifecycle: + started: true + bar: + name: app-bar-$UNIQUE_NAME + source_code_path: ./app + lifecycle: + started: ${resources.apps.foo.lifecycle.started} diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py b/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py new file mode 100644 index 0000000000..d56323cf53 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/app/app.py @@ -0,0 +1 @@ +print("Hello world\!") diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl b/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl new file mode 100644 index 0000000000..ca2eb952d0 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: lifecycle-started-omitted-$UNIQUE_NAME + +resources: + apps: + mykey: + name: $UNIQUE_NAME + description: my_app_description + source_code_path: ./app diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt new file mode 100644 index 0000000000..8f5e0b61bf --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/output.txt @@ -0,0 +1,81 @@ + +=== (started omitted) -> deploy: app created with no_compute=true, no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": { + "description": "my_app_description", + "name": "[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + } +} + +=== (started omitted) -> started: false -> deploy: no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== started: false -> (started omitted) -> deploy: no start/stop requests +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== (started omitted) -> started: true -> deploy: app started +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +>>> print_app_requests +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" +} + +=== started: true -> (started omitted) -> deploy: no start/stop requests (compute stays as-is) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_app_requests + +=== (started omitted, app running) -> bundle plan shows no driftPlan: no drift detected + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.mykey + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/lifecycle-started-omitted-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script new file mode 100644 index 0000000000..ca97563ffa --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -0,0 +1,85 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_app_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps")))' < out.requests.txt + rm out.requests.txt +} + +title "(started omitted) -> deploy: app created with no_compute=true, no start/stop requests" +trace $CLI bundle deploy +trace print_app_requests + +title "(started omitted) -> started: false -> deploy: no start/stop requests" +cat > databricks.yml < (started omitted) -> deploy: no start/stop requests" +cat > databricks.yml < started: true -> deploy: app started" +cat > databricks.yml < (started omitted) -> deploy: no start/stop requests (compute stays as-is)" +cat > databricks.yml < bundle plan shows no drift" +$CLI bundle plan -o json > LOG.planjson +verify_no_drift.py LOG.planjson +echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 1c5edb64c9..00770575a4 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -33,11 +33,13 @@ type AppState struct { Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } -// AppRemote extends apps.App with lifecycle.started so that it appears in -// RemoteType and can be used for $resource resolution. +// AppRemote extends apps.App with lifecycle.started and deployment fields so +// that they appear in RemoteType and can be used for $resource resolution and drift detection. type AppRemote struct { apps.App - Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } // Custom marshalers needed because embedded apps.App has its own MarshalJSON @@ -81,16 +83,17 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { } // RemapState maps the remote AppRemote to AppState for diff comparison. -// Deploy-only fields (SourceCodePath, Config, GitSource) are not in remote state, -// so they default to zero values, which prevents false drift detection. +// Config and GitSource are populated from the active deployment when one exists, +// enabling drift detection for out-of-band redeploys. +// SourceCodePath is not tracked for drift (it's a local-only deployment path). // Started is derived from compute status so the planner can detect start/stop changes. func (*ResourceApp) RemapState(remote *AppRemote) *AppState { started := !isComputeStopped(&remote.App) return &AppState{ App: remote.App, SourceCodePath: "", - Config: nil, - GitSource: nil, + Config: remote.Config, + GitSource: remote.GitSource, Lifecycle: &AppStateLifecycle{Started: &started}, } } @@ -101,7 +104,17 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) return nil, err } started := !isComputeStopped(app) - return &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}}, nil + remote := &AppRemote{ + App: *app, + Config: nil, + GitSource: nil, + Lifecycle: &AppStateLifecycle{Started: &started}, + } + if app.ActiveDeployment != nil { + remote.GitSource = app.ActiveDeployment.GitSource + remote.Config = deploymentToAppConfig(app.ActiveDeployment) + } + return remote, nil } func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, *AppRemote, error) { @@ -173,7 +186,7 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } if response.Status.State != apps.AppUpdateUpdateStatusUpdateStateSucceeded { - return nil, fmt.Errorf("failed to update app %s: %s", id, response.Status.Message) + return nil, fmt.Errorf("update status: %s: %s", response.Status.State, response.Status.Message) } } @@ -181,20 +194,19 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, return nil, nil } - // The planner computes the remote started value in RemapState based on compute status, - // so changes["lifecycle.started"].Action == Update means the compute state differs from the desired state. - startedChange := entry.Changes["lifecycle.started"] + desiredStarted := *config.Lifecycle.Started + remoteStarted := remoteIsStarted(entry) - if *config.Lifecycle.Started { + if desiredStarted { // lifecycle.started=true: ensure the app compute is running and deploy the latest code. - if startedChange != nil && startedChange.Action == deployplan.Update { + if !remoteStarted { startWaiter, err := r.client.Apps.Start(ctx, apps.StartAppRequest{Name: id}) if err != nil { - return nil, fmt.Errorf("failed to start app %s: %w", id, err) + return nil, err } startedApp, err := startWaiter.Get() if err != nil { - return nil, fmt.Errorf("failed to wait for app %s to start: %w", id, err) + return nil, err } if err := appdeploy.WaitForDeploymentToComplete(ctx, r.client, startedApp); err != nil { return nil, err @@ -206,13 +218,13 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, } } else { // lifecycle.started=false: ensure the app compute is stopped. - if startedChange != nil && startedChange.Action == deployplan.Update { + if remoteStarted { stopWaiter, err := r.client.Apps.Stop(ctx, apps.StopAppRequest{Name: id}) if err != nil { - return nil, fmt.Errorf("failed to stop app %s: %w", id, err) + return nil, err } if _, err = stopWaiter.Get(); err != nil { - return nil, fmt.Errorf("failed to wait for app %s to stop: %w", id, err) + return nil, err } } } @@ -236,6 +248,41 @@ func isComputeStopped(app *apps.App) bool { app.ComputeStatus.State == apps.ComputeStateError } +// remoteIsStarted reads the compute started state from the plan entry's remote state. +func remoteIsStarted(entry *PlanEntry) bool { + if entry.RemoteState == nil { + return false + } + remote, ok := entry.RemoteState.(*AppRemote) + if !ok || remote.Lifecycle == nil || remote.Lifecycle.Started == nil { + return false + } + return *remote.Lifecycle.Started +} + +// deploymentToAppConfig extracts an AppConfig from an active deployment. +// Returns nil if the deployment has no command or env vars. +func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { + if len(d.Command) == 0 && len(d.EnvVars) == 0 { + return nil + } + config := &resources.AppConfig{} + if len(d.Command) > 0 { + config.Command = d.Command + } + if len(d.EnvVars) > 0 { + config.Env = make([]resources.AppEnvVar, len(d.EnvVars)) + for i, ev := range d.EnvVars { + config.Env[i] = resources.AppEnvVar{ + Name: ev.Name, + Value: ev.Value, + ValueFrom: ev.ValueFrom, + } + } + } + return config +} + func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { _, err := r.client.Apps.DeleteByName(ctx, id) return err @@ -274,5 +321,10 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli return nil, err } started := !isComputeStopped(app) - return &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}}, nil + remote := &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}} + if app.ActiveDeployment != nil { + remote.GitSource = app.ActiveDeployment.GitSource + remote.Config = deploymentToAppConfig(app.ActiveDeployment) + } + return remote, nil } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index f7a3fc92c3..019ec6ada4 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -376,15 +376,11 @@ resources: - field: name reason: immutable ignore_remote_changes: - # source_code_path, config, git_source, and lifecycle are bundle-local fields - # not stored in remote state. Remote always returns empty; skip the spurious diff - # when the local config hasn't changed these fields. - field: source_code_path reason: local_only - - field: config - reason: local_only - - field: git_source - reason: local_only + # lifecycle fields are managed by the planner based on compute status. + # When lifecycle.started is omitted from config, it should be untracked + # and remote changes to compute state should not cause drift. - field: lifecycle reason: local_only - field: lifecycle.started @@ -393,19 +389,28 @@ resources: # https://github.com/databricks/terraform-provider-databricks/blob/4eba541abe1a9f50993ea7b9dd83874207e224a1/internal/providers/pluginfw/products/app/resource_app.go#L41 # s["compute_size"] = s["compute_size"].SetComputed() - field: compute_size - # Output-only fields assigned by the platform; never set by users in bundle config. - field: app_status + reason: output_only - field: compute_status + reason: output_only - field: url - - field: id + reason: output_only - field: service_principal_client_id + reason: output_only - field: service_principal_id + reason: output_only - field: service_principal_name + reason: output_only - field: create_time + reason: output_only - field: update_time + reason: output_only - field: creator + reason: output_only - field: updater + reason: output_only - field: effective_budget_policy_id + reason: output_only secret_scopes: backend_defaults: diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index d9d13db66b..7d4086e40a 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,10 +16,7 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ - // source_code_path, config, and git_source are bundle-specific deployment fields not present in the remote App state. "apps": { - "config", - "git_source", "source_code_path", }, "clusters": { diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index fd692c9614..0af9394f24 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" @@ -14,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/dyn" ) // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". @@ -33,27 +33,31 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { + root := b.Config.Value() var errs []error for _, action := range actions { if action.ActionType != deployplan.Recreate && action.ActionType != deployplan.Delete { continue } - // ResourceKey format: "resources.{type}.{key}" - parts := strings.SplitN(action.ResourceKey, ".", 3) - if len(parts) != 3 || parts[0] != "resources" { + path, err := dyn.NewPathFromString(action.ResourceKey) + if err != nil { + return fmt.Errorf("failed to parse %q", action.ResourceKey) + } + + path = append(path, dyn.Key("lifecycle"), dyn.Key("prevent_destroy")) + + preventDestroyV, err := dyn.GetByPath(root, path) + if err != nil { continue } - resourceType, resourceKey := parts[1], parts[2] - - for _, group := range b.Config.Resources.AllResources() { - if group.Description.PluralName != resourceType { - continue - } - if r, ok := group.Resources[resourceKey]; ok && r.GetLifecycle().HasPreventDestroy() { - errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) - } - break + + preventDestroy, ok := preventDestroyV.AsBool() + if !ok { + return fmt.Errorf("internal error: prevent_destroy is not a boolean for %s", action.ResourceKey) + } + if preventDestroy { + errs = append(errs, fmt.Errorf("%s has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for %s", action.ResourceKey, action.ResourceKey)) } } diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 1318b63912..73249b4e13 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -5,11 +5,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/databricks-sdk-go/service/apps" - "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/require" ) @@ -48,20 +45,19 @@ func TestCheckPreventDestroyForAllResources(t *testing.T) { } func TestCheckPreventDestroyForJob(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test_resource": { - BaseResource: resources.BaseResource{ - Lifecycle: resources.Lifecycle{PreventDestroy: true}, - }, - JobSettings: jobs.JobSettings{}, - }, - }, - }, - }, - } + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) actions := []deployplan.Action{ { @@ -70,7 +66,7 @@ func TestCheckPreventDestroyForJob(t *testing.T) { }, } - err := checkForPreventDestroy(b, actions) + err = checkForPreventDestroy(b, actions) require.Error(t, err) require.Contains(t, err.Error(), "resources.jobs.test_resource has lifecycle.prevent_destroy set") require.Contains(t, err.Error(), "but the plan calls for this resource to be recreated or destroyed") @@ -78,17 +74,19 @@ func TestCheckPreventDestroyForJob(t *testing.T) { } func TestCheckPreventDestroyForApp(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Apps: map[string]*resources.App{ - "test_resource": { - Lifecycle: resources.LifecycleWithStarted{Lifecycle: resources.Lifecycle{PreventDestroy: true}}, - }, - }, - }, - }, - } + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "apps": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) actions := []deployplan.Action{ { @@ -97,23 +95,21 @@ func TestCheckPreventDestroyForApp(t *testing.T) { }, } - err := checkForPreventDestroy(b, actions) + err = checkForPreventDestroy(b, actions) require.Error(t, err) require.Contains(t, err.Error(), "resources.apps.test_resource has lifecycle.prevent_destroy set") } func TestCheckPreventDestroyNoError(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test_resource": { - JobSettings: jobs.JobSettings{}, - }, - }, - }, - }, - } + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_resource": dyn.NewValue(map[string]dyn.Value{}, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) actions := []deployplan.Action{ { @@ -122,33 +118,27 @@ func TestCheckPreventDestroyNoError(t *testing.T) { }, } - err := checkForPreventDestroy(b, actions) + err = checkForPreventDestroy(b, actions) require.NoError(t, err) } func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Bundle: config.Bundle{ - Name: "test", - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "test_job": { - JobSettings: jobs.JobSettings{}, - }, - }, - Apps: map[string]*resources.App{ - "test_app": { - App: apps.App{ - Name: "Test App", - }, - Lifecycle: resources.LifecycleWithStarted{Lifecycle: resources.Lifecycle{PreventDestroy: true}}, - }, - }, - }, - }, - } + b := &bundle.Bundle{} + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.Set(v, "resources", dyn.NewValue(map[string]dyn.Value{ + "jobs": dyn.NewValue(map[string]dyn.Value{ + "test_job": dyn.NewValue(map[string]dyn.Value{}, nil), + }, nil), + "apps": dyn.NewValue(map[string]dyn.Value{ + "test_app": dyn.NewValue(map[string]dyn.Value{ + "lifecycle": dyn.NewValue(map[string]dyn.Value{ + "prevent_destroy": dyn.NewValue(true, nil), + }, nil), + }, nil), + }, nil), + }, nil)) + }) + require.NoError(t, err) actions := []deployplan.Action{ { @@ -161,7 +151,7 @@ func TestCheckForPreventDestroyWhenFirstHasNoPreventDestroy(t *testing.T) { }, } - err := checkForPreventDestroy(b, actions) + err = checkForPreventDestroy(b, actions) require.Error(t, err) require.Contains(t, err.Error(), "resources.apps.test_app has lifecycle.prevent_destroy set") } From 7b39509f3e150bb8d2b7a4d2dcfdeff6f5ffec15 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 27 Mar 2026 15:13:15 +0100 Subject: [PATCH 34/42] added test for changing config --- .../resources/apps/config-drift/app/app.py | 1 + .../apps/config-drift/databricks.yml.tmpl | 18 +++ .../apps/config-drift/out.plan.direct.json | 118 ++++++++++++++++++ .../resources/apps/config-drift/out.test.toml | 5 + .../resources/apps/config-drift/output.txt | 54 ++++++++ .../bundle/resources/apps/config-drift/script | 49 ++++++++ .../resources/apps/config-drift/test.toml | 8 ++ bundle/direct/dresources/app.go | 13 +- libs/testserver/apps.go | 5 +- 9 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 acceptance/bundle/resources/apps/config-drift/app/app.py create mode 100644 acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/apps/config-drift/out.plan.direct.json create mode 100644 acceptance/bundle/resources/apps/config-drift/out.test.toml create mode 100644 acceptance/bundle/resources/apps/config-drift/output.txt create mode 100644 acceptance/bundle/resources/apps/config-drift/script create mode 100644 acceptance/bundle/resources/apps/config-drift/test.toml diff --git a/acceptance/bundle/resources/apps/config-drift/app/app.py b/acceptance/bundle/resources/apps/config-drift/app/app.py new file mode 100644 index 0000000000..f1a18139c8 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/app/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl b/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl new file mode 100644 index 0000000000..b38d8f92f5 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: config-drift-$UNIQUE_NAME + +resources: + apps: + myapp: + name: $UNIQUE_NAME + description: my_app + source_code_path: ./app + config: + command: + - python + - app.py + env: + - name: MY_VAR + value: original_value + lifecycle: + started: true diff --git a/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json new file mode 100644 index 0000000000..009f17fed0 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json @@ -0,0 +1,118 @@ +{ + "active_deployment": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "command": [ + "streamlit", + "run", + "dashboard.py" + ], + "deployment_id": "deploy-[NUMID]", + "env_vars": [ + { + "name": "MY_VAR", + "value": "changed_value" + }, + { + "name": "NEW_VAR", + "value": "new_value" + } + ], + "mode": "SNAPSHOT", + "source_code_path": "./app", + "status": { + "message": "Deployment succeeded.", + "state": "SUCCEEDED" + } + } + }, + "app_status": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "message": "Application is running.", + "state": "RUNNING" + } + }, + "compute_status": { + "action": "skip", + "reason": "spec:output_only", + "remote": { + "message": "App compute is active.", + "state": "ACTIVE" + } + }, + "config.command": { + "action": "update", + "old": [ + "python", + "app.py" + ], + "new": [ + "python", + "app.py" + ], + "remote": [ + "streamlit", + "run", + "dashboard.py" + ] + }, + "config.env": { + "action": "update", + "old": [ + { + "name": "MY_VAR", + "value": "original_value" + } + ], + "new": [ + { + "name": "MY_VAR", + "value": "original_value" + } + ], + "remote": [ + { + "name": "MY_VAR", + "value": "changed_value" + }, + { + "name": "NEW_VAR", + "value": "new_value" + } + ] + }, + "id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "1000" + }, + "service_principal_client_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[UUID]" + }, + "service_principal_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": [NUMID] + }, + "service_principal_name": { + "action": "skip", + "reason": "spec:output_only", + "remote": "app-[UNIQUE_NAME]" + }, + "source_code_path": { + "action": "skip", + "reason": "local_only", + "old": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app", + "new": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app" + }, + "url": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[UNIQUE_NAME]-123.cloud.databricksapps.com" + } +} diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/output.txt b/acceptance/bundle/resources/apps/config-drift/output.txt new file mode 100644 index 0000000000..1965decd17 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/output.txt @@ -0,0 +1,54 @@ + +=== First deploy: creates app +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Second deploy: pushes code with config +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify no drift after deploy +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Simulate out-of-band deployment with changed command and env +=== Plan should detect config drift +>>> [CLI] bundle plan +update apps.myapp + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Redeploy to fix drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files... +Deploying resources... +✓ Deployment succeeded. +Updating deployment state... +Deployment complete! + +=== Verify no drift after fix +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Simulate out-of-band deployment with git_source added +=== Plan should detect git_source drift +>>> [CLI] bundle plan +update apps.myapp + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.myapp + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script new file mode 100644 index 0000000000..e37ac80fde --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -0,0 +1,49 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "First deploy: creates app" +trace $CLI bundle deploy + +title "Second deploy: pushes code with config" +trace $CLI bundle deploy + +title "Verify no drift after deploy" +trace $CLI bundle plan + +title "Simulate out-of-band deployment with changed command and env" +$CLI apps deploy $UNIQUE_NAME --no-wait --json '{ + "source_code_path": "./app", + "mode": "SNAPSHOT", + "command": ["streamlit", "run", "dashboard.py"], + "env_vars": [ + {"name": "MY_VAR", "value": "changed_value"}, + {"name": "NEW_VAR", "value": "new_value"} + ] + }' > /dev/null + +title "Plan should detect config drift" +trace $CLI bundle plan +$CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' > out.plan.direct.json + +title "Redeploy to fix drift" +trace $CLI bundle deploy + +title "Verify no drift after fix" +trace $CLI bundle plan + +title "Simulate out-of-band deployment with git_source added" +$CLI apps deploy $UNIQUE_NAME --no-wait --json '{ + "source_code_path": "./app", + "mode": "SNAPSHOT", + "git_source": {"branch": "feature-branch"}, + "command": ["python", "app.py"], + "env_vars": [{"name": "MY_VAR", "value": "original_value"}] + }' > /dev/null + +title "Plan should detect git_source drift" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/apps/config-drift/test.toml b/acceptance/bundle/resources/apps/config-drift/test.toml new file mode 100644 index 0000000000..bfe2b2f2a7 --- /dev/null +++ b/acceptance/bundle/resources/apps/config-drift/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", "databricks.yml"] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 00770575a4..5053c6db2a 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -266,9 +266,9 @@ func deploymentToAppConfig(d *apps.AppDeployment) *resources.AppConfig { if len(d.Command) == 0 && len(d.EnvVars) == 0 { return nil } - config := &resources.AppConfig{} - if len(d.Command) > 0 { - config.Command = d.Command + config := &resources.AppConfig{ + Command: d.Command, + Env: nil, } if len(d.EnvVars) > 0 { config.Env = make([]resources.AppEnvVar, len(d.EnvVars)) @@ -321,7 +321,12 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli return nil, err } started := !isComputeStopped(app) - remote := &AppRemote{App: *app, Lifecycle: &AppStateLifecycle{Started: &started}} + remote := &AppRemote{ + App: *app, + Config: nil, + GitSource: nil, + Lifecycle: &AppStateLifecycle{Started: &started}, + } if app.ActiveDeployment != nil { remote.GitSource = app.ActiveDeployment.GitSource remote.Config = deploymentToAppConfig(app.ActiveDeployment) diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index c2530fcb29..6b0d2a7b98 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -92,7 +92,7 @@ func (s *FakeWorkspace) AppsGetUpdate(_ Request, name string) Response { func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response { defer s.LockUnlock()() - _, ok := s.Apps[name] + app, ok := s.Apps[name] if !ok { return Response{StatusCode: 404} } @@ -108,6 +108,9 @@ func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response Message: "Deployment succeeded.", } + app.ActiveDeployment = &deployment + s.Apps[name] = app + return Response{Body: deployment} } From 5494b19e4834bed1fceb344d4af18c48b26adeda Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 27 Mar 2026 15:15:21 +0100 Subject: [PATCH 35/42] update refschema --- acceptance/bundle/refschema/out.fields.txt | 34 +++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index ae571612bb..191d98f156 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -93,14 +93,14 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.active_instances int ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL -resources.apps.*.config *resources.AppConfig INPUT STATE -resources.apps.*.config.command []string INPUT STATE -resources.apps.*.config.command[*] string INPUT STATE -resources.apps.*.config.env []resources.AppEnvVar INPUT STATE -resources.apps.*.config.env[*] resources.AppEnvVar INPUT STATE -resources.apps.*.config.env[*].name string INPUT STATE -resources.apps.*.config.env[*].value string INPUT STATE -resources.apps.*.config.env[*].value_from string INPUT STATE +resources.apps.*.config *resources.AppConfig ALL +resources.apps.*.config.command []string ALL +resources.apps.*.config.command[*] string ALL +resources.apps.*.config.env []resources.AppEnvVar ALL +resources.apps.*.config.env[*] resources.AppEnvVar ALL +resources.apps.*.config.env[*].name string ALL +resources.apps.*.config.env[*].value string ALL +resources.apps.*.config.env[*].value_from string ALL resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -112,15 +112,15 @@ resources.apps.*.effective_user_api_scopes[*] string ALL resources.apps.*.git_repository *apps.GitRepository ALL resources.apps.*.git_repository.provider string ALL resources.apps.*.git_repository.url string ALL -resources.apps.*.git_source *apps.GitSource INPUT STATE -resources.apps.*.git_source.branch string INPUT STATE -resources.apps.*.git_source.commit string INPUT STATE -resources.apps.*.git_source.git_repository *apps.GitRepository INPUT STATE -resources.apps.*.git_source.git_repository.provider string INPUT STATE -resources.apps.*.git_source.git_repository.url string INPUT STATE -resources.apps.*.git_source.resolved_commit string INPUT STATE -resources.apps.*.git_source.source_code_path string INPUT STATE -resources.apps.*.git_source.tag string INPUT STATE +resources.apps.*.git_source *apps.GitSource ALL +resources.apps.*.git_source.branch string ALL +resources.apps.*.git_source.commit string ALL +resources.apps.*.git_source.git_repository *apps.GitRepository ALL +resources.apps.*.git_source.git_repository.provider string ALL +resources.apps.*.git_source.git_repository.url string ALL +resources.apps.*.git_source.resolved_commit string ALL +resources.apps.*.git_source.source_code_path string ALL +resources.apps.*.git_source.tag string ALL resources.apps.*.id string ALL resources.apps.*.lifecycle *dresources.AppStateLifecycle REMOTE STATE resources.apps.*.lifecycle resources.Lifecycle INPUT From f2b107b658b3ee60697c2911c978182e14154212 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 27 Mar 2026 15:42:23 +0100 Subject: [PATCH 36/42] fixed git_source test --- acceptance/bundle/apps/git_source/output.txt | 17 ++++++--- acceptance/bundle/apps/git_source/script | 1 + acceptance/bundle/apps/git_source/test.toml | 39 -------------------- 3 files changed, 13 insertions(+), 44 deletions(-) diff --git a/acceptance/bundle/apps/git_source/output.txt b/acceptance/bundle/apps/git_source/output.txt index dee5a96faf..e9d7487bec 100644 --- a/acceptance/bundle/apps/git_source/output.txt +++ b/acceptance/bundle/apps/git_source/output.txt @@ -22,6 +22,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run my_app +✓ Getting the status of the app [APP_NAME] +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app [APP_NAME] +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at [APP_NAME]-123.cloud.databricksapps.com + === Get app details and verify git_source configuration >>> [CLI] bundle summary --output json @@ -44,11 +54,8 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> cat tmp.app-run ✓ Getting the status of the app [APP_NAME] ✓ App is in RUNNING state -✓ App compute is in STOPPED state -✓ Starting the app [APP_NAME] -✓ App is starting... -✓ App is started! -✓ Deployment succeeded +✓ App compute is in ACTIVE state +✓ Deployment succeeded. You can access the app at [APP_NAME]-123.cloud.databricksapps.com === Update git_source branch and redeploy diff --git a/acceptance/bundle/apps/git_source/script b/acceptance/bundle/apps/git_source/script index b1241ec40a..962033f0e5 100644 --- a/acceptance/bundle/apps/git_source/script +++ b/acceptance/bundle/apps/git_source/script @@ -17,6 +17,7 @@ trace $CLI bundle plan title "Deploy bundle" trace $CLI bundle deploy +trace $CLI bundle run my_app > /dev/null || true title "Get app details and verify git_source configuration" app_name=$(trace $CLI bundle summary --output json | jq -r '.resources.apps.my_app.name') diff --git a/acceptance/bundle/apps/git_source/test.toml b/acceptance/bundle/apps/git_source/test.toml index 819e361dcd..db551625c7 100644 --- a/acceptance/bundle/apps/git_source/test.toml +++ b/acceptance/bundle/apps/git_source/test.toml @@ -9,42 +9,3 @@ Ignore = [".databricks", "databricks.yml", "databricks.yml.bak", "tmp.app-run"] # Apps can take longer to deploy TimeoutCloud = "5m" - -# Mock responses for app deployment -[[Server]] -Pattern = "POST /api/2.0/apps/{app_name}/deployments" -Response.Body = ''' -{ - "deployment_id": "test-deployment-123", - "status": { - "state": "SUCCEEDED", - "message": "Deployment succeeded" - }, - "git_source": { - "branch": "main", - "source_code_path": "internal/testdata/simple-app", - "resolved_commit": "abc123def456" - }, - "source_code_path": "/Workspace/Users/tester@databricks.com/.bundle/files/app", - "mode": "SNAPSHOT" -} -''' - -[[Server]] -Pattern = "GET /api/2.0/apps/{app_name}/deployments/{deployment_id}" -Response.Body = ''' -{ - "deployment_id": "test-deployment-123", - "status": { - "state": "SUCCEEDED", - "message": "Deployment succeeded" - }, - "git_source": { - "branch": "main", - "source_code_path": "internal/testdata/simple-app", - "resolved_commit": "abc123def456" - }, - "source_code_path": "/Workspace/Users/tester@databricks.com/.bundle/files/app", - "mode": "SNAPSHOT" -} -''' From 96425adca940789c0aa3935ec6a29859b19ae740 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 11:56:20 +0200 Subject: [PATCH 37/42] fixes --- acceptance/bundle/refschema/out.fields.txt | 4 +- bundle/config/resources/apps.go | 7 ++- bundle/direct/dresources/app.go | 52 +++++++++++++--------- bundle/direct/dresources/resources.yml | 2 - bundle/direct/dresources/type_test.go | 3 -- bundle/direct/dresources/util.go | 14 ++++++ bundle/internal/validation/required.go | 8 ++-- 7 files changed, 58 insertions(+), 32 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 191d98f156..ecfab1562e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -123,8 +123,8 @@ resources.apps.*.git_source.source_code_path string ALL resources.apps.*.git_source.tag string ALL resources.apps.*.id string ALL resources.apps.*.lifecycle *dresources.AppStateLifecycle REMOTE STATE +resources.apps.*.lifecycle *resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle resources.Lifecycle INPUT -resources.apps.*.lifecycle resources.LifecycleWithStarted INPUT resources.apps.*.lifecycle.prevent_destroy bool INPUT resources.apps.*.lifecycle.started *bool ALL resources.apps.*.modified_status string INPUT @@ -200,7 +200,7 @@ resources.apps.*.resources[*].uc_securable.securable_type apps.AppResourceUcSecu resources.apps.*.service_principal_client_id string ALL resources.apps.*.service_principal_id int64 ALL resources.apps.*.service_principal_name string ALL -resources.apps.*.source_code_path string INPUT STATE +resources.apps.*.source_code_path string ALL resources.apps.*.space string ALL resources.apps.*.telemetry_export_destinations []apps.TelemetryExportDestination ALL resources.apps.*.telemetry_export_destinations[*] apps.TelemetryExportDestination ALL diff --git a/bundle/config/resources/apps.go b/bundle/config/resources/apps.go index 590029b3d1..f17aa4c22b 100644 --- a/bundle/config/resources/apps.go +++ b/bundle/config/resources/apps.go @@ -38,7 +38,7 @@ type App struct { // Note: apps.App already includes GitRepository field from the SDK // Lifecycle shadows BaseResource.Lifecycle to add support for lifecycle.started. - Lifecycle LifecycleWithStarted `json:"lifecycle,omitempty"` + Lifecycle *LifecycleWithStarted `json:"lifecycle,omitempty"` // SourceCodePath is a required field used by DABs to point to Databricks app source code // on local disk and to the corresponding workspace path during app deployment. @@ -58,7 +58,10 @@ type App struct { // GetLifecycle returns the lifecycle settings, using LifecycleWithStarted. func (a *App) GetLifecycle() LifecycleConfig { - return a.Lifecycle + if a.Lifecycle == nil { + return LifecycleWithStarted{} + } + return *a.Lifecycle } func (a *App) UnmarshalJSON(b []byte) error { diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 5053c6db2a..3ddb914094 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -23,8 +23,8 @@ type AppStateLifecycle struct { Started *bool `json:"started,omitempty"` } -// AppState is the state type for App resources. It extends apps.App with fields -// needed for app deployments (Apps.Deploy) that are not part of the remote state. +// AppState is the state type for App resources. It extends apps.App with deployment-related +// fields (source_code_path, config, git_source, lifecycle) that are persisted in state. type AppState struct { apps.App SourceCodePath string `json:"source_code_path,omitempty"` @@ -33,13 +33,14 @@ type AppState struct { Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } -// AppRemote extends apps.App with lifecycle.started and deployment fields so -// that they appear in RemoteType and can be used for $resource resolution and drift detection. +// AppRemote extends apps.App with the same deployment fields as AppState so they +// appear in RemoteType and can be used for $resource resolution and drift detection. type AppRemote struct { apps.App - Config *resources.AppConfig `json:"config,omitempty"` - GitSource *apps.GitSource `json:"git_source,omitempty"` - Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` + SourceCodePath string `json:"source_code_path,omitempty"` + Config *resources.AppConfig `json:"config,omitempty"` + GitSource *apps.GitSource `json:"git_source,omitempty"` + Lifecycle *AppStateLifecycle `json:"lifecycle,omitempty"` } // Custom marshalers needed because embedded apps.App has its own MarshalJSON @@ -74,24 +75,22 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, - Lifecycle: nil, } - if input.Lifecycle.Started != nil { + if input.Lifecycle != nil && input.Lifecycle.Started != nil { s.Lifecycle = &AppStateLifecycle{Started: input.Lifecycle.Started} } return s } // RemapState maps the remote AppRemote to AppState for diff comparison. -// Config and GitSource are populated from the active deployment when one exists, -// enabling drift detection for out-of-band redeploys. -// SourceCodePath is not tracked for drift (it's a local-only deployment path). +// Config, GitSource, and SourceCodePath are populated from the active deployment +// when one exists, enabling drift detection for out-of-band redeploys. // Started is derived from compute status so the planner can detect start/stop changes. func (*ResourceApp) RemapState(remote *AppRemote) *AppState { started := !isComputeStopped(&remote.App) return &AppState{ App: remote.App, - SourceCodePath: "", + SourceCodePath: remote.SourceCodePath, Config: remote.Config, GitSource: remote.GitSource, Lifecycle: &AppStateLifecycle{Started: &started}, @@ -111,6 +110,7 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { + remote.SourceCodePath = app.ActiveDeployment.SourceCodePath remote.GitSource = app.ActiveDeployment.GitSource remote.Config = deploymentToAppConfig(app.ActiveDeployment) } @@ -159,13 +159,23 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, * } func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, entry *PlanEntry) (*AppRemote, error) { - // Build update mask excluding local-only fields that have no counterpart in the API. - var maskPaths []string + // Build update mask excluding local-only fields that have no counterpart in the App Update API. + // Paths are truncated at the first index because the API only supports entire collection fields, + // not individual elements (e.g. "resources" instead of "resources[0].name"). + maskSet := make(map[string]bool) for path, change := range entry.Changes { - if change.Action == deployplan.Update && !localOnlyFields[path] { - maskPaths = append(maskPaths, path) + if change.Action != deployplan.Update { + continue + } + truncated := truncateAtIndex(path) + if !deployOnlyFields[truncated] { + maskSet[truncated] = true } } + maskPaths := make([]string, 0, len(maskSet)) + for p := range maskSet { + maskPaths = append(maskPaths, p) + } slices.Sort(maskPaths) updateMask := strings.Join(maskPaths, ",") @@ -232,9 +242,10 @@ func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *AppState, return nil, nil } -// localOnlyFields are AppState fields that have no counterpart in the remote state. -// They must not appear in the App update_mask. -var localOnlyFields = map[string]bool{ +// deployOnlyFields are AppState fields managed via the Deploy API, not the App Update API. +// They have remote counterparts (populated from active deployment and compute status), +// but must not appear in the App update_mask. +var deployOnlyFields = map[string]bool{ "source_code_path": true, "config": true, "git_source": true, @@ -328,6 +339,7 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { + remote.SourceCodePath = app.ActiveDeployment.SourceCodePath remote.GitSource = app.ActiveDeployment.GitSource remote.Config = deploymentToAppConfig(app.ActiveDeployment) } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 019ec6ada4..a58022aa70 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -376,8 +376,6 @@ resources: - field: name reason: immutable ignore_remote_changes: - - field: source_code_path - reason: local_only # lifecycle fields are managed by the planner based on compute status. # When lifecycle.started is omitted from config, it should be untracked # and remote changes to compute state should not cause drift. diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 7d4086e40a..3321725049 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -16,9 +16,6 @@ import ( // These are known issues that should be fixed. If a field listed here is found in RemoteType, // the test fails to ensure the entry is removed from this map. var knownMissingInRemoteType = map[string][]string{ - "apps": { - "source_code_path", - }, "clusters": { "apply_policy_default_values", }, diff --git a/bundle/direct/dresources/util.go b/bundle/direct/dresources/util.go index 5a7671e8bc..66b3ef243d 100644 --- a/bundle/direct/dresources/util.go +++ b/bundle/direct/dresources/util.go @@ -61,3 +61,17 @@ func collectUpdatePathsWithPrefix(changes Changes, prefix string) []string { } return paths } + +// truncateAtIndex truncates a field path at the first bracket index (e.g. "[0]", "[*]", +// "[key=value]"). Most update_mask APIs only support referencing entire collection +// fields, not individual elements within them. +// Examples: "resources[0].name" -> "resources", "description" -> "description", +// "config.env[0].name" -> "config.env". +func truncateAtIndex(path string) string { + for i, c := range path { + if c == '[' { + return path[:i] + } + } + return path +} diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index 87cd15a50c..584f63ba8d 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -52,8 +52,10 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } - // Anonymous embedded structs are transparent in JSON; skip them as standalone fields. - if field.Anonymous { + // Fields without a json tag (e.g. anonymous embeds) have no independent + // JSON key and cannot be required. Continue walking their children. + rawTag := field.Tag.Get("json") + if rawTag == "" { return true } @@ -64,7 +66,7 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { } // The "omitempty" tag indicates the field is optional in bundle config. - jsonTag := structtag.JSONTag(field.Tag.Get("json")) + jsonTag := structtag.JSONTag(rawTag) if jsonTag.OmitEmpty() { return true } From 29d5c8a469782201b6c5a34496f465829e8a2bdb Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 12:00:33 +0200 Subject: [PATCH 38/42] fix lint --- bundle/direct/dresources/app.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 3ddb914094..619b55c944 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -75,6 +75,7 @@ func (*ResourceApp) PrepareState(input *resources.App) *AppState { SourceCodePath: input.SourceCodePath, Config: input.Config, GitSource: input.GitSource, + Lifecycle: nil, } if input.Lifecycle != nil && input.Lifecycle.Started != nil { s.Lifecycle = &AppStateLifecycle{Started: input.Lifecycle.Started} @@ -104,10 +105,11 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) } started := !isComputeStopped(app) remote := &AppRemote{ - App: *app, - Config: nil, - GitSource: nil, - Lifecycle: &AppStateLifecycle{Started: &started}, + App: *app, + Config: nil, + GitSource: nil, + SourceCodePath: "", + Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { remote.SourceCodePath = app.ActiveDeployment.SourceCodePath @@ -333,10 +335,11 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli } started := !isComputeStopped(app) remote := &AppRemote{ - App: *app, - Config: nil, - GitSource: nil, - Lifecycle: &AppStateLifecycle{Started: &started}, + App: *app, + Config: nil, + GitSource: nil, + SourceCodePath: "", + Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { remote.SourceCodePath = app.ActiveDeployment.SourceCodePath From beed2c52cee0ad248bd90e20497141d56694179d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 15:51:12 +0200 Subject: [PATCH 39/42] fixed app update test --- .../apps/update/out.requests.direct.json | 34 +++++++++++++++++++ .../apps/update/out.requests.terraform.json | 34 +++++++++++++++++++ .../bundle/resources/apps/update/output.txt | 27 +++++++++++++++ .../bundle/resources/apps/update/script | 3 ++ 4 files changed, 98 insertions(+) diff --git a/acceptance/bundle/resources/apps/update/out.requests.direct.json b/acceptance/bundle/resources/apps/update/out.requests.direct.json index 85a9ac2bc6..8bf3fbbf82 100644 --- a/acceptance/bundle/resources/apps/update/out.requests.direct.json +++ b/acceptance/bundle/resources/apps/update/out.requests.direct.json @@ -9,6 +9,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/myappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": { "app": { @@ -20,6 +33,14 @@ "method": "POST", "path": "/api/2.0/apps/myappname/update" } +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": {}, "method": "DELETE", @@ -36,6 +57,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/deployments" +} { "body": {}, "method": "DELETE", diff --git a/acceptance/bundle/resources/apps/update/out.requests.terraform.json b/acceptance/bundle/resources/apps/update/out.requests.terraform.json index 98dfdcccdc..253def8245 100644 --- a/acceptance/bundle/resources/apps/update/out.requests.terraform.json +++ b/acceptance/bundle/resources/apps/update/out.requests.terraform.json @@ -9,6 +9,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/myappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": { "description": "MY_APP_DESCRIPTION", @@ -20,6 +33,14 @@ "method": "PATCH", "path": "/api/2.0/apps/myappname" } +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/myappname/deployments" +} { "body": {}, "method": "DELETE", @@ -36,6 +57,19 @@ "no_compute": "true" } } +{ + "body": {}, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/start" +} +{ + "body": { + "mode": "SNAPSHOT", + "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app" + }, + "method": "POST", + "path": "/api/2.0/apps/mynewappname/deployments" +} { "body": {}, "method": "DELETE", diff --git a/acceptance/bundle/resources/apps/update/output.txt b/acceptance/bundle/resources/apps/update/output.txt index 021b10a9cd..5b94b6d4af 100644 --- a/acceptance/bundle/resources/apps/update/output.txt +++ b/acceptance/bundle/resources/apps/update/output.txt @@ -10,6 +10,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app myappname +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app myappname +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at myappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle summary @@ -38,6 +48,13 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app myappname +✓ App is in RUNNING state +✓ App compute is in ACTIVE state +✓ Deployment succeeded. +You can access the app at myappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle summary @@ -78,6 +95,16 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle run mykey +✓ Getting the status of the app mynewappname +✓ App is in RUNNING state +✓ App compute is in STOPPED state +✓ Starting the app mynewappname +✓ App is starting... +✓ App is started! +✓ Deployment succeeded. +You can access the app at mynewappname-123.cloud.databricksapps.com + >>> print_requests >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/apps/update/script b/acceptance/bundle/resources/apps/update/script index a66ff34b8c..22e832b035 100644 --- a/acceptance/bundle/resources/apps/update/script +++ b/acceptance/bundle/resources/apps/update/script @@ -6,6 +6,7 @@ print_requests() { trace $CLI bundle plan trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests > out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle summary @@ -13,6 +14,7 @@ title "Update description and re-deploy" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION trace $CLI bundle plan trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests >> out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle summary @@ -21,6 +23,7 @@ trace update_file.py databricks.yml myappname mynewappname trace $CLI bundle plan trace $CLI bundle summary trace $CLI bundle deploy +trace $CLI bundle run mykey trace print_requests >> out.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan From fcbd2ec211bc0b76a7b15b1f6fe015caf1755060 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 16:00:38 +0200 Subject: [PATCH 40/42] fix for source_code_path drift --- .../resources/apps/config-drift/out.plan.direct.json | 11 ++++++++--- bundle/direct/dresources/app.go | 6 ++++-- libs/testserver/apps.go | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json index 009f17fed0..e3571f45fa 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json +++ b/acceptance/bundle/resources/apps/config-drift/out.plan.direct.json @@ -84,6 +84,11 @@ } ] }, + "default_source_code_path": { + "action": "skip", + "reason": "spec:output_only", + "remote": "./app" + }, "id": { "action": "skip", "reason": "spec:output_only", @@ -105,10 +110,10 @@ "remote": "app-[UNIQUE_NAME]" }, "source_code_path": { - "action": "skip", - "reason": "local_only", + "action": "update", "old": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app", - "new": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app" + "new": "/Workspace/Users/[USERNAME]/.bundle/config-drift-[UNIQUE_NAME]/default/files/app", + "remote": "./app" }, "url": { "action": "skip", diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 619b55c944..3f4b4f7401 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -112,7 +112,9 @@ func (r *ResourceApp) DoRead(ctx context.Context, id string) (*AppRemote, error) Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { - remote.SourceCodePath = app.ActiveDeployment.SourceCodePath + // The source code path in active deployment is snapshotted version of the source code path in the app. + // We need to use the default source code path to get the correct source code path for drift detection. + remote.SourceCodePath = app.DefaultSourceCodePath remote.GitSource = app.ActiveDeployment.GitSource remote.Config = deploymentToAppConfig(app.ActiveDeployment) } @@ -342,7 +344,7 @@ func (r *ResourceApp) waitForApp(ctx context.Context, w *databricks.WorkspaceCli Lifecycle: &AppStateLifecycle{Started: &started}, } if app.ActiveDeployment != nil { - remote.SourceCodePath = app.ActiveDeployment.SourceCodePath + remote.SourceCodePath = app.DefaultSourceCodePath remote.GitSource = app.ActiveDeployment.GitSource remote.Config = deploymentToAppConfig(app.ActiveDeployment) } diff --git a/libs/testserver/apps.go b/libs/testserver/apps.go index 6b0d2a7b98..3a4521266b 100644 --- a/libs/testserver/apps.go +++ b/libs/testserver/apps.go @@ -109,6 +109,7 @@ func (s *FakeWorkspace) AppsCreateDeployment(req Request, name string) Response } app.ActiveDeployment = &deployment + app.DefaultSourceCodePath = deployment.SourceCodePath s.Apps[name] = app return Response{Body: deployment} From b93f33f8160810edb0a73322b78bd3e1b7c61681 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 16:20:33 +0200 Subject: [PATCH 41/42] fix migrate test --- .../bundle/invariant/configs/app.yml.tmpl-post-deploy.sh | 3 +++ acceptance/bundle/invariant/migrate/script | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh diff --git a/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh b/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh new file mode 100644 index 0000000000..a303e44c14 --- /dev/null +++ b/acceptance/bundle/invariant/configs/app.yml.tmpl-post-deploy.sh @@ -0,0 +1,3 @@ +# Run the app after the deploy otherwise migrate will show the drift on the source code path. +# This happens because source code path is set remotely only after the deploy. +$CLI bundle run foo diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index d02200cb53..dba524bd8b 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -35,6 +35,10 @@ cat LOG.deploy | contains.py '!panic:' '!internal error' > /dev/null echo INPUT_CONFIG_OK trace $CLI bundle deployment migrate &> LOG.migrate +POST_DEPLOY_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-post-deploy.sh" +if [ -f "$POST_DEPLOY_SCRIPT" ]; then + source "$POST_DEPLOY_SCRIPT" &> LOG.post_deploy +fi cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null From ef5f09a2152e9e96d86facf741d30b1abfe9514c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 30 Mar 2026 16:31:54 +0200 Subject: [PATCH 42/42] fix source_code_path no_drift test --- acceptance/bundle/invariant/no_drift/script | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/invariant/no_drift/script b/acceptance/bundle/invariant/no_drift/script index 0473672e16..2cf8ace75d 100644 --- a/acceptance/bundle/invariant/no_drift/script +++ b/acceptance/bundle/invariant/no_drift/script @@ -34,6 +34,10 @@ trap cleanup EXIT trace $CLI bundle deploy &> LOG.deploy cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null +POST_DEPLOY_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-post-deploy.sh" +if [ -f "$POST_DEPLOY_SCRIPT" ]; then + source "$POST_DEPLOY_SCRIPT" &> LOG.post_deploy +fi # Special message to fuzzer that generated config was fine. # Any failures after this point will be considered as "bug detected" by fuzzer.