[direct] Implement bundle summary/run/open and resource deletion (#3162)

denik · web-flow · commit d1c34da081e0 · 2025-07-03T11:27:12.000+02:00
## Changes - Fix resource deletion in direct backend: resource removed from the config are now deleted. - Implement bundle summary/run/open in direct backend. ## Depends on - #3134 - #3161 ## Tests Enabled a bunch of tests & added more precise comments on remaining disabled.
diff --git a/acceptance/bundle/deploy/jobs/check-metadata/test.toml b/acceptance/bundle/deploy/jobs/check-metadata/test.toml
@@ -1,8 +1,6 @@
 Local = false
 Cloud = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # require "bundle summary"
-
 Ignore = [
     "databricks.yml",
     "a/b/resources.yml",
diff --git a/acceptance/bundle/deploy/jobs/double-underscore-keys/test.toml b/acceptance/bundle/deploy/jobs/double-underscore-keys/test.toml
@@ -1,7 +1,7 @@
 Local = true
 Cloud = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # needs investigation Error: deploying jobs.foo: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="Missing required field: settings.tasks.task_key."
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # (only on Cloud/direct) Error: deploying jobs.foo: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="Missing required field: settings.tasks.task_key."
 
 Ignore = [
     "databricks.yml",
diff --git a/acceptance/bundle/deploy/mlops-stacks/test.toml b/acceptance/bundle/deploy/mlops-stacks/test.toml
@@ -3,7 +3,7 @@ Local=false
 
 Badness = "the newly initialized bundle from the 'mlops-stacks' template contains two validation warnings in the configuration"
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # requires "bundle summary"
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # uses 'experiments' resource
 
 Ignore = [
     "config.json"
diff --git a/acceptance/bundle/deploy/pipeline/auto-approve/test.toml b/acceptance/bundle/deploy/pipeline/auto-approve/test.toml
@@ -1,8 +1,6 @@
 Local = true
 Cloud = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle summary"
-
 Ignore = [
     "databricks.yml"
 ]
diff --git a/acceptance/bundle/deploy/pipeline/recreate/test.toml b/acceptance/bundle/deploy/pipeline/recreate/test.toml
@@ -2,7 +2,7 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # pipeline recreation and ${resources}
 
 Ignore = [
     "databricks.yml"
diff --git a/acceptance/bundle/deploy/schema/auto-approve/test.toml b/acceptance/bundle/deploy/schema/auto-approve/test.toml
@@ -2,7 +2,7 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle summary"
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "${resources}"
 
 Ignore = [
     "databricks.yml",
diff --git a/acceptance/bundle/deploy/secret-scope/test.toml b/acceptance/bundle/deploy/secret-scope/test.toml
@@ -1,7 +1,7 @@
 Cloud = true
 Local = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # secret scopes not implemented
 
 Ignore = [
     "databricks.yml",
diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml
@@ -1,3 +1,3 @@
 Cloud = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # summary,bind,unbind not implemented
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # bind,unbind not implemented
diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml
@@ -1,3 +1,5 @@
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # clusters not implemented
+
 [EnvMatrix]
 DATA_SECURITY_MODE = [
     "USER_ISOLATION",
diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml
@@ -1,8 +1,6 @@
 Local = false
 CloudSlow = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "bundle run"; some tests require "clusters"
-
 Ignore = [
     ".databricks",
     ".venv",
diff --git a/acceptance/bundle/open/test.toml b/acceptance/bundle/open/test.toml
@@ -1,3 +1,2 @@
 GOOS.windows = false
 GOOS.linux = false
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # "bundle open" not implemented yet
diff --git a/acceptance/bundle/python/restricted-execution/test.toml b/acceptance/bundle/python/restricted-execution/test.toml
diff --git a/acceptance/bundle/run/basic/test.toml b/acceptance/bundle/run/basic/test.toml
diff --git a/acceptance/bundle/run/no-state/test.toml b/acceptance/bundle/run/no-state/test.toml
diff --git a/acceptance/bundle/summary/modified_status/test.toml b/acceptance/bundle/summary/modified_status/test.toml
@@ -1,2 +1 @@
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # "bundle summary" not implemented for direct deployment
 EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"]
diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml
@@ -1,9 +1,6 @@
 RecordRequests = true
 IncludeRequestHeaders = ["User-Agent"]
 
-# Retrieves job/pipelines IDs, not populated yet
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
-
 [[Repls]]
 Old = '"execution_time_ms": \d{1,5},'
 New = '"execution_time_ms": SMALL_INT,'
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json
@@ -25,7 +25,6 @@
     "mode": "development",
     "name": "project_name_[UNIQUE_NAME]",
     "target": "dev",
-    "terraform": {},
     "uuid": "[UUID]"
   },
   "include": [
diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt
@@ -54,27 +54,27 @@ Resources:
 >>> diff.py ../out.validate.dev.json ../out.summary.dev.json
 --- ../out.validate.dev.json
 +++ ../out.summary.dev.json
-@@ -52,4 +52,5 @@
+@@ -51,4 +51,5 @@
          "edit_mode": "UI_LOCKED",
          "format": "MULTI_TASK",
 +        "id": "[NUMID]",
          "job_clusters": [
            {
-@@ -120,5 +121,6 @@
+@@ -119,5 +120,6 @@
              "unit": "DAYS"
            }
 -        }
 +        },
 +        "url": "[DATABRICKS_URL]/jobs/[NUMID]"
        }
      },
-@@ -135,4 +137,5 @@
+@@ -134,4 +136,5 @@
          "development": true,
          "edition": "ADVANCED",
 +        "id": "[UUID]",
          "libraries": [
            {
-@@ -144,5 +147,6 @@
+@@ -143,5 +146,6 @@
          "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
          "permissions": [],
 -        "schema": "project_name_[UNIQUE_NAME]_dev"
@@ -132,9 +132,9 @@ Validation OK!
      "name": "project_name_[UNIQUE_NAME]",
 -    "target": "dev",
 +    "target": "prod",
-     "terraform": {},
      "uuid": "[UUID]"
-@@ -33,14 +27,10 @@
+   },
+@@ -32,14 +26,10 @@
      "resources/project_name_[UNIQUE_NAME].pipeline.yml"
    ],
 -  "presets": {
@@ -155,14 +155,14 @@ Validation OK!
 +  ],
    "resources": {
      "jobs": {
-@@ -48,5 +38,5 @@
+@@ -47,5 +37,5 @@
          "deployment": {
            "kind": "BUNDLE",
 -          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
 +          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json"
          },
          "edit_mode": "UI_LOCKED",
-@@ -67,12 +57,9 @@
+@@ -66,12 +56,9 @@
            }
          ],
 -        "max_concurrent_runs": 4,
@@ -177,21 +177,21 @@ Validation OK!
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -80,5 +67,5 @@
+@@ -79,5 +66,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
 +              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook"
              },
              "task_key": "notebook_task"
-@@ -115,5 +102,5 @@
+@@ -114,5 +101,5 @@
          ],
          "trigger": {
 -          "pause_status": "PAUSED",
 +          "pause_status": "UNPAUSED",
            "periodic": {
              "interval": 1,
-@@ -127,22 +114,21 @@
+@@ -126,22 +113,21 @@
          "channel": "CURRENT",
          "configuration": {
 -          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
@@ -219,7 +219,7 @@ Validation OK!
 +        "schema": "project_name_[UNIQUE_NAME]_prod"
        }
      }
-@@ -155,10 +141,10 @@
+@@ -154,10 +140,10 @@
    "targets": null,
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",
@@ -290,9 +290,9 @@ Resources:
      "name": "project_name_[UNIQUE_NAME]",
 -    "target": "dev",
 +    "target": "prod",
-     "terraform": {},
      "uuid": "[UUID]"
-@@ -33,14 +27,10 @@
+   },
+@@ -32,14 +26,10 @@
      "resources/project_name_[UNIQUE_NAME].pipeline.yml"
    ],
 -  "presets": {
@@ -313,14 +313,14 @@ Resources:
 +  ],
    "resources": {
      "jobs": {
-@@ -48,5 +38,5 @@
+@@ -47,5 +37,5 @@
          "deployment": {
            "kind": "BUNDLE",
 -          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
 +          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json"
          },
          "edit_mode": "UI_LOCKED",
-@@ -68,12 +58,9 @@
+@@ -67,12 +57,9 @@
            }
          ],
 -        "max_concurrent_runs": 4,
@@ -335,21 +335,21 @@ Resources:
 -          "dev": "[USERNAME]"
          },
          "tasks": [
-@@ -81,5 +68,5 @@
+@@ -80,5 +67,5 @@
              "job_cluster_key": "job_cluster",
              "notebook_task": {
 -              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
 +              "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook"
              },
              "task_key": "notebook_task"
-@@ -116,5 +103,5 @@
+@@ -115,5 +102,5 @@
          ],
          "trigger": {
 -          "pause_status": "PAUSED",
 +          "pause_status": "UNPAUSED",
            "periodic": {
              "interval": 1,
-@@ -129,11 +116,10 @@
+@@ -128,11 +115,10 @@
          "channel": "CURRENT",
          "configuration": {
 -          "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
@@ -363,7 +363,7 @@ Resources:
 -        "development": true,
          "edition": "ADVANCED",
          "id": "[UUID]",
-@@ -141,11 +127,11 @@
+@@ -140,11 +126,11 @@
            {
              "notebook": {
 -              "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline"
@@ -378,7 +378,7 @@ Resources:
 +        "schema": "project_name_[UNIQUE_NAME]_prod",
          "url": "[DATABRICKS_URL]/pipelines/[UUID]"
        }
-@@ -159,10 +145,10 @@
+@@ -158,10 +144,10 @@
    "targets": null,
    "workspace": {
 -    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",
diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script
@@ -10,7 +10,7 @@ rm input.json
 cd project_name_${UNIQUE_NAME}
 
 export JQ_DELETE='
-  .bundle.terraform.exec_path,
+  .bundle.terraform,
   .workspace.current_user
 '
 
diff --git a/acceptance/bundle/templates/default-python/integration_classic/test.toml b/acceptance/bundle/templates/default-python/integration_classic/test.toml
@@ -1,9 +1,6 @@
 Cloud = true
 Timeout = '1m'
 
-# "bundle summary" is not supported yet
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
-
 Ignore = [
     '.venv',
 ]
diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go
@@ -145,19 +145,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diags
 	}
 
-	if b.DirectDeployment {
-		// TODO: terraform.Load alternative
-	} else {
-		newDiags := bundle.Apply(ctx, b,
-			statemgmt.Load(),
-		)
-		diags = diags.Extend(newDiags)
-		if newDiags.HasError() {
-			return diags
-		}
-	}
-
 	diags = diags.Extend(bundle.ApplySeq(ctx, b,
+		statemgmt.Load(),
 		apps.InterpolateVariables(),
 		apps.UploadConfig(),
 		metadata.Compute(),
@@ -192,8 +181,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		diags = diags.Extend(bundle.Apply(ctx, b, lock.Release(lock.GoalDeploy)))
 	}()
 
-	// TODO: StatePull and OpenResourceDatabase both parse resources.json; we should do it only once
-
 	diags = diags.Extend(bundle.ApplySeq(ctx, b,
 		statemgmt.StatePull(),
 		terraform.CheckDashboardsModifiedRemotely(),
@@ -222,13 +209,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		return diags
 	}
 
-	if b.DirectDeployment {
-		err := b.OpenResourceDatabase(ctx)
-		if err != nil {
-			diags = diags.Extend(diag.FromErr(err))
-			return diags
-		}
-	} else {
+	if !b.DirectDeployment {
 		diags = diags.Extend(bundle.ApplySeq(ctx, b,
 			terraform.Interpolate(),
 			terraform.Write(),
diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go
@@ -135,13 +135,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle) (diags diag.Diagnostics) {
 		return diags
 	}
 
-	if b.DirectDeployment {
-		err := b.OpenResourceDatabase(ctx)
-		if err != nil {
-			diags = diags.Extend(diag.FromErr(err))
-			return diags
-		}
-	} else {
+	if !b.DirectDeployment {
 		diags = diags.Extend(bundle.ApplySeq(ctx, b,
 			terraform.Interpolate(),
 			terraform.Write(),
diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go
@@ -33,17 +33,28 @@ func (l *load) Name() string {
 }
 
 func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
-	tf := b.Terraform
-	if tf == nil {
-		return diag.Errorf("terraform not initialized")
-	}
+	var state ExportedResourcesMap
+	var err error
 
-	err := tf.Init(ctx, tfexec.Upgrade(true))
-	if err != nil {
-		return diag.Errorf("terraform init: %v", err)
-	}
+	if b.DirectDeployment {
+		err = b.OpenResourceDatabase(ctx)
+		if err != nil {
+			return diag.FromErr(err)
+		}
+		state = b.ResourceDatabase.ExportState(ctx)
+	} else {
+		tf := b.Terraform
+		if tf == nil {
+			return diag.Errorf("terraform not initialized")
+		}
 
-	state, err := terraform.ParseResourcesState(ctx, b)
+		err = tf.Init(ctx, tfexec.Upgrade(true))
+		if err != nil {
+			return diag.Errorf("terraform init: %v", err)
+		}
+
+		state, err = terraform.ParseResourcesState(ctx, b)
+	}
 	if err != nil {
 		return diag.FromErr(err)
 	}
diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go
diff --git a/bundle/terranova/apply.go b/bundle/terranova/apply.go
diff --git a/bundle/terranova/plan.go b/bundle/terranova/plan.go
diff --git a/bundle/terranova/tnstate/state.go b/bundle/terranova/tnstate/state.go
diff --git a/cmd/bundle/open.go b/cmd/bundle/open.go
diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go
diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,6 @@`
`1`	`1`	`Local = true`
`2`	`2`	`Cloud = true`
`3`	`3`
`4`		`-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # requires "bundle summary"`
`5`		`-`
`6`	`4`	`Ignore = [`
`7`	`5`	`"databricks.yml"`
`8`	`6`	`]`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`Cloud = true`
`2`	`2`
`3`		`-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # summary,bind,unbind not implemented`
	`3`	`+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # bind,unbind not implemented`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"] # clusters not implemented`
	`2`	`+`
`1`	`3`	`[EnvMatrix]`
`2`	`4`	`DATA_SECURITY_MODE = [`
`3`	`5`	`"USER_ISOLATION",`