Generalize terraform to bundle conversion, prepare for direct deployment (#3134)

## Changes
- Introduce ExportedResourcesMap type that is deployment-method neutral.
Modify terraform.ParseResourcesState to return this type instead of
struct matching terraform state format.
- Modify terraform.TerraformToBundle() to rely on this new format.
- Make terraform.TerraformToBundle() handle all resources automatically
and operate on dynamic bundle structure.
- Remove ModifedStatusUpdated constant and usage in apps.
- Fix Secret scopes resource to properly set modified_status. Previously
it was always "created".

## Why
ExportedResourcesMap allows implementing terraform.Load() and thus
bundle summary/run/open with direct deployment method
(#2926) as it is not tied to
terraform resource names or internal state structure.

When implementing new resource, we no longer need to add an entry in
TerraformToBundle, this both saves time and avoids bugs.

ModifiedStatusUpdated is removed because it's not implemented. Only app
resource ever sets it and it sets it always, not based on the actual
situation.

## Tests
summary/modified_status is expanded to handle both cases where
resources: block is present and not present (regression during
development of this PR).

Author: denik

acceptance/bundle/deploy/secret-scope/output.txt

@@ -23,12 +23,11 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
-=== Print summary after deploy; it should have id and no modified_status. Badness: incorrectly shows modified_status=created
+=== Print summary after deploy; it should have id and no modified_status
 >>> [CLI] bundle summary --output json
 {
   "backend_type": "DATABRICKS",
   "id": "my-secrets-[UUID]",
-  "modified_status": "created",
   "name": "my-secrets-[UUID]",
   "permissions": [
     {

acceptance/bundle/deploy/secret-scope/script

@@ -14,7 +14,7 @@ trap cleanup EXIT
 title "Print summary before deploy; it should have modified_status=created and no id"
 trace $CLI bundle summary --output json | jq '.resources.secret_scopes.secret_scope1'
 trace $CLI bundle deploy
-title "Print summary after deploy; it should have id and no modified_status. Badness: incorrectly shows modified_status=created"
+title "Print summary after deploy; it should have id and no modified_status"
 trace $CLI bundle summary --output json | jq '.resources.secret_scopes.secret_scope1'
 trace $CLI secrets list-scopes -o json | jq --arg value ${SECRET_SCOPE_NAME} '.[] | select(.name == $value)'
 

…bundle/summary/modified_status/empty.yml …mary/modified_status/empty_resources.ymlacceptance/bundle/summary/modified_status/empty.yml renamed to acceptance/bundle/summary/modified_status/empty_resources.yml acceptance/bundle/summary/modified_status/empty.yml renamed to acceptance/bundle/summary/modified_status/empty_resources.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: test-bundle

acceptance/bundle/summary/modified_status/no_resources.yml

@@ -5,7 +5,7 @@ trace $CLI bundle deploy
 
 title "Post-deployment view of resources with id and without modified_status"
 trace $CLI bundle summary -o json | jq .resources
-mv empty.yml databricks.yml
+mv $VARIANT databricks.yml
 
 title "Expecting all resources to have modified_status=deleted"
 trace $CLI bundle summary -o json | jq .resources

acceptance/bundle/summary/modified_status/script

@@ -1 +1,2 @@
 EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # "bundle summary" not implemented for direct deployment
+EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"]

acceptance/bundle/summary/modified_status/test.toml

@@ -10,6 +10,5 @@ type ModifiedStatus = string
 
 const (
 	ModifiedStatusCreated ModifiedStatus = "created"
-	ModifiedStatusUpdated ModifiedStatus = "updated"
 	ModifiedStatusDeleted ModifiedStatus = "deleted"
 )

bundle/config/resources/modified_status.go

@@ -7,7 +7,6 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
-	tfjson "github.com/hashicorp/terraform-json"
 )
 
 type dashboardState struct {
@@ -23,20 +22,12 @@ func collectDashboardsFromState(ctx context.Context, b *bundle.Bundle) ([]dashbo
 	}
 
 	var dashboards []dashboardState
-	for _, resource := range state.Resources {
-		if resource.Mode != tfjson.ManagedResourceMode {
-			continue
-		}
-		for _, instance := range resource.Instances {
-			switch resource.Type {
-			case "databricks_dashboard":
-				dashboards = append(dashboards, dashboardState{
-					Name: resource.Name,
-					ID:   instance.Attributes.ID,
-					ETag: instance.Attributes.ETag,
-				})
-			}
-		}
+	for resourceName, instance := range state["dashboards"] {
+		dashboards = append(dashboards, dashboardState{
+			Name: resourceName,
+			ID:   instance.ID,
+			ETag: instance.ETag,
+		})
 	}
 
 	return dashboards, nil

bundle/deploy/terraform/check_dashboards_modified_remotely.go

@@ -10,7 +10,6 @@ import (
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
-	tfjson "github.com/hashicorp/terraform-json"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -51,50 +50,45 @@ func CheckRunningResource() *checkRunningResources {
 	return &checkRunningResources{}
 }
 
-func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state *resourcesState) error {
-	if state == nil {
-		return nil
-	}
-
+func checkAnyResourceRunning(ctx context.Context, w *databricks.WorkspaceClient, state ExportedResourcesMap) error {
 	errs, errCtx := errgroup.WithContext(ctx)
 
-	for _, resource := range state.Resources {
-		if resource.Mode != tfjson.ManagedResourceMode {
+	for _, jobAttrs := range state["jobs"] {
+		id := jobAttrs.ID
+		if id == "" {
 			continue
 		}
-		for _, instance := range resource.Instances {
-			id := instance.Attributes.ID
-			if id == "" {
-				continue
-			}
 
-			switch resource.Type {
-			case "databricks_job":
-				errs.Go(func() error {
-					isRunning, err := IsJobRunning(errCtx, w, id)
-					// If there's an error retrieving the job, we assume it's not running
-					if err != nil {
-						return err
-					}
-					if isRunning {
-						return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
-					}
-					return nil
-				})
-			case "databricks_pipeline":
-				errs.Go(func() error {
-					isRunning, err := IsPipelineRunning(errCtx, w, id)
-					// If there's an error retrieving the pipeline, we assume it's not running
-					if err != nil {
-						return nil
-					}
-					if isRunning {
-						return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
-					}
-					return nil
-				})
+		errs.Go(func() error {
+			isRunning, err := IsJobRunning(errCtx, w, id)
+			// If there's an error retrieving the job, we assume it's not running
+			if err != nil {
+				return err
+			}
+			if isRunning {
+				return &ErrResourceIsRunning{resourceType: "job", resourceId: id}
 			}
+			return nil
+		})
+	}
+
+	for _, pipelineAttrs := range state["pipelines"] {
+		id := pipelineAttrs.ID
+		if id == "" {
+			continue
 		}
+
+		errs.Go(func() error {
+			isRunning, err := IsPipelineRunning(errCtx, w, id)
+			// If there's an error retrieving the pipeline, we assume it's not running
+			if err != nil {
+				return nil
+			}
+			if isRunning {
+				return &ErrResourceIsRunning{resourceType: "pipeline", resourceId: id}
+			}
+			return nil
+		})
 	}
 
 	return errs.Wait()

bundle/deploy/terraform/check_running_resources.go

@@ -14,22 +14,15 @@ import (
 
 func TestIsAnyResourceRunningWithEmptyState(t *testing.T) {
 	mock := mocks.NewMockWorkspaceClient(t)
-	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, &resourcesState{})
+	err := checkAnyResourceRunning(context.Background(), mock.WorkspaceClient, nil)
 	require.NoError(t, err)
 }
 
 func TestIsAnyResourceRunningWithJob(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &resourcesState{
-		Resources: []stateResource{
-			{
-				Type: "databricks_job",
-				Mode: "managed",
-				Name: "job1",
-				Instances: []stateResourceInstance{
-					{Attributes: stateInstanceAttributes{ID: "123"}},
-				},
-			},
+	resources := ExportedResourcesMap{
+		"jobs": map[string]ResourceState{
+			"job1": {ID: "123"},
 		},
 	}
 
@@ -55,16 +48,9 @@ func TestIsAnyResourceRunningWithJob(t *testing.T) {
 
 func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &resourcesState{
-		Resources: []stateResource{
-			{
-				Type: "databricks_pipeline",
-				Mode: "managed",
-				Name: "pipeline1",
-				Instances: []stateResourceInstance{
-					{Attributes: stateInstanceAttributes{ID: "123"}},
-				},
-			},
+	resources := ExportedResourcesMap{
+		"pipelines": map[string]ResourceState{
+			"pipeline1": {ID: "123"},
 		},
 	}
 
@@ -91,16 +77,10 @@ func TestIsAnyResourceRunningWithPipeline(t *testing.T) {
 
 func TestIsAnyResourceRunningWithAPIFailure(t *testing.T) {
 	m := mocks.NewMockWorkspaceClient(t)
-	resources := &resourcesState{
-		Resources: []stateResource{
-			{
-				Type: "databricks_pipeline",
-				Mode: "managed",
-				Name: "pipeline1",
-				Instances: []stateResourceInstance{
-					{Attributes: stateInstanceAttributes{ID: "123"}},
-				},
-			},
+
+	resources := ExportedResourcesMap{
+		"pipelines": map[string]ResourceState{
+			"pipeline1": {ID: "123"},
 		},
 	}