Add bundle deployment telemetry for variables, validation gap, and dependency graph

Add several new telemetry signals to improve understanding of DABs usage patterns:

- Detect nested variable references (var-in-var patterns like ${var.foo_${var.bar}})
- Track variable defaults count, lookup type distribution, override sources (env var, file, CLI flag), and resolution depth
- Track validation-pass-deploy-fail gap via "validation_passed" metric with deferred telemetry logging
- Track direct engine deployment plan metrics: action types, resource counts, dependency graph depth/width
- Remove unused telemetry fields: SetFields, TerraformStateSizeBytes, FromWebTerminal

Co-authored-by: Isaac

Author: shreyas-goenka

acceptance/bundle/telemetry/deploy-compute-type/output.txt

@@ -37,6 +37,10 @@ Deployment complete!
     "key": "python_wheel_wrapper_is_set",
     "value": false
   },
+  {
+    "key": "validation_passed",
+    "value": true
+  },
   {
     "key": "skip_artifact_cleanup",
     "value": false
@@ -79,6 +83,10 @@ Deployment complete!
     "key": "python_wheel_wrapper_is_set",
     "value": false
   },
+  {
+    "key": "validation_passed",
+    "value": true
+  },
   {
     "key": "skip_artifact_cleanup",
     "value": false

acceptance/bundle/telemetry/deploy-experimental/output.txt

@@ -36,6 +36,10 @@ Deployment complete!
       "key": "python_wheel_wrapper_is_set",
       "value": false
     },
+    {
+      "key": "validation_passed",
+      "value": true
+    },
     {
       "key": "skip_artifact_cleanup",
       "value": false

acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt

@@ -32,6 +32,10 @@ Deployment complete!
       "key": "python_wheel_wrapper_is_set",
       "value": false
     },
+    {
+      "key": "validation_passed",
+      "value": true
+    },
     {
       "key": "skip_artifact_cleanup",
       "value": false

acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt

@@ -32,6 +32,10 @@ Deployment complete!
       "key": "python_wheel_wrapper_is_set",
       "value": false
     },
+    {
+      "key": "validation_passed",
+      "value": true
+    },
     {
       "key": "skip_artifact_cleanup",
       "value": false

acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt

@@ -36,6 +36,10 @@ Deployment complete!
       "key": "python_wheel_wrapper_is_set",
       "value": false
     },
+    {
+      "key": "validation_passed",
+      "value": true
+    },
     {
       "key": "skip_artifact_cleanup",
       "value": false
@@ -80,6 +84,10 @@ Deployment complete!
       "key": "python_wheel_wrapper_is_set",
       "value": true
     },
+    {
+      "key": "validation_passed",
+      "value": true
+    },
     {
       "key": "skip_artifact_cleanup",
       "value": true

acceptance/bundle/telemetry/deploy/out.telemetry.txt

@@ -66,6 +66,10 @@
               "key": "python_wheel_wrapper_is_set",
               "value": false
             },
+            {
+              "key": "validation_passed",
+              "value": true
+            },
             {
               "key": "skip_artifact_cleanup",
               "value": false

bundle/bundle.go

@@ -56,6 +56,7 @@ type Metrics struct {
 	PythonUpdatedResourcesCount int64
 	ExecutionTimes              []protos.IntMapEntry
 	LocalCacheMeasurementsMs    []protos.IntMapEntry // Local cache measurements stored as milliseconds
+	DeployPlanMetrics           []protos.IntMapEntry // Deployment plan and graph metrics (direct engine)
 }
 
 // SetBoolValue sets the value of a boolean metric.

bundle/config/mutator/resolve_variable_references.go

@@ -150,11 +150,18 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 	// We rewrite it here to make the resolution logic simpler.
 	varPath := dyn.NewPath(dyn.Key("var"))
 
+	// Detect nested variable references like ${var.foo_${var.bar}} and log
+	// a telemetry event if found. These patterns are not supported by the
+	// interpolation regex and silently fail to resolve.
+	m.detectNestedVariableReferences(b)
+
 	var diags diag.Diagnostics
 	maxRounds := 1 + m.extraRounds
+	roundsUsed := 0
 
 	for round := range maxRounds {
 		hasUpdates, newDiags := m.resolveOnce(b, prefixes, varPath)
+		roundsUsed = round + 1
 
 		diags = diags.Extend(newDiags)
 
@@ -180,6 +187,13 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle)
 		b.Metrics.SetBoolValue("artifacts_reference_used", true)
 	}
 
+	if roundsUsed > 1 {
+		b.Metrics.SetBoolValue("variable_resolution_rounds_gt_1", true)
+	}
+	if roundsUsed > 3 {
+		b.Metrics.SetBoolValue("variable_resolution_rounds_gt_3", true)
+	}
+
 	return diags
 }
 
@@ -294,6 +308,30 @@ func (m *resolveVariableReferences) selectivelyMutate(b *bundle.Bundle, fn func(
 	})
 }
 
+// errNestedVarRefFound is a sentinel error used to short-circuit WalkReadOnly
+// once a nested variable reference is detected.
+var errNestedVarRefFound = errors.New("nested variable reference found")
+
+// detectNestedVariableReferences walks the bundle configuration and checks for
+// nested variable references like ${var.foo_${var.bar}}. These patterns are not
+// supported and are tracked via telemetry to understand how common they are.
+func (m *resolveVariableReferences) detectNestedVariableReferences(b *bundle.Bundle) {
+	err := dyn.WalkReadOnly(b.Config.Value(), func(_ dyn.Path, v dyn.Value) error {
+		s, ok := v.AsString()
+		if !ok {
+			return nil
+		}
+
+		if dynvar.ContainsNestedVariableReference(s) {
+			return errNestedVarRefFound
+		}
+		return nil
+	})
+	if err == errNestedVarRefFound {
+		b.Metrics.SetBoolValue("nested_var_reference_used", true)
+	}
+}
+
 func getAllKeys(root dyn.Value) ([]string, error) {
 	var keys []string
 

bundle/config/mutator/resolve_variable_references_test.go

@@ -6,10 +6,46 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/config/variable"
+	"github.com/databricks/cli/libs/telemetry/protos"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func TestResolveVariableReferencesDetectsNestedVarRef(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "${var.env_${var.region}}",
+			},
+		},
+	}
+
+	diags := bundle.Apply(t.Context(), b, ResolveVariableReferencesWithoutResources())
+	// The nested reference won't resolve, but we should still detect it.
+	_ = diags
+
+	assert.Contains(t, b.Metrics.BoolValues, protos.BoolMapEntry{Key: "nested_var_reference_used", Value: true})
+}
+
+func TestResolveVariableReferencesNoNestedVarRef(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "${var.env}",
+			},
+		},
+	}
+
+	diags := bundle.Apply(t.Context(), b, ResolveVariableReferencesWithoutResources())
+	_ = diags
+
+	for _, entry := range b.Metrics.BoolValues {
+		assert.NotEqual(t, "nested_var_reference_used", entry.Key)
+	}
+}
+
 func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 	testCases := []struct {
 		enabled bool
@@ -63,3 +99,52 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 		testCase.assert(t, b)
 	}
 }
+
+func TestResolveVariableReferencesRoundsNoReferences(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "literal-name",
+			},
+		},
+	}
+
+	diags := bundle.Apply(t.Context(), b, ResolveVariableReferencesWithoutResources())
+	require.NoError(t, diags.Error())
+
+	// No references means a single round with no updates, so gt_1 should not be set.
+	for _, entry := range b.Metrics.BoolValues {
+		assert.NotEqual(t, "variable_resolution_rounds_gt_1", entry.Key)
+		assert.NotEqual(t, "variable_resolution_rounds_gt_3", entry.Key)
+	}
+}
+
+func TestResolveVariableReferencesRoundsGt1MultiRound(t *testing.T) {
+	// Set up a chain: bundle.name -> var.a -> var.b -> literal.
+	// This requires 2 rounds to fully resolve.
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Bundle: config.Bundle{
+				Name: "${var.a}",
+			},
+			Variables: map[string]*variable.Variable{
+				"a": {
+					Value: "${var.b}",
+				},
+				"b": {
+					Value: "final",
+				},
+			},
+		},
+	}
+
+	diags := bundle.Apply(t.Context(), b, ResolveVariableReferencesWithoutResources())
+	require.NoError(t, diags.Error())
+	assert.Equal(t, "final", b.Config.Bundle.Name)
+
+	assert.Contains(t, b.Metrics.BoolValues, protos.BoolMapEntry{Key: "variable_resolution_rounds_gt_1", Value: true})
+	// 2 rounds should not trigger gt_3.
+	for _, entry := range b.Metrics.BoolValues {
+		assert.NotEqual(t, "variable_resolution_rounds_gt_3", entry.Key)
+	}
+}

bundle/config/mutator/set_variables.go

@@ -30,30 +30,42 @@ func getDefaultVariableFilePath(target string) string {
 	return ".databricks/bundle/" + target + "/variable-overrides.json"
 }
 
-func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, error) {
-	// case: variable already has value initialized, so skip
+// variableOverrideSource tracks which override method resolved a variable's value.
+type variableOverrideSource int
+
+const (
+	variableOverrideSourceCLI     variableOverrideSource = iota // --var flag (value already set)
+	variableOverrideSourceEnvVar                                // BUNDLE_VAR_* environment variable
+	variableOverrideSourceFile                                  // variable-overrides.json
+	variableOverrideSourceDefault                               // default value from config
+	variableOverrideSourceNone                                  // no value resolved (lookup or error)
+)
+
+func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, name string, fileDefault dyn.Value) (dyn.Value, variableOverrideSource, error) {
+	// case: variable already has value initialized, so skip.
+	// This happens when the value is set via the --var CLI flag.
 	if variable.HasValue() {
-		return v, nil
+		return v, variableOverrideSourceCLI, nil
 	}
 
 	// case: read and set variable value from process environment
 	envVarName := bundleVarPrefix + name
 	if val, ok := env.Lookup(ctx, envVarName); ok {
 		if variable.IsComplex() {
-			return dyn.InvalidValue, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`setting via environment variables (%s) is not supported for complex variable %s`, envVarName, name)
 		}
 
 		v, err := dyn.Set(v, "value", dyn.V(val))
 		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`failed to assign value "%s" to variable %s from environment variable %s with error: %v`, val, name, envVarName, err)
 		}
-		return v, nil
+		return v, variableOverrideSourceEnvVar, nil
 	}
 
 	// case: Defined a variable for named lookup for a resource
 	// It will be resolved later in ResolveResourceReferences mutator
 	if variable.Lookup != nil {
-		return v, nil
+		return v, variableOverrideSourceNone, nil
 	}
 
 	// case: Set the variable to the default value from the variable file
@@ -62,36 +74,36 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable,
 		hasComplexValue := fileDefault.Kind() == dyn.KindMap || fileDefault.Kind() == dyn.KindSequence
 
 		if hasComplexType && !hasComplexValue {
-			return dyn.InvalidValue, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`variable %s is of type complex, but the value in the variable file is not a complex type`, name)
 		}
 		if !hasComplexType && hasComplexValue {
-			return dyn.InvalidValue, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`variable %s is not of type complex, but the value in the variable file is a complex type`, name)
 		}
 
 		v, err := dyn.Set(v, "value", fileDefault)
 		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`failed to assign default value from variable file to variable %s with error: %v`, name, err)
 		}
 
-		return v, nil
+		return v, variableOverrideSourceFile, nil
 	}
 
 	// case: Set the variable to its default value
 	if variable.HasDefault() {
 		vDefault, err := dyn.Get(v, "default")
 		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`failed to get default value from config "%s" for variable %s with error: %v`, variable.Default, name, err)
 		}
 
 		v, err := dyn.Set(v, "value", vDefault)
 		if err != nil {
-			return dyn.InvalidValue, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err)
+			return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`failed to assign default value from config "%s" to variable %s with error: %v`, variable.Default, name, err)
 		}
-		return v, nil
+		return v, variableOverrideSourceDefault, nil
 	}
 
 	// We should have had a value to set for the variable at this point.
-	return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("<target>"))
+	return dyn.InvalidValue, variableOverrideSourceNone, fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("<target>"))
 }
 
 func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) {
@@ -128,6 +140,11 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 	if diags.HasError() {
 		return diags
 	}
+
+	envVarUsed := false
+	fileUsed := false
+	cliUsed := false
+
 	err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) {
 		return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) {
 			name := p[1].Key()
@@ -137,9 +154,28 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
 			}
 
 			fileDefault, _ := dyn.Get(defaults, name)
-			return setVariable(ctx, variable, v, name, fileDefault)
+			result, source, err := setVariable(ctx, variable, v, name, fileDefault)
+			switch source {
+			case variableOverrideSourceCLI:
+				cliUsed = true
+			case variableOverrideSourceEnvVar:
+				envVarUsed = true
+			case variableOverrideSourceFile:
+				fileUsed = true
+			}
+			return result, err
 		}))
 	})
 
+	if envVarUsed {
+		b.Metrics.SetBoolValue("variable_override_env_var_used", true)
+	}
+	if fileUsed {
+		b.Metrics.SetBoolValue("variable_override_file_used", true)
+	}
+	if cliUsed {
+		b.Metrics.SetBoolValue("variable_override_cli_flag_used", true)
+	}
+
 	return diags.Extend(diag.FromErr(err))
 }