Support nested variable references like ${var.foo_${var.tail}}

The parser now treats outer ${...} prefixes as literal text when a
nested ${ is encountered, allowing inner references to be resolved
first. Multi-round resolution progressively resolves from inside out.

Co-authored-by: Isaac

Author: shreyas-goenka

NEXT_CHANGELOG.md

@@ -12,7 +12,7 @@
 * Deduplicate grant entries with duplicate principals or privileges during initialization ([#4801](https://github.com/databricks/cli/pull/4801))
 * engine/direct: Fix unwanted recreation of secret scopes when scope_backend_type is not set ([#4834](https://github.com/databricks/cli/pull/4834))
 
-* **Breaking**: Nested variable references like `${var.foo_${var.tail}}` are now rejected with a warning and left unresolved. Previously the regex-based parser matched only the innermost `${var.tail}` by coincidence, which silently produced incorrect results. If you rely on dynamic variable name construction, use separate variables or target overrides instead ([#4747](https://github.com/databricks/cli/pull/4747)).
+* Replace regex-based variable interpolation with a character scanner. Escape sequences `\$` and `\\` are now supported ([#4747](https://github.com/databricks/cli/pull/4747)).
 
 ### Dependency updates
 

acceptance/bundle/variables/var_in_var/databricks.yml

@@ -9,8 +9,6 @@ variables:
   tail:
     default: x
   final:
-    # This works but it's not officially supported, please do not rely on it.
-    # We might start to reject this config in the future.
     default: ${var.foo_${var.tail}}
 
 targets:

acceptance/bundle/variables/var_in_var/output.txt

@@ -1,13 +1,9 @@
 
 >>> [CLI] bundle validate -o json -t target_x
-Warning: nested variable references are not supported
-  at variables.final.default
-  in databricks.yml:14:14
-
 {
   "final": {
-    "default": "${var.foo_${var.tail}}",
-    "value": "${var.foo_${var.tail}}"
+    "default": "hello from foo x",
+    "value": "hello from foo x"
   },
   "foo_x": {
     "default": "hello from foo x",
@@ -24,14 +20,10 @@ Warning: nested variable references are not supported
 }
 
 >>> [CLI] bundle validate -o json -t target_y
-Warning: nested variable references are not supported
-  at variables.final.default
-  in databricks.yml:14:14
-
 {
   "final": {
-    "default": "${var.foo_${var.tail}}",
-    "value": "${var.foo_${var.tail}}"
+    "default": "hi from foo y",
+    "value": "hi from foo y"
   },
   "foo_x": {
     "default": "hello from foo x",

acceptance/bundle/variables/var_in_var_3level/databricks.yml

@@ -0,0 +1,12 @@
+bundle:
+  name: test-bundle
+
+variables:
+  env:
+    default: prod
+  region_prod:
+    default: us
+  endpoint_us:
+    default: https://us.example.com
+  result:
+    default: ${var.endpoint_${var.region_${var.env}}}

acceptance/bundle/variables/var_in_var_3level/out.test.toml

@@ -0,0 +1,20 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "endpoint_us": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  },
+  "env": {
+    "default": "prod",
+    "value": "prod"
+  },
+  "region_prod": {
+    "default": "us",
+    "value": "us"
+  },
+  "result": {
+    "default": "https://us.example.com",
+    "value": "https://us.example.com"
+  }
+}

acceptance/bundle/variables/var_in_var_3level/output.txt

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq .variables

acceptance/bundle/variables/var_in_var_3level/script

@@ -394,6 +394,35 @@ func TestResolveSequenceVariable(t *testing.T) {
 	assert.Equal(t, "value2", seq[1].MustString())
 }
 
+func TestResolveNestedVariableReference(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"tail":  dyn.V("x"),
+		"foo_x": dyn.V("hello"),
+		"final": dyn.V("${foo_${tail}}"),
+	})
+
+	// First pass resolves ${tail} -> "x", producing "${foo_x}" for final.
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+
+	// After one pass, the inner ref is resolved but the outer is not yet.
+	assert.Equal(t, "${foo_x}", getByPath(t, out, "final").MustString())
+}
+
+func TestResolveThreeLevelNestedVariableReference(t *testing.T) {
+	in := dyn.V(map[string]dyn.Value{
+		"c":     dyn.V("z"),
+		"b_z":   dyn.V("y"),
+		"a_y":   dyn.V("hello"),
+		"final": dyn.V("${a_${b_${c}}}"),
+	})
+
+	// First pass resolves ${c} -> "z", producing "${a_${b_z}}" for final.
+	out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
+	require.NoError(t, err)
+	assert.Equal(t, "${a_${b_z}}", getByPath(t, out, "final").MustString())
+}
+
 func TestResolveEscapedRef(t *testing.T) {
 	in := dyn.V(map[string]dyn.Value{
 		"a": dyn.V("a"),

libs/dyn/dynvar/resolve_test.go

@@ -57,11 +57,16 @@ var indexPattern = regexp.MustCompile(`^(\[[0-9]+\])+$`)
 //   - "\$" produces a literal "$"
 //   - "\\" produces a literal "\"
 //
+// Nested references like "${a.${b}}" are supported by treating the outer
+// "${a." as literal text so that inner references are resolved first.
+// After resolution the resulting string (e.g. "${a.x}") is re-parsed.
+//
 // Examples:
 //   - "hello"            -> [Literal("hello")]
 //   - "${a.b}"           -> [Ref("a.b")]
 //   - "pre ${a.b} post"  -> [Literal("pre "), Ref("a.b"), Literal(" post")]
 //   - "\${a.b}"          -> [Literal("${a.b}")]
+//   - "${a.${b}}"        -> [Literal("${a."), Ref("b"), Literal("}")]
 func Parse(s string) ([]Token, error) {
 	if len(s) == 0 {
 		return nil, nil
@@ -136,18 +141,30 @@ func Parse(s string) ([]Token, error) {
 			refStart := i
 			j := i + 2 // skip "${"
 
-			// Scan until closing '}', rejecting nested '${'.
+			// Scan until closing '}', handling nested '${'.
 			pathStart := j
+			nested := false
 			for j < len(s) && s[j] != closeBrace {
 				if s[j] == dollarChar && j+1 < len(s) && s[j+1] == openBrace {
-					return nil, &ParseError{
-						Msg: "nested variable references are not supported",
-						Pos: refStart,
-					}
+					// Nested '${' found. Treat the outer "${..." prefix as
+					// literal so inner references get resolved first.
+					// E.g. "${a.${b}}" produces:
+					//   [Literal("${a."), Ref("b"), Literal("}")]
+					nested = true
+					break
 				}
 				j++
 			}
 
+			if nested {
+				if buf.Len() == 0 {
+					litStart = refStart
+				}
+				buf.WriteString(s[refStart:j])
+				i = j
+				continue
+			}
+
 			if j >= len(s) {
 				return nil, &ParseError{
 					Msg: "unterminated variable reference",

libs/interpolation/parse.go

@@ -178,6 +178,33 @@ func TestParse(t *testing.T) {
 				{Kind: TokenRef, Value: "b", Start: 6, End: 10},
 			},
 		},
+		{
+			"nested_ref",
+			"${var.foo_${var.tail}}",
+			[]Token{
+				{Kind: TokenLiteral, Value: "${var.foo_", Start: 0, End: 10},
+				{Kind: TokenRef, Value: "var.tail", Start: 10, End: 21},
+				{Kind: TokenLiteral, Value: "}", Start: 21, End: 22},
+			},
+		},
+		{
+			"three_level_nested_ref",
+			"${a_${b_${c}}}",
+			[]Token{
+				{Kind: TokenLiteral, Value: "${a_${b_", Start: 0, End: 8},
+				{Kind: TokenRef, Value: "c", Start: 8, End: 12},
+				{Kind: TokenLiteral, Value: "}}", Start: 12, End: 14},
+			},
+		},
+		{
+			"nested_ref_mid_path",
+			"${a.${b.c}.d}",
+			[]Token{
+				{Kind: TokenLiteral, Value: "${a.", Start: 0, End: 4},
+				{Kind: TokenRef, Value: "b.c", Start: 4, End: 10},
+				{Kind: TokenLiteral, Value: ".d}", Start: 10, End: 13},
+			},
+		},
 	}
 
 	for _, tt := range tests {
@@ -195,7 +222,6 @@ func TestParseErrors(t *testing.T) {
 		input   string
 		errContains string
 	}{
-		{"nested_reference", "${var.foo_${var.tail}}", "nested variable references are not supported"},
 		{"unterminated_ref", "${a.b", "unterminated"},
 		{"empty_ref", "${}", "empty"},
 		{"trailing_hyphen", "${foo.bar-}", "invalid"},