From 30161ec9c79a8debdc555980feaca9c22d7a8672 Mon Sep 17 00:00:00 2001 From: Aditya Menon Date: Mon, 16 Feb 2026 08:02:37 +0530 Subject: [PATCH] feat: support nested ref+ expressions Resolve nested ref+ expressions like ref+echo://ref+envsubst://$VAR/path by pre-processing inside-out before the main expansion loop runs. Closes #794 Signed-off-by: Aditya Menon --- pkg/expansion/expand_match.go | 78 +++++++++++++ pkg/expansion/expand_match_test.go | 178 +++++++++++++++++++++++++++++ vals_test.go | 41 +++++++ 3 files changed, 297 insertions(+) diff --git a/pkg/expansion/expand_match.go b/pkg/expansion/expand_match.go index fc7e928b..f2a86ff4 100644 --- a/pkg/expansion/expand_match.go +++ b/pkg/expansion/expand_match.go @@ -15,11 +15,83 @@ type ExpandRegexMatch struct { var DefaultRefRegexp = regexp.MustCompile(`((secret)?ref)\+([^\+:]*:\/\/[^\+\n ]+[^\+\n ",])\+?`) +const maxNestingDepth = 10 + +// refPrefixRegexp detects ref+ or secretref+ prefixes to identify potential nesting. +var refPrefixRegexp = regexp.MustCompile(`(secret)?ref\+`) + func (e *ExpandRegexMatch) shouldExpand(kind string) bool { return len(e.Only) == 0 || slices.Contains(e.Only, kind) } +// resolveInnerRefs finds nested ref+ expressions and resolves them inside-out. +// For example, ref+echo://ref+envsubst://$VAR/path will first resolve the inner +// ref+envsubst expression, then the outer ref+echo expression. +func (e *ExpandRegexMatch) resolveInnerRefs(s string, depth int) (string, error) { + if depth >= maxNestingDepth { + return "", fmt.Errorf("maximum nesting depth (%d) exceeded", maxNestingDepth) + } + + positions := refPrefixRegexp.FindAllStringIndex(s, -1) + if len(positions) <= 1 { + return s, nil + } + + // Find the rightmost ref that is actually nested (no separator between it + // and the preceding ref+ prefix). Work from innermost outward. + for i := len(positions) - 1; i >= 1; i-- { + between := s[positions[i-1][1]:positions[i][0]] + if strings.ContainsAny(between, " \n\r\t\",") { + continue // Independent ref, not nested + } + + // This ref is nested — resolve it + start := positions[i][0] + substring := s[start:] + ixs := e.Target.FindStringSubmatchIndex(substring) + if ixs == nil { + continue + } + kind := substring[ixs[2]:ixs[3]] + if !e.shouldExpand(kind) { + continue + } + + ref := substring[ixs[6]:ixs[7]] + val, err := e.Lookup(ref) + if err != nil { + return "", fmt.Errorf("expand %s: %v", ref, err) + } + + // Nested refs become part of an outer URI, so they must resolve to scalar values. + if val == nil { + return "", fmt.Errorf("nested ref %s resolved to nil", ref) + } + switch val.(type) { + case string, bool, + int, int8, int16, int32, int64, + uint, uint8, uint16, uint32, uint64, + float32, float64: + // Scalar types that format cleanly into a URI. + default: + return "", fmt.Errorf("nested ref %s resolved to %T; nested refs must resolve to scalar values", ref, val) + } + + replaceStart := start + ixs[0] + replaceEnd := start + ixs[1] + result := s[:replaceStart] + fmt.Sprintf("%v", val) + s[replaceEnd:] + return e.resolveInnerRefs(result, depth+1) + } + + return s, nil +} + func (e *ExpandRegexMatch) InString(s string) (string, error) { + s, err := e.resolveInnerRefs(s, 0) + if err != nil { + return "", err + } + var sb strings.Builder for { ixs := e.Target.FindStringSubmatchIndex(s) @@ -50,6 +122,12 @@ func (e *ExpandRegexMatch) InString(s string) (string, error) { // If the entire string matches the regex, it expands and preserves the type. // If only part of the string matches, it expands as a string. func (e *ExpandRegexMatch) InValue(s string) (interface{}, error) { + resolved, err := e.resolveInnerRefs(s, 0) + if err != nil { + return nil, err + } + s = resolved + ixs := e.Target.FindStringSubmatchIndex(s) switch { // No match, return as is diff --git a/pkg/expansion/expand_match_test.go b/pkg/expansion/expand_match_test.go index 6e546f91..990a9be8 100644 --- a/pkg/expansion/expand_match_test.go +++ b/pkg/expansion/expand_match_test.go @@ -4,6 +4,7 @@ import ( "net/url" "reflect" "regexp" + "strings" "testing" ) @@ -15,6 +16,56 @@ func TestExpandRegexpMatchInString(t *testing.T) { expected string only []string }{ + { + name: "nested ref", + regex: DefaultRefRegexp, + input: "ref+echo://ref+echo://inner/value", + expected: "echo-echo-inner--/value", + }, + { + name: "triple nested ref", + regex: DefaultRefRegexp, + input: "ref+echo://ref+echo://ref+echo://deep/value", + expected: "echo-echo-echo-deep---/value", + }, + { + name: "nested ref with surrounding text", + regex: DefaultRefRegexp, + input: "prefix ref+echo://ref+echo://inner/value suffix", + expected: "prefix echo-echo-inner--/value suffix", + }, + { + name: "mixed nested and independent refs", + regex: DefaultRefRegexp, + input: "ref+echo://simple ref+echo://ref+echo://inner/value", + expected: "echo-simple- echo-echo-inner--/value", + }, + { + name: "nested secretref", + regex: DefaultRefRegexp, + input: "secretref+echo://secretref+echo://inner/value", + expected: "echo-echo-inner--/value", + }, + { + name: "mixed nested ref and secretref", + regex: DefaultRefRegexp, + input: "ref+echo://secretref+echo://inner/value", + expected: "echo-echo-inner--/value", + }, + { + name: "nested ref with only filter on inner", + regex: DefaultRefRegexp, + only: []string{"ref"}, + input: "ref+echo://secretref+echo://inner/value", + expected: "echo-secretref-echo://inner/value", + }, + { + name: "nested ref with only filter on outer", + regex: DefaultRefRegexp, + only: []string{"secretref"}, + input: "secretref+echo://ref+echo://inner/value", + expected: "echo-ref-echo://inner/value", + }, { name: "ref", regex: DefaultRefRegexp, @@ -251,3 +302,130 @@ func TestExpandRegexpMatchInMap(t *testing.T) { }) } } + +func TestResolveInnerRefs(t *testing.T) { + lookup := func(m string) (interface{}, error) { + parsed, err := url.Parse(m) + if err != nil { + return "", err + } + return parsed.Scheme + "-" + parsed.Host + "-" + parsed.Path, nil + } + + expand := ExpandRegexMatch{ + Target: DefaultRefRegexp, + Lookup: lookup, + } + + testcases := []struct { + name string + input string + expected string + }{ + { + name: "no ref prefix", + input: "just a plain string", + expected: "just a plain string", + }, + { + name: "single ref unchanged", + input: "ref+echo://simple/value", + expected: "ref+echo://simple/value", + }, + { + name: "single nesting resolved", + input: "ref+echo://ref+echo://inner/value", + expected: "ref+echo://echo-inner-/value", + }, + { + name: "double nesting resolved", + input: "ref+echo://ref+echo://ref+echo://deep/value", + expected: "ref+echo://echo-echo-deep--/value", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + actual, err := expand.resolveInnerRefs(tc.input, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if actual != tc.expected { + t.Errorf("expected: %s, got: %s", tc.expected, actual) + } + }) + } +} + +func TestResolveInnerRefsNonScalarError(t *testing.T) { + input := "ref+echo://ref+echo://trigger/value" + + t.Run("map value", func(t *testing.T) { + expand := ExpandRegexMatch{ + Target: DefaultRefRegexp, + Lookup: func(m string) (interface{}, error) { + return map[string]interface{}{"key": "value"}, nil + }, + } + _, err := expand.resolveInnerRefs(input, 0) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "scalar") { + t.Fatalf("expected scalar type error, got: %v", err) + } + }) + + t.Run("nil value", func(t *testing.T) { + expand := ExpandRegexMatch{ + Target: DefaultRefRegexp, + Lookup: func(m string) (interface{}, error) { + return nil, nil + }, + } + _, err := expand.resolveInnerRefs(input, 0) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "nil") { + t.Fatalf("expected nil error, got: %v", err) + } + }) + + t.Run("slice value", func(t *testing.T) { + expand := ExpandRegexMatch{ + Target: DefaultRefRegexp, + Lookup: func(m string) (interface{}, error) { + return []string{"a", "b"}, nil + }, + } + _, err := expand.resolveInnerRefs(input, 0) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "scalar") { + t.Fatalf("expected scalar type error, got: %v", err) + } + }) +} + +func TestResolveInnerRefsDepthLimit(t *testing.T) { + lookup := func(m string) (interface{}, error) { + // Always returns another nested ref to trigger infinite recursion + return "ref+echo://nested/value", nil + } + + expand := ExpandRegexMatch{ + Target: DefaultRefRegexp, + Lookup: lookup, + } + + input := "ref+echo://ref+echo://trigger/value" + _, err := expand.resolveInnerRefs(input, 0) + if err == nil { + t.Fatal("expected depth limit error, got nil") + } + if !strings.Contains(err.Error(), "maximum nesting depth") { + t.Fatalf("expected depth limit error, got: %v", err) + } +} diff --git a/vals_test.go b/vals_test.go index f038fdca..3a306e3e 100644 --- a/vals_test.go +++ b/vals_test.go @@ -236,6 +236,31 @@ func TestFlatten(t *testing.T) { } } +func TestGetNested(t *testing.T) { + testCases := []struct { + envVars map[string]string + name string + code string + expected string + }{ + {nil, "nested echo", "ref+echo://ref+echo://inner/value", "inner/value"}, + {nil, "triple nested echo", "ref+echo://ref+echo://ref+echo://deep/value", "deep/value"}, + {nil, "non-nested unchanged", "ref+echo://simple/value", "simple/value"}, + {map[string]string{"TEST_NESTED_VAR": "resolved"}, "nested envsubst in echo", "ref+echo://ref+envsubst://$TEST_NESTED_VAR/path", "resolved/path"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for k, v := range tc.envVars { + t.Setenv(k, v) + } + got, err := Get(tc.code, Options{}) + require.NoError(t, err) + require.Equal(t, tc.expected, got) + }) + } +} + func TestTextInput(t *testing.T) { t.Run("read file", func(t *testing.T) { dir := t.TempDir() @@ -286,6 +311,22 @@ func TestTextInput(t *testing.T) { }) } +func TestEvalNested(t *testing.T) { + t.Setenv("TEST_NESTED_VAR", "resolved") + + template := map[string]interface{}{ + "key1": "ref+echo://ref+echo://inner/value", + "key2": "ref+echo://simple/value", + "key3": "ref+echo://ref+envsubst://$TEST_NESTED_VAR/path", + } + + got, err := Eval(template) + require.NoError(t, err) + require.Equal(t, "inner/value", got["key1"]) + require.Equal(t, "simple/value", got["key2"]) + require.Equal(t, "resolved/path", got["key3"]) +} + func TestEvalNodesWithDictionaries(t *testing.T) { yamlDocs := `- entry: first username: ref+echo://secrets.enc.yaml