diff --git a/.gitignore b/.gitignore index 14f32407..c9cfad7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .idea *~ bin/ +dist/ cover.out vals .vscode diff --git a/pkg/expansion/expand_match.go b/pkg/expansion/expand_match.go index 1aa2c253..fc7e928b 100644 --- a/pkg/expansion/expand_match.go +++ b/pkg/expansion/expand_match.go @@ -3,17 +3,22 @@ package expansion import ( "fmt" "regexp" + "slices" "strings" ) type ExpandRegexMatch struct { Target *regexp.Regexp - Lookup func(string) (string, error) + Lookup func(string) (interface{}, error) Only []string } var DefaultRefRegexp = regexp.MustCompile(`((secret)?ref)\+([^\+:]*:\/\/[^\+\n ]+[^\+\n ",])\+?`) +func (e *ExpandRegexMatch) shouldExpand(kind string) bool { + return len(e.Only) == 0 || slices.Contains(e.Only, kind) +} + func (e *ExpandRegexMatch) InString(s string) (string, error) { var sb strings.Builder for { @@ -22,40 +27,60 @@ func (e *ExpandRegexMatch) InString(s string) (string, error) { sb.WriteString(s) return sb.String(), nil } + kind := s[ixs[2]:ixs[3]] - if len(e.Only) > 0 { - var shouldExpand bool - for _, k := range e.Only { - if k == kind { - shouldExpand = true - break - } - } - if !shouldExpand { - sb.WriteString(s) - return sb.String(), nil - } + if !e.shouldExpand(kind) { + sb.WriteString(s) + // FIXME: this skips the rest of the string, is this intended? + return sb.String(), nil } + ref := s[ixs[6]:ixs[7]] val, err := e.Lookup(ref) if err != nil { return "", fmt.Errorf("expand %s: %v", ref, err) } sb.WriteString(s[:ixs[0]]) - sb.WriteString(val) + fmt.Fprintf(&sb, "%v", val) s = s[ixs[1]:] } } +// InValue expands matches in the given string value. +// If the entire string matches the regex, it expands and preserves the type. +// If only part of the string matches, it expands as a string. +func (e *ExpandRegexMatch) InValue(s string) (interface{}, error) { + ixs := e.Target.FindStringSubmatchIndex(s) + switch { + // No match, return as is + case ixs == nil: + return s, nil + // Full match, expand preserving type + case ixs[0] == 0 && ixs[1] == len(s): + kind := s[ixs[2]:ixs[3]] + ref := s[ixs[6]:ixs[7]] + if !e.shouldExpand(kind) { + return s, nil + } + val, err := e.Lookup(ref) + if err != nil { + return nil, fmt.Errorf("expand %s: %v", ref, err) + } + return val, nil + // Partial match, expand as string + default: + return e.InString(s) + } +} + func (e *ExpandRegexMatch) InMap(target map[string]interface{}) (map[string]interface{}, error) { ret, err := ModifyStringValues(target, func(p string) (interface{}, error) { - ret, err := e.InString(p) + ret, err := e.InValue(p) if err != nil { return nil, err } return ret, nil }) - if err != nil { return nil, err } diff --git a/pkg/expansion/expand_match_test.go b/pkg/expansion/expand_match_test.go index 0fb834b1..6e546f91 100644 --- a/pkg/expansion/expand_match_test.go +++ b/pkg/expansion/expand_match_test.go @@ -144,7 +144,7 @@ func TestExpandRegexpMatchInString(t *testing.T) { tc := testcases[i] t.Run(tc.name, func(t *testing.T) { - lookup := func(m string) (string, error) { + lookup := func(m string) (interface{}, error) { parsed, err := url.Parse(m) if err != nil { return "", err @@ -160,7 +160,6 @@ func TestExpandRegexpMatchInString(t *testing.T) { } actual, err := expand.InString(tc.input) - if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -227,7 +226,7 @@ func TestExpandRegexpMatchInMap(t *testing.T) { tc := testcases[i] t.Run(tc.name, func(t *testing.T) { - lookup := func(m string) (string, error) { + lookup := func(m string) (interface{}, error) { parsed, err := url.Parse(m) if err != nil { return "", err @@ -242,7 +241,6 @@ func TestExpandRegexpMatchInMap(t *testing.T) { } actual, err := expand.InMap(tc.input) - if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/vals.go b/vals.go index a770213b..ef3a644c 100644 --- a/vals.go +++ b/vals.go @@ -119,9 +119,7 @@ const ( ProviderServercore = "servercore" ) -var ( - EnvFallbackPrefix = "VALS_" -) +var EnvFallbackPrefix = "VALS_" type Evaluator interface { Eval(map[string]interface{}) (map[string]interface{}, error) @@ -344,25 +342,23 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { expand := expansion.ExpandRegexMatch{ Only: only, Target: expansion.DefaultRefRegexp, - Lookup: func(key string) (string, error) { + Lookup: func(key string) (interface{}, error) { if val, ok := r.docCache.Get(key); ok { - valStr, ok := val.(string) - if ok { - return valStr, nil + if isTerminalValue(val) { + return val, nil } } uri, err := url.Parse(key) if err != nil { - return "", err + return nil, err } hash := uriToProviderHash(uri) p, err := updateProviders(uri, hash) - if err != nil { - return "", err + return nil, err } var frag string @@ -401,12 +397,12 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { if cachedStr, ok := r.strCache.Get(cacheKey); ok { str, ok = cachedStr.(string) if !ok { - return "", fmt.Errorf("error reading str from cache: unsupported value type %T", cachedStr) + return nil, fmt.Errorf("error reading str from cache: unsupported value type %T", cachedStr) } } else { str, err = p.GetString(path) if err != nil { - return "", err + return nil, err } r.strCache.Add(cacheKey, str) } @@ -418,7 +414,7 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { if cachedMap, ok := r.docCache.Get(mapRequestURI); ok { obj, ok = cachedMap.(map[string]interface{}) if !ok { - return "", fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap) + return nil, fmt.Errorf("error reading map from cache: unsupported value type %T", cachedMap) } } else if uri.Scheme == "httpjson" { // Due to the unpredictability in the structure of the JSON object, @@ -430,42 +426,45 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { // object, accommodating different configurations and variations. value, err := p.GetString(key) if err != nil { - return "", err + return nil, err } return value, nil } else { obj, err = p.GetStringMap(path) if err != nil { - return "", err + return nil, err } r.docCache.Add(mapRequestURI, obj) } keys := strings.Split(frag, "/") for i, k := range keys { - newobj := map[string]interface{}{} - switch t := obj[k].(type) { - case string: + t := obj[k] + if isTerminalValue(t) { if i != len(keys)-1 { - return "", fmt.Errorf("unexpected type of value for key at %d=%s in %v: expected map[string]interface{}, got %v(%T)", i, k, keys, t, t) + return nil, fmt.Errorf("unexpected type of value for key at %d=%s in %v: expected map[string]interface{}, got %v(%T)", i, k, keys, t, t) } r.docCache.Add(key, t) return t, nil + } + switch t := t.(type) { case map[string]interface{}: - newobj = t + obj = t case map[interface{}]interface{}: + obj := map[string]interface{}{} for k, v := range t { - newobj[fmt.Sprintf("%v", k)] = v + obj[fmt.Sprintf("%v", k)] = v } + default: + return nil, fmt.Errorf("unsupported type for key at %d=%s in %v: %v(%T)", i, k, keys, t, t) } - obj = newobj } if r.Options.FailOnMissingKeyInMap { - return "", fmt.Errorf("no value found for key %s", frag) + return nil, fmt.Errorf("no value found for key %s", frag) } - return "", nil + return nil, nil } }, } @@ -473,6 +472,15 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { return &expand, nil } +func isTerminalValue(v any) bool { + switch v.(type) { + case bool, int, string: + return true + default: + return false + } +} + // Eval replaces 'ref+://xxxxx' entries by their actual values func (r *Runtime) Eval(template map[string]interface{}) (map[string]interface{}, error) { expand, err := r.prepare() @@ -597,8 +605,10 @@ func applyEnvWithQuote(quote bool) func(map[string]interface{}, ...Options) ([]s } } -var Env = applyEnvWithQuote(false) -var QuotedEnv = applyEnvWithQuote(true) +var ( + Env = applyEnvWithQuote(false) + QuotedEnv = applyEnvWithQuote(true) +) type ExecConfig struct { Stdout io.Writer diff --git a/vals_test.go b/vals_test.go index cfe96f0b..f60c8aaa 100644 --- a/vals_test.go +++ b/vals_test.go @@ -18,7 +18,7 @@ func TestExec(t *testing.T) { // should evaluate to "x: baz" data := []byte("x: ref+echo://foo/bar/baz#/foo/bar") - require.NoError(t, os.WriteFile(filepath.Join(dir, "input.yaml"), data, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "input.yaml"), data, 0o644)) stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} @@ -43,7 +43,7 @@ func TestEnv(t *testing.T) { input["var3"] = "ref+echo://val'ue" input["var4"] = "ref+echo://'value" - var expected = []string{ + expected := []string{ "var1=value", "var2=val;ue", "var3=val'ue", @@ -65,7 +65,7 @@ func TestQuotedEnv(t *testing.T) { input["var3"] = "ref+echo://val'ue" input["var4"] = "ref+echo://'value" - var expected = []string{ + expected := []string{ "var1=value", `var2='val;ue'`, `var3='val'"'"'ue'`, @@ -155,13 +155,13 @@ func TestGet(t *testing.T) { } func TestEvalNodesWithDictionaries(t *testing.T) { - var yamlDocs = `- entry: first + yamlDocs := `- entry: first username: ref+echo://secrets.enc.yaml - entry: second username: ref+echo://secrets.enc.yaml ` - var expected = `- entry: first + expected := `- entry: first username: secrets.enc.yaml - entry: second username: secrets.enc.yaml @@ -188,16 +188,16 @@ func TestEvalNodesWithDictionaries(t *testing.T) { } func TestEvalNodesWithTime(t *testing.T) { - var yamlDocs = ` + yamlDocs := ` date: 2025-01-01 -datet_in_list: +datet_in_list: - from: 2025-01-01 datetime: 2025-01-01T12:34:56Z datetime_millis: 2025-01-01T12:34:56.789Z datetime_offset: 2025-01-01T12:34:56+01:00 ` - var expected = `date: "2025-01-01" + expected := `date: "2025-01-01" datet_in_list: - from: "2025-01-01" datetime: "2025-01-01T12:34:56Z" @@ -224,3 +224,46 @@ datetime_offset: "2025-01-01T12:34:56+01:00" require.Equal(t, expected, buf.String()) } + +func TestEvalNodesTypes(t *testing.T) { + tmpDir := t.TempDir() + + createTmpFile := func(t *testing.T, dir, name, content string) string { + tmpFilePath := filepath.Join(dir, name) + err := os.WriteFile(tmpFilePath, []byte(content), 0o600) + require.NoError(t, err) + return tmpFilePath + } + + secretYaml := ` +bool: true +int: 42 +string: "It's a string" +` + secretsFile := createTmpFile(t, tmpDir, "secrets.yaml", secretYaml) + + replacer := strings.NewReplacer("{file-ref}", "ref+file://"+secretsFile) + inputYaml := replacer.Replace(` +bool_value: {file-ref}#/bool +int_value: {file-ref}#/int +string_value: {file-ref}#/string +`) + inputFile := createTmpFile(t, tmpDir, "input.yaml", inputYaml) + + expected := `bool_value: true +int_value: 42 +string_value: It's a string +` + + input, err := Inputs(inputFile) + require.NoError(t, err) + + nodes, err := EvalNodes(input, Options{}) + require.NoError(t, err) + buf := new(strings.Builder) + + err = Output(buf, "", nodes) + require.NoError(t, err) + + require.Equal(t, expected, buf.String()) +}