From f7fc8552ed7ecb29c6583099a30c8007752e1bc4 Mon Sep 17 00:00:00 2001 From: Marco Bulgarini Date: Wed, 11 Mar 2026 10:53:45 +0100 Subject: [PATCH] Add {version} placeholder interpolation for URL-type sources The url field now supports a {version} placeholder that gets resolved to the source's version field value before fetching. This makes version the single source of truth, preventing drift between the URL path and the version field (e.g. kubevirt url pointing at v1.7.1 while version says v1.6.1). Renovate only needs to bump version, and the URL stays correct automatically. Validation requires {version} to be present in all url-type source URLs and rejects it when version is empty. Co-Authored-By: Claude Opus 4.6 --- cmd/crd-schema-extractor/validate.go | 7 +++++ internal/fetcher/url.go | 14 +++++----- internal/fetcher/url_test.go | 27 ++++++++++++++++++++ internal/source/source.go | 7 +++++ internal/source/source_test.go | 38 ++++++++++++++++++++++++++++ source.schema.json | 2 +- 6 files changed, 88 insertions(+), 7 deletions(-) diff --git a/cmd/crd-schema-extractor/validate.go b/cmd/crd-schema-extractor/validate.go index 87ce76a..98edf3f 100644 --- a/cmd/crd-schema-extractor/validate.go +++ b/cmd/crd-schema-extractor/validate.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "github.com/spf13/cobra" @@ -61,6 +62,12 @@ func validateRunE(cmd *cobra.Command, args []string) error { if src.URL == "" { errs = append(errs, fmt.Sprintf("source %s: url type requires 'url' field", src.Name)) } + if strings.Contains(src.URL, "{version}") && src.Version == "" { + errs = append(errs, fmt.Sprintf("source %s: url contains {version} placeholder but version is empty", src.Name)) + } + if src.URL != "" && !strings.Contains(src.URL, "{version}") { + errs = append(errs, fmt.Sprintf("source %s: url must contain {version} placeholder to keep version in sync", src.Name)) + } case "": errs = append(errs, fmt.Sprintf("source %s: missing 'type' field", src.Name)) default: diff --git a/internal/fetcher/url.go b/internal/fetcher/url.go index fe3e55f..737a001 100644 --- a/internal/fetcher/url.go +++ b/internal/fetcher/url.go @@ -30,16 +30,18 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro client = &http.Client{Timeout: httpTimeout} } + resolvedURL := src.ResolvedURL() + var resp *http.Response var lastErr error for attempt := 1; attempt <= maxRetries; attempt++ { - log.Debug().Str("url", src.URL).Int("attempt", attempt).Msg("fetching manifest") + log.Debug().Str("url", resolvedURL).Int("attempt", attempt).Msg("fetching manifest") var err error - resp, err = client.Get(src.URL) + resp, err = client.Get(resolvedURL) if err != nil { - lastErr = fmt.Errorf("downloading %s: %w", src.URL, err) + lastErr = fmt.Errorf("downloading %s: %w", resolvedURL, err) log.Warn().Err(err).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch failed") if attempt < maxRetries { time.Sleep(retryInterval) @@ -48,7 +50,7 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro } if resp.StatusCode != http.StatusOK { resp.Body.Close() - lastErr = fmt.Errorf("downloading %s: HTTP %d", src.URL, resp.StatusCode) + lastErr = fmt.Errorf("downloading %s: HTTP %d", resolvedURL, resp.StatusCode) log.Warn().Int("status", resp.StatusCode).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch returned non-200") if attempt < maxRetries { time.Sleep(retryInterval) @@ -67,9 +69,9 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro data, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("reading response from %s: %w", src.URL, err) + return nil, fmt.Errorf("reading response from %s: %w", resolvedURL, err) } - log.Debug().Str("url", src.URL).Int("bytes", len(data)).Msg("manifest downloaded") + log.Debug().Str("url", resolvedURL).Int("bytes", len(data)).Msg("manifest downloaded") return &Result{Data: data}, nil } diff --git a/internal/fetcher/url_test.go b/internal/fetcher/url_test.go index 3a9d5e3..e6c7453 100644 --- a/internal/fetcher/url_test.go +++ b/internal/fetcher/url_test.go @@ -85,6 +85,33 @@ func TestURLFetcherRetryExhaustion(t *testing.T) { } } +func TestURLFetcherVersionInterpolation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/releases/download/v1.7.1/crds.yaml" { + t.Errorf("unexpected path: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + w.Write([]byte("interpolated")) + })) + defer srv.Close() + + f := &URLFetcher{Client: srv.Client()} + result, err := f.Fetch(nopLog, source.Source{ + Name: "test-interp", + Type: "url", + URL: srv.URL + "/releases/download/{version}/crds.yaml", + Version: "v1.7.1", + }) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + + if string(result.Data) != "interpolated" { + t.Errorf("Data = %q, want %q", string(result.Data), "interpolated") + } +} + func TestURLFetcherConnectionError(t *testing.T) { // Use a URL that will fail to connect f := &URLFetcher{Client: &http.Client{}} diff --git a/internal/source/source.go b/internal/source/source.go index bb3a2ce..e4c68c9 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -99,6 +99,13 @@ func LoadAll(dir string) (map[string][]Source, error) { return result, nil } +// ResolvedURL returns the source URL with {version} placeholders replaced +// by the source's Version field value. If the URL contains no placeholders, +// it is returned as-is. +func (s Source) ResolvedURL() string { + return strings.ReplaceAll(s.URL, "{version}", s.Version) +} + // All returns a flat list of all sources across all API groups. func All(grouped map[string][]Source) []Source { var all []Source diff --git a/internal/source/source_test.go b/internal/source/source_test.go index e2723bd..1b90178 100644 --- a/internal/source/source_test.go +++ b/internal/source/source_test.go @@ -227,6 +227,44 @@ func TestLoadMissingPath(t *testing.T) { } } +func TestResolvedURL(t *testing.T) { + tests := []struct { + name string + src Source + want string + }{ + { + name: "placeholder replaced", + src: Source{URL: "https://github.com/org/repo/releases/download/{version}/crds.yaml", Version: "v1.7.1"}, + want: "https://github.com/org/repo/releases/download/v1.7.1/crds.yaml", + }, + { + name: "multiple placeholders replaced", + src: Source{URL: "https://example.com/{version}/download/{version}/crds.yaml", Version: "v2.0.0"}, + want: "https://example.com/v2.0.0/download/v2.0.0/crds.yaml", + }, + { + name: "no placeholder returns url as-is", + src: Source{URL: "https://example.com/crds.yaml", Version: "v1.0.0"}, + want: "https://example.com/crds.yaml", + }, + { + name: "empty url", + src: Source{URL: "", Version: "v1.0.0"}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.src.ResolvedURL() + if got != tt.want { + t.Errorf("ResolvedURL() = %q, want %q", got, tt.want) + } + }) + } +} + func TestAll(t *testing.T) { grouped := map[string][]Source{ "a.example.com": {{Name: "a1"}, {Name: "a2"}}, diff --git a/source.schema.json b/source.schema.json index c66c024..c5e1ea1 100644 --- a/source.schema.json +++ b/source.schema.json @@ -48,7 +48,7 @@ "url": { "type": "string", "format": "uri", - "description": "Direct URL to a YAML manifest containing CRDs. Required when type is url." + "description": "Direct URL to a YAML manifest containing CRDs. Required when type is url. Supports {version} placeholder which is replaced with the version field value before fetching." }, "version": { "type": "string",