diff --git a/cmd/crd-schema-extractor/validate.go b/cmd/crd-schema-extractor/validate.go index 87ce76a..98edf3f 100644 --- a/cmd/crd-schema-extractor/validate.go +++ b/cmd/crd-schema-extractor/validate.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strings" "github.com/spf13/cobra" @@ -61,6 +62,12 @@ func validateRunE(cmd *cobra.Command, args []string) error { if src.URL == "" { errs = append(errs, fmt.Sprintf("source %s: url type requires 'url' field", src.Name)) } + if strings.Contains(src.URL, "{version}") && src.Version == "" { + errs = append(errs, fmt.Sprintf("source %s: url contains {version} placeholder but version is empty", src.Name)) + } + if src.URL != "" && !strings.Contains(src.URL, "{version}") { + errs = append(errs, fmt.Sprintf("source %s: url must contain {version} placeholder to keep version in sync", src.Name)) + } case "": errs = append(errs, fmt.Sprintf("source %s: missing 'type' field", src.Name)) default: diff --git a/internal/fetcher/url.go b/internal/fetcher/url.go index fe3e55f..737a001 100644 --- a/internal/fetcher/url.go +++ b/internal/fetcher/url.go @@ -30,16 +30,18 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro client = &http.Client{Timeout: httpTimeout} } + resolvedURL := src.ResolvedURL() + var resp *http.Response var lastErr error for attempt := 1; attempt <= maxRetries; attempt++ { - log.Debug().Str("url", src.URL).Int("attempt", attempt).Msg("fetching manifest") + log.Debug().Str("url", resolvedURL).Int("attempt", attempt).Msg("fetching manifest") var err error - resp, err = client.Get(src.URL) + resp, err = client.Get(resolvedURL) if err != nil { - lastErr = fmt.Errorf("downloading %s: %w", src.URL, err) + lastErr = fmt.Errorf("downloading %s: %w", resolvedURL, err) log.Warn().Err(err).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch failed") if attempt < maxRetries { time.Sleep(retryInterval) @@ -48,7 +50,7 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro } if resp.StatusCode != http.StatusOK { resp.Body.Close() - lastErr = fmt.Errorf("downloading %s: HTTP %d", src.URL, resp.StatusCode) + lastErr = fmt.Errorf("downloading %s: HTTP %d", resolvedURL, resp.StatusCode) log.Warn().Int("status", resp.StatusCode).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch returned non-200") if attempt < maxRetries { time.Sleep(retryInterval) @@ -67,9 +69,9 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro data, err := io.ReadAll(resp.Body) if err != nil { - return nil, fmt.Errorf("reading response from %s: %w", src.URL, err) + return nil, fmt.Errorf("reading response from %s: %w", resolvedURL, err) } - log.Debug().Str("url", src.URL).Int("bytes", len(data)).Msg("manifest downloaded") + log.Debug().Str("url", resolvedURL).Int("bytes", len(data)).Msg("manifest downloaded") return &Result{Data: data}, nil } diff --git a/internal/fetcher/url_test.go b/internal/fetcher/url_test.go index 3a9d5e3..e6c7453 100644 --- a/internal/fetcher/url_test.go +++ b/internal/fetcher/url_test.go @@ -85,6 +85,33 @@ func TestURLFetcherRetryExhaustion(t *testing.T) { } } +func TestURLFetcherVersionInterpolation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/releases/download/v1.7.1/crds.yaml" { + t.Errorf("unexpected path: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + return + } + w.Write([]byte("interpolated")) + })) + defer srv.Close() + + f := &URLFetcher{Client: srv.Client()} + result, err := f.Fetch(nopLog, source.Source{ + Name: "test-interp", + Type: "url", + URL: srv.URL + "/releases/download/{version}/crds.yaml", + Version: "v1.7.1", + }) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + + if string(result.Data) != "interpolated" { + t.Errorf("Data = %q, want %q", string(result.Data), "interpolated") + } +} + func TestURLFetcherConnectionError(t *testing.T) { // Use a URL that will fail to connect f := &URLFetcher{Client: &http.Client{}} diff --git a/internal/source/source.go b/internal/source/source.go index bb3a2ce..e4c68c9 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -99,6 +99,13 @@ func LoadAll(dir string) (map[string][]Source, error) { return result, nil } +// ResolvedURL returns the source URL with {version} placeholders replaced +// by the source's Version field value. If the URL contains no placeholders, +// it is returned as-is. +func (s Source) ResolvedURL() string { + return strings.ReplaceAll(s.URL, "{version}", s.Version) +} + // All returns a flat list of all sources across all API groups. func All(grouped map[string][]Source) []Source { var all []Source diff --git a/internal/source/source_test.go b/internal/source/source_test.go index e2723bd..1b90178 100644 --- a/internal/source/source_test.go +++ b/internal/source/source_test.go @@ -227,6 +227,44 @@ func TestLoadMissingPath(t *testing.T) { } } +func TestResolvedURL(t *testing.T) { + tests := []struct { + name string + src Source + want string + }{ + { + name: "placeholder replaced", + src: Source{URL: "https://github.com/org/repo/releases/download/{version}/crds.yaml", Version: "v1.7.1"}, + want: "https://github.com/org/repo/releases/download/v1.7.1/crds.yaml", + }, + { + name: "multiple placeholders replaced", + src: Source{URL: "https://example.com/{version}/download/{version}/crds.yaml", Version: "v2.0.0"}, + want: "https://example.com/v2.0.0/download/v2.0.0/crds.yaml", + }, + { + name: "no placeholder returns url as-is", + src: Source{URL: "https://example.com/crds.yaml", Version: "v1.0.0"}, + want: "https://example.com/crds.yaml", + }, + { + name: "empty url", + src: Source{URL: "", Version: "v1.0.0"}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.src.ResolvedURL() + if got != tt.want { + t.Errorf("ResolvedURL() = %q, want %q", got, tt.want) + } + }) + } +} + func TestAll(t *testing.T) { grouped := map[string][]Source{ "a.example.com": {{Name: "a1"}, {Name: "a2"}}, diff --git a/source.schema.json b/source.schema.json index c66c024..c5e1ea1 100644 --- a/source.schema.json +++ b/source.schema.json @@ -48,7 +48,7 @@ "url": { "type": "string", "format": "uri", - "description": "Direct URL to a YAML manifest containing CRDs. Required when type is url." + "description": "Direct URL to a YAML manifest containing CRDs. Required when type is url. Supports {version} placeholder which is replaced with the version field value before fetching." }, "version": { "type": "string",