Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/crd-schema-extractor/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"fmt"
"strings"

"github.com/spf13/cobra"

Expand Down Expand Up @@ -61,6 +62,12 @@ func validateRunE(cmd *cobra.Command, args []string) error {
if src.URL == "" {
errs = append(errs, fmt.Sprintf("source %s: url type requires 'url' field", src.Name))
}
if strings.Contains(src.URL, "{version}") && src.Version == "" {
errs = append(errs, fmt.Sprintf("source %s: url contains {version} placeholder but version is empty", src.Name))
}
if src.URL != "" && !strings.Contains(src.URL, "{version}") {
errs = append(errs, fmt.Sprintf("source %s: url must contain {version} placeholder to keep version in sync", src.Name))
}
case "":
errs = append(errs, fmt.Sprintf("source %s: missing 'type' field", src.Name))
default:
Expand Down
14 changes: 8 additions & 6 deletions internal/fetcher/url.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,18 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro
client = &http.Client{Timeout: httpTimeout}
}

resolvedURL := src.ResolvedURL()

var resp *http.Response
var lastErr error

for attempt := 1; attempt <= maxRetries; attempt++ {
log.Debug().Str("url", src.URL).Int("attempt", attempt).Msg("fetching manifest")
log.Debug().Str("url", resolvedURL).Int("attempt", attempt).Msg("fetching manifest")

var err error
resp, err = client.Get(src.URL)
resp, err = client.Get(resolvedURL)
if err != nil {
lastErr = fmt.Errorf("downloading %s: %w", src.URL, err)
lastErr = fmt.Errorf("downloading %s: %w", resolvedURL, err)
log.Warn().Err(err).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch failed")
if attempt < maxRetries {
time.Sleep(retryInterval)
Expand All @@ -48,7 +50,7 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
lastErr = fmt.Errorf("downloading %s: HTTP %d", src.URL, resp.StatusCode)
lastErr = fmt.Errorf("downloading %s: HTTP %d", resolvedURL, resp.StatusCode)
log.Warn().Int("status", resp.StatusCode).Int("attempt", attempt).Int("max", maxRetries).Msg("fetch returned non-200")
if attempt < maxRetries {
time.Sleep(retryInterval)
Expand All @@ -67,9 +69,9 @@ func (f *URLFetcher) Fetch(log zerolog.Logger, src source.Source) (*Result, erro

data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading response from %s: %w", src.URL, err)
return nil, fmt.Errorf("reading response from %s: %w", resolvedURL, err)
}

log.Debug().Str("url", src.URL).Int("bytes", len(data)).Msg("manifest downloaded")
log.Debug().Str("url", resolvedURL).Int("bytes", len(data)).Msg("manifest downloaded")
return &Result{Data: data}, nil
}
27 changes: 27 additions & 0 deletions internal/fetcher/url_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,33 @@ func TestURLFetcherRetryExhaustion(t *testing.T) {
}
}

func TestURLFetcherVersionInterpolation(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/releases/download/v1.7.1/crds.yaml" {
t.Errorf("unexpected path: %s", r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
w.Write([]byte("interpolated"))
}))
defer srv.Close()

f := &URLFetcher{Client: srv.Client()}
result, err := f.Fetch(nopLog, source.Source{
Name: "test-interp",
Type: "url",
URL: srv.URL + "/releases/download/{version}/crds.yaml",
Version: "v1.7.1",
})
if err != nil {
t.Fatalf("Fetch: %v", err)
}

if string(result.Data) != "interpolated" {
t.Errorf("Data = %q, want %q", string(result.Data), "interpolated")
}
}

func TestURLFetcherConnectionError(t *testing.T) {
// Use a URL that will fail to connect
f := &URLFetcher{Client: &http.Client{}}
Expand Down
7 changes: 7 additions & 0 deletions internal/source/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ func LoadAll(dir string) (map[string][]Source, error) {
return result, nil
}

// ResolvedURL returns the source URL with {version} placeholders replaced
// by the source's Version field value. If the URL contains no placeholders,
// it is returned as-is.
func (s Source) ResolvedURL() string {
return strings.ReplaceAll(s.URL, "{version}", s.Version)
}

// All returns a flat list of all sources across all API groups.
func All(grouped map[string][]Source) []Source {
var all []Source
Expand Down
38 changes: 38 additions & 0 deletions internal/source/source_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,44 @@ func TestLoadMissingPath(t *testing.T) {
}
}

func TestResolvedURL(t *testing.T) {
tests := []struct {
name string
src Source
want string
}{
{
name: "placeholder replaced",
src: Source{URL: "https://github.com/org/repo/releases/download/{version}/crds.yaml", Version: "v1.7.1"},
want: "https://github.com/org/repo/releases/download/v1.7.1/crds.yaml",
},
{
name: "multiple placeholders replaced",
src: Source{URL: "https://example.com/{version}/download/{version}/crds.yaml", Version: "v2.0.0"},
want: "https://example.com/v2.0.0/download/v2.0.0/crds.yaml",
},
{
name: "no placeholder returns url as-is",
src: Source{URL: "https://example.com/crds.yaml", Version: "v1.0.0"},
want: "https://example.com/crds.yaml",
},
{
name: "empty url",
src: Source{URL: "", Version: "v1.0.0"},
want: "",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.src.ResolvedURL()
if got != tt.want {
t.Errorf("ResolvedURL() = %q, want %q", got, tt.want)
}
})
}
}

func TestAll(t *testing.T) {
grouped := map[string][]Source{
"a.example.com": {{Name: "a1"}, {Name: "a2"}},
Expand Down
2 changes: 1 addition & 1 deletion source.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"url": {
"type": "string",
"format": "uri",
"description": "Direct URL to a YAML manifest containing CRDs. Required when type is url."
"description": "Direct URL to a YAML manifest containing CRDs. Required when type is url. Supports {version} placeholder which is replaced with the version field value before fetching."
},
"version": {
"type": "string",
Expand Down