From 7824ef9c36813e2eb487908e80c880bc0f0303e8 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Tue, 6 Jan 2026 15:47:20 -0600 Subject: [PATCH 1/3] go.mod: update toolkit Signed-off-by: Hank Donnay --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 95b6258ea..48ac68090 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936 github.com/package-url/packageurl-go v0.1.3 github.com/prometheus/client_golang v1.23.2 - github.com/quay/claircore/toolkit v1.3.0 + github.com/quay/claircore/toolkit v1.4.0 github.com/quay/claircore/updater/driver v1.0.0 github.com/quay/goval-parser v0.8.8 github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319 diff --git a/go.sum b/go.sum index 3498f4ec4..2178c2fbb 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/quay/claircore/toolkit v1.3.0 h1:QncygaArnuSKbkPESD2zMDz5xJLWlJxAGPZ69XCYA5o= -github.com/quay/claircore/toolkit v1.3.0/go.mod h1:REVn3WdU+9yMurWa+h9mfuiPWYzKqSaKIGWZ4Xanj5g= +github.com/quay/claircore/toolkit v1.4.0 h1:ygHG1pLAOTSk7r2Wmo/UbIz9WHJb+K3hhQnNIvLrBSQ= +github.com/quay/claircore/toolkit v1.4.0/go.mod h1:0cQXEt/BIYSxo/Wq6ItyAJOJzdvD6ty1wPZJ9xR3b6E= github.com/quay/goval-parser v0.8.8 h1:Uf+f9iF2GIR5GPUY2pGoa9il2+4cdES44ZlM0mWm4cA= github.com/quay/goval-parser v0.8.8/go.mod h1:Y0NTNfPYOC7yxsYKzJOrscTWUPq1+QbtHw4XpPXWPMc= github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319 h1:ukjThsA2ou7AmovpwtMVkNQSuoN/v5U16+JomTz3c7o= From 0b6ab33d13b722a60850e23bd6d5316c76f021e6 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Tue, 6 Jan 2026 15:47:03 -0600 Subject: [PATCH 2/3] wip: internal/matcher: add events Signed-off-by: Hank Donnay --- internal/matcher/controller.go | 157 ----------------------------- internal/matcher/match.go | 177 ++++++++++++++++++++++++++++++++- internal/matcher/match_test.go | 126 +++++++++++++++++++++++ 3 files changed, 299 insertions(+), 161 deletions(-) delete mode 100644 internal/matcher/controller.go create mode 100644 internal/matcher/match_test.go diff --git a/internal/matcher/controller.go b/internal/matcher/controller.go deleted file mode 100644 index a5e17e363..000000000 --- a/internal/matcher/controller.go +++ /dev/null @@ -1,157 +0,0 @@ -package matcher - -import ( - "context" - "log/slog" - "time" - - "github.com/quay/claircore" - "github.com/quay/claircore/datastore" - "github.com/quay/claircore/libvuln/driver" -) - -// Controller is a control structure used to find vulnerabilities affecting -// a set of packages. -type Controller struct { - // an implemented Matcher - m driver.Matcher - // a vulnstore.Vulnerability instance for querying vulnerabilities - store datastore.Vulnerability -} - -// NewController is a constructor for a Controller -func NewController(m driver.Matcher, store datastore.Vulnerability) *Controller { - return &Controller{ - m: m, - store: store, - } -} - -// Match is the entrypoint for [Controller]. -func (mc *Controller) Match(ctx context.Context, records []*claircore.IndexRecord) (map[string][]*claircore.Vulnerability, error) { - log := slog.With("matcher", mc.m.Name()) - // find the packages the matcher is interested in. - interested := mc.findInterested(records) - log.DebugContext(ctx, "interest", - "interested", len(interested), - "records", len(records)) - - // early return; do not call db at all - if len(interested) == 0 { - return map[string][]*claircore.Vulnerability{}, nil - } - - remoteMatcher, matchedVulns, err := mc.queryRemoteMatcher(ctx, interested) - if remoteMatcher { - if err != nil { - log.ErrorContext(ctx, "remote matcher error, returning empty results", "reason", err) - return map[string][]*claircore.Vulnerability{}, nil - } - return matchedVulns, nil - } - - dbSide, authoritative := mc.dbFilter() - log.DebugContext(ctx, "version filter compatible?", - "opt-in", dbSide, - "authoritative", authoritative) - - // query the vulnstore - vulns, err := mc.query(ctx, interested, dbSide) - if err != nil { - return nil, err - } - log.DebugContext(ctx, "query", "count", len(vulns)) - - if authoritative { - return vulns, nil - } - // filter the vulns - filteredVulns, err := mc.filter(ctx, interested, vulns) - if err != nil { - return nil, err - } - log.DebugContext(ctx, "filtered", "count", len(filteredVulns)) - return filteredVulns, nil -} - -// If RemoteMatcher exists, it will call the matcher service which runs on a remote -// machine and fetches the vulnerabilities associated with the IndexRecords. -func (mc *Controller) queryRemoteMatcher(ctx context.Context, interested []*claircore.IndexRecord) (bool, map[string][]*claircore.Vulnerability, error) { - f, ok := mc.m.(driver.RemoteMatcher) - if !ok { - return false, nil, nil - } - tctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - vulns, err := f.QueryRemoteMatcher(tctx, interested) - return true, vulns, err -} - -// DbFilter reports whether the db-side version filtering can be used, and -// whether it's authoritative. -func (mc *Controller) dbFilter() (bool, bool) { - f, ok := mc.m.(driver.VersionFilter) - if !ok { - return false, false - } - return true, f.VersionAuthoritative() -} - -func (mc *Controller) findInterested(records []*claircore.IndexRecord) []*claircore.IndexRecord { - out := []*claircore.IndexRecord{} - for _, record := range records { - if record.Package.NormalizedVersion.Kind == claircore.UnmatchableKind { - continue - } - if mc.m.Filter(record) { - out = append(out, record) - } - } - return out -} - -// Query asks the Matcher how we should query the vulnstore then performs the query and returns all -// matched vulnerabilities. -func (mc *Controller) query(ctx context.Context, interested []*claircore.IndexRecord, dbSide bool) (map[string][]*claircore.Vulnerability, error) { - // ask the matcher how we should query the vulnstore - matchers := mc.m.Query() - getOpts := datastore.GetOpts{ - Matchers: matchers, - Debug: true, - VersionFiltering: dbSide, - } - matches, err := mc.store.Get(ctx, interested, getOpts) - if err != nil { - return nil, err - } - return matches, nil -} - -// Filter method asks the matcher if the given package is affected by the returned vulnerability. if so; its added to a result map where the key is the package ID -// and the value is a Vulnerability. if not it is not added to the result. -func (mc *Controller) filter(ctx context.Context, interested []*claircore.IndexRecord, vulns map[string][]*claircore.Vulnerability) (map[string][]*claircore.Vulnerability, error) { - filtered := map[string][]*claircore.Vulnerability{} - for _, record := range interested { - match, err := filterVulns(ctx, mc.m, record, vulns[record.Package.ID]) - if err != nil { - return nil, err - } - filtered[record.Package.ID] = append(filtered[record.Package.ID], match...) - } - return filtered, nil -} - -// filter returns only the vulnerabilities affected by the provided package. -func filterVulns(ctx context.Context, m driver.Matcher, record *claircore.IndexRecord, vulns []*claircore.Vulnerability) ([]*claircore.Vulnerability, error) { - filtered := []*claircore.Vulnerability{} - for _, vuln := range vulns { - match, err := m.Vulnerable(ctx, record, vuln) - if err != nil { - return nil, err - } - if match { - filtered = append(filtered, vuln) - } - } - return filtered, nil -} diff --git a/internal/matcher/match.go b/internal/matcher/match.go index 3ba8eb372..0068cad3b 100644 --- a/internal/matcher/match.go +++ b/internal/matcher/match.go @@ -9,12 +9,14 @@ import ( "runtime" "sync" "sync/atomic" + "time" "golang.org/x/sync/errgroup" "github.com/quay/claircore" "github.com/quay/claircore/datastore" "github.com/quay/claircore/libvuln/driver" + "github.com/quay/claircore/toolkit/events" ) // BUG(hank) Match and EnrichedMatch have different semantics for errors @@ -46,7 +48,7 @@ func Match(ctx context.Context, ir *claircore.IndexReport, matchers []driver.Mat ctrlC := make(chan map[string][]*claircore.Vulnerability, lim) var errMu sync.Mutex errs := make([]error, 0, lim) - // fan out all controllers, write their output to ctrlC, close ctrlC once all writers finish + // fan out all workers, write their output to ctrlC, close ctrlC once all writers finish go func() { defer close(ctrlC) var wg sync.WaitGroup @@ -55,8 +57,7 @@ func Match(ctx context.Context, ir *claircore.IndexReport, matchers []driver.Mat m := matchers[i] go func() { defer wg.Done() - mc := NewController(m, store) - vulns, err := mc.Match(ctx, records) + vulns, err := matchOne(ctx, store, m, records) if err != nil { errMu.Lock() errs = append(errs, err) @@ -119,7 +120,7 @@ func EnrichedMatch(ctx context.Context, ir *claircore.IndexReport, ms []driver.M return mctx.Err() default: } - vs, err := NewController(m, s).Match(mctx, records) + vs, err := matchOne(ctx, s, m, records) if err != nil { return fmt.Errorf("matcher error: %w", err) } @@ -244,3 +245,171 @@ var _ driver.EnrichmentGetter = (*enrichmentGetter)(nil) func (e *enrichmentGetter) GetEnrichment(ctx context.Context, tags []string) ([]driver.EnrichmentRecord, error) { return e.s.GetEnrichment(ctx, e.name, tags) } + +type event struct { + remote bool + dbfilter bool + dbfilterAuthoritative bool + numRecords int + numInterested int + numVulnerabilities int + numMatched int +} + +func newEvent(m driver.Matcher) *event { + _, remote := m.(driver.RemoteMatcher) + var dbfilterAuthoritative bool + f, dbfilter := m.(driver.VersionFilter) + if dbfilter { + dbfilterAuthoritative = f.VersionAuthoritative() + } + return &event{ + remote: remote, + dbfilter: dbfilter, + dbfilterAuthoritative: dbfilterAuthoritative, + numRecords: -1, + numInterested: -1, + numVulnerabilities: -1, + numMatched: -1, + } +} + +func (ev *event) LogValue() slog.Value { + as := make([]slog.Attr, 3, 7) // Capacity for the number of fields in [event]. + as[0] = slog.Bool("remote", ev.remote) + as[1] = slog.Bool("dbfilter", ev.dbfilter) + as[2] = slog.Bool("dbfilter_authoritative", ev.dbfilterAuthoritative) + if ev.numRecords >= 0 { + as = append(as, slog.Int("records", ev.numRecords)) + } + if ev.numInterested >= 0 { + as = append(as, slog.Int("interested", ev.numInterested)) + } + if ev.numVulnerabilities >= 0 { + as = append(as, slog.Int("vulnerabilities", ev.numVulnerabilities)) + } + if ev.numMatched >= 0 { + as = append(as, slog.Int("matched", ev.numMatched)) + } + return slog.GroupValue(as...) +} + +// MatchOne uses the passed [driver.Matcher] to find vulnerabilities affecting +// the recorded packages. +func matchOne(ctx context.Context, + store datastore.Vulnerability, + m driver.Matcher, + records []*claircore.IndexRecord, +) (out map[string][]*claircore.Vulnerability, err error) { + name := m.Name() + ev := newEvent(m) + ev.numRecords = len(records) + defer func() { + l := events.Logger(ctx) + if err != nil { + l.ErrorContext(ctx, "match", + name, ev, + "reason", err, + ) + return + } + l.InfoContext(ctx, "match", + name, ev) + }() + log := slog.With("matcher", name) + + // Find the packages the matcher is interested in. + interested := make([]*claircore.IndexRecord, 0, len(records)) + for _, record := range records { + if record.Package.NormalizedVersion.Kind == claircore.UnmatchableKind { + continue + } + if m.Filter(record) { + interested = append(interested, record) + } + } + ev.numInterested = len(interested) + log.DebugContext(ctx, "interest", + "interested", len(interested), + "records", len(records)) + + // Early return; do not call DB at all. + if len(interested) == 0 { + return map[string][]*claircore.Vulnerability{}, nil + } + + // Attempt "remote" matching if relevant. + if f, ok := m.(driver.RemoteMatcher); ok { + // TODO(hank) Remove/modify this timeout? + tctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + out, err = f.QueryRemoteMatcher(tctx, interested) + if err != nil { + log.ErrorContext(ctx, "remote matcher error, returning empty results", "reason", err) + return map[string][]*claircore.Vulnerability{}, nil + } + return out, nil + } + + // Check whether the db-side version filtering can be used and whether it's authoritative. + var authoritative bool + vf, dbSide := m.(driver.VersionFilter) + if dbSide { + authoritative = vf.VersionAuthoritative() + } + log.DebugContext(ctx, "version filter compatible?", + "opt-in", dbSide, + "authoritative", authoritative) + + // Query the Vulnerability database. + out, err = store.Get(ctx, interested, datastore.GetOpts{ + Matchers: m.Query(), + VersionFiltering: dbSide, + }) + if err != nil { + return nil, err + } + ev.numVulnerabilities = len(out) + log.DebugContext(ctx, "query", "count", len(out)) + + // If the DB filtering is authoritative, this process is done. + if authoritative { + ev.numMatched = len(out) + return out, nil + } + // Filter the vulnerabilities locally: + vulns := out + out = make(map[string][]*claircore.Vulnerability) + ev.numMatched = 0 + for _, r := range interested { + var match []*claircore.Vulnerability + match, err = selectVulnerabilities(ctx, m, r, vulns[r.Package.ID]) + if err != nil { + return nil, err + } + if len(match) == 0 { + continue + } + ev.numMatched++ + out[r.Package.ID] = append(out[r.Package.ID], match...) + } + log.DebugContext(ctx, "filtered", "count", len(out)) + + return out, nil +} + +// SelectVulnerabilities returns only the vulnerabilities affected by the +// provided package. +func selectVulnerabilities(ctx context.Context, m driver.Matcher, r *claircore.IndexRecord, vs []*claircore.Vulnerability) ([]*claircore.Vulnerability, error) { + out := []*claircore.Vulnerability{} + for _, vuln := range vs { + match, err := m.Vulnerable(ctx, r, vuln) + if err != nil { + return nil, err + } + if match { + out = append(out, vuln) + } + } + return out, nil +} diff --git a/internal/matcher/match_test.go b/internal/matcher/match_test.go new file mode 100644 index 000000000..1358c9755 --- /dev/null +++ b/internal/matcher/match_test.go @@ -0,0 +1,126 @@ +package matcher + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "testing" + + "github.com/google/go-cmp/cmp" + "go.uber.org/mock/gomock" + + "github.com/quay/claircore" + "github.com/quay/claircore/datastore" + "github.com/quay/claircore/libvuln/driver" + mock_datastore "github.com/quay/claircore/test/mock/datastore" + mock_driver "github.com/quay/claircore/test/mock/driver" + "github.com/quay/claircore/toolkit/events" +) + +// TestEvent does just enough to test the event emitting bits. +func TestEvent(t *testing.T) { + t.Run("New", func(t *testing.T) { + ctrl := gomock.NewController(t) + m := mock_driver.NewMockMatcher(ctrl) + ev := newEvent(m) + got := ev.LogValue() + want := slog.GroupValue( + slog.Bool("remote", false), + slog.Bool("dbfilter", false), + slog.Bool("dbfilter_authoritative", false), + ) + if !got.Equal(want) { + t.Error(cmp.Diff(got, want)) + } + }) + + t.Run("NoHandler", func(t *testing.T) { + // Cannot use the "test.Logging" helper because of an import cycle. + ctx := t.Context() + runMatchOne(t, ctx) + }) + + t.Run("MatchOne", func(t *testing.T) { + var buf bytes.Buffer + // Cannot use the "test.Logging" helper because of an import cycle. + h := slog.NewJSONHandler(&buf, nil) + ctx := events.WithHandler(t.Context(), h) + runMatchOne(t, ctx) + + var evGot map[string]any + dec := json.NewDecoder(&buf) + dec.UseNumber() + if err := dec.Decode(&evGot); err != nil { + t.Errorf("decoding JSON: %v", err) + } + delete(evGot, slog.TimeKey) + evWant := map[string]any{ + slog.LevelKey: "INFO", + slog.MessageKey: "match", + matcherName: map[string]any{ + "dbfilter": false, + "dbfilter_authoritative": false, + "remote": false, + "interested": json.Number("1"), + "records": json.Number("1"), + "matched": json.Number("0"), + "vulnerabilities": json.Number("0"), + }, + } + if !cmp.Equal(evGot, evWant) { + t.Error(cmp.Diff(evGot, evWant)) + } + }) +} + +const matcherName = "mockmatcher" + +func runMatchOne(t *testing.T, ctx context.Context) { + t.Helper() + records := []*claircore.IndexRecord{ + { + Package: &claircore.Package{ + ID: "pkg1", + Name: "hello", + Version: "1-1", + Kind: claircore.BINARY, + Arch: "amd64", + }, + Distribution: &claircore.Distribution{ + ID: "dist1", + DID: "test", + }, + }, + } + + ctrl := gomock.NewController(t) + store := mock_datastore.NewMockVulnerability(ctrl) + store.EXPECT().Get( + gomock.Any(), + gomock.AssignableToTypeOf([]*claircore.IndexRecord{}), + gomock.AssignableToTypeOf(datastore.GetOpts{}), + ).Return( + map[string][]*claircore.Vulnerability{}, nil, + ) + + m := mock_driver.NewMockMatcher(ctrl) + m.EXPECT().Name().Return(matcherName) + m.EXPECT().Query().Return([]driver.MatchConstraint{ + driver.DistributionDID, + }) + m.EXPECT().Filter(records[0]).Return(true) + + got, err := matchOne(ctx, store, m, records) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + want := map[string][]*claircore.Vulnerability{} + if !cmp.Equal(got, want) { + t.Error(cmp.Diff(got, want)) + } + + if t.Failed() { + t.FailNow() + } +} From 9571ff711efd08a37dc94d474ef4a3c3addddcb2 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Tue, 6 Jan 2026 16:00:35 -0600 Subject: [PATCH 3/3] fixup! wip: internal/matcher: add events --- internal/matcher/match.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/matcher/match.go b/internal/matcher/match.go index 0068cad3b..2c3b425a8 100644 --- a/internal/matcher/match.go +++ b/internal/matcher/match.go @@ -305,16 +305,13 @@ func matchOne(ctx context.Context, ev := newEvent(m) ev.numRecords = len(records) defer func() { + lvl := slog.LevelInfo l := events.Logger(ctx) if err != nil { - l.ErrorContext(ctx, "match", - name, ev, - "reason", err, - ) - return + lvl = slog.LevelError + l = l.With("reason", err) } - l.InfoContext(ctx, "match", - name, ev) + l.Log(ctx, lvl, "match", name, ev) }() log := slog.With("matcher", name)