From fb3e0af7fbc3f15c53f374135f72c5e7e328447d Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 21:58:31 -0400 Subject: [PATCH 1/7] Add framework evidence paths and bump Go to 1.26.1 --- IMPLEMENTATION_CHECK.md | 2 +- README.md | 26 +- core/framework/eu-ai-act.yaml | 141 +++++++++- core/framework/evaluate.go | 255 ++++++++++++++++++ core/framework/evaluate_test.go | 137 ++++++++++ core/framework/framework.go | 106 ++++++-- core/framework/framework_test.go | 107 +++++++- core/framework/pci-dss.yaml | 68 ++++- core/framework/soc2.yaml | 103 ++++++- core/schema/schema_test.go | 44 +++ .../v1/framework-definition.schema.json | 39 ++- docs/api-contract.md | 2 +- frameworks/eu-ai-act.yaml | 141 +++++++++- frameworks/pci-dss.yaml | 68 ++++- frameworks/soc2.yaml | 103 ++++++- go.mod | 2 +- internal/tools/syncframeworks/main.go | 105 ++++++++ proof.go | 7 + proof_test.go | 36 +++ schemas/v1/framework-definition.schema.json | 39 ++- 20 files changed, 1437 insertions(+), 94 deletions(-) create mode 100644 core/framework/evaluate.go create mode 100644 core/framework/evaluate_test.go create mode 100644 internal/tools/syncframeworks/main.go diff --git a/IMPLEMENTATION_CHECK.md b/IMPLEMENTATION_CHECK.md index b5acfdf..76dfc6c 100644 --- a/IMPLEMENTATION_CHECK.md +++ b/IMPLEMENTATION_CHECK.md @@ -21,7 +21,7 @@ Status key: | FR4 Signing | PASS | Ed25519 + cosign signing/verification paths implemented for records/chains and bundle manifests, including cert/identity/issuer verify options and revocation-list verification. | | FR5 Canonicalization | PASS | JSON/SQL/URL/text/prompt canonicalization plus digest metadata (`algo_id`, `salt_id`) and HMAC-SHA-256 helpers in `core/canon`. | | FR6 Verification CLI | PASS | `verify`, `inspect`, `chain verify`, `types`, `frameworks`; bundle signature verification, custom type schema mapping, `--explain`, and exit code contract implemented. | -| FR7 Framework definitions | PASS | 8 frameworks in `frameworks/` and `core/framework/`; list/show implemented. | +| FR7 Framework definitions | PASS | 10 frameworks in `frameworks/` and `core/framework/`; list/show, schema validation, and deterministic evidence coverage evaluation implemented. | | FR8 Go module API | PASS | Primary API surface exported from `proof.go`. | | FR9 JSON schemas | PASS | Base + type schemas + chain/bundle/framework schemas in `schemas/v1/`. | diff --git a/README.md b/README.md index 7c4a86e..38ed347 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ PROOF_VERSION="$(gh release view --repo Clyra-AI/proof --json tagName -q .tagNam go install github.com/Clyra-AI/proof/cmd/proof@"${PROOF_VERSION}" proof types list # 18 built-in record types -proof frameworks list # 10 built-in starter frameworks (73 controls) +proof frameworks list # 10 built-in starter frameworks (79 controls) proof verify ./artifact # Verify any proof artifact offline ``` @@ -250,25 +250,33 @@ All digests carry `algo_id` (sha256 or hmac-sha256) and optional `salt_id` metad ## Compliance Framework Definitions -YAML files that declare what regulatory controls require — which record types, required fields, and evidence frequency. Zero evaluation logic. Configuration data consumed by downstream compliance tools. +YAML files declare what regulatory controls require and which evidence paths can satisfy them. Proof evaluates deterministic evidence coverage only. It does not decide regulatory applicability, scope gating, or compliance status. ```yaml controls: - id: article-12 title: Record-Keeping - required_record_types: [tool_invocation, decision, guardrail_activation, permission_check, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + evidence_sets: + - id: runtime_control + source_products: [gait] + required_record_types: [tool_invocation, permission_check, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: combined + source_products: [wrkr, gait] + required_record_types: [scan_finding, tool_invocation, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous ``` -10 built-in starter frameworks ship with v1 (73 controls total). Add custom frameworks via YAML. +10 built-in starter frameworks ship with v1 (79 controls total). Add custom frameworks via YAML. | Framework | Scope | |---|---| -| EU AI Act | Articles 9, 12, 14 (starter mapping) | -| SOC 2 | CC6, CC7, CC8 (starter mapping) | +| EU AI Act | Articles 9, 12, 13, 14, 15, 26 (starter mapping) | +| SOC 2 | CC6.1, CC6.3, CC7.1, CC8.1 (starter mapping) | | SOX | Change management (starter mapping) | -| PCI-DSS | Requirement 10 (logging and monitoring) | +| PCI-DSS | Requirements 6.5, 7.2, 12.8 (starter mapping) | | Texas TRAIGA | State AI regulation | | Colorado AI Act | State AI regulation | | ISO 42001 | AI Management System | diff --git a/core/framework/eu-ai-act.yaml b/core/framework/eu-ai-act.yaml index 56947e4..4197c52 100644 --- a/core/framework/eu-ai-act.yaml +++ b/core/framework/eu-ai-act.yaml @@ -5,16 +5,139 @@ framework: controls: - id: article-9 title: Risk Management - required_record_types: [risk_assessment] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: quarterly + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery risk evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym risk assessment evidence + source_products: [axym] + required_record_types: [risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and assessment evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly - id: article-12 title: Record-Keeping - required_record_types: [tool_invocation, decision, guardrail_activation, permission_check, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery record evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime record evidence + source_products: [gait] + required_record_types: [tool_invocation, permission_check, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance record evidence + source_products: [axym] + required_record_types: [decision, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime record evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, tool_invocation, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: article-13 + title: Transparency and Information to Deployers + evidence_sets: + - id: runtime_control + title: Gait runtime decision evidence + source_products: [gait] + required_record_types: [decision, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym transparency evidence + source_products: [axym] + required_record_types: [decision, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and decision evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, decision] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event - id: article-14 title: Human Oversight - required_record_types: [human_oversight, approval, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: per-event + evidence_sets: + - id: runtime_control + title: Gait human oversight evidence + source_products: [gait] + required_record_types: [human_oversight, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym oversight evidence + source_products: [axym] + required_record_types: [approval, decision] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and oversight evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, human_oversight, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: article-15 + title: Accuracy, Robustness, and Cybersecurity + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery security evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime safety evidence + source_products: [gait] + required_record_types: [guardrail_activation, incident, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym resilience evidence + source_products: [axym] + required_record_types: [risk_assessment, incident] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime safety evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, guardrail_activation, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: article-26 + title: Deployer Obligations + evidence_sets: + - id: runtime_control + title: Gait deployer control evidence + source_products: [gait] + required_record_types: [compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym deployer governance evidence + source_products: [axym] + required_record_types: [deployment, risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and deployer evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, deployment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly diff --git a/core/framework/evaluate.go b/core/framework/evaluate.go new file mode 100644 index 0000000..12384de --- /dev/null +++ b/core/framework/evaluate.go @@ -0,0 +1,255 @@ +package framework + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/Clyra-AI/proof/core/record" +) + +type Coverage struct { + FrameworkID string `json:"framework_id"` + FrameworkVersion string `json:"framework_version"` + TotalControls int `json:"total_controls"` + CoveredControls int `json:"covered_controls"` + Controls []ControlCoverage `json:"controls"` +} + +type ControlCoverage struct { + ID string `json:"id"` + Title string `json:"title"` + Covered bool `json:"covered"` + MatchedEvidenceSetIDs []string `json:"matched_evidence_set_ids,omitempty"` + EvidenceSets []EvidenceSetCoverage `json:"evidence_sets,omitempty"` + Children []ControlCoverage `json:"children,omitempty"` +} + +type EvidenceSetCoverage struct { + ID string `json:"id"` + Title string `json:"title,omitempty"` + Covered bool `json:"covered"` + SourceProducts []string `json:"source_products,omitempty"` + RequiredRecordTypes []string `json:"required_record_types"` + MinimumFrequency string `json:"minimum_frequency"` + RequiredFields []string `json:"required_fields"` + MatchingRecordIDs []string `json:"matching_record_ids,omitempty"` + MissingRecordTypes []string `json:"missing_record_types,omitempty"` +} + +type indexedRecord struct { + record record.Record + raw map[string]any +} + +func EvaluateCoverage(f *Framework, records []record.Record) (*Coverage, error) { + if f == nil { + return nil, fmt.Errorf("framework is nil") + } + + indexed := make([]indexedRecord, 0, len(records)) + for i := range records { + raw, err := recordMap(records[i]) + if err != nil { + return nil, err + } + indexed = append(indexed, indexedRecord{record: records[i], raw: raw}) + } + + controls := make([]ControlCoverage, 0, len(f.Controls)) + for _, control := range f.Controls { + controls = append(controls, evaluateControl(control, indexed)) + } + + coveredControls := countCoveredControls(controls) + return &Coverage{ + FrameworkID: f.Framework.ID, + FrameworkVersion: f.Framework.Version, + TotalControls: countControls(f.Controls), + CoveredControls: coveredControls, + Controls: controls, + }, nil +} + +func evaluateControl(control Control, indexed []indexedRecord) ControlCoverage { + sets := controlEvidenceSets(control) + setCoverage := make([]EvidenceSetCoverage, 0, len(sets)) + matched := make([]string, 0, len(sets)) + for _, set := range sets { + coverage := evaluateEvidenceSet(set, indexed) + setCoverage = append(setCoverage, coverage) + if coverage.Covered { + matched = append(matched, coverage.ID) + } + } + + children := make([]ControlCoverage, 0, len(control.Children)) + for _, child := range control.Children { + children = append(children, evaluateControl(child, indexed)) + } + + return ControlCoverage{ + ID: control.ID, + Title: control.Title, + Covered: len(matched) > 0, + MatchedEvidenceSetIDs: matched, + EvidenceSets: setCoverage, + Children: children, + } +} + +func evaluateEvidenceSet(set EvidenceSet, indexed []indexedRecord) EvidenceSetCoverage { + matching := make([]string, 0, len(set.RequiredRecordTypes)) + missing := make([]string, 0, len(set.RequiredRecordTypes)) + for _, requiredType := range set.RequiredRecordTypes { + recordID, ok := matchRecord(requiredType, set, indexed) + if !ok { + missing = append(missing, requiredType) + continue + } + matching = append(matching, recordID) + } + matching = sortedUniqueStrings(matching) + return EvidenceSetCoverage{ + ID: set.ID, + Title: set.Title, + Covered: len(missing) == 0, + SourceProducts: append([]string(nil), set.SourceProducts...), + RequiredRecordTypes: append([]string(nil), set.RequiredRecordTypes...), + MinimumFrequency: set.MinimumFrequency, + RequiredFields: append([]string(nil), set.RequiredFields...), + MatchingRecordIDs: matching, + MissingRecordTypes: missing, + } +} + +func matchRecord(requiredType string, set EvidenceSet, indexed []indexedRecord) (string, bool) { + requiredType = strings.TrimSpace(requiredType) + for _, candidate := range indexed { + if candidate.record.RecordType != requiredType { + continue + } + if !matchesSourceProduct(candidate.record.SourceProduct, set.SourceProducts) { + continue + } + if !hasRequiredFields(candidate.raw, set.RequiredFields) { + continue + } + return candidate.record.RecordID, true + } + return "", false +} + +func matchesSourceProduct(sourceProduct string, allowed []string) bool { + if len(allowed) == 0 { + return true + } + sourceProduct = strings.ToLower(strings.TrimSpace(sourceProduct)) + for _, product := range allowed { + if sourceProduct == strings.ToLower(strings.TrimSpace(product)) { + return true + } + } + return false +} + +func hasRequiredFields(raw map[string]any, fields []string) bool { + for _, field := range fields { + value, ok := fieldValue(raw, field) + if !ok || !presentValue(value) { + return false + } + } + return true +} + +func fieldValue(raw map[string]any, path string) (any, bool) { + current := any(raw) + for _, part := range strings.Split(path, ".") { + part = strings.TrimSpace(part) + if part == "" { + return nil, false + } + m, ok := current.(map[string]any) + if !ok { + return nil, false + } + next, exists := m[part] + if !exists { + return nil, false + } + current = next + } + return current, true +} + +func presentValue(value any) bool { + switch typed := value.(type) { + case nil: + return false + case string: + return strings.TrimSpace(typed) != "" + case []any: + return len(typed) > 0 + case map[string]any: + return len(typed) > 0 + default: + return true + } +} + +func controlEvidenceSets(control Control) []EvidenceSet { + if len(control.EvidenceSets) > 0 { + out := make([]EvidenceSet, len(control.EvidenceSets)) + copy(out, control.EvidenceSets) + return out + } + return []EvidenceSet{{ + ID: "legacy", + Title: control.Title, + RequiredRecordTypes: append([]string(nil), control.RequiredRecordTypes...), + MinimumFrequency: control.MinimumFrequency, + RequiredFields: append([]string(nil), control.RequiredFields...), + }} +} + +func countCoveredControls(controls []ControlCoverage) int { + total := 0 + for _, control := range controls { + if control.Covered { + total++ + } + total += countCoveredControls(control.Children) + } + return total +} + +func recordMap(in record.Record) (map[string]any, error) { + raw, err := json.Marshal(in) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +func sortedUniqueStrings(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, item := range in { + if _, exists := seen[item]; exists { + continue + } + seen[item] = struct{}{} + out = append(out, item) + } + sort.Strings(out) + return out +} diff --git a/core/framework/evaluate_test.go b/core/framework/evaluate_test.go new file mode 100644 index 0000000..193cc7b --- /dev/null +++ b/core/framework/evaluate_test.go @@ -0,0 +1,137 @@ +package framework + +import ( + "encoding/json" + "testing" + "time" + + "github.com/Clyra-AI/proof/core/record" + "github.com/stretchr/testify/require" +) + +func TestEvaluateCoverageAlternativeEvidencePaths(t *testing.T) { + f := &Framework{} + f.Framework.ID = "starter" + f.Framework.Version = "1" + f.Framework.Title = "Starter" + f.Controls = []Control{{ + ID: "cc7.1", + Title: "Monitoring", + EvidenceSets: []EvidenceSet{ + { + ID: "wrkr-discovery", + SourceProducts: []string{"wrkr"}, + RequiredRecordTypes: []string{"scan_finding"}, + RequiredFields: []string{"record_id", "source_product", "event.entity_id"}, + MinimumFrequency: "continuous", + }, + { + ID: "runtime-control", + SourceProducts: []string{"gait"}, + RequiredRecordTypes: []string{"permission_check", "policy_enforcement"}, + RequiredFields: []string{"record_id", "source_product", "event"}, + MinimumFrequency: "continuous", + }, + { + ID: "combined", + SourceProducts: []string{"wrkr", "gait"}, + RequiredRecordTypes: []string{"scan_finding", "permission_check"}, + RequiredFields: []string{"record_id", "source_product"}, + MinimumFrequency: "continuous", + }, + }, + }} + + wrkrRecord := mustRecord(t, "wrkr", "scan_finding", map[string]any{"entity_id": "tool:filesystem.write"}) + gaitPermission := mustRecord(t, "gait", "permission_check", map[string]any{"verdict": "allow"}) + gaitPolicy := mustRecord(t, "gait", "policy_enforcement", map[string]any{"policy_id": "policy-a"}) + wrkrMissingField := mustRecord(t, "wrkr", "scan_finding", map[string]any{"severity": "medium"}) + + t.Run("wrkr only", func(t *testing.T) { + coverage, err := EvaluateCoverage(f, []record.Record{*wrkrRecord}) + require.NoError(t, err) + require.Equal(t, 1, coverage.CoveredControls) + require.Equal(t, []string{"wrkr-discovery"}, coverage.Controls[0].MatchedEvidenceSetIDs) + }) + + t.Run("gait only", func(t *testing.T) { + coverage, err := EvaluateCoverage(f, []record.Record{*gaitPermission, *gaitPolicy}) + require.NoError(t, err) + require.Equal(t, []string{"runtime-control"}, coverage.Controls[0].MatchedEvidenceSetIDs) + }) + + t.Run("combined", func(t *testing.T) { + coverage, err := EvaluateCoverage(f, []record.Record{*wrkrRecord, *gaitPermission}) + require.NoError(t, err) + require.Equal(t, []string{"wrkr-discovery", "combined"}, coverage.Controls[0].MatchedEvidenceSetIDs) + }) + + t.Run("missing field blocks coverage", func(t *testing.T) { + coverage, err := EvaluateCoverage(f, []record.Record{*wrkrMissingField}) + require.NoError(t, err) + require.False(t, coverage.Controls[0].Covered) + require.Equal(t, []string{"scan_finding"}, coverage.Controls[0].EvidenceSets[0].MissingRecordTypes) + }) +} + +func TestEvaluateCoverageLegacyControl(t *testing.T) { + f := &Framework{} + f.Framework.ID = "legacy" + f.Framework.Version = "1" + f.Controls = []Control{{ + ID: "legacy-control", + Title: "Legacy Control", + RequiredRecordTypes: []string{"decision"}, + RequiredFields: []string{"record_id", "integrity.record_hash"}, + MinimumFrequency: "continuous", + }} + + decision := mustRecord(t, "axym", "decision", map[string]any{"action": "allow"}) + + coverage, err := EvaluateCoverage(f, []record.Record{*decision}) + require.NoError(t, err) + require.True(t, coverage.Controls[0].Covered) + require.Equal(t, []string{"legacy"}, coverage.Controls[0].MatchedEvidenceSetIDs) +} + +func TestEvaluateCoverageDeterministic(t *testing.T) { + f := &Framework{} + f.Framework.ID = "deterministic" + f.Framework.Version = "1" + f.Controls = []Control{{ + ID: "control-1", + Title: "Control 1", + EvidenceSets: []EvidenceSet{{ + ID: "set-1", + RequiredRecordTypes: []string{"scan_finding"}, + RequiredFields: []string{"record_id"}, + MinimumFrequency: "continuous", + }}, + }} + + rec := mustRecord(t, "wrkr", "scan_finding", map[string]any{"entity_id": "tool:x"}) + + first, err := EvaluateCoverage(f, []record.Record{*rec}) + require.NoError(t, err) + second, err := EvaluateCoverage(f, []record.Record{*rec}) + require.NoError(t, err) + + firstRaw, err := json.Marshal(first) + require.NoError(t, err) + secondRaw, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, string(firstRaw), string(secondRaw)) +} + +func mustRecord(t *testing.T, sourceProduct, recordType string, event map[string]any) *record.Record { + t.Helper() + r, err := record.New(record.RecordOpts{ + Timestamp: time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC), + Source: sourceProduct, + SourceProduct: sourceProduct, + Type: recordType, + Event: event, + }) + require.NoError(t, err) + return r +} diff --git a/core/framework/framework.go b/core/framework/framework.go index a8849c5..cbe6ecc 100644 --- a/core/framework/framework.go +++ b/core/framework/framework.go @@ -1,16 +1,20 @@ package framework import ( + "bytes" "embed" + "encoding/json" "fmt" "os" "path/filepath" "sort" "strings" + coreschema "github.com/Clyra-AI/proof/core/schema" "gopkg.in/yaml.v3" ) +//go:generate go run ../../internal/tools/syncframeworks //go:embed *.yaml var frameworkFS embed.FS @@ -24,12 +28,22 @@ type Framework struct { } type Control struct { - ID string `yaml:"id" json:"id"` - Title string `yaml:"title" json:"title"` - RequiredRecordTypes []string `yaml:"required_record_types,omitempty" json:"required_record_types,omitempty"` - MinimumFrequency string `yaml:"minimum_frequency,omitempty" json:"minimum_frequency,omitempty"` - RequiredFields []string `yaml:"required_fields,omitempty" json:"required_fields,omitempty"` - Children []Control `yaml:"children,omitempty" json:"children,omitempty"` + ID string `yaml:"id" json:"id"` + Title string `yaml:"title" json:"title"` + RequiredRecordTypes []string `yaml:"required_record_types,omitempty" json:"required_record_types,omitempty"` + MinimumFrequency string `yaml:"minimum_frequency,omitempty" json:"minimum_frequency,omitempty"` + RequiredFields []string `yaml:"required_fields,omitempty" json:"required_fields,omitempty"` + EvidenceSets []EvidenceSet `yaml:"evidence_sets,omitempty" json:"evidence_sets,omitempty"` + Children []Control `yaml:"children,omitempty" json:"children,omitempty"` +} + +type EvidenceSet struct { + ID string `yaml:"id" json:"id"` + Title string `yaml:"title,omitempty" json:"title,omitempty"` + SourceProducts []string `yaml:"source_products,omitempty" json:"source_products,omitempty"` + RequiredRecordTypes []string `yaml:"required_record_types,omitempty" json:"required_record_types,omitempty"` + MinimumFrequency string `yaml:"minimum_frequency,omitempty" json:"minimum_frequency,omitempty"` + RequiredFields []string `yaml:"required_fields,omitempty" json:"required_fields,omitempty"` } type Info struct { @@ -132,7 +146,8 @@ func isLikelyPath(value string) bool { func parseFramework(idOrFile string, raw []byte) (*Framework, error) { var f Framework - if err := yaml.Unmarshal(raw, &f); err != nil { + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + if err := decoder.Decode(&f); err != nil { return nil, err } if f.Framework.ID == "" { @@ -144,6 +159,9 @@ func parseFramework(idOrFile string, raw []byte) (*Framework, error) { if err := validateControls(f.Controls, "controls"); err != nil { return nil, fmt.Errorf("framework %s invalid: %w", idOrFile, err) } + if err := validateFrameworkSchema(&f); err != nil { + return nil, fmt.Errorf("framework %s schema invalid: %w", idOrFile, err) + } return &f, nil } @@ -156,23 +174,63 @@ func validateControls(controls []Control, path string) error { if strings.TrimSpace(c.Title) == "" { return fmt.Errorf("%s (%s) missing title", controlPath, c.ID) } - if len(c.RequiredRecordTypes) == 0 { - return fmt.Errorf("%s (%s) missing required_record_types", controlPath, c.ID) + hasLegacy := len(c.RequiredRecordTypes) > 0 || strings.TrimSpace(c.MinimumFrequency) != "" || len(c.RequiredFields) > 0 + hasEvidenceSets := len(c.EvidenceSets) > 0 + if hasLegacy && hasEvidenceSets { + return fmt.Errorf("%s (%s) mixes legacy required_record_types with evidence_sets", controlPath, c.ID) } - if strings.TrimSpace(c.MinimumFrequency) == "" { - return fmt.Errorf("%s (%s) missing minimum_frequency", controlPath, c.ID) + if !hasLegacy && !hasEvidenceSets { + return fmt.Errorf("%s (%s) missing evidence definition", controlPath, c.ID) } - if len(c.RequiredFields) == 0 { - return fmt.Errorf("%s (%s) missing required_fields", controlPath, c.ID) - } - for _, t := range c.RequiredRecordTypes { - if strings.TrimSpace(t) == "" { - return fmt.Errorf("%s (%s) has blank required_record_types entry", controlPath, c.ID) + if hasLegacy { + if len(c.RequiredRecordTypes) == 0 { + return fmt.Errorf("%s (%s) missing required_record_types", controlPath, c.ID) + } + if strings.TrimSpace(c.MinimumFrequency) == "" { + return fmt.Errorf("%s (%s) missing minimum_frequency", controlPath, c.ID) + } + if len(c.RequiredFields) == 0 { + return fmt.Errorf("%s (%s) missing required_fields", controlPath, c.ID) + } + for _, t := range c.RequiredRecordTypes { + if strings.TrimSpace(t) == "" { + return fmt.Errorf("%s (%s) has blank required_record_types entry", controlPath, c.ID) + } + } + for _, field := range c.RequiredFields { + if strings.TrimSpace(field) == "" { + return fmt.Errorf("%s (%s) has blank required_fields entry", controlPath, c.ID) + } } } - for _, field := range c.RequiredFields { - if strings.TrimSpace(field) == "" { - return fmt.Errorf("%s (%s) has blank required_fields entry", controlPath, c.ID) + for j, set := range c.EvidenceSets { + setPath := fmt.Sprintf("%s.evidence_sets[%d]", controlPath, j) + if strings.TrimSpace(set.ID) == "" { + return fmt.Errorf("%s missing id", setPath) + } + if len(set.RequiredRecordTypes) == 0 { + return fmt.Errorf("%s (%s) missing required_record_types", setPath, set.ID) + } + if strings.TrimSpace(set.MinimumFrequency) == "" { + return fmt.Errorf("%s (%s) missing minimum_frequency", setPath, set.ID) + } + if len(set.RequiredFields) == 0 { + return fmt.Errorf("%s (%s) missing required_fields", setPath, set.ID) + } + for _, t := range set.RequiredRecordTypes { + if strings.TrimSpace(t) == "" { + return fmt.Errorf("%s (%s) has blank required_record_types entry", setPath, set.ID) + } + } + for _, field := range set.RequiredFields { + if strings.TrimSpace(field) == "" { + return fmt.Errorf("%s (%s) has blank required_fields entry", setPath, set.ID) + } + } + for _, product := range set.SourceProducts { + if strings.TrimSpace(product) == "" { + return fmt.Errorf("%s (%s) has blank source_products entry", setPath, set.ID) + } } } if err := validateControls(c.Children, controlPath+".children"); err != nil { @@ -190,3 +248,11 @@ func countControls(in []Control) int { } return total } + +func validateFrameworkSchema(f *Framework) error { + raw, err := json.Marshal(f) + if err != nil { + return err + } + return coreschema.ValidateAgainstSchema(raw, "v1/framework-definition.schema.json") +} diff --git a/core/framework/framework_test.go b/core/framework/framework_test.go index dc271d5..6585d4f 100644 --- a/core/framework/framework_test.go +++ b/core/framework/framework_test.go @@ -1,6 +1,7 @@ package framework import ( + "encoding/json" "os" "path/filepath" "testing" @@ -40,9 +41,12 @@ framework: controls: - id: custom-control title: Custom Control - required_record_types: [decision] - required_fields: [record_id] - minimum_frequency: continuous + evidence_sets: + - id: wrkr-discovery + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, source_product] + minimum_frequency: continuous `), 0o644)) f, err := Load(path) @@ -73,7 +77,7 @@ func TestLoadPrefersEmbeddedOverLocalCollision(t *testing.T) { for _, info := range list { if info.ID == "eu-ai-act" { found = true - require.Equal(t, 3, info.ControlCount) + require.Equal(t, 6, info.ControlCount) } } require.True(t, found) @@ -112,6 +116,21 @@ func TestValidateControls(t *testing.T) { } require.NoError(t, validateControls(valid, "controls")) + validEvidenceSets := []Control{ + { + ID: "c3", + Title: "Control 3", + EvidenceSets: []EvidenceSet{{ + ID: "wrkr-discovery", + SourceProducts: []string{"wrkr"}, + RequiredRecordTypes: []string{"scan_finding"}, + MinimumFrequency: "continuous", + RequiredFields: []string{"record_id", "source_product"}, + }}, + }, + } + require.NoError(t, validateControls(validEvidenceSets, "controls")) + missingFields := []Control{ { ID: "c2", @@ -255,6 +274,51 @@ func TestValidateControlsErrors(t *testing.T) { }}, needle: "missing required_record_types", }, + { + name: "mixed legacy and evidence sets", + in: []Control{{ + ID: "c1", + Title: "Control", + RequiredRecordTypes: []string{"decision"}, + MinimumFrequency: "continuous", + RequiredFields: []string{"record_id"}, + EvidenceSets: []EvidenceSet{{ + ID: "set-1", + RequiredRecordTypes: []string{"scan_finding"}, + MinimumFrequency: "continuous", + RequiredFields: []string{"record_id"}, + }}, + }}, + needle: "mixes legacy required_record_types with evidence_sets", + }, + { + name: "missing evidence set id", + in: []Control{{ + ID: "c1", + Title: "Control", + EvidenceSets: []EvidenceSet{{ + RequiredRecordTypes: []string{"scan_finding"}, + MinimumFrequency: "continuous", + RequiredFields: []string{"record_id"}, + }}, + }}, + needle: "missing id", + }, + { + name: "blank evidence set source product", + in: []Control{{ + ID: "c1", + Title: "Control", + EvidenceSets: []EvidenceSet{{ + ID: "set-1", + SourceProducts: []string{"wrkr", " "}, + RequiredRecordTypes: []string{"scan_finding"}, + MinimumFrequency: "continuous", + RequiredFields: []string{"record_id"}, + }}, + }}, + needle: "blank source_products entry", + }, } for _, tc := range cases { @@ -266,3 +330,38 @@ func TestValidateControlsErrors(t *testing.T) { }) } } + +func TestLoadFrameworkDeterministic(t *testing.T) { + first, err := Load("eu-ai-act") + require.NoError(t, err) + second, err := Load("eu-ai-act") + require.NoError(t, err) + + firstRaw, err := json.Marshal(first) + require.NoError(t, err) + secondRaw, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, string(firstRaw), string(secondRaw)) +} + +func TestBuiltInControlPresence(t *testing.T) { + cases := map[string][]string{ + "eu-ai-act": {"article-9", "article-12", "article-13", "article-14", "article-15", "article-26"}, + "pci-dss": {"req-6.5", "req-7.2", "req-12.8"}, + "soc2": {"cc6.1", "cc6.3", "cc7.1", "cc8.1"}, + } + + for frameworkID, wantControls := range cases { + f, err := Load(frameworkID) + require.NoError(t, err) + + got := make(map[string]struct{}, len(wantControls)) + for _, control := range f.Controls { + got[control.ID] = struct{}{} + } + for _, want := range wantControls { + _, ok := got[want] + require.Truef(t, ok, "framework %s missing control %s", frameworkID, want) + } + } +} diff --git a/core/framework/pci-dss.yaml b/core/framework/pci-dss.yaml index 95fcc39..b751c6c 100644 --- a/core/framework/pci-dss.yaml +++ b/core/framework/pci-dss.yaml @@ -3,8 +3,66 @@ framework: version: "4.0" title: PCI-DSS controls: - - id: req-10 - title: Logging and Monitoring - required_record_types: [tool_invocation, permission_check, incident, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + - id: req-6.5 + title: Secure Systems and Software + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime hardening evidence + source_products: [gait] + required_record_types: [guardrail_activation, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and validation evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: req-7.2 + title: Access Control by Need to Know + evidence_sets: + - id: runtime_control + title: Gait access control evidence + source_products: [gait] + required_record_types: [permission_check, policy_enforcement] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym approval evidence + source_products: [axym] + required_record_types: [approval, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime access evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: req-12.8 + title: Service Provider Governance + evidence_sets: + - id: axym_compliance + title: Axym provider governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval, deployment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and governance evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly diff --git a/core/framework/soc2.yaml b/core/framework/soc2.yaml index da95b4d..94ea175 100644 --- a/core/framework/soc2.yaml +++ b/core/framework/soc2.yaml @@ -3,18 +3,93 @@ framework: version: "2026" title: SOC 2 AI Controls controls: - - id: cc6 - title: Logical Access - required_record_types: [permission_check, approval] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous - - id: cc7 - title: System Operations - required_record_types: [incident, guardrail_activation, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous - - id: cc8 + - id: cc6.1 + title: Logical Access Controls + evidence_sets: + - id: runtime_control + title: Gait runtime access enforcement + source_products: [gait] + required_record_types: [permission_check, policy_enforcement] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: cc6.3 + title: Authorization and Approval + evidence_sets: + - id: runtime_control + title: Gait runtime approval gates + source_products: [gait] + required_record_types: [permission_check, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym governance approvals + source_products: [axym] + required_record_types: [approval, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and approval evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: cc7.1 + title: System Monitoring + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery monitoring evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime monitoring evidence + source_products: [gait] + required_record_types: [guardrail_activation, incident] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym operations evidence + source_products: [axym] + required_record_types: [incident, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime monitoring evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, guardrail_activation] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: cc8.1 title: Change Management - required_record_types: [approval, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + evidence_sets: + - id: runtime_control + title: Gait change gating evidence + source_products: [gait] + required_record_types: [compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym deployment governance evidence + source_products: [axym] + required_record_types: [deployment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and change control evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event diff --git a/core/schema/schema_test.go b/core/schema/schema_test.go index 04222ec..26cb857 100644 --- a/core/schema/schema_test.go +++ b/core/schema/schema_test.go @@ -349,3 +349,47 @@ func TestValidateRelationshipEnvelopeSchema(t *testing.T) { }`) require.NoError(t, ValidateAgainstSchema(valid, "v1/relationship_envelope.schema.json")) } + +func TestValidateFrameworkDefinitionSchema(t *testing.T) { + valid := []byte(`{ + "framework":{"id":"starter","version":"1","title":"Starter"}, + "controls":[ + { + "id":"cc7.1", + "title":"Monitoring", + "evidence_sets":[ + { + "id":"wrkr-discovery", + "source_products":["wrkr"], + "required_record_types":["scan_finding"], + "required_fields":["record_id","source_product"], + "minimum_frequency":"continuous" + } + ] + } + ] + }`) + require.NoError(t, ValidateAgainstSchema(valid, "v1/framework-definition.schema.json")) + + invalid := []byte(`{ + "framework":{"id":"starter","version":"1","title":"Starter"}, + "controls":[ + { + "id":"cc7.1", + "title":"Monitoring", + "required_record_types":["scan_finding"], + "required_fields":["record_id"], + "minimum_frequency":"continuous", + "evidence_sets":[ + { + "id":"wrkr-discovery", + "required_record_types":["scan_finding"], + "required_fields":["record_id"], + "minimum_frequency":"continuous" + } + ] + } + ] + }`) + require.Error(t, ValidateAgainstSchema(invalid, "v1/framework-definition.schema.json")) +} diff --git a/core/schema/v1/framework-definition.schema.json b/core/schema/v1/framework-definition.schema.json index ddbd758..da0b487 100644 --- a/core/schema/v1/framework-definition.schema.json +++ b/core/schema/v1/framework-definition.schema.json @@ -4,9 +4,41 @@ "type": "object", "required": ["framework", "controls"], "definitions": { + "evidence_set": { + "type": "object", + "required": ["id", "required_record_types", "required_fields", "minimum_frequency"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "source_products": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "required_record_types": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "minimum_frequency": { "type": "string", "minLength": 1 }, + "required_fields": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + }, "control": { "type": "object", - "required": ["id", "title", "required_record_types", "required_fields", "minimum_frequency"], + "required": ["id", "title"], + "oneOf": [ + { + "required": ["required_record_types", "required_fields", "minimum_frequency"] + }, + { + "required": ["evidence_sets"] + } + ], "properties": { "id": { "type": "string", "minLength": 1 }, "title": { "type": "string", "minLength": 1 }, @@ -21,6 +53,11 @@ "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "evidence_sets": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/evidence_set" } + }, "children": { "type": "array", "items": { "$ref": "#/definitions/control" } diff --git a/docs/api-contract.md b/docs/api-contract.md index 95e17a6..f538240 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -13,7 +13,7 @@ Use this contract before refactors to avoid accidental breakage for downstream u | `github.com/Clyra-AI/proof/core/signing` | Low-level signing primitives | Supported (stable) | Backward compatible within major version for exported symbols. | | `github.com/Clyra-AI/proof/core/canon` | Low-level canonicalization primitives | Supported (stable) | Backward compatible within major version for exported symbols. | | `github.com/Clyra-AI/proof/core/schema` | Low-level schema/type primitives | Supported (stable) | Backward compatible within major version for exported symbols. | -| `github.com/Clyra-AI/proof/core/framework` | Framework definition loading | Supported (stable) | Backward compatible within major version for exported symbols. | +| `github.com/Clyra-AI/proof/core/framework` | Framework definitions and evidence coverage evaluation | Supported (stable) | Backward compatible within major version for exported symbols. | | `github.com/Clyra-AI/proof/core/bundle` | Bundle manifest/sign/verify primitives | Supported (stable) | Backward compatible within major version for exported symbols. | | `github.com/Clyra-AI/proof/core/exitcode` | Exit-code constants | Supported (stable) | Exit code values `0-8` are contractually stable. | | `github.com/Clyra-AI/proof/signing` | Compatibility shim | Supported (compatibility) | Kept for migration compatibility. New code should prefer `github.com/Clyra-AI/proof` or `.../core/signing`. | diff --git a/frameworks/eu-ai-act.yaml b/frameworks/eu-ai-act.yaml index 56947e4..4197c52 100644 --- a/frameworks/eu-ai-act.yaml +++ b/frameworks/eu-ai-act.yaml @@ -5,16 +5,139 @@ framework: controls: - id: article-9 title: Risk Management - required_record_types: [risk_assessment] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: quarterly + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery risk evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym risk assessment evidence + source_products: [axym] + required_record_types: [risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and assessment evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly - id: article-12 title: Record-Keeping - required_record_types: [tool_invocation, decision, guardrail_activation, permission_check, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery record evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime record evidence + source_products: [gait] + required_record_types: [tool_invocation, permission_check, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance record evidence + source_products: [axym] + required_record_types: [decision, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime record evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, tool_invocation, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: article-13 + title: Transparency and Information to Deployers + evidence_sets: + - id: runtime_control + title: Gait runtime decision evidence + source_products: [gait] + required_record_types: [decision, compiled_action] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym transparency evidence + source_products: [axym] + required_record_types: [decision, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and decision evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, decision] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event - id: article-14 title: Human Oversight - required_record_types: [human_oversight, approval, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: per-event + evidence_sets: + - id: runtime_control + title: Gait human oversight evidence + source_products: [gait] + required_record_types: [human_oversight, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym oversight evidence + source_products: [axym] + required_record_types: [approval, decision] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and oversight evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, human_oversight, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: article-15 + title: Accuracy, Robustness, and Cybersecurity + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery security evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime safety evidence + source_products: [gait] + required_record_types: [guardrail_activation, incident, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym resilience evidence + source_products: [axym] + required_record_types: [risk_assessment, incident] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime safety evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, guardrail_activation, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: article-26 + title: Deployer Obligations + evidence_sets: + - id: runtime_control + title: Gait deployer control evidence + source_products: [gait] + required_record_types: [compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym deployer governance evidence + source_products: [axym] + required_record_types: [deployment, risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and deployer evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, deployment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly diff --git a/frameworks/pci-dss.yaml b/frameworks/pci-dss.yaml index 95fcc39..b751c6c 100644 --- a/frameworks/pci-dss.yaml +++ b/frameworks/pci-dss.yaml @@ -3,8 +3,66 @@ framework: version: "4.0" title: PCI-DSS controls: - - id: req-10 - title: Logging and Monitoring - required_record_types: [tool_invocation, permission_check, incident, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + - id: req-6.5 + title: Secure Systems and Software + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime hardening evidence + source_products: [gait] + required_record_types: [guardrail_activation, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and validation evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, test_result] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: req-7.2 + title: Access Control by Need to Know + evidence_sets: + - id: runtime_control + title: Gait access control evidence + source_products: [gait] + required_record_types: [permission_check, policy_enforcement] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym approval evidence + source_products: [axym] + required_record_types: [approval, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime access evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: req-12.8 + title: Service Provider Governance + evidence_sets: + - id: axym_compliance + title: Axym provider governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval, deployment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and governance evidence + source_products: [wrkr, axym] + required_record_types: [scan_finding, risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly diff --git a/frameworks/soc2.yaml b/frameworks/soc2.yaml index da95b4d..94ea175 100644 --- a/frameworks/soc2.yaml +++ b/frameworks/soc2.yaml @@ -3,18 +3,93 @@ framework: version: "2026" title: SOC 2 AI Controls controls: - - id: cc6 - title: Logical Access - required_record_types: [permission_check, approval] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous - - id: cc7 - title: System Operations - required_record_types: [incident, guardrail_activation, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous - - id: cc8 + - id: cc6.1 + title: Logical Access Controls + evidence_sets: + - id: runtime_control + title: Gait runtime access enforcement + source_products: [gait] + required_record_types: [permission_check, policy_enforcement] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym governance evidence + source_products: [axym] + required_record_types: [risk_assessment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: cc6.3 + title: Authorization and Approval + evidence_sets: + - id: runtime_control + title: Gait runtime approval gates + source_products: [gait] + required_record_types: [permission_check, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym governance approvals + source_products: [axym] + required_record_types: [approval, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and approval evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, permission_check, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: cc7.1 + title: System Monitoring + evidence_sets: + - id: wrkr_discovery + title: Wrkr discovery monitoring evidence + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: runtime_control + title: Gait runtime monitoring evidence + source_products: [gait] + required_record_types: [guardrail_activation, incident] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: axym_compliance + title: Axym operations evidence + source_products: [axym] + required_record_types: [incident, risk_assessment] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: quarterly + - id: combined + title: Discovery and runtime monitoring evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, guardrail_activation] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: continuous + - id: cc8.1 title: Change Management - required_record_types: [approval, compiled_action] - required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] - minimum_frequency: continuous + evidence_sets: + - id: runtime_control + title: Gait change gating evidence + source_products: [gait] + required_record_types: [compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: axym_compliance + title: Axym deployment governance evidence + source_products: [axym] + required_record_types: [deployment, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event + - id: combined + title: Discovery and change control evidence + source_products: [wrkr, gait] + required_record_types: [scan_finding, compiled_action, approval] + required_fields: [record_id, timestamp, source, source_product, record_type, event, integrity.record_hash] + minimum_frequency: per-event diff --git a/go.mod b/go.mod index f176ab5..5627592 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Clyra-AI/proof -go 1.25.7 +go 1.26.1 require ( github.com/gowebpki/jcs v1.0.1 diff --git a/internal/tools/syncframeworks/main.go b/internal/tools/syncframeworks/main.go new file mode 100644 index 0000000..7c3048d --- /dev/null +++ b/internal/tools/syncframeworks/main.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sort" + "strings" +) + +func main() { + root, err := repoRoot() + if err != nil { + fail(err) + } + srcDir := filepath.Join(root, "core", "framework") + dstDir := filepath.Join(root, "frameworks") + if err := syncFrameworks(srcDir, dstDir); err != nil { + fail(err) + } +} + +func repoRoot() (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("locate source file") + } + return filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", "..")), nil +} + +func syncFrameworks(srcDir, dstDir string) error { + entries, err := os.ReadDir(srcDir) + if err != nil { + return err + } + if err := os.MkdirAll(dstDir, 0o755); err != nil { + return err + } + + sourceFiles := make(map[string]struct{}) + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".yaml" { + continue + } + sourceFiles[entry.Name()] = struct{}{} + names = append(names, entry.Name()) + } + sort.Strings(names) + + for _, name := range names { + srcPath := filepath.Join(srcDir, name) + dstPath := filepath.Join(dstDir, name) + if err := copyFile(srcPath, dstPath); err != nil { + return err + } + } + + dstEntries, err := os.ReadDir(dstDir) + if err != nil { + return err + } + for _, entry := range dstEntries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".yaml" { + continue + } + if _, ok := sourceFiles[entry.Name()]; ok { + continue + } + if err := os.Remove(filepath.Join(dstDir, entry.Name())); err != nil { + return err + } + } + return nil +} + +func copyFile(srcPath, dstPath string) error { + src, err := os.Open(srcPath) + if err != nil { + return err + } + defer func() { + _ = src.Close() + }() + + dst, err := os.Create(dstPath) + if err != nil { + return err + } + if _, err := io.Copy(dst, src); err != nil { + _ = dst.Close() + return err + } + if err := dst.Close(); err != nil { + return err + } + return os.Chmod(dstPath, 0o644) +} + +func fail(err error) { + _, _ = fmt.Fprintf(os.Stderr, "sync frameworks: %s\n", strings.TrimSpace(err.Error())) + os.Exit(1) +} diff --git a/proof.go b/proof.go index 35d53eb..cec0e68 100644 --- a/proof.go +++ b/proof.go @@ -34,6 +34,9 @@ type RevocationList = signing.RevocationList type RevocationEntry = signing.RevocationEntry type CosignVerifyOpts = signing.CosignVerifyOpts type Framework = framework.Framework +type FrameworkCoverage = framework.Coverage +type FrameworkControlCoverage = framework.ControlCoverage +type FrameworkEvidenceSetCoverage = framework.EvidenceSetCoverage type RecordType = schema.RecordType type CanonDomain = canon.Domain type Digest = canon.Digest @@ -92,6 +95,10 @@ func LoadFramework(pathOrID string) (*Framework, error) { return framework.Load(pathOrID) } +func EvaluateFrameworkCoverage(f *Framework, records []Record) (*FrameworkCoverage, error) { + return framework.EvaluateCoverage(f, records) +} + func ListRecordTypes() []RecordType { return schema.ListRecordTypes() } diff --git a/proof_test.go b/proof_test.go index 10fd65a..de95b93 100644 --- a/proof_test.go +++ b/proof_test.go @@ -170,6 +170,42 @@ controls: require.Equal(t, "custom-framework", custom.Framework.ID) } +func TestEvaluateFrameworkCoverage(t *testing.T) { + path := filepath.Join(t.TempDir(), "custom-framework.yaml") + require.NoError(t, os.WriteFile(path, []byte(` +framework: + id: custom-framework + version: "1" + title: Custom Framework +controls: + - id: custom-control + title: Custom Control + evidence_sets: + - id: wrkr-discovery + source_products: [wrkr] + required_record_types: [scan_finding] + required_fields: [record_id, source_product, event.entity_id] + minimum_frequency: continuous +`), 0o644)) + + f, err := LoadFramework(path) + require.NoError(t, err) + + r, err := NewRecord(RecordOpts{ + Timestamp: time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC), + Source: "wrkr", + SourceProduct: "wrkr", + Type: "scan_finding", + Event: map[string]any{"entity_id": "tool:filesystem.write"}, + }) + require.NoError(t, err) + + coverage, err := EvaluateFrameworkCoverage(f, []Record{*r}) + require.NoError(t, err) + require.Equal(t, 1, coverage.CoveredControls) + require.Equal(t, []string{"wrkr-discovery"}, coverage.Controls[0].MatchedEvidenceSetIDs) +} + func TestWriteReadAndCustomSchemaValidation(t *testing.T) { ResetCustomTypes() t.Cleanup(ResetCustomTypes) diff --git a/schemas/v1/framework-definition.schema.json b/schemas/v1/framework-definition.schema.json index ddbd758..da0b487 100644 --- a/schemas/v1/framework-definition.schema.json +++ b/schemas/v1/framework-definition.schema.json @@ -4,9 +4,41 @@ "type": "object", "required": ["framework", "controls"], "definitions": { + "evidence_set": { + "type": "object", + "required": ["id", "required_record_types", "required_fields", "minimum_frequency"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "title": { "type": "string", "minLength": 1 }, + "source_products": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "required_record_types": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "minimum_frequency": { "type": "string", "minLength": 1 }, + "required_fields": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + } + }, "control": { "type": "object", - "required": ["id", "title", "required_record_types", "required_fields", "minimum_frequency"], + "required": ["id", "title"], + "oneOf": [ + { + "required": ["required_record_types", "required_fields", "minimum_frequency"] + }, + { + "required": ["evidence_sets"] + } + ], "properties": { "id": { "type": "string", "minLength": 1 }, "title": { "type": "string", "minLength": 1 }, @@ -21,6 +53,11 @@ "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "evidence_sets": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/definitions/evidence_set" } + }, "children": { "type": "array", "items": { "$ref": "#/definitions/control" } From e878e1681af87e1d594f56b69f65b602bb0ed179 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:07:31 -0400 Subject: [PATCH 2/7] Address PR review feedback --- .tool-versions | 2 +- core/framework/evaluate.go | 3 +++ core/framework/evaluate_test.go | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 8472001..aced889 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -golang 1.25.7 +golang 1.26.1 python 3.13.7 diff --git a/core/framework/evaluate.go b/core/framework/evaluate.go index 12384de..bb82482 100644 --- a/core/framework/evaluate.go +++ b/core/framework/evaluate.go @@ -47,6 +47,9 @@ func EvaluateCoverage(f *Framework, records []record.Record) (*Coverage, error) if f == nil { return nil, fmt.Errorf("framework is nil") } + if err := validateControls(f.Controls, "controls"); err != nil { + return nil, fmt.Errorf("framework invalid: %w", err) + } indexed := make([]indexedRecord, 0, len(records)) for i := range records { diff --git a/core/framework/evaluate_test.go b/core/framework/evaluate_test.go index 193cc7b..5809df1 100644 --- a/core/framework/evaluate_test.go +++ b/core/framework/evaluate_test.go @@ -123,6 +123,20 @@ func TestEvaluateCoverageDeterministic(t *testing.T) { require.Equal(t, string(firstRaw), string(secondRaw)) } +func TestEvaluateCoverageRejectsInvalidControls(t *testing.T) { + f := &Framework{} + f.Framework.ID = "invalid" + f.Framework.Version = "1" + f.Controls = []Control{{ + ID: "control-1", + Title: "Control 1", + }} + + _, err := EvaluateCoverage(f, nil) + require.Error(t, err) + require.ErrorContains(t, err, "missing evidence definition") +} + func mustRecord(t *testing.T, sourceProduct, recordType string, event map[string]any) *record.Record { t.Helper() r, err := record.New(record.RecordOpts{ From 5941027ce588bb669b4df5e5086501c3a0169b06 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:12:55 -0400 Subject: [PATCH 3/7] Speed up Windows test runs --- .github/workflows/pr.yml | 2 +- internal/testutil/testutil.go | 35 +++++++++++++++++++++++++++++- internal/testutil/testutil_test.go | 2 ++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6f59b8c..fdb9353 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -36,4 +36,4 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod - - run: go test ./... + - run: go test -v ./... diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index f95af69..30857bd 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -5,9 +5,15 @@ import ( "os/exec" "path/filepath" "runtime" + "sync" "testing" ) +var ( + buildMu sync.Mutex + binaryCache = map[string]string{} +) + func RepoRoot(t *testing.T) string { t.Helper() _, file, _, ok := runtime.Caller(0) @@ -19,10 +25,26 @@ func RepoRoot(t *testing.T) string { func BuildBinary(t *testing.T, root string) string { t.Helper() - bin := filepath.Join(t.TempDir(), "proof") + + buildMu.Lock() + if cached, ok := binaryCache[root]; ok { + if _, err := os.Stat(cached); err == nil { + buildMu.Unlock() + return cached + } + delete(binaryCache, root) + } + buildMu.Unlock() + + dir, err := os.MkdirTemp("", "proof-test-bin-*") + if err != nil { + t.Fatalf("create temp dir for test binary: %v", err) + } + bin := filepath.Join(dir, "proof") if runtime.GOOS == "windows" { bin += ".exe" } + // #nosec G204 -- test helper executes a fixed go build command. cmd := exec.Command("go", "build", "-o", bin, "./cmd/proof") cmd.Dir = root @@ -33,6 +55,17 @@ func BuildBinary(t *testing.T, root string) string { if _, err := os.Stat(bin); err != nil { t.Fatalf("build binary output not found at %s: %v", bin, err) } + + buildMu.Lock() + if cached, ok := binaryCache[root]; ok { + if _, err := os.Stat(cached); err == nil { + buildMu.Unlock() + return cached + } + } + binaryCache[root] = bin + buildMu.Unlock() + return bin } diff --git a/internal/testutil/testutil_test.go b/internal/testutil/testutil_test.go index 5cd7406..5248b41 100644 --- a/internal/testutil/testutil_test.go +++ b/internal/testutil/testutil_test.go @@ -27,8 +27,10 @@ func TestWriteFile(t *testing.T) { func TestBuildBinaryAndExitCode(t *testing.T) { root := RepoRoot(t) bin := BuildBinary(t, root) + binAgain := BuildBinary(t, root) _, err := os.Stat(bin) require.NoError(t, err) + require.Equal(t, bin, binAgain) cmd := exec.Command("sh", "-c", "exit 7") err = cmd.Run() From bbaa12b10048f77b2f2547a87ef2ccfa1294c8c9 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:13:57 -0400 Subject: [PATCH 4/7] Make framework matches deterministic --- core/framework/evaluate.go | 10 ++++++-- core/framework/evaluate_test.go | 44 ++++++++++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/core/framework/evaluate.go b/core/framework/evaluate.go index bb82482..5568644 100644 --- a/core/framework/evaluate.go +++ b/core/framework/evaluate.go @@ -129,6 +129,7 @@ func evaluateEvidenceSet(set EvidenceSet, indexed []indexedRecord) EvidenceSetCo func matchRecord(requiredType string, set EvidenceSet, indexed []indexedRecord) (string, bool) { requiredType = strings.TrimSpace(requiredType) + bestRecordID := "" for _, candidate := range indexed { if candidate.record.RecordType != requiredType { continue @@ -139,9 +140,14 @@ func matchRecord(requiredType string, set EvidenceSet, indexed []indexedRecord) if !hasRequiredFields(candidate.raw, set.RequiredFields) { continue } - return candidate.record.RecordID, true + if bestRecordID == "" || candidate.record.RecordID < bestRecordID { + bestRecordID = candidate.record.RecordID + } + } + if bestRecordID == "" { + return "", false } - return "", false + return bestRecordID, true } func matchesSourceProduct(sourceProduct string, allowed []string) bool { diff --git a/core/framework/evaluate_test.go b/core/framework/evaluate_test.go index 5809df1..368707a 100644 --- a/core/framework/evaluate_test.go +++ b/core/framework/evaluate_test.go @@ -123,6 +123,43 @@ func TestEvaluateCoverageDeterministic(t *testing.T) { require.Equal(t, string(firstRaw), string(secondRaw)) } +func TestEvaluateCoverageDeterministicAcrossRecordOrder(t *testing.T) { + f := &Framework{} + f.Framework.ID = "deterministic-order" + f.Framework.Version = "1" + f.Controls = []Control{{ + ID: "control-1", + Title: "Control 1", + EvidenceSets: []EvidenceSet{{ + ID: "set-1", + SourceProducts: []string{"wrkr"}, + RequiredRecordTypes: []string{"scan_finding"}, + RequiredFields: []string{"record_id", "event.entity_id"}, + MinimumFrequency: "continuous", + }}, + }} + + recA := mustRecordAt(t, "wrkr", "scan_finding", time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC), map[string]any{"entity_id": "tool:b"}) + recB := mustRecordAt(t, "wrkr", "scan_finding", time.Date(2026, 3, 10, 12, 1, 0, 0, time.UTC), map[string]any{"entity_id": "tool:a"}) + + first, err := EvaluateCoverage(f, []record.Record{*recA, *recB}) + require.NoError(t, err) + second, err := EvaluateCoverage(f, []record.Record{*recB, *recA}) + require.NoError(t, err) + + firstRaw, err := json.Marshal(first) + require.NoError(t, err) + secondRaw, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, string(firstRaw), string(secondRaw)) + + expected := recA.RecordID + if recB.RecordID < expected { + expected = recB.RecordID + } + require.Equal(t, []string{expected}, first.Controls[0].EvidenceSets[0].MatchingRecordIDs) +} + func TestEvaluateCoverageRejectsInvalidControls(t *testing.T) { f := &Framework{} f.Framework.ID = "invalid" @@ -138,9 +175,14 @@ func TestEvaluateCoverageRejectsInvalidControls(t *testing.T) { } func mustRecord(t *testing.T, sourceProduct, recordType string, event map[string]any) *record.Record { + t.Helper() + return mustRecordAt(t, sourceProduct, recordType, time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC), event) +} + +func mustRecordAt(t *testing.T, sourceProduct, recordType string, ts time.Time, event map[string]any) *record.Record { t.Helper() r, err := record.New(record.RecordOpts{ - Timestamp: time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC), + Timestamp: ts, Source: sourceProduct, SourceProduct: sourceProduct, Type: recordType, From cc18cd360778e2bfd2e46791fd3b7c6a1399ccc0 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:17:15 -0400 Subject: [PATCH 5/7] Cancel duplicate PR workflow runs --- .github/workflows/pr.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index fdb9353..df1aa67 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,6 +6,10 @@ on: branches-ignore: - main +concurrency: + group: pr-${{ github.workflow }}-${{ github.event.pull_request.head.ref || github.ref_name }} + cancel-in-progress: true + jobs: pr-lint: runs-on: ubuntu-latest From f83cebcd45c2d0fe7ebb065f9a793748aaf573b3 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:25:52 -0400 Subject: [PATCH 6/7] Validate frameworks against raw YAML --- core/framework/framework.go | 12 ++++++++---- core/framework/framework_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/core/framework/framework.go b/core/framework/framework.go index cbe6ecc..106eb76 100644 --- a/core/framework/framework.go +++ b/core/framework/framework.go @@ -159,7 +159,7 @@ func parseFramework(idOrFile string, raw []byte) (*Framework, error) { if err := validateControls(f.Controls, "controls"); err != nil { return nil, fmt.Errorf("framework %s invalid: %w", idOrFile, err) } - if err := validateFrameworkSchema(&f); err != nil { + if err := validateFrameworkSchema(raw); err != nil { return nil, fmt.Errorf("framework %s schema invalid: %w", idOrFile, err) } return &f, nil @@ -249,10 +249,14 @@ func countControls(in []Control) int { return total } -func validateFrameworkSchema(f *Framework) error { - raw, err := json.Marshal(f) +func validateFrameworkSchema(raw []byte) error { + var document any + if err := yaml.Unmarshal(raw, &document); err != nil { + return err + } + normalized, err := json.Marshal(document) if err != nil { return err } - return coreschema.ValidateAgainstSchema(raw, "v1/framework-definition.schema.json") + return coreschema.ValidateAgainstSchema(normalized, "v1/framework-definition.schema.json") } diff --git a/core/framework/framework_test.go b/core/framework/framework_test.go index 6585d4f..8a41048 100644 --- a/core/framework/framework_test.go +++ b/core/framework/framework_test.go @@ -187,6 +187,23 @@ framework: controls: [] `)) require.ErrorContains(t, err, "has no controls") + + _, err = parseFramework("empty-source-products", []byte(` +framework: + id: test + version: "1" + title: Empty Source Products +controls: + - id: c1 + title: Control + evidence_sets: + - id: set-1 + source_products: [] + required_record_types: [scan_finding] + required_fields: [record_id] + minimum_frequency: continuous +`)) + require.ErrorContains(t, err, "schema invalid") } func TestValidateControlsErrors(t *testing.T) { From d1b972f8b1c4ab10cbd7e96fd932b3b23c5747b5 Mon Sep 17 00:00:00 2001 From: David Ahmann Date: Tue, 10 Mar 2026 22:36:27 -0400 Subject: [PATCH 7/7] Fix Windows CLI test stdout deadlock --- cmd/proof/root_cmd_test.go | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/cmd/proof/root_cmd_test.go b/cmd/proof/root_cmd_test.go index bba4e4b..a1ce2e4 100644 --- a/cmd/proof/root_cmd_test.go +++ b/cmd/proof/root_cmd_test.go @@ -3,13 +3,16 @@ package main import ( "bytes" "encoding/json" + "fmt" "io" "os" "path/filepath" + "strings" "testing" "time" "github.com/Clyra-AI/proof" + "github.com/spf13/cobra" "github.com/stretchr/testify/require" ) @@ -107,22 +110,48 @@ func TestInspectChainAndBundleCommands(t *testing.T) { require.Contains(t, out, "\"files\"") } +func TestRunCommandForTestDrainsLargeStdout(t *testing.T) { + cmd := &cobra.Command{ + RunE: func(cmd *cobra.Command, args []string) error { + _, err := fmt.Fprint(os.Stdout, strings.Repeat("x", 1024*1024)) + return err + }, + } + + out, err := runCommandForTest(t, cmd, nil) + require.NoError(t, err) + require.Len(t, out, 1024*1024) +} + func runCLIForTest(t *testing.T, args []string) (string, error) { t.Helper() cmd := newRootCmd("test") + return runCommandForTest(t, cmd, args) +} + +func runCommandForTest(t *testing.T, cmd *cobra.Command, args []string) (string, error) { + t.Helper() cmd.SetArgs(args) oldStdout := os.Stdout - r, w, _ := os.Pipe() + r, w, err := os.Pipe() + require.NoError(t, err) os.Stdout = w + defer func() { + os.Stdout = oldStdout + }() + + var buf bytes.Buffer + copyDone := make(chan error, 1) + go func() { + _, copyErr := io.Copy(&buf, r) + copyDone <- copyErr + }() - err := cmd.Execute() + err = cmd.Execute() _ = w.Close() - os.Stdout = oldStdout - - var buf bytes.Buffer - _, _ = io.Copy(&buf, r) + require.NoError(t, <-copyDone) _ = r.Close() return buf.String(), err }