diff --git a/src/cache/cache-store.ts b/src/cache/cache-store.ts new file mode 100644 index 0000000..74cd515 --- /dev/null +++ b/src/cache/cache-store.ts @@ -0,0 +1,115 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; +import * as path from 'path'; +import type { CacheData, CachedResult } from './types'; +import { CACHE_SCHEMA } from '../schemas/cache-schema'; + +/** + * Cache schema version. Bump this to invalidate existing caches + * when the internal data structure changes in a future release. + */ +const CACHE_VERSION = 1; +const DEFAULT_CACHE_DIR = '.vectorlint'; +const CACHE_FILENAME = 'cache.json'; + +/** + * Persistent cache store for evaluation results. + * Stores cache in .vectorlint/cache.json by default. + */ +export class CacheStore { + private readonly cacheDir: string; + private readonly cacheFile: string; + private data: CacheData; + private dirty: boolean = false; + + constructor(cwd: string = process.cwd(), cacheDir: string = DEFAULT_CACHE_DIR) { + this.cacheDir = path.resolve(cwd, cacheDir); + this.cacheFile = path.join(this.cacheDir, CACHE_FILENAME); + this.data = this.load(); + } + + /** + * Load cache from disk or create empty cache. + */ + private load(): CacheData { + try { + if (existsSync(this.cacheFile)) { + const raw = readFileSync(this.cacheFile, 'utf-8'); + const json: unknown = JSON.parse(raw); + + const result = CACHE_SCHEMA.safeParse(json); + + if (!result.success) { + console.warn(`[vectorlint] Cache validation failed, starting fresh: ${result.error.message}`); + return { version: CACHE_VERSION, entries: {} }; + } + + const parsed = result.data; + + /* + * Cache version invalidation: Bump CACHE_VERSION when CachedResult structure changes. + * + * When to bump: + * - Adding/removing fields in CachedResult, CachedIssue, or CachedScore + * - Changing hash algorithms (content or prompts) + * - Modifying score calculation logic that affects cached components + * + * Migration strategy: On version mismatch, clear entire cache and rebuild. + */ + + if (parsed.version !== CACHE_VERSION) { + console.warn(`[vectorlint] Cache version mismatch, clearing cache`); + return { version: CACHE_VERSION, entries: {} }; + } + + return parsed; + } + } catch (e: unknown) { + // If cache is corrupted, start fresh + const err = e instanceof Error ? e : new Error(String(e)); + console.warn(`[vectorlint] Could not read cache, starting fresh: ${err.message}`); + } + + return { version: CACHE_VERSION, entries: {} }; + } + + get(key: string): CachedResult | undefined { + return this.data.entries[key]; + } + + set(key: string, result: CachedResult): void { + this.data.entries[key] = result; + this.dirty = true; + } + + has(key: string): boolean { + return key in this.data.entries; + } + + clear(): void { + this.data.entries = {}; + this.dirty = true; + } + + size(): number { + return Object.keys(this.data.entries).length; + } + + save(): void { + if (!this.dirty) return; + + try { + // Create cache directory if missing + if (!existsSync(this.cacheDir)) { + mkdirSync(this.cacheDir, { recursive: true }); + } + + const json = JSON.stringify(this.data, null, 2); + writeFileSync(this.cacheFile, json, 'utf-8'); + this.dirty = false; + } catch (e) { + // Don't fail the run if cache can't be written + const msg = e instanceof Error ? e.message : String(e); + console.warn(`[vectorlint] Warning: Could not save cache: ${msg}`); + } + } +} diff --git a/src/cache/content-hasher.ts b/src/cache/content-hasher.ts new file mode 100644 index 0000000..40ce1df --- /dev/null +++ b/src/cache/content-hasher.ts @@ -0,0 +1,50 @@ +import { createHash } from 'crypto'; +import type { PromptFile } from '../prompts/prompt-loader'; + +const HASH_TRUNCATE_LENGTH = 16; + +/** + * Computes a SHA256 hash of normalized content. + * + * Normalization rationale: + * - Line endings (\r\n -> \n): Ensures consistent hashing across Windows/Unix. + * - Trim whitespace: Trailing whitespace is irrelevant for content quality. + * + * IMPORTANT: Changing normalization invalidates ALL cache entries. + */ +export function hashContent(content: string): string { + const normalized = content.replace(/\r\n/g, '\n').trim(); + return createHash('sha256').update(normalized, 'utf8').digest('hex'); +} + +/** + * Computes a SHA256 hash of prompt configurations. + * This includes prompt id, meta, and body to detect rule changes. + */ +export function hashPrompts(prompts: PromptFile[]): string { + // Sort prompts by id for deterministic hashing. + const sorted = [...prompts].sort((a, b) => a.id.localeCompare(b.id)); + + // Extract hashable parts: id, meta (serialized), and body + const parts = sorted.map(p => ({ + id: p.id, + meta: JSON.stringify(p.meta), + body: p.body.trim(), + pack: p.pack || '', + })); + + const serialized = JSON.stringify(parts); + return createHash('sha256').update(serialized, 'utf8').digest('hex'); +} + +/** + * Creates a cache key string from components. + * Format: "filePath|contentHash(16)|promptsHash(16)" + */ +export function createCacheKeyString( + filePath: string, + contentHash: string, + promptsHash: string +): string { + return `${filePath}|${contentHash.substring(0, HASH_TRUNCATE_LENGTH)}|${promptsHash.substring(0, HASH_TRUNCATE_LENGTH)}`; +} diff --git a/src/cache/index.ts b/src/cache/index.ts new file mode 100644 index 0000000..0a70ebb --- /dev/null +++ b/src/cache/index.ts @@ -0,0 +1,3 @@ +export * from './types'; +export * from './content-hasher'; +export * from './cache-store'; diff --git a/src/cache/types.ts b/src/cache/types.ts new file mode 100644 index 0000000..2d1eb99 --- /dev/null +++ b/src/cache/types.ts @@ -0,0 +1,64 @@ +import { Severity } from '../evaluators/types'; +import { ScoreComponent } from '../output/json-formatter'; + +/** + * Unique key for cache entries consisting of file path and content/prompt hashes. + */ +export interface CacheKey { + filePath: string; + contentHash: string; + promptsHash: string; +} + + +/** + * Minimal issue data stored in cache for replay. + */ +export interface CachedIssue { + line: number; + column: number; + severity: Severity; + summary: string; + ruleName: string; + suggestion?: string | undefined; + scoreText?: string | undefined; + match?: string | undefined; +} + +export interface CachedEvaluationSummary { + id: string; + scoreText: string; + score?: number | undefined; +} + +/** + * Grouped scores by rule/prompt. + */ +export interface CachedScore { + ruleName: string; + items: CachedEvaluationSummary[]; + components?: ScoreComponent[] | undefined; +} + +export interface CachedResult { + errors: number; + warnings: number; + hadOperationalErrors: boolean; + hadSeverityErrors: boolean; + requestFailures: number; + issues?: CachedIssue[] | undefined; + scores?: CachedScore[] | undefined; + jsonOutput?: unknown; + timestamp: number; +} + +export interface CacheData { + version: number; + entries: Record; +} + +export interface CacheOptions { + enabled: boolean; + forceFullRun: boolean; + cacheDir?: string; +} diff --git a/src/cli/commands.ts b/src/cli/commands.ts index 70e9f38..64c3d58 100644 --- a/src/cli/commands.ts +++ b/src/cli/commands.ts @@ -1,20 +1,21 @@ -import type { Command } from 'commander'; -import { existsSync } from 'fs'; -import * as path from 'path'; -import { createProvider } from '../providers/provider-factory'; -import { PerplexitySearchProvider } from '../providers/perplexity-provider'; -import type { SearchProvider } from '../providers/search-provider'; -import { loadConfig } from '../boundaries/config-loader'; -import { loadPromptFile, type PromptFile } from '../prompts/prompt-loader'; -import { EvalPackLoader } from '../boundaries/eval-pack-loader'; -import { printGlobalSummary } from '../output/reporter'; -import { DefaultRequestBuilder } from '../providers/request-builder'; -import { loadDirective } from '../prompts/directive-loader'; -import { resolveTargets } from '../scan/file-resolver'; -import { parseCliOptions, parseEnvironment } from '../boundaries/index'; -import { handleUnknownError } from '../errors/index'; -import { evaluateFiles } from './orchestrator'; -import { DEFAULT_CONFIG_FILENAME } from '../config/constants'; +import type { Command } from "commander"; +import { existsSync } from "fs"; +import * as path from "path"; +import { createProvider } from "../providers/provider-factory"; +import { PerplexitySearchProvider } from "../providers/perplexity-provider"; +import type { SearchProvider } from "../providers/search-provider"; +import { loadConfig } from "../boundaries/config-loader"; +import { loadPromptFile, type PromptFile } from "../prompts/prompt-loader"; +import { EvalPackLoader } from "../boundaries/eval-pack-loader"; +import { printGlobalSummary } from "../output/reporter"; +import { DefaultRequestBuilder } from "../providers/request-builder"; +import { loadDirective } from "../prompts/directive-loader"; +import { resolveTargets } from "../scan/file-resolver"; +import { parseCliOptions, parseEnvironment } from "../boundaries/index"; +import { handleUnknownError } from "../errors/index"; +import { evaluateFiles } from "./orchestrator"; +import { OutputFormat } from "./types"; +import { DEFAULT_CONFIG_FILENAME } from "../config/constants"; /* * Registers the main evaluation command with Commander. @@ -22,21 +23,32 @@ import { DEFAULT_CONFIG_FILENAME } from '../config/constants'; */ export function registerMainCommand(program: Command): void { program - .option('-v, --verbose', 'Enable verbose logging') - .option('--show-prompt', 'Print full prompt and injected content') - .option('--show-prompt-trunc', 'Print truncated prompt/content previews (500 chars)') - .option('--debug-json', 'Print full JSON response from the API') - .option('--output ', 'Output format: line (default), json, or vale-json, rdjson', 'line') - .option(`--config ', 'Path to custom ${DEFAULT_CONFIG_FILENAME} config file`) - .argument('[paths...]', 'files or directories to check (optional)') + .option("-v, --verbose", "Enable verbose logging") + .option("--show-prompt", "Print full prompt and injected content") + .option( + "--show-prompt-trunc", + "Print truncated prompt/content previews (500 chars)" + ) + .option("--debug-json", "Print full JSON response from the API") + .option( + "--output ", + "Output format: line (default), json, or vale-json, rdjson", + "line" + ) + .option( + "--config ", + `Path to custom ${DEFAULT_CONFIG_FILENAME} config file` + ) + .option("--full", "Force full evaluation, ignore cache") + .option("--no-cache", "Disable caching entirely") + .argument("[paths...]", "files or directories to check (optional)") .action(async (paths: string[] = []) => { - // Parse and validate CLI options let cliOptions; try { cliOptions = parseCliOptions(program.opts()); } catch (e: unknown) { - const err = handleUnknownError(e, 'Parsing CLI options'); + const err = handleUnknownError(e, "Parsing CLI options"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -46,9 +58,9 @@ export function registerMainCommand(program: Command): void { try { env = parseEnvironment(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Validating environment variables'); + const err = handleUnknownError(e, "Validating environment variables"); console.error(`Error: ${err.message}`); - console.error('Please set these in your .env file or environment.'); + console.error("Please set these in your .env file or environment."); process.exit(1); } @@ -57,7 +69,7 @@ export function registerMainCommand(program: Command): void { try { directive = loadDirective(); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading directive'); + const err = handleUnknownError(e, "Loading directive"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -83,7 +95,7 @@ export function registerMainCommand(program: Command): void { try { config = loadConfig(process.cwd(), cliOptions.config); } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading configuration'); + const err = handleUnknownError(e, "Loading configuration"); console.error(`Error: ${err.message}`); process.exit(1); } @@ -100,8 +112,12 @@ export function registerMainCommand(program: Command): void { const packs = await loader.findAllPacks(rulesPath); if (packs.length === 0) { - console.warn(`[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath}.`); - console.warn(`[vectorlint] Please organize your rules into subdirectories (e.g., ${rulesPath}/VectorLint/ or ${rulesPath}/MyPack/).`); + console.warn( + `[vectorlint] Warning: No rule packs (subdirectories) found in ${rulesPath}.` + ); + console.warn( + `[vectorlint] Please organize your rules into subdirectories (e.g., ${rulesPath}/VectorLint/ or ${rulesPath}/MyPack/).` + ); } for (const packName of packs) { @@ -111,7 +127,8 @@ export function registerMainCommand(program: Command): void { for (const filePath of evalPaths) { const result = loadPromptFile(filePath, packName); if (result.warning) { - if (cliOptions.verbose) console.warn(`[vectorlint] ${result.warning}`); + if (cliOptions.verbose) + console.warn(`[vectorlint] ${result.warning}`); } if (result.prompt) { prompts.push(result.prompt); @@ -120,15 +137,24 @@ export function registerMainCommand(program: Command): void { } if (prompts.length === 0) { - console.error(`Error: no .md rules found in any packs in ${rulesPath}`); + console.error( + `Error: no .md rules found in any packs in ${rulesPath}` + ); process.exit(1); } } catch (e: unknown) { - const err = handleUnknownError(e, 'Loading prompts'); + const err = handleUnknownError(e, "Loading prompts"); console.error(`Error: failed to load prompts: ${err.message}`); process.exit(1); } + const normalizedScanPaths = config.scanPaths.map( + ({ runRules, ...rest }) => ({ + ...rest, + ...(runRules !== undefined ? { runRules } : {}), + }) + ); + // Resolve target files let targets: string[] = []; try { @@ -136,28 +162,35 @@ export function registerMainCommand(program: Command): void { cliArgs: paths, cwd: process.cwd(), rulesPath, - scanPaths: config.scanPaths, + scanPaths: normalizedScanPaths, configDir: config.configDir, }); } catch (e: unknown) { - const err = handleUnknownError(e, 'Resolving target files'); + const err = handleUnknownError(e, "Resolving target files"); console.error(`Error: failed to resolve target files: ${err.message}`); process.exit(1); } if (targets.length === 0) { - console.error('Error: no target files found to evaluate.'); + console.error("Error: no target files found to evaluate."); process.exit(1); } - - // Create search provider if API key is available - const searchProvider: SearchProvider | undefined = process.env.PERPLEXITY_API_KEY + const searchProvider: SearchProvider | undefined = process.env + .PERPLEXITY_API_KEY ? new PerplexitySearchProvider({ debug: false }) : undefined; - const outputFormat = cliOptions.output === 'JSON' ? 'json' : cliOptions.output; + // Convert string to OutputFormat enum + const outputFormatMap: Record = { + line: OutputFormat.Line, + json: OutputFormat.Json, + "vale-json": OutputFormat.ValeJson, + rdjson: OutputFormat.RdJson, + }; + const outputFormat = + outputFormatMap[cliOptions.output] || OutputFormat.Line; // Run evaluations via orchestrator const result = await evaluateFiles(targets, { @@ -168,11 +201,13 @@ export function registerMainCommand(program: Command): void { concurrency: config.concurrency, verbose: cliOptions.verbose, outputFormat: outputFormat, - scanPaths: config.scanPaths, + scanPaths: normalizedScanPaths, + cacheEnabled: !cliOptions.noCache, + forceFullRun: cliOptions.full, }); // Print global summary (only for line format) - if (cliOptions.output === 'line') { + if (outputFormat === OutputFormat.Line) { printGlobalSummary( result.totalFiles, result.totalErrors, @@ -182,6 +217,8 @@ export function registerMainCommand(program: Command): void { } // Exit with appropriate code - process.exit(result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0); + process.exit( + result.hadOperationalErrors || result.hadSeverityErrors ? 1 : 0 + ); }); } diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index caa14e3..2c8eea7 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -25,6 +25,14 @@ import { handleUnknownError, MissingDependencyError } from "../errors/index"; import { createEvaluator } from "../evaluators/index"; import { Type, Severity } from "../evaluators/types"; import { OutputFormat } from "./types"; +import { + CacheStore, + hashContent, + hashPrompts, + createCacheKeyString, + type CachedIssue, + type CachedScore, +} from "../cache/index"; import type { EvaluationOptions, EvaluationResult, @@ -78,7 +86,10 @@ async function runWithConcurrency( /* * Reports an issue in either line or JSON format. */ -function reportIssue(params: ReportIssueParams): void { +function reportIssue( + params: ReportIssueParams, + issueCollector?: CachedIssue[] +): void { const { file, line, @@ -93,6 +104,19 @@ function reportIssue(params: ReportIssueParams): void { match, } = params; + if (issueCollector) { + issueCollector.push({ + line, + column, + severity, + summary, + ruleName, + suggestion, + scoreText, + match, + }); + } + if (outputFormat === OutputFormat.Line) { const locStr = `${line}:${column}`; printIssueRow( @@ -155,6 +179,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { outputFormat, jsonFormatter, verbose, + issueCollector, } = params; let hadOperationalErrors = false; @@ -229,19 +254,22 @@ function locateAndReportViolations(params: ProcessViolationsParams): { matchedText, rowSummary, } of verifiedViolations) { - reportIssue({ - file: relFile, - line, - column, - severity, - summary: rowSummary, - ruleName, - outputFormat, - jsonFormatter, - ...(v.suggestion !== undefined && { suggestion: v.suggestion }), - scoreText, - match: matchedText, - }); + reportIssue( + { + file: relFile, + line, + column, + severity, + summary: rowSummary, + ruleName, + outputFormat, + jsonFormatter, + ...(v.suggestion !== undefined && { suggestion: v.suggestion }), + scoreText, + match: matchedText, + }, + issueCollector + ); } return { hadOperationalErrors }; @@ -266,6 +294,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose, + issueCollector, } = params; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -303,19 +332,22 @@ function extractAndReportCriterion( expTargetSpec?.suggestion || metaTargetSpec?.suggestion || "Add the required target section."; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity: Severity.ERROR, - summary, - ruleName, - outputFormat, - jsonFormatter, - suggestion, - scoreText: "nil", - match: "", - }); + reportIssue( + { + file: relFile, + line: 1, + column: 1, + severity: Severity.ERROR, + summary, + ruleName, + outputFormat, + jsonFormatter, + suggestion, + scoreText: "nil", + match: "", + }, + issueCollector + ); return { errors: 1, warnings: 0, @@ -412,6 +444,7 @@ function extractAndReportCriterion( outputFormat, jsonFormatter, verbose: !!verbose, + issueCollector, }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; @@ -429,18 +462,21 @@ function extractAndReportCriterion( const words = sum.split(/\s+/).filter(Boolean); const limited = words.slice(0, 15).join(" "); const summaryText = limited || "No findings"; - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: summaryText, - ruleName, - outputFormat, - jsonFormatter, - scoreText, - match: "", - }); + reportIssue( + { + file: relFile, + line: 1, + column: 1, + severity, + summary: summaryText, + ruleName, + outputFormat, + jsonFormatter, + scoreText, + match: "", + }, + issueCollector + ); } return { @@ -553,6 +589,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose, + issueCollector, } = params; const meta = promptFile.meta; const promptId = (meta.id || "").toString(); @@ -579,6 +616,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose: !!verbose, + issueCollector, }); hadOperationalErrors = hadOperationalErrors || violationResult.hadOperationalErrors; @@ -588,17 +626,20 @@ function routePromptResult( result.message ) { // For JSON, if there's a message but no violations, report it as a general issue - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: result.message, - ruleName, - outputFormat, - jsonFormatter, - match: "", - }); + reportIssue( + { + file: relFile, + line: 1, + column: 1, + severity, + summary: result.message, + ruleName, + outputFormat, + jsonFormatter, + match: "", + }, + issueCollector + ); } // Create scoreEntry for Quality Scores display @@ -614,6 +655,7 @@ function routePromptResult( hadOperationalErrors, hadSeverityErrors: severity === Severity.ERROR, scoreEntries: [scoreEntry], + scoreComponents: [], }; } @@ -643,6 +685,7 @@ function routePromptResult( outputFormat, jsonFormatter, verbose: !!verbose, + issueCollector, }); promptErrors += criterionResult.errors; @@ -673,6 +716,7 @@ function routePromptResult( hadOperationalErrors, hadSeverityErrors, scoreEntries: criterionScores, + scoreComponents: scoreComponents, }; } @@ -726,7 +770,8 @@ async function runPromptEvaluation( async function evaluateFile( params: EvaluateFileParams ): Promise { - const { file, options, jsonFormatter } = params; + const { file, options, jsonFormatter, cacheStore, promptsHash, useCache } = + params; const { prompts, provider, @@ -737,15 +782,86 @@ async function evaluateFile( verbose, } = options; + const content = readFileSync(file, "utf-8"); + const relFile = path.relative(process.cwd(), file) || file; + + // Check cache before running evaluations + if (useCache && cacheStore && promptsHash) { + const contentHash = hashContent(content); + const cacheKey = createCacheKeyString(relFile, contentHash, promptsHash); + const cached = cacheStore.get(cacheKey); + + if (cached) { + // Cache hit - return cached result without running LLM + if (outputFormat === OutputFormat.Line) { + printFileHeader(relFile); + console.log(" (cached)"); + } + + // Replay cached issues + if (cached.issues) { + for (const issue of cached.issues) { + reportIssue({ + file: relFile, + line: issue.line, + column: issue.column, + severity: issue.severity, + summary: issue.summary, + ruleName: issue.ruleName, + outputFormat, + jsonFormatter, + ...(issue.suggestion !== undefined + ? { suggestion: issue.suggestion } + : {}), + ...(issue.scoreText !== undefined + ? { scoreText: issue.scoreText } + : {}), + ...(issue.match !== undefined ? { match: issue.match } : {}), + }); + } + } + + if (cached.scores) { + if (outputFormat === OutputFormat.Line) { + const replayScores = new Map(); + for (const s of cached.scores) { + replayScores.set(s.ruleName, s.items); + } + printEvaluationSummaries(replayScores); + console.log(""); + } else if (outputFormat === OutputFormat.Json) { + for (const s of cached.scores) { + if (s.components && s.components.length > 0) { + ( + jsonFormatter as JsonFormatter | RdJsonFormatter + ).addEvaluationScore(relFile, { + id: s.ruleName, + scores: s.components, + }); + } + } + } + } + + return { + errors: cached.errors, + warnings: cached.warnings, + requestFailures: cached.requestFailures, + hadOperationalErrors: cached.hadOperationalErrors, + hadSeverityErrors: cached.hadSeverityErrors, + wasCacheHit: true, + }; + } + } + let hadOperationalErrors = false; let hadSeverityErrors = false; let totalErrors = 0; let totalWarnings = 0; let requestFailures = 0; - const allScores = new Map(); - - const content = readFileSync(file, "utf-8"); - const relFile = path.relative(process.cwd(), file) || file; + const allScores = new Map(); + // Collect issues for caching + const issueCollector: CachedIssue[] = []; if (outputFormat === OutputFormat.Line) { printFileHeader(relFile); @@ -833,6 +949,7 @@ async function evaluateFile( outputFormat, jsonFormatter, verbose, + issueCollector, }); totalErrors += promptResult.errors; totalWarnings += promptResult.warnings; @@ -842,22 +959,54 @@ async function evaluateFile( if (promptResult.scoreEntries && promptResult.scoreEntries.length > 0) { const ruleName = (p.meta.id || p.filename).toString(); - allScores.set(ruleName, promptResult.scoreEntries); + allScores.set(ruleName, { + ruleName, + items: promptResult.scoreEntries, + ...(promptResult.scoreComponents + ? { components: promptResult.scoreComponents } + : {}), + }); } } if (outputFormat === OutputFormat.Line) { - printEvaluationSummaries(allScores); + const summaryMap = new Map(); + for (const [key, val] of allScores.entries()) { + summaryMap.set(key, val.items); + } + printEvaluationSummaries(summaryMap); console.log(""); } - return { + const result: EvaluateFileResult = { errors: totalErrors, warnings: totalWarnings, requestFailures, hadOperationalErrors, hadSeverityErrors, + wasCacheHit: false, }; + + // Store result in cache + if (cacheStore && promptsHash) { + const contentHash = hashContent(content); + const cacheKey = createCacheKeyString(relFile, contentHash, promptsHash); + + const cachedScores: CachedScore[] = Array.from(allScores.values()); + + cacheStore.set(cacheKey, { + errors: totalErrors, + warnings: totalWarnings, + hadOperationalErrors, + hadSeverityErrors, + requestFailures, + issues: issueCollector, + scores: cachedScores, + timestamp: Date.now(), + }); + } + + return result; } /* @@ -869,7 +1018,13 @@ export async function evaluateFiles( targets: string[], options: EvaluationOptions ): Promise { - const { outputFormat = OutputFormat.Line } = options; + const { + outputFormat = OutputFormat.Line, + cacheEnabled = true, + forceFullRun = false, + prompts, + verbose, + } = options; let hadOperationalErrors = false; let hadSeverityErrors = false; @@ -877,6 +1032,28 @@ export async function evaluateFiles( let totalErrors = 0; let totalWarnings = 0; let requestFailures = 0; + let cacheHits = 0; + + // Initialize cache if enabled + const useCache = cacheEnabled && !forceFullRun; + let cacheStore: CacheStore | undefined; + let promptsHash: string | undefined; + + if (cacheEnabled) { + cacheStore = new CacheStore(); + promptsHash = hashPrompts(prompts); + + if (verbose) { + const cacheSize = cacheStore.size(); + if (forceFullRun) { + console.log( + `[vectorlint] Cache: --full flag, ignoring ${cacheSize} cached entries` + ); + } else if (cacheSize > 0) { + console.log(`[vectorlint] Cache: ${cacheSize} entries loaded`); + } + } + } let jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; if (outputFormat === OutputFormat.Json) { @@ -890,7 +1067,20 @@ export async function evaluateFiles( for (const file of targets) { try { totalFiles += 1; - const fileResult = await evaluateFile({ file, options, jsonFormatter }); + const fileResult = await evaluateFile({ + file, + options, + jsonFormatter, + cacheStore, + promptsHash, + useCache, + }); + + // Track cache hits + if (fileResult.wasCacheHit) { + cacheHits += 1; + } + totalErrors += fileResult.errors; totalWarnings += fileResult.warnings; requestFailures += fileResult.requestFailures; @@ -904,6 +1094,17 @@ export async function evaluateFiles( } } + // Save cache after all evaluations + if (cacheStore) { + cacheStore.save(); + + if (verbose && cacheHits > 0) { + console.log( + `[vectorlint] Cache: ${cacheHits}/${totalFiles} files from cache` + ); + } + } + // Output results based on format (always to stdout for JSON formats) if ( outputFormat === OutputFormat.Json || diff --git a/src/cli/types.ts b/src/cli/types.ts index f1e36b0..072dfa8 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -15,6 +15,7 @@ import type { SubjectiveResult, } from "../prompts/schema"; import { Severity } from "../evaluators/types"; +import type { CacheStore, CachedIssue } from "../cache/index"; export enum OutputFormat { Line = "line", @@ -32,6 +33,8 @@ export interface EvaluationOptions { verbose: boolean; scanPaths: FilePatternConfig[]; outputFormat?: OutputFormat; + cacheEnabled?: boolean; + forceFullRun?: boolean; } export interface EvaluationResult { @@ -49,6 +52,7 @@ export interface ErrorTrackingResult { hadOperationalErrors: boolean; hadSeverityErrors: boolean; scoreEntries?: EvaluationSummary[]; + scoreComponents?: ScoreComponent[]; } export interface EvaluationContext { @@ -85,6 +89,7 @@ export interface ProcessViolationsParams extends EvaluationContext { severity: Severity; ruleName: string; scoreText: string; + issueCollector?: CachedIssue[] | undefined; } export interface ProcessCriterionParams extends EvaluationContext { @@ -93,6 +98,7 @@ export interface ProcessCriterionParams extends EvaluationContext { promptId: string; promptFilename: string; meta: PromptMeta; + issueCollector?: CachedIssue[] | undefined; } export interface ProcessCriterionResult extends ErrorTrackingResult { @@ -110,6 +116,7 @@ export interface ValidationParams { export interface ProcessPromptResultParams extends EvaluationContext { promptFile: PromptFile; result: PromptEvaluationResult; + issueCollector?: CachedIssue[] | undefined; } export interface RunPromptEvaluationParams { @@ -128,8 +135,12 @@ export interface EvaluateFileParams { file: string; options: EvaluationOptions; jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + cacheStore?: CacheStore | undefined; + promptsHash?: string | undefined; + useCache?: boolean | undefined; } export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; + wasCacheHit?: boolean; } diff --git a/src/output/reporter.ts b/src/output/reporter.ts index abc4c06..c204c51 100644 --- a/src/output/reporter.ts +++ b/src/output/reporter.ts @@ -6,7 +6,7 @@ import { Severity } from '../evaluators/types'; export interface EvaluationSummary { id: string; scoreText: string; - score?: number; + score?: number | undefined; } export type Status = Severity; diff --git a/src/schemas/cache-schema.ts b/src/schemas/cache-schema.ts new file mode 100644 index 0000000..a1ea06f --- /dev/null +++ b/src/schemas/cache-schema.ts @@ -0,0 +1,59 @@ +import { z } from 'zod'; + +import { Severity } from '../evaluators/types'; + +// Schema for cached issues +const CACHED_ISSUE_SCHEMA = z.object({ + line: z.number(), + column: z.number(), + severity: z.nativeEnum(Severity), + summary: z.string(), + ruleName: z.string(), + suggestion: z.string().optional(), + scoreText: z.string().optional(), + match: z.string().optional(), +}); + +// Schema for evaluation summary in scores +const CACHED_EVALUATION_SUMMARY_SCHEMA = z.object({ + id: z.string(), + scoreText: z.string(), + score: z.number().optional(), +}); + +// Schema for granular score components +const SCORE_COMPONENT_SCHEMA = z.object({ + criterion: z.string(), + rawScore: z.number(), + maxScore: z.number(), + weightedScore: z.number(), + weightedMaxScore: z.number(), + normalizedScore: z.number(), + normalizedMaxScore: z.number(), +}); + +// Schema for grouped scores +const CACHED_SCORE_SCHEMA = z.object({ + ruleName: z.string(), + items: z.array(CACHED_EVALUATION_SUMMARY_SCHEMA), + components: z.array(SCORE_COMPONENT_SCHEMA).optional(), +}); + +// Schema for the main cached result +const CACHED_RESULT_SCHEMA = z.object({ + errors: z.number(), + warnings: z.number(), + hadOperationalErrors: z.boolean(), + hadSeverityErrors: z.boolean(), + requestFailures: z.number(), + issues: z.array(CACHED_ISSUE_SCHEMA).optional(), + scores: z.array(CACHED_SCORE_SCHEMA).optional(), + // Use unknown for jsonOutput as it can be any valid JSON structure + jsonOutput: z.unknown().optional(), + timestamp: z.number(), +}); + +export const CACHE_SCHEMA = z.object({ + version: z.number(), + entries: z.record(z.string(), CACHED_RESULT_SCHEMA) +}); diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index f216d16..e7c6cbe 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -10,6 +10,8 @@ export const CLI_OPTIONS_SCHEMA = z.object({ prompts: z.string().optional(), evals: z.string().optional(), config: z.string().optional(), + full: z.boolean().default(false), + noCache: z.boolean().default(false), }); // Validate command options schema diff --git a/tests/cache-store.test.ts b/tests/cache-store.test.ts new file mode 100644 index 0000000..34b5133 --- /dev/null +++ b/tests/cache-store.test.ts @@ -0,0 +1,256 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdirSync, rmSync, existsSync } from 'fs'; +import * as path from 'path'; +import { hashContent, hashPrompts, createCacheKeyString } from '../src/cache/content-hasher'; +import { CacheStore } from '../src/cache/cache-store'; +import type { PromptFile } from '../src/prompts/prompt-loader'; +import type { CachedResult } from '../src/cache/types'; + +describe('Content Hasher', () => { + describe('hashContent', () => { + it('produces consistent hashes for same content', () => { + const content = 'Hello, World!'; + const hash1 = hashContent(content); + const hash2 = hashContent(content); + expect(hash1).toBe(hash2); + }); + + it('normalizes line endings', () => { + const contentCRLF = 'Line 1\r\nLine 2'; + const contentLF = 'Line 1\nLine 2'; + expect(hashContent(contentCRLF)).toBe(hashContent(contentLF)); + }); + + it('trims whitespace', () => { + const content1 = ' Hello '; + const content2 = 'Hello'; + expect(hashContent(content1)).toBe(hashContent(content2)); + }); + + it('produces different hashes for different content', () => { + const hash1 = hashContent('Content A'); + const hash2 = hashContent('Content B'); + expect(hash1).not.toBe(hash2); + }); + }); + + describe('hashPrompts', () => { + const createMockPrompt = (id: string, body: string): PromptFile => ({ + id, + filename: `${id}.md`, + fullPath: `/path/to/${id}.md`, + meta: { + id, + name: `Test ${id}`, + }, + body, + pack: 'TestPack', + }); + + it('produces consistent hashes for same prompts', () => { + const prompts = [createMockPrompt('rule1', 'Body 1')]; + const hash1 = hashPrompts(prompts); + const hash2 = hashPrompts(prompts); + expect(hash1).toBe(hash2); + }); + + it('produces same hash regardless of prompt order', () => { + const prompt1 = createMockPrompt('a-rule', 'Body A'); + const prompt2 = createMockPrompt('b-rule', 'Body B'); + + const hash1 = hashPrompts([prompt1, prompt2]); + const hash2 = hashPrompts([prompt2, prompt1]); + expect(hash1).toBe(hash2); + }); + + it('produces different hash when prompt body changes', () => { + const v1 = [createMockPrompt('rule1', 'Original body')]; + const v2 = [createMockPrompt('rule1', 'Modified body')]; + + expect(hashPrompts(v1)).not.toBe(hashPrompts(v2)); + }); + + it('produces different hash when prompt meta changes', () => { + const v1: PromptFile[] = [{ + id: 'rule1', + filename: 'rule1.md', + fullPath: '/path/rule1.md', + meta: { id: 'rule1', name: 'Rule One' }, + body: 'Same body', + }]; + const v2: PromptFile[] = [{ + id: 'rule1', + filename: 'rule1.md', + fullPath: '/path/rule1.md', + meta: { id: 'rule1', name: 'Rule One Modified' }, + body: 'Same body', + }]; + + expect(hashPrompts(v1)).not.toBe(hashPrompts(v2)); + }); + }); + + describe('createCacheKeyString', () => { + it('creates a key with truncated hashes', () => { + const key = createCacheKeyString( + 'path/to/file.md', + 'abc123def456ghi789jkl012mno345pqr678', + 'xyz987wvu654tsr321qpo098nml765kji432' + ); + expect(key).toBe('path/to/file.md|abc123def456ghi7|xyz987wvu654tsr3'); + }); + }); +}); + +describe('CacheStore', () => { + const testDir = path.join(__dirname, 'fixtures', 'cache-test'); + const cacheDir = '.test-vectorlint'; + + beforeEach(() => { + // Ensure test directory exists + if (!existsSync(testDir)) { + mkdirSync(testDir, { recursive: true }); + } + // Clean up any existing test cache + const fullCacheDir = path.join(testDir, cacheDir); + if (existsSync(fullCacheDir)) { + rmSync(fullCacheDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up test cache + const fullCacheDir = path.join(testDir, cacheDir); + if (existsSync(fullCacheDir)) { + rmSync(fullCacheDir, { recursive: true }); + } + }); + + it('returns undefined for missing keys', () => { + const store = new CacheStore(testDir, cacheDir); + expect(store.get('nonexistent-key')).toBeUndefined(); + }); + + it('stores and retrieves results', () => { + const store = new CacheStore(testDir, cacheDir); + const result: CachedResult = { + errors: 2, + warnings: 1, + hadOperationalErrors: false, + hadSeverityErrors: true, + requestFailures: 0, + timestamp: Date.now(), + }; + + store.set('test-key', result); + const retrieved = store.get('test-key'); + + expect(retrieved).toBeDefined(); + expect(retrieved?.errors).toBe(2); + expect(retrieved?.warnings).toBe(1); + expect(retrieved?.hadSeverityErrors).toBe(true); + }); + + it('persists cache to disk', () => { + const store1 = new CacheStore(testDir, cacheDir); + const result: CachedResult = { + errors: 1, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }; + + store1.set('persistent-key', result); + store1.save(); + + // Create new store instance to load from disk + const store2 = new CacheStore(testDir, cacheDir); + const retrieved = store2.get('persistent-key'); + + expect(retrieved).toBeDefined(); + expect(retrieved?.errors).toBe(1); + }); + + it('creates cache directory if missing', () => { + const store = new CacheStore(testDir, cacheDir); + store.set('key', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + store.save(); + + const fullCacheDir = path.join(testDir, cacheDir); + expect(existsSync(fullCacheDir)).toBe(true); + expect(existsSync(path.join(fullCacheDir, 'cache.json'))).toBe(true); + }); + + it('clears all entries', () => { + const store = new CacheStore(testDir, cacheDir); + store.set('key1', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + store.set('key2', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + + expect(store.size()).toBe(2); + store.clear(); + expect(store.size()).toBe(0); + }); + + it('reports correct size', () => { + const store = new CacheStore(testDir, cacheDir); + expect(store.size()).toBe(0); + + store.set('key1', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + expect(store.size()).toBe(1); + + store.set('key2', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + expect(store.size()).toBe(2); + }); + + it('checks key existence with has()', () => { + const store = new CacheStore(testDir, cacheDir); + expect(store.has('missing')).toBe(false); + + store.set('exists', { + errors: 0, + warnings: 0, + hadOperationalErrors: false, + hadSeverityErrors: false, + requestFailures: 0, + timestamp: Date.now(), + }); + expect(store.has('exists')).toBe(true); + }); +});