From 79ad8df195bebd94c32df99a339a101c5596b9c0 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 14:40:41 +0100 Subject: [PATCH 1/4] refactor: remove dummy JSON issues and tighten schema guidance - Stop emitting synthetic JSON/vale-json issues when filtered violations leave no surfaced findings, so clean runs no longer include noise cards. - Add explicit analysis/suggestion descriptions to judge/check structured output schemas to keep model responses concise and actionable. - Add regression tests for zero-violation JSON paths and schema description constraints to prevent future drift. --- src/cli/orchestrator.ts | 16 ------ src/prompts/schema.ts | 22 ++++++-- tests/orchestrator-filtering.test.ts | 81 ++++++++++++++++++++++++++++ tests/prompt-schema.test.ts | 35 ++++++++++++ 4 files changed, 134 insertions(+), 20 deletions(-) create mode 100644 tests/prompt-schema.test.ts diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 4bc4828..be5ab5e 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -670,22 +670,6 @@ function routePromptResult( } } - // If no violations but we have a message (JSON output), report it - if (violationCount === 0 && (outputFormat === OutputFormat.Json || outputFormat === OutputFormat.ValeJson) && result.message) { - const ruleName = buildRuleName(promptFile.pack, promptId, undefined); - reportIssue({ - file: relFile, - line: 1, - column: 1, - severity, - summary: result.message, - ruleName, - outputFormat, - jsonFormatter, - match: "", - }); - } - // Create scoreEntry for Quality Scores display const scoreEntry: EvaluationSummary = { id: buildRuleName(promptFile.pack, promptId, undefined), diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index daf65b3..ee6c08e 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -52,8 +52,15 @@ export function buildJudgeLLMSchema() { context_before: { type: "string" }, context_after: { type: "string" }, description: { type: "string" }, - analysis: { type: "string" }, - suggestion: { type: "string" }, + analysis: { + type: "string", + description: + "A concise 1-2 sentence explanation of the specific issue.", + }, + suggestion: { + type: "string", + description: "Suggest a fix in 15 words or less.", + }, fix: { type: "string" }, rule_quote: { type: "string" }, checks: { @@ -156,8 +163,15 @@ export function buildCheckLLMSchema() { context_before: { type: "string" }, context_after: { type: "string" }, description: { type: "string" }, - analysis: { type: "string" }, - suggestion: { type: "string" }, + analysis: { + type: "string", + description: + "A concise 1-2 sentence explanation of the specific issue.", + }, + suggestion: { + type: "string", + description: "Suggest a fix in 15 words or less.", + }, fix: { type: "string" }, rule_quote: { type: "string" }, checks: { diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 8a740b1..f3b300c 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -339,4 +339,85 @@ describe("CLI violation filtering", () => { ); expect(zeroThresholdRun.totalWarnings).toBe(2); }); + + it("does not emit dummy issues in JSON output when no violations are surfaced", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckJsonPrompt", + name: "Check JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + severity: Severity.WARNING, + finalScore: 10, + percentage: 100, + message: "No issues found", + violations: [ + makeCheckViolation({ + confidence: 0.2, + }), + ], + }) + ); + + const run = await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.Json, + }); + + expect(run.totalWarnings).toBe(0); + + const loggedJson = vi.mocked(console.log).mock.calls.at(-1)?.[0]; + const parsed = JSON.parse(String(loggedJson)) as { + files: Record }>; + }; + const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); + + expect(allIssues).toHaveLength(0); + expect(JSON.stringify(parsed)).not.toContain("No issues found"); + }); + + it("does not emit dummy issues in Vale JSON output when no violations are surfaced", async () => { + const targetFile = createTempFile("Alpha text\n"); + const prompt = createPrompt({ + id: "CheckValeJsonPrompt", + name: "Check Vale JSON Prompt", + type: "check", + severity: Severity.WARNING, + }); + + EVALUATE_MOCK.mockResolvedValue( + makeCheckResult({ + severity: Severity.WARNING, + finalScore: 10, + percentage: 100, + message: "No issues found", + violations: [ + makeCheckViolation({ + confidence: 0.2, + }), + ], + }) + ); + + const run = await evaluateFiles([targetFile], { + ...createBaseOptions([prompt]), + outputFormat: OutputFormat.ValeJson, + }); + + expect(run.totalWarnings).toBe(0); + + const loggedJson = vi.mocked(console.log).mock.calls.at(-1)?.[0]; + const parsed = JSON.parse(String(loggedJson)) as Record< + string, + Array<{ Message: string }> + >; + const allIssues = Object.values(parsed).flat(); + + expect(allIssues).toHaveLength(0); + expect(JSON.stringify(parsed)).not.toContain("No issues found"); + }); }); diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts new file mode 100644 index 0000000..43c6748 --- /dev/null +++ b/tests/prompt-schema.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { buildCheckLLMSchema, buildJudgeLLMSchema } from "../src/prompts/schema"; + +describe("prompt schema verbosity constraints", () => { + it("includes concise analysis and suggestion descriptions for check schema", () => { + const schema = buildCheckLLMSchema(); + const violationProperties = + schema.schema.properties.violations.items.properties; + + expect(violationProperties.analysis).toEqual({ + type: "string", + description: "A concise 1-2 sentence explanation of the specific issue.", + }); + expect(violationProperties.suggestion).toEqual({ + type: "string", + description: "Suggest a fix in 15 words or less.", + }); + }); + + it("includes concise analysis and suggestion descriptions for judge schema", () => { + const schema = buildJudgeLLMSchema(); + const violationProperties = + schema.schema.properties.criteria.items.properties.violations.items + .properties; + + expect(violationProperties.analysis).toEqual({ + type: "string", + description: "A concise 1-2 sentence explanation of the specific issue.", + }); + expect(violationProperties.suggestion).toEqual({ + type: "string", + description: "Suggest a fix in 15 words or less.", + }); + }); +}); From 8fe23dfc388247aaa700ccecf370362570105df5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 14:45:02 +0100 Subject: [PATCH 2/4] chore: sync package-lock to 2.4.0 --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4db4783..0477c9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "vectorlint", - "version": "2.3.0", + "version": "2.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "vectorlint", - "version": "2.3.0", + "version": "2.4.0", "license": "Apache-2.0", "dependencies": { "@ai-sdk/amazon-bedrock": "^4.0.64", From c58729b86bc399ca483d699a8bb978f20203dc23 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 14:51:55 +0100 Subject: [PATCH 3/4] refactor(cli): satisfy lint in filtering path - Remove an unused check-result variable left after JSON-noise cleanup. - Parse mocked console output inline in tests to avoid unsafe-any assignments while preserving assertions. --- src/cli/orchestrator.ts | 2 -- tests/orchestrator-filtering.test.ts | 13 ++++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index be5ab5e..dbd1ecc 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -619,8 +619,6 @@ function routePromptResult( const { decisions, surfacedViolations } = getViolationFilterResults( result.violations ); - const violationCount = surfacedViolations.length; - // Group violations by criterionName const violationsByCriterion = new Map< string | undefined, diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index f3b300c..245f447 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -370,8 +370,9 @@ describe("CLI violation filtering", () => { expect(run.totalWarnings).toBe(0); - const loggedJson = vi.mocked(console.log).mock.calls.at(-1)?.[0]; - const parsed = JSON.parse(String(loggedJson)) as { + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as { files: Record }>; }; const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); @@ -410,11 +411,9 @@ describe("CLI violation filtering", () => { expect(run.totalWarnings).toBe(0); - const loggedJson = vi.mocked(console.log).mock.calls.at(-1)?.[0]; - const parsed = JSON.parse(String(loggedJson)) as Record< - string, - Array<{ Message: string }> - >; + const parsed = JSON.parse( + String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) + ) as Record>; const allIssues = Object.values(parsed).flat(); expect(allIssues).toHaveLength(0); From 2f233a99e49c922bf361b5ad5a2c7d4e425195a5 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 14:56:00 +0100 Subject: [PATCH 4/4] test(cli): use concrete output types in JSON assertions - Replace inline parsed-output shapes with Result and ValeOutput types to simplify test code and keep contract checks explicit. - Preserve existing no-dummy-issue assertions and runtime behavior. --- tests/orchestrator-filtering.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/orchestrator-filtering.test.ts b/tests/orchestrator-filtering.test.ts index 245f447..6bbe6d1 100644 --- a/tests/orchestrator-filtering.test.ts +++ b/tests/orchestrator-filtering.test.ts @@ -5,8 +5,10 @@ import path from "path"; import { evaluateFiles } from "../src/cli/orchestrator"; import { OutputFormat, type EvaluationOptions } from "../src/cli/types"; import { EvaluationType, Severity } from "../src/evaluators/types"; +import type { Result } from "../src/output/json-formatter"; import type { PromptFile } from "../src/prompts/prompt-loader"; import type { CheckResult, JudgeResult } from "../src/prompts/schema"; +import type { ValeOutput } from "../src/schemas/vale-responses"; const { EVALUATE_MOCK } = vi.hoisted(() => ({ EVALUATE_MOCK: vi.fn(), @@ -372,9 +374,7 @@ describe("CLI violation filtering", () => { const parsed = JSON.parse( String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as { - files: Record }>; - }; + ) as Result; const allIssues = Object.values(parsed.files).flatMap((file) => file.issues); expect(allIssues).toHaveLength(0); @@ -413,7 +413,7 @@ describe("CLI violation filtering", () => { const parsed = JSON.parse( String(vi.mocked(console.log).mock.calls.at(-1)?.[0]) - ) as Record>; + ) as ValeOutput; const allIssues = Object.values(parsed).flat(); expect(allIssues).toHaveLength(0);