From 6b88da8f807ec175bfd157b9a7c11a260c0b23fd Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:33:19 +0100 Subject: [PATCH 1/8] feat: add message field to check and judge LLM schemas --- src/prompts/schema.ts | 12 ++++++++++++ tests/prompt-schema.test.ts | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index f168bf6..c0af435 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -57,6 +57,11 @@ export function buildJudgeLLMSchema() { description: "A concise 1-2 sentence explanation of the specific issue.", }, + message: { + type: "string", + description: + "Under 15 words. State the issue directly to the document author. No rule references.", + }, suggestion: { type: "string", description: "Suggest a fix in 15 words or less.", @@ -124,6 +129,7 @@ export function buildJudgeLLMSchema() { "context_after", "description", "analysis", + "message", "suggestion", "fix", "rule_quote", @@ -168,6 +174,11 @@ export function buildCheckLLMSchema() { description: "A concise 1-2 sentence explanation of the specific issue.", }, + message: { + type: "string", + description: + "Under 15 words. State the issue directly to the document author. No rule references.", + }, suggestion: { type: "string", description: "Suggest a fix in 15 words or less.", @@ -235,6 +246,7 @@ export function buildCheckLLMSchema() { "context_after", "description", "analysis", + "message", "suggestion", "fix", "rule_quote", diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index 43c6748..123079f 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -32,4 +32,27 @@ describe("prompt schema verbosity constraints", () => { description: "Suggest a fix in 15 words or less.", }); }); + + it("check schema violation includes message field with correct description", () => { + const schema = buildCheckLLMSchema(); + const violationProperties = + schema.schema.properties.violations.items.properties; + + expect(violationProperties.message).toEqual({ + type: "string", + description: "Under 15 words. State the issue directly to the document author. No rule references.", + }); + }); + + it("judge schema violation includes message field with correct description", () => { + const schema = buildJudgeLLMSchema(); + const violationProperties = + schema.schema.properties.criteria.items.properties.violations.items + .properties; + + expect(violationProperties.message).toEqual({ + type: "string", + description: "Under 15 words. State the issue directly to the document author. No rule references.", + }); + }); }); From 6076e1b19a8bcefa8a7bacb02e9b8b6af03a7496 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:40:28 +0100 Subject: [PATCH 2/8] feat: add message field to violation TypeScript types - Add `message: string` to JudgeLLMResult violation array - Add `message: string` to CheckLLMResult violation array - Add `message: string` to JudgeResult criteria violations array - Add `message?: string` (optional) to CheckResult violations array - Add `message?: string` (optional) to CheckItem type - Add required-array assertions for message in prompt-schema tests --- src/prompts/schema.ts | 5 +++++ tests/prompt-schema.test.ts | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index c0af435..9faa05e 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -275,6 +275,7 @@ export type JudgeLLMResult = { context_after: string; description: string; analysis: string; + message: string; suggestion: string; fix: string; rule_quote: string; @@ -291,6 +292,7 @@ export type CheckLLMResult = { line: number; description: string; analysis: string; + message: string; suggestion: string; fix: string; quoted_text: string; @@ -321,6 +323,7 @@ export type JudgeResult = { context_after: string; description: string; analysis: string; + message: string; suggestion: string; fix: string; rule_quote: string; @@ -337,6 +340,7 @@ export type CheckItem = { line: number; description: string; analysis: string; + message?: string; suggestion?: string; fix?: string; quoted_text?: string; @@ -360,6 +364,7 @@ export type CheckResult = { violations: Array<{ line?: number; analysis: string; + message?: string; suggestion?: string; fix?: string; quoted_text?: string; diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index 123079f..1bd11ae 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -55,4 +55,17 @@ describe("prompt schema verbosity constraints", () => { description: "Under 15 words. State the issue directly to the document author. No rule references.", }); }); + + it("check schema violation required array includes message", () => { + const schema = buildCheckLLMSchema(); + const required = schema.schema.properties.violations.items.required; + expect(required).toContain("message"); + }); + + it("judge schema violation required array includes message", () => { + const schema = buildJudgeLLMSchema(); + const required = + schema.schema.properties.criteria.items.properties.violations.items.required; + expect(required).toContain("message"); + }); }); From ea04e7b72f1c10560747d733e835cd94c6e51bc2 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:42:22 +0100 Subject: [PATCH 3/8] feat: display message field instead of analysis in terminal output --- src/cli/orchestrator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index ee3cf08..77f9f59 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -209,7 +209,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { for (const v of violations) { if (!v) continue; - const rowSummary = (v.analysis || "").trim(); + const rowSummary = (v.message || v.analysis || "").trim(); try { const locWithMatch = locateQuotedText( From 410e52b1949f98659465678cad307b14d9fd4b8e Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:43:22 +0100 Subject: [PATCH 4/8] docs: add directive guidance distinguishing message from analysis --- src/prompts/directive-loader.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index f4ed43b..63de626 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -77,6 +77,8 @@ filter can decide whether to surface it. Return valid JSON matching the required schema exactly. +The \`message\` field is shown directly to the document author in the terminal and UI. Keep it under 15 words, author-addressed, with no rule references. The \`analysis\` field is your internal reasoning and is not shown in the UI. + ## Hard constraints - Do NOT invent evidence. Every quoted span must be copied exactly from the Input. - Use the provided line numbers. From d3556ba0f358e523a793450e402b7e206da5d669 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:46:27 +0100 Subject: [PATCH 5/8] fix: remove unnecessary analysis fallback from rowSummary --- src/cli/orchestrator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 77f9f59..2a3831c 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -209,7 +209,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { for (const v of violations) { if (!v) continue; - const rowSummary = (v.message || v.analysis || "").trim(); + const rowSummary = (v.message || "").trim(); try { const locWithMatch = locateQuotedText( From 23379b8bf9a64bebef3fa81c92242948e4b5077a Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:47:57 +0100 Subject: [PATCH 6/8] fix: use 'user' instead of 'document author' in message field guidance --- src/prompts/directive-loader.ts | 2 +- src/prompts/schema.ts | 4 ++-- tests/prompt-schema.test.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/prompts/directive-loader.ts b/src/prompts/directive-loader.ts index 63de626..13e28ca 100644 --- a/src/prompts/directive-loader.ts +++ b/src/prompts/directive-loader.ts @@ -77,7 +77,7 @@ filter can decide whether to surface it. Return valid JSON matching the required schema exactly. -The \`message\` field is shown directly to the document author in the terminal and UI. Keep it under 15 words, author-addressed, with no rule references. The \`analysis\` field is your internal reasoning and is not shown in the UI. +The \`message\` field is shown directly to the user in the terminal and UI. Keep it under 15 words, with no rule references. The \`analysis\` field is your internal reasoning and is not shown in the UI. ## Hard constraints - Do NOT invent evidence. Every quoted span must be copied exactly from the Input. diff --git a/src/prompts/schema.ts b/src/prompts/schema.ts index 9faa05e..26c137e 100644 --- a/src/prompts/schema.ts +++ b/src/prompts/schema.ts @@ -60,7 +60,7 @@ export function buildJudgeLLMSchema() { message: { type: "string", description: - "Under 15 words. State the issue directly to the document author. No rule references.", + "Under 15 words. State the issue directly to the user. No rule references.", }, suggestion: { type: "string", @@ -177,7 +177,7 @@ export function buildCheckLLMSchema() { message: { type: "string", description: - "Under 15 words. State the issue directly to the document author. No rule references.", + "Under 15 words. State the issue directly to the user. No rule references.", }, suggestion: { type: "string", diff --git a/tests/prompt-schema.test.ts b/tests/prompt-schema.test.ts index 1bd11ae..bf651a8 100644 --- a/tests/prompt-schema.test.ts +++ b/tests/prompt-schema.test.ts @@ -40,7 +40,7 @@ describe("prompt schema verbosity constraints", () => { expect(violationProperties.message).toEqual({ type: "string", - description: "Under 15 words. State the issue directly to the document author. No rule references.", + description: "Under 15 words. State the issue directly to the user. No rule references.", }); }); @@ -52,7 +52,7 @@ describe("prompt schema verbosity constraints", () => { expect(violationProperties.message).toEqual({ type: "string", - description: "Under 15 words. State the issue directly to the document author. No rule references.", + description: "Under 15 words. State the issue directly to the user. No rule references.", }); }); From f7b4599df6006b216e1491393bba9c481d3c456b Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 15:58:09 +0100 Subject: [PATCH 7/8] feat: include analysis field in native JSON output --- src/cli/orchestrator.ts | 3 +++ src/cli/types.ts | 1 + src/output/json-formatter.ts | 1 + 3 files changed, 5 insertions(+) diff --git a/src/cli/orchestrator.ts b/src/cli/orchestrator.ts index 2a3831c..9fd50a0 100644 --- a/src/cli/orchestrator.ts +++ b/src/cli/orchestrator.ts @@ -121,6 +121,7 @@ function reportIssue(params: ReportIssueParams): void { ruleName, outputFormat, jsonFormatter, + analysis, suggestion, fix, scoreText, @@ -165,6 +166,7 @@ function reportIssue(params: ReportIssueParams): void { message: summary, rule: ruleName, match: match || "", + ...(analysis ? { analysis } : {}), ...(suggestion ? { suggestion } : {}), ...(fix ? { fix } : {}), }; @@ -274,6 +276,7 @@ function locateAndReportViolations(params: ProcessViolationsParams): { ruleName, outputFormat, jsonFormatter, + ...(v.analysis !== undefined && { analysis: v.analysis }), ...(v.suggestion !== undefined && { suggestion: v.suggestion }), ...(v.fix !== undefined && { fix: v.fix }), scoreText, diff --git a/src/cli/types.ts b/src/cli/types.ts index 902ea2f..fc26a58 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -68,6 +68,7 @@ export interface ReportIssueParams { ruleName: string; outputFormat: OutputFormat; jsonFormatter: ValeJsonFormatter | JsonFormatter | RdJsonFormatter; + analysis?: string; suggestion?: string; fix?: string; scoreText?: string; diff --git a/src/output/json-formatter.ts b/src/output/json-formatter.ts index 54a7a64..eeddac6 100644 --- a/src/output/json-formatter.ts +++ b/src/output/json-formatter.ts @@ -23,6 +23,7 @@ export interface Issue { message: string; rule: string; match: string; + analysis?: string; suggestion?: string; fix?: string; } From 18e923ecb071cf40cab94082ca0648f55a45d0e3 Mon Sep 17 00:00:00 2001 From: Osho Emmanuel Date: Wed, 4 Mar 2026 16:03:44 +0100 Subject: [PATCH 8/8] fix: add message to ProcessViolationsParams violations type --- src/cli/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cli/types.ts b/src/cli/types.ts index fc26a58..31c4b6d 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -81,6 +81,7 @@ export interface ProcessViolationsParams extends EvaluationContext { quoted_text?: string; context_before?: string; context_after?: string; + message?: string; analysis?: string; suggestion?: string; fix?: string;