From f8414a00da62baeaa064f4ca5166e56dc716c645 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:30:20 +0100 Subject: [PATCH 01/32] Create types for style guide feature --- src/style-guide/types.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/style-guide/types.ts diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts new file mode 100644 index 0000000..14b8096 --- /dev/null +++ b/src/style-guide/types.ts @@ -0,0 +1,26 @@ +export enum StyleGuideFormat { + MARKDOWN = 'markdown', + AUTO = 'auto', +} + +export enum RuleCategory { + GRAMMAR = 'grammar', + TONE = 'tone', + TERMINOLOGY = 'terminology', + STRUCTURE = 'structure', + FORMATTING = 'formatting', + ACCESSIBILITY = 'accessibility', + SEO = 'seo', + CUSTOM = 'custom', +} + +export interface ParserOptions { + format?: StyleGuideFormat; + verbose?: boolean; + strict?: boolean; +} + +export interface ParserResult { + data: T; + warnings: string[]; +} From ea314da7e25bdc8099e9250efec96f86947b63a2 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:31:37 +0100 Subject: [PATCH 02/32] Create schemas for style guide feature --- src/schemas/style-guide-schemas.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/schemas/style-guide-schemas.ts diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts new file mode 100644 index 0000000..898161b --- /dev/null +++ b/src/schemas/style-guide-schemas.ts @@ -0,0 +1,28 @@ +import { z } from 'zod'; + +export const STYLE_GUIDE_EXAMPLES_SCHEMA = z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), +}).strict(); + +export const STYLE_GUIDE_RULE_SCHEMA = z.object({ + id: z.string(), + category: z.string(), + description: z.string(), + severity: z.enum(['error', 'warning']).optional(), + examples: STYLE_GUIDE_EXAMPLES_SCHEMA.optional(), + weight: z.number().positive().optional(), + metadata: z.record(z.unknown()).optional(), +}).strict(); + +export const STYLE_GUIDE_SCHEMA = z.object({ + name: z.string(), + version: z.string().optional(), + description: z.string().optional(), + rules: z.array(STYLE_GUIDE_RULE_SCHEMA), + metadata: z.record(z.unknown()).optional(), +}).strict(); + +export type StyleGuideExamples = z.infer; +export type StyleGuideRule = z.infer; +export type ParsedStyleGuide = z.infer; From 11d0cfa4515d41262c0a5e0ee45257a2e7e7aad4 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:38:06 +0100 Subject: [PATCH 03/32] Implement style guide error handling --- src/errors/style-guide-errors.ts | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 src/errors/style-guide-errors.ts diff --git a/src/errors/style-guide-errors.ts b/src/errors/style-guide-errors.ts new file mode 100644 index 0000000..d22d26f --- /dev/null +++ b/src/errors/style-guide-errors.ts @@ -0,0 +1,62 @@ +/** + * Base error for style guide operations + */ +export class StyleGuideError extends Error { + constructor(message: string) { + super(message); + this.name = 'StyleGuideError'; + } +} + +/** + * Error thrown when parsing style guide fails + */ +export class StyleGuideParseError extends StyleGuideError { + constructor( + message: string, + public filePath?: string, + public line?: number + ) { + super(message); + this.name = 'StyleGuideParseError'; + } +} + +/** + * Error thrown when style guide validation fails + */ +export class StyleGuideValidationError extends StyleGuideError { + constructor( + message: string, + public issues?: string[] + ) { + super(message); + this.name = 'StyleGuideValidationError'; + } +} + +/** + * Error thrown when eval generation fails + */ +export class EvalGenerationError extends StyleGuideError { + constructor( + message: string, + public ruleId?: string + ) { + super(message); + this.name = 'EvalGenerationError'; + } +} + +/** + * Error thrown when unsupported format is encountered + */ +export class UnsupportedFormatError extends StyleGuideError { + constructor( + message: string, + public format?: string + ) { + super(message); + this.name = 'UnsupportedFormatError'; + } +} From 454a6520cad3d19cfa75ef80092641321a38c347 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:39:53 +0100 Subject: [PATCH 04/32] feat: Add style guide parser, tests --- src/style-guide/style-guide-parser.ts | 360 ++++++++++++++++++ tests/style-guide/fixtures/invalid.txt | 3 + .../fixtures/sample-style-guide.md | 28 ++ tests/style-guide/parser.test.ts | 178 +++++++++ 4 files changed, 569 insertions(+) create mode 100644 src/style-guide/style-guide-parser.ts create mode 100644 tests/style-guide/fixtures/invalid.txt create mode 100644 tests/style-guide/fixtures/sample-style-guide.md create mode 100644 tests/style-guide/parser.test.ts diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts new file mode 100644 index 0000000..16f54f6 --- /dev/null +++ b/src/style-guide/style-guide-parser.ts @@ -0,0 +1,360 @@ +import { readFileSync } from 'fs'; +import * as path from 'path'; +import YAML from 'yaml'; +import { + STYLE_GUIDE_SCHEMA, + type ParsedStyleGuide, + type StyleGuideRule, +} from '../schemas/style-guide-schemas'; +import { + StyleGuideParseError, + StyleGuideValidationError, + UnsupportedFormatError, +} from '../errors/style-guide-errors'; +import { + StyleGuideFormat, + RuleCategory, + type ParserOptions, + type ParserResult, +} from './types'; + +/** + * Parser for converting style guide documents into structured format + */ +export class StyleGuideParser { + private warnings: string[] = []; + + parse(filePath: string, options: ParserOptions = {}): ParserResult { + this.warnings = []; + + try { + const content = readFileSync(filePath, 'utf-8'); + const format = options.format || this.detectFormat(filePath); + + let result: ParsedStyleGuide; + + switch (format) { + case StyleGuideFormat.MARKDOWN: + result = this.parseMarkdown(content); + break; + default: + throw new UnsupportedFormatError( + `Unsupported format: ${format}`, + format + ); + } + + this.validate(result); + + // Auto-categorize rules if not already categorized + result.rules = result.rules.map((rule) => this.categorizeRule(rule)); + + if (options.verbose && this.warnings.length > 0) { + console.warn('[StyleGuideParser] Warnings:'); + this.warnings.forEach((w) => console.warn(` - ${w}`)); + } + + return { + data: result, + warnings: this.warnings, + }; + } catch (error) { + if (error instanceof StyleGuideParseError || error instanceof UnsupportedFormatError) { + throw error; + } + const err = error instanceof Error ? error : new Error(String(error)); + throw new StyleGuideParseError( + `Failed to parse style guide: ${err.message}`, + filePath + ); + } + } + + /** + * Parse Markdown format style guide + */ + parseMarkdown(content: string): ParsedStyleGuide { + const rules: StyleGuideRule[] = []; + let name = 'Untitled Style Guide'; + let version: string | undefined; + let description: string | undefined; + + // Check for YAML frontmatter + let bodyContent = content; + if (content.startsWith('---')) { + const endIndex = content.indexOf('\n---', 3); + if (endIndex !== -1) { + const frontmatter = content.slice(3, endIndex).trim(); + bodyContent = content.slice(endIndex + 4); + + try { + const meta = YAML.parse(frontmatter); + if (meta.name) name = meta.name; + if (meta.version) version = meta.version; + if (meta.description) description = meta.description; + } catch (e) { + this.warnings.push('Failed to parse YAML frontmatter, using defaults'); + } + } + } + + // Extract title from first H1 if no name in frontmatter + const h1Match = bodyContent.match(/^#\s+(.+)$/m); + if (h1Match && h1Match[1] && name === 'Untitled Style Guide') { + name = h1Match[1].trim(); + } + + // Parse rules from sections + const sections = this.extractMarkdownSections(bodyContent); + let ruleCounter = 0; + + for (const section of sections) { + // Each H2 or H3 section could be a category + // If it's H2, it's likely a category header. If H3, it might be a rule. + let category = 'general'; + + // Try to find the parent H2 for this section if possible, + // but for now let's just look for category in the current section title if it's H2 + if (section.level === 2) { + category = section.title.replace(/^\d+\.\s*/, '').trim(); // Remove "1. " prefix + } + + // If section is H3, treat the title itself as a rule + if (section.level === 3) { + ruleCounter++; + const ruleId = this.generateRuleId(section.title, ruleCounter); + rules.push({ + id: ruleId, + category: 'general', // We'd need state to know the parent category, but 'general' is safe for now + description: section.title.replace(/^\*\*|\*\*$/g, '').trim(), // Remove bold markers + severity: 'warning', + }); + } + + // Extract rules from list items (bullets and bold lines) + const listItems = this.extractListItems(section.content); + + for (const item of listItems) { + ruleCounter++; + const rule = this.parseMarkdownRule(item, category, ruleCounter); + if (rule) { + rules.push(rule); + } + } + } + + if (rules.length === 0) { + this.warnings.push('No rules found in style guide'); + } + + return { + name, + version, + description, + rules, + }; + } + + /** + * Detect format from file extension + */ + private detectFormat(filePath: string): StyleGuideFormat { + const ext = path.extname(filePath).toLowerCase(); + + switch (ext) { + case '.md': + case '.markdown': + return StyleGuideFormat.MARKDOWN; + default: + throw new UnsupportedFormatError( + `Cannot detect format from extension: ${ext}`, + ext + ); + } + } + + /** + * Validate parsed style guide against schema + */ + private validate(styleGuide: ParsedStyleGuide): void { + try { + STYLE_GUIDE_SCHEMA.parse(styleGuide); + } catch (error) { + const err = error instanceof Error ? error : new Error(String(error)); + throw new StyleGuideValidationError( + `Style guide validation failed: ${err.message}` + ); + } + } + + /** + * Auto-categorize a rule based on its content + */ + private categorizeRule(rule: StyleGuideRule): StyleGuideRule { + // Check existing category first + if (rule.category && rule.category !== 'general') { + const category = rule.category.toLowerCase(); + + if (category.includes('grammar') || category.includes('mechanics')) { + return { ...rule, category: RuleCategory.GRAMMAR }; + } + if (category.includes('tone') || category.includes('voice')) { + return { ...rule, category: RuleCategory.TONE }; + } + if (category.includes('term')) { + return { ...rule, category: RuleCategory.TERMINOLOGY }; + } + if (category.includes('structure') || category.includes('heading')) { + return { ...rule, category: RuleCategory.STRUCTURE }; + } + + // Keep original if no mapping found + return rule; + } + + const description = rule.description.toLowerCase(); + + // Simple keyword-based categorization + if ( + description.includes('grammar') || + description.includes('spelling') || + description.includes('punctuation') + ) { + return { ...rule, category: RuleCategory.GRAMMAR }; + } + + if ( + description.includes('tone') || + description.includes('voice') || + description.includes('style') + ) { + return { ...rule, category: RuleCategory.TONE }; + } + + if ( + description.includes('term') || + description.includes('word') || + description.includes('phrase') + ) { + return { ...rule, category: RuleCategory.TERMINOLOGY }; + } + + if ( + description.includes('heading') || + description.includes('paragraph') || + description.includes('structure') + ) { + return { ...rule, category: RuleCategory.STRUCTURE }; + } + + return { ...rule, category: RuleCategory.CUSTOM }; + } + + /** + * Extract sections from markdown content + */ + private extractMarkdownSections(content: string): Array<{ level: number; title: string; content: string }> { + const sections: Array<{ level: number; title: string; content: string }> = []; + const lines = content.split(/\r?\n/); + let currentSection: { level: number; title: string; content: string } | null = null; + + for (const line of lines) { + const headingMatch = line.match(/^(#{2,3})\s+(.+)$/); + + if (headingMatch && headingMatch[1] && headingMatch[2]) { + if (currentSection) { + sections.push(currentSection); + } + currentSection = { + level: headingMatch[1].length, + title: headingMatch[2].trim(), + content: line + '\n', + }; + } else if (currentSection) { + currentSection.content += line + '\n'; + } + } + + if (currentSection) { + sections.push(currentSection); + } + + return sections; + } + + /** + * Extract list items from markdown content + */ + private extractListItems(content: string): string[] { + const items: string[] = []; + const lines = content.split(/\r?\n/); + let currentItem = ''; + + for (const line of lines) { + // Match list items starting with - or * + const listMatch = line.match(/^\s*[-*]\s+(.+)$/); + // Match lines starting with bold text: **Rule** + const boldMatch = line.match(/^\s*\*\*(.+?)\*\*(.*)$/); + + if (listMatch) { + if (currentItem) { + items.push(currentItem.trim()); + } + currentItem = listMatch[1]!; + } else if (boldMatch) { + if (currentItem) { + items.push(currentItem.trim()); + } + // For bold rules, we take the whole line usually, or just the bold part? + // User example: "**Write in Second Person...** Address your readers..." + // Let's take the bold part + the rest of the line + currentItem = boldMatch[1]! + boldMatch[2]; + } else if (currentItem && line.trim()) { + // Continuation of previous item + currentItem += ' ' + line.trim(); + } + } + + if (currentItem) { + items.push(currentItem.trim()); + } + + return items; + } + + /** + * Parse a single markdown rule from list item + */ + private parseMarkdownRule( + item: string, + category: string, + index: number + ): StyleGuideRule | null { + if (!item.trim()) return null; + + // Generate ID from content + const id = this.generateRuleId(item, index); + + return { + id, + category, + description: item, + severity: 'warning', // Default severity + }; + } + + /** + * Generate a rule ID from description + */ + private generateRuleId(description: string, index: number): string { + // Create slug from first few words + const words = description + .toLowerCase() + .replace(/[^a-z0-9\s]/g, '') + .split(/\s+/) + .slice(0, 4) + .join('-'); + + return `rule-${words || index}`; + } +} diff --git a/tests/style-guide/fixtures/invalid.txt b/tests/style-guide/fixtures/invalid.txt new file mode 100644 index 0000000..55de3c9 --- /dev/null +++ b/tests/style-guide/fixtures/invalid.txt @@ -0,0 +1,3 @@ +# Invalid File + +This file has an unsupported extension for testing. diff --git a/tests/style-guide/fixtures/sample-style-guide.md b/tests/style-guide/fixtures/sample-style-guide.md new file mode 100644 index 0000000..632751f --- /dev/null +++ b/tests/style-guide/fixtures/sample-style-guide.md @@ -0,0 +1,28 @@ +# Acme Corp Writing Style Guide + +## Voice and Tone + +- Use active voice whenever possible +- Be conversational but professional +- Avoid jargon unless necessary for technical accuracy +- Write in second person ("you") to address readers directly + +## Terminology + +- Use "customer" not "client" +- Use "dashboard" not "control panel" +- Use "sign in" not "log in" +- Always capitalize product names: "Acme Platform", "Acme API" + +## Structure + +- Headings must clearly communicate value to the reader +- Paragraphs should be 3-4 sentences maximum +- Use bullet points for lists of 3 or more items +- Include code examples for technical content + +## Grammar and Mechanics + +- Use Oxford comma in lists +- Spell out numbers one through nine, use numerals for 10 and above +- Use sentence case for headings, not title case diff --git a/tests/style-guide/parser.test.ts b/tests/style-guide/parser.test.ts new file mode 100644 index 0000000..9fb6c78 --- /dev/null +++ b/tests/style-guide/parser.test.ts @@ -0,0 +1,178 @@ +import { describe, it, expect } from 'vitest'; +import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; +import { StyleGuideFormat, RuleCategory } from '../../src/style-guide/types'; +import { + StyleGuideParseError, + UnsupportedFormatError, +} from '../../src/errors/style-guide-errors'; +import * as path from 'path'; + +describe('StyleGuideParser', () => { + const fixturesDir = path.join(__dirname, 'fixtures'); + + describe('Markdown parsing', () => { + it('should parse markdown style guide', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md'), + { format: StyleGuideFormat.MARKDOWN } + ); + + expect(result.data.name).toBe('Acme Corp Writing Style Guide'); + expect(result.data.rules.length).toBeGreaterThan(0); + }); + + it('should parse TinyRocket style guide (headers and bold text)', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'tinyrocket-style-guide.md') + ); + + console.log(`[DEBUG] TinyRocket Rules Found: ${result.data.rules.length}`); + result.data.rules.forEach(r => console.log(`[DEBUG] Rule: ${r.id} - ${r.description.substring(0, 50)}...`)); + + // We expect this to fail initially or find very few rules + expect(result.data.rules.length).toBeGreaterThan(5); + }); + + it('should extract rules from markdown sections', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md') + ); + + const rules = result.data.rules; + expect(rules.length).toBeGreaterThan(10); // Should have multiple rules + + // Check that rules have required fields + rules.forEach((rule) => { + expect(rule.id).toBeDefined(); + expect(rule.category).toBeDefined(); + expect(rule.description).toBeDefined(); + }); + }); + + it('should auto-categorize rules', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md') + ); + + const rules = result.data.rules; + + // Should have tone rules + const toneRules = rules.filter((r) => r.category === RuleCategory.TONE); + expect(toneRules.length).toBeGreaterThan(0); + + // Should have terminology rules + const termRules = rules.filter( + (r) => r.category === RuleCategory.TERMINOLOGY + ); + expect(termRules.length).toBeGreaterThan(0); + + // Should have structure rules + const structureRules = rules.filter( + (r) => r.category === RuleCategory.STRUCTURE + ); + expect(structureRules.length).toBeGreaterThan(0); + }); + }); + + describe('Format detection', () => { + it('should auto-detect markdown format', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md') + ); + + expect(result.data.name).toBeDefined(); + expect(result.data.rules.length).toBeGreaterThan(0); + }); + + it('should throw error for unsupported format', () => { + const parser = new StyleGuideParser(); + + expect(() => { + parser.parse(path.join(fixturesDir, 'invalid.txt')); + }).toThrow(UnsupportedFormatError); + }); + }); + + describe('Error handling', () => { + it('should throw error for non-existent file', () => { + const parser = new StyleGuideParser(); + + expect(() => { + parser.parse(path.join(fixturesDir, 'non-existent.md')); + }).toThrow(StyleGuideParseError); + }); + }); + + describe('Warnings', () => { + it('should collect warnings during parsing', () => { + const parser = new StyleGuideParser(); + const emptyMarkdown = '# Empty Style Guide\n\nNo rules here.'; + + // Write temporary file + const fs = require('fs'); + const tempFile = path.join(fixturesDir, 'empty-warnings.md'); + fs.writeFileSync(tempFile, emptyMarkdown); + + try { + const result = parser.parse(tempFile); + expect(result.warnings).toBeDefined(); + expect(Array.isArray(result.warnings)).toBe(true); + } finally { + if (fs.existsSync(tempFile)) { + fs.unlinkSync(tempFile); + } + } + }); + + it('should warn when no rules found', () => { + const parser = new StyleGuideParser(); + const emptyMarkdown = '# Empty Style Guide\n\nNo rules here.'; + + // Write temporary file + const fs = require('fs'); + const tempFile = path.join(fixturesDir, 'empty-no-rules.md'); + fs.writeFileSync(tempFile, emptyMarkdown); + + try { + const result = parser.parse(tempFile); + expect(result.warnings.some((w) => w.includes('No rules found'))).toBe( + true + ); + } finally { + if (fs.existsSync(tempFile)) { + fs.unlinkSync(tempFile); + } + } + }); + }); + + describe('Rule ID generation', () => { + it('should generate unique IDs for rules', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md') + ); + + const ids = result.data.rules.map((r) => r.id); + const uniqueIds = new Set(ids); + + expect(ids.length).toBe(uniqueIds.size); // All IDs should be unique + }); + + it('should generate readable IDs from descriptions', () => { + const parser = new StyleGuideParser(); + const result = parser.parse( + path.join(fixturesDir, 'sample-style-guide.md') + ); + + result.data.rules.forEach((rule) => { + expect(rule.id).toMatch(/^rule-[a-z0-9-]+$/); + }); + }); + }); +}); From fd02594e9690aa0744ce913daf367e4c699c3a3e Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:46:23 +0100 Subject: [PATCH 05/32] feat: introduce LLM-powered evaluation generation from style guide rules with new generator, schema, and tests --- package-lock.json | 12 +- package.json | 5 +- src/style-guide/eval-generation-schema.ts | 25 ++++ src/style-guide/eval-generator.ts | 169 ++++++++++++++++++++++ tests/style-guide/generator.test.ts | 71 +++++++++ 5 files changed, 279 insertions(+), 3 deletions(-) create mode 100644 src/style-guide/eval-generation-schema.ts create mode 100644 src/style-guide/eval-generator.ts create mode 100644 tests/style-guide/generator.test.ts diff --git a/package-lock.json b/package-lock.json index ed4a266..b3fd1cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,8 @@ "openai": "^4.0.0", "strip-ansi": "^7.1.0", "yaml": "^2.5.0", - "zod": "^3.25.76" + "zod": "^3.25.76", + "zod-to-json-schema": "^3.25.0" }, "bin": { "vectorlint": "dist/index.js" @@ -6294,6 +6295,15 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } } } } diff --git a/package.json b/package.json index b748a57..480443a 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,8 @@ "openai": "^4.0.0", "strip-ansi": "^7.1.0", "yaml": "^2.5.0", - "zod": "^3.25.76" + "zod": "^3.25.76", + "zod-to-json-schema": "^3.25.0" }, "devDependencies": { "@eslint/js": "^9.37.0", @@ -61,4 +62,4 @@ "typescript-eslint": "^8.46.1", "vitest": "^2.0.0" } -} \ No newline at end of file +} diff --git a/src/style-guide/eval-generation-schema.ts b/src/style-guide/eval-generation-schema.ts new file mode 100644 index 0000000..94edc29 --- /dev/null +++ b/src/style-guide/eval-generation-schema.ts @@ -0,0 +1,25 @@ +import { z } from 'zod'; + +/** + * Schema for the LLM output when generating an evaluation prompt + */ +export const EVAL_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective']), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string(), + id: z.string(), + weight: z.number().positive(), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional(), + })).optional(), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + +export type EvalGenerationOutput = z.infer; diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/eval-generator.ts new file mode 100644 index 0000000..d21d0ec --- /dev/null +++ b/src/style-guide/eval-generator.ts @@ -0,0 +1,169 @@ +import { z } from 'zod'; +import { LLMProvider } from '../providers/llm-provider'; +import { StyleGuideRule, ParsedStyleGuide } from '../schemas/style-guide-schemas'; +import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from './eval-generation-schema'; +import { EvalGenerationError } from '../errors/style-guide-errors'; +import { zodToJsonSchema } from 'zod-to-json-schema'; + +export interface EvalGenerationOptions { + llmProvider: LLMProvider; + templateDir?: string; + defaultSeverity?: 'error' | 'warning'; + strictness?: 'lenient' | 'standard' | 'strict'; +} + +export interface GeneratedEval { + filename: string; + content: string; + meta: { + id: string; + name: string; + severity: string; + type: string; + }; +} + +export class EvalGenerator { + private llmProvider: LLMProvider; + private options: EvalGenerationOptions; + + constructor(options: EvalGenerationOptions) { + this.llmProvider = options.llmProvider; + this.options = { + defaultSeverity: 'warning', + strictness: 'standard', + ...options, + }; + } + + /** + * Generate evaluations from a parsed style guide + */ + public async generateEvalsFromStyleGuide( + styleGuide: ParsedStyleGuide + ): Promise { + const evals: GeneratedEval[] = []; + + for (const rule of styleGuide.rules) { + try { + const generatedEval = await this.generateEval(rule); + evals.push(generatedEval); + } catch (error) { + console.error(`Failed to generate eval for rule ${rule.id}:`, error); + // Continue with other rules + } + } + + return evals; + } + + /** + * Generate a single evaluation from a rule + */ + public async generateEval(rule: StyleGuideRule): Promise { + const prompt = this.buildPrompt(rule); + + try { + const schemaJson = zodToJsonSchema(EVAL_GENERATION_SCHEMA, 'evalGeneration'); + + // The LLMProvider expects { name: string; schema: Record } + // zodToJsonSchema returns a schema object that might have $schema, etc. + // We need to cast or massage it to fit the interface if strict. + // Assuming schema property expects the JSON schema object. + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(rule), // Context + prompt, // System/User prompt + { + name: 'evalGeneration', + schema: schemaJson as Record + } + ); + + return this.formatEval(rule, result); + } catch (error) { + throw new EvalGenerationError( + `LLM generation failed: ${(error as Error).message}`, + rule.id + ); + } + } + + /** + * Build the prompt for the LLM + */ + private buildPrompt(rule: StyleGuideRule): string { + return ` +You are an expert in creating automated content evaluation prompts. +Your task is to convert a style guide rule into a structured evaluation prompt for an LLM. + +Rule ID: ${rule.id} +Category: ${rule.category} +Description: ${rule.description} +Severity: ${rule.severity || this.options.defaultSeverity} +${rule.examples ? `Examples:\nGood: ${rule.examples.good?.join(', ')}\nBad: ${rule.examples.bad?.join(', ')}` : ''} + +Strictness Level: ${this.options.strictness} + +Instructions: +1. Analyze the rule to determine if it requires 'subjective' (nuanced, requires judgement) or 'semi-objective' (clear pattern matching but needs context) evaluation. +2. Create a clear, concise prompt body that instructs an LLM how to check for this rule. +3. Define criteria with weights. Total weight usually sums to 10 or 100, but for single rule it can be just the weight of that rule (e.g. 1-10). +4. If subjective, create a 1-4 rubric where 4 is perfect adherence and 1 is a severe violation. +5. Use the provided examples to guide the prompt generation. + +Output the result in the specified JSON format. +`; + } + + /** + * Format the LLM output into a Markdown file content + */ + private formatEval(rule: StyleGuideRule, output: EvalGenerationOutput): GeneratedEval { + const severity = rule.severity || this.options.defaultSeverity || 'warning'; + + // YAML Frontmatter + let content = `--- +evaluator: base +type: ${output.evaluationType} +id: ${rule.id} +name: ${rule.id.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase())} +severity: ${severity} +`; + + if (output.criteria && output.criteria.length > 0) { + content += `criteria:\n`; + output.criteria.forEach(c => { + content += ` - name: ${c.name}\n`; + content += ` id: ${c.id}\n`; + content += ` weight: ${c.weight}\n`; + }); + } + + content += `---\n\n`; + content += `${output.promptBody}\n\n`; + + if (output.criteria) { + output.criteria.forEach(c => { + if (c.rubric) { + content += `## Rubric for ${c.name}\n\n`; + c.rubric.forEach(r => { + content += `- **${r.score} (${r.label})**: ${r.description}\n`; + }); + content += `\n`; + } + }); + } + + return { + filename: `${rule.id}.md`, + content, + meta: { + id: rule.id, + name: rule.id, + severity, + type: output.evaluationType, + } + }; + } +} diff --git a/tests/style-guide/generator.test.ts b/tests/style-guide/generator.test.ts new file mode 100644 index 0000000..7c880a7 --- /dev/null +++ b/tests/style-guide/generator.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, vi } from 'vitest'; +import { EvalGenerator } from '../../src/style-guide/eval-generator'; +import { LLMProvider } from '../../src/providers/llm-provider'; +import { StyleGuideRule } from '../../src/schemas/style-guide-schemas'; +import { EvalGenerationOutput } from '../../src/style-guide/eval-generation-schema'; + +// Mock LLM Provider +class MockLLMProvider implements LLMProvider { + async runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record } + ): Promise { + // Return a dummy response matching the schema + const response: EvalGenerationOutput = { + evaluationType: 'subjective', + promptBody: 'Check if the content follows the rule.', + criteria: [ + { + name: 'Adherence', + id: 'adherence', + weight: 10, + rubric: [ + { score: 4, label: 'Excellent', description: 'Perfect adherence' }, + { score: 1, label: 'Poor', description: 'Severe violation' } + ] + } + ] + }; + return response as unknown as T; + } +} + +describe('EvalGenerator', () => { + it('should generate an eval from a rule', async () => { + const mockProvider = new MockLLMProvider(); + const generator = new EvalGenerator({ llmProvider: mockProvider }); + + const rule: StyleGuideRule = { + id: 'test-rule', + category: 'tone', + description: 'Use a friendly tone.', + severity: 'warning' + }; + + const result = await generator.generateEval(rule); + + expect(result).toBeDefined(); + expect(result.filename).toBe('test-rule.md'); + expect(result.content).toContain('evaluator: base'); + expect(result.content).toContain('type: subjective'); + expect(result.content).toContain('id: test-rule'); + expect(result.content).toContain('severity: warning'); + expect(result.content).toContain('Check if the content follows the rule.'); + expect(result.content).toContain('## Rubric for Adherence'); + }); + + it('should handle errors gracefully', async () => { + const mockProvider = new MockLLMProvider(); + vi.spyOn(mockProvider, 'runPromptStructured').mockRejectedValue(new Error('LLM Error')); + + const generator = new EvalGenerator({ llmProvider: mockProvider }); + const rule: StyleGuideRule = { + id: 'test-rule', + category: 'tone', + description: 'Use a friendly tone.' + }; + + await expect(generator.generateEval(rule)).rejects.toThrow('LLM generation failed: LLM Error'); + }); +}); From 7fe7d1e59cf4951a9ecb73e9e4869a3211b96a59 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 4 Dec 2025 09:55:33 +0100 Subject: [PATCH 06/32] feat: Implement LLM-based evaluation generation for style guide rules with Handlebars templates. --- package-lock.json | 65 +++++++++++++++ package.json | 1 + src/style-guide/eval-generator.ts | 3 + src/style-guide/template-renderer.ts | 94 ++++++++++++++++++++++ src/style-guide/templates/base-template.md | 21 +++++ 5 files changed, 184 insertions(+) create mode 100644 src/style-guide/template-renderer.ts create mode 100644 src/style-guide/templates/base-template.md diff --git a/package-lock.json b/package-lock.json index b3fd1cf..6a4c110 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", + "handlebars": "^4.7.8", "micromatch": "^4.0.5", "openai": "^4.0.0", "strip-ansi": "^7.1.0", @@ -3580,6 +3581,36 @@ "dev": true, "license": "MIT" }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/handlebars/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4096,6 +4127,15 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -4186,6 +4226,12 @@ "dev": true, "license": "MIT" }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -5438,6 +5484,19 @@ "dev": true, "license": "MIT" }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -6167,6 +6226,12 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", diff --git a/package.json b/package.json index 480443a..8f11bae 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", + "handlebars": "^4.7.8", "micromatch": "^4.0.5", "openai": "^4.0.0", "strip-ansi": "^7.1.0", diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/eval-generator.ts index d21d0ec..ce54538 100644 --- a/src/style-guide/eval-generator.ts +++ b/src/style-guide/eval-generator.ts @@ -4,6 +4,7 @@ import { StyleGuideRule, ParsedStyleGuide } from '../schemas/style-guide-schemas import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from './eval-generation-schema'; import { EvalGenerationError } from '../errors/style-guide-errors'; import { zodToJsonSchema } from 'zod-to-json-schema'; +import { TemplateRenderer } from './template-renderer'; export interface EvalGenerationOptions { llmProvider: LLMProvider; @@ -26,6 +27,7 @@ export interface GeneratedEval { export class EvalGenerator { private llmProvider: LLMProvider; private options: EvalGenerationOptions; + private renderer: TemplateRenderer; constructor(options: EvalGenerationOptions) { this.llmProvider = options.llmProvider; @@ -34,6 +36,7 @@ export class EvalGenerator { strictness: 'standard', ...options, }; + this.renderer = new TemplateRenderer(options.templateDir); } /** diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts new file mode 100644 index 0000000..b5ab394 --- /dev/null +++ b/src/style-guide/template-renderer.ts @@ -0,0 +1,94 @@ +import Handlebars from 'handlebars'; +import { readFileSync, existsSync } from 'fs'; +import { join } from 'path'; +import { EvalGenerationOutput } from './eval-generation-schema'; +import { StyleGuideRule } from '../schemas/style-guide-schemas'; + +export interface TemplateContext { + EVALUATION_TYPE: string; + RULE_ID: string; + RULE_NAME: string; + SEVERITY: string; + PROMPT_BODY: string; + CRITERIA?: Array<{ + name: string; + id: string; + weight: number; + }> | undefined; + RUBRIC?: string | undefined; + [key: string]: unknown; +} + +export class TemplateRenderer { + private templateDir: string; + + constructor(templateDir?: string) { + this.templateDir = templateDir || join(__dirname, 'templates'); + this.registerHelpers(); + } + + /** + * Register Handlebars helpers + */ + private registerHelpers(): void { + Handlebars.registerHelper('uppercase', (str: string) => str.toUpperCase()); + Handlebars.registerHelper('lowercase', (str: string) => str.toLowerCase()); + } + + /** + * Render a template with the given context + */ + public render(templateName: string, context: TemplateContext): string { + const templatePath = join(this.templateDir, templateName); + + if (!existsSync(templatePath)) { + throw new Error(`Template not found: ${templatePath}`); + } + + const templateContent = readFileSync(templatePath, 'utf-8'); + const template = Handlebars.compile(templateContent); + + return template(context); + } + + /** + * Create context from rule and LLM output + */ + public createContext( + rule: StyleGuideRule, + output: EvalGenerationOutput, + defaultSeverity: string + ): TemplateContext { + const severity = rule.severity || defaultSeverity; + const ruleName = rule.id + .replace(/-/g, ' ') + .replace(/\b\w/g, (l) => l.toUpperCase()); + + let rubricStr = ''; + if (output.criteria) { + output.criteria.forEach(c => { + if (c.rubric) { + rubricStr += `## Rubric for ${c.name}\n\n`; + c.rubric.forEach(r => { + rubricStr += `- **${r.score} (${r.label})**: ${r.description}\n`; + }); + rubricStr += `\n`; + } + }); + } + + return { + EVALUATION_TYPE: output.evaluationType, + RULE_ID: rule.id, + RULE_NAME: ruleName, + SEVERITY: severity, + PROMPT_BODY: output.promptBody, + CRITERIA: output.criteria?.map(c => ({ + name: c.name, + id: c.id, + weight: c.weight + })), + RUBRIC: rubricStr.trim() + }; + } +} diff --git a/src/style-guide/templates/base-template.md b/src/style-guide/templates/base-template.md new file mode 100644 index 0000000..8f50b26 --- /dev/null +++ b/src/style-guide/templates/base-template.md @@ -0,0 +1,21 @@ +--- +evaluator: base +type: {{EVALUATION_TYPE}} +id: {{RULE_ID}} +name: {{RULE_NAME}} +severity: {{SEVERITY}} +{{#if CRITERIA}} +criteria: +{{#each CRITERIA}} + - name: {{name}} + id: {{id}} + weight: {{weight}} +{{/each}} +{{/if}} +--- + +{{PROMPT_BODY}} + +{{#if RUBRIC}} +{{RUBRIC}} +{{/if}} From 553a1ab8068e7dc20523ba1f96cea8782fa08e0e Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:22:33 +0100 Subject: [PATCH 07/32] Delete rule category enum --- src/style-guide/types.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts index 14b8096..65777de 100644 --- a/src/style-guide/types.ts +++ b/src/style-guide/types.ts @@ -3,17 +3,6 @@ export enum StyleGuideFormat { AUTO = 'auto', } -export enum RuleCategory { - GRAMMAR = 'grammar', - TONE = 'tone', - TERMINOLOGY = 'terminology', - STRUCTURE = 'structure', - FORMATTING = 'formatting', - ACCESSIBILITY = 'accessibility', - SEO = 'seo', - CUSTOM = 'custom', -} - export interface ParserOptions { format?: StyleGuideFormat; verbose?: boolean; From 129ea2e6c97df707e66ae5f3a9201e1bd6c16d14 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:25:02 +0100 Subject: [PATCH 08/32] Implement eval generation schema --- src/{style-guide => schemas}/eval-generation-schema.ts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{style-guide => schemas}/eval-generation-schema.ts (100%) diff --git a/src/style-guide/eval-generation-schema.ts b/src/schemas/eval-generation-schema.ts similarity index 100% rename from src/style-guide/eval-generation-schema.ts rename to src/schemas/eval-generation-schema.ts From fdd42fa049031982e2d2260b2fb773b723414c4a Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:26:10 +0100 Subject: [PATCH 09/32] Implement template renderer for rules --- src/style-guide/template-renderer.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts index b5ab394..03141be 100644 --- a/src/style-guide/template-renderer.ts +++ b/src/style-guide/template-renderer.ts @@ -1,7 +1,8 @@ import Handlebars from 'handlebars'; import { readFileSync, existsSync } from 'fs'; -import { join } from 'path'; -import { EvalGenerationOutput } from './eval-generation-schema'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { EvalGenerationOutput } from '../schemas/eval-generation-schema'; import { StyleGuideRule } from '../schemas/style-guide-schemas'; export interface TemplateContext { @@ -23,7 +24,14 @@ export class TemplateRenderer { private templateDir: string; constructor(templateDir?: string) { - this.templateDir = templateDir || join(__dirname, 'templates'); + if (templateDir) { + this.templateDir = templateDir; + } else { + // ESM compatible __dirname + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + this.templateDir = join(__dirname, 'templates'); + } this.registerHelpers(); } From 48684d358a1b0d9b9e3c9f2b6cb11cbc472319ac Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:26:51 +0100 Subject: [PATCH 10/32] Implement category schema --- src/schemas/category-schema.ts | 52 ++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/schemas/category-schema.ts diff --git a/src/schemas/category-schema.ts b/src/schemas/category-schema.ts new file mode 100644 index 0000000..466af22 --- /dev/null +++ b/src/schemas/category-schema.ts @@ -0,0 +1,52 @@ +import { z } from 'zod'; + +/** + * Schema for generating a category-level evaluation prompt + * This handles multiple related rules in a single eval + */ +export const CATEGORY_EVAL_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective', 'objective']), + categoryName: z.string().describe('The category this eval covers'), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string().describe('Criterion name (usually the rule description)'), + id: z.string().describe('Criterion ID (slug-friendly)'), + weight: z.number().positive().describe('Weight of this criterion in overall score'), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional().describe('Rubric for subjective criteria'), + })), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + +export type CategoryEvalGenerationOutput = z.infer; + + + +/** + * Schema for extracting and categorizing rules from a style guide + */ +export const CATEGORY_EXTRACTION_SCHEMA = z.object({ + categories: z.array(z.object({ + name: z.string().describe('Category name (e.g., "Voice & Tone", "Evidence & Citations")'), + id: z.string().describe('Slug-friendly category ID (e.g., "voice-tone")'), + type: z.enum(['subjective', 'semi-objective', 'objective']).describe('Evaluation type for this category'), + description: z.string().describe('Brief description of what this category covers'), + rules: z.array(z.object({ + description: z.string().describe('The rule text from the style guide'), + severity: z.enum(['error', 'warning']).optional().describe('Suggested severity level'), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), + })), + priority: z.number().int().min(1).max(10).describe('Priority level (1=highest, 10=lowest)'), + })), +}); +export type CategoryExtractionOutput = z.infer; + From e13fd83f59d4cc5f5fab5cb113906ae5119d17c7 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:50:52 +0100 Subject: [PATCH 11/32] refactor: Update rule categorization to preserve extracted categories from markdown and remove auto-categorization logic --- src/style-guide/style-guide-parser.ts | 70 +++++---------------------- 1 file changed, 11 insertions(+), 59 deletions(-) diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index 16f54f6..b7123a8 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -13,7 +13,6 @@ import { } from '../errors/style-guide-errors'; import { StyleGuideFormat, - RuleCategory, type ParserOptions, type ParserResult, } from './types'; @@ -29,7 +28,10 @@ export class StyleGuideParser { try { const content = readFileSync(filePath, 'utf-8'); - const format = options.format || this.detectFormat(filePath); + let format = options.format; + if (!format || format === StyleGuideFormat.AUTO) { + format = this.detectFormat(filePath); + } let result: ParsedStyleGuide; @@ -188,66 +190,16 @@ export class StyleGuideParser { } /** - * Auto-categorize a rule based on its content + * Process a rule - category is now determined dynamically by LLM, not preset here + * Just preserve whatever category was extracted from the markdown structure */ private categorizeRule(rule: StyleGuideRule): StyleGuideRule { - // Check existing category first - if (rule.category && rule.category !== 'general') { - const category = rule.category.toLowerCase(); - - if (category.includes('grammar') || category.includes('mechanics')) { - return { ...rule, category: RuleCategory.GRAMMAR }; - } - if (category.includes('tone') || category.includes('voice')) { - return { ...rule, category: RuleCategory.TONE }; - } - if (category.includes('term')) { - return { ...rule, category: RuleCategory.TERMINOLOGY }; - } - if (category.includes('structure') || category.includes('heading')) { - return { ...rule, category: RuleCategory.STRUCTURE }; - } - - // Keep original if no mapping found - return rule; + // Keep the category as-is from the markdown section title + // If no category was set, use 'uncategorized' + if (!rule.category || rule.category.trim() === '') { + return { ...rule, category: 'uncategorized' }; } - - const description = rule.description.toLowerCase(); - - // Simple keyword-based categorization - if ( - description.includes('grammar') || - description.includes('spelling') || - description.includes('punctuation') - ) { - return { ...rule, category: RuleCategory.GRAMMAR }; - } - - if ( - description.includes('tone') || - description.includes('voice') || - description.includes('style') - ) { - return { ...rule, category: RuleCategory.TONE }; - } - - if ( - description.includes('term') || - description.includes('word') || - description.includes('phrase') - ) { - return { ...rule, category: RuleCategory.TERMINOLOGY }; - } - - if ( - description.includes('heading') || - description.includes('paragraph') || - description.includes('structure') - ) { - return { ...rule, category: RuleCategory.STRUCTURE }; - } - - return { ...rule, category: RuleCategory.CUSTOM }; + return rule; } /** From 97c155fc23a069f967bd1dff5b90f615859a1a2b Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:51:32 +0100 Subject: [PATCH 12/32] refactor: Update import paths and enhance progress logging in EvalGenerator --- src/style-guide/eval-generator.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/eval-generator.ts index ce54538..c9ef2c2 100644 --- a/src/style-guide/eval-generator.ts +++ b/src/style-guide/eval-generator.ts @@ -1,16 +1,16 @@ import { z } from 'zod'; import { LLMProvider } from '../providers/llm-provider'; import { StyleGuideRule, ParsedStyleGuide } from '../schemas/style-guide-schemas'; -import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from './eval-generation-schema'; +import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from '../schemas/eval-generation-schema'; import { EvalGenerationError } from '../errors/style-guide-errors'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { TemplateRenderer } from './template-renderer'; export interface EvalGenerationOptions { llmProvider: LLMProvider; - templateDir?: string; - defaultSeverity?: 'error' | 'warning'; - strictness?: 'lenient' | 'standard' | 'strict'; + templateDir?: string | undefined; + defaultSeverity?: 'error' | 'warning' | undefined; + strictness?: 'lenient' | 'standard' | 'strict' | undefined; } export interface GeneratedEval { @@ -47,10 +47,15 @@ export class EvalGenerator { ): Promise { const evals: GeneratedEval[] = []; + let completed = 0; for (const rule of styleGuide.rules) { try { const generatedEval = await this.generateEval(rule); evals.push(generatedEval); + completed++; + if (completed % 5 === 0 || completed === styleGuide.rules.length) { + console.log(`[EvalGenerator] Progress: ${completed}/${styleGuide.rules.length} rules processed`); + } } catch (error) { console.error(`Failed to generate eval for rule ${rule.id}:`, error); // Continue with other rules @@ -67,7 +72,7 @@ export class EvalGenerator { const prompt = this.buildPrompt(rule); try { - const schemaJson = zodToJsonSchema(EVAL_GENERATION_SCHEMA, 'evalGeneration'); + const schemaJson = zodToJsonSchema(EVAL_GENERATION_SCHEMA); // The LLMProvider expects { name: string; schema: Record } // zodToJsonSchema returns a schema object that might have $schema, etc. From 23829e4a93183e224f32f4a75c23697c67ef474b Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:57:22 +0100 Subject: [PATCH 13/32] feat: Implement StyleGuideProcessor for category extraction and evaluation generation --- src/style-guide/style-guide-processor.ts | 371 +++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 src/style-guide/style-guide-processor.ts diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts new file mode 100644 index 0000000..9008322 --- /dev/null +++ b/src/style-guide/style-guide-processor.ts @@ -0,0 +1,371 @@ +import { LLMProvider } from '../providers/llm-provider'; +import { ParsedStyleGuide } from '../schemas/style-guide-schemas'; +import { EvalGenerationError } from '../errors/style-guide-errors'; +import { zodToJsonSchema } from 'zod-to-json-schema'; +import { + CATEGORY_EXTRACTION_SCHEMA, + CategoryExtractionOutput, + CATEGORY_EVAL_GENERATION_SCHEMA, + CategoryEvalGenerationOutput +} from '../schemas/category-schema'; +import { TemplateRenderer } from './template-renderer'; + +export interface StyleGuideProcessorOptions { + llmProvider: LLMProvider; + // Extraction options + maxCategories?: number | undefined; + filterRule?: string | undefined; + // Generation options + templateDir?: string | undefined; + defaultSeverity?: 'error' | 'warning' | undefined; + strictness?: 'lenient' | 'standard' | 'strict' | undefined; + verbose?: boolean | undefined; +} + +export interface GeneratedCategoryEval { + filename: string; + content: string; + meta: { + id: string; + name: string; + categoryType: string; + ruleCount: number; + }; +} + +export class StyleGuideProcessor { + private llmProvider: LLMProvider; + private options: Required> & Omit; + private renderer: TemplateRenderer; + + constructor(options: StyleGuideProcessorOptions) { + this.llmProvider = options.llmProvider; + this.renderer = new TemplateRenderer(options.templateDir); + this.options = { + maxCategories: 10, + verbose: false, + defaultSeverity: 'warning', + strictness: 'standard', + ...options + }; + } + + /** + * Process a style guide: Extract categories and generate evals + */ + public async process(styleGuide: ParsedStyleGuide): Promise { + // 1. Extract Categories (Organizer Role) + const extractionOutput = await this.extractCategories(styleGuide); + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Extracted ${extractionOutput.categories.length} categories`); + } + + // 2. Generate Evals (Author Role) + return this.generateCategoryEvals(extractionOutput); + } + + /** + * Extract and categorize rules from a parsed style guide + */ + private async extractCategories(styleGuide: ParsedStyleGuide): Promise { + // If filterRule is specified, generate ONLY ONE category for that specific rule + if (this.options.filterRule) { + return this.extractSingleRule(styleGuide); + } + + // Otherwise, do full category extraction + return this.extractAllCategories(styleGuide); + } + + private async extractSingleRule(styleGuide: ParsedStyleGuide): Promise { + const filterTerm = this.options.filterRule!.toLowerCase(); + + // Find rules matching the filter + const matchingRules = styleGuide.rules.filter(r => + r.description.toLowerCase().includes(filterTerm) || + r.id.toLowerCase().includes(filterTerm) || + r.category.toLowerCase().includes(filterTerm) + ); + + if (matchingRules.length === 0) { + throw new EvalGenerationError( + `No rules found matching filter: "${this.options.filterRule}"`, + 'category-extraction' + ); + } + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Found ${matchingRules.length} rules matching "${this.options.filterRule}"`); + console.log(`[StyleGuideProcessor] Generating ONE consolidated eval for this rule`); + } + + const prompt = this.buildSingleRulePrompt(styleGuide, matchingRules, filterTerm); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify({ name: styleGuide.name, matchingRules }), + prompt, + { + name: 'singleRuleExtraction', + schema: schemaJson as Record + } + ); + + // Ensure we return exactly ONE category + if (result.categories.length > 1) { + const firstCategory = result.categories[0]; + if (firstCategory) { + result.categories = [firstCategory]; + } + } + + if (result.categories.length === 0) { + throw new EvalGenerationError( + `No category generated for rule: "${this.options.filterRule}"`, + 'category-extraction' + ); + } + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Extracted 1 category: "${result.categories[0]?.name}"`); + } + + return result; + } catch (error) { + if (error instanceof EvalGenerationError) throw error; + throw new EvalGenerationError( + `Single rule extraction failed: ${(error as Error).message}`, + 'category-extraction' + ); + } + } + + private async extractAllCategories(styleGuide: ParsedStyleGuide): Promise { + const prompt = this.buildFullPrompt(styleGuide); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(styleGuide), + prompt, + { + name: 'categoryExtraction', + schema: schemaJson as Record + } + ); + + // Sort categories by priority (1=highest) and limit to maxCategories + const sortedCategories = [...result.categories] + .sort((a, b) => a.priority - b.priority) + .slice(0, this.options.maxCategories); + + const finalResult: CategoryExtractionOutput = { categories: sortedCategories }; + + if (this.options.verbose) { + const totalRules = finalResult.categories.reduce((sum: number, cat) => sum + cat.rules.length, 0); + console.log(`[StyleGuideProcessor] Extracted ${finalResult.categories.length} categories with ${totalRules} total rules`); + finalResult.categories.forEach(cat => { + console.log(` - ${cat.name} (priority: ${cat.priority}, ${cat.type}): ${cat.rules.length} rules`); + }); + } + + return finalResult; + } catch (error) { + if (error instanceof EvalGenerationError) throw error; + throw new EvalGenerationError( + `Category extraction failed: ${(error as Error).message}`, + 'category-extraction' + ); + } + } + + /** + * Generate category-level evals from extracted categories + */ + private async generateCategoryEvals( + categories: CategoryExtractionOutput + ): Promise { + const evals: GeneratedCategoryEval[] = []; + let completed = 0; + + for (const category of categories.categories) { + try { + const generatedEval = await this.generateCategoryEval(category); + evals.push(generatedEval); + completed++; + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Progress: ${completed}/${categories.categories.length} categories processed`); + } + } catch (error) { + console.error(`Failed to generate eval for category ${category.id}:`, error); + // Continue with other categories + } + } + + return evals; + } + + private async generateCategoryEval( + category: CategoryExtractionOutput['categories'][0] + ): Promise { + const prompt = this.buildEvalPrompt(category); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_EVAL_GENERATION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(category), + prompt, + { + name: 'categoryEvalGeneration', + schema: schemaJson as Record + } + ); + + return this.formatCategoryEval(category, result); + } catch (error) { + throw new EvalGenerationError( + `Category eval generation failed: ${(error as Error).message}`, + category.id + ); + } + } + + // --- Prompt Builders --- + + private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, matchingRules: typeof styleGuide.rules, filterTerm: string): string { + return ` +You are an expert in creating content evaluation prompts from style guides. + +The user wants to generate an evaluation for a SPECIFIC rule: "${filterTerm}" + +I found these matching rules in the style guide: +${matchingRules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} + +Your task: +1. Create EXACTLY ONE category that consolidates all matching rules into a single cohesive evaluation +2. Name the category based on what "${filterTerm}" refers to in the style guide +3. Classify it as subjective, semi-objective, or objective based on the rule nature +4. Include ALL matching rules under this single category + +DO NOT create multiple categories. Create exactly ONE category that covers the "${filterTerm}" topic. + +Style Guide Name: ${styleGuide.name} +`; + } + + private buildFullPrompt(styleGuide: ParsedStyleGuide): string { + return ` +You are an expert in analyzing style guides and organizing rules into logical categories. + +Your task is to analyze the provided style guide and DYNAMICALLY identify categories based on the content. +DO NOT use predefined categories. Let the content guide what categories emerge naturally. + +Instructions: +1. Read all the rules in the style guide +2. Identify natural thematic groupings (e.g., if many rules discuss tone, create a "Voice & Tone" category) +3. Create up to ${this.options.maxCategories} categories based on what you find +4. Classify each category as: + - Subjective: Requires judgment (tone, style, clarity) + - Semi-objective: Clear patterns but needs context (citations, evidence) + - Objective: Can be mechanically checked (formatting, word count) +5. Assign priority (1=highest, 10=lowest) based on impact on content quality + +Important: +- Categories should emerge from the ACTUAL content of the style guide +- Do not force rules into predefined buckets +- Each category should have 3-10 related rules +- Preserve original rule text and examples + +Style Guide Name: ${styleGuide.name} +Total Rules: ${styleGuide.rules.length} + +Analyze the style guide and output categories based on what you find. +`; + } + + private buildEvalPrompt(category: CategoryExtractionOutput['categories'][0]): string { + return ` +You are an expert in creating automated content evaluation prompts. + +Your task is to create a comprehensive evaluation prompt that checks ALL rules in the "${category.name}" category. + +Category: ${category.name} +Type: ${category.type} +Description: ${category.description} +Number of Rules: ${category.rules.length} + +Rules to evaluate: +${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} + +Strictness Level: ${this.options.strictness} + +Instructions: +1. Create a single prompt that evaluates ALL rules in this category together +2. Each rule becomes a separate criterion with its own weight +3. The prompt body should instruct the LLM to check all criteria +4. For ${category.type} evaluation, ${category.type === 'subjective' ? 'create 1-4 rubrics for each criterion' : 'provide clear pass/fail guidance'} +5. Total weight across all criteria should sum to 100 +6. Use examples from the rules when available + +Output a structured evaluation prompt that covers the entire category. +`; + } + + // --- Helpers --- + + private formatCategoryEval( + category: CategoryExtractionOutput['categories'][0], + output: CategoryEvalGenerationOutput + ): GeneratedCategoryEval { + // Build YAML frontmatter + let content = `--- +evaluator: base +type: ${output.evaluationType} +id: ${category.id} +name: ${category.name} +severity: ${this.options.defaultSeverity} +`; + + if (output.criteria && output.criteria.length > 0) { + content += `criteria:\n`; + output.criteria.forEach(c => { + content += ` - name: ${c.name}\n`; + content += ` id: ${c.id}\n`; + content += ` weight: ${c.weight}\n`; + }); + } + + content += `---\n\n`; + content += `# ${category.name}\n\n`; + content += `${output.promptBody}\n\n`; + + // Add rubrics if present + if (output.criteria) { + output.criteria.forEach(c => { + if (c.rubric && c.rubric.length > 0) { + content += `## Rubric for ${c.name}\n\n`; + c.rubric.forEach(r => { + content += `- **${r.score} (${r.label})**: ${r.description}\n`; + }); + content += `\n`; + } + }); + } + + return { + filename: `${category.id}.md`, + content, + meta: { + id: category.id, + name: category.name, + categoryType: category.type, + ruleCount: category.rules.length, + } + }; + } +} From a65a22afa77fb227a54371eb21d85b1f30e7ea65 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:59:30 +0100 Subject: [PATCH 14/32] feat: Add convert command for style guide to VectorLint evaluation prompts --- src/boundaries/file-section-resolver.ts | 4 +- src/cli/convert-command.ts | 194 ++++++++++++++++++++++++ src/index.ts | 2 + 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 src/cli/convert-command.ts diff --git a/src/boundaries/file-section-resolver.ts b/src/boundaries/file-section-resolver.ts index 1ec7835..681837c 100644 --- a/src/boundaries/file-section-resolver.ts +++ b/src/boundaries/file-section-resolver.ts @@ -1,4 +1,4 @@ -import { isMatch } from 'micromatch'; +import micromatch from 'micromatch'; import { FilePatternConfig } from './file-section-parser'; export interface FileResolution { @@ -23,7 +23,7 @@ export class FileSectionResolver { let mergedOverrides: Record = {}; for (const section of sections) { - if ((isMatch as (filePath: string, pattern: string) => boolean)(filePath, section.pattern)) { + if (micromatch.isMatch(filePath, section.pattern)) { // Handle RunEvals if (section.runEvals.length === 0) { // Explicit exclusion: clear active packs diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts new file mode 100644 index 0000000..b7ea37c --- /dev/null +++ b/src/cli/convert-command.ts @@ -0,0 +1,194 @@ +import type { Command } from 'commander'; +import { existsSync, mkdirSync, writeFileSync } from 'fs'; +import * as path from 'path'; +import { StyleGuideParser } from '../style-guide/style-guide-parser'; +import { EvalGenerator } from '../style-guide/eval-generator'; +import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; +import { createProvider } from '../providers/provider-factory'; +import { DefaultRequestBuilder } from '../providers/request-builder'; +import { loadDirective } from '../prompts/directive-loader'; +import { parseEnvironment } from '../boundaries/index'; +import { loadConfig } from '../boundaries/config-loader'; +import { handleUnknownError } from '../errors/index'; +import { StyleGuideFormat } from '../style-guide/types'; + +interface ConvertOptions { + output: string; + format: string; + template?: string; + strictness: 'lenient' | 'standard' | 'strict'; + severity: 'error' | 'warning'; + force: boolean; + dryRun: boolean; + verbose: boolean; + groupByCategory: boolean; + maxCategories?: string; + rule?: string; +} + +export function registerConvertCommand(program: Command): void { + program + .command('convert') + .description('Convert a style guide into VectorLint evaluation prompts') + .argument('', 'Path to the style guide file') + .option('-o, --output ', 'Output directory for generated evals (defaults to PromptsPath from config)') + .option('-f, --format ', 'Input format: markdown, auto', 'auto') + .option('-t, --template ', 'Custom template directory') + .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') + .option('--severity ', 'Default severity: error, warning', 'warning') + .option('--group-by-category', 'Group rules by category (recommended, reduces eval count)', true) + .option('--max-categories ', 'Limit to N most important categories (default: 10)', '10') + .option('--rule ', 'Generate only rule matching this name/keyword') + .option('--force', 'Overwrite existing files', false) + .option('--dry-run', 'Preview generated evals without writing files', false) + .option('-v, --verbose', 'Enable verbose logging', false) + .action(async (styleGuidePath: string, options: ConvertOptions) => { + try { + if (!existsSync(styleGuidePath)) { + console.error(`Error: Style guide file not found: ${styleGuidePath}`); + process.exit(1); + } + + // Determine output directory: CLI option > config PromptsPath + let outputDir = options.output; + if (!outputDir) { + try { + const config = loadConfig(); + outputDir = config.promptsPath; + if (options.verbose) { + console.log(`[vectorlint] Using PromptsPath from config: ${outputDir}`); + } + } catch { + console.error('Error: No output directory specified and no vectorlint.ini found.'); + console.error('Please either use -o/--output or create a vectorlint.ini with PromptsPath.'); + process.exit(1); + } + } + + if (options.verbose) { + console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); + console.log(`[vectorlint] Output directory: ${outputDir}`); + } + + // 1. Parse style guide + const parser = new StyleGuideParser(); + const parseOptions = { + format: options.format === 'auto' ? StyleGuideFormat.AUTO : options.format as StyleGuideFormat, + verbose: options.verbose + }; + + const styleGuide = parser.parse(styleGuidePath, parseOptions); + + if (options.verbose) { + console.log(`[vectorlint] Parsed ${styleGuide.data.rules.length} rules from style guide`); + } + + // 2. Initialize LLM provider + // Parse and validate environment variables + let env; + try { + env = parseEnvironment(); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Validating environment variables'); + console.error(`Error: ${err.message}`); + console.error('Please set these in your .env file or environment.'); + process.exit(1); + } + + const directive = loadDirective(); + const provider = createProvider( + env, + { debug: options.verbose }, + new DefaultRequestBuilder(directive) + ); + + // 3. Generate evals + if (options.verbose) { + console.log(`[vectorlint] Generating evals using ${env.LLM_PROVIDER}...`); + if (options.groupByCategory) { + console.log(`[vectorlint] Using category-based generation (max ${options.maxCategories || 10} categories)`); + } + } + + let evals: Array<{ filename: string; content: string }> = []; + + if (options.groupByCategory) { + const processor = new StyleGuideProcessor({ + llmProvider: provider, + maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, + filterRule: options.rule, + templateDir: options.template || undefined, + defaultSeverity: options.severity, + strictness: options.strictness, + verbose: options.verbose, + }); + + const categoryEvals = await processor.process(styleGuide.data); + evals = categoryEvals.map(e => ({ filename: e.filename, content: e.content })); + } else { + // Original rule-by-rule generation + const generator = new EvalGenerator({ + llmProvider: provider, + templateDir: options.template || undefined, + defaultSeverity: options.severity, + strictness: options.strictness, + }); + + const ruleEvals = await generator.generateEvalsFromStyleGuide(styleGuide.data); + evals = ruleEvals.map(e => ({ filename: e.filename, content: e.content })); + } + + if (evals.length === 0) { + console.warn('[vectorlint] No evals were generated. Check your style guide format.'); + process.exit(0); + } + + // 4. Write to files or preview + if (options.dryRun) { + console.log('\n--- DRY RUN PREVIEW ---\n'); + for (const eva of evals) { + console.log(`File: ${eva.filename}`); + console.log('---'); + console.log(eva.content); + console.log('---\n'); + } + console.log(`[vectorlint] Would generate ${evals.length} files in ${outputDir}`); + } else { + if (!existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + + let writtenCount = 0; + let skippedCount = 0; + + for (const eva of evals) { + const filePath = path.join(outputDir, eva.filename); + + if (existsSync(filePath) && !options.force) { + if (options.verbose) { + console.warn(`[vectorlint] Skipping existing file: ${filePath} (use --force to overwrite)`); + } + skippedCount++; + continue; + } + + writeFileSync(filePath, eva.content, 'utf-8'); + writtenCount++; + if (options.verbose) { + console.log(`[vectorlint] Wrote: ${filePath}`); + } + } + + console.log(`\n[vectorlint] Successfully generated ${writtenCount} evaluation files.`); + if (skippedCount > 0) { + console.log(`[vectorlint] Skipped ${skippedCount} existing files.`); + } + } + + } catch (error) { + const err = handleUnknownError(error, 'Converting style guide'); + console.error(`Error: ${err.message}`); + process.exit(1); + } + }); +} diff --git a/src/index.ts b/src/index.ts index d5a817a..a441540 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import * as path from "path"; import { handleUnknownError } from "./errors/index"; import { registerValidateCommand } from "./cli/validate-command"; import { registerMainCommand } from "./cli/commands"; +import { registerConvertCommand } from "./cli/convert-command"; // Import evaluators module to trigger self-registration of all evaluators import "./evaluators/index"; @@ -62,6 +63,7 @@ program // Register commands registerValidateCommand(program); registerMainCommand(program); +registerConvertCommand(program); // Parse command line arguments program.parse(); From a4dd4f7db778592170750fcc73d1fac7b6f508ba Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 09:59:53 +0100 Subject: [PATCH 15/32] test: Add unit and integration tests for convert command and style guide conversion --- tests/cli/convert-command.test.ts | 50 +++++++++++ .../style-guide-conversion.test.ts | 85 +++++++++++++++++++ tests/style-guide/generator.test.ts | 2 +- vectorlint.ini | 20 +++++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/cli/convert-command.test.ts create mode 100644 tests/integration/style-guide-conversion.test.ts create mode 100644 vectorlint.ini diff --git a/tests/cli/convert-command.test.ts b/tests/cli/convert-command.test.ts new file mode 100644 index 0000000..dc649f3 --- /dev/null +++ b/tests/cli/convert-command.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; +import { registerConvertCommand } from '../../src/cli/convert-command'; +import * as fs from 'fs'; +import * as path from 'path'; + +// Mocks +vi.mock('fs'); +vi.mock('../../src/style-guide/style-guide-parser'); +vi.mock('../../src/style-guide/eval-generator'); +vi.mock('../../src/providers/provider-factory'); +vi.mock('../../src/boundaries/index'); +vi.mock('../../src/prompts/directive-loader'); + +describe('convert command', () => { + let program: Command; + + beforeEach(() => { + program = new Command(); + vi.clearAllMocks(); + // Mock fs.existsSync to return true for style guide + vi.mocked(fs.existsSync).mockReturnValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should register convert command', () => { + registerConvertCommand(program); + const command = program.commands.find(c => c.name() === 'convert'); + expect(command).toBeDefined(); + expect(command?.description()).toContain('Convert a style guide'); + }); + + it('should have correct options', () => { + registerConvertCommand(program); + const command = program.commands.find(c => c.name() === 'convert'); + const options = command?.options.map(o => o.name()); + + expect(options).toContain('output'); + expect(options).toContain('format'); + expect(options).toContain('template'); + expect(options).toContain('strictness'); + expect(options).toContain('severity'); + expect(options).toContain('force'); + expect(options).toContain('dry-run'); + expect(options).toContain('verbose'); + }); +}); diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts new file mode 100644 index 0000000..4ac8ea8 --- /dev/null +++ b/tests/integration/style-guide-conversion.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; +import { EvalGenerator } from '../../src/style-guide/eval-generator'; +import { LLMProvider } from '../../src/providers/llm-provider'; +import * as path from 'path'; +import * as fs from 'fs'; + +// Mock LLM Provider +class MockLLMProvider implements LLMProvider { + async runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record } + ): Promise { + // Return a dummy response matching the schema + return { + evaluationType: 'subjective', + promptBody: 'Check if the content follows the rule.', + criteria: [ + { + name: 'Adherence', + id: 'adherence', + weight: 10, + rubric: [ + { score: 4, label: 'Excellent', description: 'Perfect adherence' }, + { score: 1, label: 'Poor', description: 'Severe violation' } + ] + } + ] + } as unknown as T; + } +} + +describe('Style Guide Conversion Integration', () => { + const fixturesDir = path.join(__dirname, '../style-guide/fixtures'); + const outputDir = path.join(__dirname, 'temp-evals'); + + beforeEach(() => { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up output directory + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + }); + + it('should convert a markdown style guide to eval files', async () => { + // 1. Parse Style Guide + const parser = new StyleGuideParser(); + const styleGuidePath = path.join(fixturesDir, 'sample-style-guide.md'); + const styleGuide = parser.parse(styleGuidePath); + + expect(styleGuide.data.rules.length).toBeGreaterThan(0); + + // 2. Generate Evals + const mockProvider = new MockLLMProvider(); + const generator = new EvalGenerator({ + llmProvider: mockProvider, + defaultSeverity: 'warning' + }); + + const evals = await generator.generateEvalsFromStyleGuide(styleGuide.data); + + expect(evals.length).toBe(styleGuide.data.rules.length); + + // 3. Write Files + for (const eva of evals) { + const filePath = path.join(outputDir, eva.filename); + fs.writeFileSync(filePath, eva.content, 'utf-8'); + } + + // 4. Verify Files Exist + const files = fs.readdirSync(outputDir); + expect(files.length).toBe(evals.length); + + // 5. Verify Content + const firstFile = fs.readFileSync(path.join(outputDir, files[0]), 'utf-8'); + expect(firstFile).toContain('evaluator: base'); + expect(firstFile).toContain('type: subjective'); + }); +}); diff --git a/tests/style-guide/generator.test.ts b/tests/style-guide/generator.test.ts index 7c880a7..8b9b20c 100644 --- a/tests/style-guide/generator.test.ts +++ b/tests/style-guide/generator.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi } from 'vitest'; import { EvalGenerator } from '../../src/style-guide/eval-generator'; import { LLMProvider } from '../../src/providers/llm-provider'; import { StyleGuideRule } from '../../src/schemas/style-guide-schemas'; -import { EvalGenerationOutput } from '../../src/style-guide/eval-generation-schema'; +import { EvalGenerationOutput } from '../../src/schemas/eval-generation-schema'; // Mock LLM Provider class MockLLMProvider implements LLMProvider { diff --git a/vectorlint.ini b/vectorlint.ini new file mode 100644 index 0000000..ed28c85 --- /dev/null +++ b/vectorlint.ini @@ -0,0 +1,20 @@ +PromptsPath=.github/evals +ScanPaths=[*.md] +Concurrency=4 +DefaultSeverity=warning + +# Optional: map prompts to files +[Prompts] +paths=["Default:prompts", "Blog:prompts/blog"] + +[Defaults] +include=["**/*.md"] +exclude=["archived/**"] + +[Directory:Blog] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] + +[Prompt:Headline] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] \ No newline at end of file From 1dbe1d9e09205481fd7b126d5e92194ed6d48513 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 10:53:58 +0100 Subject: [PATCH 16/32] refactor: Simplify content generation in EvalGenerator and StyleGuideProcessor using TemplateRenderer --- src/style-guide/eval-generator.ts | 45 +++---------------- src/style-guide/style-guide-parser.ts | 36 +++++++++++++--- src/style-guide/style-guide-processor.ts | 37 ++-------------- src/style-guide/template-renderer.ts | 55 ++++++++++++++++++------ tests/style-guide/parser.test.ts | 8 ++-- 5 files changed, 84 insertions(+), 97 deletions(-) diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/eval-generator.ts index c9ef2c2..ec93cf2 100644 --- a/src/style-guide/eval-generator.ts +++ b/src/style-guide/eval-generator.ts @@ -74,11 +74,6 @@ export class EvalGenerator { try { const schemaJson = zodToJsonSchema(EVAL_GENERATION_SCHEMA); - // The LLMProvider expects { name: string; schema: Record } - // zodToJsonSchema returns a schema object that might have $schema, etc. - // We need to cast or massage it to fit the interface if strict. - // Assuming schema property expects the JSON schema object. - const result = await this.llmProvider.runPromptStructured( JSON.stringify(rule), // Context prompt, // System/User prompt @@ -128,48 +123,20 @@ Output the result in the specified JSON format. * Format the LLM output into a Markdown file content */ private formatEval(rule: StyleGuideRule, output: EvalGenerationOutput): GeneratedEval { - const severity = rule.severity || this.options.defaultSeverity || 'warning'; - - // YAML Frontmatter - let content = `--- -evaluator: base -type: ${output.evaluationType} -id: ${rule.id} -name: ${rule.id.replace(/-/g, ' ').replace(/\b\w/g, l => l.toUpperCase())} -severity: ${severity} -`; - - if (output.criteria && output.criteria.length > 0) { - content += `criteria:\n`; - output.criteria.forEach(c => { - content += ` - name: ${c.name}\n`; - content += ` id: ${c.id}\n`; - content += ` weight: ${c.weight}\n`; - }); - } + const defaultSeverity = this.options.defaultSeverity || 'warning'; - content += `---\n\n`; - content += `${output.promptBody}\n\n`; - - if (output.criteria) { - output.criteria.forEach(c => { - if (c.rubric) { - content += `## Rubric for ${c.name}\n\n`; - c.rubric.forEach(r => { - content += `- **${r.score} (${r.label})**: ${r.description}\n`; - }); - content += `\n`; - } - }); - } + // Use TemplateRenderer to generate content + const context = this.renderer.createContext(rule, output, defaultSeverity); + const content = this.renderer.render('base-template.md', context); + // Extract meta for return value (still needed for CLI/API) return { filename: `${rule.id}.md`, content, meta: { id: rule.id, name: rule.id, - severity, + severity: context.SEVERITY as string, type: output.evaluationType, } }; diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index b7123a8..c017f93 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -1,6 +1,7 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import YAML from 'yaml'; +import { z } from 'zod'; import { STYLE_GUIDE_SCHEMA, type ParsedStyleGuide, @@ -17,6 +18,12 @@ import { type ParserResult, } from './types'; +const STYLE_GUIDE_FRONTMATTER_SCHEMA = z.object({ + name: z.string().optional(), + version: z.string().optional(), + description: z.string().optional(), +}); + /** * Parser for converting style guide documents into structured format */ @@ -90,10 +97,17 @@ export class StyleGuideParser { bodyContent = content.slice(endIndex + 4); try { - const meta = YAML.parse(frontmatter); - if (meta.name) name = meta.name; - if (meta.version) version = meta.version; - if (meta.description) description = meta.description; + const raw: unknown = YAML.parse(frontmatter); + const parsed = STYLE_GUIDE_FRONTMATTER_SCHEMA.safeParse(raw); + + if (parsed.success) { + const meta = parsed.data; + if (meta.name) name = meta.name; + if (meta.version) version = meta.version; + if (meta.description) description = meta.description; + } else { + this.warnings.push('Invalid YAML frontmatter format'); + } } catch (e) { this.warnings.push('Failed to parse YAML frontmatter, using defaults'); } @@ -118,7 +132,8 @@ export class StyleGuideParser { // Try to find the parent H2 for this section if possible, // but for now let's just look for category in the current section title if it's H2 if (section.level === 2) { - category = section.title.replace(/^\d+\.\s*/, '').trim(); // Remove "1. " prefix + const rawCategory = section.title.replace(/^\d+\.\s*/, '').trim(); + category = this.normalizeCategory(rawCategory); } // If section is H3, treat the title itself as a rule @@ -178,6 +193,9 @@ export class StyleGuideParser { /** * Validate parsed style guide against schema */ + /** + * Notify user if validation fails + */ private validate(styleGuide: ParsedStyleGuide): void { try { STYLE_GUIDE_SCHEMA.parse(styleGuide); @@ -190,9 +208,13 @@ export class StyleGuideParser { } /** - * Process a rule - category is now determined dynamically by LLM, not preset here - * Just preserve whatever category was extracted from the markdown structure + * Normalize category name to standard ID format */ + private normalizeCategory(raw: string): string { + return raw.toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/(^-|-$)/g, ''); + } private categorizeRule(rule: StyleGuideRule): StyleGuideRule { // Keep the category as-is from the markdown section title // If no category was set, use 'uncategorized' diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 9008322..c4a7514 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -322,40 +322,11 @@ Output a structured evaluation prompt that covers the entire category. category: CategoryExtractionOutput['categories'][0], output: CategoryEvalGenerationOutput ): GeneratedCategoryEval { - // Build YAML frontmatter - let content = `--- -evaluator: base -type: ${output.evaluationType} -id: ${category.id} -name: ${category.name} -severity: ${this.options.defaultSeverity} -`; - - if (output.criteria && output.criteria.length > 0) { - content += `criteria:\n`; - output.criteria.forEach(c => { - content += ` - name: ${c.name}\n`; - content += ` id: ${c.id}\n`; - content += ` weight: ${c.weight}\n`; - }); - } + const defaultSeverity = this.options.defaultSeverity || 'warning'; - content += `---\n\n`; - content += `# ${category.name}\n\n`; - content += `${output.promptBody}\n\n`; - - // Add rubrics if present - if (output.criteria) { - output.criteria.forEach(c => { - if (c.rubric && c.rubric.length > 0) { - content += `## Rubric for ${c.name}\n\n`; - c.rubric.forEach(r => { - content += `- **${r.score} (${r.label})**: ${r.description}\n`; - }); - content += `\n`; - } - }); - } + // Use TemplateRenderer + const context = this.renderer.createCategoryContext(category, output, defaultSeverity); + const content = this.renderer.render('base-template.md', context); return { filename: `${category.id}.md`, diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts index 03141be..09c1307 100644 --- a/src/style-guide/template-renderer.ts +++ b/src/style-guide/template-renderer.ts @@ -3,6 +3,7 @@ import { readFileSync, existsSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; import { EvalGenerationOutput } from '../schemas/eval-generation-schema'; +import { CategoryEvalGenerationOutput } from '../schemas/category-schema'; import { StyleGuideRule } from '../schemas/style-guide-schemas'; export interface TemplateContext { @@ -72,19 +73,6 @@ export class TemplateRenderer { .replace(/-/g, ' ') .replace(/\b\w/g, (l) => l.toUpperCase()); - let rubricStr = ''; - if (output.criteria) { - output.criteria.forEach(c => { - if (c.rubric) { - rubricStr += `## Rubric for ${c.name}\n\n`; - c.rubric.forEach(r => { - rubricStr += `- **${r.score} (${r.label})**: ${r.description}\n`; - }); - rubricStr += `\n`; - } - }); - } - return { EVALUATION_TYPE: output.evaluationType, RULE_ID: rule.id, @@ -96,7 +84,46 @@ export class TemplateRenderer { id: c.id, weight: c.weight })), - RUBRIC: rubricStr.trim() + RUBRIC: this.buildRubricString(output.criteria) }; } + + /** + * Create context from category and LLM output + */ + public createCategoryContext( + category: { id: string; name: string }, + output: CategoryEvalGenerationOutput, + defaultSeverity: string + ): TemplateContext { + return { + EVALUATION_TYPE: output.evaluationType, + RULE_ID: category.id, + RULE_NAME: category.name, + SEVERITY: defaultSeverity, + PROMPT_BODY: `# ${category.name}\n\n${output.promptBody}`, + CRITERIA: output.criteria?.map(c => ({ + name: c.name, + id: c.id, + weight: c.weight + })), + RUBRIC: this.buildRubricString(output.criteria) + }; + } + + private buildRubricString(criteria?: Array<{ name: string; rubric?: Array<{ score: number; label: string; description: string }> | undefined }>): string { + let rubricStr = ''; + if (criteria) { + criteria.forEach(c => { + if (c.rubric) { + rubricStr += `## Rubric for ${c.name}\n\n`; + c.rubric.forEach(r => { + rubricStr += `- **${r.score} (${r.label})**: ${r.description}\n`; + }); + rubricStr += `\n`; + } + }); + } + return rubricStr.trim(); + } } diff --git a/tests/style-guide/parser.test.ts b/tests/style-guide/parser.test.ts index 9fb6c78..6b61a8b 100644 --- a/tests/style-guide/parser.test.ts +++ b/tests/style-guide/parser.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; -import { StyleGuideFormat, RuleCategory } from '../../src/style-guide/types'; +import { StyleGuideFormat } from '../../src/style-guide/types'; import { StyleGuideParseError, UnsupportedFormatError, @@ -61,18 +61,18 @@ describe('StyleGuideParser', () => { const rules = result.data.rules; // Should have tone rules - const toneRules = rules.filter((r) => r.category === RuleCategory.TONE); + const toneRules = rules.filter((r) => r.category === 'voice-and-tone'); expect(toneRules.length).toBeGreaterThan(0); // Should have terminology rules const termRules = rules.filter( - (r) => r.category === RuleCategory.TERMINOLOGY + (r) => r.category === 'terminology' ); expect(termRules.length).toBeGreaterThan(0); // Should have structure rules const structureRules = rules.filter( - (r) => r.category === RuleCategory.STRUCTURE + (r) => r.category === 'structure' ); expect(structureRules.length).toBeGreaterThan(0); }); From de78faa821e1191bdd875beed080214fa1f46ae4 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 12:52:14 +0100 Subject: [PATCH 17/32] refactor: Update output directory references and enhance category ID formatting in conversion process --- src/cli/convert-command.ts | 6 +++--- src/schemas/category-schema.ts | 4 ++-- src/style-guide/style-guide-processor.ts | 20 ++++++++++++++++---- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index b7ea37c..9c8f43b 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -31,7 +31,7 @@ export function registerConvertCommand(program: Command): void { .command('convert') .description('Convert a style guide into VectorLint evaluation prompts') .argument('', 'Path to the style guide file') - .option('-o, --output ', 'Output directory for generated evals (defaults to PromptsPath from config)') + .option('-o, --output ', 'Output directory for generated evals (defaults to RulesPath from config)') .option('-f, --format ', 'Input format: markdown, auto', 'auto') .option('-t, --template ', 'Custom template directory') .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') @@ -54,9 +54,9 @@ export function registerConvertCommand(program: Command): void { if (!outputDir) { try { const config = loadConfig(); - outputDir = config.promptsPath; + outputDir = config.rulesPath; if (options.verbose) { - console.log(`[vectorlint] Using PromptsPath from config: ${outputDir}`); + console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); } } catch { console.error('Error: No output directory specified and no vectorlint.ini found.'); diff --git a/src/schemas/category-schema.ts b/src/schemas/category-schema.ts index 466af22..09a27ca 100644 --- a/src/schemas/category-schema.ts +++ b/src/schemas/category-schema.ts @@ -10,7 +10,7 @@ export const CATEGORY_EVAL_GENERATION_SCHEMA = z.object({ promptBody: z.string().describe('The main instruction for the LLM evaluator'), criteria: z.array(z.object({ name: z.string().describe('Criterion name (usually the rule description)'), - id: z.string().describe('Criterion ID (slug-friendly)'), + id: z.string().describe('PascalCase criterion ID (e.g., "VoiceSecondPersonPreferred")'), weight: z.number().positive().describe('Weight of this criterion in overall score'), rubric: z.array(z.object({ score: z.number().int().min(1).max(4), @@ -34,7 +34,7 @@ export type CategoryEvalGenerationOutput = z.infer `${i + 1}. ${r.description}`).join('\n')} Your task: 1. Create EXACTLY ONE category that consolidates all matching rules into a single cohesive evaluation 2. Name the category based on what "${filterTerm}" refers to in the style guide -3. Classify it as subjective, semi-objective, or objective based on the rule nature -4. Include ALL matching rules under this single category +3. Create a PascalCase ID for the category (e.g., "VoiceSecondPersonPreferred") +4. Classify it as subjective, semi-objective, or objective based on the rule nature +5. Include ALL matching rules under this single category DO NOT create multiple categories. Create exactly ONE category that covers the "${filterTerm}" topic. @@ -274,6 +275,7 @@ Instructions: - Semi-objective: Clear patterns but needs context (citations, evidence) - Objective: Can be mechanically checked (formatting, word count) 5. Assign priority (1=highest, 10=lowest) based on impact on content quality +6. Use PascalCase for all category IDs (e.g., "VoiceTone", "EvidenceCredibility") Important: - Categories should emerge from the ACTUAL content of the style guide @@ -307,7 +309,8 @@ Strictness Level: ${this.options.strictness} Instructions: 1. Create a single prompt that evaluates ALL rules in this category together 2. Each rule becomes a separate criterion with its own weight -3. The prompt body should instruct the LLM to check all criteria +3. Use PascalCase for all criterion IDs (e.g., "VoiceSecondPersonPreferred") +4. The prompt body should instruct the LLM to check all criteria 4. For ${category.type} evaluation, ${category.type === 'subjective' ? 'create 1-4 rubrics for each criterion' : 'provide clear pass/fail guidance'} 5. Total weight across all criteria should sum to 100 6. Use examples from the rules when available @@ -324,12 +327,21 @@ Output a structured evaluation prompt that covers the entire category. ): GeneratedCategoryEval { const defaultSeverity = this.options.defaultSeverity || 'warning'; + // Helpers for ID formatting + const toKebabCase = (str: string) => str + .replace(/([a-z])([A-Z])/g, '$1-$2') + .replace(/[\s_]+/g, '-') + .toLowerCase(); + // Use TemplateRenderer const context = this.renderer.createCategoryContext(category, output, defaultSeverity); const content = this.renderer.render('base-template.md', context); + // Ensure filename is kebab-case even if ID is PascalCase + const filenameId = toKebabCase(category.id); + return { - filename: `${category.id}.md`, + filename: `${filenameId}.md`, content, meta: { id: category.id, From 5f8684287ddece1466731a308bc450d5593174ef Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 13:22:44 +0100 Subject: [PATCH 18/32] feat: Implement convert options parsing and schema for convert command --- src/boundaries/cli-parser.ts | 15 +++- src/cli/convert-command.ts | 144 ++++++++++++++++++----------------- src/cli/types.ts | 2 + src/schemas/cli-schemas.ts | 16 ++++ 4 files changed, 106 insertions(+), 71 deletions(-) diff --git a/src/boundaries/cli-parser.ts b/src/boundaries/cli-parser.ts index 6091336..2b74235 100644 --- a/src/boundaries/cli-parser.ts +++ b/src/boundaries/cli-parser.ts @@ -1,4 +1,4 @@ -import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions } from '../schemas/cli-schemas'; +import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, CONVERT_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions, type ConvertOptions } from '../schemas/cli-schemas'; import { ValidationError, handleUnknownError } from '../errors/index'; export function parseCliOptions(raw: unknown): CliOptions { @@ -26,3 +26,16 @@ export function parseValidateOptions(raw: unknown): ValidateOptions { throw new ValidationError(`Validate option parsing failed: ${err.message}`); } } + +export function parseConvertOptions(raw: unknown): ConvertOptions { + try { + return CONVERT_OPTIONS_SCHEMA.parse(raw); + } catch (e: unknown) { + if (e instanceof Error && 'issues' in e) { + // Zod error + throw new ValidationError(`Invalid convert options: ${e.message}`); + } + const err = handleUnknownError(e, 'Convert option parsing'); + throw new ValidationError(`Convert option parsing failed: ${err.message}`); + } +} diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index 9c8f43b..b462fd0 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -7,24 +7,11 @@ import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; import { createProvider } from '../providers/provider-factory'; import { DefaultRequestBuilder } from '../providers/request-builder'; import { loadDirective } from '../prompts/directive-loader'; -import { parseEnvironment } from '../boundaries/index'; +import { parseEnvironment, parseConvertOptions } from '../boundaries/index'; import { loadConfig } from '../boundaries/config-loader'; import { handleUnknownError } from '../errors/index'; import { StyleGuideFormat } from '../style-guide/types'; - -interface ConvertOptions { - output: string; - format: string; - template?: string; - strictness: 'lenient' | 'standard' | 'strict'; - severity: 'error' | 'warning'; - force: boolean; - dryRun: boolean; - verbose: boolean; - groupByCategory: boolean; - maxCategories?: string; - rule?: string; -} +import { ConvertOptions } from './types'; export function registerConvertCommand(program: Command): void { program @@ -42,59 +29,68 @@ export function registerConvertCommand(program: Command): void { .option('--force', 'Overwrite existing files', false) .option('--dry-run', 'Preview generated evals without writing files', false) .option('-v, --verbose', 'Enable verbose logging', false) - .action(async (styleGuidePath: string, options: ConvertOptions) => { + .action(async (styleGuidePath: string, rawOptions: unknown) => { + // 1. Parse CLI options + let options: ConvertOptions; try { - if (!existsSync(styleGuidePath)) { - console.error(`Error: Style guide file not found: ${styleGuidePath}`); - process.exit(1); - } - - // Determine output directory: CLI option > config PromptsPath - let outputDir = options.output; - if (!outputDir) { - try { - const config = loadConfig(); - outputDir = config.rulesPath; - if (options.verbose) { - console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); - } - } catch { - console.error('Error: No output directory specified and no vectorlint.ini found.'); - console.error('Please either use -o/--output or create a vectorlint.ini with PromptsPath.'); - process.exit(1); - } - } + options = parseConvertOptions(rawOptions); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Parsing CLI options'); + console.error(`Error: ${err.message}`); + process.exit(1); + } - if (options.verbose) { - console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); - console.log(`[vectorlint] Output directory: ${outputDir}`); - } + // 2. Validate input file + if (!existsSync(styleGuidePath)) { + console.error(`Error: Style guide file not found: ${styleGuidePath}`); + process.exit(1); + } - // 1. Parse style guide - const parser = new StyleGuideParser(); - const parseOptions = { - format: options.format === 'auto' ? StyleGuideFormat.AUTO : options.format as StyleGuideFormat, - verbose: options.verbose - }; + // 3. Load configuration & determine output directory + let config; + let outputDir = options.output; - const styleGuide = parser.parse(styleGuidePath, parseOptions); + try { + // Determine output directory: CLI option > config RulesPath + config = loadConfig(process.cwd()); - if (options.verbose) { - console.log(`[vectorlint] Parsed ${styleGuide.data.rules.length} rules from style guide`); + if (!outputDir) { + outputDir = config.rulesPath; + if (options.verbose) { + console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); + } } - - // 2. Initialize LLM provider - // Parse and validate environment variables - let env; - try { - env = parseEnvironment(); - } catch (e: unknown) { - const err = handleUnknownError(e, 'Validating environment variables'); - console.error(`Error: ${err.message}`); - console.error('Please set these in your .env file or environment.'); + } catch (e: unknown) { + if (!outputDir) { + const err = handleUnknownError(e, 'Loading configuration'); + console.error('Error: No output directory specified and failed to load vectorlint.ini.'); + console.error(`Details: ${err.message}`); + console.error('Please either use -o/--output or create a valid vectorlint.ini.'); process.exit(1); } + const err = handleUnknownError(e, 'Loading configuration'); + console.error(`Error: ${err.message}`); + process.exit(1); + } + + if (options.verbose) { + console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); + console.log(`[vectorlint] Output directory: ${outputDir}`); + } + + // 4. Parse Environment + let env; + try { + env = parseEnvironment(); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Validating environment variables'); + console.error(`Error: ${err.message}`); + console.error('Please set these in your .env file or environment.'); + process.exit(1); + } + // 5. Load Directive & Initialize Provider + try { const directive = loadDirective(); const provider = createProvider( env, @@ -102,14 +98,23 @@ export function registerConvertCommand(program: Command): void { new DefaultRequestBuilder(directive) ); - // 3. Generate evals + // 6. Parse Style Guide if (options.verbose) { + console.log(`[vectorlint] Parsing style guide...`); + } + const parser = new StyleGuideParser(); + const parseOptions = { + format: options.format === 'auto' ? StyleGuideFormat.AUTO : options.format as StyleGuideFormat, + verbose: options.verbose + }; + const styleGuide = parser.parse(styleGuidePath, parseOptions); + + if (options.verbose) { + console.log(`[vectorlint] Parsed ${styleGuide.data.rules.length} rules from style guide`); console.log(`[vectorlint] Generating evals using ${env.LLM_PROVIDER}...`); - if (options.groupByCategory) { - console.log(`[vectorlint] Using category-based generation (max ${options.maxCategories || 10} categories)`); - } } + // 7. Generate Evals let evals: Array<{ filename: string; content: string }> = []; if (options.groupByCategory) { @@ -126,7 +131,6 @@ export function registerConvertCommand(program: Command): void { const categoryEvals = await processor.process(styleGuide.data); evals = categoryEvals.map(e => ({ filename: e.filename, content: e.content })); } else { - // Original rule-by-rule generation const generator = new EvalGenerator({ llmProvider: provider, templateDir: options.template || undefined, @@ -143,7 +147,7 @@ export function registerConvertCommand(program: Command): void { process.exit(0); } - // 4. Write to files or preview + // 8. Write Output if (options.dryRun) { console.log('\n--- DRY RUN PREVIEW ---\n'); for (const eva of evals) { @@ -154,15 +158,15 @@ export function registerConvertCommand(program: Command): void { } console.log(`[vectorlint] Would generate ${evals.length} files in ${outputDir}`); } else { - if (!existsSync(outputDir)) { - mkdirSync(outputDir, { recursive: true }); + if (!existsSync(outputDir!)) { + mkdirSync(outputDir!, { recursive: true }); } let writtenCount = 0; let skippedCount = 0; for (const eva of evals) { - const filePath = path.join(outputDir, eva.filename); + const filePath = path.join(outputDir!, eva.filename); // outputDir is guaranteed string here if (existsSync(filePath) && !options.force) { if (options.verbose) { @@ -185,8 +189,8 @@ export function registerConvertCommand(program: Command): void { } } - } catch (error) { - const err = handleUnknownError(error, 'Converting style guide'); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Converting style guide'); console.error(`Error: ${err.message}`); process.exit(1); } diff --git a/src/cli/types.ts b/src/cli/types.ts index 5f3818b..5035909 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -130,3 +130,5 @@ export interface EvaluateFileParams { export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; } + +export type { ConvertOptions } from '../schemas/cli-schemas'; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 753f7bf..12aea6d 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -16,6 +16,22 @@ export const VALIDATE_OPTIONS_SCHEMA = z.object({ evals: z.string().optional(), }); +// Convert command options schema +export const CONVERT_OPTIONS_SCHEMA = z.object({ + output: z.string().optional(), + format: z.string().default('auto'), + template: z.string().optional(), + strictness: z.enum(['lenient', 'standard', 'strict']).default('standard'), + severity: z.enum(['error', 'warning']).default('warning'), + groupByCategory: z.boolean().default(true), + maxCategories: z.string().optional().default('10'), + rule: z.string().optional(), + force: z.boolean().default(false), + dryRun: z.boolean().default(false), + verbose: z.boolean().default(false), +}); + // Inferred types export type CliOptions = z.infer; export type ValidateOptions = z.infer; +export type ConvertOptions = z.infer; From aac9e0277b002b6bfaf95c546a5cd88152ff1df8 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 14:01:56 +0100 Subject: [PATCH 19/32] refactor: Update error handling and import structure in style guide processing and parsing --- src/cli/convert-command.ts | 2 +- src/cli/types.ts | 2 - src/errors/style-guide-errors.ts | 62 ------------------------ src/style-guide/eval-generator.ts | 7 ++- src/style-guide/style-guide-parser.ts | 30 +++++------- src/style-guide/style-guide-processor.ts | 31 +++++------- 6 files changed, 29 insertions(+), 105 deletions(-) delete mode 100644 src/errors/style-guide-errors.ts diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index b462fd0..1fba130 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -11,7 +11,7 @@ import { parseEnvironment, parseConvertOptions } from '../boundaries/index'; import { loadConfig } from '../boundaries/config-loader'; import { handleUnknownError } from '../errors/index'; import { StyleGuideFormat } from '../style-guide/types'; -import { ConvertOptions } from './types'; +import { ConvertOptions } from '../schemas'; export function registerConvertCommand(program: Command): void { program diff --git a/src/cli/types.ts b/src/cli/types.ts index 5035909..5f3818b 100644 --- a/src/cli/types.ts +++ b/src/cli/types.ts @@ -130,5 +130,3 @@ export interface EvaluateFileParams { export interface EvaluateFileResult extends ErrorTrackingResult { requestFailures: number; } - -export type { ConvertOptions } from '../schemas/cli-schemas'; diff --git a/src/errors/style-guide-errors.ts b/src/errors/style-guide-errors.ts deleted file mode 100644 index d22d26f..0000000 --- a/src/errors/style-guide-errors.ts +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Base error for style guide operations - */ -export class StyleGuideError extends Error { - constructor(message: string) { - super(message); - this.name = 'StyleGuideError'; - } -} - -/** - * Error thrown when parsing style guide fails - */ -export class StyleGuideParseError extends StyleGuideError { - constructor( - message: string, - public filePath?: string, - public line?: number - ) { - super(message); - this.name = 'StyleGuideParseError'; - } -} - -/** - * Error thrown when style guide validation fails - */ -export class StyleGuideValidationError extends StyleGuideError { - constructor( - message: string, - public issues?: string[] - ) { - super(message); - this.name = 'StyleGuideValidationError'; - } -} - -/** - * Error thrown when eval generation fails - */ -export class EvalGenerationError extends StyleGuideError { - constructor( - message: string, - public ruleId?: string - ) { - super(message); - this.name = 'EvalGenerationError'; - } -} - -/** - * Error thrown when unsupported format is encountered - */ -export class UnsupportedFormatError extends StyleGuideError { - constructor( - message: string, - public format?: string - ) { - super(message); - this.name = 'UnsupportedFormatError'; - } -} diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/eval-generator.ts index ec93cf2..d7a85f1 100644 --- a/src/style-guide/eval-generator.ts +++ b/src/style-guide/eval-generator.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { LLMProvider } from '../providers/llm-provider'; import { StyleGuideRule, ParsedStyleGuide } from '../schemas/style-guide-schemas'; import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from '../schemas/eval-generation-schema'; -import { EvalGenerationError } from '../errors/style-guide-errors'; +import { ProcessingError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { TemplateRenderer } from './template-renderer'; @@ -85,9 +85,8 @@ export class EvalGenerator { return this.formatEval(rule, result); } catch (error) { - throw new EvalGenerationError( - `LLM generation failed: ${(error as Error).message}`, - rule.id + throw new ProcessingError( + `LLM generation failed: ${(error as Error).message}` ); } } diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index c017f93..d523f75 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -8,10 +8,10 @@ import { type StyleGuideRule, } from '../schemas/style-guide-schemas'; import { - StyleGuideParseError, - StyleGuideValidationError, - UnsupportedFormatError, -} from '../errors/style-guide-errors'; + ValidationError, + ProcessingError, + ConfigError, +} from '../errors/index'; import { StyleGuideFormat, type ParserOptions, @@ -47,9 +47,8 @@ export class StyleGuideParser { result = this.parseMarkdown(content); break; default: - throw new UnsupportedFormatError( - `Unsupported format: ${format}`, - format + throw new ConfigError( + `Unsupported format: ${format}` ); } @@ -68,13 +67,12 @@ export class StyleGuideParser { warnings: this.warnings, }; } catch (error) { - if (error instanceof StyleGuideParseError || error instanceof UnsupportedFormatError) { + if (error instanceof ProcessingError || error instanceof ConfigError || error instanceof ValidationError) { throw error; } const err = error instanceof Error ? error : new Error(String(error)); - throw new StyleGuideParseError( - `Failed to parse style guide: ${err.message}`, - filePath + throw new ProcessingError( + `Failed to parse style guide: ${err.message}` ); } } @@ -183,9 +181,8 @@ export class StyleGuideParser { case '.markdown': return StyleGuideFormat.MARKDOWN; default: - throw new UnsupportedFormatError( - `Cannot detect format from extension: ${ext}`, - ext + throw new ConfigError( + `Cannot detect format from extension: ${ext}` ); } } @@ -193,15 +190,12 @@ export class StyleGuideParser { /** * Validate parsed style guide against schema */ - /** - * Notify user if validation fails - */ private validate(styleGuide: ParsedStyleGuide): void { try { STYLE_GUIDE_SCHEMA.parse(styleGuide); } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); - throw new StyleGuideValidationError( + throw new ValidationError( `Style guide validation failed: ${err.message}` ); } diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 8f23e7b..d62cb06 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -1,6 +1,6 @@ import { LLMProvider } from '../providers/llm-provider'; import { ParsedStyleGuide } from '../schemas/style-guide-schemas'; -import { EvalGenerationError } from '../errors/style-guide-errors'; +import { ProcessingError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { CATEGORY_EXTRACTION_SCHEMA, @@ -89,9 +89,8 @@ export class StyleGuideProcessor { ); if (matchingRules.length === 0) { - throw new EvalGenerationError( - `No rules found matching filter: "${this.options.filterRule}"`, - 'category-extraction' + throw new ProcessingError( + `No rules found matching filter: "${this.options.filterRule}" (context: category-extraction)` ); } @@ -123,9 +122,8 @@ export class StyleGuideProcessor { } if (result.categories.length === 0) { - throw new EvalGenerationError( - `No category generated for rule: "${this.options.filterRule}"`, - 'category-extraction' + throw new ProcessingError( + `No category generated for rule: "${this.options.filterRule}" (context: category-extraction)` ); } @@ -135,10 +133,9 @@ export class StyleGuideProcessor { return result; } catch (error) { - if (error instanceof EvalGenerationError) throw error; - throw new EvalGenerationError( - `Single rule extraction failed: ${(error as Error).message}`, - 'category-extraction' + if (error instanceof ProcessingError) throw error; + throw new ProcessingError( + `Single rule extraction failed: ${(error as Error).message}` ); } } @@ -175,10 +172,9 @@ export class StyleGuideProcessor { return finalResult; } catch (error) { - if (error instanceof EvalGenerationError) throw error; - throw new EvalGenerationError( - `Category extraction failed: ${(error as Error).message}`, - 'category-extraction' + if (error instanceof ProcessingError) throw error; + throw new ProcessingError( + `Category extraction failed: ${(error as Error).message}` ); } } @@ -228,9 +224,8 @@ export class StyleGuideProcessor { return this.formatCategoryEval(category, result); } catch (error) { - throw new EvalGenerationError( - `Category eval generation failed: ${(error as Error).message}`, - category.id + throw new ProcessingError( + `Category eval generation failed for ${category.id}: ${(error as Error).message}` ); } } From 4d1d855085b32d28f08baee1c5b86b3ff5772fe1 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 16:40:13 +0100 Subject: [PATCH 20/32] refactor: Rename evals to rules in conversion and validation commands, update related schemas and tests --- src/cli/convert-command.ts | 40 +++++++------- src/cli/validate-command.ts | 4 +- src/schemas/category-schema.ts | 6 +-- src/schemas/cli-schemas.ts | 4 +- .../{eval-generator.ts => rule-generator.ts} | 34 ++++++------ src/style-guide/style-guide-processor.ts | 52 +++++++++---------- src/style-guide/template-renderer.ts | 4 +- .../style-guide-conversion.test.ts | 22 ++++---- 8 files changed, 83 insertions(+), 83 deletions(-) rename src/style-guide/{eval-generator.ts => rule-generator.ts} (83%) diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index 1fba130..1c21620 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -2,7 +2,7 @@ import type { Command } from 'commander'; import { existsSync, mkdirSync, writeFileSync } from 'fs'; import * as path from 'path'; import { StyleGuideParser } from '../style-guide/style-guide-parser'; -import { EvalGenerator } from '../style-guide/eval-generator'; +import { RuleGenerator } from '../style-guide/rule-generator'; import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; import { createProvider } from '../providers/provider-factory'; import { DefaultRequestBuilder } from '../providers/request-builder'; @@ -18,7 +18,7 @@ export function registerConvertCommand(program: Command): void { .command('convert') .description('Convert a style guide into VectorLint evaluation prompts') .argument('', 'Path to the style guide file') - .option('-o, --output ', 'Output directory for generated evals (defaults to RulesPath from config)') + .option('-o, --output ', 'Output directory for generated rules (defaults to RulesPath from config)') .option('-f, --format ', 'Input format: markdown, auto', 'auto') .option('-t, --template ', 'Custom template directory') .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') @@ -27,7 +27,7 @@ export function registerConvertCommand(program: Command): void { .option('--max-categories ', 'Limit to N most important categories (default: 10)', '10') .option('--rule ', 'Generate only rule matching this name/keyword') .option('--force', 'Overwrite existing files', false) - .option('--dry-run', 'Preview generated evals without writing files', false) + .option('--dry-run', 'Preview generated rules without writing files', false) .option('-v, --verbose', 'Enable verbose logging', false) .action(async (styleGuidePath: string, rawOptions: unknown) => { // 1. Parse CLI options @@ -111,11 +111,11 @@ export function registerConvertCommand(program: Command): void { if (options.verbose) { console.log(`[vectorlint] Parsed ${styleGuide.data.rules.length} rules from style guide`); - console.log(`[vectorlint] Generating evals using ${env.LLM_PROVIDER}...`); + console.log(`[vectorlint] Generating rules using ${env.LLM_PROVIDER}...`); } - // 7. Generate Evals - let evals: Array<{ filename: string; content: string }> = []; + // 7. Generate Rules + let rules: Array<{ filename: string; content: string }> = []; if (options.groupByCategory) { const processor = new StyleGuideProcessor({ @@ -128,35 +128,35 @@ export function registerConvertCommand(program: Command): void { verbose: options.verbose, }); - const categoryEvals = await processor.process(styleGuide.data); - evals = categoryEvals.map(e => ({ filename: e.filename, content: e.content })); + const categoryRules = await processor.process(styleGuide.data); + rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); } else { - const generator = new EvalGenerator({ + const generator = new RuleGenerator({ llmProvider: provider, templateDir: options.template || undefined, defaultSeverity: options.severity, strictness: options.strictness, }); - const ruleEvals = await generator.generateEvalsFromStyleGuide(styleGuide.data); - evals = ruleEvals.map(e => ({ filename: e.filename, content: e.content })); + const ruleRules = await generator.generateRulesFromStyleGuide(styleGuide.data); + rules = ruleRules.map(e => ({ filename: e.filename, content: e.content })); } - if (evals.length === 0) { - console.warn('[vectorlint] No evals were generated. Check your style guide format.'); + if (rules.length === 0) { + console.warn('[vectorlint] No rules were generated. Check your style guide format.'); process.exit(0); } // 8. Write Output if (options.dryRun) { console.log('\n--- DRY RUN PREVIEW ---\n'); - for (const eva of evals) { - console.log(`File: ${eva.filename}`); + for (const rule of rules) { + console.log(`File: ${rule.filename}`); console.log('---'); - console.log(eva.content); + console.log(rule.content); console.log('---\n'); } - console.log(`[vectorlint] Would generate ${evals.length} files in ${outputDir}`); + console.log(`[vectorlint] Would generate ${rules.length} files in ${outputDir}`); } else { if (!existsSync(outputDir!)) { mkdirSync(outputDir!, { recursive: true }); @@ -165,8 +165,8 @@ export function registerConvertCommand(program: Command): void { let writtenCount = 0; let skippedCount = 0; - for (const eva of evals) { - const filePath = path.join(outputDir!, eva.filename); // outputDir is guaranteed string here + for (const rule of rules) { + const filePath = path.join(outputDir!, rule.filename); // outputDir is guaranteed string here if (existsSync(filePath) && !options.force) { if (options.verbose) { @@ -176,7 +176,7 @@ export function registerConvertCommand(program: Command): void { continue; } - writeFileSync(filePath, eva.content, 'utf-8'); + writeFileSync(filePath, rule.content, 'utf-8'); writtenCount++; if (options.verbose) { console.log(`[vectorlint] Wrote: ${filePath}`); diff --git a/src/cli/validate-command.ts b/src/cli/validate-command.ts index 4ce7012..e00d169 100644 --- a/src/cli/validate-command.ts +++ b/src/cli/validate-command.ts @@ -20,7 +20,7 @@ export function registerValidateCommand(program: Command): void { program .command('validate') .description('Validate prompt configuration files') - .option('--evals ', 'override evals directory') + .option('--rules ', 'override rules directory') .action(async (rawOpts: unknown) => { // Parse and validate command options let validateOptions; @@ -33,7 +33,7 @@ export function registerValidateCommand(program: Command): void { } // Determine rules path (from option or config) - let rulesPath = validateOptions.evals; + let rulesPath = validateOptions.rules; if (!rulesPath) { try { rulesPath = loadConfig().rulesPath; diff --git a/src/schemas/category-schema.ts b/src/schemas/category-schema.ts index 09a27ca..4c124eb 100644 --- a/src/schemas/category-schema.ts +++ b/src/schemas/category-schema.ts @@ -2,9 +2,9 @@ import { z } from 'zod'; /** * Schema for generating a category-level evaluation prompt - * This handles multiple related rules in a single eval + * This handles multiple related rules in a single rule */ -export const CATEGORY_EVAL_GENERATION_SCHEMA = z.object({ +export const CATEGORY_RULE_GENERATION_SCHEMA = z.object({ evaluationType: z.enum(['subjective', 'semi-objective', 'objective']), categoryName: z.string().describe('The category this eval covers'), promptBody: z.string().describe('The main instruction for the LLM evaluator'), @@ -24,7 +24,7 @@ export const CATEGORY_EVAL_GENERATION_SCHEMA = z.object({ }).optional(), }); -export type CategoryEvalGenerationOutput = z.infer; +export type CategoryRuleGenerationOutput = z.infer; diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 12aea6d..96ab2b5 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,13 +7,13 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'json', 'vale-json', 'JSON']).default('line'), - evals: z.string().optional(), + rules: z.string().optional(), config: z.string().optional(), }); // Validate command options schema export const VALIDATE_OPTIONS_SCHEMA = z.object({ - evals: z.string().optional(), + rules: z.string().optional(), }); // Convert command options schema diff --git a/src/style-guide/eval-generator.ts b/src/style-guide/rule-generator.ts similarity index 83% rename from src/style-guide/eval-generator.ts rename to src/style-guide/rule-generator.ts index d7a85f1..81b04fc 100644 --- a/src/style-guide/eval-generator.ts +++ b/src/style-guide/rule-generator.ts @@ -6,14 +6,14 @@ import { ProcessingError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { TemplateRenderer } from './template-renderer'; -export interface EvalGenerationOptions { +export interface RuleGenerationOptions { llmProvider: LLMProvider; templateDir?: string | undefined; defaultSeverity?: 'error' | 'warning' | undefined; strictness?: 'lenient' | 'standard' | 'strict' | undefined; } -export interface GeneratedEval { +export interface GeneratedRule { filename: string; content: string; meta: { @@ -24,12 +24,12 @@ export interface GeneratedEval { }; } -export class EvalGenerator { +export class RuleGenerator { private llmProvider: LLMProvider; - private options: EvalGenerationOptions; + private options: RuleGenerationOptions; private renderer: TemplateRenderer; - constructor(options: EvalGenerationOptions) { + constructor(options: RuleGenerationOptions) { this.llmProvider = options.llmProvider; this.options = { defaultSeverity: 'warning', @@ -42,33 +42,33 @@ export class EvalGenerator { /** * Generate evaluations from a parsed style guide */ - public async generateEvalsFromStyleGuide( + public async generateRulesFromStyleGuide( styleGuide: ParsedStyleGuide - ): Promise { - const evals: GeneratedEval[] = []; + ): Promise { + const rules: GeneratedRule[] = []; let completed = 0; for (const rule of styleGuide.rules) { try { - const generatedEval = await this.generateEval(rule); - evals.push(generatedEval); + const generatedRule = await this.generateRule(rule); + rules.push(generatedRule); completed++; if (completed % 5 === 0 || completed === styleGuide.rules.length) { - console.log(`[EvalGenerator] Progress: ${completed}/${styleGuide.rules.length} rules processed`); + console.log(`[RuleGenerator] Progress: ${completed}/${styleGuide.rules.length} rules processed`); } } catch (error) { - console.error(`Failed to generate eval for rule ${rule.id}:`, error); + console.error(`Failed to generate rule ${rule.id}:`, error); // Continue with other rules } } - return evals; + return rules; } /** * Generate a single evaluation from a rule */ - public async generateEval(rule: StyleGuideRule): Promise { + public async generateRule(rule: StyleGuideRule): Promise { const prompt = this.buildPrompt(rule); try { @@ -78,12 +78,12 @@ export class EvalGenerator { JSON.stringify(rule), // Context prompt, // System/User prompt { - name: 'evalGeneration', + name: 'ruleGeneration', schema: schemaJson as Record } ); - return this.formatEval(rule, result); + return this.formatRule(rule, result); } catch (error) { throw new ProcessingError( `LLM generation failed: ${(error as Error).message}` @@ -121,7 +121,7 @@ Output the result in the specified JSON format. /** * Format the LLM output into a Markdown file content */ - private formatEval(rule: StyleGuideRule, output: EvalGenerationOutput): GeneratedEval { + private formatRule(rule: StyleGuideRule, output: EvalGenerationOutput): GeneratedRule { const defaultSeverity = this.options.defaultSeverity || 'warning'; // Use TemplateRenderer to generate content diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index d62cb06..1e6442b 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -5,8 +5,8 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; import { CATEGORY_EXTRACTION_SCHEMA, CategoryExtractionOutput, - CATEGORY_EVAL_GENERATION_SCHEMA, - CategoryEvalGenerationOutput + CATEGORY_RULE_GENERATION_SCHEMA, + CategoryRuleGenerationOutput } from '../schemas/category-schema'; import { TemplateRenderer } from './template-renderer'; @@ -22,7 +22,7 @@ export interface StyleGuideProcessorOptions { verbose?: boolean | undefined; } -export interface GeneratedCategoryEval { +export interface GeneratedCategoryRule { filename: string; content: string; meta: { @@ -51,9 +51,9 @@ export class StyleGuideProcessor { } /** - * Process a style guide: Extract categories and generate evals + * Process a style guide: Extract categories and generate rules */ - public async process(styleGuide: ParsedStyleGuide): Promise { + public async process(styleGuide: ParsedStyleGuide): Promise { // 1. Extract Categories (Organizer Role) const extractionOutput = await this.extractCategories(styleGuide); @@ -61,8 +61,8 @@ export class StyleGuideProcessor { console.log(`[StyleGuideProcessor] Extracted ${extractionOutput.categories.length} categories`); } - // 2. Generate Evals (Author Role) - return this.generateCategoryEvals(extractionOutput); + // 2. Generate Rules (Author Role) + return this.generateCategoryRules(extractionOutput); } /** @@ -180,18 +180,18 @@ export class StyleGuideProcessor { } /** - * Generate category-level evals from extracted categories + * Generate category-level rules from extracted categories */ - private async generateCategoryEvals( + private async generateCategoryRules( categories: CategoryExtractionOutput - ): Promise { - const evals: GeneratedCategoryEval[] = []; + ): Promise { + const rules: GeneratedCategoryRule[] = []; let completed = 0; for (const category of categories.categories) { try { - const generatedEval = await this.generateCategoryEval(category); - evals.push(generatedEval); + const generatedEval = await this.generateCategoryRule(category); + rules.push(generatedEval); completed++; if (this.options.verbose) { console.log(`[StyleGuideProcessor] Progress: ${completed}/${categories.categories.length} categories processed`); @@ -202,30 +202,30 @@ export class StyleGuideProcessor { } } - return evals; + return rules; } - private async generateCategoryEval( + private async generateCategoryRule( category: CategoryExtractionOutput['categories'][0] - ): Promise { - const prompt = this.buildEvalPrompt(category); + ): Promise { + const prompt = this.buildRulePrompt(category); try { - const schemaJson = zodToJsonSchema(CATEGORY_EVAL_GENERATION_SCHEMA); + const schemaJson = zodToJsonSchema(CATEGORY_RULE_GENERATION_SCHEMA); - const result = await this.llmProvider.runPromptStructured( + const result = await this.llmProvider.runPromptStructured( JSON.stringify(category), prompt, { - name: 'categoryEvalGeneration', + name: 'categoryRuleGeneration', schema: schemaJson as Record } ); - return this.formatCategoryEval(category, result); + return this.formatCategoryRule(category, result); } catch (error) { throw new ProcessingError( - `Category eval generation failed for ${category.id}: ${(error as Error).message}` + `Category rule generation failed for ${category.id}: ${(error as Error).message}` ); } } @@ -285,7 +285,7 @@ Analyze the style guide and output categories based on what you find. `; } - private buildEvalPrompt(category: CategoryExtractionOutput['categories'][0]): string { + private buildRulePrompt(category: CategoryExtractionOutput['categories'][0]): string { return ` You are an expert in creating automated content evaluation prompts. @@ -316,10 +316,10 @@ Output a structured evaluation prompt that covers the entire category. // --- Helpers --- - private formatCategoryEval( + private formatCategoryRule( category: CategoryExtractionOutput['categories'][0], - output: CategoryEvalGenerationOutput - ): GeneratedCategoryEval { + output: CategoryRuleGenerationOutput + ): GeneratedCategoryRule { const defaultSeverity = this.options.defaultSeverity || 'warning'; // Helpers for ID formatting diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts index 09c1307..b512fd3 100644 --- a/src/style-guide/template-renderer.ts +++ b/src/style-guide/template-renderer.ts @@ -3,7 +3,7 @@ import { readFileSync, existsSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; import { EvalGenerationOutput } from '../schemas/eval-generation-schema'; -import { CategoryEvalGenerationOutput } from '../schemas/category-schema'; +import { CategoryRuleGenerationOutput } from '../schemas/category-schema'; import { StyleGuideRule } from '../schemas/style-guide-schemas'; export interface TemplateContext { @@ -93,7 +93,7 @@ export class TemplateRenderer { */ public createCategoryContext( category: { id: string; name: string }, - output: CategoryEvalGenerationOutput, + output: CategoryRuleGenerationOutput, defaultSeverity: string ): TemplateContext { return { diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts index 4ac8ea8..70c424d 100644 --- a/tests/integration/style-guide-conversion.test.ts +++ b/tests/integration/style-guide-conversion.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; -import { EvalGenerator } from '../../src/style-guide/eval-generator'; +import { RuleGenerator } from '../../src/style-guide/rule-generator'; import { LLMProvider } from '../../src/providers/llm-provider'; import * as path from 'path'; import * as fs from 'fs'; @@ -33,7 +33,7 @@ class MockLLMProvider implements LLMProvider { describe('Style Guide Conversion Integration', () => { const fixturesDir = path.join(__dirname, '../style-guide/fixtures'); - const outputDir = path.join(__dirname, 'temp-evals'); + const outputDir = path.join(__dirname, 'temp-rules'); beforeEach(() => { if (!fs.existsSync(outputDir)) { @@ -48,7 +48,7 @@ describe('Style Guide Conversion Integration', () => { } }); - it('should convert a markdown style guide to eval files', async () => { + it('should convert a markdown style guide to rule files', async () => { // 1. Parse Style Guide const parser = new StyleGuideParser(); const styleGuidePath = path.join(fixturesDir, 'sample-style-guide.md'); @@ -56,26 +56,26 @@ describe('Style Guide Conversion Integration', () => { expect(styleGuide.data.rules.length).toBeGreaterThan(0); - // 2. Generate Evals + // 2. Generate Rules const mockProvider = new MockLLMProvider(); - const generator = new EvalGenerator({ + const generator = new RuleGenerator({ llmProvider: mockProvider, defaultSeverity: 'warning' }); - const evals = await generator.generateEvalsFromStyleGuide(styleGuide.data); + const rules = await generator.generateRulesFromStyleGuide(styleGuide.data); - expect(evals.length).toBe(styleGuide.data.rules.length); + expect(rules.length).toBe(styleGuide.data.rules.length); // 3. Write Files - for (const eva of evals) { - const filePath = path.join(outputDir, eva.filename); - fs.writeFileSync(filePath, eva.content, 'utf-8'); + for (const rule of rules) { + const filePath = path.join(outputDir, rule.filename); + fs.writeFileSync(filePath, rule.content, 'utf-8'); } // 4. Verify Files Exist const files = fs.readdirSync(outputDir); - expect(files.length).toBe(evals.length); + expect(files.length).toBe(rules.length); // 5. Verify Content const firstFile = fs.readFileSync(path.join(outputDir, files[0]), 'utf-8'); From cd503eebfc93b7c5b5b7da5ea3763a253d89547f Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 18:07:58 +0100 Subject: [PATCH 21/32] refactor: Update style guide schemas and processing logic to allow optional categories and improve rule handling --- src/schemas/style-guide-schemas.ts | 2 +- src/style-guide/rule-generator.ts | 21 ++----------- src/style-guide/style-guide-parser.ts | 39 ++---------------------- src/style-guide/style-guide-processor.ts | 2 +- src/style-guide/types.ts | 20 ++++++++++++ 5 files changed, 27 insertions(+), 57 deletions(-) diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts index 898161b..e297d89 100644 --- a/src/schemas/style-guide-schemas.ts +++ b/src/schemas/style-guide-schemas.ts @@ -7,7 +7,7 @@ export const STYLE_GUIDE_EXAMPLES_SCHEMA = z.object({ export const STYLE_GUIDE_RULE_SCHEMA = z.object({ id: z.string(), - category: z.string(), + category: z.string().optional(), description: z.string(), severity: z.enum(['error', 'warning']).optional(), examples: STYLE_GUIDE_EXAMPLES_SCHEMA.optional(), diff --git a/src/style-guide/rule-generator.ts b/src/style-guide/rule-generator.ts index 81b04fc..e5747bd 100644 --- a/src/style-guide/rule-generator.ts +++ b/src/style-guide/rule-generator.ts @@ -5,24 +5,7 @@ import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from '../schemas/eval-ge import { ProcessingError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { TemplateRenderer } from './template-renderer'; - -export interface RuleGenerationOptions { - llmProvider: LLMProvider; - templateDir?: string | undefined; - defaultSeverity?: 'error' | 'warning' | undefined; - strictness?: 'lenient' | 'standard' | 'strict' | undefined; -} - -export interface GeneratedRule { - filename: string; - content: string; - meta: { - id: string; - name: string; - severity: string; - type: string; - }; -} +import { GeneratedRule, RuleGenerationOptions } from './types'; export class RuleGenerator { private llmProvider: LLMProvider; @@ -100,7 +83,7 @@ You are an expert in creating automated content evaluation prompts. Your task is to convert a style guide rule into a structured evaluation prompt for an LLM. Rule ID: ${rule.id} -Category: ${rule.category} +${rule.category ? `Category: ${rule.category}` : ''} Description: ${rule.description} Severity: ${rule.severity || this.options.defaultSeverity} ${rule.examples ? `Examples:\nGood: ${rule.examples.good?.join(', ')}\nBad: ${rule.examples.bad?.join(', ')}` : ''} diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index d523f75..8a7074a 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -54,9 +54,6 @@ export class StyleGuideParser { this.validate(result); - // Auto-categorize rules if not already categorized - result.rules = result.rules.map((rule) => this.categorizeRule(rule)); - if (options.verbose && this.warnings.length > 0) { console.warn('[StyleGuideParser] Warnings:'); this.warnings.forEach((w) => console.warn(` - ${w}`)); @@ -123,25 +120,13 @@ export class StyleGuideParser { let ruleCounter = 0; for (const section of sections) { - // Each H2 or H3 section could be a category - // If it's H2, it's likely a category header. If H3, it might be a rule. - let category = 'general'; - - // Try to find the parent H2 for this section if possible, - // but for now let's just look for category in the current section title if it's H2 - if (section.level === 2) { - const rawCategory = section.title.replace(/^\d+\.\s*/, '').trim(); - category = this.normalizeCategory(rawCategory); - } - // If section is H3, treat the title itself as a rule if (section.level === 3) { ruleCounter++; const ruleId = this.generateRuleId(section.title, ruleCounter); rules.push({ id: ruleId, - category: 'general', // We'd need state to know the parent category, but 'general' is safe for now - description: section.title.replace(/^\*\*|\*\*$/g, '').trim(), // Remove bold markers + description: section.title.replace(/^\*\*|\*\*$/g, '').trim(), severity: 'warning', }); } @@ -151,7 +136,7 @@ export class StyleGuideParser { for (const item of listItems) { ruleCounter++; - const rule = this.parseMarkdownRule(item, category, ruleCounter); + const rule = this.parseMarkdownRule(item, ruleCounter); if (rule) { rules.push(rule); } @@ -201,22 +186,6 @@ export class StyleGuideParser { } } - /** - * Normalize category name to standard ID format - */ - private normalizeCategory(raw: string): string { - return raw.toLowerCase() - .replace(/[^a-z0-9]+/g, '-') - .replace(/(^-|-$)/g, ''); - } - private categorizeRule(rule: StyleGuideRule): StyleGuideRule { - // Keep the category as-is from the markdown section title - // If no category was set, use 'uncategorized' - if (!rule.category || rule.category.trim() === '') { - return { ...rule, category: 'uncategorized' }; - } - return rule; - } /** * Extract sections from markdown content @@ -295,7 +264,6 @@ export class StyleGuideParser { */ private parseMarkdownRule( item: string, - category: string, index: number ): StyleGuideRule | null { if (!item.trim()) return null; @@ -305,9 +273,8 @@ export class StyleGuideParser { return { id, - category, description: item, - severity: 'warning', // Default severity + severity: 'warning', }; } diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 1e6442b..ee4f88a 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -85,7 +85,7 @@ export class StyleGuideProcessor { const matchingRules = styleGuide.rules.filter(r => r.description.toLowerCase().includes(filterTerm) || r.id.toLowerCase().includes(filterTerm) || - r.category.toLowerCase().includes(filterTerm) + r.category?.toLowerCase().includes(filterTerm) ); if (matchingRules.length === 0) { diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts index 65777de..7fa72b2 100644 --- a/src/style-guide/types.ts +++ b/src/style-guide/types.ts @@ -1,3 +1,5 @@ +import { LLMProvider } from "../providers"; + export enum StyleGuideFormat { MARKDOWN = 'markdown', AUTO = 'auto', @@ -13,3 +15,21 @@ export interface ParserResult { data: T; warnings: string[]; } + +export interface RuleGenerationOptions { + llmProvider: LLMProvider; + templateDir?: string | undefined; + defaultSeverity?: 'error' | 'warning' | undefined; + strictness?: 'lenient' | 'standard' | 'strict' | undefined; +} + +export interface GeneratedRule { + filename: string; + content: string; + meta: { + id: string; + name: string; + severity: string; + type: string; + }; +} From 203012de1356ebeac8a3c28b456254dac4121311 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 19:35:49 +0100 Subject: [PATCH 22/32] refactor: Remove RuleGenerator and streamline style guide processing to enhance category extraction and rule generation --- src/cli/convert-command.ts | 40 ++---- src/schemas/cli-schemas.ts | 1 - src/style-guide/rule-generator.ts | 126 ------------------ src/style-guide/style-guide-processor.ts | 60 ++++----- src/style-guide/types.ts | 19 +-- .../style-guide-conversion.test.ts | 51 +++++-- 6 files changed, 79 insertions(+), 218 deletions(-) delete mode 100644 src/style-guide/rule-generator.ts diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index 1c21620..60210fa 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -2,7 +2,6 @@ import type { Command } from 'commander'; import { existsSync, mkdirSync, writeFileSync } from 'fs'; import * as path from 'path'; import { StyleGuideParser } from '../style-guide/style-guide-parser'; -import { RuleGenerator } from '../style-guide/rule-generator'; import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; import { createProvider } from '../providers/provider-factory'; import { DefaultRequestBuilder } from '../providers/request-builder'; @@ -23,7 +22,6 @@ export function registerConvertCommand(program: Command): void { .option('-t, --template ', 'Custom template directory') .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') .option('--severity ', 'Default severity: error, warning', 'warning') - .option('--group-by-category', 'Group rules by category (recommended, reduces eval count)', true) .option('--max-categories ', 'Limit to N most important categories (default: 10)', '10') .option('--rule ', 'Generate only rule matching this name/keyword') .option('--force', 'Overwrite existing files', false) @@ -115,32 +113,18 @@ export function registerConvertCommand(program: Command): void { } // 7. Generate Rules - let rules: Array<{ filename: string; content: string }> = []; - - if (options.groupByCategory) { - const processor = new StyleGuideProcessor({ - llmProvider: provider, - maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, - filterRule: options.rule, - templateDir: options.template || undefined, - defaultSeverity: options.severity, - strictness: options.strictness, - verbose: options.verbose, - }); - - const categoryRules = await processor.process(styleGuide.data); - rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); - } else { - const generator = new RuleGenerator({ - llmProvider: provider, - templateDir: options.template || undefined, - defaultSeverity: options.severity, - strictness: options.strictness, - }); - - const ruleRules = await generator.generateRulesFromStyleGuide(styleGuide.data); - rules = ruleRules.map(e => ({ filename: e.filename, content: e.content })); - } + const processor = new StyleGuideProcessor({ + llmProvider: provider, + maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, + filterRule: options.rule, + templateDir: options.template || undefined, + defaultSeverity: options.severity, + strictness: options.strictness, + verbose: options.verbose, + }); + + const categoryRules = await processor.process(styleGuide.data); + const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); if (rules.length === 0) { console.warn('[vectorlint] No rules were generated. Check your style guide format.'); diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 96ab2b5..8a4cf48 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -23,7 +23,6 @@ export const CONVERT_OPTIONS_SCHEMA = z.object({ template: z.string().optional(), strictness: z.enum(['lenient', 'standard', 'strict']).default('standard'), severity: z.enum(['error', 'warning']).default('warning'), - groupByCategory: z.boolean().default(true), maxCategories: z.string().optional().default('10'), rule: z.string().optional(), force: z.boolean().default(false), diff --git a/src/style-guide/rule-generator.ts b/src/style-guide/rule-generator.ts deleted file mode 100644 index e5747bd..0000000 --- a/src/style-guide/rule-generator.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { z } from 'zod'; -import { LLMProvider } from '../providers/llm-provider'; -import { StyleGuideRule, ParsedStyleGuide } from '../schemas/style-guide-schemas'; -import { EVAL_GENERATION_SCHEMA, EvalGenerationOutput } from '../schemas/eval-generation-schema'; -import { ProcessingError } from '../errors/index'; -import { zodToJsonSchema } from 'zod-to-json-schema'; -import { TemplateRenderer } from './template-renderer'; -import { GeneratedRule, RuleGenerationOptions } from './types'; - -export class RuleGenerator { - private llmProvider: LLMProvider; - private options: RuleGenerationOptions; - private renderer: TemplateRenderer; - - constructor(options: RuleGenerationOptions) { - this.llmProvider = options.llmProvider; - this.options = { - defaultSeverity: 'warning', - strictness: 'standard', - ...options, - }; - this.renderer = new TemplateRenderer(options.templateDir); - } - - /** - * Generate evaluations from a parsed style guide - */ - public async generateRulesFromStyleGuide( - styleGuide: ParsedStyleGuide - ): Promise { - const rules: GeneratedRule[] = []; - - let completed = 0; - for (const rule of styleGuide.rules) { - try { - const generatedRule = await this.generateRule(rule); - rules.push(generatedRule); - completed++; - if (completed % 5 === 0 || completed === styleGuide.rules.length) { - console.log(`[RuleGenerator] Progress: ${completed}/${styleGuide.rules.length} rules processed`); - } - } catch (error) { - console.error(`Failed to generate rule ${rule.id}:`, error); - // Continue with other rules - } - } - - return rules; - } - - /** - * Generate a single evaluation from a rule - */ - public async generateRule(rule: StyleGuideRule): Promise { - const prompt = this.buildPrompt(rule); - - try { - const schemaJson = zodToJsonSchema(EVAL_GENERATION_SCHEMA); - - const result = await this.llmProvider.runPromptStructured( - JSON.stringify(rule), // Context - prompt, // System/User prompt - { - name: 'ruleGeneration', - schema: schemaJson as Record - } - ); - - return this.formatRule(rule, result); - } catch (error) { - throw new ProcessingError( - `LLM generation failed: ${(error as Error).message}` - ); - } - } - - /** - * Build the prompt for the LLM - */ - private buildPrompt(rule: StyleGuideRule): string { - return ` -You are an expert in creating automated content evaluation prompts. -Your task is to convert a style guide rule into a structured evaluation prompt for an LLM. - -Rule ID: ${rule.id} -${rule.category ? `Category: ${rule.category}` : ''} -Description: ${rule.description} -Severity: ${rule.severity || this.options.defaultSeverity} -${rule.examples ? `Examples:\nGood: ${rule.examples.good?.join(', ')}\nBad: ${rule.examples.bad?.join(', ')}` : ''} - -Strictness Level: ${this.options.strictness} - -Instructions: -1. Analyze the rule to determine if it requires 'subjective' (nuanced, requires judgement) or 'semi-objective' (clear pattern matching but needs context) evaluation. -2. Create a clear, concise prompt body that instructs an LLM how to check for this rule. -3. Define criteria with weights. Total weight usually sums to 10 or 100, but for single rule it can be just the weight of that rule (e.g. 1-10). -4. If subjective, create a 1-4 rubric where 4 is perfect adherence and 1 is a severe violation. -5. Use the provided examples to guide the prompt generation. - -Output the result in the specified JSON format. -`; - } - - /** - * Format the LLM output into a Markdown file content - */ - private formatRule(rule: StyleGuideRule, output: EvalGenerationOutput): GeneratedRule { - const defaultSeverity = this.options.defaultSeverity || 'warning'; - - // Use TemplateRenderer to generate content - const context = this.renderer.createContext(rule, output, defaultSeverity); - const content = this.renderer.render('base-template.md', context); - - // Extract meta for return value (still needed for CLI/API) - return { - filename: `${rule.id}.md`, - content, - meta: { - id: rule.id, - name: rule.id, - severity: context.SEVERITY as string, - type: output.evaluationType, - } - }; - } -} diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index ee4f88a..954c41e 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -79,33 +79,21 @@ export class StyleGuideProcessor { } private async extractSingleRule(styleGuide: ParsedStyleGuide): Promise { - const filterTerm = this.options.filterRule!.toLowerCase(); - - // Find rules matching the filter - const matchingRules = styleGuide.rules.filter(r => - r.description.toLowerCase().includes(filterTerm) || - r.id.toLowerCase().includes(filterTerm) || - r.category?.toLowerCase().includes(filterTerm) - ); - - if (matchingRules.length === 0) { - throw new ProcessingError( - `No rules found matching filter: "${this.options.filterRule}" (context: category-extraction)` - ); - } + const filterTerm = this.options.filterRule!; if (this.options.verbose) { - console.log(`[StyleGuideProcessor] Found ${matchingRules.length} rules matching "${this.options.filterRule}"`); - console.log(`[StyleGuideProcessor] Generating ONE consolidated eval for this rule`); + console.log(`[StyleGuideProcessor] Using LLM to find rules related to "${filterTerm}"`); + console.log(`[StyleGuideProcessor] Passing full style guide (${styleGuide.rules.length} rules) for semantic matching`); } - const prompt = this.buildSingleRulePrompt(styleGuide, matchingRules, filterTerm); + // Pass full style guide to LLM - let LLM semantically find matching rules + const prompt = this.buildSingleRulePrompt(styleGuide, filterTerm); try { const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); const result = await this.llmProvider.runPromptStructured( - JSON.stringify({ name: styleGuide.name, matchingRules }), + JSON.stringify(styleGuide), prompt, { name: 'singleRuleExtraction', @@ -123,12 +111,13 @@ export class StyleGuideProcessor { if (result.categories.length === 0) { throw new ProcessingError( - `No category generated for rule: "${this.options.filterRule}" (context: category-extraction)` + `LLM could not find rules related to "${filterTerm}" in the style guide` ); } if (this.options.verbose) { - console.log(`[StyleGuideProcessor] Extracted 1 category: "${result.categories[0]?.name}"`); + const cat = result.categories[0]; + console.log(`[StyleGuideProcessor] LLM extracted category: "${cat?.name}" with ${cat?.rules.length} rules`); } return result; @@ -232,25 +221,30 @@ export class StyleGuideProcessor { // --- Prompt Builders --- - private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, matchingRules: typeof styleGuide.rules, filterTerm: string): string { + private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { return ` -You are an expert in creating content evaluation prompts from style guides. - -The user wants to generate an evaluation for a SPECIFIC rule: "${filterTerm}" +You are an expert in analyzing style guides and creating content evaluation prompts. -I found these matching rules in the style guide: -${matchingRules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} +The user wants to generate an evaluation for a SPECIFIC topic: "${filterTerm}" Your task: -1. Create EXACTLY ONE category that consolidates all matching rules into a single cohesive evaluation -2. Name the category based on what "${filterTerm}" refers to in the style guide -3. Create a PascalCase ID for the category (e.g., "VoiceSecondPersonPreferred") -4. Classify it as subjective, semi-objective, or objective based on the rule nature -5. Include ALL matching rules under this single category - -DO NOT create multiple categories. Create exactly ONE category that covers the "${filterTerm}" topic. +1. Analyze the ENTIRE style guide provided as context +2. Semantically identify ALL rules that relate to "${filterTerm}" (understand synonyms, related concepts, abbreviations like "pov" = "point of view" = "second person") +3. Create EXACTLY ONE category that consolidates all related rules into a single cohesive evaluation +4. Name the category based on the topic (e.g., if "${filterTerm}" is about voice/perspective, name it accordingly) +5. Create a PascalCase ID for the category (e.g., "VoiceSecondPerson", "ToneFormality") +6. Classify it as subjective, semi-objective, or objective based on the rule nature +7. Include ALL semantically matching rules under this single category + +IMPORTANT: +- Use semantic understanding, not just string matching +- "${filterTerm}" may be an abbreviation (pov, cta, seo) - understand what it means +- Look for rules that are RELATED to the topic, even if they don't use the exact term Style Guide Name: ${styleGuide.name} +Total Rules: ${styleGuide.rules.length} + +Analyze the style guide and create ONE category covering the "${filterTerm}" topic. `; } diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts index 7fa72b2..5fbd4fd 100644 --- a/src/style-guide/types.ts +++ b/src/style-guide/types.ts @@ -1,4 +1,4 @@ -import { LLMProvider } from "../providers"; + export enum StyleGuideFormat { MARKDOWN = 'markdown', @@ -16,20 +16,3 @@ export interface ParserResult { warnings: string[]; } -export interface RuleGenerationOptions { - llmProvider: LLMProvider; - templateDir?: string | undefined; - defaultSeverity?: 'error' | 'warning' | undefined; - strictness?: 'lenient' | 'standard' | 'strict' | undefined; -} - -export interface GeneratedRule { - filename: string; - content: string; - meta: { - id: string; - name: string; - severity: string; - type: string; - }; -} diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts index 70c424d..f5f0c2e 100644 --- a/tests/integration/style-guide-conversion.test.ts +++ b/tests/integration/style-guide-conversion.test.ts @@ -1,26 +1,49 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; -import { RuleGenerator } from '../../src/style-guide/rule-generator'; +import { StyleGuideProcessor } from '../../src/style-guide/style-guide-processor'; import { LLMProvider } from '../../src/providers/llm-provider'; import * as path from 'path'; import * as fs from 'fs'; -// Mock LLM Provider +// Mock LLM Provider that handles both category extraction and rule generation class MockLLMProvider implements LLMProvider { + private callCount = 0; + async runPromptStructured( content: string, promptText: string, schema: { name: string; schema: Record } ): Promise { - // Return a dummy response matching the schema + this.callCount++; + + // First call: category extraction + if (schema.name === 'categoryExtraction') { + return { + categories: [ + { + id: 'VoiceTone', + name: 'Voice & Tone', + description: 'Guidelines for voice and tone', + type: 'subjective', + priority: 1, + rules: [ + { id: 'rule-1', description: 'Write in second person' }, + { id: 'rule-2', description: 'Use active voice' } + ] + } + ] + } as unknown as T; + } + + // Second call: rule generation return { evaluationType: 'subjective', promptBody: 'Check if the content follows the rule.', criteria: [ { name: 'Adherence', - id: 'adherence', - weight: 10, + id: 'Adherence', + weight: 100, rubric: [ { score: 4, label: 'Excellent', description: 'Perfect adherence' }, { score: 1, label: 'Poor', description: 'Severe violation' } @@ -48,7 +71,7 @@ describe('Style Guide Conversion Integration', () => { } }); - it('should convert a markdown style guide to rule files', async () => { + it('should convert a markdown style guide to category-based rule files', async () => { // 1. Parse Style Guide const parser = new StyleGuideParser(); const styleGuidePath = path.join(fixturesDir, 'sample-style-guide.md'); @@ -56,16 +79,19 @@ describe('Style Guide Conversion Integration', () => { expect(styleGuide.data.rules.length).toBeGreaterThan(0); - // 2. Generate Rules + // 2. Process Style Guide (extract categories + generate rules) const mockProvider = new MockLLMProvider(); - const generator = new RuleGenerator({ + const processor = new StyleGuideProcessor({ llmProvider: mockProvider, - defaultSeverity: 'warning' + maxCategories: 10, + defaultSeverity: 'warning', + verbose: false, }); - const rules = await generator.generateRulesFromStyleGuide(styleGuide.data); + const rules = await processor.process(styleGuide.data); - expect(rules.length).toBe(styleGuide.data.rules.length); + // Expect at least one category-based rule + expect(rules.length).toBeGreaterThan(0); // 3. Write Files for (const rule of rules) { @@ -78,8 +104,9 @@ describe('Style Guide Conversion Integration', () => { expect(files.length).toBe(rules.length); // 5. Verify Content - const firstFile = fs.readFileSync(path.join(outputDir, files[0]), 'utf-8'); + const firstFile = fs.readFileSync(path.join(outputDir, files[0]!), 'utf-8'); expect(firstFile).toContain('evaluator: base'); expect(firstFile).toContain('type: subjective'); }); }); + From 8320dd5f48976183a3cb26f02cfa1ae0975ddc31 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 20:14:06 +0100 Subject: [PATCH 23/32] refactor: Simplify style guide parser by removing frontmatter handling and updating parseMarkdown method to use filePath for name extraction --- src/schemas/style-guide-schemas.ts | 6 +++ src/style-guide/style-guide-parser.ts | 53 +++------------------------ 2 files changed, 11 insertions(+), 48 deletions(-) diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts index e297d89..349e71d 100644 --- a/src/schemas/style-guide-schemas.ts +++ b/src/schemas/style-guide-schemas.ts @@ -23,6 +23,12 @@ export const STYLE_GUIDE_SCHEMA = z.object({ metadata: z.record(z.unknown()).optional(), }).strict(); +export const STYLE_GUIDE_FRONTMATTER_SCHEMA = z.object({ + name: z.string().optional(), + version: z.string().optional(), + description: z.string().optional(), +}); + export type StyleGuideExamples = z.infer; export type StyleGuideRule = z.infer; export type ParsedStyleGuide = z.infer; diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index 8a7074a..1e0d688 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -1,7 +1,5 @@ import { readFileSync } from 'fs'; import * as path from 'path'; -import YAML from 'yaml'; -import { z } from 'zod'; import { STYLE_GUIDE_SCHEMA, type ParsedStyleGuide, @@ -18,12 +16,6 @@ import { type ParserResult, } from './types'; -const STYLE_GUIDE_FRONTMATTER_SCHEMA = z.object({ - name: z.string().optional(), - version: z.string().optional(), - description: z.string().optional(), -}); - /** * Parser for converting style guide documents into structured format */ @@ -44,7 +36,7 @@ export class StyleGuideParser { switch (format) { case StyleGuideFormat.MARKDOWN: - result = this.parseMarkdown(content); + result = this.parseMarkdown(content, filePath); break; default: throw new ConfigError( @@ -77,46 +69,13 @@ export class StyleGuideParser { /** * Parse Markdown format style guide */ - parseMarkdown(content: string): ParsedStyleGuide { + parseMarkdown(content: string, filePath: string): ParsedStyleGuide { const rules: StyleGuideRule[] = []; - let name = 'Untitled Style Guide'; - let version: string | undefined; - let description: string | undefined; - - // Check for YAML frontmatter - let bodyContent = content; - if (content.startsWith('---')) { - const endIndex = content.indexOf('\n---', 3); - if (endIndex !== -1) { - const frontmatter = content.slice(3, endIndex).trim(); - bodyContent = content.slice(endIndex + 4); - - try { - const raw: unknown = YAML.parse(frontmatter); - const parsed = STYLE_GUIDE_FRONTMATTER_SCHEMA.safeParse(raw); - - if (parsed.success) { - const meta = parsed.data; - if (meta.name) name = meta.name; - if (meta.version) version = meta.version; - if (meta.description) description = meta.description; - } else { - this.warnings.push('Invalid YAML frontmatter format'); - } - } catch (e) { - this.warnings.push('Failed to parse YAML frontmatter, using defaults'); - } - } - } - - // Extract title from first H1 if no name in frontmatter - const h1Match = bodyContent.match(/^#\s+(.+)$/m); - if (h1Match && h1Match[1] && name === 'Untitled Style Guide') { - name = h1Match[1].trim(); - } + // Name from filename (without extension) + const name = path.basename(filePath, path.extname(filePath)); // Parse rules from sections - const sections = this.extractMarkdownSections(bodyContent); + const sections = this.extractMarkdownSections(content); let ruleCounter = 0; for (const section of sections) { @@ -149,8 +108,6 @@ export class StyleGuideParser { return { name, - version, - description, rules, }; } From 54487ae7636006c1509181aa6d520930a063a632 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 20:33:39 +0100 Subject: [PATCH 24/32] refactor: Update style guide schemas and parser to streamline processing and enhance content handling --- src/schemas/style-guide-schemas.ts | 12 +- src/style-guide/style-guide-parser.ts | 179 ++--------------------- src/style-guide/style-guide-processor.ts | 8 +- 3 files changed, 19 insertions(+), 180 deletions(-) diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts index 349e71d..b26646b 100644 --- a/src/schemas/style-guide-schemas.ts +++ b/src/schemas/style-guide-schemas.ts @@ -17,18 +17,10 @@ export const STYLE_GUIDE_RULE_SCHEMA = z.object({ export const STYLE_GUIDE_SCHEMA = z.object({ name: z.string(), - version: z.string().optional(), - description: z.string().optional(), - rules: z.array(STYLE_GUIDE_RULE_SCHEMA), - metadata: z.record(z.unknown()).optional(), + content: z.string(), // Raw markdown content for LLM processing }).strict(); -export const STYLE_GUIDE_FRONTMATTER_SCHEMA = z.object({ - name: z.string().optional(), - version: z.string().optional(), - description: z.string().optional(), -}); - export type StyleGuideExamples = z.infer; export type StyleGuideRule = z.infer; export type ParsedStyleGuide = z.infer; + diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts index 1e0d688..30721bd 100644 --- a/src/style-guide/style-guide-parser.ts +++ b/src/style-guide/style-guide-parser.ts @@ -3,7 +3,6 @@ import * as path from 'path'; import { STYLE_GUIDE_SCHEMA, type ParsedStyleGuide, - type StyleGuideRule, } from '../schemas/style-guide-schemas'; import { ValidationError, @@ -17,7 +16,8 @@ import { } from './types'; /** - * Parser for converting style guide documents into structured format + * Parser for reading style guide documents. + * Returns raw content - LLM handles extraction and categorization. */ export class StyleGuideParser { private warnings: string[] = []; @@ -32,18 +32,19 @@ export class StyleGuideParser { format = this.detectFormat(filePath); } - let result: ParsedStyleGuide; - - switch (format) { - case StyleGuideFormat.MARKDOWN: - result = this.parseMarkdown(content, filePath); - break; - default: - throw new ConfigError( - `Unsupported format: ${format}` - ); + // Validate format is supported + if (format !== StyleGuideFormat.MARKDOWN) { + throw new ConfigError(`Unsupported format: ${format}`); } + // Extract name from filename + const name = path.basename(filePath, path.extname(filePath)); + + const result: ParsedStyleGuide = { + name, + content, + }; + this.validate(result); if (options.verbose && this.warnings.length > 0) { @@ -66,52 +67,6 @@ export class StyleGuideParser { } } - /** - * Parse Markdown format style guide - */ - parseMarkdown(content: string, filePath: string): ParsedStyleGuide { - const rules: StyleGuideRule[] = []; - // Name from filename (without extension) - const name = path.basename(filePath, path.extname(filePath)); - - // Parse rules from sections - const sections = this.extractMarkdownSections(content); - let ruleCounter = 0; - - for (const section of sections) { - // If section is H3, treat the title itself as a rule - if (section.level === 3) { - ruleCounter++; - const ruleId = this.generateRuleId(section.title, ruleCounter); - rules.push({ - id: ruleId, - description: section.title.replace(/^\*\*|\*\*$/g, '').trim(), - severity: 'warning', - }); - } - - // Extract rules from list items (bullets and bold lines) - const listItems = this.extractListItems(section.content); - - for (const item of listItems) { - ruleCounter++; - const rule = this.parseMarkdownRule(item, ruleCounter); - if (rule) { - rules.push(rule); - } - } - } - - if (rules.length === 0) { - this.warnings.push('No rules found in style guide'); - } - - return { - name, - rules, - }; - } - /** * Detect format from file extension */ @@ -142,111 +97,5 @@ export class StyleGuideParser { ); } } - - - /** - * Extract sections from markdown content - */ - private extractMarkdownSections(content: string): Array<{ level: number; title: string; content: string }> { - const sections: Array<{ level: number; title: string; content: string }> = []; - const lines = content.split(/\r?\n/); - let currentSection: { level: number; title: string; content: string } | null = null; - - for (const line of lines) { - const headingMatch = line.match(/^(#{2,3})\s+(.+)$/); - - if (headingMatch && headingMatch[1] && headingMatch[2]) { - if (currentSection) { - sections.push(currentSection); - } - currentSection = { - level: headingMatch[1].length, - title: headingMatch[2].trim(), - content: line + '\n', - }; - } else if (currentSection) { - currentSection.content += line + '\n'; - } - } - - if (currentSection) { - sections.push(currentSection); - } - - return sections; - } - - /** - * Extract list items from markdown content - */ - private extractListItems(content: string): string[] { - const items: string[] = []; - const lines = content.split(/\r?\n/); - let currentItem = ''; - - for (const line of lines) { - // Match list items starting with - or * - const listMatch = line.match(/^\s*[-*]\s+(.+)$/); - // Match lines starting with bold text: **Rule** - const boldMatch = line.match(/^\s*\*\*(.+?)\*\*(.*)$/); - - if (listMatch) { - if (currentItem) { - items.push(currentItem.trim()); - } - currentItem = listMatch[1]!; - } else if (boldMatch) { - if (currentItem) { - items.push(currentItem.trim()); - } - // For bold rules, we take the whole line usually, or just the bold part? - // User example: "**Write in Second Person...** Address your readers..." - // Let's take the bold part + the rest of the line - currentItem = boldMatch[1]! + boldMatch[2]; - } else if (currentItem && line.trim()) { - // Continuation of previous item - currentItem += ' ' + line.trim(); - } - } - - if (currentItem) { - items.push(currentItem.trim()); - } - - return items; - } - - /** - * Parse a single markdown rule from list item - */ - private parseMarkdownRule( - item: string, - index: number - ): StyleGuideRule | null { - if (!item.trim()) return null; - - // Generate ID from content - const id = this.generateRuleId(item, index); - - return { - id, - description: item, - severity: 'warning', - }; - } - - /** - * Generate a rule ID from description - */ - private generateRuleId(description: string, index: number): string { - // Create slug from first few words - const words = description - .toLowerCase() - .replace(/[^a-z0-9\s]/g, '') - .split(/\s+/) - .slice(0, 4) - .join('-'); - - return `rule-${words || index}`; - } } + diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 954c41e..d07e8b2 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -83,7 +83,7 @@ export class StyleGuideProcessor { if (this.options.verbose) { console.log(`[StyleGuideProcessor] Using LLM to find rules related to "${filterTerm}"`); - console.log(`[StyleGuideProcessor] Passing full style guide (${styleGuide.rules.length} rules) for semantic matching`); + console.log(`[StyleGuideProcessor] Passing raw style guide content for semantic matching`); } // Pass full style guide to LLM - let LLM semantically find matching rules @@ -242,9 +242,8 @@ IMPORTANT: - Look for rules that are RELATED to the topic, even if they don't use the exact term Style Guide Name: ${styleGuide.name} -Total Rules: ${styleGuide.rules.length} -Analyze the style guide and create ONE category covering the "${filterTerm}" topic. +Analyze the style guide content and create ONE category covering the "${filterTerm}" topic. `; } @@ -273,9 +272,8 @@ Important: - Preserve original rule text and examples Style Guide Name: ${styleGuide.name} -Total Rules: ${styleGuide.rules.length} -Analyze the style guide and output categories based on what you find. +Analyze the style guide content and output categories based on what you find. `; } From da4b1b4ebdc4c6fdfd403d57cc9f8eca502c7e69 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 20:50:41 +0100 Subject: [PATCH 25/32] refactor: Remove StyleGuideParser and enhance StyleGuideProcessor to streamline style guide processing --- src/cli/convert-command.ts | 21 +---- src/style-guide/style-guide-parser.ts | 101 ----------------------- src/style-guide/style-guide-processor.ts | 50 ++++++++++- 3 files changed, 52 insertions(+), 120 deletions(-) delete mode 100644 src/style-guide/style-guide-parser.ts diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index 60210fa..c83fad9 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -1,7 +1,6 @@ import type { Command } from 'commander'; import { existsSync, mkdirSync, writeFileSync } from 'fs'; import * as path from 'path'; -import { StyleGuideParser } from '../style-guide/style-guide-parser'; import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; import { createProvider } from '../providers/provider-factory'; import { DefaultRequestBuilder } from '../providers/request-builder'; @@ -9,7 +8,6 @@ import { loadDirective } from '../prompts/directive-loader'; import { parseEnvironment, parseConvertOptions } from '../boundaries/index'; import { loadConfig } from '../boundaries/config-loader'; import { handleUnknownError } from '../errors/index'; -import { StyleGuideFormat } from '../style-guide/types'; import { ConvertOptions } from '../schemas'; export function registerConvertCommand(program: Command): void { @@ -96,23 +94,12 @@ export function registerConvertCommand(program: Command): void { new DefaultRequestBuilder(directive) ); - // 6. Parse Style Guide + // 6. Process Style Guide if (options.verbose) { - console.log(`[vectorlint] Parsing style guide...`); + console.log(`[vectorlint] Processing style guide...`); + console.log(`[vectorlint] Using ${env.LLM_PROVIDER}...`); } - const parser = new StyleGuideParser(); - const parseOptions = { - format: options.format === 'auto' ? StyleGuideFormat.AUTO : options.format as StyleGuideFormat, - verbose: options.verbose - }; - const styleGuide = parser.parse(styleGuidePath, parseOptions); - if (options.verbose) { - console.log(`[vectorlint] Parsed ${styleGuide.data.rules.length} rules from style guide`); - console.log(`[vectorlint] Generating rules using ${env.LLM_PROVIDER}...`); - } - - // 7. Generate Rules const processor = new StyleGuideProcessor({ llmProvider: provider, maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, @@ -123,7 +110,7 @@ export function registerConvertCommand(program: Command): void { verbose: options.verbose, }); - const categoryRules = await processor.process(styleGuide.data); + const categoryRules = await processor.processFile(styleGuidePath); const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); if (rules.length === 0) { diff --git a/src/style-guide/style-guide-parser.ts b/src/style-guide/style-guide-parser.ts deleted file mode 100644 index 30721bd..0000000 --- a/src/style-guide/style-guide-parser.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { readFileSync } from 'fs'; -import * as path from 'path'; -import { - STYLE_GUIDE_SCHEMA, - type ParsedStyleGuide, -} from '../schemas/style-guide-schemas'; -import { - ValidationError, - ProcessingError, - ConfigError, -} from '../errors/index'; -import { - StyleGuideFormat, - type ParserOptions, - type ParserResult, -} from './types'; - -/** - * Parser for reading style guide documents. - * Returns raw content - LLM handles extraction and categorization. - */ -export class StyleGuideParser { - private warnings: string[] = []; - - parse(filePath: string, options: ParserOptions = {}): ParserResult { - this.warnings = []; - - try { - const content = readFileSync(filePath, 'utf-8'); - let format = options.format; - if (!format || format === StyleGuideFormat.AUTO) { - format = this.detectFormat(filePath); - } - - // Validate format is supported - if (format !== StyleGuideFormat.MARKDOWN) { - throw new ConfigError(`Unsupported format: ${format}`); - } - - // Extract name from filename - const name = path.basename(filePath, path.extname(filePath)); - - const result: ParsedStyleGuide = { - name, - content, - }; - - this.validate(result); - - if (options.verbose && this.warnings.length > 0) { - console.warn('[StyleGuideParser] Warnings:'); - this.warnings.forEach((w) => console.warn(` - ${w}`)); - } - - return { - data: result, - warnings: this.warnings, - }; - } catch (error) { - if (error instanceof ProcessingError || error instanceof ConfigError || error instanceof ValidationError) { - throw error; - } - const err = error instanceof Error ? error : new Error(String(error)); - throw new ProcessingError( - `Failed to parse style guide: ${err.message}` - ); - } - } - - /** - * Detect format from file extension - */ - private detectFormat(filePath: string): StyleGuideFormat { - const ext = path.extname(filePath).toLowerCase(); - - switch (ext) { - case '.md': - case '.markdown': - return StyleGuideFormat.MARKDOWN; - default: - throw new ConfigError( - `Cannot detect format from extension: ${ext}` - ); - } - } - - /** - * Validate parsed style guide against schema - */ - private validate(styleGuide: ParsedStyleGuide): void { - try { - STYLE_GUIDE_SCHEMA.parse(styleGuide); - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - throw new ValidationError( - `Style guide validation failed: ${err.message}` - ); - } - } -} - diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index d07e8b2..f9cf03b 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -1,6 +1,8 @@ +import { readFileSync } from 'fs'; +import * as path from 'path'; import { LLMProvider } from '../providers/llm-provider'; -import { ParsedStyleGuide } from '../schemas/style-guide-schemas'; -import { ProcessingError } from '../errors/index'; +import { ParsedStyleGuide, STYLE_GUIDE_SCHEMA } from '../schemas/style-guide-schemas'; +import { ProcessingError, ConfigError, ValidationError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { CATEGORY_EXTRACTION_SCHEMA, @@ -50,6 +52,50 @@ export class StyleGuideProcessor { }; } + /** + * Process a style guide file: Read, extract categories, and generate rules + */ + public async processFile(filePath: string): Promise { + // 1. Read and parse the style guide file + const styleGuide = this.readStyleGuide(filePath); + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Loaded style guide: ${styleGuide.name}`); + } + + // 2. Process the style guide content + return this.process(styleGuide); + } + + /** + * Read a style guide file and return parsed content + */ + private readStyleGuide(filePath: string): ParsedStyleGuide { + // Validate file extension + const ext = path.extname(filePath).toLowerCase(); + if (ext !== '.md' && ext !== '.markdown') { + throw new ConfigError(`Unsupported format: ${ext}. Only .md or .markdown files are supported.`); + } + + try { + const content = readFileSync(filePath, 'utf-8'); + const name = path.basename(filePath, path.extname(filePath)); + + const result: ParsedStyleGuide = { name, content }; + + // Validate against schema + STYLE_GUIDE_SCHEMA.parse(result); + + return result; + } catch (error) { + if (error instanceof ConfigError || error instanceof ValidationError) { + throw error; + } + const err = error instanceof Error ? error : new Error(String(error)); + throw new ProcessingError(`Failed to read style guide: ${err.message}`); + } + } + /** * Process a style guide: Extract categories and generate rules */ From 29fde91fc4ffeb80c91a55475f5221e8ffb7d791 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 22:09:42 +0100 Subject: [PATCH 26/32] refactor: Consolidate style guide types and remove unused interfaces to streamline type management --- src/style-guide/style-guide-processor.ts | 148 ++++++++++------------- src/style-guide/types.ts | 31 +++-- 2 files changed, 81 insertions(+), 98 deletions(-) diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index f9cf03b..48708b5 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -11,29 +11,7 @@ import { CategoryRuleGenerationOutput } from '../schemas/category-schema'; import { TemplateRenderer } from './template-renderer'; - -export interface StyleGuideProcessorOptions { - llmProvider: LLMProvider; - // Extraction options - maxCategories?: number | undefined; - filterRule?: string | undefined; - // Generation options - templateDir?: string | undefined; - defaultSeverity?: 'error' | 'warning' | undefined; - strictness?: 'lenient' | 'standard' | 'strict' | undefined; - verbose?: boolean | undefined; -} - -export interface GeneratedCategoryRule { - filename: string; - content: string; - meta: { - id: string; - name: string; - categoryType: string; - ruleCount: number; - }; -} +import { GeneratedCategoryRule, StyleGuideProcessorOptions } from './types'; export class StyleGuideProcessor { private llmProvider: LLMProvider; @@ -269,87 +247,87 @@ export class StyleGuideProcessor { private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { return ` -You are an expert in analyzing style guides and creating content evaluation prompts. + You are an expert in analyzing style guides and creating content evaluation prompts. -The user wants to generate an evaluation for a SPECIFIC topic: "${filterTerm}" + The user wants to generate an evaluation for a SPECIFIC topic: "${filterTerm}" -Your task: -1. Analyze the ENTIRE style guide provided as context -2. Semantically identify ALL rules that relate to "${filterTerm}" (understand synonyms, related concepts, abbreviations like "pov" = "point of view" = "second person") -3. Create EXACTLY ONE category that consolidates all related rules into a single cohesive evaluation -4. Name the category based on the topic (e.g., if "${filterTerm}" is about voice/perspective, name it accordingly) -5. Create a PascalCase ID for the category (e.g., "VoiceSecondPerson", "ToneFormality") -6. Classify it as subjective, semi-objective, or objective based on the rule nature -7. Include ALL semantically matching rules under this single category + Your task: + 1. Analyze the ENTIRE style guide provided as context + 2. Semantically identify ALL rules that relate to "${filterTerm}" (understand synonyms, related concepts, abbreviations like "pov" = "point of view" = "second person") + 3. Create EXACTLY ONE category that consolidates all related rules into a single cohesive evaluation + 4. Name the category based on the topic (e.g., if "${filterTerm}" is about voice/perspective, name it accordingly) + 5. Create a PascalCase ID for the category (e.g., "VoiceSecondPerson", "ToneFormality") + 6. Classify it as subjective, semi-objective, or objective based on the rule nature + 7. Include ALL semantically matching rules under this single category -IMPORTANT: -- Use semantic understanding, not just string matching -- "${filterTerm}" may be an abbreviation (pov, cta, seo) - understand what it means -- Look for rules that are RELATED to the topic, even if they don't use the exact term + IMPORTANT: + - Use semantic understanding, not just string matching + - "${filterTerm}" may be an abbreviation (pov, cta, seo) - understand what it means + - Look for rules that are RELATED to the topic, even if they don't use the exact term -Style Guide Name: ${styleGuide.name} + Style Guide Name: ${styleGuide.name} -Analyze the style guide content and create ONE category covering the "${filterTerm}" topic. -`; + Analyze the style guide content and create ONE category covering the "${filterTerm}" topic. + `; } private buildFullPrompt(styleGuide: ParsedStyleGuide): string { return ` -You are an expert in analyzing style guides and organizing rules into logical categories. - -Your task is to analyze the provided style guide and DYNAMICALLY identify categories based on the content. -DO NOT use predefined categories. Let the content guide what categories emerge naturally. - -Instructions: -1. Read all the rules in the style guide -2. Identify natural thematic groupings (e.g., if many rules discuss tone, create a "Voice & Tone" category) -3. Create up to ${this.options.maxCategories} categories based on what you find -4. Classify each category as: - - Subjective: Requires judgment (tone, style, clarity) - - Semi-objective: Clear patterns but needs context (citations, evidence) - - Objective: Can be mechanically checked (formatting, word count) -5. Assign priority (1=highest, 10=lowest) based on impact on content quality -6. Use PascalCase for all category IDs (e.g., "VoiceTone", "EvidenceCredibility") - -Important: -- Categories should emerge from the ACTUAL content of the style guide -- Do not force rules into predefined buckets -- Each category should have 3-10 related rules -- Preserve original rule text and examples - -Style Guide Name: ${styleGuide.name} - -Analyze the style guide content and output categories based on what you find. -`; + You are an expert in analyzing style guides and organizing rules into logical categories. + + Your task is to analyze the provided style guide and DYNAMICALLY identify categories based on the content. + DO NOT use predefined categories. Let the content guide what categories emerge naturally. + + Instructions: + 1. Read all the rules in the style guide + 2. Identify natural thematic groupings (e.g., if many rules discuss tone, create a "Voice & Tone" category) + 3. Create up to ${this.options.maxCategories} categories based on what you find + 4. Classify each category as: + - Subjective: Requires judgment (tone, style, clarity) + - Semi-objective: Clear patterns but needs context (citations, evidence) + - Objective: Can be mechanically checked (formatting, word count) + 5. Assign priority (1=highest, 10=lowest) based on impact on content quality + 6. Use PascalCase for all category IDs (e.g., "VoiceTone", "EvidenceCredibility") + + Important: + - Categories should emerge from the ACTUAL content of the style guide + - Do not force rules into predefined buckets + - Each category should have 3-10 related rules + - Preserve original rule text and examples + + Style Guide Name: ${styleGuide.name} + + Analyze the style guide content and output categories based on what you find. + `; } private buildRulePrompt(category: CategoryExtractionOutput['categories'][0]): string { return ` -You are an expert in creating automated content evaluation prompts. + You are an expert in creating automated content evaluation prompts. -Your task is to create a comprehensive evaluation prompt that checks ALL rules in the "${category.name}" category. + Your task is to create a comprehensive evaluation prompt that checks ALL rules in the "${category.name}" category. -Category: ${category.name} -Type: ${category.type} -Description: ${category.description} -Number of Rules: ${category.rules.length} + Category: ${category.name} + Type: ${category.type} + Description: ${category.description} + Number of Rules: ${category.rules.length} -Rules to evaluate: -${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} + Rules to evaluate: + ${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} -Strictness Level: ${this.options.strictness} + Strictness Level: ${this.options.strictness} -Instructions: -1. Create a single prompt that evaluates ALL rules in this category together -2. Each rule becomes a separate criterion with its own weight -3. Use PascalCase for all criterion IDs (e.g., "VoiceSecondPersonPreferred") -4. The prompt body should instruct the LLM to check all criteria -4. For ${category.type} evaluation, ${category.type === 'subjective' ? 'create 1-4 rubrics for each criterion' : 'provide clear pass/fail guidance'} -5. Total weight across all criteria should sum to 100 -6. Use examples from the rules when available + Instructions: + 1. Create a single prompt that evaluates ALL rules in this category together + 2. Each rule becomes a separate criterion with its own weight + 3. Use PascalCase for all criterion IDs (e.g., "VoiceSecondPersonPreferred") + 4. The prompt body should instruct the LLM to check all criteria + 4. For ${category.type} evaluation, ${category.type === 'subjective' ? 'create 1-4 rubrics for each criterion' : 'provide clear pass/fail guidance'} + 5. Total weight across all criteria should sum to 100 + 6. Use examples from the rules when available -Output a structured evaluation prompt that covers the entire category. -`; + Output a structured evaluation prompt that covers the entire category. + `; } // --- Helpers --- diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts index 5fbd4fd..015a6d1 100644 --- a/src/style-guide/types.ts +++ b/src/style-guide/types.ts @@ -1,18 +1,23 @@ +import { LLMProvider } from "../providers"; - -export enum StyleGuideFormat { - MARKDOWN = 'markdown', - AUTO = 'auto', -} - -export interface ParserOptions { - format?: StyleGuideFormat; - verbose?: boolean; - strict?: boolean; +export interface StyleGuideProcessorOptions { + llmProvider: LLMProvider; + maxCategories?: number | undefined; + filterRule?: string | undefined; + templateDir?: string | undefined; + defaultSeverity?: 'error' | 'warning' | undefined; + strictness?: 'lenient' | 'standard' | 'strict' | undefined; + verbose?: boolean | undefined; } -export interface ParserResult { - data: T; - warnings: string[]; +export interface GeneratedCategoryRule { + filename: string; + content: string; + meta: { + id: string; + name: string; + categoryType: string; + ruleCount: number; + }; } From 9b184311c6f09b5b02dd2d5e671a68a5cc947a83 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Mon, 8 Dec 2025 22:31:01 +0100 Subject: [PATCH 27/32] refactor: Remove deprecated schemas and consolidate evaluation generation logic in style guide schemas --- src/schemas/category-schema.ts | 52 ----------------- src/schemas/eval-generation-schema.ts | 25 -------- src/schemas/style-guide-schemas.ts | 72 ++++++++++++++++++++++++ src/style-guide/style-guide-processor.ts | 18 +++--- src/style-guide/template-renderer.ts | 21 +------ src/style-guide/types.ts | 23 ++++++++ 6 files changed, 107 insertions(+), 104 deletions(-) delete mode 100644 src/schemas/category-schema.ts delete mode 100644 src/schemas/eval-generation-schema.ts diff --git a/src/schemas/category-schema.ts b/src/schemas/category-schema.ts deleted file mode 100644 index 4c124eb..0000000 --- a/src/schemas/category-schema.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { z } from 'zod'; - -/** - * Schema for generating a category-level evaluation prompt - * This handles multiple related rules in a single rule - */ -export const CATEGORY_RULE_GENERATION_SCHEMA = z.object({ - evaluationType: z.enum(['subjective', 'semi-objective', 'objective']), - categoryName: z.string().describe('The category this eval covers'), - promptBody: z.string().describe('The main instruction for the LLM evaluator'), - criteria: z.array(z.object({ - name: z.string().describe('Criterion name (usually the rule description)'), - id: z.string().describe('PascalCase criterion ID (e.g., "VoiceSecondPersonPreferred")'), - weight: z.number().positive().describe('Weight of this criterion in overall score'), - rubric: z.array(z.object({ - score: z.number().int().min(1).max(4), - label: z.string(), - description: z.string(), - })).optional().describe('Rubric for subjective criteria'), - })), - examples: z.object({ - good: z.array(z.string()).optional(), - bad: z.array(z.string()).optional(), - }).optional(), -}); - -export type CategoryRuleGenerationOutput = z.infer; - - - -/** - * Schema for extracting and categorizing rules from a style guide - */ -export const CATEGORY_EXTRACTION_SCHEMA = z.object({ - categories: z.array(z.object({ - name: z.string().describe('Category name (e.g., "Voice & Tone", "Evidence & Citations")'), - id: z.string().describe('PascalCase category ID (e.g., "VoiceTone")'), - type: z.enum(['subjective', 'semi-objective', 'objective']).describe('Evaluation type for this category'), - description: z.string().describe('Brief description of what this category covers'), - rules: z.array(z.object({ - description: z.string().describe('The rule text from the style guide'), - severity: z.enum(['error', 'warning']).optional().describe('Suggested severity level'), - examples: z.object({ - good: z.array(z.string()).optional(), - bad: z.array(z.string()).optional(), - }).optional(), - })), - priority: z.number().int().min(1).max(10).describe('Priority level (1=highest, 10=lowest)'), - })), -}); -export type CategoryExtractionOutput = z.infer; - diff --git a/src/schemas/eval-generation-schema.ts b/src/schemas/eval-generation-schema.ts deleted file mode 100644 index 94edc29..0000000 --- a/src/schemas/eval-generation-schema.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { z } from 'zod'; - -/** - * Schema for the LLM output when generating an evaluation prompt - */ -export const EVAL_GENERATION_SCHEMA = z.object({ - evaluationType: z.enum(['subjective', 'semi-objective']), - promptBody: z.string().describe('The main instruction for the LLM evaluator'), - criteria: z.array(z.object({ - name: z.string(), - id: z.string(), - weight: z.number().positive(), - rubric: z.array(z.object({ - score: z.number().int().min(1).max(4), - label: z.string(), - description: z.string(), - })).optional(), - })).optional(), - examples: z.object({ - good: z.array(z.string()).optional(), - bad: z.array(z.string()).optional(), - }).optional(), -}); - -export type EvalGenerationOutput = z.infer; diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts index b26646b..4395a03 100644 --- a/src/schemas/style-guide-schemas.ts +++ b/src/schemas/style-guide-schemas.ts @@ -20,7 +20,79 @@ export const STYLE_GUIDE_SCHEMA = z.object({ content: z.string(), // Raw markdown content for LLM processing }).strict(); + +/** + * Schema for generating a category-level evaluation prompt + * This handles multiple related rules in a single rule + */ +export const CATEGORY_RULE_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective', 'objective']), + categoryName: z.string().describe('The category this eval covers'), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string().describe('Criterion name (usually the rule description)'), + id: z.string().describe('PascalCase criterion ID (e.g., "VoiceSecondPersonPreferred")'), + weight: z.number().positive().describe('Weight of this criterion in overall score'), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional().describe('Rubric for subjective criteria'), + })), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + + +/** + * Schema for extracting and categorizing rules from a style guide + */ +export const CATEGORY_EXTRACTION_SCHEMA = z.object({ + categories: z.array(z.object({ + name: z.string().describe('Category name (e.g., "Voice & Tone", "Evidence & Citations")'), + id: z.string().describe('PascalCase category ID (e.g., "VoiceTone")'), + type: z.enum(['subjective', 'semi-objective', 'objective']).describe('Evaluation type for this category'), + description: z.string().describe('Brief description of what this category covers'), + rules: z.array(z.object({ + description: z.string().describe('The rule text from the style guide'), + severity: z.enum(['error', 'warning']).optional().describe('Suggested severity level'), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), + })), + priority: z.number().int().min(1).max(10).describe('Priority level (1=highest, 10=lowest)'), + })), +}); + +/** + * Schema for the LLM output when generating an evaluation prompt + */ +export const RULE_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective']), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string(), + id: z.string(), + weight: z.number().positive(), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional(), + })).optional(), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + +export type RuleGenerationOutput = z.infer; +export type CategoryRuleGenerationOutput = z.infer; export type StyleGuideExamples = z.infer; export type StyleGuideRule = z.infer; export type ParsedStyleGuide = z.infer; +export type CategoryExtractionOutput = z.infer; diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 48708b5..3ab4622 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -9,24 +9,25 @@ import { CategoryExtractionOutput, CATEGORY_RULE_GENERATION_SCHEMA, CategoryRuleGenerationOutput -} from '../schemas/category-schema'; +} from '../schemas/style-guide-schemas'; import { TemplateRenderer } from './template-renderer'; -import { GeneratedCategoryRule, StyleGuideProcessorOptions } from './types'; +import { GeneratedCategoryRule, StyleGuideProcessorOptions, ResolvedProcessorOptions } from './types'; export class StyleGuideProcessor { private llmProvider: LLMProvider; - private options: Required> & Omit; + private options: ResolvedProcessorOptions; private renderer: TemplateRenderer; constructor(options: StyleGuideProcessorOptions) { this.llmProvider = options.llmProvider; this.renderer = new TemplateRenderer(options.templateDir); this.options = { - maxCategories: 10, - verbose: false, - defaultSeverity: 'warning', - strictness: 'standard', - ...options + maxCategories: options.maxCategories ?? 10, + verbose: options.verbose ?? false, + defaultSeverity: options.defaultSeverity ?? 'warning', + strictness: options.strictness ?? 'standard', + filterRule: options.filterRule, + templateDir: options.templateDir, }; } @@ -110,7 +111,6 @@ export class StyleGuideProcessor { console.log(`[StyleGuideProcessor] Passing raw style guide content for semantic matching`); } - // Pass full style guide to LLM - let LLM semantically find matching rules const prompt = this.buildSingleRulePrompt(styleGuide, filterTerm); try { diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts index b512fd3..77a1abb 100644 --- a/src/style-guide/template-renderer.ts +++ b/src/style-guide/template-renderer.ts @@ -2,24 +2,9 @@ import Handlebars from 'handlebars'; import { readFileSync, existsSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; -import { EvalGenerationOutput } from '../schemas/eval-generation-schema'; -import { CategoryRuleGenerationOutput } from '../schemas/category-schema'; +import { CategoryRuleGenerationOutput, RuleGenerationOutput } from '../schemas/style-guide-schemas'; import { StyleGuideRule } from '../schemas/style-guide-schemas'; - -export interface TemplateContext { - EVALUATION_TYPE: string; - RULE_ID: string; - RULE_NAME: string; - SEVERITY: string; - PROMPT_BODY: string; - CRITERIA?: Array<{ - name: string; - id: string; - weight: number; - }> | undefined; - RUBRIC?: string | undefined; - [key: string]: unknown; -} +import { TemplateContext } from './types'; export class TemplateRenderer { private templateDir: string; @@ -65,7 +50,7 @@ export class TemplateRenderer { */ public createContext( rule: StyleGuideRule, - output: EvalGenerationOutput, + output: RuleGenerationOutput, defaultSeverity: string ): TemplateContext { const severity = rule.severity || defaultSeverity; diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts index 015a6d1..514e18b 100644 --- a/src/style-guide/types.ts +++ b/src/style-guide/types.ts @@ -10,6 +10,15 @@ export interface StyleGuideProcessorOptions { verbose?: boolean | undefined; } +export type ResolvedProcessorOptions = { + maxCategories: number; + verbose: boolean; + defaultSeverity: 'error' | 'warning'; + strictness: 'lenient' | 'standard' | 'strict'; + filterRule?: string | undefined; + templateDir?: string | undefined; +}; + export interface GeneratedCategoryRule { filename: string; content: string; @@ -21,3 +30,17 @@ export interface GeneratedCategoryRule { }; } +export interface TemplateContext { + EVALUATION_TYPE: string; + RULE_ID: string; + RULE_NAME: string; + SEVERITY: string; + PROMPT_BODY: string; + CRITERIA?: Array<{ + name: string; + id: string; + weight: number; + }> | undefined; + RUBRIC?: string | undefined; + [key: string]: unknown; +} From 91fb1e83a54971852b0fbb53946d5f229f7be8a5 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 09:13:52 +0100 Subject: [PATCH 28/32] refactor: Enhance error handling in convert command and streamline processing logic --- src/cli/convert-command.ts | 270 +++++++++--------- src/style-guide/style-guide-processor.ts | 134 ++++----- src/style-guide/template-renderer.ts | 15 +- tests/cli/convert-command.test.ts | 4 +- .../style-guide-conversion.test.ts | 94 ++++-- tests/style-guide/generator.test.ts | 71 ----- tests/style-guide/parser.test.ts | 178 ------------ 7 files changed, 275 insertions(+), 491 deletions(-) delete mode 100644 tests/style-guide/generator.test.ts delete mode 100644 tests/style-guide/parser.test.ts diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index c83fad9..3bb5e27 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -10,6 +10,17 @@ import { loadConfig } from '../boundaries/config-loader'; import { handleUnknownError } from '../errors/index'; import { ConvertOptions } from '../schemas'; +/** + * Custom error class for convert command failures. + * Includes exit code for CLI handling. + */ +class ConvertCommandError extends Error { + constructor(message: string, public readonly exitCode: number = 1) { + super(message); + this.name = 'ConvertCommandError'; + } +} + export function registerConvertCommand(program: Command): void { program .command('convert') @@ -26,144 +37,145 @@ export function registerConvertCommand(program: Command): void { .option('--dry-run', 'Preview generated rules without writing files', false) .option('-v, --verbose', 'Enable verbose logging', false) .action(async (styleGuidePath: string, rawOptions: unknown) => { - // 1. Parse CLI options - let options: ConvertOptions; - try { - options = parseConvertOptions(rawOptions); - } catch (e: unknown) { - const err = handleUnknownError(e, 'Parsing CLI options'); - console.error(`Error: ${err.message}`); - process.exit(1); - } - - // 2. Validate input file - if (!existsSync(styleGuidePath)) { - console.error(`Error: Style guide file not found: ${styleGuidePath}`); - process.exit(1); - } - - // 3. Load configuration & determine output directory - let config; - let outputDir = options.output; - try { - // Determine output directory: CLI option > config RulesPath - config = loadConfig(process.cwd()); - - if (!outputDir) { - outputDir = config.rulesPath; - if (options.verbose) { - console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); - } - } + await executeConvert(styleGuidePath, rawOptions); } catch (e: unknown) { - if (!outputDir) { - const err = handleUnknownError(e, 'Loading configuration'); - console.error('Error: No output directory specified and failed to load vectorlint.ini.'); - console.error(`Details: ${err.message}`); - console.error('Please either use -o/--output or create a valid vectorlint.ini.'); - process.exit(1); + if (e instanceof ConvertCommandError) { + console.error(`Error: ${e.message}`); + // Re-throw to let Commander handle the exit + throw e; } - const err = handleUnknownError(e, 'Loading configuration'); - console.error(`Error: ${err.message}`); - process.exit(1); + throw e; } + }); +} +async function executeConvert(styleGuidePath: string, rawOptions: unknown): Promise { + // 1. Parse CLI options + let options: ConvertOptions; + try { + options = parseConvertOptions(rawOptions); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Parsing CLI options'); + throw new ConvertCommandError(err.message); + } + + // 2. Validate input file + if (!existsSync(styleGuidePath)) { + throw new ConvertCommandError(`Style guide file not found: ${styleGuidePath}`); + } + + // 3. Load configuration & determine output directory + let config; + let outputDir = options.output; + + try { + // Determine output directory: CLI option > config RulesPath + config = loadConfig(process.cwd()); + + if (!outputDir) { + outputDir = config.rulesPath; if (options.verbose) { - console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); - console.log(`[vectorlint] Output directory: ${outputDir}`); + console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); } - - // 4. Parse Environment - let env; - try { - env = parseEnvironment(); - } catch (e: unknown) { - const err = handleUnknownError(e, 'Validating environment variables'); - console.error(`Error: ${err.message}`); - console.error('Please set these in your .env file or environment.'); - process.exit(1); - } - - // 5. Load Directive & Initialize Provider - try { - const directive = loadDirective(); - const provider = createProvider( - env, - { debug: options.verbose }, - new DefaultRequestBuilder(directive) - ); - - // 6. Process Style Guide + } + } catch (e: unknown) { + if (!outputDir) { + const err = handleUnknownError(e, 'Loading configuration'); + console.error('Error: No output directory specified and failed to load vectorlint.ini.'); + console.error(`Details: ${err.message}`); + throw new ConvertCommandError('Please either use -o/--output or create a valid vectorlint.ini.'); + } + const err = handleUnknownError(e, 'Loading configuration'); + throw new ConvertCommandError(err.message); + } + + if (options.verbose) { + console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); + console.log(`[vectorlint] Output directory: ${outputDir}`); + } + + // 4. Parse Environment + let env; + try { + env = parseEnvironment(); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Validating environment variables'); + console.error('Please set these in your .env file or environment.'); + throw new ConvertCommandError(err.message); + } + + // 5. Load Directive & Initialize Provider + const directive = loadDirective(); + const provider = createProvider( + env, + { debug: options.verbose }, + new DefaultRequestBuilder(directive) + ); + + // 6. Process Style Guide + if (options.verbose) { + console.log(`[vectorlint] Processing style guide...`); + console.log(`[vectorlint] Using ${env.LLM_PROVIDER}...`); + } + + const processor = new StyleGuideProcessor({ + llmProvider: provider, + maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, + filterRule: options.rule, + templateDir: options.template || undefined, + defaultSeverity: options.severity, + strictness: options.strictness, + verbose: options.verbose, + }); + + const categoryRules = await processor.processFile(styleGuidePath); + const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); + + if (rules.length === 0) { + console.warn('[vectorlint] No rules were generated. Check your style guide format.'); + return; // Exit gracefully with success since no error occurred + } + + // 7. Write Output + if (options.dryRun) { + console.log('\n--- DRY RUN PREVIEW ---\n'); + for (const rule of rules) { + console.log(`File: ${rule.filename}`); + console.log('---'); + console.log(rule.content); + console.log('---\n'); + } + console.log(`[vectorlint] Would generate ${rules.length} files in ${outputDir}`); + } else { + if (!existsSync(outputDir)) { + mkdirSync(outputDir, { recursive: true }); + } + + let writtenCount = 0; + let skippedCount = 0; + + for (const rule of rules) { + const filePath = path.join(outputDir, rule.filename); + + if (existsSync(filePath) && !options.force) { if (options.verbose) { - console.log(`[vectorlint] Processing style guide...`); - console.log(`[vectorlint] Using ${env.LLM_PROVIDER}...`); - } - - const processor = new StyleGuideProcessor({ - llmProvider: provider, - maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, - filterRule: options.rule, - templateDir: options.template || undefined, - defaultSeverity: options.severity, - strictness: options.strictness, - verbose: options.verbose, - }); - - const categoryRules = await processor.processFile(styleGuidePath); - const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); - - if (rules.length === 0) { - console.warn('[vectorlint] No rules were generated. Check your style guide format.'); - process.exit(0); - } - - // 8. Write Output - if (options.dryRun) { - console.log('\n--- DRY RUN PREVIEW ---\n'); - for (const rule of rules) { - console.log(`File: ${rule.filename}`); - console.log('---'); - console.log(rule.content); - console.log('---\n'); - } - console.log(`[vectorlint] Would generate ${rules.length} files in ${outputDir}`); - } else { - if (!existsSync(outputDir!)) { - mkdirSync(outputDir!, { recursive: true }); - } - - let writtenCount = 0; - let skippedCount = 0; - - for (const rule of rules) { - const filePath = path.join(outputDir!, rule.filename); // outputDir is guaranteed string here - - if (existsSync(filePath) && !options.force) { - if (options.verbose) { - console.warn(`[vectorlint] Skipping existing file: ${filePath} (use --force to overwrite)`); - } - skippedCount++; - continue; - } - - writeFileSync(filePath, rule.content, 'utf-8'); - writtenCount++; - if (options.verbose) { - console.log(`[vectorlint] Wrote: ${filePath}`); - } - } - - console.log(`\n[vectorlint] Successfully generated ${writtenCount} evaluation files.`); - if (skippedCount > 0) { - console.log(`[vectorlint] Skipped ${skippedCount} existing files.`); - } + console.warn(`[vectorlint] Skipping existing file: ${filePath} (use --force to overwrite)`); } + skippedCount++; + continue; + } - } catch (e: unknown) { - const err = handleUnknownError(e, 'Converting style guide'); - console.error(`Error: ${err.message}`); - process.exit(1); + writeFileSync(filePath, rule.content, 'utf-8'); + writtenCount++; + if (options.verbose) { + console.log(`[vectorlint] Wrote: ${filePath}`); } - }); + } + + console.log(`\n[vectorlint] Successfully generated ${writtenCount} evaluation files.`); + if (skippedCount > 0) { + console.log(`[vectorlint] Skipped ${skippedCount} existing files.`); + } + } } diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 3ab4622..692f6e4 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -1,15 +1,16 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import { LLMProvider } from '../providers/llm-provider'; -import { ParsedStyleGuide, STYLE_GUIDE_SCHEMA } from '../schemas/style-guide-schemas'; -import { ProcessingError, ConfigError, ValidationError } from '../errors/index'; -import { zodToJsonSchema } from 'zod-to-json-schema'; import { + ParsedStyleGuide, + STYLE_GUIDE_SCHEMA, CATEGORY_EXTRACTION_SCHEMA, CategoryExtractionOutput, CATEGORY_RULE_GENERATION_SCHEMA, CategoryRuleGenerationOutput } from '../schemas/style-guide-schemas'; +import { ProcessingError, ConfigError, ValidationError } from '../errors/index'; +import { zodToJsonSchema } from 'zod-to-json-schema'; import { TemplateRenderer } from './template-renderer'; import { GeneratedCategoryRule, StyleGuideProcessorOptions, ResolvedProcessorOptions } from './types'; @@ -35,14 +36,12 @@ export class StyleGuideProcessor { * Process a style guide file: Read, extract categories, and generate rules */ public async processFile(filePath: string): Promise { - // 1. Read and parse the style guide file const styleGuide = this.readStyleGuide(filePath); if (this.options.verbose) { console.log(`[StyleGuideProcessor] Loaded style guide: ${styleGuide.name}`); } - // 2. Process the style guide content return this.process(styleGuide); } @@ -62,7 +61,6 @@ export class StyleGuideProcessor { const result: ParsedStyleGuide = { name, content }; - // Validate against schema STYLE_GUIDE_SCHEMA.parse(result); return result; @@ -94,12 +92,9 @@ export class StyleGuideProcessor { * Extract and categorize rules from a parsed style guide */ private async extractCategories(styleGuide: ParsedStyleGuide): Promise { - // If filterRule is specified, generate ONLY ONE category for that specific rule if (this.options.filterRule) { return this.extractSingleRule(styleGuide); } - - // Otherwise, do full category extraction return this.extractAllCategories(styleGuide); } @@ -125,7 +120,6 @@ export class StyleGuideProcessor { } ); - // Ensure we return exactly ONE category if (result.categories.length > 1) { const firstCategory = result.categories[0]; if (firstCategory) { @@ -246,88 +240,78 @@ export class StyleGuideProcessor { // --- Prompt Builders --- private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { - return ` - You are an expert in analyzing style guides and creating content evaluation prompts. + return `You are a **style guide analyzer** designed to extract and categorize rules from style guides. - The user wants to generate an evaluation for a SPECIFIC topic: "${filterTerm}" +## Task - Your task: - 1. Analyze the ENTIRE style guide provided as context - 2. Semantically identify ALL rules that relate to "${filterTerm}" (understand synonyms, related concepts, abbreviations like "pov" = "point of view" = "second person") - 3. Create EXACTLY ONE category that consolidates all related rules into a single cohesive evaluation - 4. Name the category based on the topic (e.g., if "${filterTerm}" is about voice/perspective, name it accordingly) - 5. Create a PascalCase ID for the category (e.g., "VoiceSecondPerson", "ToneFormality") - 6. Classify it as subjective, semi-objective, or objective based on the rule nature - 7. Include ALL semantically matching rules under this single category +Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** - IMPORTANT: - - Use semantic understanding, not just string matching - - "${filterTerm}" may be an abbreviation (pov, cta, seo) - understand what it means - - Look for rules that are RELATED to the topic, even if they don't use the exact term +## Output Requirements - Style Guide Name: ${styleGuide.name} +- Create **exactly one** category that consolidates all related rules +- Use **PascalCase** for the category ID +- Classify as: **subjective**, **semi-objective**, or **objective** +- Include all semantically matching rules - Analyze the style guide content and create ONE category covering the "${filterTerm}" topic. - `; +## Guidelines + +- Look for rules **related** to the topic, not just exact matches +- Consolidate similar rules into a cohesive category +- Preserve original rule text + +Style Guide: **${styleGuide.name}**`; } private buildFullPrompt(styleGuide: ParsedStyleGuide): string { - return ` - You are an expert in analyzing style guides and organizing rules into logical categories. - - Your task is to analyze the provided style guide and DYNAMICALLY identify categories based on the content. - DO NOT use predefined categories. Let the content guide what categories emerge naturally. - - Instructions: - 1. Read all the rules in the style guide - 2. Identify natural thematic groupings (e.g., if many rules discuss tone, create a "Voice & Tone" category) - 3. Create up to ${this.options.maxCategories} categories based on what you find - 4. Classify each category as: - - Subjective: Requires judgment (tone, style, clarity) - - Semi-objective: Clear patterns but needs context (citations, evidence) - - Objective: Can be mechanically checked (formatting, word count) - 5. Assign priority (1=highest, 10=lowest) based on impact on content quality - 6. Use PascalCase for all category IDs (e.g., "VoiceTone", "EvidenceCredibility") - - Important: - - Categories should emerge from the ACTUAL content of the style guide - - Do not force rules into predefined buckets - - Each category should have 3-10 related rules - - Preserve original rule text and examples - - Style Guide Name: ${styleGuide.name} - - Analyze the style guide content and output categories based on what you find. - `; + return `You are a **style guide categorizer** designed to organize style guide rules into logical groups. + +## Task + +Analyze the provided style guide and identify up to **${this.options.maxCategories}** natural categories. + +## Category Types + +- **Subjective**: Requires judgment (tone, style, clarity) +- **Semi-objective**: Clear patterns but needs context (citations, evidence) +- **Objective**: Can be mechanically checked (formatting, word count) + +## Output Requirements + +- Use **PascalCase** for all category IDs +- Assign **priority** (1=highest, 10=lowest) based on quality impact +- Include **3-10 rules** per category +- Preserve original rule text and examples + + + +Style Guide: **${styleGuide.name}**`; } private buildRulePrompt(category: CategoryExtractionOutput['categories'][0]): string { - return ` - You are an expert in creating automated content evaluation prompts. + return `You are an **evaluation prompt generator** designed to create content evaluation prompts. + +## Task + +Create a comprehensive evaluation prompt for the **"${category.name}"** category. - Your task is to create a comprehensive evaluation prompt that checks ALL rules in the "${category.name}" category. +## Category Details - Category: ${category.name} - Type: ${category.type} - Description: ${category.description} - Number of Rules: ${category.rules.length} +- **Name**: ${category.name} +- **Type**: ${category.type} +- **Description**: ${category.description} +- **Strictness**: ${this.options.strictness} - Rules to evaluate: - ${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} +## Rules to Evaluate - Strictness Level: ${this.options.strictness} +${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} - Instructions: - 1. Create a single prompt that evaluates ALL rules in this category together - 2. Each rule becomes a separate criterion with its own weight - 3. Use PascalCase for all criterion IDs (e.g., "VoiceSecondPersonPreferred") - 4. The prompt body should instruct the LLM to check all criteria - 4. For ${category.type} evaluation, ${category.type === 'subjective' ? 'create 1-4 rubrics for each criterion' : 'provide clear pass/fail guidance'} - 5. Total weight across all criteria should sum to 100 - 6. Use examples from the rules when available +## Output Requirements - Output a structured evaluation prompt that covers the entire category. - `; +- Each rule becomes a **separate criterion** with its own weight +- Use **PascalCase** for all criterion IDs +- Total weight must sum to **100** +${category.type === 'subjective' ? '- Create **1-4 rubric levels** for each criterion' : '- Provide **pass/fail** guidance for each criterion'} +- Include examples from rules when available`; } // --- Helpers --- diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts index 77a1abb..e75d0a0 100644 --- a/src/style-guide/template-renderer.ts +++ b/src/style-guide/template-renderer.ts @@ -2,8 +2,11 @@ import Handlebars from 'handlebars'; import { readFileSync, existsSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; -import { CategoryRuleGenerationOutput, RuleGenerationOutput } from '../schemas/style-guide-schemas'; -import { StyleGuideRule } from '../schemas/style-guide-schemas'; +import { + CategoryRuleGenerationOutput, + RuleGenerationOutput, + StyleGuideRule +} from '../schemas/style-guide-schemas'; import { TemplateContext } from './types'; export class TemplateRenderer { @@ -13,10 +16,10 @@ export class TemplateRenderer { if (templateDir) { this.templateDir = templateDir; } else { - // ESM compatible __dirname - const __filename = fileURLToPath(import.meta.url); - const __dirname = dirname(__filename); - this.templateDir = join(__dirname, 'templates'); + // ESM compatible current directory resolution + const currentFilePath = fileURLToPath(import.meta.url); + const currentDirPath = dirname(currentFilePath); + this.templateDir = join(currentDirPath, 'templates'); } this.registerHelpers(); } diff --git a/tests/cli/convert-command.test.ts b/tests/cli/convert-command.test.ts index dc649f3..649f4ec 100644 --- a/tests/cli/convert-command.test.ts +++ b/tests/cli/convert-command.test.ts @@ -2,12 +2,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { Command } from 'commander'; import { registerConvertCommand } from '../../src/cli/convert-command'; import * as fs from 'fs'; -import * as path from 'path'; // Mocks vi.mock('fs'); -vi.mock('../../src/style-guide/style-guide-parser'); -vi.mock('../../src/style-guide/eval-generator'); +vi.mock('../../src/style-guide/style-guide-processor'); vi.mock('../../src/providers/provider-factory'); vi.mock('../../src/boundaries/index'); vi.mock('../../src/prompts/directive-loader'); diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts index f5f0c2e..dfd797c 100644 --- a/tests/integration/style-guide-conversion.test.ts +++ b/tests/integration/style-guide-conversion.test.ts @@ -1,24 +1,22 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { StyleGuideProcessor } from '../../src/style-guide/style-guide-processor'; import { LLMProvider } from '../../src/providers/llm-provider'; +import type { ParsedStyleGuide, CategoryExtractionOutput, CategoryRuleGenerationOutput } from '../../src/schemas/style-guide-schemas'; import * as path from 'path'; import * as fs from 'fs'; -// Mock LLM Provider that handles both category extraction and rule generation +/** + * Mock LLM Provider that returns structured responses for category extraction and rule generation + */ class MockLLMProvider implements LLMProvider { - private callCount = 0; - - async runPromptStructured( + runPromptStructured( content: string, promptText: string, schema: { name: string; schema: Record } ): Promise { - this.callCount++; - - // First call: category extraction + // Category extraction call if (schema.name === 'categoryExtraction') { - return { + const categoryResult: CategoryExtractionOutput = { categories: [ { id: 'VoiceTone', @@ -27,30 +25,42 @@ class MockLLMProvider implements LLMProvider { type: 'subjective', priority: 1, rules: [ - { id: 'rule-1', description: 'Write in second person' }, - { id: 'rule-2', description: 'Use active voice' } + { description: 'Write in second person' }, + { description: 'Use active voice' } ] } ] - } as unknown as T; + }; + return Promise.resolve(categoryResult as T); } - // Second call: rule generation - return { + // Category rule generation call + const ruleResult: CategoryRuleGenerationOutput = { evaluationType: 'subjective', - promptBody: 'Check if the content follows the rule.', + categoryName: 'Voice & Tone', + promptBody: 'Evaluate the content for voice and tone adherence.', criteria: [ { - name: 'Adherence', - id: 'Adherence', - weight: 100, + name: 'Second Person Voice', + id: 'SecondPersonVoice', + weight: 50, + rubric: [ + { score: 4, label: 'Excellent', description: 'Consistent second person usage' }, + { score: 1, label: 'Poor', description: 'No second person usage' } + ] + }, + { + name: 'Active Voice', + id: 'ActiveVoice', + weight: 50, rubric: [ - { score: 4, label: 'Excellent', description: 'Perfect adherence' }, - { score: 1, label: 'Poor', description: 'Severe violation' } + { score: 4, label: 'Excellent', description: 'Strong active voice throughout' }, + { score: 1, label: 'Poor', description: 'Predominantly passive voice' } ] } ] - } as unknown as T; + }; + return Promise.resolve(ruleResult as T); } } @@ -72,14 +82,15 @@ describe('Style Guide Conversion Integration', () => { }); it('should convert a markdown style guide to category-based rule files', async () => { - // 1. Parse Style Guide - const parser = new StyleGuideParser(); const styleGuidePath = path.join(fixturesDir, 'sample-style-guide.md'); - const styleGuide = parser.parse(styleGuidePath); - expect(styleGuide.data.rules.length).toBeGreaterThan(0); + // Skip test if fixture doesn't exist + if (!fs.existsSync(styleGuidePath)) { + console.log('[SKIP] sample-style-guide.md fixture not found'); + return; + } - // 2. Process Style Guide (extract categories + generate rules) + // 1. Create processor with mock LLM provider const mockProvider = new MockLLMProvider(); const processor = new StyleGuideProcessor({ llmProvider: mockProvider, @@ -88,7 +99,8 @@ describe('Style Guide Conversion Integration', () => { verbose: false, }); - const rules = await processor.process(styleGuide.data); + // 2. Process the style guide file directly + const rules = await processor.processFile(styleGuidePath); // Expect at least one category-based rule expect(rules.length).toBeGreaterThan(0); @@ -108,5 +120,29 @@ describe('Style Guide Conversion Integration', () => { expect(firstFile).toContain('evaluator: base'); expect(firstFile).toContain('type: subjective'); }); -}); + it('should process a style guide object with process method', async () => { + const mockProvider = new MockLLMProvider(); + const processor = new StyleGuideProcessor({ + llmProvider: mockProvider, + maxCategories: 5, + verbose: false, + }); + + // Create a ParsedStyleGuide object directly + const styleGuide: ParsedStyleGuide = { + name: 'Test Style Guide', + content: '# Test Guide\n\nUse second person (you/your).\nPrefer active voice over passive voice.' + }; + + // Process the style guide + const rules = await processor.process(styleGuide); + + expect(rules.length).toBe(1); + expect(rules[0]?.meta.id).toBe('VoiceTone'); + expect(rules[0]?.meta.name).toBe('Voice & Tone'); + expect(rules[0]?.meta.categoryType).toBe('subjective'); + expect(rules[0]?.meta.ruleCount).toBe(2); + expect(rules[0]?.filename).toBe('voice-tone.md'); + }); +}); diff --git a/tests/style-guide/generator.test.ts b/tests/style-guide/generator.test.ts deleted file mode 100644 index 8b9b20c..0000000 --- a/tests/style-guide/generator.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { describe, it, expect, vi } from 'vitest'; -import { EvalGenerator } from '../../src/style-guide/eval-generator'; -import { LLMProvider } from '../../src/providers/llm-provider'; -import { StyleGuideRule } from '../../src/schemas/style-guide-schemas'; -import { EvalGenerationOutput } from '../../src/schemas/eval-generation-schema'; - -// Mock LLM Provider -class MockLLMProvider implements LLMProvider { - async runPromptStructured( - content: string, - promptText: string, - schema: { name: string; schema: Record } - ): Promise { - // Return a dummy response matching the schema - const response: EvalGenerationOutput = { - evaluationType: 'subjective', - promptBody: 'Check if the content follows the rule.', - criteria: [ - { - name: 'Adherence', - id: 'adherence', - weight: 10, - rubric: [ - { score: 4, label: 'Excellent', description: 'Perfect adherence' }, - { score: 1, label: 'Poor', description: 'Severe violation' } - ] - } - ] - }; - return response as unknown as T; - } -} - -describe('EvalGenerator', () => { - it('should generate an eval from a rule', async () => { - const mockProvider = new MockLLMProvider(); - const generator = new EvalGenerator({ llmProvider: mockProvider }); - - const rule: StyleGuideRule = { - id: 'test-rule', - category: 'tone', - description: 'Use a friendly tone.', - severity: 'warning' - }; - - const result = await generator.generateEval(rule); - - expect(result).toBeDefined(); - expect(result.filename).toBe('test-rule.md'); - expect(result.content).toContain('evaluator: base'); - expect(result.content).toContain('type: subjective'); - expect(result.content).toContain('id: test-rule'); - expect(result.content).toContain('severity: warning'); - expect(result.content).toContain('Check if the content follows the rule.'); - expect(result.content).toContain('## Rubric for Adherence'); - }); - - it('should handle errors gracefully', async () => { - const mockProvider = new MockLLMProvider(); - vi.spyOn(mockProvider, 'runPromptStructured').mockRejectedValue(new Error('LLM Error')); - - const generator = new EvalGenerator({ llmProvider: mockProvider }); - const rule: StyleGuideRule = { - id: 'test-rule', - category: 'tone', - description: 'Use a friendly tone.' - }; - - await expect(generator.generateEval(rule)).rejects.toThrow('LLM generation failed: LLM Error'); - }); -}); diff --git a/tests/style-guide/parser.test.ts b/tests/style-guide/parser.test.ts deleted file mode 100644 index 6b61a8b..0000000 --- a/tests/style-guide/parser.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { StyleGuideParser } from '../../src/style-guide/style-guide-parser'; -import { StyleGuideFormat } from '../../src/style-guide/types'; -import { - StyleGuideParseError, - UnsupportedFormatError, -} from '../../src/errors/style-guide-errors'; -import * as path from 'path'; - -describe('StyleGuideParser', () => { - const fixturesDir = path.join(__dirname, 'fixtures'); - - describe('Markdown parsing', () => { - it('should parse markdown style guide', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md'), - { format: StyleGuideFormat.MARKDOWN } - ); - - expect(result.data.name).toBe('Acme Corp Writing Style Guide'); - expect(result.data.rules.length).toBeGreaterThan(0); - }); - - it('should parse TinyRocket style guide (headers and bold text)', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'tinyrocket-style-guide.md') - ); - - console.log(`[DEBUG] TinyRocket Rules Found: ${result.data.rules.length}`); - result.data.rules.forEach(r => console.log(`[DEBUG] Rule: ${r.id} - ${r.description.substring(0, 50)}...`)); - - // We expect this to fail initially or find very few rules - expect(result.data.rules.length).toBeGreaterThan(5); - }); - - it('should extract rules from markdown sections', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md') - ); - - const rules = result.data.rules; - expect(rules.length).toBeGreaterThan(10); // Should have multiple rules - - // Check that rules have required fields - rules.forEach((rule) => { - expect(rule.id).toBeDefined(); - expect(rule.category).toBeDefined(); - expect(rule.description).toBeDefined(); - }); - }); - - it('should auto-categorize rules', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md') - ); - - const rules = result.data.rules; - - // Should have tone rules - const toneRules = rules.filter((r) => r.category === 'voice-and-tone'); - expect(toneRules.length).toBeGreaterThan(0); - - // Should have terminology rules - const termRules = rules.filter( - (r) => r.category === 'terminology' - ); - expect(termRules.length).toBeGreaterThan(0); - - // Should have structure rules - const structureRules = rules.filter( - (r) => r.category === 'structure' - ); - expect(structureRules.length).toBeGreaterThan(0); - }); - }); - - describe('Format detection', () => { - it('should auto-detect markdown format', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md') - ); - - expect(result.data.name).toBeDefined(); - expect(result.data.rules.length).toBeGreaterThan(0); - }); - - it('should throw error for unsupported format', () => { - const parser = new StyleGuideParser(); - - expect(() => { - parser.parse(path.join(fixturesDir, 'invalid.txt')); - }).toThrow(UnsupportedFormatError); - }); - }); - - describe('Error handling', () => { - it('should throw error for non-existent file', () => { - const parser = new StyleGuideParser(); - - expect(() => { - parser.parse(path.join(fixturesDir, 'non-existent.md')); - }).toThrow(StyleGuideParseError); - }); - }); - - describe('Warnings', () => { - it('should collect warnings during parsing', () => { - const parser = new StyleGuideParser(); - const emptyMarkdown = '# Empty Style Guide\n\nNo rules here.'; - - // Write temporary file - const fs = require('fs'); - const tempFile = path.join(fixturesDir, 'empty-warnings.md'); - fs.writeFileSync(tempFile, emptyMarkdown); - - try { - const result = parser.parse(tempFile); - expect(result.warnings).toBeDefined(); - expect(Array.isArray(result.warnings)).toBe(true); - } finally { - if (fs.existsSync(tempFile)) { - fs.unlinkSync(tempFile); - } - } - }); - - it('should warn when no rules found', () => { - const parser = new StyleGuideParser(); - const emptyMarkdown = '# Empty Style Guide\n\nNo rules here.'; - - // Write temporary file - const fs = require('fs'); - const tempFile = path.join(fixturesDir, 'empty-no-rules.md'); - fs.writeFileSync(tempFile, emptyMarkdown); - - try { - const result = parser.parse(tempFile); - expect(result.warnings.some((w) => w.includes('No rules found'))).toBe( - true - ); - } finally { - if (fs.existsSync(tempFile)) { - fs.unlinkSync(tempFile); - } - } - }); - }); - - describe('Rule ID generation', () => { - it('should generate unique IDs for rules', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md') - ); - - const ids = result.data.rules.map((r) => r.id); - const uniqueIds = new Set(ids); - - expect(ids.length).toBe(uniqueIds.size); // All IDs should be unique - }); - - it('should generate readable IDs from descriptions', () => { - const parser = new StyleGuideParser(); - const result = parser.parse( - path.join(fixturesDir, 'sample-style-guide.md') - ); - - result.data.rules.forEach((rule) => { - expect(rule.id).toMatch(/^rule-[a-z0-9-]+$/); - }); - }); - }); -}); From 2108b7ccf486e12f61ac6bac26f11ef59e539b7b Mon Sep 17 00:00:00 2001 From: Ayomide Date: Tue, 9 Dec 2025 09:30:50 +0100 Subject: [PATCH 29/32] refactor: Introduce type identification schema and enhance style guide processing logic --- src/schemas/style-guide-schemas.ts | 14 ++ src/style-guide/style-guide-processor.ts | 211 +++++++++++------- .../style-guide-conversion.test.ts | 94 +++++--- 3 files changed, 206 insertions(+), 113 deletions(-) diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts index 4395a03..a546256 100644 --- a/src/schemas/style-guide-schemas.ts +++ b/src/schemas/style-guide-schemas.ts @@ -46,6 +46,19 @@ export const CATEGORY_RULE_GENERATION_SCHEMA = z.object({ }); + +/** + * Schema for the first step: Identifying evaluation types and their descriptions + */ +export const TYPE_IDENTIFICATION_SCHEMA = z.object({ + types: z.array(z.object({ + type: z.enum(['objective', 'semi-objective', 'subjective']), + description: z.string().describe('Description of the rule type'), + ruleCount: z.number().int().describe('Estimated number of rules for this type'), + rules: z.array(z.string()).describe('Raw text of the rules belonging to this type'), + })), +}); + /** * Schema for extracting and categorizing rules from a style guide */ @@ -95,4 +108,5 @@ export type StyleGuideExamples = z.infer; export type StyleGuideRule = z.infer; export type ParsedStyleGuide = z.infer; export type CategoryExtractionOutput = z.infer; +export type TypeIdentificationOutput = z.infer; diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index 692f6e4..b988766 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -7,7 +7,9 @@ import { CATEGORY_EXTRACTION_SCHEMA, CategoryExtractionOutput, CATEGORY_RULE_GENERATION_SCHEMA, - CategoryRuleGenerationOutput + CategoryRuleGenerationOutput, + TYPE_IDENTIFICATION_SCHEMA, + TypeIdentificationOutput } from '../schemas/style-guide-schemas'; import { ProcessingError, ConfigError, ValidationError } from '../errors/index'; import { zodToJsonSchema } from 'zod-to-json-schema'; @@ -74,39 +76,81 @@ export class StyleGuideProcessor { } /** - * Process a style guide: Extract categories and generate rules + * Process a style guide: Identify types, extract categories, and generate rules */ public async process(styleGuide: ParsedStyleGuide): Promise { - // 1. Extract Categories (Organizer Role) - const extractionOutput = await this.extractCategories(styleGuide); + // Handle single rule extraction separately + if (this.options.filterRule) { + return this.processSingleRule(styleGuide); + } + + // 1. Identify Types Strategy (Planner Role) + const typeIdentification = await this.identifyTypes(styleGuide); if (this.options.verbose) { - console.log(`[StyleGuideProcessor] Extracted ${extractionOutput.categories.length} categories`); + console.log(`[StyleGuideProcessor] Identified ${typeIdentification.types.length} evaluation types`); } - // 2. Generate Rules (Author Role) - return this.generateCategoryRules(extractionOutput); + const allCategories: CategoryExtractionOutput['categories'] = []; + + // 2. Extract Categories for each type (Organizer Role) + for (const typeInfo of typeIdentification.types) { + const categories = await this.extractCategoriesForType(styleGuide, typeInfo); + allCategories.push(...categories.categories); + } + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Extracted ${allCategories.length} total categories`); + } + + // 3. Generate Rules (Author Role) + return this.generateCategoryRules({ categories: allCategories }); } /** - * Extract and categorize rules from a parsed style guide + * Process a single rule based on filter term */ - private async extractCategories(styleGuide: ParsedStyleGuide): Promise { - if (this.options.filterRule) { - return this.extractSingleRule(styleGuide); + private async processSingleRule(styleGuide: ParsedStyleGuide): Promise { + const filterTerm = this.options.filterRule!; + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Processing single rule filter: "${filterTerm}"`); } - return this.extractAllCategories(styleGuide); + + const extractionOutput = await this.extractSingleRule(styleGuide, filterTerm); + return this.generateCategoryRules(extractionOutput); } - private async extractSingleRule(styleGuide: ParsedStyleGuide): Promise { - const filterTerm = this.options.filterRule!; + /** + * Step 1: Identify evaluation types present in the style guide + */ + private async identifyTypes(styleGuide: ParsedStyleGuide): Promise { + const prompt = this.buildTypeIdentificationPrompt(styleGuide); - if (this.options.verbose) { - console.log(`[StyleGuideProcessor] Using LLM to find rules related to "${filterTerm}"`); - console.log(`[StyleGuideProcessor] Passing raw style guide content for semantic matching`); + try { + const schemaJson = zodToJsonSchema(TYPE_IDENTIFICATION_SCHEMA); + + return await this.llmProvider.runPromptStructured( + JSON.stringify(styleGuide), + prompt, + { + name: 'typeIdentification', + schema: schemaJson as Record + } + ); + } catch (error) { + throw new ProcessingError(`Type identification failed: ${(error as Error).message}`); } + } - const prompt = this.buildSingleRulePrompt(styleGuide, filterTerm); + /** + * Step 2: Extract categories for a specific evaluation type + */ + private async extractCategoriesForType( + styleGuide: ParsedStyleGuide, + typeInfo: TypeIdentificationOutput['types'][0] + ): Promise { + const prompt = this.buildCategoryExtractionPrompt(styleGuide, typeInfo); try { const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); @@ -115,40 +159,24 @@ export class StyleGuideProcessor { JSON.stringify(styleGuide), prompt, { - name: 'singleRuleExtraction', + name: 'categoryExtraction', schema: schemaJson as Record } ); - if (result.categories.length > 1) { - const firstCategory = result.categories[0]; - if (firstCategory) { - result.categories = [firstCategory]; - } - } - - if (result.categories.length === 0) { - throw new ProcessingError( - `LLM could not find rules related to "${filterTerm}" in the style guide` - ); - } - - if (this.options.verbose) { - const cat = result.categories[0]; - console.log(`[StyleGuideProcessor] LLM extracted category: "${cat?.name}" with ${cat?.rules.length} rules`); - } + // Ensure extracted categories match the requested type + result.categories.forEach(cat => cat.type = typeInfo.type); return result; } catch (error) { - if (error instanceof ProcessingError) throw error; - throw new ProcessingError( - `Single rule extraction failed: ${(error as Error).message}` - ); + console.warn(`Category extraction failed for type ${typeInfo.type}:`, error); + return { categories: [] }; } } - private async extractAllCategories(styleGuide: ParsedStyleGuide): Promise { - const prompt = this.buildFullPrompt(styleGuide); + + private async extractSingleRule(styleGuide: ParsedStyleGuide, filterTerm: string): Promise { + const prompt = this.buildFilteredRulePrompt(styleGuide, filterTerm); try { const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); @@ -157,31 +185,24 @@ export class StyleGuideProcessor { JSON.stringify(styleGuide), prompt, { - name: 'categoryExtraction', + name: 'singleRuleExtraction', schema: schemaJson as Record } ); - // Sort categories by priority (1=highest) and limit to maxCategories - const sortedCategories = [...result.categories] - .sort((a, b) => a.priority - b.priority) - .slice(0, this.options.maxCategories); - - const finalResult: CategoryExtractionOutput = { categories: sortedCategories }; - - if (this.options.verbose) { - const totalRules = finalResult.categories.reduce((sum: number, cat) => sum + cat.rules.length, 0); - console.log(`[StyleGuideProcessor] Extracted ${finalResult.categories.length} categories with ${totalRules} total rules`); - finalResult.categories.forEach(cat => { - console.log(` - ${cat.name} (priority: ${cat.priority}, ${cat.type}): ${cat.rules.length} rules`); - }); + if (result.categories.length === 0) { + throw new ProcessingError( + `LLM could not find rules related to "${filterTerm}" in the style guide` + ); } - return finalResult; + // Take only the first category if multiple returned + return { categories: [result.categories[0]!] }; + } catch (error) { if (error instanceof ProcessingError) throw error; throw new ProcessingError( - `Category extraction failed: ${(error as Error).message}` + `Single rule extraction failed: ${(error as Error).message}` ); } } @@ -215,7 +236,7 @@ export class StyleGuideProcessor { private async generateCategoryRule( category: CategoryExtractionOutput['categories'][0] ): Promise { - const prompt = this.buildRulePrompt(category); + const prompt = this.buildRuleGenerationPrompt(category); try { const schemaJson = zodToJsonSchema(CATEGORY_RULE_GENERATION_SCHEMA); @@ -239,55 +260,77 @@ export class StyleGuideProcessor { // --- Prompt Builders --- - private buildSingleRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { - return `You are a **style guide analyzer** designed to extract and categorize rules from style guides. + private buildTypeIdentificationPrompt(styleGuide: ParsedStyleGuide): string { + return `You are a **style guide planning agent**. ## Task +Analyze the style guide and identify which evaluation types are present. -Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** +## Evaluation Types +- **objective**: Formatting, structure, casing, specific disallowed words. +- **semi-objective**: Grammar, spelling, specific prohibited patterns, clear violations. +- **subjective**: Tone, voice, clarity, audience, engagement, flow. ## Output Requirements +- Identify **all** applicable types found in the content +- Estimate the number of rules for each type +- Provide the raw text of the rules belonging to each type -- Create **exactly one** category that consolidates all related rules -- Use **PascalCase** for the category ID -- Classify as: **subjective**, **semi-objective**, or **objective** -- Include all semantically matching rules +Style Guide: **${styleGuide.name}**`; + } -## Guidelines + private buildCategoryExtractionPrompt( + styleGuide: ParsedStyleGuide, + typeInfo: TypeIdentificationOutput['types'][0] + ): string { + return `You are a **rule categorizer** agent. -- Look for rules **related** to the topic, not just exact matches -- Consolidate similar rules into a cohesive category -- Preserve original rule text +## Task +Extract and categorize rules specifically for the **${typeInfo.type}** evaluation type. + +## Context +${typeInfo.description} +Raw Rules Text: +${typeInfo.rules.join('\n\n')} + +## Type: ${typeInfo.type} +${typeInfo.type === 'subjective' ? '- Focus on tone, voice, clarity' : ''} +${typeInfo.type === 'semi-objective' ? '- Focus on repeatable patterns and clear violations' : ''} +${typeInfo.type === 'objective' ? '- Focus on formatting, structure, and existence checks' : ''} + +## Output Requirements +- Create logical categories for these rules (e.g., "Voice & Tone", "Grammar", "Formatting") +- Use **PascalCase** for IDs +- Group related rules together (3-10 rules per category) +- Preserve original instructions Style Guide: **${styleGuide.name}**`; } - private buildFullPrompt(styleGuide: ParsedStyleGuide): string { - return `You are a **style guide categorizer** designed to organize style guide rules into logical groups. + private buildFilteredRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { + return `You are a **style guide analyzer** designed to extract and categorize rules from style guides. ## Task -Analyze the provided style guide and identify up to **${this.options.maxCategories}** natural categories. - -## Category Types - -- **Subjective**: Requires judgment (tone, style, clarity) -- **Semi-objective**: Clear patterns but needs context (citations, evidence) -- **Objective**: Can be mechanically checked (formatting, word count) +Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** ## Output Requirements -- Use **PascalCase** for all category IDs -- Assign **priority** (1=highest, 10=lowest) based on quality impact -- Include **3-10 rules** per category -- Preserve original rule text and examples +- Create **exactly one** category that consolidates all related rules +- Use **PascalCase** for the category ID +- Classify as: **subjective**, **semi-objective**, or **objective** +- Include all semantically matching rules +## Guidelines +- Look for rules **related** to the topic, not just exact matches +- Consolidate similar rules into a cohesive category +- Preserve original rule text Style Guide: **${styleGuide.name}**`; } - private buildRulePrompt(category: CategoryExtractionOutput['categories'][0]): string { + private buildRuleGenerationPrompt(category: CategoryExtractionOutput['categories'][0]): string { return `You are an **evaluation prompt generator** designed to create content evaluation prompts. ## Task diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts index dfd797c..8d4cfac 100644 --- a/tests/integration/style-guide-conversion.test.ts +++ b/tests/integration/style-guide-conversion.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { StyleGuideProcessor } from '../../src/style-guide/style-guide-processor'; import { LLMProvider } from '../../src/providers/llm-provider'; -import type { ParsedStyleGuide, CategoryExtractionOutput, CategoryRuleGenerationOutput } from '../../src/schemas/style-guide-schemas'; +import type { ParsedStyleGuide, CategoryExtractionOutput, CategoryRuleGenerationOutput, TypeIdentificationOutput } from '../../src/schemas/style-guide-schemas'; import * as path from 'path'; import * as fs from 'fs'; @@ -14,7 +14,22 @@ class MockLLMProvider implements LLMProvider { promptText: string, schema: { name: string; schema: Record } ): Promise { - // Category extraction call + // 1. Type Identification Call + if (schema.name === 'typeIdentification') { + const typeResult: TypeIdentificationOutput = { + types: [ + { + type: 'subjective', + description: 'Tone and Voice rules', + ruleCount: 5, + rules: ['Use second person', 'Use active voice'] + } + ] + }; + return Promise.resolve(typeResult as T); + } + + // 2. Category Extraction Call if (schema.name === 'categoryExtraction') { const categoryResult: CategoryExtractionOutput = { categories: [ @@ -34,33 +49,54 @@ class MockLLMProvider implements LLMProvider { return Promise.resolve(categoryResult as T); } - // Category rule generation call - const ruleResult: CategoryRuleGenerationOutput = { - evaluationType: 'subjective', - categoryName: 'Voice & Tone', - promptBody: 'Evaluate the content for voice and tone adherence.', - criteria: [ - { - name: 'Second Person Voice', - id: 'SecondPersonVoice', - weight: 50, - rubric: [ - { score: 4, label: 'Excellent', description: 'Consistent second person usage' }, - { score: 1, label: 'Poor', description: 'No second person usage' } - ] - }, - { - name: 'Active Voice', - id: 'ActiveVoice', - weight: 50, - rubric: [ - { score: 4, label: 'Excellent', description: 'Strong active voice throughout' }, - { score: 1, label: 'Poor', description: 'Predominantly passive voice' } - ] - } - ] - }; - return Promise.resolve(ruleResult as T); + // 3. Category Rule Generation Call + if (schema.name === 'categoryRuleGeneration') { + const ruleResult: CategoryRuleGenerationOutput = { + evaluationType: 'subjective', + categoryName: 'Voice & Tone', + promptBody: 'Evaluate the content for voice and tone adherence.', + criteria: [ + { + name: 'Second Person Voice', + id: 'SecondPersonVoice', + weight: 50, + rubric: [ + { score: 4, label: 'Excellent', description: 'Consistent second person usage' }, + { score: 1, label: 'Poor', description: 'No second person usage' } + ] + }, + { + name: 'Active Voice', + id: 'ActiveVoice', + weight: 50, + rubric: [ + { score: 4, label: 'Excellent', description: 'Strong active voice throughout' }, + { score: 1, label: 'Poor', description: 'Predominantly passive voice' } + ] + } + ] + }; + return Promise.resolve(ruleResult as T); + } + + // Default or Single Rule Extraction + if (schema.name === 'singleRuleExtraction') { + const categoryResult: CategoryExtractionOutput = { + categories: [ + { + id: 'SingleRule', + name: 'Single Rule Category', + description: 'A single extracted rule', + type: 'semi-objective', + priority: 1, + rules: [{ description: 'Single rule description' }] + } + ] + }; + return Promise.resolve(categoryResult as T); + } + + return Promise.reject(new Error(`Unknown schema name: ${schema.name}`)); } } From afd717f2f0f5dc69355ea8c1ea10cb64e9f1c774 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Wed, 10 Dec 2025 12:05:33 +0100 Subject: [PATCH 30/32] Update convert rule to create directory for new rules --- src/cli/convert-command.ts | 43 ++++++++++-------------- src/style-guide/style-guide-processor.ts | 2 +- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index 3bb5e27..acc6294 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -10,17 +10,6 @@ import { loadConfig } from '../boundaries/config-loader'; import { handleUnknownError } from '../errors/index'; import { ConvertOptions } from '../schemas'; -/** - * Custom error class for convert command failures. - * Includes exit code for CLI handling. - */ -class ConvertCommandError extends Error { - constructor(message: string, public readonly exitCode: number = 1) { - super(message); - this.name = 'ConvertCommandError'; - } -} - export function registerConvertCommand(program: Command): void { program .command('convert') @@ -40,12 +29,10 @@ export function registerConvertCommand(program: Command): void { try { await executeConvert(styleGuidePath, rawOptions); } catch (e: unknown) { - if (e instanceof ConvertCommandError) { - console.error(`Error: ${e.message}`); - // Re-throw to let Commander handle the exit - throw e; - } - throw e; + + const err = handleUnknownError(e, 'execute convert'); + console.error(`Error: ${err.message}`); + throw err; } }); } @@ -57,12 +44,12 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom options = parseConvertOptions(rawOptions); } catch (e: unknown) { const err = handleUnknownError(e, 'Parsing CLI options'); - throw new ConvertCommandError(err.message); + throw new Error(err.message); } // 2. Validate input file if (!existsSync(styleGuidePath)) { - throw new ConvertCommandError(`Style guide file not found: ${styleGuidePath}`); + throw new Error(`Style guide file not found: ${styleGuidePath}`); } // 3. Load configuration & determine output directory @@ -84,10 +71,10 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom const err = handleUnknownError(e, 'Loading configuration'); console.error('Error: No output directory specified and failed to load vectorlint.ini.'); console.error(`Details: ${err.message}`); - throw new ConvertCommandError('Please either use -o/--output or create a valid vectorlint.ini.'); + throw new Error('Please either use -o/--output or create a valid vectorlint.ini.'); } const err = handleUnknownError(e, 'Loading configuration'); - throw new ConvertCommandError(err.message); + throw new Error(err.message); } if (options.verbose) { @@ -102,7 +89,7 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom } catch (e: unknown) { const err = handleUnknownError(e, 'Validating environment variables'); console.error('Please set these in your .env file or environment.'); - throw new ConvertCommandError(err.message); + throw new Error(err.message); } // 5. Load Directive & Initialize Provider @@ -138,6 +125,10 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom } // 7. Write Output + // Create subdirectory named after the style guide + const styleGuideName = path.basename(styleGuidePath, path.extname(styleGuidePath)); + const finalOutputDir = path.join(outputDir, styleGuideName); + if (options.dryRun) { console.log('\n--- DRY RUN PREVIEW ---\n'); for (const rule of rules) { @@ -146,17 +137,17 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom console.log(rule.content); console.log('---\n'); } - console.log(`[vectorlint] Would generate ${rules.length} files in ${outputDir}`); + console.log(`[vectorlint] Would generate ${rules.length} files in ${finalOutputDir}`); } else { - if (!existsSync(outputDir)) { - mkdirSync(outputDir, { recursive: true }); + if (!existsSync(finalOutputDir)) { + mkdirSync(finalOutputDir, { recursive: true }); } let writtenCount = 0; let skippedCount = 0; for (const rule of rules) { - const filePath = path.join(outputDir, rule.filename); + const filePath = path.join(finalOutputDir, rule.filename); if (existsSync(filePath) && !options.force) { if (options.verbose) { diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index b988766..daec668 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -283,7 +283,7 @@ Style Guide: **${styleGuide.name}**`; styleGuide: ParsedStyleGuide, typeInfo: TypeIdentificationOutput['types'][0] ): string { - return `You are a **rule categorizer** agent. + return `You are a **style guide rule categorizer** agent. ## Task Extract and categorize rules specifically for the **${typeInfo.type}** evaluation type. From a3773359141df7caca7a36dc56d6ba216dbdf53a Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 11 Dec 2025 11:08:49 +0100 Subject: [PATCH 31/32] feat: Add custom name option for output subdirectory in convert command --- src/cli/convert-command.ts | 6 ++++-- src/schemas/cli-schemas.ts | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts index acc6294..53f843e 100644 --- a/src/cli/convert-command.ts +++ b/src/cli/convert-command.ts @@ -16,6 +16,7 @@ export function registerConvertCommand(program: Command): void { .description('Convert a style guide into VectorLint evaluation prompts') .argument('', 'Path to the style guide file') .option('-o, --output ', 'Output directory for generated rules (defaults to RulesPath from config)') + .option('-n, --name ', 'Custom name for the output subdirectory') .option('-f, --format ', 'Input format: markdown, auto', 'auto') .option('-t, --template ', 'Custom template directory') .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') @@ -125,8 +126,9 @@ async function executeConvert(styleGuidePath: string, rawOptions: unknown): Prom } // 7. Write Output - // Create subdirectory named after the style guide - const styleGuideName = path.basename(styleGuidePath, path.extname(styleGuidePath)); + // 7. Write Output + // Create subdirectory named after the style guide OR custom name + const styleGuideName = options.name || path.basename(styleGuidePath, path.extname(styleGuidePath)); const finalOutputDir = path.join(outputDir, styleGuideName); if (options.dryRun) { diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 8a4cf48..48bd5f7 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -28,6 +28,7 @@ export const CONVERT_OPTIONS_SCHEMA = z.object({ force: z.boolean().default(false), dryRun: z.boolean().default(false), verbose: z.boolean().default(false), + name: z.string().optional(), }); // Inferred types From 8fc0c45b67bc1e8ea823b852b80a982f953569a3 Mon Sep 17 00:00:00 2001 From: Ayomide Date: Thu, 11 Dec 2025 11:17:12 +0100 Subject: [PATCH 32/32] refactor: Improve prompt formatting in style guide processing for better readability --- src/style-guide/style-guide-processor.ts | 112 ++++++++++++----------- 1 file changed, 58 insertions(+), 54 deletions(-) diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts index daec668..cfeb68f 100644 --- a/src/style-guide/style-guide-processor.ts +++ b/src/style-guide/style-guide-processor.ts @@ -263,20 +263,21 @@ export class StyleGuideProcessor { private buildTypeIdentificationPrompt(styleGuide: ParsedStyleGuide): string { return `You are a **style guide planning agent**. -## Task -Analyze the style guide and identify which evaluation types are present. + ## Task + Analyze the style guide and identify which evaluation types are present. -## Evaluation Types -- **objective**: Formatting, structure, casing, specific disallowed words. -- **semi-objective**: Grammar, spelling, specific prohibited patterns, clear violations. -- **subjective**: Tone, voice, clarity, audience, engagement, flow. + ## Evaluation Types + - **objective**: Formatting, structure, casing, specific disallowed words. + - **semi-objective**: Grammar, spelling, specific prohibited patterns, clear violations. + - **subjective**: Tone, voice, clarity, audience, engagement, flow. -## Output Requirements -- Identify **all** applicable types found in the content -- Estimate the number of rules for each type -- Provide the raw text of the rules belonging to each type + ## Output Requirements + - Identify **all** applicable types found in the content + - Estimate the number of rules for each type + - Provide the raw text of the rules belonging to each type -Style Guide: **${styleGuide.name}**`; + Style Guide: **${styleGuide.name}** + `; } private buildCategoryExtractionPrompt( @@ -285,76 +286,79 @@ Style Guide: **${styleGuide.name}**`; ): string { return `You are a **style guide rule categorizer** agent. -## Task -Extract and categorize rules specifically for the **${typeInfo.type}** evaluation type. + ## Task + Extract and categorize rules specifically for the **${typeInfo.type}** evaluation type. -## Context -${typeInfo.description} -Raw Rules Text: -${typeInfo.rules.join('\n\n')} + ## Context + ${typeInfo.description} + Raw Rules Text: + ${typeInfo.rules.join('\n\n')} -## Type: ${typeInfo.type} -${typeInfo.type === 'subjective' ? '- Focus on tone, voice, clarity' : ''} -${typeInfo.type === 'semi-objective' ? '- Focus on repeatable patterns and clear violations' : ''} -${typeInfo.type === 'objective' ? '- Focus on formatting, structure, and existence checks' : ''} + ## Type: ${typeInfo.type} + ${typeInfo.type === 'subjective' ? '- Focus on tone, voice, clarity' : ''} + ${typeInfo.type === 'semi-objective' ? '- Focus on repeatable patterns and clear violations' : ''} + ${typeInfo.type === 'objective' ? '- Focus on formatting, structure, and existence checks' : ''} -## Output Requirements -- Create logical categories for these rules (e.g., "Voice & Tone", "Grammar", "Formatting") -- Use **PascalCase** for IDs -- Group related rules together (3-10 rules per category) -- Preserve original instructions + ## Output Requirements + - Create logical categories for these rules (e.g., "Voice & Tone", "Grammar", "Formatting") + - Use **PascalCase** for IDs + - Group related rules together (3-10 rules per category) + - Preserve original instructions -Style Guide: **${styleGuide.name}**`; + Style Guide: **${styleGuide.name}**` + ; } private buildFilteredRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { return `You are a **style guide analyzer** designed to extract and categorize rules from style guides. -## Task + ## Task -Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** + Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** -## Output Requirements + ## Output Requirements -- Create **exactly one** category that consolidates all related rules -- Use **PascalCase** for the category ID -- Classify as: **subjective**, **semi-objective**, or **objective** -- Include all semantically matching rules + - Create **exactly one** category that consolidates all related rules + - Use **PascalCase** for the category ID + - Classify as: **subjective**, **semi-objective**, or **objective** + - Include all semantically matching rules -## Guidelines + ## Guidelines -- Look for rules **related** to the topic, not just exact matches -- Consolidate similar rules into a cohesive category -- Preserve original rule text + - Look for rules **related** to the topic, not just exact matches + - Consolidate similar rules into a cohesive category + - Preserve original rule text -Style Guide: **${styleGuide.name}**`; + Style Guide: **${styleGuide.name}** + `; } private buildRuleGenerationPrompt(category: CategoryExtractionOutput['categories'][0]): string { return `You are an **evaluation prompt generator** designed to create content evaluation prompts. -## Task + ## Task -Create a comprehensive evaluation prompt for the **"${category.name}"** category. + Create a comprehensive evaluation prompt for the **"${category.name}"** category. -## Category Details + ## Category Details -- **Name**: ${category.name} -- **Type**: ${category.type} -- **Description**: ${category.description} -- **Strictness**: ${this.options.strictness} + - **Name**: ${category.name} + - **Type**: ${category.type} + - **Description**: ${category.description} + - **Strictness**: ${this.options.strictness} -## Rules to Evaluate + ## Rules to Evaluate -${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} + ${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} -## Output Requirements + ## Output Requirements -- Each rule becomes a **separate criterion** with its own weight -- Use **PascalCase** for all criterion IDs -- Total weight must sum to **100** -${category.type === 'subjective' ? '- Create **1-4 rubric levels** for each criterion' : '- Provide **pass/fail** guidance for each criterion'} -- Include examples from rules when available`; + - Each rule becomes a **separate criterion** with its own weight + - Use **PascalCase** for all criterion IDs + - Total weight must sum to **100** + ${category.type === 'subjective' ? '- Create **1-4 rubric levels** for each criterion' : '- Provide **pass/fail** guidance for each criterion'} + - Include examples from rules when available + `; } // --- Helpers ---