diff --git a/package-lock.json b/package-lock.json index d837d89..71e2fdf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,11 +16,13 @@ "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", + "handlebars": "^4.7.8", "micromatch": "^4.0.5", "openai": "^4.0.0", "strip-ansi": "^7.1.0", "yaml": "^2.5.0", - "zod": "^3.25.76" + "zod": "^3.25.76", + "zod-to-json-schema": "^3.25.0" }, "bin": { "veclint": "dist/index.js", @@ -3580,6 +3582,36 @@ "dev": true, "license": "MIT" }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/handlebars/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -4096,6 +4128,15 @@ "node": "*" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -4186,6 +4227,12 @@ "dev": true, "license": "MIT" }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -5438,6 +5485,19 @@ "dev": true, "license": "MIT" }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -6167,6 +6227,12 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", @@ -6295,6 +6361,15 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } } } } diff --git a/package.json b/package.json index 2e3c109..e91212a 100644 --- a/package.json +++ b/package.json @@ -58,11 +58,13 @@ "chalk": "^5.3.0", "commander": "^12.0.0", "fast-glob": "^3.3.2", + "handlebars": "^4.7.8", "micromatch": "^4.0.5", "openai": "^4.0.0", "strip-ansi": "^7.1.0", "yaml": "^2.5.0", - "zod": "^3.25.76" + "zod": "^3.25.76", + "zod-to-json-schema": "^3.25.0" }, "devDependencies": { "@eslint/js": "^9.37.0", @@ -81,4 +83,4 @@ "typescript-eslint": "^8.46.1", "vitest": "^2.0.0" } -} \ No newline at end of file +} diff --git a/src/boundaries/cli-parser.ts b/src/boundaries/cli-parser.ts index 6091336..2b74235 100644 --- a/src/boundaries/cli-parser.ts +++ b/src/boundaries/cli-parser.ts @@ -1,4 +1,4 @@ -import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions } from '../schemas/cli-schemas'; +import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, CONVERT_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions, type ConvertOptions } from '../schemas/cli-schemas'; import { ValidationError, handleUnknownError } from '../errors/index'; export function parseCliOptions(raw: unknown): CliOptions { @@ -26,3 +26,16 @@ export function parseValidateOptions(raw: unknown): ValidateOptions { throw new ValidationError(`Validate option parsing failed: ${err.message}`); } } + +export function parseConvertOptions(raw: unknown): ConvertOptions { + try { + return CONVERT_OPTIONS_SCHEMA.parse(raw); + } catch (e: unknown) { + if (e instanceof Error && 'issues' in e) { + // Zod error + throw new ValidationError(`Invalid convert options: ${e.message}`); + } + const err = handleUnknownError(e, 'Convert option parsing'); + throw new ValidationError(`Convert option parsing failed: ${err.message}`); + } +} diff --git a/src/cli/convert-command.ts b/src/cli/convert-command.ts new file mode 100644 index 0000000..53f843e --- /dev/null +++ b/src/cli/convert-command.ts @@ -0,0 +1,174 @@ +import type { Command } from 'commander'; +import { existsSync, mkdirSync, writeFileSync } from 'fs'; +import * as path from 'path'; +import { StyleGuideProcessor } from '../style-guide/style-guide-processor'; +import { createProvider } from '../providers/provider-factory'; +import { DefaultRequestBuilder } from '../providers/request-builder'; +import { loadDirective } from '../prompts/directive-loader'; +import { parseEnvironment, parseConvertOptions } from '../boundaries/index'; +import { loadConfig } from '../boundaries/config-loader'; +import { handleUnknownError } from '../errors/index'; +import { ConvertOptions } from '../schemas'; + +export function registerConvertCommand(program: Command): void { + program + .command('convert') + .description('Convert a style guide into VectorLint evaluation prompts') + .argument('', 'Path to the style guide file') + .option('-o, --output ', 'Output directory for generated rules (defaults to RulesPath from config)') + .option('-n, --name ', 'Custom name for the output subdirectory') + .option('-f, --format ', 'Input format: markdown, auto', 'auto') + .option('-t, --template ', 'Custom template directory') + .option('--strictness ', 'Strictness level: lenient, standard, strict', 'standard') + .option('--severity ', 'Default severity: error, warning', 'warning') + .option('--max-categories ', 'Limit to N most important categories (default: 10)', '10') + .option('--rule ', 'Generate only rule matching this name/keyword') + .option('--force', 'Overwrite existing files', false) + .option('--dry-run', 'Preview generated rules without writing files', false) + .option('-v, --verbose', 'Enable verbose logging', false) + .action(async (styleGuidePath: string, rawOptions: unknown) => { + try { + await executeConvert(styleGuidePath, rawOptions); + } catch (e: unknown) { + + const err = handleUnknownError(e, 'execute convert'); + console.error(`Error: ${err.message}`); + throw err; + } + }); +} + +async function executeConvert(styleGuidePath: string, rawOptions: unknown): Promise { + // 1. Parse CLI options + let options: ConvertOptions; + try { + options = parseConvertOptions(rawOptions); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Parsing CLI options'); + throw new Error(err.message); + } + + // 2. Validate input file + if (!existsSync(styleGuidePath)) { + throw new Error(`Style guide file not found: ${styleGuidePath}`); + } + + // 3. Load configuration & determine output directory + let config; + let outputDir = options.output; + + try { + // Determine output directory: CLI option > config RulesPath + config = loadConfig(process.cwd()); + + if (!outputDir) { + outputDir = config.rulesPath; + if (options.verbose) { + console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`); + } + } + } catch (e: unknown) { + if (!outputDir) { + const err = handleUnknownError(e, 'Loading configuration'); + console.error('Error: No output directory specified and failed to load vectorlint.ini.'); + console.error(`Details: ${err.message}`); + throw new Error('Please either use -o/--output or create a valid vectorlint.ini.'); + } + const err = handleUnknownError(e, 'Loading configuration'); + throw new Error(err.message); + } + + if (options.verbose) { + console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`); + console.log(`[vectorlint] Output directory: ${outputDir}`); + } + + // 4. Parse Environment + let env; + try { + env = parseEnvironment(); + } catch (e: unknown) { + const err = handleUnknownError(e, 'Validating environment variables'); + console.error('Please set these in your .env file or environment.'); + throw new Error(err.message); + } + + // 5. Load Directive & Initialize Provider + const directive = loadDirective(); + const provider = createProvider( + env, + { debug: options.verbose }, + new DefaultRequestBuilder(directive) + ); + + // 6. Process Style Guide + if (options.verbose) { + console.log(`[vectorlint] Processing style guide...`); + console.log(`[vectorlint] Using ${env.LLM_PROVIDER}...`); + } + + const processor = new StyleGuideProcessor({ + llmProvider: provider, + maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10, + filterRule: options.rule, + templateDir: options.template || undefined, + defaultSeverity: options.severity, + strictness: options.strictness, + verbose: options.verbose, + }); + + const categoryRules = await processor.processFile(styleGuidePath); + const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content })); + + if (rules.length === 0) { + console.warn('[vectorlint] No rules were generated. Check your style guide format.'); + return; // Exit gracefully with success since no error occurred + } + + // 7. Write Output + // 7. Write Output + // Create subdirectory named after the style guide OR custom name + const styleGuideName = options.name || path.basename(styleGuidePath, path.extname(styleGuidePath)); + const finalOutputDir = path.join(outputDir, styleGuideName); + + if (options.dryRun) { + console.log('\n--- DRY RUN PREVIEW ---\n'); + for (const rule of rules) { + console.log(`File: ${rule.filename}`); + console.log('---'); + console.log(rule.content); + console.log('---\n'); + } + console.log(`[vectorlint] Would generate ${rules.length} files in ${finalOutputDir}`); + } else { + if (!existsSync(finalOutputDir)) { + mkdirSync(finalOutputDir, { recursive: true }); + } + + let writtenCount = 0; + let skippedCount = 0; + + for (const rule of rules) { + const filePath = path.join(finalOutputDir, rule.filename); + + if (existsSync(filePath) && !options.force) { + if (options.verbose) { + console.warn(`[vectorlint] Skipping existing file: ${filePath} (use --force to overwrite)`); + } + skippedCount++; + continue; + } + + writeFileSync(filePath, rule.content, 'utf-8'); + writtenCount++; + if (options.verbose) { + console.log(`[vectorlint] Wrote: ${filePath}`); + } + } + + console.log(`\n[vectorlint] Successfully generated ${writtenCount} evaluation files.`); + if (skippedCount > 0) { + console.log(`[vectorlint] Skipped ${skippedCount} existing files.`); + } + } +} diff --git a/src/cli/validate-command.ts b/src/cli/validate-command.ts index 4ce7012..e00d169 100644 --- a/src/cli/validate-command.ts +++ b/src/cli/validate-command.ts @@ -20,7 +20,7 @@ export function registerValidateCommand(program: Command): void { program .command('validate') .description('Validate prompt configuration files') - .option('--evals ', 'override evals directory') + .option('--rules ', 'override rules directory') .action(async (rawOpts: unknown) => { // Parse and validate command options let validateOptions; @@ -33,7 +33,7 @@ export function registerValidateCommand(program: Command): void { } // Determine rules path (from option or config) - let rulesPath = validateOptions.evals; + let rulesPath = validateOptions.rules; if (!rulesPath) { try { rulesPath = loadConfig().rulesPath; diff --git a/src/index.ts b/src/index.ts index dc1d8f2..c91f224 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ import * as path from "path"; import { handleUnknownError } from "./errors/index"; import { registerValidateCommand } from "./cli/validate-command"; import { registerMainCommand } from "./cli/commands"; +import { registerConvertCommand } from "./cli/convert-command"; // Import evaluators module to trigger self-registration of all evaluators import "./evaluators/index"; @@ -62,6 +63,7 @@ program // Register commands registerValidateCommand(program); registerMainCommand(program); +registerConvertCommand(program); // Parse command line arguments program.parse(); diff --git a/src/schemas/cli-schemas.ts b/src/schemas/cli-schemas.ts index 753f7bf..48bd5f7 100644 --- a/src/schemas/cli-schemas.ts +++ b/src/schemas/cli-schemas.ts @@ -7,15 +7,31 @@ export const CLI_OPTIONS_SCHEMA = z.object({ showPromptTrunc: z.boolean().default(false), debugJson: z.boolean().default(false), output: z.enum(['line', 'json', 'vale-json', 'JSON']).default('line'), - evals: z.string().optional(), + rules: z.string().optional(), config: z.string().optional(), }); // Validate command options schema export const VALIDATE_OPTIONS_SCHEMA = z.object({ - evals: z.string().optional(), + rules: z.string().optional(), +}); + +// Convert command options schema +export const CONVERT_OPTIONS_SCHEMA = z.object({ + output: z.string().optional(), + format: z.string().default('auto'), + template: z.string().optional(), + strictness: z.enum(['lenient', 'standard', 'strict']).default('standard'), + severity: z.enum(['error', 'warning']).default('warning'), + maxCategories: z.string().optional().default('10'), + rule: z.string().optional(), + force: z.boolean().default(false), + dryRun: z.boolean().default(false), + verbose: z.boolean().default(false), + name: z.string().optional(), }); // Inferred types export type CliOptions = z.infer; export type ValidateOptions = z.infer; +export type ConvertOptions = z.infer; diff --git a/src/schemas/style-guide-schemas.ts b/src/schemas/style-guide-schemas.ts new file mode 100644 index 0000000..a546256 --- /dev/null +++ b/src/schemas/style-guide-schemas.ts @@ -0,0 +1,112 @@ +import { z } from 'zod'; + +export const STYLE_GUIDE_EXAMPLES_SCHEMA = z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), +}).strict(); + +export const STYLE_GUIDE_RULE_SCHEMA = z.object({ + id: z.string(), + category: z.string().optional(), + description: z.string(), + severity: z.enum(['error', 'warning']).optional(), + examples: STYLE_GUIDE_EXAMPLES_SCHEMA.optional(), + weight: z.number().positive().optional(), + metadata: z.record(z.unknown()).optional(), +}).strict(); + +export const STYLE_GUIDE_SCHEMA = z.object({ + name: z.string(), + content: z.string(), // Raw markdown content for LLM processing +}).strict(); + + +/** + * Schema for generating a category-level evaluation prompt + * This handles multiple related rules in a single rule + */ +export const CATEGORY_RULE_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective', 'objective']), + categoryName: z.string().describe('The category this eval covers'), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string().describe('Criterion name (usually the rule description)'), + id: z.string().describe('PascalCase criterion ID (e.g., "VoiceSecondPersonPreferred")'), + weight: z.number().positive().describe('Weight of this criterion in overall score'), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional().describe('Rubric for subjective criteria'), + })), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + + + +/** + * Schema for the first step: Identifying evaluation types and their descriptions + */ +export const TYPE_IDENTIFICATION_SCHEMA = z.object({ + types: z.array(z.object({ + type: z.enum(['objective', 'semi-objective', 'subjective']), + description: z.string().describe('Description of the rule type'), + ruleCount: z.number().int().describe('Estimated number of rules for this type'), + rules: z.array(z.string()).describe('Raw text of the rules belonging to this type'), + })), +}); + +/** + * Schema for extracting and categorizing rules from a style guide + */ +export const CATEGORY_EXTRACTION_SCHEMA = z.object({ + categories: z.array(z.object({ + name: z.string().describe('Category name (e.g., "Voice & Tone", "Evidence & Citations")'), + id: z.string().describe('PascalCase category ID (e.g., "VoiceTone")'), + type: z.enum(['subjective', 'semi-objective', 'objective']).describe('Evaluation type for this category'), + description: z.string().describe('Brief description of what this category covers'), + rules: z.array(z.object({ + description: z.string().describe('The rule text from the style guide'), + severity: z.enum(['error', 'warning']).optional().describe('Suggested severity level'), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), + })), + priority: z.number().int().min(1).max(10).describe('Priority level (1=highest, 10=lowest)'), + })), +}); + +/** + * Schema for the LLM output when generating an evaluation prompt + */ +export const RULE_GENERATION_SCHEMA = z.object({ + evaluationType: z.enum(['subjective', 'semi-objective']), + promptBody: z.string().describe('The main instruction for the LLM evaluator'), + criteria: z.array(z.object({ + name: z.string(), + id: z.string(), + weight: z.number().positive(), + rubric: z.array(z.object({ + score: z.number().int().min(1).max(4), + label: z.string(), + description: z.string(), + })).optional(), + })).optional(), + examples: z.object({ + good: z.array(z.string()).optional(), + bad: z.array(z.string()).optional(), + }).optional(), +}); + +export type RuleGenerationOutput = z.infer; +export type CategoryRuleGenerationOutput = z.infer; +export type StyleGuideExamples = z.infer; +export type StyleGuideRule = z.infer; +export type ParsedStyleGuide = z.infer; +export type CategoryExtractionOutput = z.infer; +export type TypeIdentificationOutput = z.infer; + diff --git a/src/style-guide/style-guide-processor.ts b/src/style-guide/style-guide-processor.ts new file mode 100644 index 0000000..cfeb68f --- /dev/null +++ b/src/style-guide/style-guide-processor.ts @@ -0,0 +1,396 @@ +import { readFileSync } from 'fs'; +import * as path from 'path'; +import { LLMProvider } from '../providers/llm-provider'; +import { + ParsedStyleGuide, + STYLE_GUIDE_SCHEMA, + CATEGORY_EXTRACTION_SCHEMA, + CategoryExtractionOutput, + CATEGORY_RULE_GENERATION_SCHEMA, + CategoryRuleGenerationOutput, + TYPE_IDENTIFICATION_SCHEMA, + TypeIdentificationOutput +} from '../schemas/style-guide-schemas'; +import { ProcessingError, ConfigError, ValidationError } from '../errors/index'; +import { zodToJsonSchema } from 'zod-to-json-schema'; +import { TemplateRenderer } from './template-renderer'; +import { GeneratedCategoryRule, StyleGuideProcessorOptions, ResolvedProcessorOptions } from './types'; + +export class StyleGuideProcessor { + private llmProvider: LLMProvider; + private options: ResolvedProcessorOptions; + private renderer: TemplateRenderer; + + constructor(options: StyleGuideProcessorOptions) { + this.llmProvider = options.llmProvider; + this.renderer = new TemplateRenderer(options.templateDir); + this.options = { + maxCategories: options.maxCategories ?? 10, + verbose: options.verbose ?? false, + defaultSeverity: options.defaultSeverity ?? 'warning', + strictness: options.strictness ?? 'standard', + filterRule: options.filterRule, + templateDir: options.templateDir, + }; + } + + /** + * Process a style guide file: Read, extract categories, and generate rules + */ + public async processFile(filePath: string): Promise { + const styleGuide = this.readStyleGuide(filePath); + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Loaded style guide: ${styleGuide.name}`); + } + + return this.process(styleGuide); + } + + /** + * Read a style guide file and return parsed content + */ + private readStyleGuide(filePath: string): ParsedStyleGuide { + // Validate file extension + const ext = path.extname(filePath).toLowerCase(); + if (ext !== '.md' && ext !== '.markdown') { + throw new ConfigError(`Unsupported format: ${ext}. Only .md or .markdown files are supported.`); + } + + try { + const content = readFileSync(filePath, 'utf-8'); + const name = path.basename(filePath, path.extname(filePath)); + + const result: ParsedStyleGuide = { name, content }; + + STYLE_GUIDE_SCHEMA.parse(result); + + return result; + } catch (error) { + if (error instanceof ConfigError || error instanceof ValidationError) { + throw error; + } + const err = error instanceof Error ? error : new Error(String(error)); + throw new ProcessingError(`Failed to read style guide: ${err.message}`); + } + } + + /** + * Process a style guide: Identify types, extract categories, and generate rules + */ + public async process(styleGuide: ParsedStyleGuide): Promise { + // Handle single rule extraction separately + if (this.options.filterRule) { + return this.processSingleRule(styleGuide); + } + + // 1. Identify Types Strategy (Planner Role) + const typeIdentification = await this.identifyTypes(styleGuide); + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Identified ${typeIdentification.types.length} evaluation types`); + } + + const allCategories: CategoryExtractionOutput['categories'] = []; + + // 2. Extract Categories for each type (Organizer Role) + for (const typeInfo of typeIdentification.types) { + const categories = await this.extractCategoriesForType(styleGuide, typeInfo); + allCategories.push(...categories.categories); + } + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Extracted ${allCategories.length} total categories`); + } + + // 3. Generate Rules (Author Role) + return this.generateCategoryRules({ categories: allCategories }); + } + + /** + * Process a single rule based on filter term + */ + private async processSingleRule(styleGuide: ParsedStyleGuide): Promise { + const filterTerm = this.options.filterRule!; + + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Processing single rule filter: "${filterTerm}"`); + } + + const extractionOutput = await this.extractSingleRule(styleGuide, filterTerm); + return this.generateCategoryRules(extractionOutput); + } + + /** + * Step 1: Identify evaluation types present in the style guide + */ + private async identifyTypes(styleGuide: ParsedStyleGuide): Promise { + const prompt = this.buildTypeIdentificationPrompt(styleGuide); + + try { + const schemaJson = zodToJsonSchema(TYPE_IDENTIFICATION_SCHEMA); + + return await this.llmProvider.runPromptStructured( + JSON.stringify(styleGuide), + prompt, + { + name: 'typeIdentification', + schema: schemaJson as Record + } + ); + } catch (error) { + throw new ProcessingError(`Type identification failed: ${(error as Error).message}`); + } + } + + /** + * Step 2: Extract categories for a specific evaluation type + */ + private async extractCategoriesForType( + styleGuide: ParsedStyleGuide, + typeInfo: TypeIdentificationOutput['types'][0] + ): Promise { + const prompt = this.buildCategoryExtractionPrompt(styleGuide, typeInfo); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(styleGuide), + prompt, + { + name: 'categoryExtraction', + schema: schemaJson as Record + } + ); + + // Ensure extracted categories match the requested type + result.categories.forEach(cat => cat.type = typeInfo.type); + + return result; + } catch (error) { + console.warn(`Category extraction failed for type ${typeInfo.type}:`, error); + return { categories: [] }; + } + } + + + private async extractSingleRule(styleGuide: ParsedStyleGuide, filterTerm: string): Promise { + const prompt = this.buildFilteredRulePrompt(styleGuide, filterTerm); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_EXTRACTION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(styleGuide), + prompt, + { + name: 'singleRuleExtraction', + schema: schemaJson as Record + } + ); + + if (result.categories.length === 0) { + throw new ProcessingError( + `LLM could not find rules related to "${filterTerm}" in the style guide` + ); + } + + // Take only the first category if multiple returned + return { categories: [result.categories[0]!] }; + + } catch (error) { + if (error instanceof ProcessingError) throw error; + throw new ProcessingError( + `Single rule extraction failed: ${(error as Error).message}` + ); + } + } + + /** + * Generate category-level rules from extracted categories + */ + private async generateCategoryRules( + categories: CategoryExtractionOutput + ): Promise { + const rules: GeneratedCategoryRule[] = []; + let completed = 0; + + for (const category of categories.categories) { + try { + const generatedEval = await this.generateCategoryRule(category); + rules.push(generatedEval); + completed++; + if (this.options.verbose) { + console.log(`[StyleGuideProcessor] Progress: ${completed}/${categories.categories.length} categories processed`); + } + } catch (error) { + console.error(`Failed to generate eval for category ${category.id}:`, error); + // Continue with other categories + } + } + + return rules; + } + + private async generateCategoryRule( + category: CategoryExtractionOutput['categories'][0] + ): Promise { + const prompt = this.buildRuleGenerationPrompt(category); + + try { + const schemaJson = zodToJsonSchema(CATEGORY_RULE_GENERATION_SCHEMA); + + const result = await this.llmProvider.runPromptStructured( + JSON.stringify(category), + prompt, + { + name: 'categoryRuleGeneration', + schema: schemaJson as Record + } + ); + + return this.formatCategoryRule(category, result); + } catch (error) { + throw new ProcessingError( + `Category rule generation failed for ${category.id}: ${(error as Error).message}` + ); + } + } + + // --- Prompt Builders --- + + private buildTypeIdentificationPrompt(styleGuide: ParsedStyleGuide): string { + return `You are a **style guide planning agent**. + + ## Task + Analyze the style guide and identify which evaluation types are present. + + ## Evaluation Types + - **objective**: Formatting, structure, casing, specific disallowed words. + - **semi-objective**: Grammar, spelling, specific prohibited patterns, clear violations. + - **subjective**: Tone, voice, clarity, audience, engagement, flow. + + ## Output Requirements + - Identify **all** applicable types found in the content + - Estimate the number of rules for each type + - Provide the raw text of the rules belonging to each type + + Style Guide: **${styleGuide.name}** + `; + } + + private buildCategoryExtractionPrompt( + styleGuide: ParsedStyleGuide, + typeInfo: TypeIdentificationOutput['types'][0] + ): string { + return `You are a **style guide rule categorizer** agent. + + ## Task + Extract and categorize rules specifically for the **${typeInfo.type}** evaluation type. + + ## Context + ${typeInfo.description} + Raw Rules Text: + ${typeInfo.rules.join('\n\n')} + + ## Type: ${typeInfo.type} + ${typeInfo.type === 'subjective' ? '- Focus on tone, voice, clarity' : ''} + ${typeInfo.type === 'semi-objective' ? '- Focus on repeatable patterns and clear violations' : ''} + ${typeInfo.type === 'objective' ? '- Focus on formatting, structure, and existence checks' : ''} + + ## Output Requirements + - Create logical categories for these rules (e.g., "Voice & Tone", "Grammar", "Formatting") + - Use **PascalCase** for IDs + - Group related rules together (3-10 rules per category) + - Preserve original instructions + + Style Guide: **${styleGuide.name}**` + ; + } + + private buildFilteredRulePrompt(styleGuide: ParsedStyleGuide, filterTerm: string): string { + return `You are a **style guide analyzer** designed to extract and categorize rules from style guides. + + ## Task + + Analyze the provided style guide and extract all rules related to: **"${filterTerm}"** + + ## Output Requirements + + - Create **exactly one** category that consolidates all related rules + - Use **PascalCase** for the category ID + - Classify as: **subjective**, **semi-objective**, or **objective** + - Include all semantically matching rules + + ## Guidelines + + - Look for rules **related** to the topic, not just exact matches + - Consolidate similar rules into a cohesive category + - Preserve original rule text + + Style Guide: **${styleGuide.name}** + `; + } + + private buildRuleGenerationPrompt(category: CategoryExtractionOutput['categories'][0]): string { + return `You are an **evaluation prompt generator** designed to create content evaluation prompts. + + ## Task + + Create a comprehensive evaluation prompt for the **"${category.name}"** category. + + ## Category Details + + - **Name**: ${category.name} + - **Type**: ${category.type} + - **Description**: ${category.description} + - **Strictness**: ${this.options.strictness} + + ## Rules to Evaluate + + ${category.rules.map((r, i) => `${i + 1}. ${r.description}`).join('\n')} + + ## Output Requirements + + - Each rule becomes a **separate criterion** with its own weight + - Use **PascalCase** for all criterion IDs + - Total weight must sum to **100** + ${category.type === 'subjective' ? '- Create **1-4 rubric levels** for each criterion' : '- Provide **pass/fail** guidance for each criterion'} + - Include examples from rules when available + `; + } + + // --- Helpers --- + + private formatCategoryRule( + category: CategoryExtractionOutput['categories'][0], + output: CategoryRuleGenerationOutput + ): GeneratedCategoryRule { + const defaultSeverity = this.options.defaultSeverity || 'warning'; + + // Helpers for ID formatting + const toKebabCase = (str: string) => str + .replace(/([a-z])([A-Z])/g, '$1-$2') + .replace(/[\s_]+/g, '-') + .toLowerCase(); + + // Use TemplateRenderer + const context = this.renderer.createCategoryContext(category, output, defaultSeverity); + const content = this.renderer.render('base-template.md', context); + + // Ensure filename is kebab-case even if ID is PascalCase + const filenameId = toKebabCase(category.id); + + return { + filename: `${filenameId}.md`, + content, + meta: { + id: category.id, + name: category.name, + categoryType: category.type, + ruleCount: category.rules.length, + } + }; + } +} diff --git a/src/style-guide/template-renderer.ts b/src/style-guide/template-renderer.ts new file mode 100644 index 0000000..e75d0a0 --- /dev/null +++ b/src/style-guide/template-renderer.ts @@ -0,0 +1,117 @@ +import Handlebars from 'handlebars'; +import { readFileSync, existsSync } from 'fs'; +import { join, dirname } from 'path'; +import { fileURLToPath } from 'url'; +import { + CategoryRuleGenerationOutput, + RuleGenerationOutput, + StyleGuideRule +} from '../schemas/style-guide-schemas'; +import { TemplateContext } from './types'; + +export class TemplateRenderer { + private templateDir: string; + + constructor(templateDir?: string) { + if (templateDir) { + this.templateDir = templateDir; + } else { + // ESM compatible current directory resolution + const currentFilePath = fileURLToPath(import.meta.url); + const currentDirPath = dirname(currentFilePath); + this.templateDir = join(currentDirPath, 'templates'); + } + this.registerHelpers(); + } + + /** + * Register Handlebars helpers + */ + private registerHelpers(): void { + Handlebars.registerHelper('uppercase', (str: string) => str.toUpperCase()); + Handlebars.registerHelper('lowercase', (str: string) => str.toLowerCase()); + } + + /** + * Render a template with the given context + */ + public render(templateName: string, context: TemplateContext): string { + const templatePath = join(this.templateDir, templateName); + + if (!existsSync(templatePath)) { + throw new Error(`Template not found: ${templatePath}`); + } + + const templateContent = readFileSync(templatePath, 'utf-8'); + const template = Handlebars.compile(templateContent); + + return template(context); + } + + /** + * Create context from rule and LLM output + */ + public createContext( + rule: StyleGuideRule, + output: RuleGenerationOutput, + defaultSeverity: string + ): TemplateContext { + const severity = rule.severity || defaultSeverity; + const ruleName = rule.id + .replace(/-/g, ' ') + .replace(/\b\w/g, (l) => l.toUpperCase()); + + return { + EVALUATION_TYPE: output.evaluationType, + RULE_ID: rule.id, + RULE_NAME: ruleName, + SEVERITY: severity, + PROMPT_BODY: output.promptBody, + CRITERIA: output.criteria?.map(c => ({ + name: c.name, + id: c.id, + weight: c.weight + })), + RUBRIC: this.buildRubricString(output.criteria) + }; + } + + /** + * Create context from category and LLM output + */ + public createCategoryContext( + category: { id: string; name: string }, + output: CategoryRuleGenerationOutput, + defaultSeverity: string + ): TemplateContext { + return { + EVALUATION_TYPE: output.evaluationType, + RULE_ID: category.id, + RULE_NAME: category.name, + SEVERITY: defaultSeverity, + PROMPT_BODY: `# ${category.name}\n\n${output.promptBody}`, + CRITERIA: output.criteria?.map(c => ({ + name: c.name, + id: c.id, + weight: c.weight + })), + RUBRIC: this.buildRubricString(output.criteria) + }; + } + + private buildRubricString(criteria?: Array<{ name: string; rubric?: Array<{ score: number; label: string; description: string }> | undefined }>): string { + let rubricStr = ''; + if (criteria) { + criteria.forEach(c => { + if (c.rubric) { + rubricStr += `## Rubric for ${c.name}\n\n`; + c.rubric.forEach(r => { + rubricStr += `- **${r.score} (${r.label})**: ${r.description}\n`; + }); + rubricStr += `\n`; + } + }); + } + return rubricStr.trim(); + } +} diff --git a/src/style-guide/templates/base-template.md b/src/style-guide/templates/base-template.md new file mode 100644 index 0000000..8f50b26 --- /dev/null +++ b/src/style-guide/templates/base-template.md @@ -0,0 +1,21 @@ +--- +evaluator: base +type: {{EVALUATION_TYPE}} +id: {{RULE_ID}} +name: {{RULE_NAME}} +severity: {{SEVERITY}} +{{#if CRITERIA}} +criteria: +{{#each CRITERIA}} + - name: {{name}} + id: {{id}} + weight: {{weight}} +{{/each}} +{{/if}} +--- + +{{PROMPT_BODY}} + +{{#if RUBRIC}} +{{RUBRIC}} +{{/if}} diff --git a/src/style-guide/types.ts b/src/style-guide/types.ts new file mode 100644 index 0000000..514e18b --- /dev/null +++ b/src/style-guide/types.ts @@ -0,0 +1,46 @@ +import { LLMProvider } from "../providers"; + +export interface StyleGuideProcessorOptions { + llmProvider: LLMProvider; + maxCategories?: number | undefined; + filterRule?: string | undefined; + templateDir?: string | undefined; + defaultSeverity?: 'error' | 'warning' | undefined; + strictness?: 'lenient' | 'standard' | 'strict' | undefined; + verbose?: boolean | undefined; +} + +export type ResolvedProcessorOptions = { + maxCategories: number; + verbose: boolean; + defaultSeverity: 'error' | 'warning'; + strictness: 'lenient' | 'standard' | 'strict'; + filterRule?: string | undefined; + templateDir?: string | undefined; +}; + +export interface GeneratedCategoryRule { + filename: string; + content: string; + meta: { + id: string; + name: string; + categoryType: string; + ruleCount: number; + }; +} + +export interface TemplateContext { + EVALUATION_TYPE: string; + RULE_ID: string; + RULE_NAME: string; + SEVERITY: string; + PROMPT_BODY: string; + CRITERIA?: Array<{ + name: string; + id: string; + weight: number; + }> | undefined; + RUBRIC?: string | undefined; + [key: string]: unknown; +} diff --git a/tests/cli/convert-command.test.ts b/tests/cli/convert-command.test.ts new file mode 100644 index 0000000..649f4ec --- /dev/null +++ b/tests/cli/convert-command.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Command } from 'commander'; +import { registerConvertCommand } from '../../src/cli/convert-command'; +import * as fs from 'fs'; + +// Mocks +vi.mock('fs'); +vi.mock('../../src/style-guide/style-guide-processor'); +vi.mock('../../src/providers/provider-factory'); +vi.mock('../../src/boundaries/index'); +vi.mock('../../src/prompts/directive-loader'); + +describe('convert command', () => { + let program: Command; + + beforeEach(() => { + program = new Command(); + vi.clearAllMocks(); + // Mock fs.existsSync to return true for style guide + vi.mocked(fs.existsSync).mockReturnValue(true); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should register convert command', () => { + registerConvertCommand(program); + const command = program.commands.find(c => c.name() === 'convert'); + expect(command).toBeDefined(); + expect(command?.description()).toContain('Convert a style guide'); + }); + + it('should have correct options', () => { + registerConvertCommand(program); + const command = program.commands.find(c => c.name() === 'convert'); + const options = command?.options.map(o => o.name()); + + expect(options).toContain('output'); + expect(options).toContain('format'); + expect(options).toContain('template'); + expect(options).toContain('strictness'); + expect(options).toContain('severity'); + expect(options).toContain('force'); + expect(options).toContain('dry-run'); + expect(options).toContain('verbose'); + }); +}); diff --git a/tests/integration/style-guide-conversion.test.ts b/tests/integration/style-guide-conversion.test.ts new file mode 100644 index 0000000..8d4cfac --- /dev/null +++ b/tests/integration/style-guide-conversion.test.ts @@ -0,0 +1,184 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { StyleGuideProcessor } from '../../src/style-guide/style-guide-processor'; +import { LLMProvider } from '../../src/providers/llm-provider'; +import type { ParsedStyleGuide, CategoryExtractionOutput, CategoryRuleGenerationOutput, TypeIdentificationOutput } from '../../src/schemas/style-guide-schemas'; +import * as path from 'path'; +import * as fs from 'fs'; + +/** + * Mock LLM Provider that returns structured responses for category extraction and rule generation + */ +class MockLLMProvider implements LLMProvider { + runPromptStructured( + content: string, + promptText: string, + schema: { name: string; schema: Record } + ): Promise { + // 1. Type Identification Call + if (schema.name === 'typeIdentification') { + const typeResult: TypeIdentificationOutput = { + types: [ + { + type: 'subjective', + description: 'Tone and Voice rules', + ruleCount: 5, + rules: ['Use second person', 'Use active voice'] + } + ] + }; + return Promise.resolve(typeResult as T); + } + + // 2. Category Extraction Call + if (schema.name === 'categoryExtraction') { + const categoryResult: CategoryExtractionOutput = { + categories: [ + { + id: 'VoiceTone', + name: 'Voice & Tone', + description: 'Guidelines for voice and tone', + type: 'subjective', + priority: 1, + rules: [ + { description: 'Write in second person' }, + { description: 'Use active voice' } + ] + } + ] + }; + return Promise.resolve(categoryResult as T); + } + + // 3. Category Rule Generation Call + if (schema.name === 'categoryRuleGeneration') { + const ruleResult: CategoryRuleGenerationOutput = { + evaluationType: 'subjective', + categoryName: 'Voice & Tone', + promptBody: 'Evaluate the content for voice and tone adherence.', + criteria: [ + { + name: 'Second Person Voice', + id: 'SecondPersonVoice', + weight: 50, + rubric: [ + { score: 4, label: 'Excellent', description: 'Consistent second person usage' }, + { score: 1, label: 'Poor', description: 'No second person usage' } + ] + }, + { + name: 'Active Voice', + id: 'ActiveVoice', + weight: 50, + rubric: [ + { score: 4, label: 'Excellent', description: 'Strong active voice throughout' }, + { score: 1, label: 'Poor', description: 'Predominantly passive voice' } + ] + } + ] + }; + return Promise.resolve(ruleResult as T); + } + + // Default or Single Rule Extraction + if (schema.name === 'singleRuleExtraction') { + const categoryResult: CategoryExtractionOutput = { + categories: [ + { + id: 'SingleRule', + name: 'Single Rule Category', + description: 'A single extracted rule', + type: 'semi-objective', + priority: 1, + rules: [{ description: 'Single rule description' }] + } + ] + }; + return Promise.resolve(categoryResult as T); + } + + return Promise.reject(new Error(`Unknown schema name: ${schema.name}`)); + } +} + +describe('Style Guide Conversion Integration', () => { + const fixturesDir = path.join(__dirname, '../style-guide/fixtures'); + const outputDir = path.join(__dirname, 'temp-rules'); + + beforeEach(() => { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + }); + + afterEach(() => { + // Clean up output directory + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true, force: true }); + } + }); + + it('should convert a markdown style guide to category-based rule files', async () => { + const styleGuidePath = path.join(fixturesDir, 'sample-style-guide.md'); + + // Skip test if fixture doesn't exist + if (!fs.existsSync(styleGuidePath)) { + console.log('[SKIP] sample-style-guide.md fixture not found'); + return; + } + + // 1. Create processor with mock LLM provider + const mockProvider = new MockLLMProvider(); + const processor = new StyleGuideProcessor({ + llmProvider: mockProvider, + maxCategories: 10, + defaultSeverity: 'warning', + verbose: false, + }); + + // 2. Process the style guide file directly + const rules = await processor.processFile(styleGuidePath); + + // Expect at least one category-based rule + expect(rules.length).toBeGreaterThan(0); + + // 3. Write Files + for (const rule of rules) { + const filePath = path.join(outputDir, rule.filename); + fs.writeFileSync(filePath, rule.content, 'utf-8'); + } + + // 4. Verify Files Exist + const files = fs.readdirSync(outputDir); + expect(files.length).toBe(rules.length); + + // 5. Verify Content + const firstFile = fs.readFileSync(path.join(outputDir, files[0]!), 'utf-8'); + expect(firstFile).toContain('evaluator: base'); + expect(firstFile).toContain('type: subjective'); + }); + + it('should process a style guide object with process method', async () => { + const mockProvider = new MockLLMProvider(); + const processor = new StyleGuideProcessor({ + llmProvider: mockProvider, + maxCategories: 5, + verbose: false, + }); + + // Create a ParsedStyleGuide object directly + const styleGuide: ParsedStyleGuide = { + name: 'Test Style Guide', + content: '# Test Guide\n\nUse second person (you/your).\nPrefer active voice over passive voice.' + }; + + // Process the style guide + const rules = await processor.process(styleGuide); + + expect(rules.length).toBe(1); + expect(rules[0]?.meta.id).toBe('VoiceTone'); + expect(rules[0]?.meta.name).toBe('Voice & Tone'); + expect(rules[0]?.meta.categoryType).toBe('subjective'); + expect(rules[0]?.meta.ruleCount).toBe(2); + expect(rules[0]?.filename).toBe('voice-tone.md'); + }); +}); diff --git a/tests/style-guide/fixtures/invalid.txt b/tests/style-guide/fixtures/invalid.txt new file mode 100644 index 0000000..55de3c9 --- /dev/null +++ b/tests/style-guide/fixtures/invalid.txt @@ -0,0 +1,3 @@ +# Invalid File + +This file has an unsupported extension for testing. diff --git a/tests/style-guide/fixtures/sample-style-guide.md b/tests/style-guide/fixtures/sample-style-guide.md new file mode 100644 index 0000000..632751f --- /dev/null +++ b/tests/style-guide/fixtures/sample-style-guide.md @@ -0,0 +1,28 @@ +# Acme Corp Writing Style Guide + +## Voice and Tone + +- Use active voice whenever possible +- Be conversational but professional +- Avoid jargon unless necessary for technical accuracy +- Write in second person ("you") to address readers directly + +## Terminology + +- Use "customer" not "client" +- Use "dashboard" not "control panel" +- Use "sign in" not "log in" +- Always capitalize product names: "Acme Platform", "Acme API" + +## Structure + +- Headings must clearly communicate value to the reader +- Paragraphs should be 3-4 sentences maximum +- Use bullet points for lists of 3 or more items +- Include code examples for technical content + +## Grammar and Mechanics + +- Use Oxford comma in lists +- Spell out numbers one through nine, use numerals for 10 and above +- Use sentence case for headings, not title case diff --git a/vectorlint.ini b/vectorlint.ini new file mode 100644 index 0000000..ed28c85 --- /dev/null +++ b/vectorlint.ini @@ -0,0 +1,20 @@ +PromptsPath=.github/evals +ScanPaths=[*.md] +Concurrency=4 +DefaultSeverity=warning + +# Optional: map prompts to files +[Prompts] +paths=["Default:prompts", "Blog:prompts/blog"] + +[Defaults] +include=["**/*.md"] +exclude=["archived/**"] + +[Directory:Blog] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] + +[Prompt:Headline] +include=["content/blog/**/*.md"] +exclude=["content/blog/drafts/**"] \ No newline at end of file