Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
f8414a0
Create types for style guide feature
ayo6706 Dec 4, 2025
ea314da
Create schemas for style guide feature
ayo6706 Dec 4, 2025
11d0cfa
Implement style guide error handling
ayo6706 Dec 4, 2025
454a652
feat: Add style guide parser, tests
ayo6706 Dec 4, 2025
fd02594
feat: introduce LLM-powered evaluation generation from style guide ru…
ayo6706 Dec 4, 2025
7fe7d1e
feat: Implement LLM-based evaluation generation for style guide rules…
ayo6706 Dec 4, 2025
553a1ab
Delete rule category enum
ayo6706 Dec 8, 2025
129ea2e
Implement eval generation schema
ayo6706 Dec 8, 2025
fdd42fa
Implement template renderer for rules
ayo6706 Dec 8, 2025
48684d3
Implement category schema
ayo6706 Dec 8, 2025
e13fd83
refactor: Update rule categorization to preserve extracted categories…
ayo6706 Dec 8, 2025
97c155f
refactor: Update import paths and enhance progress logging in EvalGen…
ayo6706 Dec 8, 2025
23829e4
feat: Implement StyleGuideProcessor for category extraction and evalu…
ayo6706 Dec 8, 2025
a65a22a
feat: Add convert command for style guide to VectorLint evaluation pr…
ayo6706 Dec 8, 2025
a4dd4f7
test: Add unit and integration tests for convert command and style gu…
ayo6706 Dec 8, 2025
1dbe1d9
refactor: Simplify content generation in EvalGenerator and StyleGuide…
ayo6706 Dec 8, 2025
36e2f8d
Merge branch 'main' into ft/style-guide
ayo6706 Dec 8, 2025
de78faa
refactor: Update output directory references and enhance category ID …
ayo6706 Dec 8, 2025
5f86842
feat: Implement convert options parsing and schema for convert command
ayo6706 Dec 8, 2025
aac9e02
refactor: Update error handling and import structure in style guide p…
ayo6706 Dec 8, 2025
4d1d855
refactor: Rename evals to rules in conversion and validation commands…
ayo6706 Dec 8, 2025
cd503ee
refactor: Update style guide schemas and processing logic to allow op…
ayo6706 Dec 8, 2025
203012d
refactor: Remove RuleGenerator and streamline style guide processing …
ayo6706 Dec 8, 2025
8320dd5
refactor: Simplify style guide parser by removing frontmatter handlin…
ayo6706 Dec 8, 2025
54487ae
refactor: Update style guide schemas and parser to streamline process…
ayo6706 Dec 8, 2025
da4b1b4
refactor: Remove StyleGuideParser and enhance StyleGuideProcessor to …
ayo6706 Dec 8, 2025
29fde91
refactor: Consolidate style guide types and remove unused interfaces …
ayo6706 Dec 8, 2025
9b18431
refactor: Remove deprecated schemas and consolidate evaluation genera…
ayo6706 Dec 8, 2025
91fb1e8
refactor: Enhance error handling in convert command and streamline pr…
ayo6706 Dec 9, 2025
2108b7c
refactor: Introduce type identification schema and enhance style guid…
ayo6706 Dec 9, 2025
afd717f
Update convert rule to create directory for new rules
ayo6706 Dec 10, 2025
a377335
feat: Add custom name option for output subdirectory in convert command
ayo6706 Dec 11, 2025
8fc0c45
refactor: Improve prompt formatting in style guide processing for bet…
ayo6706 Dec 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 76 additions & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,13 @@
"chalk": "^5.3.0",
"commander": "^12.0.0",
"fast-glob": "^3.3.2",
"handlebars": "^4.7.8",
"micromatch": "^4.0.5",
"openai": "^4.0.0",
"strip-ansi": "^7.1.0",
"yaml": "^2.5.0",
"zod": "^3.25.76"
"zod": "^3.25.76",
"zod-to-json-schema": "^3.25.0"
},
"devDependencies": {
"@eslint/js": "^9.37.0",
Expand All @@ -81,4 +83,4 @@
"typescript-eslint": "^8.46.1",
"vitest": "^2.0.0"
}
}
}
15 changes: 14 additions & 1 deletion src/boundaries/cli-parser.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions } from '../schemas/cli-schemas';
import { CLI_OPTIONS_SCHEMA, VALIDATE_OPTIONS_SCHEMA, CONVERT_OPTIONS_SCHEMA, type CliOptions, type ValidateOptions, type ConvertOptions } from '../schemas/cli-schemas';
import { ValidationError, handleUnknownError } from '../errors/index';

export function parseCliOptions(raw: unknown): CliOptions {
Expand Down Expand Up @@ -26,3 +26,16 @@ export function parseValidateOptions(raw: unknown): ValidateOptions {
throw new ValidationError(`Validate option parsing failed: ${err.message}`);
}
}

export function parseConvertOptions(raw: unknown): ConvertOptions {
try {
return CONVERT_OPTIONS_SCHEMA.parse(raw);
} catch (e: unknown) {
if (e instanceof Error && 'issues' in e) {
// Zod error
throw new ValidationError(`Invalid convert options: ${e.message}`);
}
const err = handleUnknownError(e, 'Convert option parsing');
throw new ValidationError(`Convert option parsing failed: ${err.message}`);
}
}
174 changes: 174 additions & 0 deletions src/cli/convert-command.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
import type { Command } from 'commander';
import { existsSync, mkdirSync, writeFileSync } from 'fs';
import * as path from 'path';
import { StyleGuideProcessor } from '../style-guide/style-guide-processor';
import { createProvider } from '../providers/provider-factory';
import { DefaultRequestBuilder } from '../providers/request-builder';
import { loadDirective } from '../prompts/directive-loader';
import { parseEnvironment, parseConvertOptions } from '../boundaries/index';
import { loadConfig } from '../boundaries/config-loader';
import { handleUnknownError } from '../errors/index';
import { ConvertOptions } from '../schemas';

export function registerConvertCommand(program: Command): void {
program
.command('convert')
.description('Convert a style guide into VectorLint evaluation prompts')
.argument('<style-guide-path>', 'Path to the style guide file')
.option('-o, --output <dir>', 'Output directory for generated rules (defaults to RulesPath from config)')
.option('-n, --name <name>', 'Custom name for the output subdirectory')
.option('-f, --format <format>', 'Input format: markdown, auto', 'auto')
.option('-t, --template <dir>', 'Custom template directory')
.option('--strictness <level>', 'Strictness level: lenient, standard, strict', 'standard')
.option('--severity <level>', 'Default severity: error, warning', 'warning')
.option('--max-categories <number>', 'Limit to N most important categories (default: 10)', '10')
.option('--rule <name>', 'Generate only rule matching this name/keyword')
.option('--force', 'Overwrite existing files', false)
.option('--dry-run', 'Preview generated rules without writing files', false)
.option('-v, --verbose', 'Enable verbose logging', false)
.action(async (styleGuidePath: string, rawOptions: unknown) => {
try {
await executeConvert(styleGuidePath, rawOptions);
} catch (e: unknown) {

const err = handleUnknownError(e, 'execute convert');
console.error(`Error: ${err.message}`);
throw err;
}
});
}

async function executeConvert(styleGuidePath: string, rawOptions: unknown): Promise<void> {
// 1. Parse CLI options
let options: ConvertOptions;
try {
options = parseConvertOptions(rawOptions);
} catch (e: unknown) {
const err = handleUnknownError(e, 'Parsing CLI options');
throw new Error(err.message);
}

// 2. Validate input file
if (!existsSync(styleGuidePath)) {
throw new Error(`Style guide file not found: ${styleGuidePath}`);
}

// 3. Load configuration & determine output directory
let config;
let outputDir = options.output;

try {
// Determine output directory: CLI option > config RulesPath
config = loadConfig(process.cwd());

if (!outputDir) {
outputDir = config.rulesPath;
if (options.verbose) {
console.log(`[vectorlint] Using RulesPath from config: ${outputDir}`);
}
}
} catch (e: unknown) {
if (!outputDir) {
const err = handleUnknownError(e, 'Loading configuration');
console.error('Error: No output directory specified and failed to load vectorlint.ini.');
console.error(`Details: ${err.message}`);
throw new Error('Please either use -o/--output or create a valid vectorlint.ini.');
}
const err = handleUnknownError(e, 'Loading configuration');
throw new Error(err.message);
}

if (options.verbose) {
console.log(`[vectorlint] Reading style guide from: ${styleGuidePath}`);
console.log(`[vectorlint] Output directory: ${outputDir}`);
}

// 4. Parse Environment
let env;
try {
env = parseEnvironment();
} catch (e: unknown) {
const err = handleUnknownError(e, 'Validating environment variables');
console.error('Please set these in your .env file or environment.');
throw new Error(err.message);
}

// 5. Load Directive & Initialize Provider
const directive = loadDirective();
const provider = createProvider(
env,
{ debug: options.verbose },
new DefaultRequestBuilder(directive)
);

// 6. Process Style Guide
if (options.verbose) {
console.log(`[vectorlint] Processing style guide...`);
console.log(`[vectorlint] Using ${env.LLM_PROVIDER}...`);
}

const processor = new StyleGuideProcessor({
llmProvider: provider,
maxCategories: options.maxCategories ? parseInt(options.maxCategories) : 10,
filterRule: options.rule,
templateDir: options.template || undefined,
defaultSeverity: options.severity,
strictness: options.strictness,
verbose: options.verbose,
});

const categoryRules = await processor.processFile(styleGuidePath);
const rules = categoryRules.map(e => ({ filename: e.filename, content: e.content }));

if (rules.length === 0) {
console.warn('[vectorlint] No rules were generated. Check your style guide format.');
return; // Exit gracefully with success since no error occurred
}

// 7. Write Output
// 7. Write Output
// Create subdirectory named after the style guide OR custom name
const styleGuideName = options.name || path.basename(styleGuidePath, path.extname(styleGuidePath));
const finalOutputDir = path.join(outputDir, styleGuideName);

if (options.dryRun) {
console.log('\n--- DRY RUN PREVIEW ---\n');
for (const rule of rules) {
console.log(`File: ${rule.filename}`);
console.log('---');
console.log(rule.content);
console.log('---\n');
}
console.log(`[vectorlint] Would generate ${rules.length} files in ${finalOutputDir}`);
} else {
if (!existsSync(finalOutputDir)) {
mkdirSync(finalOutputDir, { recursive: true });
}

let writtenCount = 0;
let skippedCount = 0;

for (const rule of rules) {
const filePath = path.join(finalOutputDir, rule.filename);

if (existsSync(filePath) && !options.force) {
if (options.verbose) {
console.warn(`[vectorlint] Skipping existing file: ${filePath} (use --force to overwrite)`);
}
skippedCount++;
continue;
}

writeFileSync(filePath, rule.content, 'utf-8');
writtenCount++;
if (options.verbose) {
console.log(`[vectorlint] Wrote: ${filePath}`);
}
}

console.log(`\n[vectorlint] Successfully generated ${writtenCount} evaluation files.`);
if (skippedCount > 0) {
console.log(`[vectorlint] Skipped ${skippedCount} existing files.`);
}
}
}
4 changes: 2 additions & 2 deletions src/cli/validate-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export function registerValidateCommand(program: Command): void {
program
.command('validate')
.description('Validate prompt configuration files')
.option('--evals <dir>', 'override evals directory')
.option('--rules <dir>', 'override rules directory')
.action(async (rawOpts: unknown) => {
// Parse and validate command options
let validateOptions;
Expand All @@ -33,7 +33,7 @@ export function registerValidateCommand(program: Command): void {
}

// Determine rules path (from option or config)
let rulesPath = validateOptions.evals;
let rulesPath = validateOptions.rules;
if (!rulesPath) {
try {
rulesPath = loadConfig().rulesPath;
Expand Down
2 changes: 2 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import * as path from "path";
import { handleUnknownError } from "./errors/index";
import { registerValidateCommand } from "./cli/validate-command";
import { registerMainCommand } from "./cli/commands";
import { registerConvertCommand } from "./cli/convert-command";

// Import evaluators module to trigger self-registration of all evaluators
import "./evaluators/index";
Expand Down Expand Up @@ -62,6 +63,7 @@ program
// Register commands
registerValidateCommand(program);
registerMainCommand(program);
registerConvertCommand(program);

// Parse command line arguments
program.parse();
Loading