From 0ed86d42e7cf31b7f71ef9a73d055267a4e17047 Mon Sep 17 00:00:00 2001 From: HiranoMasaaki Date: Sat, 3 Jan 2026 04:22:19 +0000 Subject: [PATCH 1/2] feat(create-expert): Generate Expert Ecosystems with Usability Focus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transform create-expert from generating single experts to generating expert ecosystems (main + demo + setup + doctor). - Add usability properties to property-extractor - Rename expert-builder to ecosystem-builder - Add usability-manager for PDCA on usability - Update expert-tester with usability stage - Update report-generator for ecosystems Closes #337 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .changeset/expert-ecosystems.md | 21 ++ .../src/lib/create-expert-toml.ts | 347 ++++++++++++++++-- 2 files changed, 335 insertions(+), 33 deletions(-) create mode 100644 .changeset/expert-ecosystems.md diff --git a/.changeset/expert-ecosystems.md b/.changeset/expert-ecosystems.md new file mode 100644 index 00000000..1d15316c --- /dev/null +++ b/.changeset/expert-ecosystems.md @@ -0,0 +1,21 @@ +--- +"create-expert": minor +--- + +feat(create-expert): Generate Expert Ecosystems with Usability Focus + +Transform create-expert from generating single experts to generating expert ecosystems: + +- **Main expert**: Core functionality +- **Demo expert**: Works immediately with sample data (no setup required) +- **Setup expert**: Automated configuration wizard (if external deps exist) +- **Doctor expert**: Troubleshooting assistant (if external deps exist) + +Changes: +- Add usability properties to property-extractor (Zero-Config, Setup-Automation, Error-Guidance) +- Rename expert-builder to ecosystem-builder with ecosystem generation templates +- Add usability-manager for PDCA cycle on usability properties +- Update expert-tester to handle usability stage testing +- Update report-generator with ecosystem-aware output + +Closes #337 diff --git a/apps/create-expert/src/lib/create-expert-toml.ts b/apps/create-expert/src/lib/create-expert-toml.ts index 316e8242..f5601bfd 100644 --- a/apps/create-expert/src/lib/create-expert-toml.ts +++ b/apps/create-expert/src/lib/create-expert-toml.ts @@ -18,10 +18,10 @@ Coordinate the Expert creation process by delegating to specialized Experts. ## Workflow 1. **Extract Properties**: Delegate to \`property-extractor\` with user requirements - - Get back: user properties + Perstack properties + - Get back: user properties + Perstack properties + usability properties + external dependencies -2. **Build Initial Expert**: Delegate to \`expert-builder\` with properties - - Get back: initial perstack.toml with Expert definition +2. **Build Expert Ecosystem**: Delegate to \`ecosystem-builder\` with properties + - Get back: perstack.toml with Expert ecosystem (main + demo + setup + doctor) 3. **Happy-Path PDCA**: Delegate to \`happy-path-manager\` - Runs PDCA until happy-path properties pass @@ -32,13 +32,19 @@ Coordinate the Expert creation process by delegating to specialized Experts. 5. **Adversarial PDCA**: Delegate to \`adversarial-manager\` - Tests prompt injection, path traversal, etc. -6. **Generate Report**: Delegate to \`report-generator\` - - Get back: final summary of all tests +6. **Usability PDCA**: Delegate to \`usability-manager\` + - Tests demo works without setup + - Tests setup automation completes successfully + - Tests error messages have actionable guidance + +7. **Generate Report**: Delegate to \`report-generator\` + - Get back: final summary of all tests including usability ## Important -- Pass context between delegates (properties, test results) +- Pass context between delegates (properties, test results, ecosystem info) - Each manager handles its own PDCA loop - You just orchestrate the flow +- The ecosystem should be immediately usable by fresh users ` const PROPERTY_EXTRACTOR_INSTRUCTION = `You extract testable properties from user requirements. @@ -57,45 +63,215 @@ Return a structured list of properties: 4. Error Handling: Graceful failure with helpful messages 5. Security: No path traversal, no instruction leakage +### Usability Properties (always verified) +1. Zero-Config: Demo mode works without any setup OR setup is fully automated +2. Setup-Automation: If external dependencies exist (API keys, etc.), setup expert guides user through configuration +3. Error-Guidance: All errors include actionable "To fix: ..." guidance with exact steps + +### External Dependencies Analysis +Identify any external dependencies that require user configuration: +- API keys (e.g., BRAVE_API_KEY, OPENAI_API_KEY) +- External services (e.g., databases, third-party APIs) +- Required environment variables + For each property, define: - What to test - Expected behavior - How to verify ` -const EXPERT_BUILDER_INSTRUCTION = `You create initial Expert definitions based on properties. +const ECOSYSTEM_BUILDER_INSTRUCTION = `You create an Expert ecosystem based on properties and dependencies. ## Input - Properties to satisfy (from property-extractor) - User's original requirements +- External dependencies analysis + +## Your Role +Generate not just a single Expert, but an **ecosystem** of experts that ensures usability: +1. **Main Expert**: Core functionality +2. **Demo Expert**: Works immediately with sample data (no setup required) +3. **Setup Expert**: Only if external dependencies exist - guides user through configuration +4. **Doctor Expert**: Only if external dependencies exist - diagnoses and fixes issues ## Output -Use editTextFile to APPEND the new Expert definition to the existing perstack.toml. -Do NOT use writeTextFile - it would overwrite the entire file and destroy existing configuration. +Use editTextFile to APPEND the Expert ecosystem to the existing perstack.toml. +Do NOT use writeTextFile - it would overwrite the entire file. ## How to Append 1. First, read perstack.toml to understand the existing structure -2. Use editTextFile to add your new Expert section at the END of the file +2. Use editTextFile to add your Expert sections at the END of the file 3. Do NOT modify model, runtime, or provider settings - they already exist -## Design Principles -1. Start minimal - only what's needed -2. Declarative instructions - policies not procedures -3. Use @perstack/base for file/shell operations -4. Add skills only if clearly needed +## Ecosystem Generation Strategy + +### Step 1: Analyze Dependencies +From the properties, identify: +- Required API keys (e.g., BRAVE_API_KEY, OPENAI_API_KEY) +- External services or tools +- Environment variables needed + +### Step 2: Generate Ecosystem + +**ALWAYS generate:** +- Main expert (core functionality) +- Demo expert (with sample/mock data) + +**IF external dependencies exist, ALSO generate:** +- Setup expert (automated configuration wizard) +- Doctor expert (troubleshooting assistant) + +## Expert Templates + +### Main Expert Template +\`\`\`toml +[experts.""] +version = "1.0.0" +description = "Brief description of main functionality" +instruction = ''' +Your role and capabilities... +''' + +[experts."".skills."@perstack/base"] +type = "mcpStdioSkill" +command = "npx" +packageName = "@perstack/base" +pick = ["readTextFile", "think", "attemptCompletion"] +\`\`\` + +### Demo Expert Template +\`\`\`toml +[experts."-demo"] +version = "1.0.0" +description = "Interactive demo with sample data - no setup required" +instruction = ''' +You demonstrate the capabilities of using built-in sample data. + +## Your Role +Show users what the expert can do WITHOUT requiring any setup or API keys. + +## Demo Mode +- Use sample/mock data instead of real API calls +- Demonstrate key features with realistic examples +- Show expected output format +- Explain what would happen with real configuration -## Expert Section Format (append this to the file) +## Sample Data +[Include relevant sample data for demonstration] -[experts."expert-name"] +## Output +Provide a realistic demonstration that helps users understand: +1. What this expert does +2. What kind of input it expects +3. What output they will get +''' + +[experts."-demo".skills."@perstack/base"] +type = "mcpStdioSkill" +command = "npx" +packageName = "@perstack/base" +pick = ["think", "attemptCompletion"] +\`\`\` + +### Setup Expert Template (only if external deps exist) +\`\`\`toml +[experts."-setup"] version = "1.0.0" -description = "Brief description" -instruction = "Clear instructions..." +description = "Automated setup wizard for " +instruction = ''' +You guide users through setting up . + +## Your Role +Help users configure required dependencies quickly and correctly. + +## Setup Process + +### 1. Check Current Configuration +- Check if .env file exists +- Check if required environment variables are set +- Report current status + +### 2. Guide Through Missing Configuration +For each missing dependency: +- Explain what it is and why it's needed +- Provide URL to obtain it (e.g., API key signup page) +- Ask user for the value +- Validate the format +- Save to .env file + +### 3. Verify Configuration +- Run a simple test to verify the configuration works +- If test fails, explain what went wrong and how to fix it -[experts."expert-name".skills."@perstack/base"] +### 4. Confirm Success +Output: "✓ Setup complete! Try: npx perstack run \\"your query\\"" + +## Required Dependencies +[List the specific dependencies this expert needs] +''' + +[experts."-setup".skills."@perstack/base"] type = "mcpStdioSkill" command = "npx" packageName = "@perstack/base" -pick = ["readTextFile", "think", "attemptCompletion"] # ALWAYS use pick for minimal privilege +pick = ["readTextFile", "editTextFile", "exec", "think", "attemptCompletion"] +\`\`\` + +### Doctor Expert Template (only if external deps exist) +\`\`\`toml +[experts."-doctor"] +version = "1.0.0" +description = "Diagnose and fix issues with " +instruction = ''' +You diagnose and help fix issues with . + +## Your Role +Run diagnostics and provide actionable fixes for common problems. + +## Diagnostic Checks + +### 1. Environment Check +- Verify required environment variables exist +- Check .env file for proper formatting +- Validate API key formats + +### 2. Connectivity Check +- Test network connectivity to required services +- Verify API endpoints are reachable + +### 3. Configuration Validation +- Run a simple test query +- Analyze any error messages + +## Output Format +For each issue found: +\`\`\` +❌ Issue: [What's wrong] + Why it matters: [Impact] + To fix: [Exact steps to resolve] +\`\`\` + +If no issues found: +\`\`\` +✓ All checks passed! is ready to use. +\`\`\` +''' + +[experts."-doctor".skills."@perstack/base"] +type = "mcpStdioSkill" +command = "npx" +packageName = "@perstack/base" +pick = ["readTextFile", "exec", "think", "attemptCompletion"] +\`\`\` + +## Design Principles +1. Start minimal - only what's needed +2. Declarative instructions - policies not procedures +3. Use @perstack/base for file/shell operations +4. ALWAYS use pick for minimal privilege +5. Demo expert must work WITHOUT any setup +6. Setup expert should complete in < 2 minutes +7. All errors must include "To fix: ..." guidance ` const HAPPY_PATH_MANAGER_INSTRUCTION = `You manage the happy-path PDCA cycle. @@ -204,6 +380,53 @@ If failures exist: All adversarial properties pass → return success to parent. ` +const USABILITY_MANAGER_INSTRUCTION = `You manage the usability PDCA cycle. + +## Your Role +Ensure the Expert ecosystem is production-ready from a UX perspective. + +## PDCA Loop + +### Plan +Define usability test scenarios: +1. **Fresh User Test**: Can someone with zero knowledge succeed? +2. **Demo Test**: Does the demo expert work without any setup? +3. **Setup Test**: If setup expert exists, does it complete in < 2 minutes? +4. **Error Recovery Test**: Do errors include "To fix:" guidance? + +### Do +Delegate to \`expert-tester\` with: +- Stage: "usability" +- Expert ecosystem to test (main, demo, setup, doctor) +- Usability properties to verify + +Test cases to run: +1. Run demo expert - should succeed without configuration +2. Run setup expert (if exists) - should guide through configuration +3. Run main expert - should work after setup +4. Run doctor expert (if exists) - should diagnose issues +5. Trigger intentional errors - should show actionable guidance + +### Check +Verify usability properties: +- [ ] Demo expert works without any configuration +- [ ] Setup expert (if exists) completes successfully in < 2 minutes +- [ ] All errors include "To fix: ..." guidance +- [ ] Doctor expert (if exists) can diagnose common issues +- [ ] Time to first success < 5 minutes for new users + +### Act +If any property fails: +- If demo missing/broken: Fix demo expert instructions +- If setup broken: Fix setup automation flow +- If errors unclear: Add actionable "To fix:" guidance +- If doctor missing: Generate doctor expert +- Loop back to Do + +## Exit Condition +All usability properties pass → return success to parent. +` + const EXPERT_TESTER_INSTRUCTION = `You test Experts and report property-wise results. ## Your Role @@ -211,7 +434,7 @@ Execute tests against an Expert and evaluate which properties pass or fail. ## Input From the stage manager: -- Stage: "happy-path", "unhappy-path", or "adversarial" +- Stage: "happy-path", "unhappy-path", "adversarial", or "usability" - Expert name to test - Properties to test - Test cases to run @@ -226,7 +449,27 @@ npx -y perstack run expert-name "test query" --workspace . --filter completeRun **CRITICAL: When there are multiple test cases, you MUST call multiple \`exec\` tools in a SINGLE response to run them in parallel.** -### 2. Evaluate Properties +### 2. Stage-Specific Testing + +#### For "usability" stage: +Test the entire expert ecosystem: +1. **Demo expert**: \`npx perstack run -demo --workspace .\` + - Should work without any configuration or API keys + - Should demonstrate capabilities with sample data + +2. **Setup expert** (if exists): \`npx perstack run -setup --workspace .\` + - Should detect missing configuration + - Should guide user through setup process + +3. **Doctor expert** (if exists): \`npx perstack run -doctor --workspace .\` + - Should run diagnostics + - Should identify any issues + +4. **Error guidance check**: + - Trigger an error condition + - Verify error message includes "To fix:" guidance + +### 3. Evaluate Properties For each property, determine: - PASS: Property is satisfied - FAIL: Property is not satisfied (with reason) @@ -252,17 +495,24 @@ const REPORT_GENERATOR_INSTRUCTION = `You generate the final Expert creation rep ## Input - Original requirements - Extracted properties -- Test results from all stages +- Test results from all stages (including usability) +- Expert ecosystem information ## Output A comprehensive report: \`\`\`markdown -## Expert Creation Complete +## Expert Ecosystem Created -### Expert: [name] +### Ecosystem: [name] [description] +### Experts Generated +- **[name]**: Main expert (core functionality) +- **[name]-demo**: Demo mode (no setup required) +- **[name]-setup**: Setup wizard (if external deps exist) +- **[name]-doctor**: Troubleshooting (if external deps exist) + ### Properties Verified #### User Properties @@ -274,14 +524,30 @@ A comprehensive report: - [x] Error Handling: [evidence] - [x] Security: [evidence] +#### Usability Properties +- [x] Zero-Config: Demo works without setup +- [x] Setup-Automation: Setup completes successfully +- [x] Error-Guidance: Errors include "To fix:" guidance + ### Test Summary - Happy-path: X/Y passed - Unhappy-path: X/Y passed - Adversarial: X/Y passed +- Usability: X/Y passed -### Usage +### Quick Start \`\`\`bash -npx perstack run expert-name "query" --workspace . +# Try the demo first (no setup required) +npx perstack run [name]-demo --workspace . + +# Set up for real use +npx perstack run [name]-setup --workspace . + +# Use the expert +npx perstack run [name] "your query" --workspace . + +# If you encounter issues +npx perstack run [name]-doctor --workspace . \`\`\` ### Notes @@ -312,7 +578,7 @@ description = "Creates and tests new Perstack Experts using Property-Based Testi instruction = ''' ${CREATE_EXPERT_INSTRUCTION} ''' -delegates = ["property-extractor", "expert-builder", "happy-path-manager", "unhappy-path-manager", "adversarial-manager", "report-generator"] +delegates = ["property-extractor", "ecosystem-builder", "happy-path-manager", "unhappy-path-manager", "adversarial-manager", "usability-manager", "report-generator"] [experts."create-expert".skills."@perstack/base"] type = "mcpStdioSkill" @@ -335,14 +601,14 @@ packageName = "@perstack/base" # ---------------------------------------------------------------------------- -[experts."expert-builder"] +[experts."ecosystem-builder"] version = "1.0.0" -description = "Creates initial Expert definitions based on properties" +description = "Creates Expert ecosystem (main + demo + setup + doctor)" instruction = ''' -${EXPERT_BUILDER_INSTRUCTION} +${ECOSYSTEM_BUILDER_INSTRUCTION} ''' -[experts."expert-builder".skills."@perstack/base"] +[experts."ecosystem-builder".skills."@perstack/base"] type = "mcpStdioSkill" command = "npx" packageName = "@perstack/base" @@ -394,6 +660,21 @@ packageName = "@perstack/base" # ---------------------------------------------------------------------------- +[experts."usability-manager"] +version = "1.0.0" +description = "Manages usability PDCA cycle for expert ecosystem" +instruction = ''' +${USABILITY_MANAGER_INSTRUCTION} +''' +delegates = ["expert-tester"] + +[experts."usability-manager".skills."@perstack/base"] +type = "mcpStdioSkill" +command = "npx" +packageName = "@perstack/base" + +# ---------------------------------------------------------------------------- + [experts."expert-tester"] version = "1.0.0" description = "Executes tests and reports property-wise results" From 84877744579f865281390145e1be594e79d0def9 Mon Sep 17 00:00:00 2001 From: HiranoMasaaki Date: Sat, 3 Jan 2026 04:26:36 +0000 Subject: [PATCH 2/2] fix: use patch bump for create-expert --- .changeset/expert-ecosystems.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/expert-ecosystems.md b/.changeset/expert-ecosystems.md index 1d15316c..df8eb613 100644 --- a/.changeset/expert-ecosystems.md +++ b/.changeset/expert-ecosystems.md @@ -1,5 +1,5 @@ --- -"create-expert": minor +"create-expert": patch --- feat(create-expert): Generate Expert Ecosystems with Usability Focus