From ce9c8358821dfcb72ab5a1843655aac673aaa978 Mon Sep 17 00:00:00 2001 From: Miles Date: Sat, 21 Feb 2026 11:07:41 +0800 Subject: [PATCH] fix(web): add post-generation verification and harden generated spec handling --- README.md | 1 + apps/web/app/api/generate/route.ts | 2 + apps/web/components/playground.tsx | 358 ++++++++++++++++++++------- apps/web/lib/render/catalog.ts | 8 +- apps/web/lib/render/option-utils.ts | 82 ++++++ apps/web/lib/render/registry.tsx | 68 +++-- apps/web/lib/render/spec-verifier.ts | 194 +++++++++++++++ 7 files changed, 601 insertions(+), 112 deletions(-) create mode 100644 apps/web/lib/render/option-utils.ts create mode 100644 apps/web/lib/render/spec-verifier.ts diff --git a/README.md b/README.md index 68229637..e1ad41b2 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ json-render is a **Generative UI** framework: AI generates interfaces from natur - **Guardrailed** - AI can only use components in your catalog - **Predictable** - JSON output matches your schema, every time - **Fast** - Stream and render progressively as the model responds +- **Verifiable** - Playground includes a post-generation verification + quick feedback loop - **Cross-Platform** - React (web) and React Native (mobile) from the same catalog - **Batteries Included** - 36 pre-built shadcn/ui components ready to use diff --git a/apps/web/app/api/generate/route.ts b/apps/web/app/api/generate/route.ts index b4740368..b3be22da 100644 --- a/apps/web/app/api/generate/route.ts +++ b/apps/web/app/api/generate/route.ts @@ -15,6 +15,8 @@ const SYSTEM_PROMPT = playgroundCatalog.prompt({ "Wrap each repeated item in a Card for visual separation and structure.", "Use realistic, professional sample data. Include 3-5 items with varied content. Never leave state arrays empty.", 'For form inputs (Input, Textarea, Select), always include checks for validation (e.g. required, email, minLength). Always pair checks with a $bindState expression on the value prop (e.g. { "$bindState": "/path" }).', + 'For Rating used in forms/feedback, make it interactive: bind props.value with { "$bindState": "/path" } and provide on.change to update state.', + "For Select and Radio options, output plain string arrays only (e.g. ['Small','Medium','Large']). Do not output raw option objects.", ], }); diff --git a/apps/web/components/playground.tsx b/apps/web/components/playground.tsx index 74325378..8c121806 100644 --- a/apps/web/components/playground.tsx +++ b/apps/web/components/playground.tsx @@ -19,6 +19,10 @@ import { Sheet, SheetContent, SheetTitle } from "./ui/sheet"; import { PlaygroundRenderer } from "@/lib/render/renderer"; import { playgroundCatalog } from "@/lib/render/catalog"; import { buildCatalogDisplayData } from "@/lib/render/catalog-display"; +import { + verifyPlaygroundSpec, + buildVerificationFixPrompt, +} from "@/lib/render/spec-verifier"; type Tab = "json" | "nested" | "stream" | "catalog"; type RenderView = "preview" | "code"; @@ -33,12 +37,21 @@ type MobileView = interface Version { id: string; prompt: string; + requestPrompt: string; + originPrompt: string; tree: Spec | null; status: "generating" | "complete" | "error"; usage: TokenUsage | null; rawLines: string[]; } +interface StartGenerationOptions { + requestPrompt: string; + displayPrompt: string; + previousSpec: Spec | null; + originPrompt: string; +} + /** * Convert a flat Spec into a nested tree structure that is easier for humans * to read. Children keys are resolved recursively into inline objects. @@ -104,6 +117,9 @@ export function Playground() { const [renderView, setRenderView] = useState("preview"); const [mobileView, setMobileView] = useState("preview"); const [versionsSheetOpen, setVersionsSheetOpen] = useState(false); + const [verificationFeedback, setVerificationFeedback] = useState< + Record + >({}); const inputRef = useRef(null); const mobileInputRef = useRef(null); const versionsEndRef = useRef(null); @@ -141,6 +157,18 @@ export function Playground() { // Get the selected version const selectedVersion = versions.find((v) => v.id === selectedVersionId); + const selectedVersionIndex = selectedVersion + ? versions.findIndex((v) => v.id === selectedVersion.id) + : -1; + const selectedVerification = useMemo(() => { + if (!selectedVersion?.tree || selectedVersion.status !== "complete") { + return null; + } + return verifyPlaygroundSpec(selectedVersion.tree); + }, [selectedVersion]); + const selectedVerificationFeedback = selectedVersionId + ? verificationFeedback[selectedVersionId] + : undefined; // Determine which tree to display: // - If streaming and selected version is the generating one, show apiSpec @@ -201,27 +229,119 @@ export function Playground() { } }, [isStreaming, apiSpec, streamUsage, streamRawLines]); + const handleClearAll = useCallback(() => { + setVersions([]); + setSelectedVersionId(null); + setVerificationFeedback({}); + clear(); + currentTreeRef.current = null; + generatingVersionIdRef.current = null; + }, [clear]); + + const startGeneration = useCallback( + async ({ + requestPrompt, + displayPrompt, + previousSpec, + originPrompt, + }: StartGenerationOptions) => { + const trimmedPrompt = requestPrompt.trim(); + if (!trimmedPrompt || isStreaming) return; + + const newVersionId = Date.now().toString(); + const newVersion: Version = { + id: newVersionId, + prompt: displayPrompt, + requestPrompt: trimmedPrompt, + originPrompt, + tree: null, + status: "generating", + usage: null, + rawLines: [], + }; + + generatingVersionIdRef.current = newVersionId; + setVersions((prev) => [...prev, newVersion]); + setSelectedVersionId(newVersionId); + + await send(trimmedPrompt, { previousSpec: previousSpec ?? undefined }); + }, + [isStreaming, send], + ); + const handleSubmit = useCallback(async () => { - if (!inputValue.trim() || isStreaming) return; - - const newVersionId = Date.now().toString(); - const newVersion: Version = { - id: newVersionId, - prompt: inputValue.trim(), - tree: null, - status: "generating", - usage: null, - rawLines: [], - }; - - generatingVersionIdRef.current = newVersionId; - setVersions((prev) => [...prev, newVersion]); - setSelectedVersionId(newVersionId); + const prompt = inputValue.trim(); + if (!prompt || isStreaming) return; + setInputValue(""); + await startGeneration({ + requestPrompt: prompt, + displayPrompt: prompt, + previousSpec: currentTreeRef.current, + originPrompt: prompt, + }); + }, [inputValue, isStreaming, startGeneration]); + + const handleVerificationAccept = useCallback(() => { + if (!selectedVersionId) return; + setVerificationFeedback((prev) => ({ + ...prev, + [selectedVersionId]: "accepted", + })); + toast.success("Verification feedback saved."); + }, [selectedVersionId]); + + const handleVerificationFix = useCallback(async () => { + if (!selectedVersion || !selectedVersion.tree || isStreaming) return; + + const issues = selectedVerification?.issues ?? []; + if (issues.length === 0) { + toast.success("No verification issues detected."); + return; + } + + setVerificationFeedback((prev) => ({ + ...prev, + [selectedVersion.id]: "fix-requested", + })); - // Pass the current tree as context so the API can iterate on it - await send(inputValue.trim(), { previousSpec: currentTreeRef.current }); - }, [inputValue, isStreaming, send]); + const label = + selectedVersionIndex >= 0 + ? `Fix verification issues for v${selectedVersionIndex + 1}` + : "Fix verification issues"; + + await startGeneration({ + requestPrompt: buildVerificationFixPrompt({ + originalPrompt: selectedVersion.originPrompt, + issues, + }), + displayPrompt: label, + previousSpec: selectedVersion.tree, + originPrompt: selectedVersion.originPrompt, + }); + }, [ + selectedVersion, + isStreaming, + selectedVerification, + selectedVersionIndex, + startGeneration, + ]); + + const handleRegenerate = useCallback(async () => { + if (!selectedVersion || isStreaming) return; + + const label = + selectedVersionIndex >= 0 + ? `Regenerate v${selectedVersionIndex + 1}` + : "Regenerate"; + + await startGeneration({ + requestPrompt: selectedVersion.originPrompt, + displayPrompt: label, + previousSpec: null, + originPrompt: selectedVersion.originPrompt, + }); + }, [selectedVersion, isStreaming, selectedVersionIndex, startGeneration]); const handleKeyDown = useCallback( (e: React.KeyboardEvent) => { @@ -304,6 +424,92 @@ ${jsx} }`; }, [currentTree]); + const verificationIssues = selectedVerification?.issues ?? []; + const verificationErrorCount = verificationIssues.filter( + (issue) => issue.severity === "error", + ).length; + const verificationWarningCount = + verificationIssues.length - verificationErrorCount; + + const verificationPanel = + selectedVersion?.status === "complete" && selectedVersion.tree ? ( +
+
+
+

+ verification + {verificationIssues.length === 0 + ? " passed" + : ` ${verificationErrorCount} error(s), ${verificationWarningCount} warning(s)`} +

+ {selectedVerificationFeedback === "accepted" && ( +

+ Feedback: looks good +

+ )} + {selectedVerificationFeedback === "fix-requested" && ( +

+ Feedback: fix requested +

+ )} +
+
+ + {verificationIssues.length > 0 && ( + + )} + +
+
+ {verificationIssues.length > 0 && ( +
+ {verificationIssues.slice(0, 3).map((issue, idx) => ( +

+ [{issue.severity}] {issue.message} +

+ ))} + {verificationIssues.length > 3 && ( +

+ +{verificationIssues.length - 3} more issue(s) +

+ )} +
+ )} +
+ ) : null; + + const previewContent = currentTree && currentTree.root ? ( +
+ +
+ ) : ( +
+ {isStreaming ? "generating..." : "// enter a prompt to generate UI"} +
+ ); + // Chat pane content const chatPane = (
@@ -410,11 +616,7 @@ ${jsx}
{versions.length > 0 ? (
{renderView === "preview" ? ( - currentTree && currentTree.root ? ( -
- -
- ) : ( -
- {isStreaming - ? "generating..." - : "// enter a prompt to generate UI"} -
- ) +
+ {verificationPanel} +
{previewContent}
+
) : ( ) : mobileView === "preview" ? ( - currentTree && currentTree.root ? ( -
- -
- ) : ( -
- {isStreaming ? ( -

- generating... -

- ) : ( - <> -

- Describe what you want to build, then iterate on it. +

+ {verificationPanel} + {currentTree && currentTree.root ? ( +
+ +
+ ) : ( +
+ {isStreaming ? ( +

+ generating...

-
- {EXAMPLE_PROMPTS.map((prompt) => ( - - ))} -
- - )} -
- ) + ) : ( + <> +

+ Describe what you want to build, then iterate on it. +

+
+ {EXAMPLE_PROMPTS.map((prompt) => ( + + ))} +
+ + )} +
+ )} +
) : ( /* generated-code */ {versions.length > 0 ? ( ))}
@@ -858,12 +881,9 @@ export const { registry, executeAction } = defineRegistry(playgroundCatalog, { const isBound = !!bindings?.value; const value = isBound ? (boundValue ?? "") : localValue; const setValue = isBound ? setBoundValue : setLocalValue; - const rawOptions = props.options ?? []; - // Coerce options to strings – AI may produce objects/numbers instead of - // plain strings which would cause duplicate `[object Object]` keys. - const options = rawOptions.map((opt) => - typeof opt === "string" ? opt : String(opt ?? ""), - ); + const rawOptions = Array.isArray(props.options) ? props.options : []; + // Normalize options so object shapes render as readable label/value pairs. + const options = normalizeChoiceOptions(rawOptions).options; const hasValidation = !!(bindings?.value && props.checks?.length); const { errors, validate } = useFieldValidation( @@ -888,10 +908,10 @@ export const { registry, executeAction } = defineRegistry(playgroundCatalog, { {options.map((opt, idx) => ( - {opt} + {opt.label} ))} @@ -931,15 +951,13 @@ export const { registry, executeAction } = defineRegistry(playgroundCatalog, { }, Radio: ({ props, bindings, emit }) => { - const rawOptions = props.options ?? []; - const options = rawOptions.map((opt) => - typeof opt === "string" ? opt : String(opt ?? ""), - ); + const rawOptions = Array.isArray(props.options) ? props.options : []; + const options = normalizeChoiceOptions(rawOptions).options; const [boundValue, setBoundValue] = useBoundProp( props.value as string | undefined, bindings?.value, ); - const [localValue, setLocalValue] = useState(options[0] ?? ""); + const [localValue, setLocalValue] = useState(options[0]?.value ?? ""); const isBound = !!bindings?.value; const value = isBound ? (boundValue ?? "") : localValue; const setValue = isBound ? setBoundValue : setLocalValue; @@ -956,18 +974,18 @@ export const { registry, executeAction } = defineRegistry(playgroundCatalog, { > {options.map((opt, idx) => (
))} diff --git a/apps/web/lib/render/spec-verifier.ts b/apps/web/lib/render/spec-verifier.ts new file mode 100644 index 00000000..19138ef6 --- /dev/null +++ b/apps/web/lib/render/spec-verifier.ts @@ -0,0 +1,194 @@ +import { validateSpec, type Spec, type UIElement } from "@json-render/core"; + +import { normalizeChoiceOptions } from "./option-utils"; + +export type SpecVerificationSeverity = "error" | "warning"; + +export type SpecVerificationCode = + | "structural_issue" + | "rating_maybe_read_only" + | "rating_value_out_of_range" + | "choice_options_not_array" + | "choice_invalid_option_shape" + | "choice_object_options_detected" + | "choice_value_not_in_options"; + +export interface SpecVerificationIssue { + severity: SpecVerificationSeverity; + code: SpecVerificationCode; + message: string; + elementKey?: string; +} + +export interface SpecVerificationResult { + passed: boolean; + issues: SpecVerificationIssue[]; +} + +function asRecord(value: unknown): Record { + if (value && typeof value === "object" && !Array.isArray(value)) { + return value as Record; + } + return {}; +} + +function hasValueBinding(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + + const ref = value as Record; + return ( + typeof ref.$bindState === "string" || typeof ref.$bindItem === "string" + ); +} + +function hasEventBinding(element: UIElement, eventName: string): boolean { + const eventBinding = element.on?.[eventName]; + if (!eventBinding) return false; + if (Array.isArray(eventBinding)) return eventBinding.length > 0; + return true; +} + +function verifyRatingElement( + key: string, + element: UIElement, +): SpecVerificationIssue[] { + const issues: SpecVerificationIssue[] = []; + const props = asRecord(element.props); + const rawValue = props.value; + const rawMax = props.max; + const value = typeof rawValue === "number" ? rawValue : null; + const max = typeof rawMax === "number" && rawMax > 0 ? rawMax : 5; + + if (value !== null && (value < 0 || value > max)) { + issues.push({ + severity: "error", + code: "rating_value_out_of_range", + elementKey: key, + message: `Rating "${key}" has value ${value} outside range 0-${max}.`, + }); + } + + const interactive = hasValueBinding(rawValue) || hasEventBinding(element, "change"); + if (!interactive && value !== null) { + issues.push({ + severity: "warning", + code: "rating_maybe_read_only", + elementKey: key, + message: `Rating "${key}" appears read-only (static value with no binding or on.change).`, + }); + } + + return issues; +} + +function verifyChoiceElement( + key: string, + element: UIElement, + type: "Select" | "Radio", +): SpecVerificationIssue[] { + const issues: SpecVerificationIssue[] = []; + const props = asRecord(element.props); + const rawOptions = props.options; + + if (!Array.isArray(rawOptions)) { + issues.push({ + severity: "error", + code: "choice_options_not_array", + elementKey: key, + message: `${type} "${key}" has invalid options. Expected an array.`, + }); + return issues; + } + + const normalized = normalizeChoiceOptions(rawOptions); + + if (normalized.invalidCount > 0) { + issues.push({ + severity: "error", + code: "choice_invalid_option_shape", + elementKey: key, + message: `${type} "${key}" has ${normalized.invalidCount} option(s) with unsupported shape.`, + }); + } + + if (normalized.objectCount > 0) { + issues.push({ + severity: "warning", + code: "choice_object_options_detected", + elementKey: key, + message: `${type} "${key}" uses object options. Verify labels/values are correct.`, + }); + } + + const value = props.value; + if ( + typeof value === "string" && + value.length > 0 && + normalized.options.length > 0 && + !normalized.options.some((option) => option.value === value) + ) { + issues.push({ + severity: "warning", + code: "choice_value_not_in_options", + elementKey: key, + message: `${type} "${key}" has value "${value}" that does not match any option.`, + }); + } + + return issues; +} + +export function verifyPlaygroundSpec(spec: Spec): SpecVerificationResult { + const issues: SpecVerificationIssue[] = []; + + const structural = validateSpec(spec); + for (const issue of structural.issues) { + issues.push({ + severity: issue.severity, + code: "structural_issue", + message: issue.message, + elementKey: issue.elementKey, + }); + } + + for (const [key, element] of Object.entries(spec.elements)) { + if (element.type === "Rating") { + issues.push(...verifyRatingElement(key, element)); + continue; + } + + if (element.type === "Select" || element.type === "Radio") { + issues.push(...verifyChoiceElement(key, element, element.type)); + } + } + + const hasErrors = issues.some((issue) => issue.severity === "error"); + return { + passed: !hasErrors, + issues, + }; +} + +export function buildVerificationFixPrompt(options: { + originalPrompt: string; + issues: SpecVerificationIssue[]; +}): string { + const { originalPrompt, issues } = options; + const lines = [ + `Fix the current generated UI spec to satisfy this request: "${originalPrompt}".`, + "Keep layout/content intent, but repair invalid or non-interactive parts.", + "Verification issues to fix:", + ...issues.map((issue) => { + const keyLabel = issue.elementKey ? ` (${issue.elementKey})` : ""; + return `- [${issue.severity}] ${issue.code}${keyLabel}: ${issue.message}`; + }), + "Required output format: JSONL patch operations only.", + "Important constraints:", + "- Rating used as user input must be interactive (bind value to state and handle on.change).", + "- Select/Radio options must be plain strings or valid {label, value} objects.", + ]; + + return lines.join("\n"); +}