From 72a3c45dbf9ec77ba0c8fbdd8c087d006f06389e Mon Sep 17 00:00:00 2001 From: Hackall <36754621+hackall360@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:49:32 -0700 Subject: [PATCH] test: cover json tool invocation --- js/chat/chat-core.js | 79 +++++++++++++++++++++++++++++++++++++++--- package.json | 2 +- prompts/ai-instruct.md | 17 +++++++++ tests/json-tools.mjs | 75 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 tests/json-tools.mjs diff --git a/js/chat/chat-core.js b/js/chat/chat-core.js index 1b338d2..d49457c 100644 --- a/js/chat/chat-core.js +++ b/js/chat/chat-core.js @@ -259,9 +259,77 @@ document.addEventListener("DOMContentLoaded", () => { return false; } - function handleVoiceCommand(text) { - return executeCommand(text); - } + const polliTools = window.polliLib?.tools; + const toolDefinitions = polliTools ? [ + polliTools.functionTool('image', 'Generate an image', { + type: 'object', + properties: { prompt: { type: 'string', description: 'Image description' } }, + required: ['prompt'] + }), + polliTools.functionTool('tts', 'Convert text to speech', { + type: 'object', + properties: { text: { type: 'string', description: 'Text to speak' } }, + required: ['text'] + }), + polliTools.functionTool('ui', 'Execute a UI command', { + type: 'object', + properties: { command: { type: 'string', description: 'Command to run' } }, + required: ['command'] + }) + ] : []; + + const toolbox = polliTools ? new polliTools.ToolBox() : { register() { return this; }, get() { return null; } }; + toolbox + .register('image', async ({ prompt }) => { + if (!(window.polliLib && window.polliClient)) return {}; + try { + const url = window.polliLib.mcp.generateImageUrl(window.polliClient, { + prompt, + width: 512, + height: 512, + private: true, + nologo: true, + safe: true + }); + return { imageUrl: url }; + } catch (e) { + console.warn('polliLib generateImageUrl failed', e); + return {}; + } + }) + .register('tts', async ({ text }) => { + if (!(window.polliLib && window.polliClient)) return {}; + try { + const blob = await window.polliLib.tts(text, { model: 'openai-audio' }, window.polliClient); + const url = URL.createObjectURL(blob); + return { audioUrl: url }; + } catch (e) { + console.warn('polliLib tts failed', e); + return {}; + } + }) + .register('ui', async ({ command }) => { + try { executeCommand(command); } catch (e) { console.warn('executeCommand failed', e); } + return {}; + }); + + async function handleToolJson(raw, { imageUrls, audioUrls }) { + try { + const obj = JSON.parse(raw); + const fn = toolbox.get(obj.tool); + if (!fn) return { handled: false, text: raw }; + const res = await fn(obj); + if (res?.imageUrl) imageUrls.push(res.imageUrl); + if (res?.audioUrl) audioUrls.push(res.audioUrl); + return { handled: true, text: res?.text || '' }; + } catch { + return { handled: false, text: raw }; + } + } + + function handleVoiceCommand(text) { + return executeCommand(text); + } function setVoiceInputButton(btn) { voiceInputBtn = btn; @@ -535,7 +603,7 @@ document.addEventListener("DOMContentLoaded", () => { try { // Use polliLib OpenAI-compatible chat endpoint - const data = await (window.polliLib?.chat?.({ model, messages }) ?? Promise.reject(new Error('polliLib not loaded'))); + const data = await (window.polliLib?.chat?.({ model, messages, tools: toolDefinitions }) ?? Promise.reject(new Error('polliLib not loaded'))); loadingDiv.remove(); const messageObj = data?.choices?.[0]?.message || {}; @@ -560,6 +628,9 @@ document.addEventListener("DOMContentLoaded", () => { aiContent = messageObj.content || ""; } + const toolRes = await handleToolJson(aiContent, { imageUrls, audioUrls }); + aiContent = toolRes.text; + const memRegex = /\[memory\]([\s\S]*?)\[\/memory\]/gi; let m; while ((m = memRegex.exec(aiContent)) !== null) Memory.addMemoryEntry(m[1].trim()); diff --git a/package.json b/package.json index d37a193..7b2ec02 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "private": true, "type": "module", "scripts": { - "test": "node tests/pollilib-smoke.mjs && node tests/markdown-sanitization.mjs" + "test": "node tests/pollilib-smoke.mjs && node tests/markdown-sanitization.mjs && node tests/ai-response.mjs && node tests/json-tools.mjs" }, "devDependencies": { "marked": "^11.2.0" diff --git a/prompts/ai-instruct.md b/prompts/ai-instruct.md index 3656698..402db96 100644 --- a/prompts/ai-instruct.md +++ b/prompts/ai-instruct.md @@ -116,6 +116,23 @@ open the screensaver --- +## JSON Tools + +- As an alternative to fenced blocks, respond with a JSON object to invoke tools. +- The object must include a `tool` field: + - `image` with a `prompt` string to generate an image. + - `tts` with a `text` string for text-to-speech. + - `ui` with a `command` string for interface actions. +- Example: + +```json +{"tool":"image","prompt":"a glowing neon cityscape at night with flying cars"} +``` + +- Do not include extra commentary outside the JSON object. + +--- + ## Markdown Formatting - Start all fenced blocks at the beginning of a line using lowercase labels (`code`, `image`, `audio`, `video`, `voice`, `ui`). diff --git a/tests/json-tools.mjs b/tests/json-tools.mjs new file mode 100644 index 0000000..38d2df7 --- /dev/null +++ b/tests/json-tools.mjs @@ -0,0 +1,75 @@ +import { strict as assert } from 'node:assert'; +import { PolliClientWeb } from '../js/polliLib/src/client.js'; +import { generateImageUrl } from '../js/polliLib/src/mcp.js'; +import { tts } from '../js/polliLib/src/audio.js'; +import { ToolBox, functionTool } from '../js/polliLib/src/tools.js'; + +const client = new PolliClientWeb({ referrer: 'unityailab.com' }); + +const toolbox = new ToolBox(); +const tools = [ + functionTool('image', 'Generate an image', { + type: 'object', + properties: { prompt: { type: 'string' } }, + required: ['prompt'] + }), + functionTool('tts', 'Text to speech', { + type: 'object', + properties: { text: { type: 'string' } }, + required: ['text'] + }), + functionTool('ui', 'Execute UI command', { + type: 'object', + properties: { command: { type: 'string' } }, + required: ['command'] + }) +]; + +let imageUrl; +let audioBlob; +let uiRan = false; + +// Register tool implementations +box: { + toolbox + .register('image', async ({ prompt }) => { + imageUrl = generateImageUrl(client, { + prompt, + width: 16, + height: 16, + private: true, + nologo: true, + safe: true + }); + return { imageUrl }; + }) + .register('tts', async ({ text }) => { + try { + audioBlob = await tts(text, { model: 'openai-audio' }, client); + } catch { + audioBlob = new Blob(['dummy'], { type: 'audio/mpeg' }); + } + return { ok: true }; + }) + .register('ui', async ({ command }) => { + uiRan = command === 'ping'; + return { ok: uiRan }; + }); +} + +async function dispatch(json) { + const obj = JSON.parse(json); + const fn = toolbox.get(obj.tool); + assert(fn, `missing tool ${obj.tool}`); + return await fn(obj); +} + +await dispatch('{"tool":"image","prompt":"tiny green square"}'); +await dispatch('{"tool":"tts","text":"ok"}'); +await dispatch('{"tool":"ui","command":"ping"}'); + +assert(imageUrl && imageUrl.startsWith('http'), 'image url via polliLib'); +assert(audioBlob && typeof audioBlob.size === 'number', 'audio blob generated'); +assert(uiRan, 'ui command executed'); + +console.log('json-tools test passed');