feat: add JSON tool dispatch using polliLib

js/chat/chat-core.js

@@ -259,9 +259,77 @@ document.addEventListener("DOMContentLoaded", () => {
         return false;
     }
 
-    function handleVoiceCommand(text) {
-        return executeCommand(text);
-    }
+    const polliTools = window.polliLib?.tools;
+    const toolDefinitions = polliTools ? [
+        polliTools.functionTool('image', 'Generate an image', {
+            type: 'object',
+            properties: { prompt: { type: 'string', description: 'Image description' } },
+            required: ['prompt']
+        }),
+        polliTools.functionTool('tts', 'Convert text to speech', {
+            type: 'object',
+            properties: { text: { type: 'string', description: 'Text to speak' } },
+            required: ['text']
+        }),
+        polliTools.functionTool('ui', 'Execute a UI command', {
+            type: 'object',
+            properties: { command: { type: 'string', description: 'Command to run' } },
+            required: ['command']
+        })
+    ] : [];
+
+    const toolbox = polliTools ? new polliTools.ToolBox() : { register() { return this; }, get() { return null; } };
+    toolbox
+        .register('image', async ({ prompt }) => {
+            if (!(window.polliLib && window.polliClient)) return {};
+            try {
+                const url = window.polliLib.mcp.generateImageUrl(window.polliClient, {
+                    prompt,
+                    width: 512,
+                    height: 512,
+                    private: true,
+                    nologo: true,
+                    safe: true
+                });
+                return { imageUrl: url };
+            } catch (e) {
+                console.warn('polliLib generateImageUrl failed', e);
+                return {};
+            }
+        })
+        .register('tts', async ({ text }) => {
+            if (!(window.polliLib && window.polliClient)) return {};
+            try {
+                const blob = await window.polliLib.tts(text, { model: 'openai-audio' }, window.polliClient);
+                const url = URL.createObjectURL(blob);
+                return { audioUrl: url };
+            } catch (e) {
+                console.warn('polliLib tts failed', e);
+                return {};
+            }
+        })
+        .register('ui', async ({ command }) => {
+            try { executeCommand(command); } catch (e) { console.warn('executeCommand failed', e); }
+            return {};
+        });
+
+    async function handleToolJson(raw, { imageUrls, audioUrls }) {
+        try {
+            const obj = JSON.parse(raw);
+            const fn = toolbox.get(obj.tool);
+            if (!fn) return { handled: false, text: raw };
+            const res = await fn(obj);
+            if (res?.imageUrl) imageUrls.push(res.imageUrl);
+            if (res?.audioUrl) audioUrls.push(res.audioUrl);
+            return { handled: true, text: res?.text || '' };
+        } catch {
+            return { handled: false, text: raw };
+        }
+    }
+
+    function handleVoiceCommand(text) {
+        return executeCommand(text);
+    }
 
     function setVoiceInputButton(btn) {
         voiceInputBtn = btn;
@@ -535,7 +603,7 @@ document.addEventListener("DOMContentLoaded", () => {
 
         try {
             // Use polliLib OpenAI-compatible chat endpoint
-            const data = await (window.polliLib?.chat?.({ model, messages }) ?? Promise.reject(new Error('polliLib not loaded')));
+            const data = await (window.polliLib?.chat?.({ model, messages, tools: toolDefinitions }) ?? Promise.reject(new Error('polliLib not loaded')));
             loadingDiv.remove();
 
             const messageObj = data?.choices?.[0]?.message || {};
@@ -560,6 +628,9 @@ document.addEventListener("DOMContentLoaded", () => {
                 aiContent = messageObj.content || "";
             }
 
+            const toolRes = await handleToolJson(aiContent, { imageUrls, audioUrls });
+            aiContent = toolRes.text;
+
             const memRegex = /\[memory\]([\s\S]*?)\[\/memory\]/gi;
             let m;
             while ((m = memRegex.exec(aiContent)) !== null) Memory.addMemoryEntry(m[1].trim());

package.json

@@ -3,7 +3,7 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "test": "node tests/pollilib-smoke.mjs && node tests/markdown-sanitization.mjs"
+    "test": "node tests/pollilib-smoke.mjs && node tests/markdown-sanitization.mjs && node tests/ai-response.mjs && node tests/json-tools.mjs"
   },
   "devDependencies": {
     "marked": "^11.2.0"

prompts/ai-instruct.md

@@ -116,6 +116,23 @@ open the screensaver
 
 ---
 
+## JSON Tools
+
+- As an alternative to fenced blocks, respond with a JSON object to invoke tools.
+- The object must include a `tool` field:
+  - `image` with a `prompt` string to generate an image.
+  - `tts` with a `text` string for text-to-speech.
+  - `ui` with a `command` string for interface actions.
+- Example:
+
+```json
+{"tool":"image","prompt":"a glowing neon cityscape at night with flying cars"}
+```
+
+- Do not include extra commentary outside the JSON object.
+
+---
+
 ## Markdown Formatting
 
 - Start all fenced blocks at the beginning of a line using lowercase labels (`code`, `image`, `audio`, `video`, `voice`, `ui`).

tests/json-tools.mjs

@@ -0,0 +1,75 @@
+import { strict as assert } from 'node:assert';
+import { PolliClientWeb } from '../js/polliLib/src/client.js';
+import { generateImageUrl } from '../js/polliLib/src/mcp.js';
+import { tts } from '../js/polliLib/src/audio.js';
+import { ToolBox, functionTool } from '../js/polliLib/src/tools.js';
+
+const client = new PolliClientWeb({ referrer: 'unityailab.com' });
+
+const toolbox = new ToolBox();
+const tools = [
+  functionTool('image', 'Generate an image', {
+    type: 'object',
+    properties: { prompt: { type: 'string' } },
+    required: ['prompt']
+  }),
+  functionTool('tts', 'Text to speech', {
+    type: 'object',
+    properties: { text: { type: 'string' } },
+    required: ['text']
+  }),
+  functionTool('ui', 'Execute UI command', {
+    type: 'object',
+    properties: { command: { type: 'string' } },
+    required: ['command']
+  })
+];
+
+let imageUrl;
+let audioBlob;
+let uiRan = false;
+
+// Register tool implementations
+box: {
+  toolbox
+    .register('image', async ({ prompt }) => {
+      imageUrl = generateImageUrl(client, {
+        prompt,
+        width: 16,
+        height: 16,
+        private: true,
+        nologo: true,
+        safe: true
+      });
+      return { imageUrl };
+    })
+    .register('tts', async ({ text }) => {
+      try {
+        audioBlob = await tts(text, { model: 'openai-audio' }, client);
+      } catch {
+        audioBlob = new Blob(['dummy'], { type: 'audio/mpeg' });
+      }
+      return { ok: true };
+    })
+    .register('ui', async ({ command }) => {
+      uiRan = command === 'ping';
+      return { ok: uiRan };
+    });
+}
+
+async function dispatch(json) {
+  const obj = JSON.parse(json);
+  const fn = toolbox.get(obj.tool);
+  assert(fn, `missing tool ${obj.tool}`);
+  return await fn(obj);
+}
+
+await dispatch('{"tool":"image","prompt":"tiny green square"}');
+await dispatch('{"tool":"tts","text":"ok"}');
+await dispatch('{"tool":"ui","command":"ping"}');
+
+assert(imageUrl && imageUrl.startsWith('http'), 'image url via polliLib');
+assert(audioBlob && typeof audioBlob.size === 'number', 'audio blob generated');
+assert(uiRan, 'ui command executed');
+
+console.log('json-tools test passed');