Use JSON tool calls for images

js/chat/chat-core.js

@@ -447,20 +447,27 @@ document.addEventListener("DOMContentLoaded", () => {
     async function handleToolJson(raw, { imageUrls, audioUrls }) {
         const obj = (window.repairJson || (() => ({ text: raw })))(raw);
         let handled = false;
+        const texts = [];
 
-        if (obj.tool) {
-            const fn = toolbox.get(obj.tool);
+        const runTool = async spec => {
+            const fn = spec && toolbox.get(spec.tool);
             if (fn) {
                 try {
-                    const res = await fn(obj);
+                    const res = await fn(spec);
                     if (res?.imageUrl) imageUrls.push(res.imageUrl);
                     if (res?.audioUrl) audioUrls.push(res.audioUrl);
+                    if (res?.text) texts.push(res.text);
                     handled = true;
-                    return { handled: true, text: res?.text || '' };
                 } catch (e) {
                     console.warn('tool execution failed', e);
                 }
             }
+        };
+
+        if (Array.isArray(obj.tools)) {
+            for (const t of obj.tools) await runTool(t);
+        } else if (obj.tool) {
+            await runTool(obj);
         }
 
         const imgPrompts = obj.image ? [obj.image] : Array.isArray(obj.images) ? obj.images : [];
@@ -497,7 +504,8 @@ document.addEventListener("DOMContentLoaded", () => {
             }
         }
 
-        const text = typeof obj.text === 'string' ? obj.text : raw;
+        if (typeof obj.text === 'string') texts.push(obj.text);
+        const text = texts.join('').trim() || raw;
         return { handled, text };
     }
 
@@ -779,7 +787,10 @@ document.addEventListener("DOMContentLoaded", () => {
         try {
             const capsInfo = capabilities?.text?.[model];
             const chatParams = { model, messages };
-            if (capsInfo?.tools) chatParams.tools = toolDefinitions;
+            if (capsInfo?.tools) {
+                chatParams.tools = toolDefinitions;
+                chatParams.json = true;
+            }
             const data = await (window.polliLib?.chat?.(chatParams) ?? Promise.reject(new Error('polliLib not loaded')));
             loadingDiv.remove();
 

prompts/ai-instruct.md

@@ -57,7 +57,13 @@ next section
 ## Images
 
 - Do not include external URLs.
-- Provide image prompts inside an `image` fenced block:
+- When image tools are available, respond with a JSON object instead of a fenced block:
+
+```json
+{"tool":"image","prompt":"a glowing neon cityscape at night with flying cars"}
+```
+
+- If tools are unavailable, provide image prompts inside an `image` fenced block:
 
 ```image
 a glowing neon cityscape at night with flying cars
@@ -121,7 +127,8 @@ tell me a joke in a calm tone
 
 - As an alternative to fenced blocks, respond with a JSON object.
 - The object may include:
-  - `tool` to invoke a tool (`image`, `tts`, or `ui`).
+  - `tool` to invoke a single tool (`image`, `tts`, or `ui`).
+  - `tools` to invoke multiple tools at once (each entry requires a `tool` field).
   - `text` for plain responses.
   - `image` or `images` with prompt strings to generate images.
   - `audio` with text for text-to-speech.
@@ -133,11 +140,11 @@ tell me a joke in a calm tone
 ```
 
 ```json
-{"text":"Hello there"}
+{"tools":[{"tool":"image","prompt":"a tiny house"},{"tool":"tts","text":"hello"}],"text":"Here you go"}
 ```
 
 ```json
-{"images":["a tiny house"],"text":"Here you go"}
+{"text":"Hello there"}
 ```
 
 - Always return valid JSON (double quotes, no trailing commas).

tests/pollilib-capability-usage.mjs

@@ -24,9 +24,17 @@ const caps = await modelCapabilities(client);
 
 function buildOptions(model) {
   const opts = { model, messages: [] };
-  if (caps.text?.[model]?.tools) opts.tools = ['toolA'];
+  if (caps.text?.[model]?.tools) {
+    opts.tools = ['toolA'];
+    opts.json = true;
+  }
   return opts;
 }
 
-assert('tools' in buildOptions('bar'));
-assert(!('tools' in buildOptions('baz')));
+const withTools = buildOptions('bar');
+assert('tools' in withTools && withTools.json === true);
+const withoutTools = buildOptions('baz');
+assert(!('tools' in withoutTools));
+assert(!('json' in withoutTools));
+
+console.log('capability-usage test passed');

tests/site-json-tools.mjs

@@ -39,6 +39,7 @@ const tools = [
 let imageUrl;
 let audioBlob;
 let uiRan = false;
+let returnedText;
 
 // Register tool implementations
 box: {
@@ -70,17 +71,36 @@ box: {
 
 async function dispatch(json) {
   const obj = JSON.parse(json);
-  const fn = toolbox.get(obj.tool);
-  assert(fn, `missing tool ${obj.tool}`);
-  return await fn(obj);
+  const texts = [];
+  if (Array.isArray(obj.tools)) {
+    for (const t of obj.tools) {
+      const fn = toolbox.get(t.tool);
+      assert(fn, `missing tool ${t.tool}`);
+      const res = await fn(t);
+      if (res?.text) texts.push(res.text);
+    }
+  } else if (obj.tool) {
+    const fn = toolbox.get(obj.tool);
+    assert(fn, `missing tool ${obj.tool}`);
+    const res = await fn(obj);
+    if (res?.text) texts.push(res.text);
+  }
+  if (obj.text) texts.push(obj.text);
+  return texts.join(' ').trim();
 }
 
-await dispatch('{"tool":"image","prompt":"tiny green square"}');
-await dispatch('{"tool":"tts","text":"ok"}');
-await dispatch('{"tool":"ui","command":{"action":"click","target":"ping"}}');
+returnedText = await dispatch(JSON.stringify({
+  tools: [
+    { tool: 'image', prompt: 'tiny green square' },
+    { tool: 'tts', text: 'ok' },
+    { tool: 'ui', command: { action: 'click', target: 'ping' } }
+  ],
+  text: 'done'
+}));
 
 assert(imageUrl && imageUrl.startsWith('http'), 'image url via polliLib');
 assert(audioBlob && typeof audioBlob.size === 'number', 'audio blob generated');
 assert(uiRan, 'ui command executed');
+assert.equal(returnedText, 'done');
 
 console.log('json-tools test passed');