diff --git a/js/chat/chat-core.js b/js/chat/chat-core.js index 14f8dab..333d801 100644 --- a/js/chat/chat-core.js +++ b/js/chat/chat-core.js @@ -447,20 +447,27 @@ document.addEventListener("DOMContentLoaded", () => { async function handleToolJson(raw, { imageUrls, audioUrls }) { const obj = (window.repairJson || (() => ({ text: raw })))(raw); let handled = false; + const texts = []; - if (obj.tool) { - const fn = toolbox.get(obj.tool); + const runTool = async spec => { + const fn = spec && toolbox.get(spec.tool); if (fn) { try { - const res = await fn(obj); + const res = await fn(spec); if (res?.imageUrl) imageUrls.push(res.imageUrl); if (res?.audioUrl) audioUrls.push(res.audioUrl); + if (res?.text) texts.push(res.text); handled = true; - return { handled: true, text: res?.text || '' }; } catch (e) { console.warn('tool execution failed', e); } } + }; + + if (Array.isArray(obj.tools)) { + for (const t of obj.tools) await runTool(t); + } else if (obj.tool) { + await runTool(obj); } const imgPrompts = obj.image ? [obj.image] : Array.isArray(obj.images) ? obj.images : []; @@ -497,7 +504,8 @@ document.addEventListener("DOMContentLoaded", () => { } } - const text = typeof obj.text === 'string' ? obj.text : raw; + if (typeof obj.text === 'string') texts.push(obj.text); + const text = texts.join('').trim() || raw; return { handled, text }; } @@ -779,7 +787,10 @@ document.addEventListener("DOMContentLoaded", () => { try { const capsInfo = capabilities?.text?.[model]; const chatParams = { model, messages }; - if (capsInfo?.tools) chatParams.tools = toolDefinitions; + if (capsInfo?.tools) { + chatParams.tools = toolDefinitions; + chatParams.json = true; + } const data = await (window.polliLib?.chat?.(chatParams) ?? Promise.reject(new Error('polliLib not loaded'))); loadingDiv.remove(); diff --git a/prompts/ai-instruct.md b/prompts/ai-instruct.md index e103d1c..d9fd7ee 100644 --- a/prompts/ai-instruct.md +++ b/prompts/ai-instruct.md @@ -57,7 +57,13 @@ next section ## Images - Do not include external URLs. -- Provide image prompts inside an `image` fenced block: +- When image tools are available, respond with a JSON object instead of a fenced block: + +```json +{"tool":"image","prompt":"a glowing neon cityscape at night with flying cars"} +``` + +- If tools are unavailable, provide image prompts inside an `image` fenced block: ```image a glowing neon cityscape at night with flying cars @@ -121,7 +127,8 @@ tell me a joke in a calm tone - As an alternative to fenced blocks, respond with a JSON object. - The object may include: - - `tool` to invoke a tool (`image`, `tts`, or `ui`). + - `tool` to invoke a single tool (`image`, `tts`, or `ui`). + - `tools` to invoke multiple tools at once (each entry requires a `tool` field). - `text` for plain responses. - `image` or `images` with prompt strings to generate images. - `audio` with text for text-to-speech. @@ -133,11 +140,11 @@ tell me a joke in a calm tone ``` ```json -{"text":"Hello there"} +{"tools":[{"tool":"image","prompt":"a tiny house"},{"tool":"tts","text":"hello"}],"text":"Here you go"} ``` ```json -{"images":["a tiny house"],"text":"Here you go"} +{"text":"Hello there"} ``` - Always return valid JSON (double quotes, no trailing commas). diff --git a/tests/pollilib-capability-usage.mjs b/tests/pollilib-capability-usage.mjs index a8d9ee3..ef859c9 100644 --- a/tests/pollilib-capability-usage.mjs +++ b/tests/pollilib-capability-usage.mjs @@ -24,9 +24,17 @@ const caps = await modelCapabilities(client); function buildOptions(model) { const opts = { model, messages: [] }; - if (caps.text?.[model]?.tools) opts.tools = ['toolA']; + if (caps.text?.[model]?.tools) { + opts.tools = ['toolA']; + opts.json = true; + } return opts; } -assert('tools' in buildOptions('bar')); -assert(!('tools' in buildOptions('baz'))); +const withTools = buildOptions('bar'); +assert('tools' in withTools && withTools.json === true); +const withoutTools = buildOptions('baz'); +assert(!('tools' in withoutTools)); +assert(!('json' in withoutTools)); + +console.log('capability-usage test passed'); diff --git a/tests/site-json-tools.mjs b/tests/site-json-tools.mjs index 7dfdb23..b7e2933 100644 --- a/tests/site-json-tools.mjs +++ b/tests/site-json-tools.mjs @@ -39,6 +39,7 @@ const tools = [ let imageUrl; let audioBlob; let uiRan = false; +let returnedText; // Register tool implementations box: { @@ -70,17 +71,36 @@ box: { async function dispatch(json) { const obj = JSON.parse(json); - const fn = toolbox.get(obj.tool); - assert(fn, `missing tool ${obj.tool}`); - return await fn(obj); + const texts = []; + if (Array.isArray(obj.tools)) { + for (const t of obj.tools) { + const fn = toolbox.get(t.tool); + assert(fn, `missing tool ${t.tool}`); + const res = await fn(t); + if (res?.text) texts.push(res.text); + } + } else if (obj.tool) { + const fn = toolbox.get(obj.tool); + assert(fn, `missing tool ${obj.tool}`); + const res = await fn(obj); + if (res?.text) texts.push(res.text); + } + if (obj.text) texts.push(obj.text); + return texts.join(' ').trim(); } -await dispatch('{"tool":"image","prompt":"tiny green square"}'); -await dispatch('{"tool":"tts","text":"ok"}'); -await dispatch('{"tool":"ui","command":{"action":"click","target":"ping"}}'); +returnedText = await dispatch(JSON.stringify({ + tools: [ + { tool: 'image', prompt: 'tiny green square' }, + { tool: 'tts', text: 'ok' }, + { tool: 'ui', command: { action: 'click', target: 'ping' } } + ], + text: 'done' +})); assert(imageUrl && imageUrl.startsWith('http'), 'image url via polliLib'); assert(audioBlob && typeof audioBlob.size === 'number', 'audio blob generated'); assert(uiRan, 'ui command executed'); +assert.equal(returnedText, 'done'); console.log('json-tools test passed');