From 40b35fef6aa5a7e2570efcb11a366b654c43217d Mon Sep 17 00:00:00 2001 From: MZ Date: Fri, 20 Feb 2026 23:39:49 -0700 Subject: [PATCH 1/3] feat: integrate OpenViking tools and prefetch memory pipeline --- AGENTS.md | 26 ++ README.md | 63 ++++ lib/agents.sh | 81 ++++ lib/daemon.sh | 8 +- .../agent-tools/openviking/README.md | 24 ++ .../agent-tools/openviking/openviking-tool.js | 274 ++++++++++++++ .../agent-tools/openviking/ovk-ls.sh | 5 + .../agent-tools/openviking/ovk-read.sh | 5 + .../agent-tools/openviking/ovk-write.sh | 5 + lib/templates/agent-tools/openviking/ovk.sh | 5 + package.json | 1 + scripts/test-openviking-prefetch-parser.js | 74 ++++ src/lib/agent-setup.ts | 29 ++ src/lib/invoke.ts | 17 +- src/lib/openviking-prefetch.ts | 103 ++++++ src/queue-processor.ts | 199 +++++++++- src/tools/openviking-tool.ts | 347 ++++++++++++++++++ tinyclaw.sh | 23 +- 18 files changed, 1283 insertions(+), 6 deletions(-) create mode 100644 lib/templates/agent-tools/openviking/README.md create mode 100644 lib/templates/agent-tools/openviking/openviking-tool.js create mode 100755 lib/templates/agent-tools/openviking/ovk-ls.sh create mode 100644 lib/templates/agent-tools/openviking/ovk-read.sh create mode 100644 lib/templates/agent-tools/openviking/ovk-write.sh create mode 100644 lib/templates/agent-tools/openviking/ovk.sh create mode 100644 scripts/test-openviking-prefetch-parser.js create mode 100644 src/lib/openviking-prefetch.ts create mode 100644 src/tools/openviking-tool.ts diff --git a/AGENTS.md b/AGENTS.md index f1d21b2..8183eee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,6 +71,32 @@ You can communicate back and forth by mentioning your teammate in your response +## Workspace Tools + +OpenViking CLI tools are available in each agent workspace: + +- Path: `.tinyclaw/tools/openviking/` +- Main command: `./ovk.sh` +- Shortcuts: `./ovk-ls.sh`, `./ovk-read.sh`, `./ovk-write.sh` + +Environment variables: + +- `OPENVIKING_BASE_URL` (default `http://127.0.0.1:8320`) +- `OPENVIKING_API_KEY` (optional) +- `OPENVIKING_PROJECT` (optional) + +Auto-sync behavior: + +- `on_turn`: each completed turn is appended and synced to `viking://resources/tinyclaw/sessions//active.md` +- `on_session_end`: when a reset is consumed, prior session is archived to `viking://resources/tinyclaw/sessions//closed/.md` +- set `TINYCLAW_OPENVIKING_AUTOSYNC=0` to disable + +Pre-prompt retrieval: + +- before each external user turn, TinyClaw attempts to read `viking://resources/tinyclaw/sessions//active.md` +- selected relevant turns are injected as `[OpenViking Retrieved Context]` +- set `TINYCLAW_OPENVIKING_PREFETCH=0` to disable + ## Soul You have a soul file at `.tinyclaw/SOUL.md`. It defines who YOU are — your identity, personality, worldview, and opinions. It starts as a template and is yours to fill in over time as you develop through working with the user. diff --git a/README.md b/README.md index ed0c76c..4097771 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,12 @@ Commands work with `tinyclaw` (if CLI installed) or `./tinyclaw.sh` (direct scri | `team remove ` | Remove a team | `tinyclaw team remove dev` | | `team visualize [id]` | Live TUI dashboard for team chains | `tinyclaw team visualize dev` | +### Tool Commands + +| Command | Description | Example | +| ------------------------ | ------------------------------------------------ | -------------------------- | +| `tools sync [agent_id]` | Sync OpenViking CLI tools into agent workspace(s) | `tinyclaw tools sync coder` | + ### Configuration Commands | Command | Description | Example | @@ -237,6 +243,63 @@ export TINYCLAW_SKIP_UPDATE_CHECK=1 | `send ` | Send message to AI manually | `tinyclaw send "Hello!"` | | `send ` | Route to specific agent | `tinyclaw send "@coder fix bug"` | +## 🧰 OpenViking Workspace Tools + +TinyClaw can provision lightweight OpenViking tools into each agent workspace: + +```bash +tinyclaw tools sync +``` + +Tools are installed at: + +```bash +//.tinyclaw/tools/openviking/ +``` + +Common usage from an agent directory: + +```bash +cd .tinyclaw/tools/openviking +./ovk-ls.sh / +./ovk-read.sh /context/spec.md +./ovk-write.sh /context/spec.md "updated content" +./ovk.sh res-get viking://workspace/resource +``` + +Environment variables: + +- `OPENVIKING_BASE_URL` (default: `http://127.0.0.1:8320`) +- `OPENVIKING_API_KEY` (optional) +- `OPENVIKING_PROJECT` (optional) + +### Auto Sync Triggers (Minimal) + +TinyClaw now supports minimal OpenViking auto-sync triggers per agent: + +- `on_turn`: after each agent response, append `user+assistant` turn to a local session file and sync to `viking://resources/tinyclaw/sessions//active.md` +- `on_session_end`: when `agent reset` is consumed, archive the previous session to `viking://resources/tinyclaw/sessions//closed/.md` + +Notes: + +- This is best-effort and non-blocking (response delivery is not blocked by sync retries). +- Disable auto-sync by setting `TINYCLAW_OPENVIKING_AUTOSYNC=0`. + +### Pre-Prompt Retrieval (OpenViking) + +Before invoking the model for an external user turn, TinyClaw can prefetch related context from: + +- `viking://resources/tinyclaw/sessions//active.md` + +It then injects a compact `[OpenViking Retrieved Context]` block into the prompt. + +Environment flags: + +- `TINYCLAW_OPENVIKING_PREFETCH=0` disable pre-prompt retrieval +- `TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS` command timeout (default: `5000`) +- `TINYCLAW_OPENVIKING_PREFETCH_MAX_CHARS` max injected chars (default: `2800`) +- `TINYCLAW_OPENVIKING_PREFETCH_MAX_TURNS` max selected turns (default: `4`) + ### In-Chat Commands These commands work in Discord, Telegram, and WhatsApp: diff --git a/lib/agents.sh b/lib/agents.sh index a39a93b..1764b34 100644 --- a/lib/agents.sh +++ b/lib/agents.sh @@ -4,6 +4,84 @@ # AGENTS_DIR set after loading settings (uses workspace path) AGENTS_DIR="" +copy_openviking_tools_to_agent_dir() { + local agent_dir="$1" + local tools_src="$SCRIPT_DIR/lib/templates/agent-tools/openviking" + local tools_dest="$agent_dir/.tinyclaw/tools/openviking" + + [ -d "$tools_src" ] || return 0 + + mkdir -p "$tools_dest" + cp "$tools_src"/* "$tools_dest/" 2>/dev/null || true + + if [ -f "$SCRIPT_DIR/dist/tools/openviking-tool.js" ]; then + cp "$SCRIPT_DIR/dist/tools/openviking-tool.js" "$tools_dest/openviking-tool.js" + fi + + if [ -f "$SCRIPT_DIR/src/tools/openviking-tool.ts" ]; then + cp "$SCRIPT_DIR/src/tools/openviking-tool.ts" "$tools_dest/openviking-tool.ts" + fi + + chmod +x "$tools_dest/ovk.sh" "$tools_dest/ovk-ls.sh" "$tools_dest/ovk-read.sh" "$tools_dest/ovk-write.sh" 2>/dev/null || true +} + +sync_agent_tools() { + load_settings || return 0 + local agents_dir="$WORKSPACE_PATH" + [ -d "$agents_dir" ] || return 0 + + local target_agent="${1:-}" + if [ -n "$target_agent" ]; then + local agent_dir="$agents_dir/$target_agent" + [ -d "$agent_dir" ] || return 0 + copy_openviking_tools_to_agent_dir "$agent_dir" + return 0 + fi + + local agent_ids + agent_ids=$(jq -r '(.agents // {}) | keys[]' "$SETTINGS_FILE" 2>/dev/null) || return 0 + for agent_id in $agent_ids; do + local agent_dir="$agents_dir/$agent_id" + [ -d "$agent_dir" ] || continue + copy_openviking_tools_to_agent_dir "$agent_dir" + done +} + +agent_tools_sync_command() { + if [ ! -f "$SETTINGS_FILE" ]; then + echo -e "${RED}No settings file found. Run setup first.${NC}" + exit 1 + fi + + load_settings + AGENTS_DIR="$WORKSPACE_PATH" + + local target_agent="${1:-}" + if [ -n "$target_agent" ]; then + local exists + exists=$(jq -r "(.agents // {}).\"${target_agent}\" // empty" "$SETTINGS_FILE" 2>/dev/null) + if [ -z "$exists" ]; then + echo -e "${RED}Agent '${target_agent}' not found.${NC}" + exit 1 + fi + copy_openviking_tools_to_agent_dir "$AGENTS_DIR/$target_agent" + echo -e "${GREEN}✓ Synced OpenViking tools to @${target_agent}${NC}" + echo " Path: $AGENTS_DIR/$target_agent/.tinyclaw/tools/openviking" + return + fi + + local count=0 + local agent_ids + agent_ids=$(jq -r '(.agents // {}) | keys[]' "$SETTINGS_FILE" 2>/dev/null) + for agent_id in $agent_ids; do + copy_openviking_tools_to_agent_dir "$AGENTS_DIR/$agent_id" + count=$((count + 1)) + done + + echo -e "${GREEN}✓ Synced OpenViking tools to ${count} agent(s)${NC}" + echo " Tools path: //.tinyclaw/tools/openviking" +} + # Ensure all agent workspaces have .agents/skills symlinked ensure_agent_skills_links() { local skills_src="$SCRIPT_DIR/.agents/skills" @@ -277,6 +355,9 @@ agent_add() { echo " → Copied SOUL.md to .tinyclaw/" fi + copy_openviking_tools_to_agent_dir "$AGENTS_DIR/$AGENT_ID" + echo " → Synced OpenViking tools to .tinyclaw/tools/openviking/" + echo "" echo -e "${GREEN}✓ Agent '${AGENT_ID}' created!${NC}" echo -e " Directory: $AGENTS_DIR/$AGENT_ID" diff --git a/lib/daemon.sh b/lib/daemon.sh index 0e2e5b5..ba1a9fd 100644 --- a/lib/daemon.sh +++ b/lib/daemon.sh @@ -23,13 +23,14 @@ start_daemon() { if [ ! -d "$SCRIPT_DIR/dist" ]; then needs_build=true else - for ts_file in "$SCRIPT_DIR"/src/*.ts; do - local js_file="$SCRIPT_DIR/dist/$(basename "${ts_file%.ts}.js")" + while IFS= read -r -d '' ts_file; do + local rel_path="${ts_file#"$SCRIPT_DIR/src/"}" + local js_file="$SCRIPT_DIR/dist/${rel_path%.ts}.js" if [ ! -f "$js_file" ] || [ "$ts_file" -nt "$js_file" ]; then needs_build=true break fi - done + done < <(find "$SCRIPT_DIR/src" -type f -name '*.ts' ! -path '*/visualizer/*' -print0) fi if [ "$needs_build" = true ]; then echo -e "${YELLOW}Building TypeScript...${NC}" @@ -87,6 +88,7 @@ start_daemon() { # Ensure all agent workspaces have .agents/skills symlink ensure_agent_skills_links + sync_agent_tools # Validate tokens for channels that need them for ch in "${ACTIVE_CHANNELS[@]}"; do diff --git a/lib/templates/agent-tools/openviking/README.md b/lib/templates/agent-tools/openviking/README.md new file mode 100644 index 0000000..db41273 --- /dev/null +++ b/lib/templates/agent-tools/openviking/README.md @@ -0,0 +1,24 @@ +# OpenViking Tools + +Lightweight workspace tools for TinyClaw agents. + +## Environment + +- `OPENVIKING_BASE_URL` (optional, default `http://127.0.0.1:8320`) +- `OPENVIKING_API_KEY` (optional, sent as `X-API-Key`) +- `OPENVIKING_PROJECT` (optional, adds `?project=...`) + +## Commands + +- `./ovk.sh ls /` - list paths +- `./ovk.sh read /path/to/file.md` - read file content +- `./ovk.sh write /path/to/file.md "new content"` - write file content +- `./ovk.sh write-file /path/to/file.md ./local-file.md` - upload local file +- `./ovk.sh res-get viking://workspace/resource` - read resource by URI +- `./ovk.sh res-put viking://workspace/resource "content"` - write resource + +Shortcut wrappers: + +- `./ovk-ls.sh /` +- `./ovk-read.sh /path/to/file.md` +- `./ovk-write.sh /path/to/file.md "new content"` diff --git a/lib/templates/agent-tools/openviking/openviking-tool.js b/lib/templates/agent-tools/openviking/openviking-tool.js new file mode 100644 index 0000000..14a620f --- /dev/null +++ b/lib/templates/agent-tools/openviking/openviking-tool.js @@ -0,0 +1,274 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs_1 = __importDefault(require("fs")); +const os_1 = __importDefault(require("os")); +const path_1 = __importDefault(require("path")); +const HELP = `OpenViking workspace tool + +Usage: + node openviking-tool.js ls [--json] + node openviking-tool.js read [--json] + node openviking-tool.js write [--json] + node openviking-tool.js write-file [--json] + node openviking-tool.js res-get [--json] + node openviking-tool.js res-put [--mime ] [--json] + +Environment: + OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) + OPENVIKING_API_KEY Optional API key for X-API-Key header + OPENVIKING_PROJECT Optional project query (e.g. my-project) +`; +const args = process.argv.slice(2); +const jsonOutput = args.includes('--json'); +function fail(message) { + console.error(`[openviking-tool] ${message}`); + process.exit(1); +} +function getFlagValue(flag) { + const idx = args.indexOf(flag); + if (idx === -1) + return undefined; + if (!args[idx + 1] || args[idx + 1].startsWith('--')) { + fail(`Missing value for ${flag}`); + } + return args[idx + 1]; +} +function positionalArguments() { + const output = []; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--json') + continue; + if (arg === '--mime') { + i += 1; + continue; + } + output.push(arg); + } + return output; +} +function asObject(value) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value; + } + return {}; +} +function asArray(value) { + if (Array.isArray(value)) + return value; + return []; +} +function toUri(input) { + if (input.startsWith('viking://')) + return input; + if (input === '/') + return 'viking://resources'; + const normalized = input.startsWith('/') ? input.slice(1) : input; + return `viking://resources/${normalized}`; +} +async function request(endpoint, init) { + const baseUrl = process.env.OPENVIKING_BASE_URL || 'http://127.0.0.1:8320'; + const project = process.env.OPENVIKING_PROJECT; + const apiKey = process.env.OPENVIKING_API_KEY; + const url = new URL(endpoint, baseUrl); + if (project) { + url.searchParams.set('project', project); + } + const headers = new Headers(init?.headers || {}); + if (apiKey) + headers.set('X-API-Key', apiKey); + if (init?.body && !headers.has('Content-Type')) { + headers.set('Content-Type', 'application/json'); + } + const response = await fetch(url.toString(), { ...init, headers }); + const text = await response.text(); + let data = null; + if (text.trim()) { + try { + data = JSON.parse(text); + } + catch { + data = text; + } + } + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`); + } + return data; +} +function printJson(data) { + console.log(JSON.stringify(data, null, 2)); +} +function printList(data) { + const root = asObject(data); + const resultNode = root.result; + const dataNode = asObject(root.data); + const items = asArray(resultNode ?? dataNode.items ?? root.items); + if (!items.length) { + console.log('(empty)'); + return; + } + for (const item of items) { + const node = asObject(item); + const itemPath = String(node.path ?? node.uri ?? ''); + const itemType = String(node.type ?? node.kind ?? 'item'); + if (itemPath) { + console.log(`${itemType}\t${itemPath}`); + } + else { + console.log(JSON.stringify(item)); + } + } +} +function printRead(data) { + const root = asObject(data); + const resultNode = root.result; + const dataNode = asObject(root.data); + const content = dataNode.content ?? root.content ?? resultNode; + if (typeof content === 'string') { + console.log(content); + return; + } + printJson(data); +} +async function run() { + const positional = positionalArguments(); + if (!positional.length || positional[0] === 'help' || positional[0] === '--help') { + console.log(HELP); + return; + } + const command = positional[0]; + let response; + switch (command) { + case 'ls': { + if (!positional[1]) + fail('Usage: ls '); + const uri = toUri(positional[1]); + try { + response = await request(`/api/v1/fs/ls?uri=${encodeURIComponent(uri)}&output=agent`); + } + catch { + response = await request(`/api/v1/fs/ls?path=${encodeURIComponent(positional[1])}`); + } + if (jsonOutput) + printJson(response); + else + printList(response); + return; + } + case 'read': { + if (!positional[1]) + fail('Usage: read '); + const uri = toUri(positional[1]); + try { + response = await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + } + catch { + response = await request(`/api/v1/content/read?path=${encodeURIComponent(positional[1])}`); + } + if (jsonOutput) + printJson(response); + else + printRead(response); + return; + } + case 'write': { + if (!positional[1] || positional[2] === undefined) + fail('Usage: write '); + const content = positional[2]; + const targetUri = toUri(positional[1]); + const tmpFile = path_1.default.join(os_1.default.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`); + fs_1.default.writeFileSync(tmpFile, content, 'utf8'); + try { + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: targetUri, wait: true }), + }); + } + catch { + response = await request('/api/v1/content/write', { + method: 'POST', + body: JSON.stringify({ path: positional[1], content }), + }); + } + } + finally { + fs_1.default.rmSync(tmpFile, { force: true }); + } + if (jsonOutput) + printJson(response); + else + console.log(`Wrote content to ${targetUri}`); + return; + } + case 'write-file': { + if (!positional[1] || !positional[2]) + fail('Usage: write-file '); + const targetUri = toUri(positional[1]); + const localFile = positional[2]; + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: localFile, target: targetUri, wait: true }), + }); + } + catch { + const content = fs_1.default.readFileSync(localFile, 'utf8'); + response = await request('/api/v1/content/write', { + method: 'POST', + body: JSON.stringify({ path: positional[1], content }), + }); + } + if (jsonOutput) + printJson(response); + else + console.log(`Uploaded ${localFile} -> ${targetUri}`); + return; + } + case 'res-get': { + if (!positional[1]) + fail('Usage: res-get '); + const uri = toUri(positional[1]); + response = await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + if (jsonOutput) + printJson(response); + else + printRead(response); + return; + } + case 'res-put': { + if (!positional[1] || positional[2] === undefined) + fail('Usage: res-put [--mime ]'); + const content = positional[2]; + const uri = toUri(positional[1]); + const mimeType = getFlagValue('--mime') || 'text/plain'; + const _ = mimeType; + const tmpFile = path_1.default.join(os_1.default.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`); + fs_1.default.writeFileSync(tmpFile, content, 'utf8'); + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: uri, wait: true }), + }); + } + finally { + fs_1.default.rmSync(tmpFile, { force: true }); + } + if (jsonOutput) + printJson(response); + else + console.log(`Wrote resource ${uri}`); + return; + } + default: + fail(`Unknown command: ${command}\n\n${HELP}`); + } +} +run().catch((error) => { + fail(error instanceof Error ? error.message : String(error)); +}); +//# sourceMappingURL=openviking-tool.js.map \ No newline at end of file diff --git a/lib/templates/agent-tools/openviking/ovk-ls.sh b/lib/templates/agent-tools/openviking/ovk-ls.sh new file mode 100755 index 0000000..553aaa9 --- /dev/null +++ b/lib/templates/agent-tools/openviking/ovk-ls.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +node "$SCRIPT_DIR/openviking-tool.js" ls "$@" diff --git a/lib/templates/agent-tools/openviking/ovk-read.sh b/lib/templates/agent-tools/openviking/ovk-read.sh new file mode 100644 index 0000000..1567fc5 --- /dev/null +++ b/lib/templates/agent-tools/openviking/ovk-read.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +node "$SCRIPT_DIR/openviking-tool.js" read "$@" diff --git a/lib/templates/agent-tools/openviking/ovk-write.sh b/lib/templates/agent-tools/openviking/ovk-write.sh new file mode 100644 index 0000000..48ec1e7 --- /dev/null +++ b/lib/templates/agent-tools/openviking/ovk-write.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +node "$SCRIPT_DIR/openviking-tool.js" write "$@" diff --git a/lib/templates/agent-tools/openviking/ovk.sh b/lib/templates/agent-tools/openviking/ovk.sh new file mode 100644 index 0000000..4e8c317 --- /dev/null +++ b/lib/templates/agent-tools/openviking/ovk.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +node "$SCRIPT_DIR/openviking-tool.js" "$@" diff --git a/package.json b/package.json index eebef9f..bd11f59 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "build": "tsc && tsc -p tsconfig.visualizer.json && echo '{\"type\":\"module\"}' > dist/visualizer/package.json", "build:main": "tsc", "build:visualizer": "tsc -p tsconfig.visualizer.json && echo '{\"type\":\"module\"}' > dist/visualizer/package.json", + "test:prefetch-parser": "npm run build:main && node scripts/test-openviking-prefetch-parser.js", "whatsapp": "node dist/channels/whatsapp-client.js", "discord": "node dist/channels/discord-client.js", "telegram": "node dist/channels/telegram-client.js", diff --git a/scripts/test-openviking-prefetch-parser.js b/scripts/test-openviking-prefetch-parser.js new file mode 100644 index 0000000..02931dd --- /dev/null +++ b/scripts/test-openviking-prefetch-parser.js @@ -0,0 +1,74 @@ +#!/usr/bin/env node + +const assert = require('assert'); +const { + parseSessionTurns, + selectRelevantTurns, + buildPrefetchBlock, +} = require('../dist/lib/openviking-prefetch.js'); + +const fixture = `# TinyClaw Session (@assistant) + +- started_at: 2026-02-21T06:24:44.461Z +------ + +## Turn 2026-02-21T06:24:44.461Z + +- message_id: diag5_1771655068 +- source: external + +### User + +please remember: user_name=cheng; favorite=db + +------ + +[OpenViking Retrieved Context] + +- turn: 2026-02-21T06:20:46.699Z | message_id: diag3_1771654833 + user: please remember: user_name=cheng; favorite=db + assistant: Got it! I've remembered user_name=cheng; favorite=db +[End OpenViking Context] + +### Assistant + +I see from the retrieved context that I've already stored this information. +------ + +## Turn 2026-02-21T06:25:23.382Z + +- message_id: diag6_1771655118 +- source: external + +### User + +user_name=cheng; favorite=? + +------ + +[OpenViking Retrieved Context] + +- turn: 2026-02-21T06:24:44.461Z | message_id: diag5_1771655068 + user: please remember: user_name=cheng; favorite=db + assistant: I see from the retrieved context that I've already stored this information. +[End OpenViking Context] + +### Assistant + +Based on what I remember: +user_name=cheng; favorite=db +`; + +const turns = parseSessionTurns(fixture); +assert.strictEqual(turns.length, 2, `expected 2 turns, got ${turns.length}`); +assert.match(turns[0].user, /please remember: user_name=cheng; favorite=db/); +assert.doesNotMatch(turns[0].user, /\[OpenViking Retrieved Context\]/); +assert.match(turns[1].user, /favorite=\?/); + +const selected = selectRelevantTurns(turns, 'user_name=cheng; favorite=?', 3); +assert.ok(selected.length >= 1, 'expected at least one selected turn'); +const block = buildPrefetchBlock(selected, 2800); +assert.match(block, /\[OpenViking Retrieved Context\]/); +assert.match(block, /favorite=db/); + +console.log('ok - openviking prefetch parser regression test passed'); diff --git a/src/lib/agent-setup.ts b/src/lib/agent-setup.ts index 821f8b4..570ee00 100644 --- a/src/lib/agent-setup.ts +++ b/src/lib/agent-setup.ts @@ -86,6 +86,35 @@ export function ensureAgentDirectory(agentDir: string): void { if (fs.existsSync(sourceSoul)) { fs.copyFileSync(sourceSoul, path.join(targetTinyclaw, 'SOUL.md')); } + + // Copy OpenViking tool templates for CLI-driven workspace integrations. + const sourceToolsDir = path.join(SCRIPT_DIR, 'lib', 'templates', 'agent-tools', 'openviking'); + const targetToolsDir = path.join(targetTinyclaw, 'tools', 'openviking'); + if (fs.existsSync(sourceToolsDir)) { + copyDirSync(sourceToolsDir, targetToolsDir); + } + + // Prefer the compiled runtime JS from dist so tools work without TS runtime deps. + const sourceToolRuntime = path.join(SCRIPT_DIR, 'dist', 'tools', 'openviking-tool.js'); + if (fs.existsSync(sourceToolRuntime)) { + fs.mkdirSync(targetToolsDir, { recursive: true }); + fs.copyFileSync(sourceToolRuntime, path.join(targetToolsDir, 'openviking-tool.js')); + } + + // Keep source TS alongside runtime JS for easy customization by agents. + const sourceToolTs = path.join(SCRIPT_DIR, 'src', 'tools', 'openviking-tool.ts'); + if (fs.existsSync(sourceToolTs)) { + fs.mkdirSync(targetToolsDir, { recursive: true }); + fs.copyFileSync(sourceToolTs, path.join(targetToolsDir, 'openviking-tool.ts')); + } + + const toolShellScripts = ['ovk.sh', 'ovk-ls.sh', 'ovk-read.sh', 'ovk-write.sh']; + for (const script of toolShellScripts) { + const scriptPath = path.join(targetToolsDir, script); + if (fs.existsSync(scriptPath)) { + fs.chmodSync(scriptPath, 0o755); + } + } } /** diff --git a/src/lib/invoke.ts b/src/lib/invoke.ts index 2c21450..2ed7df5 100644 --- a/src/lib/invoke.ts +++ b/src/lib/invoke.ts @@ -6,7 +6,7 @@ import { SCRIPT_DIR, resolveClaudeModel, resolveCodexModel, resolveOpenCodeModel import { log } from './logging'; import { ensureAgentDirectory, updateAgentTeammates } from './agent-setup'; -export async function runCommand(command: string, args: string[], cwd?: string): Promise { +export async function runCommand(command: string, args: string[], cwd?: string, timeoutMs?: number): Promise { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd: cwd || SCRIPT_DIR, @@ -27,11 +27,26 @@ export async function runCommand(command: string, args: string[], cwd?: string): stderr += chunk; }); + let timedOut = false; + let timer: NodeJS.Timeout | null = null; + if (typeof timeoutMs === 'number' && timeoutMs > 0) { + timer = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, timeoutMs); + } + child.on('error', (error) => { + if (timer) clearTimeout(timer); reject(error); }); child.on('close', (code) => { + if (timer) clearTimeout(timer); + if (timedOut) { + reject(new Error(`Command timed out after ${timeoutMs}ms`)); + return; + } if (code === 0) { resolve(stdout); return; diff --git a/src/lib/openviking-prefetch.ts b/src/lib/openviking-prefetch.ts new file mode 100644 index 0000000..08747ff --- /dev/null +++ b/src/lib/openviking-prefetch.ts @@ -0,0 +1,103 @@ +export type SessionTurn = { + timestamp: string; + messageId: string; + user: string; + assistant: string; + index: number; +}; + +export function tokenizeForMatch(text: string): string[] { + const normalized = text.toLowerCase(); + return normalized + .split(/[^a-z0-9\u4e00-\u9fff]+/g) + .map((s) => s.trim()) + .filter((s) => s.length >= 2); +} + +export function parseSessionTurns(markdown: string): SessionTurn[] { + const turns: SessionTurn[] = []; + const turnHeadingRegex = /##\s*Turn\s+([^\n]+)/g; + const turnStarts: { index: number; timestamp: string }[] = []; + let m: RegExpExecArray | null; + while ((m = turnHeadingRegex.exec(markdown)) !== null) { + turnStarts.push({ index: m.index, timestamp: (m[1] || '').trim() }); + } + + for (let i = 0; i < turnStarts.length; i++) { + const start = turnStarts[i].index; + const end = i + 1 < turnStarts.length ? turnStarts[i + 1].index : markdown.length; + const chunk = markdown.slice(start, end); + + const idMatch = chunk.match(/- message_id:\s*([^\n]+)/); + const userMarker = '### User'; + const assistantMarker = '### Assistant'; + const userPos = chunk.indexOf(userMarker); + const assistantPos = chunk.indexOf(assistantMarker); + if (userPos === -1 || assistantPos === -1 || assistantPos <= userPos) continue; + + let userSection = chunk.slice(userPos + userMarker.length, assistantPos).trim(); + // Remove injected OpenViking prefetch block if present in the stored turn. + userSection = userSection.replace(/\n------\n\n\[OpenViking Retrieved Context\][\s\S]*?\[End OpenViking Context\]\s*$/s, '').trim(); + + let assistantSection = chunk.slice(assistantPos + assistantMarker.length).trim(); + assistantSection = assistantSection.replace(/\n- ended_at:[\s\S]*$/s, '').trim(); + + const user = userSection; + const assistant = assistantSection; + if (!user && !assistant) continue; + turns.push({ + timestamp: turnStarts[i].timestamp, + messageId: (idMatch?.[1] || '').trim(), + user, + assistant, + index: turns.length, + }); + } + + return turns; +} + +export function selectRelevantTurns(turns: SessionTurn[], query: string, maxTurns: number): SessionTurn[] { + if (!turns.length) return []; + const cap = Math.max(1, maxTurns); + const qTokens = tokenizeForMatch(query); + if (!qTokens.length) { + return turns.slice(-cap); + } + + const scored = turns.map((turn) => { + const haystack = `${turn.user}\n${turn.assistant}`.toLowerCase(); + let hit = 0; + for (const token of qTokens) { + if (haystack.includes(token)) hit += 1; + } + // Prefer more recent turns when score ties. + return { turn, score: hit, recency: turn.index }; + }); + + const topHits = scored + .filter((s) => s.score > 0) + .sort((a, b) => (b.score - a.score) || (b.recency - a.recency)) + .slice(0, cap) + .map((s) => s.turn); + + if (topHits.length > 0) return topHits; + return turns.slice(-cap); +} + +export function buildPrefetchBlock(turns: SessionTurn[], maxChars: number): string { + if (!turns.length) return ''; + const lines: string[] = []; + lines.push('[OpenViking Retrieved Context]'); + for (const turn of turns) { + lines.push(''); + lines.push(`- turn: ${turn.timestamp || 'unknown'} | message_id: ${turn.messageId || 'unknown'}`); + lines.push(` user: ${turn.user.replace(/\s+/g, ' ').trim()}`); + lines.push(` assistant: ${turn.assistant.replace(/\s+/g, ' ').trim()}`); + } + let output = lines.join('\n'); + if (output.length > maxChars) { + output = output.slice(0, maxChars) + '\n...'; + } + return output; +} diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 59dcf37..4d82152 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -24,8 +24,9 @@ import { } from './lib/config'; import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; -import { invokeAgent } from './lib/invoke'; +import { invokeAgent, runCommand } from './lib/invoke'; import { jsonrepair } from 'jsonrepair'; +import { SessionTurn, parseSessionTurns, selectRelevantTurns, buildPrefetchBlock } from './lib/openviking-prefetch'; /** Parse JSON with automatic repair for malformed content (e.g. bad escapes). */ function safeParseJSON(raw: string, label?: string): T { @@ -52,6 +53,180 @@ const conversations = new Map(); const MAX_CONVERSATION_MESSAGES = 50; const LONG_RESPONSE_THRESHOLD = 4000; +const OPENVIKING_AUTOSYNC_ENABLED = process.env.TINYCLAW_OPENVIKING_AUTOSYNC !== '0'; +const OPENVIKING_PREFETCH_ENABLED = process.env.TINYCLAW_OPENVIKING_PREFETCH !== '0'; +const OPENVIKING_PREFETCH_TIMEOUT_MS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS || 5000); +const OPENVIKING_PREFETCH_MAX_CHARS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_MAX_CHARS || 2800); +const OPENVIKING_PREFETCH_MAX_TURNS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_MAX_TURNS || 4); +const OPENVIKING_SESSION_ROOT = '/tinyclaw/sessions'; +const openVikingSyncChains = new Map>(); + +function getOpenVikingToolPath(workspacePath: string, agentId: string): string | null { + const toolPath = path.join(workspacePath, agentId, '.tinyclaw', 'tools', 'openviking', 'openviking-tool.js'); + if (!fs.existsSync(toolPath)) return null; + return toolPath; +} + +function getOpenVikingRuntimeDir(workspacePath: string, agentId: string): string { + return path.join(workspacePath, agentId, '.tinyclaw', 'runtime', 'openviking'); +} + +function getActiveSessionFile(workspacePath: string, agentId: string): string { + return path.join(getOpenVikingRuntimeDir(workspacePath, agentId), 'active-session.md'); +} + +function ensureActiveSessionFile(workspacePath: string, agentId: string): string { + const runtimeDir = getOpenVikingRuntimeDir(workspacePath, agentId); + const sessionFile = getActiveSessionFile(workspacePath, agentId); + if (!fs.existsSync(runtimeDir)) { + fs.mkdirSync(runtimeDir, { recursive: true }); + } + if (!fs.existsSync(sessionFile)) { + const header = [ + `# TinyClaw Session (@${agentId})`, + '', + `- started_at: ${new Date().toISOString()}`, + '' + ].join('\n'); + fs.writeFileSync(sessionFile, header); + } + return sessionFile; +} + +function enqueueOpenVikingSync(agentId: string, task: () => Promise): void { + const current = openVikingSyncChains.get(agentId) || Promise.resolve(); + const next = current + .then(task) + .catch((error) => { + log('WARN', `OpenViking sync failed for @${agentId}: ${(error as Error).message}`); + }); + openVikingSyncChains.set(agentId, next); + next.finally(() => { + if (openVikingSyncChains.get(agentId) === next) { + openVikingSyncChains.delete(agentId); + } + }); +} + +async function writeSessionFileToOpenViking( + workspacePath: string, + agentId: string, + localFile: string, + targetPath: string +): Promise { + if (!OPENVIKING_AUTOSYNC_ENABLED) return; + const toolPath = getOpenVikingToolPath(workspacePath, agentId); + if (!toolPath) return; + await runCommand('node', [toolPath, 'write-file', targetPath, localFile], path.join(workspacePath, agentId)); +} + +async function finalizeOpenVikingSession(workspacePath: string, agentId: string): Promise { + const sessionFile = getActiveSessionFile(workspacePath, agentId); + if (!fs.existsSync(sessionFile)) return; + + const currentContent = fs.readFileSync(sessionFile, 'utf8').trim(); + if (!currentContent) return; + + const endedAt = new Date().toISOString(); + const sessionCloseNote = `\n\n- ended_at: ${endedAt}\n`; + fs.appendFileSync(sessionFile, sessionCloseNote); + + const safeTimestamp = endedAt.replace(/[:.]/g, '-'); + await writeSessionFileToOpenViking( + workspacePath, + agentId, + sessionFile, + `${OPENVIKING_SESSION_ROOT}/${agentId}/closed/${safeTimestamp}.md` + ); + + fs.rmSync(sessionFile, { force: true }); +} + +async function appendTurnAndSyncOpenViking( + workspacePath: string, + agentId: string, + messageId: string, + userMessage: string, + assistantResponse: string, + isInternal: boolean +): Promise { + const sessionFile = ensureActiveSessionFile(workspacePath, agentId); + const turnTime = new Date().toISOString(); + const turnBlock = [ + '------', + '', + `## Turn ${turnTime}`, + '', + `- message_id: ${messageId}`, + `- source: ${isInternal ? 'internal' : 'external'}`, + '', + '### User', + '', + userMessage.trim(), + '', + '### Assistant', + '', + assistantResponse.trim(), + '' + ].join('\n'); + fs.appendFileSync(sessionFile, turnBlock); + + await writeSessionFileToOpenViking( + workspacePath, + agentId, + sessionFile, + `${OPENVIKING_SESSION_ROOT}/${agentId}/active.md` + ); +} + +async function fetchOpenVikingPrefetchContext(workspacePath: string, agentId: string, query: string): Promise { + if (!OPENVIKING_PREFETCH_ENABLED) return ''; + const toolPath = getOpenVikingToolPath(workspacePath, agentId); + if (!toolPath) return ''; + + const readTargets = [ + `${OPENVIKING_SESSION_ROOT}/${agentId}/active.md`, + `${OPENVIKING_SESSION_ROOT}/${agentId}/closed`, + ]; + + const allTurns: SessionTurn[] = []; + const diagnostics: string[] = []; + for (const target of readTargets) { + try { + const output = await runCommand( + 'node', + [toolPath, 'read', target], + path.join(workspacePath, agentId), + OPENVIKING_PREFETCH_TIMEOUT_MS + ); + const content = output.trim(); + if (!content || content.startsWith('[openviking-tool]')) { + diagnostics.push(`${target}:empty`); + continue; + } + const parsed = parseSessionTurns(content); + diagnostics.push(`${target}:chars=${content.length},turns=${parsed.length}`); + allTurns.push(...parsed); + } catch { + diagnostics.push(`${target}:error`); + // Best-effort: ignore missing/unreadable targets. + } + } + + const dedup = new Map(); + for (const turn of allTurns) { + const key = `${turn.messageId}|${turn.timestamp}|${turn.user}|${turn.assistant}`; + dedup.set(key, turn); + } + const turns = Array.from(dedup.values()); + if (!turns.length) { + log('INFO', `OpenViking prefetch empty for @${agentId}: ${diagnostics.join(' | ')}`); + return ''; + } + + const selected = selectRelevantTurns(turns, query, OPENVIKING_PREFETCH_MAX_TURNS); + return buildPrefetchBlock(selected, OPENVIKING_PREFETCH_MAX_CHARS); +} /** * If a response exceeds the threshold, save full text as a .md file @@ -336,6 +511,10 @@ async function processMessage(messageFile: string): Promise { if (shouldReset) { fs.unlinkSync(agentResetFlag); + enqueueOpenVikingSync(agentId, async () => { + await finalizeOpenVikingSession(workspacePath, agentId); + log('INFO', `OpenViking session finalized for @${agentId}`); + }); } // For internal messages: append pending response indicator so the agent @@ -351,6 +530,20 @@ async function processMessage(messageFile: string): Promise { } } + if (!isInternal) { + try { + const retrievedContext = await fetchOpenVikingPrefetchContext(workspacePath, agentId, message); + if (retrievedContext) { + message += `\n\n------\n\n${retrievedContext}\n[End OpenViking Context]`; + log('INFO', `OpenViking prefetch hit for @${agentId}: injected ${retrievedContext.length} chars`); + } else { + log('INFO', `OpenViking prefetch miss for @${agentId}`); + } + } catch (error) { + log('WARN', `OpenViking prefetch skipped for @${agentId}: ${(error as Error).message}`); + } + } + // Invoke agent emitEvent('chain_step_start', { agentId, agentName: agent.name, fromAgent: messageData.fromAgent || null }); let response: string; @@ -365,6 +558,10 @@ async function processMessage(messageFile: string): Promise { emitEvent('chain_step_done', { agentId, agentName: agent.name, responseLength: response.length, responseText: response }); + enqueueOpenVikingSync(agentId, async () => { + await appendTurnAndSyncOpenViking(workspacePath, agentId, messageId, message, response, isInternal); + }); + // --- No team context: simple response to user --- if (!teamContext) { let finalResponse = response.trim(); diff --git a/src/tools/openviking-tool.ts b/src/tools/openviking-tool.ts new file mode 100644 index 0000000..da4d8f3 --- /dev/null +++ b/src/tools/openviking-tool.ts @@ -0,0 +1,347 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; + +type Command = 'ls' | 'read' | 'write' | 'write-file' | 'res-get' | 'res-put'; + +const HELP = `OpenViking workspace tool + +Usage: + node openviking-tool.js ls [--json] + node openviking-tool.js read [--json] + node openviking-tool.js write [--json] + node openviking-tool.js write-file [--json] + node openviking-tool.js res-get [--json] + node openviking-tool.js res-put [--mime ] [--json] + +Environment: + OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) + OPENVIKING_API_KEY Optional API key for X-API-Key header + OPENVIKING_PROJECT Optional project query (e.g. my-project) +`; + +const args = process.argv.slice(2); +const jsonOutput = args.includes('--json'); + +function fail(message: string): never { + console.error(`[openviking-tool] ${message}`); + process.exit(1); +} + +function getFlagValue(flag: string): string | undefined { + const idx = args.indexOf(flag); + if (idx === -1) return undefined; + if (!args[idx + 1] || args[idx + 1].startsWith('--')) { + fail(`Missing value for ${flag}`); + } + return args[idx + 1]; +} + +function positionalArguments(): string[] { + const output: string[] = []; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--json') continue; + if (arg === '--mime') { + i += 1; + continue; + } + output.push(arg); + } + return output; +} + +function asObject(value: unknown): Record { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + return {}; +} + +function asArray(value: unknown): unknown[] { + if (Array.isArray(value)) return value; + return []; +} + +type ListItem = { + uri: string; + isDir: boolean; + modTime: string; + index: number; +}; + +function toUri(input: string): string { + if (input.startsWith('viking://')) return input; + if (input === '/') return 'viking://resources'; + const normalized = input.startsWith('/') ? input.slice(1) : input; + return `viking://resources/${normalized}`; +} + +async function request(endpoint: string, init?: RequestInit): Promise { + const baseUrl = process.env.OPENVIKING_BASE_URL || 'http://127.0.0.1:8320'; + const project = process.env.OPENVIKING_PROJECT; + const apiKey = process.env.OPENVIKING_API_KEY; + const url = new URL(endpoint, baseUrl); + + if (project) { + url.searchParams.set('project', project); + } + + const headers = new Headers(init?.headers || {}); + if (apiKey) headers.set('X-API-Key', apiKey); + if (init?.body && !headers.has('Content-Type')) { + headers.set('Content-Type', 'application/json'); + } + + const response = await fetch(url.toString(), { ...init, headers }); + const text = await response.text(); + + let data: JsonValue = null; + if (text.trim()) { + try { + data = JSON.parse(text) as JsonValue; + } catch { + data = text; + } + } + + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}: ${typeof data === 'string' ? data : JSON.stringify(data)}`); + } + + return data; +} + +function printJson(data: JsonValue): void { + console.log(JSON.stringify(data, null, 2)); +} + +function printList(data: JsonValue): void { + const root = asObject(data); + const resultNode = root.result; + const dataNode = asObject(root.data); + const items = asArray(resultNode ?? dataNode.items ?? root.items); + + if (!items.length) { + console.log('(empty)'); + return; + } + + for (const item of items) { + const node = asObject(item); + const itemPath = String(node.path ?? node.uri ?? ''); + const itemType = String(node.type ?? node.kind ?? 'item'); + if (itemPath) { + console.log(`${itemType}\t${itemPath}`); + } else { + console.log(JSON.stringify(item)); + } + } +} + +function printRead(data: JsonValue): void { + const root = asObject(data); + const resultNode = root.result; + const dataNode = asObject(root.data); + const content = dataNode.content ?? root.content ?? resultNode; + + if (typeof content === 'string') { + console.log(content); + return; + } + + printJson(data); +} + +function extractUris(data: JsonValue): string[] { + const root = asObject(data); + const items = asArray(root.result ?? asObject(root.data).items ?? root.items); + return items + .map((item) => String(asObject(item).uri ?? asObject(item).path ?? '')) + .filter((uri) => uri.length > 0); +} + +function extractListItems(data: JsonValue): ListItem[] { + const root = asObject(data); + const items = asArray(root.result ?? asObject(root.data).items ?? root.items); + const out: ListItem[] = []; + for (let i = 0; i < items.length; i++) { + const node = asObject(items[i]); + const uri = String(node.uri ?? node.path ?? ''); + if (!uri) continue; + out.push({ + uri, + isDir: Boolean(node.isDir), + modTime: String(node.modTime ?? ''), + index: i, + }); + } + return out; +} + +function modTimeScore(modTime: string): number { + const m = modTime.match(/^(\d{2}):(\d{2}):(\d{2})$/); + if (!m) return -1; + const hh = Number(m[1]); + const mm = Number(m[2]); + const ss = Number(m[3]); + return hh * 3600 + mm * 60 + ss; +} + +function pickNewestReadableFileUri(listData: JsonValue): string | null { + const candidates = extractListItems(listData) + .filter((item) => !item.isDir) + .filter((item) => item.uri.endsWith('.md') || item.uri.endsWith('.txt')); + if (!candidates.length) return null; + + candidates.sort((a, b) => { + const dt = modTimeScore(b.modTime) - modTimeScore(a.modTime); + if (dt !== 0) return dt; + return b.index - a.index; + }); + return candidates[0].uri; +} + +async function readWithDirectoryFallback(uri: string, rawPath: string): Promise { + try { + const stat = await request(`/api/v1/fs/stat?uri=${encodeURIComponent(uri)}`); + const isDir = Boolean(asObject(asObject(stat).result).isDir); + if (!isDir) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + } + const listed = await request(`/api/v1/fs/ls?uri=${encodeURIComponent(uri)}&recursive=true&output=agent`); + const fileUri = pickNewestReadableFileUri(listed); + if (!fileUri) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + } + return await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUri)}`); + } catch (primaryError) { + try { + return await request(`/api/v1/content/read?path=${encodeURIComponent(rawPath)}`); + } catch { + throw primaryError; + } + } +} + +async function run(): Promise { + const positional = positionalArguments(); + if (!positional.length || positional[0] === 'help' || positional[0] === '--help') { + console.log(HELP); + return; + } + + const command = positional[0] as Command; + let response: JsonValue; + + switch (command) { + case 'ls': { + if (!positional[1]) fail('Usage: ls '); + const uri = toUri(positional[1]); + try { + response = await request(`/api/v1/fs/ls?uri=${encodeURIComponent(uri)}&output=agent`); + } catch { + response = await request(`/api/v1/fs/ls?path=${encodeURIComponent(positional[1])}`); + } + if (jsonOutput) printJson(response); + else printList(response); + return; + } + case 'read': { + if (!positional[1]) fail('Usage: read '); + const uri = toUri(positional[1]); + response = await readWithDirectoryFallback(uri, positional[1]); + if (jsonOutput) printJson(response); + else printRead(response); + return; + } + case 'write': { + if (!positional[1] || positional[2] === undefined) fail('Usage: write '); + const content = positional[2]; + const targetUri = toUri(positional[1]); + const tmpFile = path.join(os.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`); + fs.writeFileSync(tmpFile, content, 'utf8'); + try { + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: targetUri, wait: true }), + }); + } catch { + response = await request('/api/v1/content/write', { + method: 'POST', + body: JSON.stringify({ path: positional[1], content }), + }); + } + } finally { + fs.rmSync(tmpFile, { force: true }); + } + if (jsonOutput) printJson(response); + else console.log(`Wrote content to ${targetUri}`); + return; + } + case 'write-file': { + if (!positional[1] || !positional[2]) fail('Usage: write-file '); + const targetUri = toUri(positional[1]); + const localFile = positional[2]; + try { + const ext = path.extname(localFile) || '.txt'; + const tmpFile = path.join(os.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`); + fs.copyFileSync(localFile, tmpFile); + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: targetUri, wait: true }), + }); + } finally { + fs.rmSync(tmpFile, { force: true }); + } + } catch { + const content = fs.readFileSync(localFile, 'utf8'); + response = await request('/api/v1/content/write', { + method: 'POST', + body: JSON.stringify({ path: positional[1], content }), + }); + } + if (jsonOutput) printJson(response); + else console.log(`Uploaded ${localFile} -> ${targetUri}`); + return; + } + case 'res-get': { + if (!positional[1]) fail('Usage: res-get '); + const uri = toUri(positional[1]); + response = await readWithDirectoryFallback(uri, positional[1]); + if (jsonOutput) printJson(response); + else printRead(response); + return; + } + case 'res-put': { + if (!positional[1] || positional[2] === undefined) fail('Usage: res-put [--mime ]'); + const content = positional[2]; + const uri = toUri(positional[1]); + const mimeType = getFlagValue('--mime') || 'text/plain'; + const _ = mimeType; + const tmpFile = path.join(os.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}.txt`); + fs.writeFileSync(tmpFile, content, 'utf8'); + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: uri, wait: true }), + }); + } finally { + fs.rmSync(tmpFile, { force: true }); + } + if (jsonOutput) printJson(response); + else console.log(`Wrote resource ${uri}`); + return; + } + default: + fail(`Unknown command: ${command}\n\n${HELP}`); + } +} + +run().catch((error) => { + fail(error instanceof Error ? error.message : String(error)); +}); diff --git a/tinyclaw.sh b/tinyclaw.sh index 7073a24..55279be 100755 --- a/tinyclaw.sh +++ b/tinyclaw.sh @@ -375,6 +375,24 @@ case "${1:-}" in ;; esac ;; + tools) + case "${2:-}" in + sync) + agent_tools_sync_command "$3" + ;; + *) + echo "Usage: $0 tools sync [agent_id]" + echo "" + echo "Tools Commands:" + echo " sync [agent_id] Sync OpenViking tools into agent workspace(s)" + echo "" + echo "Examples:" + echo " $0 tools sync" + echo " $0 tools sync coder" + exit 1 + ;; + esac + ;; pairing) pairing_command "${2:-}" "${3:-}" ;; @@ -391,7 +409,7 @@ case "${1:-}" in local_names=$(IFS='|'; echo "${ALL_CHANNELS[*]}") echo -e "${BLUE}TinyClaw - Claude Code + Messaging Channels${NC}" echo "" - echo "Usage: $0 {start|stop|restart|status|setup|send|logs|reset |channels|provider|model|agent|team|pairing|update|attach}" + echo "Usage: $0 {start|stop|restart|status|setup|send|logs|reset |channels|provider|model|agent|team|tools|pairing|update|attach}" echo "" echo "Commands:" echo " start Start TinyClaw" @@ -407,6 +425,7 @@ case "${1:-}" in echo " model [name] Show or switch AI model" echo " agent {list|add|remove|show|reset|provider} Manage agents" echo " team {list|add|remove|show|visualize} Manage teams" + echo " tools sync [agent_id] Sync OpenViking tools into agent workspace(s)" echo " pairing {pending|approved|list|approve |unpair } Manage sender approvals" echo " update Update TinyClaw to latest version" echo " attach Attach to tmux session" @@ -422,6 +441,8 @@ case "${1:-}" in echo " $0 agent add" echo " $0 team list" echo " $0 team visualize dev" + echo " $0 tools sync" + echo " $0 tools sync coder" echo " $0 pairing pending" echo " $0 pairing approve ABCD1234" echo " $0 pairing unpair telegram 123456789" From ce6309d1eb47e0315820207b20481944b83d5fa3 Mon Sep 17 00:00:00 2001 From: MZ Date: Sun, 22 Feb 2026 09:58:38 -0700 Subject: [PATCH 2/3] feat: harden openviking prefetch and writeback sync --- .../agent-tools/openviking/openviking-tool.js | 224 +++++++++++++++++- src/lib/openviking-prefetch.ts | 39 ++- src/queue-processor.ts | 116 +++++++-- src/tools/openviking-tool.ts | 144 ++++++++++- 4 files changed, 479 insertions(+), 44 deletions(-) diff --git a/lib/templates/agent-tools/openviking/openviking-tool.js b/lib/templates/agent-tools/openviking/openviking-tool.js index 14a620f..51d0016 100644 --- a/lib/templates/agent-tools/openviking/openviking-tool.js +++ b/lib/templates/agent-tools/openviking/openviking-tool.js @@ -15,6 +15,7 @@ Usage: node openviking-tool.js write-file [--json] node openviking-tool.js res-get [--json] node openviking-tool.js res-put [--mime ] [--json] + node openviking-tool.js find-uris [--limit ] [--score-threshold ] [--json] Environment: OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) @@ -46,6 +47,10 @@ function positionalArguments() { i += 1; continue; } + if (arg === '--limit' || arg === '--score-threshold') { + i += 1; + continue; + } output.push(arg); } return output; @@ -61,6 +66,7 @@ function asArray(value) { return value; return []; } +const DIRECTORY_READ_MAX_FILES = 8; function toUri(input) { if (input.startsWith('viking://')) return input; @@ -134,6 +140,167 @@ function printRead(data) { } printJson(data); } +function extractFindMatches(data) { + const root = asObject(data); + const resultNode = asObject(root.result); + const groups = ['memories', 'resources', 'skills']; + const out = []; + for (const g of groups) { + const items = asArray(resultNode[g]); + for (const item of items) { + const node = asObject(item); + const uri = String(node.uri ?? ''); + if (!uri) + continue; + out.push({ + uri, + score: Number(node.score ?? 0), + isLeaf: Boolean(node.is_leaf), + }); + } + } + out.sort((a, b) => b.score - a.score); + return out; +} +function printFindUris(data) { + const matches = extractFindMatches(data); + if (!matches.length) + return; + for (const m of matches) { + console.log(`${m.score}\t${m.uri}`); + } +} +function extractUris(data) { + const root = asObject(data); + const items = asArray(root.result ?? asObject(root.data).items ?? root.items); + return items + .map((item) => String(asObject(item).uri ?? asObject(item).path ?? '')) + .filter((uri) => uri.length > 0); +} +function extractListItems(data) { + const root = asObject(data); + const items = asArray(root.result ?? asObject(root.data).items ?? root.items); + const out = []; + for (let i = 0; i < items.length; i++) { + const node = asObject(items[i]); + const uri = String(node.uri ?? node.path ?? ''); + if (!uri) + continue; + out.push({ + uri, + isDir: Boolean(node.isDir), + modTime: String(node.modTime ?? ''), + index: i, + }); + } + return out; +} +function modTimeScore(modTime) { + const parsed = Date.parse(modTime); + if (!Number.isNaN(parsed)) + return parsed; + const m = modTime.match(/^(\d{2}):(\d{2}):(\d{2})$/); + if (!m) + return -1; + const hh = Number(m[1]); + const mm = Number(m[2]); + const ss = Number(m[3]); + return hh * 3600 + mm * 60 + ss; +} +function uriTimestampScore(uri) { + const openVikingLeaf = uri.match(/openviking-(\d{10,})-/); + if (openVikingLeaf) { + return Number(openVikingLeaf[1]); + } + const compactTurn = uri.match(/Turn_(\d{4}-\d{2}-\d{2})T(\d{2})(\d{2})(\d{2})(\d{1,6})?Z/); + if (compactTurn) { + const [, date, hh, mm, ss, fracRaw = ''] = compactTurn; + const ms = fracRaw.slice(0, 3).padEnd(3, '0'); + const iso = `${date}T${hh}:${mm}:${ss}.${ms}Z`; + const parsed = Date.parse(iso); + if (!Number.isNaN(parsed)) + return parsed; + } + const archivedSession = uri.match(/\/(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3,6})Z\.md\//); + if (archivedSession) { + const [, date, hh, mm, ss, fracRaw] = archivedSession; + const ms = fracRaw.slice(0, 3).padEnd(3, '0'); + const iso = `${date}T${hh}:${mm}:${ss}.${ms}Z`; + const parsed = Date.parse(iso); + if (!Number.isNaN(parsed)) + return parsed; + } + return -1; +} +function isReadableCandidateUri(uri) { + const base = uri.split('/').pop() || ''; + if (!base) + return false; + if (base.startsWith('.')) + return false; // Exclude .overview/.abstract summaries. + if (!(uri.endsWith('.md') || uri.endsWith('.txt'))) + return false; + return true; +} +function pickNewestReadableFileUris(listData, limit) { + const candidates = extractListItems(listData) + .filter((item) => !item.isDir) + .filter((item) => isReadableCandidateUri(item.uri)); + if (!candidates.length) + return []; + candidates.sort((a, b) => { + const ts = uriTimestampScore(b.uri) - uriTimestampScore(a.uri); + if (ts !== 0) + return ts; + const dt = modTimeScore(b.modTime) - modTimeScore(a.modTime); + if (dt !== 0) + return dt; + return b.index - a.index; + }); + return candidates.slice(0, Math.max(1, limit)).map((item) => item.uri); +} +async function readWithDirectoryFallback(uri, rawPath) { + try { + const stat = await request(`/api/v1/fs/stat?uri=${encodeURIComponent(uri)}`); + const isDir = Boolean(asObject(asObject(stat).result).isDir); + if (!isDir) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + } + const listed = await request(`/api/v1/fs/ls?uri=${encodeURIComponent(uri)}&recursive=true&output=agent`); + const fileUris = pickNewestReadableFileUris(listed, DIRECTORY_READ_MAX_FILES); + if (!fileUris.length) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + } + const chunks = []; + for (const fileUri of fileUris) { + try { + const data = await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUri)}`); + const root = asObject(data); + const dataNode = asObject(root.data); + const resultNode = root.result; + const content = dataNode.content ?? root.content ?? resultNode; + if (typeof content === 'string' && content.trim()) { + chunks.push(content.trim()); + } + } + catch { + // Best effort: skip unreadable leaf. + } + } + if (!chunks.length) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUris[0])}`); + } + return { content: chunks.join('\n\n------\n\n') }; + } + catch (primaryError) { + try { + return await request(`/api/v1/content/read?path=${encodeURIComponent(rawPath)}`); + } + catch { + throw primaryError; + } + } +} async function run() { const positional = positionalArguments(); if (!positional.length || positional[0] === 'help' || positional[0] === '--help') { @@ -163,12 +330,7 @@ async function run() { if (!positional[1]) fail('Usage: read '); const uri = toUri(positional[1]); - try { - response = await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); - } - catch { - response = await request(`/api/v1/content/read?path=${encodeURIComponent(positional[1])}`); - } + response = await readWithDirectoryFallback(uri, positional[1]); if (jsonOutput) printJson(response); else @@ -211,10 +373,18 @@ async function run() { const targetUri = toUri(positional[1]); const localFile = positional[2]; try { - response = await request('/api/v1/resources', { - method: 'POST', - body: JSON.stringify({ path: localFile, target: targetUri, wait: true }), - }); + const ext = path_1.default.extname(localFile) || '.txt'; + const tmpFile = path_1.default.join(os_1.default.tmpdir(), `openviking-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`); + fs_1.default.copyFileSync(localFile, tmpFile); + try { + response = await request('/api/v1/resources', { + method: 'POST', + body: JSON.stringify({ path: tmpFile, target: targetUri, wait: true }), + }); + } + finally { + fs_1.default.rmSync(tmpFile, { force: true }); + } } catch { const content = fs_1.default.readFileSync(localFile, 'utf8'); @@ -233,7 +403,7 @@ async function run() { if (!positional[1]) fail('Usage: res-get '); const uri = toUri(positional[1]); - response = await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); + response = await readWithDirectoryFallback(uri, positional[1]); if (jsonOutput) printJson(response); else @@ -264,6 +434,38 @@ async function run() { console.log(`Wrote resource ${uri}`); return; } + case 'find-uris': { + if (!positional[1] || !positional[2]) + fail('Usage: find-uris [--limit ] [--score-threshold ]'); + const query = positional[1]; + const targetUri = toUri(positional[2]); + const limitRaw = getFlagValue('--limit'); + const scoreThresholdRaw = getFlagValue('--score-threshold'); + const limit = limitRaw ? Number(limitRaw) : 12; + if (!Number.isFinite(limit) || limit <= 0) + fail('Invalid --limit value'); + let scoreThreshold; + if (scoreThresholdRaw !== undefined) { + const parsed = Number(scoreThresholdRaw); + if (!Number.isFinite(parsed)) + fail('Invalid --score-threshold value'); + scoreThreshold = parsed; + } + response = await request('/api/v1/search/find', { + method: 'POST', + body: JSON.stringify({ + query, + target_uri: targetUri, + limit, + score_threshold: scoreThreshold, + }), + }); + if (jsonOutput) + printJson(response); + else + printFindUris(response); + return; + } default: fail(`Unknown command: ${command}\n\n${HELP}`); } diff --git a/src/lib/openviking-prefetch.ts b/src/lib/openviking-prefetch.ts index 08747ff..aa952bf 100644 --- a/src/lib/openviking-prefetch.ts +++ b/src/lib/openviking-prefetch.ts @@ -7,11 +7,18 @@ export type SessionTurn = { }; export function tokenizeForMatch(text: string): string[] { + const stopwords = new Set([ + 'the', 'is', 'are', 'am', 'was', 'were', 'be', 'been', 'being', + 'a', 'an', 'to', 'of', 'in', 'on', 'for', 'with', 'by', 'from', + 'what', 'when', 'where', 'which', 'who', 'whom', 'why', 'how', + 'please', 'reply', 'answer', 'only', 'just', 'tell', 'me', 'you', 'your', + ]); const normalized = text.toLowerCase(); return normalized .split(/[^a-z0-9\u4e00-\u9fff]+/g) .map((s) => s.trim()) - .filter((s) => s.length >= 2); + .filter((s) => s.length >= 2) + .filter((s) => !stopwords.has(s)); } export function parseSessionTurns(markdown: string): SessionTurn[] { @@ -41,6 +48,7 @@ export function parseSessionTurns(markdown: string): SessionTurn[] { let assistantSection = chunk.slice(assistantPos + assistantMarker.length).trim(); assistantSection = assistantSection.replace(/\n- ended_at:[\s\S]*$/s, '').trim(); + assistantSection = assistantSection.replace(/\n#\s*TinyClaw Session[\s\S]*$/s, '').trim(); const user = userSection; const assistant = assistantSection; @@ -60,26 +68,37 @@ export function parseSessionTurns(markdown: string): SessionTurn[] { export function selectRelevantTurns(turns: SessionTurn[], query: string, maxTurns: number): SessionTurn[] { if (!turns.length) return []; const cap = Math.max(1, maxTurns); - const qTokens = tokenizeForMatch(query); + const qTokens = Array.from(new Set(tokenizeForMatch(query))); if (!qTokens.length) { return turns.slice(-cap); } const scored = turns.map((turn) => { - const haystack = `${turn.user}\n${turn.assistant}`.toLowerCase(); + const userTokens = new Set(tokenizeForMatch(turn.user)); + const assistantTokens = new Set(tokenizeForMatch(turn.assistant)); + const userIsQuestion = /[??]/.test(turn.user) || /^(what|when|where|which|who|why|how)\b/i.test(turn.user.trim()); + const assistantUncertain = /(don't have|do not have|don't know|do not know|don't see|do not see|no information|not in (the )?provided context|抱歉|没有.*信息)/i.test(turn.assistant); let hit = 0; for (const token of qTokens) { - if (haystack.includes(token)) hit += 1; + if (userTokens.has(token)) hit += 2; + if (assistantTokens.has(token)) hit += 1; + } + // De-prioritize turns where assistant explicitly says it has no info. + if (assistantUncertain) { + hit -= userIsQuestion ? 8 : 3; } // Prefer more recent turns when score ties. - return { turn, score: hit, recency: turn.index }; + return { turn, score: hit, recency: turn.index, uncertain: assistantUncertain }; }); - const topHits = scored - .filter((s) => s.score > 0) - .sort((a, b) => (b.score - a.score) || (b.recency - a.recency)) - .slice(0, cap) - .map((s) => s.turn); + const minScore = qTokens.length >= 3 ? 2 : 1; + const positive = scored + .filter((s) => s.score >= minScore) + .sort((a, b) => (b.score - a.score) || (b.recency - a.recency)); + + const informative = positive.filter((s) => !s.uncertain); + const candidates = informative.length > 0 ? informative : positive; + const topHits = candidates.slice(0, cap).map((s) => s.turn); if (topHits.length > 0) return topHits; return turns.slice(-cap); diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 4d82152..81b7411 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -26,7 +26,7 @@ import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; import { invokeAgent, runCommand } from './lib/invoke'; import { jsonrepair } from 'jsonrepair'; -import { SessionTurn, parseSessionTurns, selectRelevantTurns, buildPrefetchBlock } from './lib/openviking-prefetch'; +import { SessionTurn, parseSessionTurns, buildPrefetchBlock } from './lib/openviking-prefetch'; /** Parse JSON with automatic repair for malformed content (e.g. bad escapes). */ function safeParseJSON(raw: string, label?: string): T { @@ -61,6 +61,16 @@ const OPENVIKING_PREFETCH_MAX_TURNS = Number(process.env.TINYCLAW_OPENVIKING_PRE const OPENVIKING_SESSION_ROOT = '/tinyclaw/sessions'; const openVikingSyncChains = new Map>(); +function stripInjectedOpenVikingContext(text: string): string { + const withEndMarker = /\n*------\n*\n*\[OpenViking Retrieved Context\][\s\S]*?\[End OpenViking Context\]\s*/g; + const withoutEndMarker = /\n*------\n*\n*\[OpenViking Retrieved Context\][\s\S]*$/g; + return text + .replace(withEndMarker, '\n') + .replace(withoutEndMarker, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + function getOpenVikingToolPath(workspacePath: string, agentId: string): string | null { const toolPath = path.join(workspacePath, agentId, '.tinyclaw', 'tools', 'openviking', 'openviking-tool.js'); if (!fs.existsSync(toolPath)) return null; @@ -152,6 +162,15 @@ async function appendTurnAndSyncOpenViking( ): Promise { const sessionFile = ensureActiveSessionFile(workspacePath, agentId); const turnTime = new Date().toISOString(); + const injectedMarker = '[OpenViking Retrieved Context]'; + if (userMessage.includes(injectedMarker) || assistantResponse.includes(injectedMarker)) { + log( + 'WARN', + `OpenViking writeback guard hit for @${agentId} message_id=${messageId}: injected context marker detected before sync` + ); + } + const cleanUserMessage = stripInjectedOpenVikingContext(userMessage); + const cleanAssistantResponse = stripInjectedOpenVikingContext(assistantResponse); const turnBlock = [ '------', '', @@ -162,11 +181,11 @@ async function appendTurnAndSyncOpenViking( '', '### User', '', - userMessage.trim(), + cleanUserMessage, '', '### Assistant', '', - assistantResponse.trim(), + cleanAssistantResponse, '' ].join('\n'); fs.appendFileSync(sessionFile, turnBlock); @@ -191,31 +210,98 @@ async function fetchOpenVikingPrefetchContext(workspacePath: string, agentId: st const allTurns: SessionTurn[] = []; const diagnostics: string[] = []; + const workdir = path.join(workspacePath, agentId); + const searchLimit = Math.max(OPENVIKING_PREFETCH_MAX_TURNS * 6, 12); + const candidateUris: Array<{ uri: string; score: number }> = []; + + // Prefer OpenViking semantic retrieval chain. for (const target of readTargets) { + try { + const found = await runCommand( + 'node', + [toolPath, 'find-uris', query, target, '--limit', String(searchLimit)], + workdir, + OPENVIKING_PREFETCH_TIMEOUT_MS + ); + const lines = found + .trim() + .split('\n') + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('[openviking-tool]')); + let matched = 0; + for (const line of lines) { + const tabIdx = line.indexOf('\t'); + if (tabIdx <= 0) continue; + const score = Number(line.slice(0, tabIdx)); + const uri = line.slice(tabIdx + 1).trim(); + if (!uri) continue; + matched += 1; + candidateUris.push({ uri, score: Number.isFinite(score) ? score : 0 }); + } + diagnostics.push(`${target}:find=${matched}`); + } catch { + diagnostics.push(`${target}:find_error`); + } + } + + const rankedUris: string[] = []; + const seenUris = new Set(); + for (const candidate of candidateUris) { + if (seenUris.has(candidate.uri)) continue; + seenUris.add(candidate.uri); + rankedUris.push(candidate.uri); + if (rankedUris.length >= searchLimit) break; + } + diagnostics.push(`find_total=${rankedUris.length}`); + + for (const uri of rankedUris) { try { const output = await runCommand( 'node', - [toolPath, 'read', target], - path.join(workspacePath, agentId), + [toolPath, 'read', uri], + workdir, OPENVIKING_PREFETCH_TIMEOUT_MS ); const content = output.trim(); - if (!content || content.startsWith('[openviking-tool]')) { - diagnostics.push(`${target}:empty`); - continue; - } + if (!content || content.startsWith('[openviking-tool]')) continue; const parsed = parseSessionTurns(content); - diagnostics.push(`${target}:chars=${content.length},turns=${parsed.length}`); - allTurns.push(...parsed); + if (parsed.length > 0) { + allTurns.push(...parsed); + } } catch { - diagnostics.push(`${target}:error`); - // Best-effort: ignore missing/unreadable targets. + // Best-effort: ignore individual read failures. + } + } + + // Fallback to legacy full-read path if semantic retrieval returns nothing. + if (!allTurns.length) { + for (const target of readTargets) { + try { + const output = await runCommand( + 'node', + [toolPath, 'read', target], + workdir, + OPENVIKING_PREFETCH_TIMEOUT_MS + ); + const content = output.trim(); + if (!content || content.startsWith('[openviking-tool]')) { + diagnostics.push(`${target}:fallback_empty`); + continue; + } + const parsed = parseSessionTurns(content); + diagnostics.push(`${target}:fallback_chars=${content.length},turns=${parsed.length}`); + allTurns.push(...parsed); + } catch { + diagnostics.push(`${target}:fallback_error`); + } } } const dedup = new Map(); for (const turn of allTurns) { - const key = `${turn.messageId}|${turn.timestamp}|${turn.user}|${turn.assistant}`; + const key = turn.messageId + ? `${turn.messageId}|${turn.timestamp}` + : `${turn.timestamp}|${turn.user}|${turn.assistant}`; dedup.set(key, turn); } const turns = Array.from(dedup.values()); @@ -224,7 +310,7 @@ async function fetchOpenVikingPrefetchContext(workspacePath: string, agentId: st return ''; } - const selected = selectRelevantTurns(turns, query, OPENVIKING_PREFETCH_MAX_TURNS); + const selected = turns.slice(0, OPENVIKING_PREFETCH_MAX_TURNS); return buildPrefetchBlock(selected, OPENVIKING_PREFETCH_MAX_CHARS); } diff --git a/src/tools/openviking-tool.ts b/src/tools/openviking-tool.ts index da4d8f3..9e9a1fb 100644 --- a/src/tools/openviking-tool.ts +++ b/src/tools/openviking-tool.ts @@ -4,7 +4,7 @@ import path from 'path'; type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; -type Command = 'ls' | 'read' | 'write' | 'write-file' | 'res-get' | 'res-put'; +type Command = 'ls' | 'read' | 'write' | 'write-file' | 'res-get' | 'res-put' | 'find-uris'; const HELP = `OpenViking workspace tool @@ -15,6 +15,7 @@ Usage: node openviking-tool.js write-file [--json] node openviking-tool.js res-get [--json] node openviking-tool.js res-put [--mime ] [--json] + node openviking-tool.js find-uris [--limit ] [--score-threshold ] [--json] Environment: OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) @@ -48,6 +49,10 @@ function positionalArguments(): string[] { i += 1; continue; } + if (arg === '--limit' || arg === '--score-threshold') { + i += 1; + continue; + } output.push(arg); } return output; @@ -72,6 +77,8 @@ type ListItem = { index: number; }; +const DIRECTORY_READ_MAX_FILES = 8; + function toUri(input: string): string { if (input.startsWith('viking://')) return input; if (input === '/') return 'viking://resources'; @@ -155,6 +162,42 @@ function printRead(data: JsonValue): void { printJson(data); } +type FindMatch = { + uri: string; + score: number; + isLeaf: boolean; +}; + +function extractFindMatches(data: JsonValue): FindMatch[] { + const root = asObject(data); + const resultNode = asObject(root.result); + const groups = ['memories', 'resources', 'skills']; + const out: FindMatch[] = []; + for (const g of groups) { + const items = asArray(resultNode[g]); + for (const item of items) { + const node = asObject(item); + const uri = String(node.uri ?? ''); + if (!uri) continue; + out.push({ + uri, + score: Number(node.score ?? 0), + isLeaf: Boolean(node.is_leaf), + }); + } + } + out.sort((a, b) => b.score - a.score); + return out; +} + +function printFindUris(data: JsonValue): void { + const matches = extractFindMatches(data); + if (!matches.length) return; + for (const m of matches) { + console.log(`${m.score}\t${m.uri}`); + } +} + function extractUris(data: JsonValue): string[] { const root = asObject(data); const items = asArray(root.result ?? asObject(root.data).items ?? root.items); @@ -182,6 +225,9 @@ function extractListItems(data: JsonValue): ListItem[] { } function modTimeScore(modTime: string): number { + const parsed = Date.parse(modTime); + if (!Number.isNaN(parsed)) return parsed; + const m = modTime.match(/^(\d{2}):(\d{2}):(\d{2})$/); if (!m) return -1; const hh = Number(m[1]); @@ -190,18 +236,55 @@ function modTimeScore(modTime: string): number { return hh * 3600 + mm * 60 + ss; } -function pickNewestReadableFileUri(listData: JsonValue): string | null { +function uriTimestampScore(uri: string): number { + const openVikingLeaf = uri.match(/openviking-(\d{10,})-/); + if (openVikingLeaf) { + return Number(openVikingLeaf[1]); + } + + const compactTurn = uri.match(/Turn_(\d{4}-\d{2}-\d{2})T(\d{2})(\d{2})(\d{2})(\d{1,6})?Z/); + if (compactTurn) { + const [, date, hh, mm, ss, fracRaw = ''] = compactTurn; + const ms = fracRaw.slice(0, 3).padEnd(3, '0'); + const iso = `${date}T${hh}:${mm}:${ss}.${ms}Z`; + const parsed = Date.parse(iso); + if (!Number.isNaN(parsed)) return parsed; + } + + const archivedSession = uri.match(/\/(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})-(\d{3,6})Z\.md\//); + if (archivedSession) { + const [, date, hh, mm, ss, fracRaw] = archivedSession; + const ms = fracRaw.slice(0, 3).padEnd(3, '0'); + const iso = `${date}T${hh}:${mm}:${ss}.${ms}Z`; + const parsed = Date.parse(iso); + if (!Number.isNaN(parsed)) return parsed; + } + + return -1; +} + +function isReadableCandidateUri(uri: string): boolean { + const base = uri.split('/').pop() || ''; + if (!base) return false; + if (base.startsWith('.')) return false; // Exclude .overview/.abstract summaries. + if (!(uri.endsWith('.md') || uri.endsWith('.txt'))) return false; + return true; +} + +function pickNewestReadableFileUris(listData: JsonValue, limit: number): string[] { const candidates = extractListItems(listData) .filter((item) => !item.isDir) - .filter((item) => item.uri.endsWith('.md') || item.uri.endsWith('.txt')); - if (!candidates.length) return null; + .filter((item) => isReadableCandidateUri(item.uri)); + if (!candidates.length) return []; candidates.sort((a, b) => { + const ts = uriTimestampScore(b.uri) - uriTimestampScore(a.uri); + if (ts !== 0) return ts; const dt = modTimeScore(b.modTime) - modTimeScore(a.modTime); if (dt !== 0) return dt; return b.index - a.index; }); - return candidates[0].uri; + return candidates.slice(0, Math.max(1, limit)).map((item) => item.uri); } async function readWithDirectoryFallback(uri: string, rawPath: string): Promise { @@ -212,11 +295,29 @@ async function readWithDirectoryFallback(uri: string, rawPath: string): Promise< return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); } const listed = await request(`/api/v1/fs/ls?uri=${encodeURIComponent(uri)}&recursive=true&output=agent`); - const fileUri = pickNewestReadableFileUri(listed); - if (!fileUri) { + const fileUris = pickNewestReadableFileUris(listed, DIRECTORY_READ_MAX_FILES); + if (!fileUris.length) { return await request(`/api/v1/content/read?uri=${encodeURIComponent(uri)}`); } - return await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUri)}`); + const chunks: string[] = []; + for (const fileUri of fileUris) { + try { + const data = await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUri)}`); + const root = asObject(data); + const dataNode = asObject(root.data); + const resultNode = root.result; + const content = dataNode.content ?? root.content ?? resultNode; + if (typeof content === 'string' && content.trim()) { + chunks.push(content.trim()); + } + } catch { + // Best effort: skip unreadable leaf. + } + } + if (!chunks.length) { + return await request(`/api/v1/content/read?uri=${encodeURIComponent(fileUris[0])}`); + } + return { content: chunks.join('\n\n------\n\n') }; } catch (primaryError) { try { return await request(`/api/v1/content/read?path=${encodeURIComponent(rawPath)}`); @@ -337,6 +438,33 @@ async function run(): Promise { else console.log(`Wrote resource ${uri}`); return; } + case 'find-uris': { + if (!positional[1] || !positional[2]) fail('Usage: find-uris [--limit ] [--score-threshold ]'); + const query = positional[1]; + const targetUri = toUri(positional[2]); + const limitRaw = getFlagValue('--limit'); + const scoreThresholdRaw = getFlagValue('--score-threshold'); + const limit = limitRaw ? Number(limitRaw) : 12; + if (!Number.isFinite(limit) || limit <= 0) fail('Invalid --limit value'); + let scoreThreshold: number | undefined; + if (scoreThresholdRaw !== undefined) { + const parsed = Number(scoreThresholdRaw); + if (!Number.isFinite(parsed)) fail('Invalid --score-threshold value'); + scoreThreshold = parsed; + } + response = await request('/api/v1/search/find', { + method: 'POST', + body: JSON.stringify({ + query, + target_uri: targetUri, + limit, + score_threshold: scoreThreshold, + }), + }); + if (jsonOutput) printJson(response); + else printFindUris(response); + return; + } default: fail(`Unknown command: ${command}\n\n${HELP}`); } From 0680aa63dbb058bb73cee53b0fb4e0ff4dacf006 Mon Sep 17 00:00:00 2001 From: MZ Date: Sun, 22 Feb 2026 13:57:42 -0700 Subject: [PATCH 3/3] feat: integrate OpenViking native flow and setup auto-bootstrap --- README.md | 36 +- lib/common.sh | 39 ++ lib/daemon.sh | 98 ++++- lib/setup-wizard.sh | 138 +++++++ .../agent-tools/openviking/openviking-tool.js | 245 ++++++++++- src/lib/openviking-prefetch.ts | 150 +++++++ src/lib/openviking-session-map.ts | 97 +++++ src/queue-processor.ts | 390 +++++++++++++++++- src/tools/openviking-tool.ts | 270 +++++++++++- 9 files changed, 1395 insertions(+), 68 deletions(-) create mode 100644 src/lib/openviking-session-map.ts diff --git a/README.md b/README.md index 4097771..507971b 100644 --- a/README.md +++ b/README.md @@ -273,32 +273,44 @@ Environment variables: - `OPENVIKING_API_KEY` (optional) - `OPENVIKING_PROJECT` (optional) -### Auto Sync Triggers (Minimal) +### OpenViking Native Session Write Path -TinyClaw now supports minimal OpenViking auto-sync triggers per agent: +TinyClaw supports OpenViking native session lifecycle as the primary write path: -- `on_turn`: after each agent response, append `user+assistant` turn to a local session file and sync to `viking://resources/tinyclaw/sessions//active.md` -- `on_session_end`: when `agent reset` is consumed, archive the previous session to `viking://resources/tinyclaw/sessions//closed/.md` +- `POST /api/v1/sessions` to create/reuse session IDs per `(channel, senderId, agentId)` mapping +- `POST /api/v1/sessions/{id}/messages` for `user` and `assistant` turns +- `POST /api/v1/sessions/{id}/commit` when reset/session-end is consumed -Notes: +Setup integration: -- This is best-effort and non-blocking (response delivery is not blocked by sync retries). -- Disable auto-sync by setting `TINYCLAW_OPENVIKING_AUTOSYNC=0`. +- `tinyclaw setup` now prompts whether to enable OpenViking memory +- if enabled, setup can install OpenViking CLI and generate `~/.openviking/ov.conf` (includes LLM API key for OpenViking internals) +- `tinyclaw start` auto-starts OpenViking server (when enabled + auto_start) and exports OpenViking env vars for channel/queue processes -### Pre-Prompt Retrieval (OpenViking) +Feature flags: + +- `TINYCLAW_OPENVIKING_SESSION_NATIVE=1` enable native session write path +- `TINYCLAW_OPENVIKING_AUTOSYNC=0` disable legacy markdown sync fallback (`active.md`/`closed/*.md`) -Before invoking the model for an external user turn, TinyClaw can prefetch related context from: +Legacy markdown sync remains as a compatibility fallback. + +### Pre-Prompt Retrieval (OpenViking) -- `viking://resources/tinyclaw/sessions//active.md` +Before invoking the model for an external user turn, TinyClaw can prefetch related context via: -It then injects a compact `[OpenViking Retrieved Context]` block into the prompt. +- `POST /api/v1/search/search` (native, typed `memories/resources/skills`, optionally scoped with `session_id`) +- legacy fallback (`find-uris` + `read` on `active.md` and archived sessions) when native search is disabled or misses Environment flags: - `TINYCLAW_OPENVIKING_PREFETCH=0` disable pre-prompt retrieval -- `TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS` command timeout (default: `5000`) +- `TINYCLAW_OPENVIKING_SEARCH_NATIVE=1` enable native search as primary prefetch path +- `TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS` prefetch/search timeout (default: `5000`) +- `TINYCLAW_OPENVIKING_COMMIT_TIMEOUT_MS` native session commit timeout (default: `15000`) - `TINYCLAW_OPENVIKING_PREFETCH_MAX_CHARS` max injected chars (default: `2800`) - `TINYCLAW_OPENVIKING_PREFETCH_MAX_TURNS` max selected turns (default: `4`) +- `TINYCLAW_OPENVIKING_PREFETCH_MAX_HITS` max typed native hits injected (default: `8`) +- `TINYCLAW_OPENVIKING_SEARCH_SCORE_THRESHOLD` optional native score threshold passed to OpenViking search API ### In-Chat Commands diff --git a/lib/common.sh b/lib/common.sh index ee63e9a..6e542b3 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -61,6 +61,23 @@ declare -A CHANNEL_TOKEN_ENV=( ACTIVE_CHANNELS=() declare -A CHANNEL_TOKENS=() WORKSPACE_PATH="" +OPENVIKING_ENABLED="false" +OPENVIKING_AUTO_START="false" +OPENVIKING_HOST="127.0.0.1" +OPENVIKING_PORT="8320" +OPENVIKING_BASE_URL="http://127.0.0.1:8320" +OPENVIKING_CONFIG_PATH="$HOME/.openviking/ov.conf" +OPENVIKING_PROJECT="" +OPENVIKING_API_KEY="" +OPENVIKING_NATIVE_SESSION="false" +OPENVIKING_NATIVE_SEARCH="false" +OPENVIKING_PREFETCH="false" +OPENVIKING_AUTOSYNC="true" +OPENVIKING_PREFETCH_TIMEOUT_MS="5000" +OPENVIKING_COMMIT_TIMEOUT_MS="15000" +OPENVIKING_PREFETCH_MAX_CHARS="2800" +OPENVIKING_PREFETCH_MAX_TURNS="4" +OPENVIKING_PREFETCH_MAX_HITS="8" # Logging function log() { @@ -115,6 +132,28 @@ load_settings() { fi done + # Load OpenViking settings + OPENVIKING_ENABLED=$(jq -r '.openviking.enabled // false' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_AUTO_START=$(jq -r '.openviking.auto_start // false' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_HOST=$(jq -r '.openviking.host // "127.0.0.1"' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PORT=$(jq -r '.openviking.port // 8320' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_BASE_URL=$(jq -r '.openviking.base_url // "http://127.0.0.1:8320"' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_CONFIG_PATH=$(jq -r '.openviking.config_path // empty' "$SETTINGS_FILE" 2>/dev/null) + if [ -z "$OPENVIKING_CONFIG_PATH" ]; then + OPENVIKING_CONFIG_PATH="$HOME/.openviking/ov.conf" + fi + OPENVIKING_PROJECT=$(jq -r '.openviking.project // empty' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_API_KEY=$(jq -r '.openviking.api_key // empty' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_NATIVE_SESSION=$(jq -r '.openviking.native_session // false' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_NATIVE_SEARCH=$(jq -r '.openviking.native_search // false' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PREFETCH=$(jq -r '.openviking.prefetch // false' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_AUTOSYNC=$(jq -r '.openviking.autosync // true' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PREFETCH_TIMEOUT_MS=$(jq -r '.openviking.prefetch_timeout_ms // 5000' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_COMMIT_TIMEOUT_MS=$(jq -r '.openviking.commit_timeout_ms // 15000' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PREFETCH_MAX_CHARS=$(jq -r '.openviking.prefetch_max_chars // 2800' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PREFETCH_MAX_TURNS=$(jq -r '.openviking.prefetch_max_turns // 4' "$SETTINGS_FILE" 2>/dev/null) + OPENVIKING_PREFETCH_MAX_HITS=$(jq -r '.openviking.prefetch_max_hits // 8' "$SETTINGS_FILE" 2>/dev/null) + return 0 } diff --git a/lib/daemon.sh b/lib/daemon.sh index ba1a9fd..cfef2cb 100644 --- a/lib/daemon.sh +++ b/lib/daemon.sh @@ -86,6 +86,39 @@ start_daemon() { return 1 fi + local openviking_enabled=false + local openviking_autostart=false + local openviking_started_outside=false + local openviking_start_in_tmux=false + local openviking_bin="" + if [ "$OPENVIKING_ENABLED" = "true" ]; then + openviking_enabled=true + fi + if [ "$OPENVIKING_AUTO_START" = "true" ]; then + openviking_autostart=true + fi + if [ "$openviking_enabled" = true ] && [ "$openviking_autostart" = true ]; then + openviking_bin="$(command -v openviking || true)" + if [ -z "$openviking_bin" ] && [ -x "$HOME/.local/bin/openviking" ]; then + openviking_bin="$HOME/.local/bin/openviking" + fi + if [ -z "$openviking_bin" ]; then + echo -e "${RED}OpenViking is enabled but CLI is not installed${NC}" + echo "Run 'tinyclaw setup' again to install OpenViking, or disable OpenViking in settings." + return 1 + fi + if [ ! -f "$OPENVIKING_CONFIG_PATH" ]; then + echo -e "${RED}OpenViking is enabled but config file is missing: $OPENVIKING_CONFIG_PATH${NC}" + echo "Run 'tinyclaw setup' again to regenerate OpenViking config." + return 1 + fi + if curl -fsS --max-time 2 "$OPENVIKING_BASE_URL/health" >/dev/null 2>&1; then + openviking_started_outside=true + else + openviking_start_in_tmux=true + fi + fi + # Ensure all agent workspaces have .agents/skills symlink ensure_agent_skills_links sync_agent_tools @@ -109,6 +142,36 @@ start_daemon() { echo "${env_var}=${CHANNEL_TOKENS[$ch]}" >> "$env_file" fi done + if [ "$openviking_enabled" = true ]; then + echo "OPENVIKING_BASE_URL=${OPENVIKING_BASE_URL}" >> "$env_file" + if [ -n "$OPENVIKING_API_KEY" ]; then + echo "OPENVIKING_API_KEY=${OPENVIKING_API_KEY}" >> "$env_file" + fi + if [ -n "$OPENVIKING_PROJECT" ]; then + echo "OPENVIKING_PROJECT=${OPENVIKING_PROJECT}" >> "$env_file" + fi + if [ "$OPENVIKING_NATIVE_SESSION" = "true" ]; then + echo "TINYCLAW_OPENVIKING_SESSION_NATIVE=1" >> "$env_file" + fi + if [ "$OPENVIKING_NATIVE_SEARCH" = "true" ]; then + echo "TINYCLAW_OPENVIKING_SEARCH_NATIVE=1" >> "$env_file" + fi + if [ "$OPENVIKING_PREFETCH" = "true" ]; then + echo "TINYCLAW_OPENVIKING_PREFETCH=1" >> "$env_file" + else + echo "TINYCLAW_OPENVIKING_PREFETCH=0" >> "$env_file" + fi + if [ "$OPENVIKING_AUTOSYNC" = "true" ]; then + echo "TINYCLAW_OPENVIKING_AUTOSYNC=1" >> "$env_file" + else + echo "TINYCLAW_OPENVIKING_AUTOSYNC=0" >> "$env_file" + fi + echo "TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS=${OPENVIKING_PREFETCH_TIMEOUT_MS}" >> "$env_file" + echo "TINYCLAW_OPENVIKING_COMMIT_TIMEOUT_MS=${OPENVIKING_COMMIT_TIMEOUT_MS}" >> "$env_file" + echo "TINYCLAW_OPENVIKING_PREFETCH_MAX_CHARS=${OPENVIKING_PREFETCH_MAX_CHARS}" >> "$env_file" + echo "TINYCLAW_OPENVIKING_PREFETCH_MAX_TURNS=${OPENVIKING_PREFETCH_MAX_TURNS}" >> "$env_file" + echo "TINYCLAW_OPENVIKING_PREFETCH_MAX_HITS=${OPENVIKING_PREFETCH_MAX_HITS}" >> "$env_file" + fi # Check for updates (non-blocking) local update_info @@ -123,6 +186,13 @@ start_daemon() { for ch in "${ACTIVE_CHANNELS[@]}"; do echo -e " ${GREEN}✓${NC} ${CHANNEL_DISPLAY[$ch]}" done + if [ "$openviking_enabled" = true ]; then + if [ "$openviking_started_outside" = true ]; then + echo -e " ${GREEN}✓${NC} OpenViking (already running at ${OPENVIKING_BASE_URL})" + elif [ "$openviking_start_in_tmux" = true ]; then + echo -e " ${GREEN}✓${NC} OpenViking (auto-start)" + fi + fi echo "" # Build log tail command @@ -132,8 +202,12 @@ start_daemon() { done # --- Build tmux session dynamically --- - # Total panes = N channels + 3 (queue, heartbeat, logs) - local total_panes=$(( ${#ACTIVE_CHANNELS[@]} + 3 )) + # Total panes = N channels + optional OpenViking + 3 (queue, heartbeat, logs) + local extra_panes=3 + if [ "$openviking_start_in_tmux" = true ]; then + extra_panes=$((extra_panes + 1)) + fi + local total_panes=$(( ${#ACTIVE_CHANNELS[@]} + extra_panes )) tmux new-session -d -s "$TMUX_SESSION" -n "tinyclaw" -c "$SCRIPT_DIR" @@ -153,6 +227,13 @@ start_daemon() { pane_idx=$((pane_idx + 1)) done + # OpenViking pane (optional) + if [ "$openviking_start_in_tmux" = true ]; then + tmux send-keys -t "$TMUX_SESSION:0.$pane_idx" "cd '$SCRIPT_DIR' && '$openviking_bin' serve --host '$OPENVIKING_HOST' --port '$OPENVIKING_PORT' --config '$OPENVIKING_CONFIG_PATH' 2>&1 | tee -a '$LOG_DIR/openviking.log'" C-m + tmux select-pane -t "$TMUX_SESSION:0.$pane_idx" -T "OpenViking" + pane_idx=$((pane_idx + 1)) + fi + # Queue pane tmux send-keys -t "$TMUX_SESSION:0.$pane_idx" "cd '$SCRIPT_DIR' && node dist/queue-processor.js" C-m tmux select-pane -t "$TMUX_SESSION:0.$pane_idx" -T "Queue" @@ -254,6 +335,7 @@ start_daemon() { # Stop daemon stop_daemon() { log "Stopping TinyClaw..." + load_settings >/dev/null 2>&1 || true if session_exists; then tmux kill-session -t "$TMUX_SESSION" @@ -265,6 +347,9 @@ stop_daemon() { done pkill -f "dist/queue-processor.js" || true pkill -f "heartbeat-cron.sh" || true + if [ -n "${OPENVIKING_PORT:-}" ]; then + pkill -f "openviking serve .*--port ${OPENVIKING_PORT}" || true + fi echo -e "${GREEN}✓ TinyClaw stopped${NC}" log "Daemon stopped" @@ -292,6 +377,8 @@ restart_daemon() { # Status status_daemon() { + load_settings >/dev/null 2>&1 || true + echo -e "${BLUE}TinyClaw Status${NC}" echo "===============" echo "" @@ -341,6 +428,13 @@ status_daemon() { else echo -e "Heartbeat: ${RED}Not Running${NC}" fi + if [ "$OPENVIKING_ENABLED" = "true" ]; then + if pgrep -f "openviking serve .*--port ${OPENVIKING_PORT}" > /dev/null; then + echo -e "OpenViking: ${GREEN}Running${NC}" + else + echo -e "OpenViking: ${RED}Not Running${NC}" + fi + fi # Recent activity per channel (only show if log file exists) for ch in "${ALL_CHANNELS[@]}"; do diff --git a/lib/setup-wizard.sh b/lib/setup-wizard.sh index 0565764..28523db 100755 --- a/lib/setup-wizard.sh +++ b/lib/setup-wizard.sh @@ -228,6 +228,109 @@ DEFAULT_AGENT_NAME=$(echo "$DEFAULT_AGENT_NAME" | tr ' ' '-' | tr -cd 'a-zA-Z0-9 echo -e "${GREEN}✓ Default agent: $DEFAULT_AGENT_NAME${NC}" echo "" +# OpenViking setup (optional) +OPENVIKING_ENABLED=false +OPENVIKING_AUTO_START=false +OPENVIKING_HOST="127.0.0.1" +OPENVIKING_PORT="8320" +OPENVIKING_BASE_URL="http://127.0.0.1:8320" +OPENVIKING_PROJECT="" +OPENVIKING_API_KEY="" +OPENVIKING_CONFIG_PATH="$HOME/.openviking/ov.conf" +OPENVIKING_PREFETCH_TIMEOUT_MS=5000 +OPENVIKING_COMMIT_TIMEOUT_MS=15000 +OPENVIKING_PREFETCH_MAX_CHARS=2800 +OPENVIKING_PREFETCH_MAX_TURNS=4 +OPENVIKING_PREFETCH_MAX_HITS=8 + +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${GREEN} OpenViking Memory (Optional)${NC}" +echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo "" +echo "Enable OpenViking native memory (Session + Search + Memory extraction)?" +read -rp "Enable OpenViking? [y/N]: " ENABLE_OPENVIKING +if [[ "$ENABLE_OPENVIKING" =~ ^[yY] ]]; then + OPENVIKING_ENABLED=true + OPENVIKING_AUTO_START=true + echo "" + echo -e "${GREEN}✓ OpenViking enabled${NC}" + echo -e "${BLUE}Using default OpenViking server endpoint: ${OPENVIKING_BASE_URL}${NC}" + + echo "" + echo "Now configure values that will be written into ~/.openviking/ov.conf" + echo -e "${YELLOW}Note: currently TinyClaw setup only supports OpenAI for OpenViking (tested path).${NC}" + read -rp "OpenAI API key (required for VLM + embedding): " OV_LLM_API_KEY + if [ -z "$OV_LLM_API_KEY" ]; then + echo -e "${RED}API key is required when OpenViking is enabled${NC}" + exit 1 + fi + OV_LLM_API_BASE="https://api.openai.com/v1" + OV_LLM_MODEL="gpt-4o-mini" + OV_EMBED_MODEL="text-embedding-3-large" + + OPENVIKING_CONF_DIR="$(dirname "$OPENVIKING_CONFIG_PATH")" + OPENVIKING_DATA_PATH="$HOME/.tinyclaw/openviking-data" + mkdir -p "$OPENVIKING_CONF_DIR" + mkdir -p "$OPENVIKING_DATA_PATH" + + if ! command -v openviking &> /dev/null; then + echo -e "${YELLOW}OpenViking CLI not found. Installing with pip...${NC}" + if ! command -v python3 &> /dev/null; then + echo -e "${RED}python3 is required to install OpenViking${NC}" + exit 1 + fi + if ! python3 -m pip install --user --upgrade openviking; then + echo -e "${RED}Failed to install openviking package${NC}" + exit 1 + fi + fi + + if ! command -v jq &> /dev/null; then + echo -e "${RED}jq is required for OpenViking config generation${NC}" + exit 1 + fi + + jq -n \ + --arg agfs_path "$OPENVIKING_DATA_PATH/agfs" \ + --arg vectordb_path "$OPENVIKING_DATA_PATH/vectordb" \ + --arg api_key "$OV_LLM_API_KEY" \ + --arg api_base "$OV_LLM_API_BASE" \ + --arg vlm_model "$OV_LLM_MODEL" \ + --arg embed_model "$OV_EMBED_MODEL" \ + '{ + storage: { + agfs: { + backend: "local", + path: $agfs_path + }, + vectordb: { + backend: "local", + path: $vectordb_path + } + }, + embedding: { + dense: { + provider: "openai", + model: $embed_model, + api_key: $api_key, + api_base: (if $api_base == "" then null else $api_base end) + } + }, + vlm: { + provider: "openai", + model: $vlm_model, + api_key: $api_key, + api_base: (if $api_base == "" then null else $api_base end), + temperature: 0.0 + }, + log_level: "WARNING" + }' > "$OPENVIKING_CONFIG_PATH" + + echo -e "${GREEN}✓ OpenViking config written: $OPENVIKING_CONFIG_PATH${NC}" + echo -e "${GREEN}✓ TinyClaw will auto-start OpenViking server on tinyclaw start${NC}" + echo "" +fi + # --- Additional Agents (optional) --- echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN} Additional Agents (Optional)${NC}" @@ -343,6 +446,40 @@ else MODELS_SECTION='"models": { "provider": "openai", "openai": { "model": "'"${MODEL}"'" } }' fi +OPENVIKING_JSON=$(jq -n \ + --argjson enabled "$OPENVIKING_ENABLED" \ + --argjson auto_start "$OPENVIKING_AUTO_START" \ + --arg host "$OPENVIKING_HOST" \ + --argjson port "$OPENVIKING_PORT" \ + --arg base_url "$OPENVIKING_BASE_URL" \ + --arg config_path "$OPENVIKING_CONFIG_PATH" \ + --arg project "$OPENVIKING_PROJECT" \ + --arg api_key "$OPENVIKING_API_KEY" \ + --argjson prefetch_timeout_ms "$OPENVIKING_PREFETCH_TIMEOUT_MS" \ + --argjson commit_timeout_ms "$OPENVIKING_COMMIT_TIMEOUT_MS" \ + --argjson prefetch_max_chars "$OPENVIKING_PREFETCH_MAX_CHARS" \ + --argjson prefetch_max_turns "$OPENVIKING_PREFETCH_MAX_TURNS" \ + --argjson prefetch_max_hits "$OPENVIKING_PREFETCH_MAX_HITS" \ + '{ + enabled: $enabled, + auto_start: $auto_start, + host: $host, + port: $port, + base_url: $base_url, + config_path: $config_path, + project: (if $project == "" then null else $project end), + api_key: (if $api_key == "" then null else $api_key end), + native_session: $enabled, + native_search: $enabled, + prefetch: $enabled, + autosync: true, + prefetch_timeout_ms: $prefetch_timeout_ms, + commit_timeout_ms: $commit_timeout_ms, + prefetch_max_chars: $prefetch_max_chars, + prefetch_max_turns: $prefetch_max_turns, + prefetch_max_hits: $prefetch_max_hits + }') + cat > "$SETTINGS_FILE" < "$SETTINGS_FILE" < [--json] node openviking-tool.js res-put [--mime ] [--json] node openviking-tool.js find-uris [--limit ] [--score-threshold ] [--json] + node openviking-tool.js search [--session-id ] [--limit ] [--score-threshold ] [--json] + node openviking-tool.js session-create [--agent-id ] [--channel ] [--sender-id ] [--json] + node openviking-tool.js session-message [--json] + node openviking-tool.js session-commit [--json] Environment: OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) @@ -43,11 +47,13 @@ function positionalArguments() { const arg = args[i]; if (arg === '--json') continue; - if (arg === '--mime') { - i += 1; - continue; - } - if (arg === '--limit' || arg === '--score-threshold') { + if (arg === '--mime' + || arg === '--limit' + || arg === '--score-threshold' + || arg === '--session-id' + || arg === '--agent-id' + || arg === '--channel' + || arg === '--sender-id') { i += 1; continue; } @@ -140,6 +146,73 @@ function printRead(data) { } printJson(data); } +function normalizeAbstract(value) { + const normalized = value.replace(/\s+/g, ' ').trim(); + if (!normalized) + return '(no abstract provided)'; + const MAX_ABSTRACT_CHARS = 220; + if (normalized.length <= MAX_ABSTRACT_CHARS) + return normalized; + return `${normalized.slice(0, MAX_ABSTRACT_CHARS - 3)}...`; +} +function pickAbstract(node) { + const candidates = [ + node.abstract, + node.summary, + node.snippet, + node.text, + node.content, + node.description, + node.title, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string') { + return normalizeAbstract(candidate); + } + } + const metadata = asObject(node.metadata); + for (const candidate of [metadata.abstract, metadata.summary, metadata.snippet, metadata.description]) { + if (typeof candidate === 'string') { + return normalizeAbstract(candidate); + } + } + return '(no abstract provided)'; +} +function extractSearchMatches(data) { + const root = asObject(data); + const resultNode = asObject(root.result ?? root.data ?? root); + const groups = [ + { key: 'memories', type: 'memory' }, + { key: 'resources', type: 'resource' }, + { key: 'skills', type: 'skill' }, + ]; + const out = []; + for (const group of groups) { + const items = asArray(resultNode[group.key]); + for (const item of items) { + const node = asObject(item); + const uri = String(node.uri ?? node.path ?? '').trim(); + if (!uri) + continue; + out.push({ + type: group.type, + uri, + score: Number(node.score ?? 0), + abstract: pickAbstract(node), + }); + } + } + out.sort((a, b) => b.score - a.score); + return out; +} +function printSearchMatches(data) { + const matches = extractSearchMatches(data); + if (!matches.length) + return; + for (const m of matches) { + console.log(`${m.type}\t${m.score}\t${m.uri}\t${m.abstract}`); + } +} function extractFindMatches(data) { const root = asObject(data); const resultNode = asObject(root.result); @@ -170,13 +243,6 @@ function printFindUris(data) { console.log(`${m.score}\t${m.uri}`); } } -function extractUris(data) { - const root = asObject(data); - const items = asArray(root.result ?? asObject(root.data).items ?? root.items); - return items - .map((item) => String(asObject(item).uri ?? asObject(item).path ?? '')) - .filter((uri) => uri.length > 0); -} function extractListItems(data) { const root = asObject(data); const items = asArray(root.result ?? asObject(root.data).items ?? root.items); @@ -301,6 +367,86 @@ async function readWithDirectoryFallback(uri, rawPath) { } } } +function extractSessionId(response) { + const root = asObject(response); + const resultNode = asObject(root.result); + const dataNode = asObject(root.data); + const candidates = [ + root.id, + root.session_id, + root.sessionId, + resultNode.id, + resultNode.session_id, + resultNode.sessionId, + dataNode.id, + dataNode.session_id, + dataNode.sessionId, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim()) { + return candidate.trim(); + } + } + return ''; +} +async function createSession(agentId, channel, senderId) { + const metadata = { + source: 'tinyclaw', + agent_id: agentId || 'unknown', + channel: channel || 'unknown', + sender_id: senderId || 'unknown', + created_at: new Date().toISOString(), + }; + const fallbackName = `tinyclaw:${agentId || 'agent'}:${channel || 'channel'}:${senderId || 'sender'}`; + const payloads = [ + { metadata }, + { name: fallbackName, metadata }, + { agent_id: agentId, channel, sender_id: senderId, metadata }, + {}, + ]; + let lastError = null; + for (const payload of payloads) { + try { + return await request('/api/v1/sessions', { + method: 'POST', + body: JSON.stringify(payload), + }); + } + catch (error) { + lastError = error; + } + } + throw lastError || new Error('Failed to create session'); +} +async function writeSessionMessage(sessionId, role, content) { + const endpoint = `/api/v1/sessions/${encodeURIComponent(sessionId)}/messages`; + try { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({ role, content }), + }); + } + catch { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({ message: { role, content } }), + }); + } +} +async function commitSession(sessionId) { + const endpoint = `/api/v1/sessions/${encodeURIComponent(sessionId)}/commit`; + try { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({}), + }); + } + catch { + return await request(endpoint, { + method: 'POST', + }); + } +} async function run() { const positional = positionalArguments(); if (!positional.length || positional[0] === 'help' || positional[0] === '--help') { @@ -466,6 +612,81 @@ async function run() { printFindUris(response); return; } + case 'search': { + if (!positional[1]) + fail('Usage: search [--session-id ] [--limit ] [--score-threshold ]'); + const query = positional[1]; + const sessionId = getFlagValue('--session-id'); + const limitRaw = getFlagValue('--limit'); + const scoreThresholdRaw = getFlagValue('--score-threshold'); + const limit = limitRaw ? Number(limitRaw) : 12; + if (!Number.isFinite(limit) || limit <= 0) + fail('Invalid --limit value'); + let scoreThreshold; + if (scoreThresholdRaw !== undefined) { + const parsed = Number(scoreThresholdRaw); + if (!Number.isFinite(parsed)) + fail('Invalid --score-threshold value'); + scoreThreshold = parsed; + } + response = await request('/api/v1/search/search', { + method: 'POST', + body: JSON.stringify({ + query, + session_id: sessionId, + limit, + score_threshold: scoreThreshold, + }), + }); + if (jsonOutput) + printJson(response); + else + printSearchMatches(response); + return; + } + case 'session-create': { + const agentId = getFlagValue('--agent-id'); + const channel = getFlagValue('--channel'); + const senderId = getFlagValue('--sender-id'); + response = await createSession(agentId, channel, senderId); + const sessionId = extractSessionId(response); + if (!sessionId) { + fail('Session create succeeded but no session id found in response'); + } + if (jsonOutput) + printJson(response); + else + console.log(sessionId); + return; + } + case 'session-message': { + if (!positional[1] || !positional[2] || positional[3] === undefined) { + fail('Usage: session-message '); + } + const sessionId = positional[1]; + const role = positional[2]; + const content = positional[3]; + if (!['user', 'assistant', 'system'].includes(role)) { + fail('Role must be one of: user, assistant, system'); + } + response = await writeSessionMessage(sessionId, role, content); + if (jsonOutput) + printJson(response); + else + console.log(`Session message stored: ${sessionId} (${role})`); + return; + } + case 'session-commit': { + if (!positional[1]) + fail('Usage: session-commit '); + const sessionId = positional[1]; + response = await commitSession(sessionId); + if (jsonOutput) + printJson(response); + else + console.log(`Session committed: ${sessionId}`); + return; + } default: fail(`Unknown command: ${command}\n\n${HELP}`); } diff --git a/src/lib/openviking-prefetch.ts b/src/lib/openviking-prefetch.ts index aa952bf..4c50636 100644 --- a/src/lib/openviking-prefetch.ts +++ b/src/lib/openviking-prefetch.ts @@ -6,6 +6,21 @@ export type SessionTurn = { index: number; }; +export type OpenVikingSearchHitType = 'memory' | 'resource' | 'skill'; + +export type OpenVikingSearchHit = { + type: OpenVikingSearchHitType; + uri: string; + abstract: string; + score: number; +}; + +export type OpenVikingSearchHitDistribution = { + memory: number; + resource: number; + skill: number; +}; + export function tokenizeForMatch(text: string): string[] { const stopwords = new Set([ 'the', 'is', 'are', 'am', 'was', 'were', 'be', 'been', 'being', @@ -120,3 +135,138 @@ export function buildPrefetchBlock(turns: SessionTurn[], maxChars: number): stri } return output; } + +function asRecord(value: unknown): Record { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return value as Record; + } + return {}; +} + +function asArray(value: unknown): unknown[] { + if (Array.isArray(value)) return value; + return []; +} + +function normalizeAbstract(value: string): string { + const normalized = value.replace(/\s+/g, ' ').trim(); + if (!normalized) return ''; + const MAX_ABSTRACT_CHARS = 260; + if (normalized.length <= MAX_ABSTRACT_CHARS) return normalized; + return `${normalized.slice(0, MAX_ABSTRACT_CHARS - 3)}...`; +} + +function pickAbstract(node: Record): string { + const candidates = [ + node.abstract, + node.summary, + node.snippet, + node.text, + node.content, + node.description, + node.title, + ]; + + for (const candidate of candidates) { + if (typeof candidate === 'string') { + const normalized = normalizeAbstract(candidate); + if (normalized) return normalized; + } + } + + const metadata = asRecord(node.metadata); + const metadataAbstractCandidates = [metadata.abstract, metadata.summary, metadata.snippet, metadata.description]; + for (const candidate of metadataAbstractCandidates) { + if (typeof candidate === 'string') { + const normalized = normalizeAbstract(candidate); + if (normalized) return normalized; + } + } + + return '(no abstract provided)'; +} + +function toFiniteScore(value: unknown): number { + const n = Number(value); + if (!Number.isFinite(n)) return 0; + return n; +} + +export function parseOpenVikingSearchHits(payload: unknown): OpenVikingSearchHit[] { + const root = asRecord(payload); + const resultNode = asRecord(root.result ?? root.data ?? root); + const groups: Array<{ key: string; type: OpenVikingSearchHitType }> = [ + { key: 'memories', type: 'memory' }, + { key: 'resources', type: 'resource' }, + { key: 'skills', type: 'skill' }, + ]; + + const hits: OpenVikingSearchHit[] = []; + for (const group of groups) { + const items = asArray(resultNode[group.key]); + for (const item of items) { + const node = asRecord(item); + const uri = String(node.uri ?? node.path ?? '').trim(); + if (!uri) continue; + hits.push({ + type: group.type, + uri, + abstract: pickAbstract(node), + score: toFiniteScore(node.score), + }); + } + } + + const dedup = new Map(); + for (const hit of hits) { + const key = `${hit.type}|${hit.uri}|${hit.abstract}`; + if (!dedup.has(key)) { + dedup.set(key, hit); + continue; + } + const existing = dedup.get(key)!; + if (hit.score > existing.score) { + dedup.set(key, hit); + } + } + + return Array.from(dedup.values()).sort((a, b) => b.score - a.score); +} + +export function summarizeOpenVikingSearchHitDistribution(hits: OpenVikingSearchHit[]): OpenVikingSearchHitDistribution { + const summary: OpenVikingSearchHitDistribution = { + memory: 0, + resource: 0, + skill: 0, + }; + for (const hit of hits) { + summary[hit.type] += 1; + } + return summary; +} + +export function buildOpenVikingSearchPrefetchBlock(hits: OpenVikingSearchHit[], maxChars: number, maxHits: number): string { + if (!hits.length) return ''; + const cap = Math.max(1, maxHits); + const memoryHits = hits.filter((hit) => hit.type === 'memory'); + const resourceHits = hits.filter((hit) => hit.type === 'resource'); + const skillHits = hits.filter((hit) => hit.type === 'skill'); + // Memory-first composition keeps high-value long-term facts ahead of docs/skills. + const selected = [...memoryHits, ...resourceHits, ...skillHits].slice(0, cap); + const summary = summarizeOpenVikingSearchHitDistribution(selected); + const lines: string[] = []; + lines.push('[OpenViking Retrieved Context]'); + lines.push(''); + lines.push(`- source: search_native | distribution: memory=${summary.memory}, resource=${summary.resource}, skill=${summary.skill}`); + + for (const hit of selected) { + lines.push(`- type: ${hit.type} | score: ${hit.score.toFixed(4)} | uri: ${hit.uri}`); + lines.push(` abstract: ${hit.abstract}`); + } + + let output = lines.join('\n'); + if (output.length > maxChars) { + output = output.slice(0, maxChars) + '\n...'; + } + return output; +} diff --git a/src/lib/openviking-session-map.ts b/src/lib/openviking-session-map.ts new file mode 100644 index 0000000..8ca431f --- /dev/null +++ b/src/lib/openviking-session-map.ts @@ -0,0 +1,97 @@ +import fs from 'fs'; +import path from 'path'; +import { TINYCLAW_HOME } from './config'; + +export type OpenVikingSessionMapKey = { + channel: string; + senderId: string; + agentId: string; +}; + +type OpenVikingSessionRecord = { + sessionId: string; + channel: string; + senderId: string; + agentId: string; + updatedAt: string; +}; + +type OpenVikingSessionMap = { + version: 1; + sessions: Record; +}; + +const OPENVIKING_RUNTIME_DIR = path.join(TINYCLAW_HOME, 'runtime', 'openviking'); +const OPENVIKING_SESSION_MAP_FILE = path.join(OPENVIKING_RUNTIME_DIR, 'session-map.json'); + +function ensureRuntimeDir(): void { + if (!fs.existsSync(OPENVIKING_RUNTIME_DIR)) { + fs.mkdirSync(OPENVIKING_RUNTIME_DIR, { recursive: true }); + } +} + +function toCompositeKey(key: OpenVikingSessionMapKey): string { + return `${key.channel}::${key.senderId}::${key.agentId}`; +} + +function loadMap(): OpenVikingSessionMap { + ensureRuntimeDir(); + if (!fs.existsSync(OPENVIKING_SESSION_MAP_FILE)) { + return { version: 1, sessions: {} }; + } + + try { + const raw = fs.readFileSync(OPENVIKING_SESSION_MAP_FILE, 'utf8'); + const parsed = JSON.parse(raw) as Partial; + if (!parsed || typeof parsed !== 'object') { + return { version: 1, sessions: {} }; + } + const sessions = parsed.sessions; + if (!sessions || typeof sessions !== 'object') { + return { version: 1, sessions: {} }; + } + return { version: 1, sessions: sessions as Record }; + } catch { + return { version: 1, sessions: {} }; + } +} + +function saveMap(map: OpenVikingSessionMap): void { + ensureRuntimeDir(); + fs.writeFileSync(OPENVIKING_SESSION_MAP_FILE, JSON.stringify(map, null, 2) + '\n', 'utf8'); +} + +export function buildOpenVikingSessionMapKey(channel: string, senderId: string, agentId: string): OpenVikingSessionMapKey { + return { channel, senderId, agentId }; +} + +export function getOpenVikingSessionId(key: OpenVikingSessionMapKey): string | null { + const map = loadMap(); + const record = map.sessions[toCompositeKey(key)]; + if (!record || !record.sessionId) return null; + return record.sessionId; +} + +export function upsertOpenVikingSessionId(key: OpenVikingSessionMapKey, sessionId: string): void { + const map = loadMap(); + map.sessions[toCompositeKey(key)] = { + sessionId, + channel: key.channel, + senderId: key.senderId, + agentId: key.agentId, + updatedAt: new Date().toISOString(), + }; + saveMap(map); +} + +export function deleteOpenVikingSessionId(key: OpenVikingSessionMapKey): void { + const map = loadMap(); + const composite = toCompositeKey(key); + if (!map.sessions[composite]) return; + delete map.sessions[composite]; + saveMap(map); +} + +export function getOpenVikingSessionMapFilePath(): string { + return OPENVIKING_SESSION_MAP_FILE; +} diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 81b7411..d6a61c0 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -26,7 +26,23 @@ import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; import { invokeAgent, runCommand } from './lib/invoke'; import { jsonrepair } from 'jsonrepair'; -import { SessionTurn, parseSessionTurns, buildPrefetchBlock } from './lib/openviking-prefetch'; +import { + SessionTurn, + parseSessionTurns, + buildPrefetchBlock, + parseOpenVikingSearchHits, + summarizeOpenVikingSearchHitDistribution, + buildOpenVikingSearchPrefetchBlock, + OpenVikingSearchHitDistribution, +} from './lib/openviking-prefetch'; +import { + buildOpenVikingSessionMapKey, + getOpenVikingSessionId, + upsertOpenVikingSessionId, + deleteOpenVikingSessionId, + OpenVikingSessionMapKey, +} from './lib/openviking-session-map'; +import { ensureAgentDirectory } from './lib/agent-setup'; /** Parse JSON with automatic repair for malformed content (e.g. bad escapes). */ function safeParseJSON(raw: string, label?: string): T { @@ -53,14 +69,64 @@ const conversations = new Map(); const MAX_CONVERSATION_MESSAGES = 50; const LONG_RESPONSE_THRESHOLD = 4000; -const OPENVIKING_AUTOSYNC_ENABLED = process.env.TINYCLAW_OPENVIKING_AUTOSYNC !== '0'; +const OPENVIKING_AUTOSYNC_FALLBACK_ENABLED = process.env.TINYCLAW_OPENVIKING_AUTOSYNC !== '0'; const OPENVIKING_PREFETCH_ENABLED = process.env.TINYCLAW_OPENVIKING_PREFETCH !== '0'; +const OPENVIKING_SESSION_NATIVE_ENABLED = process.env.TINYCLAW_OPENVIKING_SESSION_NATIVE === '1'; +const OPENVIKING_SEARCH_NATIVE_ENABLED = process.env.TINYCLAW_OPENVIKING_SEARCH_NATIVE === '1'; const OPENVIKING_PREFETCH_TIMEOUT_MS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_TIMEOUT_MS || 5000); +const OPENVIKING_COMMIT_TIMEOUT_MS = Number(process.env.TINYCLAW_OPENVIKING_COMMIT_TIMEOUT_MS || 15000); const OPENVIKING_PREFETCH_MAX_CHARS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_MAX_CHARS || 2800); const OPENVIKING_PREFETCH_MAX_TURNS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_MAX_TURNS || 4); +const OPENVIKING_PREFETCH_MAX_HITS = Number(process.env.TINYCLAW_OPENVIKING_PREFETCH_MAX_HITS || 8); +const OPENVIKING_SEARCH_SCORE_THRESHOLD = process.env.TINYCLAW_OPENVIKING_SEARCH_SCORE_THRESHOLD; const OPENVIKING_SESSION_ROOT = '/tinyclaw/sessions'; +const OPENVIKING_NATIVE_PREFETCH_DUMP_FILE = path.join(path.dirname(LOG_FILE), 'prefetch_dump_native_latest.txt'); const openVikingSyncChains = new Map>(); +type OpenVikingPrefetchSource = 'search_native' | 'legacy_markdown' | 'none'; + +type OpenVikingPrefetchResult = { + block: string; + source: OpenVikingPrefetchSource; + diagnostics: string[]; + fallbackReason?: string; + distribution?: OpenVikingSearchHitDistribution; +}; + +type OpenVikingLegacyPrefetchResult = { + block: string; + diagnostics: string[]; +}; + +function writeNativePrefetchDump( + agentId: string, + query: string, + sessionId: string | undefined, + prefetch: OpenVikingPrefetchResult +): void { + if (prefetch.source !== 'search_native' || !prefetch.block) return; + const lines: string[] = [ + '# OpenViking Native Prefetch Dump (latest)', + '', + `- captured_at: ${new Date().toISOString()}`, + `- agent_id: ${agentId}`, + `- session_id: ${sessionId || 'none'}`, + `- source: ${prefetch.source}`, + `- distribution: ${maybeDistributionSummary(prefetch.distribution)}`, + `- diagnostics: ${prefetch.diagnostics.join(' | ') || 'none'}`, + '', + '## Query', + '', + query, + '', + '## Injected Block', + '', + prefetch.block, + '', + ]; + fs.writeFileSync(OPENVIKING_NATIVE_PREFETCH_DUMP_FILE, lines.join('\n'), 'utf8'); +} + function stripInjectedOpenVikingContext(text: string): string { const withEndMarker = /\n*------\n*\n*\[OpenViking Retrieved Context\][\s\S]*?\[End OpenViking Context\]\s*/g; const withoutEndMarker = /\n*------\n*\n*\[OpenViking Retrieved Context\][\s\S]*$/g; @@ -124,7 +190,7 @@ async function writeSessionFileToOpenViking( localFile: string, targetPath: string ): Promise { - if (!OPENVIKING_AUTOSYNC_ENABLED) return; + if (!OPENVIKING_AUTOSYNC_FALLBACK_ENABLED) return; const toolPath = getOpenVikingToolPath(workspacePath, agentId); if (!toolPath) return; await runCommand('node', [toolPath, 'write-file', targetPath, localFile], path.join(workspacePath, agentId)); @@ -198,10 +264,133 @@ async function appendTurnAndSyncOpenViking( ); } -async function fetchOpenVikingPrefetchContext(workspacePath: string, agentId: string, query: string): Promise { - if (!OPENVIKING_PREFETCH_ENABLED) return ''; +function resolveSessionMapKey(messageData: MessageData, agentId: string): OpenVikingSessionMapKey { + const senderId = messageData.senderId || messageData.sender || 'unknown-sender'; + return buildOpenVikingSessionMapKey(messageData.channel, senderId, agentId); +} + +function maybeDistributionSummary(distribution?: OpenVikingSearchHitDistribution): string { + if (!distribution) return 'memory=0,resource=0,skill=0'; + return `memory=${distribution.memory},resource=${distribution.resource},skill=${distribution.skill}`; +} + +async function runOpenVikingToolJson( + workspacePath: string, + agentId: string, + args: string[], + timeoutMs: number = OPENVIKING_PREFETCH_TIMEOUT_MS +): Promise { const toolPath = getOpenVikingToolPath(workspacePath, agentId); - if (!toolPath) return ''; + if (!toolPath) { + throw new Error(`OpenViking tool missing for @${agentId}`); + } + const commandArgs = args.includes('--json') ? args : [...args, '--json']; + const output = await runCommand( + 'node', + [toolPath, ...commandArgs], + path.join(workspacePath, agentId), + timeoutMs + ); + const trimmed = output.trim(); + if (!trimmed) return {}; + return safeParseJSON(trimmed, `openviking-tool:${args[0] || 'unknown'}`); +} + +function extractOpenVikingSessionId(payload: unknown): string { + const root = (payload && typeof payload === 'object' && !Array.isArray(payload)) + ? payload as Record + : {}; + const resultNode = (root.result && typeof root.result === 'object' && !Array.isArray(root.result)) + ? root.result as Record + : {}; + const dataNode = (root.data && typeof root.data === 'object' && !Array.isArray(root.data)) + ? root.data as Record + : {}; + + const candidates = [ + root.id, root.session_id, root.sessionId, + resultNode.id, resultNode.session_id, resultNode.sessionId, + dataNode.id, dataNode.session_id, dataNode.sessionId, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim()) { + return candidate.trim(); + } + } + return ''; +} + +async function ensureOpenVikingNativeSession( + workspacePath: string, + agentId: string, + sessionKey: OpenVikingSessionMapKey +): Promise<{ sessionId: string; isNew: boolean }> { + const existingSessionId = getOpenVikingSessionId(sessionKey); + if (existingSessionId) { + return { sessionId: existingSessionId, isNew: false }; + } + + const created = await runOpenVikingToolJson( + workspacePath, + agentId, + [ + 'session-create', + '--agent-id', sessionKey.agentId, + '--channel', sessionKey.channel, + '--sender-id', sessionKey.senderId, + ], + OPENVIKING_PREFETCH_TIMEOUT_MS + ); + const createdSessionId = extractOpenVikingSessionId(created); + if (!createdSessionId) { + throw new Error('OpenViking session create returned no session id'); + } + upsertOpenVikingSessionId(sessionKey, createdSessionId); + return { sessionId: createdSessionId, isNew: true }; +} + +async function appendNativeOpenVikingSessionMessage( + workspacePath: string, + agentId: string, + sessionId: string, + role: 'user' | 'assistant', + content: string +): Promise { + const sanitizedContent = stripInjectedOpenVikingContext(content); + const startedAt = Date.now(); + await runOpenVikingToolJson( + workspacePath, + agentId, + ['session-message', sessionId, role, sanitizedContent], + OPENVIKING_PREFETCH_TIMEOUT_MS + ); + const elapsedMs = Date.now() - startedAt; + log('INFO', `OpenViking session write success for @${agentId}: session_id=${sessionId} role=${role} elapsed_ms=${elapsedMs}`); +} + +async function commitNativeOpenVikingSession( + workspacePath: string, + agentId: string, + sessionId: string +): Promise { + const startedAt = Date.now(); + await runOpenVikingToolJson( + workspacePath, + agentId, + ['session-commit', sessionId], + OPENVIKING_COMMIT_TIMEOUT_MS + ); + const elapsedMs = Date.now() - startedAt; + log('INFO', `OpenViking session commit success for @${agentId}: session_id=${sessionId} elapsed_ms=${elapsedMs}`); +} + +async function fetchLegacyOpenVikingPrefetchContext( + workspacePath: string, + agentId: string, + query: string +): Promise { + const toolPath = getOpenVikingToolPath(workspacePath, agentId); + if (!toolPath) return { block: '', diagnostics: ['tool_missing'] }; const readTargets = [ `${OPENVIKING_SESSION_ROOT}/${agentId}/active.md`, @@ -306,12 +495,75 @@ async function fetchOpenVikingPrefetchContext(workspacePath: string, agentId: st } const turns = Array.from(dedup.values()); if (!turns.length) { - log('INFO', `OpenViking prefetch empty for @${agentId}: ${diagnostics.join(' | ')}`); - return ''; + return { block: '', diagnostics }; } const selected = turns.slice(0, OPENVIKING_PREFETCH_MAX_TURNS); - return buildPrefetchBlock(selected, OPENVIKING_PREFETCH_MAX_CHARS); + return { + block: buildPrefetchBlock(selected, OPENVIKING_PREFETCH_MAX_CHARS), + diagnostics, + }; +} + +async function fetchOpenVikingPrefetchContext( + workspacePath: string, + agentId: string, + query: string, + sessionId?: string +): Promise { + if (!OPENVIKING_PREFETCH_ENABLED) { + return { block: '', source: 'none', diagnostics: ['prefetch_disabled'] }; + } + + const toolPath = getOpenVikingToolPath(workspacePath, agentId); + if (!toolPath) { + return { block: '', source: 'none', diagnostics: ['tool_missing'] }; + } + + const diagnostics: string[] = []; + if (OPENVIKING_SEARCH_NATIVE_ENABLED) { + try { + const searchLimit = Math.max(OPENVIKING_PREFETCH_MAX_HITS * 2, 12); + const args = [ + 'search', + query, + '--limit', String(searchLimit), + ]; + if (OPENVIKING_SEARCH_SCORE_THRESHOLD !== undefined) { + args.push('--score-threshold', OPENVIKING_SEARCH_SCORE_THRESHOLD); + } + if (sessionId) { + args.push('--session-id', sessionId); + } + const searchResponse = await runOpenVikingToolJson(workspacePath, agentId, args, OPENVIKING_PREFETCH_TIMEOUT_MS); + const searchHits = parseOpenVikingSearchHits(searchResponse); + if (searchHits.length > 0) { + const distribution = summarizeOpenVikingSearchHitDistribution(searchHits.slice(0, OPENVIKING_PREFETCH_MAX_HITS)); + return { + block: buildOpenVikingSearchPrefetchBlock(searchHits, OPENVIKING_PREFETCH_MAX_CHARS, OPENVIKING_PREFETCH_MAX_HITS), + source: 'search_native', + diagnostics: [`native_search_hits=${searchHits.length}`, sessionId ? 'session_id_used=1' : 'session_id_used=0'], + distribution, + }; + } + diagnostics.push('native_search_empty'); + } catch (error) { + diagnostics.push(`native_search_error=${(error as Error).message}`); + } + } else { + diagnostics.push('native_search_disabled'); + } + + const legacy = await fetchLegacyOpenVikingPrefetchContext(workspacePath, agentId, query); + const fallbackReason = OPENVIKING_SEARCH_NATIVE_ENABLED + ? 'native_search_no_hits_or_error' + : 'native_search_flag_disabled'; + return { + block: legacy.block, + source: legacy.block ? 'legacy_markdown' : 'none', + diagnostics: [...diagnostics, ...legacy.diagnostics], + fallbackReason, + }; } /** @@ -566,6 +818,7 @@ async function processMessage(messageFile: string): Promise { } const agent = agents[agentId]; + ensureAgentDirectory(path.join(workspacePath, agentId)); log('INFO', `Routing to agent: ${agent.name} (${agentId}) [${agent.provider}/${agent.model}]`); if (!isInternal) { emitEvent('agent_routed', { agentId, agentName: agent.name, provider: agent.provider, model: agent.model, isTeamRouted }); @@ -594,13 +847,46 @@ async function processMessage(messageFile: string): Promise { // Check for per-agent reset const agentResetFlag = getAgentResetFlag(agentId, workspacePath); const shouldReset = fs.existsSync(agentResetFlag); + const sessionMapKey = !isInternal ? resolveSessionMapKey(messageData, agentId) : null; + let openVikingSessionId: string | null = null; + let nativeSessionWriteFailed = false; + const userMessageForSession = message; if (shouldReset) { fs.unlinkSync(agentResetFlag); - enqueueOpenVikingSync(agentId, async () => { - await finalizeOpenVikingSession(workspacePath, agentId); - log('INFO', `OpenViking session finalized for @${agentId}`); - }); + if (!isInternal && OPENVIKING_SESSION_NATIVE_ENABLED && sessionMapKey) { + const existingSessionId = getOpenVikingSessionId(sessionMapKey); + if (existingSessionId) { + try { + await commitNativeOpenVikingSession(workspacePath, agentId, existingSessionId); + } catch (error) { + log('WARN', `OpenViking session commit failed for @${agentId}: session_id=${existingSessionId} error=${(error as Error).message}`); + } finally { + deleteOpenVikingSessionId(sessionMapKey); + log('INFO', `OpenViking session map cleared for @${agentId}: session_id=${existingSessionId}`); + } + } else { + log('INFO', `OpenViking reset consumed for @${agentId}: no native session mapping found`); + } + } + + if (OPENVIKING_AUTOSYNC_FALLBACK_ENABLED) { + enqueueOpenVikingSync(agentId, async () => { + await finalizeOpenVikingSession(workspacePath, agentId); + log('INFO', `OpenViking legacy markdown session finalized for @${agentId}`); + }); + } + } + + if (!isInternal && OPENVIKING_SESSION_NATIVE_ENABLED && sessionMapKey) { + try { + const ensured = await ensureOpenVikingNativeSession(workspacePath, agentId, sessionMapKey); + openVikingSessionId = ensured.sessionId; + log('INFO', `OpenViking session resolved for @${agentId}: session_id=${openVikingSessionId} status=${ensured.isNew ? 'created' : 'reused'}`); + } catch (error) { + nativeSessionWriteFailed = true; + log('WARN', `OpenViking session setup failed for @${agentId}: ${(error as Error).message}`); + } } // For internal messages: append pending response indicator so the agent @@ -618,18 +904,48 @@ async function processMessage(messageFile: string): Promise { if (!isInternal) { try { - const retrievedContext = await fetchOpenVikingPrefetchContext(workspacePath, agentId, message); - if (retrievedContext) { - message += `\n\n------\n\n${retrievedContext}\n[End OpenViking Context]`; - log('INFO', `OpenViking prefetch hit for @${agentId}: injected ${retrievedContext.length} chars`); + const prefetch = await fetchOpenVikingPrefetchContext( + workspacePath, + agentId, + message, + openVikingSessionId || undefined + ); + if (prefetch.block) { + writeNativePrefetchDump(agentId, message, openVikingSessionId || undefined, prefetch); + message += `\n\n------\n\n${prefetch.block}\n[End OpenViking Context]`; + const distributionSummary = maybeDistributionSummary(prefetch.distribution); + log('INFO', `OpenViking prefetch hit for @${agentId}: source=${prefetch.source} distribution=${distributionSummary} injected_chars=${prefetch.block.length}`); + if (prefetch.fallbackReason) { + log('INFO', `OpenViking prefetch fallback for @${agentId}: reason=${prefetch.fallbackReason} diagnostics=${prefetch.diagnostics.join(' | ')}`); + } } else { - log('INFO', `OpenViking prefetch miss for @${agentId}`); + log('INFO', `OpenViking prefetch miss for @${agentId}: source=${prefetch.source} diagnostics=${prefetch.diagnostics.join(' | ')}`); } } catch (error) { log('WARN', `OpenViking prefetch skipped for @${agentId}: ${(error as Error).message}`); } } + if (!isInternal && OPENVIKING_SESSION_NATIVE_ENABLED) { + if (openVikingSessionId) { + try { + await appendNativeOpenVikingSessionMessage( + workspacePath, + agentId, + openVikingSessionId, + 'user', + userMessageForSession + ); + } catch (error) { + nativeSessionWriteFailed = true; + log('WARN', `OpenViking session write failed for @${agentId}: session_id=${openVikingSessionId} role=user error=${(error as Error).message}`); + } + } else { + nativeSessionWriteFailed = true; + log('WARN', `OpenViking session write skipped for @${agentId}: session_id_unavailable`); + } + } + // Invoke agent emitEvent('chain_step_start', { agentId, agentName: agent.name, fromAgent: messageData.fromAgent || null }); let response: string; @@ -644,9 +960,41 @@ async function processMessage(messageFile: string): Promise { emitEvent('chain_step_done', { agentId, agentName: agent.name, responseLength: response.length, responseText: response }); - enqueueOpenVikingSync(agentId, async () => { - await appendTurnAndSyncOpenViking(workspacePath, agentId, messageId, message, response, isInternal); - }); + if (!isInternal && OPENVIKING_SESSION_NATIVE_ENABLED && openVikingSessionId) { + try { + await appendNativeOpenVikingSessionMessage( + workspacePath, + agentId, + openVikingSessionId, + 'assistant', + response + ); + } catch (error) { + nativeSessionWriteFailed = true; + log('WARN', `OpenViking session write failed for @${agentId}: session_id=${openVikingSessionId} role=assistant error=${(error as Error).message}`); + } + } + + const shouldUseLegacyWriteback = OPENVIKING_AUTOSYNC_FALLBACK_ENABLED && ( + isInternal + || !OPENVIKING_SESSION_NATIVE_ENABLED + || nativeSessionWriteFailed + || !openVikingSessionId + ); + + if (shouldUseLegacyWriteback) { + const fallbackReasons: string[] = []; + if (isInternal) fallbackReasons.push('internal_message'); + if (!OPENVIKING_SESSION_NATIVE_ENABLED) fallbackReasons.push('session_native_disabled'); + if (OPENVIKING_SESSION_NATIVE_ENABLED && !openVikingSessionId) fallbackReasons.push('session_id_unavailable'); + if (nativeSessionWriteFailed) fallbackReasons.push('native_session_write_failed'); + log('INFO', `OpenViking legacy writeback fallback for @${agentId}: reasons=${fallbackReasons.join(',') || 'unknown'}`); + enqueueOpenVikingSync(agentId, async () => { + await appendTurnAndSyncOpenViking(workspacePath, agentId, messageId, message, response, isInternal); + }); + } else if (!isInternal && OPENVIKING_SESSION_NATIVE_ENABLED && openVikingSessionId) { + log('INFO', `OpenViking native write path complete for @${agentId}: session_id=${openVikingSessionId}`); + } // --- No team context: simple response to user --- if (!teamContext) { diff --git a/src/tools/openviking-tool.ts b/src/tools/openviking-tool.ts index 9e9a1fb..097f1cb 100644 --- a/src/tools/openviking-tool.ts +++ b/src/tools/openviking-tool.ts @@ -4,7 +4,18 @@ import path from 'path'; type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; -type Command = 'ls' | 'read' | 'write' | 'write-file' | 'res-get' | 'res-put' | 'find-uris'; +type Command = + | 'ls' + | 'read' + | 'write' + | 'write-file' + | 'res-get' + | 'res-put' + | 'find-uris' + | 'search' + | 'session-create' + | 'session-message' + | 'session-commit'; const HELP = `OpenViking workspace tool @@ -16,6 +27,10 @@ Usage: node openviking-tool.js res-get [--json] node openviking-tool.js res-put [--mime ] [--json] node openviking-tool.js find-uris [--limit ] [--score-threshold ] [--json] + node openviking-tool.js search [--session-id ] [--limit ] [--score-threshold ] [--json] + node openviking-tool.js session-create [--agent-id ] [--channel ] [--sender-id ] [--json] + node openviking-tool.js session-message [--json] + node openviking-tool.js session-commit [--json] Environment: OPENVIKING_BASE_URL API base URL (default: http://127.0.0.1:8320) @@ -45,11 +60,15 @@ function positionalArguments(): string[] { for (let i = 0; i < args.length; i++) { const arg = args[i]; if (arg === '--json') continue; - if (arg === '--mime') { - i += 1; - continue; - } - if (arg === '--limit' || arg === '--score-threshold') { + if ( + arg === '--mime' + || arg === '--limit' + || arg === '--score-threshold' + || arg === '--session-id' + || arg === '--agent-id' + || arg === '--channel' + || arg === '--sender-id' + ) { i += 1; continue; } @@ -77,6 +96,13 @@ type ListItem = { index: number; }; +type SearchMatch = { + type: 'memory' | 'resource' | 'skill'; + uri: string; + score: number; + abstract: string; +}; + const DIRECTORY_READ_MAX_FILES = 8; function toUri(input: string): string { @@ -162,17 +188,80 @@ function printRead(data: JsonValue): void { printJson(data); } -type FindMatch = { - uri: string; - score: number; - isLeaf: boolean; -}; +function normalizeAbstract(value: string): string { + const normalized = value.replace(/\s+/g, ' ').trim(); + if (!normalized) return '(no abstract provided)'; + const MAX_ABSTRACT_CHARS = 220; + if (normalized.length <= MAX_ABSTRACT_CHARS) return normalized; + return `${normalized.slice(0, MAX_ABSTRACT_CHARS - 3)}...`; +} + +function pickAbstract(node: Record): string { + const candidates = [ + node.abstract, + node.summary, + node.snippet, + node.text, + node.content, + node.description, + node.title, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string') { + return normalizeAbstract(candidate); + } + } + const metadata = asObject(node.metadata); + for (const candidate of [metadata.abstract, metadata.summary, metadata.snippet, metadata.description]) { + if (typeof candidate === 'string') { + return normalizeAbstract(candidate); + } + } + return '(no abstract provided)'; +} + +function extractSearchMatches(data: JsonValue): SearchMatch[] { + const root = asObject(data); + const resultNode = asObject(root.result ?? root.data ?? root); + const groups: Array<{ key: string; type: SearchMatch['type'] }> = [ + { key: 'memories', type: 'memory' }, + { key: 'resources', type: 'resource' }, + { key: 'skills', type: 'skill' }, + ]; + + const out: SearchMatch[] = []; + for (const group of groups) { + const items = asArray(resultNode[group.key]); + for (const item of items) { + const node = asObject(item); + const uri = String(node.uri ?? node.path ?? '').trim(); + if (!uri) continue; + out.push({ + type: group.type, + uri, + score: Number(node.score ?? 0), + abstract: pickAbstract(node), + }); + } + } + + out.sort((a, b) => b.score - a.score); + return out; +} -function extractFindMatches(data: JsonValue): FindMatch[] { +function printSearchMatches(data: JsonValue): void { + const matches = extractSearchMatches(data); + if (!matches.length) return; + for (const m of matches) { + console.log(`${m.type}\t${m.score}\t${m.uri}\t${m.abstract}`); + } +} + +function extractFindMatches(data: JsonValue): Array<{ uri: string; score: number; isLeaf: boolean }> { const root = asObject(data); const resultNode = asObject(root.result); const groups = ['memories', 'resources', 'skills']; - const out: FindMatch[] = []; + const out: Array<{ uri: string; score: number; isLeaf: boolean }> = []; for (const g of groups) { const items = asArray(resultNode[g]); for (const item of items) { @@ -198,14 +287,6 @@ function printFindUris(data: JsonValue): void { } } -function extractUris(data: JsonValue): string[] { - const root = asObject(data); - const items = asArray(root.result ?? asObject(root.data).items ?? root.items); - return items - .map((item) => String(asObject(item).uri ?? asObject(item).path ?? '')) - .filter((uri) => uri.length > 0); -} - function extractListItems(data: JsonValue): ListItem[] { const root = asObject(data); const items = asArray(root.result ?? asObject(root.data).items ?? root.items); @@ -327,6 +408,90 @@ async function readWithDirectoryFallback(uri: string, rawPath: string): Promise< } } +function extractSessionId(response: JsonValue): string { + const root = asObject(response); + const resultNode = asObject(root.result); + const dataNode = asObject(root.data); + const candidates = [ + root.id, + root.session_id, + root.sessionId, + resultNode.id, + resultNode.session_id, + resultNode.sessionId, + dataNode.id, + dataNode.session_id, + dataNode.sessionId, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate.trim()) { + return candidate.trim(); + } + } + return ''; +} + +async function createSession(agentId?: string, channel?: string, senderId?: string): Promise { + const metadata = { + source: 'tinyclaw', + agent_id: agentId || 'unknown', + channel: channel || 'unknown', + sender_id: senderId || 'unknown', + created_at: new Date().toISOString(), + }; + + const fallbackName = `tinyclaw:${agentId || 'agent'}:${channel || 'channel'}:${senderId || 'sender'}`; + const payloads: Array> = [ + { metadata }, + { name: fallbackName, metadata }, + { agent_id: agentId, channel, sender_id: senderId, metadata }, + {}, + ]; + + let lastError: Error | null = null; + for (const payload of payloads) { + try { + return await request('/api/v1/sessions', { + method: 'POST', + body: JSON.stringify(payload), + }); + } catch (error) { + lastError = error as Error; + } + } + + throw lastError || new Error('Failed to create session'); +} + +async function writeSessionMessage(sessionId: string, role: string, content: string): Promise { + const endpoint = `/api/v1/sessions/${encodeURIComponent(sessionId)}/messages`; + try { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({ role, content }), + }); + } catch { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({ message: { role, content } }), + }); + } +} + +async function commitSession(sessionId: string): Promise { + const endpoint = `/api/v1/sessions/${encodeURIComponent(sessionId)}/commit`; + try { + return await request(endpoint, { + method: 'POST', + body: JSON.stringify({}), + }); + } catch { + return await request(endpoint, { + method: 'POST', + }); + } +} + async function run(): Promise { const positional = positionalArguments(); if (!positional.length || positional[0] === 'help' || positional[0] === '--help') { @@ -465,6 +630,69 @@ async function run(): Promise { else printFindUris(response); return; } + case 'search': { + if (!positional[1]) fail('Usage: search [--session-id ] [--limit ] [--score-threshold ]'); + const query = positional[1]; + const sessionId = getFlagValue('--session-id'); + const limitRaw = getFlagValue('--limit'); + const scoreThresholdRaw = getFlagValue('--score-threshold'); + const limit = limitRaw ? Number(limitRaw) : 12; + if (!Number.isFinite(limit) || limit <= 0) fail('Invalid --limit value'); + let scoreThreshold: number | undefined; + if (scoreThresholdRaw !== undefined) { + const parsed = Number(scoreThresholdRaw); + if (!Number.isFinite(parsed)) fail('Invalid --score-threshold value'); + scoreThreshold = parsed; + } + response = await request('/api/v1/search/search', { + method: 'POST', + body: JSON.stringify({ + query, + session_id: sessionId, + limit, + score_threshold: scoreThreshold, + }), + }); + if (jsonOutput) printJson(response); + else printSearchMatches(response); + return; + } + case 'session-create': { + const agentId = getFlagValue('--agent-id'); + const channel = getFlagValue('--channel'); + const senderId = getFlagValue('--sender-id'); + response = await createSession(agentId, channel, senderId); + const sessionId = extractSessionId(response); + if (!sessionId) { + fail('Session create succeeded but no session id found in response'); + } + if (jsonOutput) printJson(response); + else console.log(sessionId); + return; + } + case 'session-message': { + if (!positional[1] || !positional[2] || positional[3] === undefined) { + fail('Usage: session-message '); + } + const sessionId = positional[1]; + const role = positional[2]; + const content = positional[3]; + if (!['user', 'assistant', 'system'].includes(role)) { + fail('Role must be one of: user, assistant, system'); + } + response = await writeSessionMessage(sessionId, role, content); + if (jsonOutput) printJson(response); + else console.log(`Session message stored: ${sessionId} (${role})`); + return; + } + case 'session-commit': { + if (!positional[1]) fail('Usage: session-commit '); + const sessionId = positional[1]; + response = await commitSession(sessionId); + if (jsonOutput) printJson(response); + else console.log(`Session committed: ${sessionId}`); + return; + } default: fail(`Unknown command: ${command}\n\n${HELP}`); }