From 5b0f7ce650a96c159a445caf80bac94093df0470 Mon Sep 17 00:00:00 2001 From: MZ Date: Thu, 12 Feb 2026 21:28:57 -0700 Subject: [PATCH 1/6] feat(memory): add optional qmd retrieval with safe defaults --- README.md | 37 +++++ docs/TROUBLESHOOTING.md | 32 ++++ src/lib/invoke.ts | 10 +- src/lib/memory.ts | 359 ++++++++++++++++++++++++++++++++++++++++ src/lib/types.ts | 12 ++ src/queue-processor.ts | 49 +++++- 6 files changed, 493 insertions(+), 6 deletions(-) create mode 100644 src/lib/memory.ts diff --git a/README.md b/README.md index 490fe90..e05e02a 100644 --- a/README.md +++ b/README.md @@ -384,10 +384,47 @@ Located at `.tinyclaw/settings.json`: }, "monitoring": { "heartbeat_interval": 3600 + }, + "memory": { + "enabled": true, + "qmd": { + "enabled": true, + "command": "/home/me/.bun/bin/qmd", + "top_k": 4, + "min_score": 0.05, + "max_chars": 2500, + "update_interval_seconds": 120, + "use_semantic_search": false + } } } ``` +### Optional Memory Retrieval (QMD) + +TinyClaw can use external memory retrieval with `qmd` (BM25 by default). + +- Memory retrieval is **disabled by default**. Set `"memory.enabled": true` to enable. +- Retrieval runs only for user chat channels (`telegram`, `discord`, `whatsapp`). +- Heartbeat/system messages are excluded from retrieval and memory persistence. +- Turns are stored in `~/.tinyclaw/memory/turns//`. + +Quick setup: + +```bash +# Install qmd (recommended by qmd project) +bun install -g github:tobi/qmd + +# Linux/WSL: install sqlite-vec extension package if qmd reports missing extension +bun add -g sqlite-vec-linux-x64 + +# Verify qmd works +qmd status + +# Optional: if qmd is not in PATH, set memory.qmd.command in settings.json +tinyclaw restart +``` + ### Heartbeat Configuration Edit agent-specific heartbeat prompts: diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 3cd1797..dfb1860 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -375,6 +375,38 @@ ps aux | grep -E 'claude|codex|node' | awk '{print $4, $11}' tail -f .tinyclaw/logs/queue.log | grep "Processing completed" ``` +### QMD memory retrieval not working + +If you enabled memory retrieval and see warnings like `qmd not found` or `Memory retrieval skipped`: + +1. **Check qmd command:** + ```bash + command -v qmd + qmd status + ``` + +2. **If qmd is not in PATH, set explicit command in `~/.tinyclaw/settings.json`:** + ```json + { + "memory": { + "enabled": true, + "qmd": { + "command": "/home/you/.bun/bin/qmd" + } + } + } + ``` + +3. **Restart TinyClaw:** + ```bash + tinyclaw restart + ``` + +4. **Verify retrieval hits in logs:** + ```bash + grep "Memory retrieval hit" ~/.tinyclaw/logs/queue.log + ``` + ## Log Analysis ### Enable debug logging diff --git a/src/lib/invoke.ts b/src/lib/invoke.ts index 471eaf2..b25370f 100644 --- a/src/lib/invoke.ts +++ b/src/lib/invoke.ts @@ -1,10 +1,11 @@ import { spawn } from 'child_process'; import fs from 'fs'; import path from 'path'; -import { AgentConfig, TeamConfig } from './types'; +import { AgentConfig, Settings, TeamConfig } from './types'; import { SCRIPT_DIR, resolveClaudeModel, resolveCodexModel } from './config'; import { log } from './logging'; import { ensureAgentDirectory, updateAgentTeammates } from './agent-setup'; +import { enrichMessageWithMemory } from './memory'; export async function runCommand(command: string, args: string[], cwd?: string): Promise { return new Promise((resolve, reject) => { @@ -51,8 +52,10 @@ export async function invokeAgent( agent: AgentConfig, agentId: string, message: string, + sourceChannel: string, workspacePath: string, shouldReset: boolean, + settings: Settings, agents: Record = {}, teams: Record = {} ): Promise { @@ -74,6 +77,7 @@ export async function invokeAgent( : path.join(workspacePath, agent.working_directory)) : agentDir; + const messageForModel = await enrichMessageWithMemory(agentId, message, settings, sourceChannel); const provider = agent.provider || 'anthropic'; if (provider === 'openai') { @@ -93,7 +97,7 @@ export async function invokeAgent( if (modelId) { codexArgs.push('--model', modelId); } - codexArgs.push('--skip-git-repo-check', '--dangerously-bypass-approvals-and-sandbox', '--json', message); + codexArgs.push('--skip-git-repo-check', '--dangerously-bypass-approvals-and-sandbox', '--json', messageForModel); const codexOutput = await runCommand('codex', codexArgs, workingDir); @@ -130,7 +134,7 @@ export async function invokeAgent( if (continueConversation) { claudeArgs.push('-c'); } - claudeArgs.push('-p', message); + claudeArgs.push('-p', messageForModel); return await runCommand('claude', claudeArgs, workingDir); } diff --git a/src/lib/memory.ts b/src/lib/memory.ts new file mode 100644 index 0000000..a65eeab --- /dev/null +++ b/src/lib/memory.ts @@ -0,0 +1,359 @@ +import { spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { AgentConfig, Settings } from './types'; +import { TINYCLAW_HOME } from './config'; +import { log } from './logging'; + +const MEMORY_ROOT = path.join(TINYCLAW_HOME, 'memory'); +const MEMORY_TURNS_DIR = path.join(MEMORY_ROOT, 'turns'); + +const DEFAULT_TOP_K = 4; +const DEFAULT_MIN_SCORE = 0.0; +const DEFAULT_MAX_CHARS = 2500; +const DEFAULT_UPDATE_INTERVAL_SECONDS = 120; + +let qmdChecked = false; +let qmdAvailable = false; +let qmdUnavailableLogged = false; +let qmdCommandPath: string | null = null; +let qmdCheckKey = ''; +const collectionPrepared = new Set(); +const lastCollectionUpdateMs = new Map(); +const MEMORY_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); + +interface QmdResult { + score: number; + snippet: string; + source: string; +} + +interface QmdConfig { + enabled: boolean; + command?: string; + topK: number; + minScore: number; + maxChars: number; + updateIntervalSeconds: number; + useSemanticSearch: boolean; +} + +interface CommandResult { + stdout: string; + stderr: string; +} + +function getQmdConfig(settings: Settings): QmdConfig { + const memoryCfg = settings.memory?.qmd; + const command = typeof memoryCfg?.command === 'string' ? memoryCfg.command.trim() : ''; + return { + enabled: settings.memory?.enabled === true && memoryCfg?.enabled !== false, + command: command || undefined, + topK: Number.isFinite(memoryCfg?.top_k) ? Math.max(1, Number(memoryCfg?.top_k)) : DEFAULT_TOP_K, + minScore: Number.isFinite(memoryCfg?.min_score) ? Number(memoryCfg?.min_score) : DEFAULT_MIN_SCORE, + maxChars: Number.isFinite(memoryCfg?.max_chars) ? Math.max(500, Number(memoryCfg?.max_chars)) : DEFAULT_MAX_CHARS, + updateIntervalSeconds: Number.isFinite(memoryCfg?.update_interval_seconds) + ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) + : DEFAULT_UPDATE_INTERVAL_SECONDS, + useSemanticSearch: memoryCfg?.use_semantic_search === true, + }; +} + +function sanitizeId(raw: string): string { + return raw.toLowerCase().replace(/[^a-z0-9_-]/g, '-'); +} + +function getAgentTurnsDir(agentId: string): string { + return path.join(MEMORY_TURNS_DIR, sanitizeId(agentId)); +} + +function getCollectionName(agentId: string): string { + return `tinyclaw-${sanitizeId(agentId)}`; +} + +function ensureDir(dir: string): void { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } +} + +function runCommand(command: string, args: string[], cwd?: string, timeoutMs = 12000): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + + const timer = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, timeoutMs); + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + child.on('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.on('close', (code) => { + clearTimeout(timer); + if (timedOut) { + reject(new Error(`Command timed out after ${timeoutMs}ms`)); + return; + } + if (code === 0) { + resolve({ stdout, stderr }); + return; + } + reject(new Error(stderr.trim() || `Command exited with code ${code}`)); + }); + }); +} + +async function isQmdAvailable(preferredCommand?: string): Promise { + const key = preferredCommand || '__auto__'; + if (qmdChecked && qmdCheckKey === key) { + return qmdAvailable; + } + qmdChecked = true; + qmdCheckKey = key; + qmdAvailable = false; + qmdCommandPath = null; + + const bundledQmd = path.join(require('os').homedir(), '.bun/bin/qmd'); + const candidates = preferredCommand ? [preferredCommand] : [bundledQmd, 'qmd']; + + try { + for (const candidate of candidates) { + try { + await runCommand(candidate, ['--help'], undefined, 5000); + qmdCommandPath = candidate; + qmdAvailable = true; + break; + } catch { + // Try next candidate. + } + } + } finally { + if (!qmdAvailable) { + qmdCommandPath = null; + } + } + return qmdAvailable; +} + +function shouldUseMemoryForChannel(channel: string): boolean { + return MEMORY_CHANNELS.has(channel); +} + +async function ensureCollection(agentId: string): Promise { + ensureDir(MEMORY_ROOT); + const agentTurnsDir = getAgentTurnsDir(agentId); + ensureDir(agentTurnsDir); + + const collectionName = getCollectionName(agentId); + if (!collectionPrepared.has(collectionName)) { + try { + await runCommand(qmdCommandPath || 'qmd', ['collection', 'add', agentTurnsDir, '--name', collectionName, '--mask', '**/*.md'], undefined, 10000); + collectionPrepared.add(collectionName); + } catch (error) { + const msg = (error as Error).message.toLowerCase(); + if (msg.includes('already') || msg.includes('exists')) { + collectionPrepared.add(collectionName); + } else { + throw error; + } + } + } + + return collectionName; +} + +async function maybeUpdateCollection(collectionName: string, updateIntervalSeconds: number): Promise { + const now = Date.now(); + const last = lastCollectionUpdateMs.get(collectionName) || 0; + if (now - last < updateIntervalSeconds * 1000) { + return; + } + await runCommand(qmdCommandPath || 'qmd', ['update', '--collections', collectionName], undefined, 15000); + lastCollectionUpdateMs.set(collectionName, now); +} + +function parseQmdResults(raw: string): QmdResult[] { + const trimmed = raw.trim(); + if (!trimmed) { + return []; + } + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return []; + } + + const rows = Array.isArray(parsed) + ? parsed + : (parsed as { results?: unknown[] }).results || []; + + const results: QmdResult[] = []; + for (const row of rows) { + if (!row || typeof row !== 'object') { + continue; + } + const r = row as Record; + const score = typeof r.score === 'number' ? r.score : 0; + const snippet = String(r.snippet || r.context || r.text || r.content || '').trim(); + const source = String(r.path || r.file || r.source || r.title || '').trim(); + if (!snippet) { + continue; + } + results.push({ score, snippet, source }); + } + return results; +} + +function formatMemoryPrompt(results: QmdResult[], maxChars: number): string { + if (results.length === 0) { + return ''; + } + + const blocks: string[] = []; + let usedChars = 0; + + for (let i = 0; i < results.length; i++) { + const result = results[i]; + const block = [ + `Snippet ${i + 1} (score=${result.score.toFixed(3)}):`, + result.source ? `Source: ${result.source}` : 'Source: unknown', + result.snippet, + ].join('\n'); + + if (usedChars + block.length > maxChars) { + break; + } + blocks.push(block); + usedChars += block.length; + } + + if (blocks.length === 0) { + return ''; + } + + return [ + '', + '---', + 'Retrieved memory snippets (from past conversations):', + 'Use only if relevant. Prioritize current user instructions over old memory.', + '', + blocks.join('\n\n'), + ].join('\n'); +} + +export async function enrichMessageWithMemory( + agentId: string, + message: string, + settings: Settings, + sourceChannel: string +): Promise { + const qmdCfg = getQmdConfig(settings); + if (!qmdCfg.enabled) { + return message; + } + if (!shouldUseMemoryForChannel(sourceChannel)) { + return message; + } + + const hasQmd = await isQmdAvailable(qmdCfg.command); + if (!hasQmd) { + if (!qmdUnavailableLogged) { + log('WARN', 'qmd not found in PATH, memory retrieval disabled'); + qmdUnavailableLogged = true; + } + return message; + } + + try { + const collectionName = await ensureCollection(agentId); + await maybeUpdateCollection(collectionName, qmdCfg.updateIntervalSeconds); + + const queryArgs = qmdCfg.useSemanticSearch + ? ['vsearch', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)] + : ['search', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; + + const { stdout } = await runCommand(qmdCommandPath || 'qmd', queryArgs, undefined, 12000); + const results = parseQmdResults(stdout); + if (results.length === 0) { + return message; + } + + const memoryBlock = formatMemoryPrompt(results, qmdCfg.maxChars); + if (!memoryBlock) { + return message; + } + + log('INFO', `Memory retrieval hit for @${agentId}: ${results.length} snippet(s)`); + return `${message}${memoryBlock}`; + } catch (error) { + log('WARN', `Memory retrieval skipped for @${agentId}: ${(error as Error).message}`); + return message; + } +} + +function timestampFilename(ts: number): string { + return new Date(ts).toISOString().replace(/[:.]/g, '-'); +} + +function truncate(text: string, max = 16000): string { + if (text.length <= max) { + return text; + } + return `${text.substring(0, max)}\n\n[truncated]`; +} + +export async function saveTurnToMemory(params: { + agentId: string; + agent: AgentConfig; + channel: string; + sender: string; + messageId: string; + userMessage: string; + agentResponse: string; + timestampMs?: number; +}): Promise { + try { + const timestampMs = params.timestampMs || Date.now(); + const dir = getAgentTurnsDir(params.agentId); + ensureDir(dir); + + const fileName = `${timestampFilename(timestampMs)}-${params.messageId}.md`; + const filePath = path.join(dir, fileName); + const lines = [ + `# Turn for @${params.agentId} (${params.agent.name})`, + '', + `- Timestamp: ${new Date(timestampMs).toISOString()}`, + `- Channel: ${params.channel}`, + `- Sender: ${params.sender}`, + `- Message ID: ${params.messageId}`, + '', + '## User', + '', + truncate(params.userMessage), + '', + '## Assistant', + '', + truncate(params.agentResponse), + '', + ]; + + fs.writeFileSync(filePath, lines.join('\n')); + } catch (error) { + log('WARN', `Failed to persist memory turn for @${params.agentId}: ${(error as Error).message}`); + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 1a20267..f167377 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -41,6 +41,18 @@ export interface Settings { monitoring?: { heartbeat_interval?: number; }; + memory?: { + enabled?: boolean; + qmd?: { + enabled?: boolean; + command?: string; + top_k?: number; + min_score?: number; + max_chars?: number; + update_interval_seconds?: number; + use_semantic_search?: boolean; + }; + }; } export interface MessageData { diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 1844d6a..2024382 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -21,6 +21,13 @@ import { import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; import { invokeAgent } from './lib/invoke'; +import { saveTurnToMemory } from './lib/memory'; + +const MEMORY_PERSIST_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); + +function shouldPersistMemoryTurn(channel: string): boolean { + return MEMORY_PERSIST_CHANNELS.has(channel); +} // Ensure directories exist [QUEUE_INCOMING, QUEUE_OUTGOING, QUEUE_PROCESSING, path.dirname(LOG_FILE)].forEach(dir => { @@ -138,7 +145,19 @@ async function processMessage(messageFile: string): Promise { if (!teamContext) { // No team context — single agent invocation (backward compatible) try { - finalResponse = await invokeAgent(agent, agentId, message, workspacePath, shouldReset, agents, teams); + finalResponse = await invokeAgent(agent, agentId, message, channel, workspacePath, shouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId, + agent, + channel, + sender, + messageId, + userMessage: message, + agentResponse: finalResponse, + timestampMs: Date.now(), + }); + } } catch (error) { const provider = agent.provider || 'anthropic'; log('ERROR', `${provider === 'openai' ? 'Codex' : 'Claude'} error (agent: ${agentId}): ${(error as Error).message}`); @@ -176,7 +195,19 @@ async function processMessage(messageFile: string): Promise { let stepResponse: string; try { - stepResponse = await invokeAgent(currentAgent, currentAgentId, currentMessage, workspacePath, currentShouldReset, agents, teams); + stepResponse = await invokeAgent(currentAgent, currentAgentId, currentMessage, channel, workspacePath, currentShouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId: currentAgentId, + agent: currentAgent, + channel, + sender, + messageId, + userMessage: currentMessage, + agentResponse: stepResponse, + timestampMs: Date.now(), + }); + } } catch (error) { const provider = currentAgent.provider || 'anthropic'; log('ERROR', `${provider === 'openai' ? 'Codex' : 'Claude'} error (agent: ${currentAgentId}): ${(error as Error).message}`); @@ -236,7 +267,19 @@ async function processMessage(messageFile: string): Promise { let mResponse: string; try { const mMessage = `[Message from teammate @${currentAgentId}]:\n${mention.message}`; - mResponse = await invokeAgent(mAgent, mention.teammateId, mMessage, workspacePath, mShouldReset, agents, teams); + mResponse = await invokeAgent(mAgent, mention.teammateId, mMessage, channel, workspacePath, mShouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId: mention.teammateId, + agent: mAgent, + channel, + sender, + messageId, + userMessage: mMessage, + agentResponse: mResponse, + timestampMs: Date.now(), + }); + } } catch (error) { log('ERROR', `Fan-out error (agent: ${mention.teammateId}): ${(error as Error).message}`); mResponse = "Sorry, I encountered an error processing this request."; From 5c9fae1ee7a0cd2579f2ba779df8460419523ea6 Mon Sep 17 00:00:00 2001 From: mcz Date: Sun, 15 Feb 2026 15:05:10 -0700 Subject: [PATCH 2/6] fix(whatsapp): preflight-check Puppeteer Chrome and fail fast (#85) * fix(whatsapp): fail fast when Puppeteer Chrome is missing * fix(whatsapp): validate Puppeteer executable path exists --- README.md | 1 + docs/TROUBLESHOOTING.md | 8 ++++++++ lib/daemon.sh | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/README.md b/README.md index e05e02a..828f938 100644 --- a/README.md +++ b/README.md @@ -517,6 +517,7 @@ tinyclaw logs all - Bash version error → Install bash 4.0+: `brew install bash` - WhatsApp not connecting → Reset auth: `tinyclaw channels reset whatsapp` +- WhatsApp startup fails with `Could not find Chrome` → Install browser: `npx puppeteer browsers install chrome` - Messages stuck → Clear queue: `rm -rf .tinyclaw/queue/processing/*` - Agent not found → Check: `tinyclaw agent list` diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index dfb1860..4cd2456 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -46,10 +46,18 @@ tinyclaw channels reset whatsapp tinyclaw restart ``` +If startup fails with `Could not find Chrome (...)`, install Puppeteer Chrome first: + +```bash +npx puppeteer browsers install chrome +tinyclaw restart +``` + **Common causes:** - QR code expired (scan within 60 seconds) - Session files corrupted - Multiple WhatsApp Web sessions active +- Chrome/Chromium missing for Puppeteer **Solution:** 1. Delete session: `rm -rf .tinyclaw/whatsapp-session/` diff --git a/lib/daemon.sh b/lib/daemon.sh index 839b9cb..22a9d34 100644 --- a/lib/daemon.sh +++ b/lib/daemon.sh @@ -2,6 +2,40 @@ # Daemon lifecycle management for TinyClaw # Handles starting, stopping, restarting, and status checking +whatsapp_enabled() { + local ch + for ch in "${ACTIVE_CHANNELS[@]}"; do + if [ "$ch" = "whatsapp" ]; then + return 0 + fi + done + return 1 +} + +check_whatsapp_preflight() { + if ! whatsapp_enabled; then + return 0 + fi + + # whatsapp-web.js launches Puppeteer; fail fast if Chrome is missing. + local chrome_path + chrome_path=$(node -e "const p=require('puppeteer');process.stdout.write(p.executablePath() || '')" 2>/dev/null || true) + if [ -n "$chrome_path" ] && [ -x "$chrome_path" ]; then + return 0 + fi + + echo -e "${RED}WhatsApp preflight failed: Chrome/Chromium not found for Puppeteer.${NC}" + echo "" + echo "Install browser dependency and retry:" + echo -e " ${GREEN}npx puppeteer browsers install chrome${NC}" + echo -e " ${GREEN}./tinyclaw.sh start${NC}" + echo "" + echo "Tip: if using a custom HOME, run both commands with the same HOME value." + echo "" + log "WhatsApp preflight failed: missing Chrome/Chromium for Puppeteer" + return 1 +} + # Start daemon start_daemon() { if session_exists; then @@ -54,6 +88,10 @@ start_daemon() { return 1 fi + if ! check_whatsapp_preflight; then + return 1 + fi + # Validate tokens for channels that need them for ch in "${ACTIVE_CHANNELS[@]}"; do local token_key="${CHANNEL_TOKEN_KEY[$ch]:-}" From 03f3569a902640f5b741d3cd76e8789de9ed9dbf Mon Sep 17 00:00:00 2001 From: MZ Date: Tue, 17 Feb 2026 22:47:37 -0700 Subject: [PATCH 3/6] feat(memory): improve qmd retrieval reliability and observability --- README.md | 11 +- lib/setup-wizard.sh | 40 ++++ scripts/patch-qmd-no-expansion.sh | 54 +++++ src/lib/memory.ts | 318 +++++++++++++++++++++++++++++- src/lib/types.ts | 7 + tinyclaw.sh | 3 +- 6 files changed, 424 insertions(+), 9 deletions(-) create mode 100755 scripts/patch-qmd-no-expansion.sh diff --git a/README.md b/README.md index 828f938..d80b890 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The setup wizard will guide you through: 5. **AI provider** - Select Anthropic (Claude) or OpenAI 6. **Model selection** - Choose model (e.g., Sonnet, Opus, GPT-5.3) 7. **Heartbeat interval** - Set proactive check-in frequency +8. **Memory retrieval (optional)** - Enable/disable QMD-based memory at setup time
📱 Channel Setup Guides @@ -394,7 +395,10 @@ Located at `.tinyclaw/settings.json`: "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 120, - "use_semantic_search": false + "use_semantic_search": false, + "disable_query_expansion": true, + "allow_unsafe_vsearch": false, + "debug_logging": false } } } @@ -425,6 +429,11 @@ qmd status tinyclaw restart ``` +Safety note: +- If `use_semantic_search` is enabled, TinyClaw defaults to safe mode and will fall back to BM25 unless disable-expansion support is detected. +- Set `memory.qmd.allow_unsafe_vsearch: true` only if you explicitly want to allow unguarded `vsearch`. +- Set `memory.qmd.debug_logging: true` to print QMD memory debug logs (command/mode/timeout) in `queue.log`. + ### Heartbeat Configuration Edit agent-specific heartbeat prompts: diff --git a/lib/setup-wizard.sh b/lib/setup-wizard.sh index c7acfe2..0200832 100755 --- a/lib/setup-wizard.sh +++ b/lib/setup-wizard.sh @@ -174,6 +174,39 @@ DEFAULT_AGENT_NAME=$(echo "$DEFAULT_AGENT_NAME" | tr ' ' '-' | tr -cd 'a-zA-Z0-9 echo -e "${GREEN}✓ Default agent: $DEFAULT_AGENT_NAME${NC}" echo "" +# Optional memory retrieval setup +echo "Enable memory retrieval (QMD)?" +echo -e "${YELLOW}(Optional. If disabled, TinyClaw still works normally.)${NC}" +echo "" +read -rp "Enable memory retrieval? [y/N]: " ENABLE_MEMORY + +MEMORY_ENABLED=false +USE_SEMANTIC_SEARCH=false +QMD_COMMAND="" + +if [[ "$ENABLE_MEMORY" =~ ^[yY] ]]; then + MEMORY_ENABLED=true + + if [ -x "$HOME/.bun/bin/qmd" ]; then + QMD_COMMAND="$HOME/.bun/bin/qmd" + elif command -v qmd >/dev/null 2>&1; then + QMD_COMMAND="$(command -v qmd)" + fi + + if [ -n "$QMD_COMMAND" ]; then + echo -e "${GREEN}✓ Found qmd: $QMD_COMMAND${NC}" + else + echo -e "${YELLOW}⚠ qmd not found in PATH right now.${NC}" + echo -e "${YELLOW} You can install later with: bun install -g github:tobi/qmd${NC}" + fi + + read -rp "Use semantic search (vector)? [y/N]: " USE_SEMANTIC + if [[ "$USE_SEMANTIC" =~ ^[yY] ]]; then + USE_SEMANTIC_SEARCH=true + fi +fi +echo "" + # --- Additional Agents (optional) --- echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN} Additional Agents (Optional)${NC}" @@ -274,6 +307,12 @@ else MODELS_SECTION='"models": { "provider": "openai", "openai": { "model": "'"${MODEL}"'" } }' fi +if [ "$MEMORY_ENABLED" = true ]; then + MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' +else + MEMORY_SECTION='"memory": { "enabled": false },' +fi + cat > "$SETTINGS_FILE" < "$SETTINGS_FILE" <&2 + exit 1 +fi + +if grep -q "QMD_VSEARCH_DISABLE_EXPANSION" "$TARGET_PATH"; then + echo "Already patched: $TARGET_PATH" + exit 0 +fi + +node - "$TARGET_PATH" <<'NODE' +const fs = require('fs'); +const target = process.argv[2]; +const src = fs.readFileSync(target, 'utf8'); + +const marker = 'QMD_VSEARCH_DISABLE_EXPANSION'; +if (src.includes(marker)) { + console.log(`Already patched: ${target}`); + process.exit(0); +} + +const pattern = / \/\/ Expand query — filter to vec\/hyde only \(lex queries target FTS, not vector\)\n const allExpanded = await store\.expandQuery\(query\);\n const vecExpanded = allExpanded\.filter\(q => q\.type !== 'lex'\);\n options\?\.hooks\?\.onExpand\?\.\(query, vecExpanded\);/; + +const replacement = [ + " // Optional: disable query expansion to run embedding-only vector search.", + " const disableExpansion = process.env.QMD_VSEARCH_DISABLE_EXPANSION === '1';", + " const vecExpanded = disableExpansion", + " ? []", + " : (await store.expandQuery(query)).filter(q => q.type !== 'lex');", + " options?.hooks?.onExpand?.(query, vecExpanded);", +].join('\n'); + +if (!pattern.test(src)) { + console.error('Patch anchor not found. qmd source layout may have changed.'); + process.exit(2); +} + +const out = src.replace(pattern, replacement); +fs.writeFileSync(target, out); +console.log(`Patched: ${target}`); +NODE diff --git a/src/lib/memory.ts b/src/lib/memory.ts index a65eeab..37f0502 100644 --- a/src/lib/memory.ts +++ b/src/lib/memory.ts @@ -12,12 +12,19 @@ const DEFAULT_TOP_K = 4; const DEFAULT_MIN_SCORE = 0.0; const DEFAULT_MAX_CHARS = 2500; const DEFAULT_UPDATE_INTERVAL_SECONDS = 120; +const DEFAULT_PRECHECK_TIMEOUT_MS = 800; +const DEFAULT_TEXT_SEARCH_TIMEOUT_MS = 3000; +const DEFAULT_VECTOR_SEARCH_TIMEOUT_MS = 10000; let qmdChecked = false; let qmdAvailable = false; let qmdUnavailableLogged = false; let qmdCommandPath: string | null = null; let qmdCheckKey = ''; +let qmdDisableExpansionCheckKey = ''; +let qmdDisableExpansionSupported = false; +let qmdUnsafeFallbackLogged = false; + const collectionPrepared = new Set(); const lastCollectionUpdateMs = new Map(); const MEMORY_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); @@ -36,6 +43,13 @@ interface QmdConfig { maxChars: number; updateIntervalSeconds: number; useSemanticSearch: boolean; + disableQueryExpansion: boolean; + allowUnsafeVsearch: boolean; + quickPrecheckEnabled: boolean; + precheckTimeoutMs: number; + searchTimeoutMs: number; + vectorSearchTimeoutMs: number; + debugLogging: boolean; } interface CommandResult { @@ -43,6 +57,16 @@ interface CommandResult { stderr: string; } +interface QueryResult { + results: QmdResult[]; + query: string; +} + +interface TurnSections { + user: string; + assistant: string; +} + function getQmdConfig(settings: Settings): QmdConfig { const memoryCfg = settings.memory?.qmd; const command = typeof memoryCfg?.command === 'string' ? memoryCfg.command.trim() : ''; @@ -56,9 +80,29 @@ function getQmdConfig(settings: Settings): QmdConfig { ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) : DEFAULT_UPDATE_INTERVAL_SECONDS, useSemanticSearch: memoryCfg?.use_semantic_search === true, + disableQueryExpansion: memoryCfg?.disable_query_expansion !== false, + allowUnsafeVsearch: memoryCfg?.allow_unsafe_vsearch === true, + quickPrecheckEnabled: memoryCfg?.quick_precheck_enabled !== false, + precheckTimeoutMs: Number.isFinite(memoryCfg?.precheck_timeout_ms) + ? Math.max(100, Number(memoryCfg?.precheck_timeout_ms)) + : DEFAULT_PRECHECK_TIMEOUT_MS, + searchTimeoutMs: Number.isFinite(memoryCfg?.search_timeout_ms) + ? Math.max(500, Number(memoryCfg?.search_timeout_ms)) + : DEFAULT_TEXT_SEARCH_TIMEOUT_MS, + vectorSearchTimeoutMs: Number.isFinite(memoryCfg?.vector_search_timeout_ms) + ? Math.max(1000, Number(memoryCfg?.vector_search_timeout_ms)) + : DEFAULT_VECTOR_SEARCH_TIMEOUT_MS, + debugLogging: memoryCfg?.debug_logging === true, }; } +function logQmdDebug(agentId: string, qmdCfg: QmdConfig, stage: string, details: string): void { + if (!qmdCfg.debugLogging) { + return; + } + log('INFO', `Memory debug @${agentId} [${stage}]: ${details}`); +} + function sanitizeId(raw: string): string { return raw.toLowerCase().replace(/[^a-z0-9_-]/g, '-'); } @@ -77,11 +121,18 @@ function ensureDir(dir: string): void { } } -function runCommand(command: string, args: string[], cwd?: string, timeoutMs = 12000): Promise { +function runCommand( + command: string, + args: string[], + cwd?: string, + timeoutMs = 12000, + env?: Record +): Promise { return new Promise((resolve, reject) => { const child = spawn(command, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'], + env: env ? { ...process.env, ...env } : process.env, }); let stdout = ''; @@ -121,6 +172,7 @@ async function isQmdAvailable(preferredCommand?: string): Promise { if (qmdChecked && qmdCheckKey === key) { return qmdAvailable; } + qmdChecked = true; qmdCheckKey = key; qmdAvailable = false; @@ -145,9 +197,40 @@ async function isQmdAvailable(preferredCommand?: string): Promise { qmdCommandPath = null; } } + return qmdAvailable; } +function isDisableExpansionPatchedQmd(commandPath: string | null): boolean { + if (!commandPath) { + return false; + } + + const bunGlobalQmd = path.join(require('os').homedir(), '.bun/bin/qmd'); + const patchedStoreTs = path.join(require('os').homedir(), '.bun/install/global/node_modules/qmd/src/store.ts'); + if (commandPath !== bunGlobalQmd || !fs.existsSync(patchedStoreTs)) { + return false; + } + + try { + const src = fs.readFileSync(patchedStoreTs, 'utf8'); + return src.includes('QMD_VSEARCH_DISABLE_EXPANSION'); + } catch { + return false; + } +} + +function isDisableExpansionSupported(): boolean { + const key = qmdCommandPath || '__unknown__'; + if (qmdDisableExpansionCheckKey === key) { + return qmdDisableExpansionSupported; + } + + qmdDisableExpansionCheckKey = key; + qmdDisableExpansionSupported = isDisableExpansionPatchedQmd(qmdCommandPath); + return qmdDisableExpansionSupported; +} + function shouldUseMemoryForChannel(channel: string): boolean { return MEMORY_CHANNELS.has(channel); } @@ -185,6 +268,41 @@ async function maybeUpdateCollection(collectionName: string, updateIntervalSecon lastCollectionUpdateMs.set(collectionName, now); } +function buildLexicalQueryVariants(message: string): string[] { + const variants: string[] = []; + const push = (value: string) => { + const cleaned = value.trim().replace(/\s+/g, ' '); + if (!cleaned) { + return; + } + if (!variants.includes(cleaned)) { + variants.push(cleaned); + } + }; + + push(message); + + const noPunct = message.replace(/[?？!！,，.。;；:：]/g, ' '); + push(noPunct); + + // Chinese question-particle normalization to reduce BM25 false negatives. + const zhSimplified = noPunct + .replace(/是什么|是啥|什么|多少|几点|哪里|哪儿|哪个|哪位|谁|吗|呢|来着/g, ' ') + .replace(/\s+/g, ' '); + push(zhSimplified); + + // English question-word normalization. + const enSimplified = noPunct + .replace(/\b(what|which|who|where|when|why|how)\b/gi, ' ') + .replace(/\s+/g, ' '); + push(enSimplified); + + // Code-friendly variant: treat hyphen as delimiter. + push(noPunct.replace(/-/g, ' ')); + + return variants; +} + function parseQmdResults(raw: string): QmdResult[] { const trimmed = raw.trim(); if (!trimmed) { @@ -219,6 +337,102 @@ function parseQmdResults(raw: string): QmdResult[] { return results; } +function normalizeInline(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +function truncateInline(text: string, max: number): string { + if (text.length <= max) { + return text; + } + return `${text.slice(0, max)}...`; +} + +function parseTurnSections(content: string): TurnSections { + const userMarker = '\n## User\n'; + const assistantMarker = '\n## Assistant\n'; + const userPos = content.indexOf(userMarker); + const assistantPos = content.indexOf(assistantMarker); + if (userPos < 0 || assistantPos < 0 || assistantPos <= userPos) { + return { user: '', assistant: '' }; + } + const userStart = userPos + userMarker.length; + const userText = content.slice(userStart, assistantPos).trim(); + const assistantStart = assistantPos + assistantMarker.length; + const assistantText = content.slice(assistantStart).trim(); + return { user: userText, assistant: assistantText }; +} + +function loadTurnSectionsFromSource(source: string, agentId: string): TurnSections | null { + const m = source.match(/^qmd:\/\/[^/]+\/(.+)$/); + if (!m) { + return null; + } + const rel = decodeURIComponent(m[1]); + const fullPath = path.join(getAgentTurnsDir(agentId), rel); + if (!fs.existsSync(fullPath)) { + return null; + } + try { + const content = fs.readFileSync(fullPath, 'utf8'); + return parseTurnSections(content); + } catch { + return null; + } +} + +function isLowConfidenceAnswer(text: string): boolean { + return /不知道|没有.*信息|无法|不清楚|need more context|don't have any information|i don't have|not enough information/i.test(text); +} + +function rerankAndHydrateResults(results: QmdResult[], message: string, agentId: string): QmdResult[] { + if (results.length === 0) { + return results; + } + const terms = Array.from(new Set((message.toLowerCase().match(/[a-z0-9_-]{2,}|[\u4e00-\u9fff]{1,3}/g) || []))); + const codePattern = /\b[A-Z]{3,}(?:-[A-Z0-9]+){2,}\b/; + + return results + .map((result) => { + let score = result.score; + let snippet = result.snippet; + + const sections = loadTurnSectionsFromSource(result.source, agentId); + if (sections && sections.assistant) { + const user = normalizeInline(sections.user); + const assistant = normalizeInline(sections.assistant); + if (assistant) { + snippet = `User: ${truncateInline(user, 180)}\nAssistant: ${truncateInline(assistant, 260)}`; + } + + if (codePattern.test(assistant)) score += 0.5; + if (/代号|key|code|是|喜欢|likes?/i.test(assistant)) score += 0.2; + if (isLowConfidenceAnswer(assistant)) score -= 0.5; + + const hay = `${user} ${assistant}`.toLowerCase(); + for (const t of terms) { + if (hay.includes(t)) score += 0.04; + } + } + + return { score, snippet, source: result.source }; + }) + .sort((a, b) => b.score - a.score); +} + +async function quickHasLexicalHit(message: string, collectionName: string, qmdCfg: QmdConfig): Promise { + const variants = buildLexicalQueryVariants(message); + for (const query of variants) { + const args = ['search', query, '--json', '-c', collectionName, '-n', '1', '--min-score', String(qmdCfg.minScore)]; + const { stdout } = await runCommand(qmdCommandPath || 'qmd', args, undefined, qmdCfg.precheckTimeoutMs); + const hits = parseQmdResults(stdout); + if (hits.length > 0) { + return true; + } + } + return false; +} + function formatMemoryPrompt(results: QmdResult[], maxChars: number): string { if (results.length === 0) { return ''; @@ -256,6 +470,49 @@ function formatMemoryPrompt(results: QmdResult[], maxChars: number): string { ].join('\n'); } +function resolveRetrievalMode(qmdCfg: QmdConfig): { useVsearch: boolean; label: 'qmd-bm25' | 'qmd-vsearch' } { + let useVsearch = qmdCfg.useSemanticSearch; + if (useVsearch && !qmdCfg.allowUnsafeVsearch) { + if (!qmdCfg.disableQueryExpansion) { + useVsearch = false; + if (!qmdUnsafeFallbackLogged) { + log('WARN', 'QMD vsearch requested without disable_query_expansion; fallback to BM25 for safety. Set memory.qmd.allow_unsafe_vsearch=true to override.'); + qmdUnsafeFallbackLogged = true; + } + } else if (!isDisableExpansionSupported()) { + useVsearch = false; + if (!qmdUnsafeFallbackLogged) { + log('WARN', 'QMD disable-query-expansion support not detected; fallback to BM25 to avoid unexpected model downloads. Run scripts/patch-qmd-no-expansion.sh or set memory.qmd.allow_unsafe_vsearch=true.'); + qmdUnsafeFallbackLogged = true; + } + } + } + + return { + useVsearch, + label: useVsearch ? 'qmd-vsearch' : 'qmd-bm25', + }; +} + +async function runBm25WithVariants( + message: string, + collectionName: string, + qmdCfg: QmdConfig +): Promise { + const variants = buildLexicalQueryVariants(message); + let lastQuery = message; + for (const query of variants) { + lastQuery = query; + const args = ['search', query, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; + const { stdout } = await runCommand(qmdCommandPath || 'qmd', args, undefined, qmdCfg.searchTimeoutMs); + const results = parseQmdResults(stdout); + if (results.length > 0) { + return { results, query }; + } + } + return { results: [], query: lastQuery }; +} + export async function enrichMessageWithMemory( agentId: string, message: string, @@ -276,32 +533,79 @@ export async function enrichMessageWithMemory( log('WARN', 'qmd not found in PATH, memory retrieval disabled'); qmdUnavailableLogged = true; } + log('INFO', `Memory source for @${agentId}: none (qmd unavailable)`); return message; } + logQmdDebug(agentId, qmdCfg, 'qmd', `command=${qmdCommandPath || 'qmd'}`); try { const collectionName = await ensureCollection(agentId); + logQmdDebug(agentId, qmdCfg, 'collection', `name=${collectionName}`); await maybeUpdateCollection(collectionName, qmdCfg.updateIntervalSeconds); + logQmdDebug(agentId, qmdCfg, 'update', `interval=${qmdCfg.updateIntervalSeconds}s`); + + if (qmdCfg.quickPrecheckEnabled) { + try { + logQmdDebug( + agentId, + qmdCfg, + 'precheck', + `cmd=search timeout=${qmdCfg.precheckTimeoutMs}ms min_score=${qmdCfg.minScore} variants=${buildLexicalQueryVariants(message).length}` + ); + const hasQuickHit = await quickHasLexicalHit(message, collectionName, qmdCfg); + if (!hasQuickHit) { + log('INFO', `Memory source for @${agentId}: none (qmd precheck no-hit)`); + return message; + } + } catch (error) { + log('WARN', `Memory quick precheck skipped for @${agentId}: ${(error as Error).message}`); + log('INFO', `Memory source for @${agentId}: none (qmd precheck error)`); + return message; + } + } - const queryArgs = qmdCfg.useSemanticSearch + const mode = resolveRetrievalMode(qmdCfg); + const queryArgs = mode.useVsearch ? ['vsearch', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)] : ['search', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; - - const { stdout } = await runCommand(qmdCommandPath || 'qmd', queryArgs, undefined, 12000); - const results = parseQmdResults(stdout); + const queryEnv = mode.useVsearch && qmdCfg.disableQueryExpansion + ? { QMD_VSEARCH_DISABLE_EXPANSION: '1' } + : undefined; + const queryTimeoutMs = mode.useVsearch ? qmdCfg.vectorSearchTimeoutMs : qmdCfg.searchTimeoutMs; + logQmdDebug( + agentId, + qmdCfg, + 'query', + `mode=${mode.label} timeout=${queryTimeoutMs}ms top_k=${qmdCfg.topK} min_score=${qmdCfg.minScore} disable_expansion=${qmdCfg.disableQueryExpansion}` + ); + + const queryResult = mode.useVsearch + ? (() => runCommand(qmdCommandPath || 'qmd', queryArgs, undefined, queryTimeoutMs, queryEnv).then(({ stdout }) => ({ + results: parseQmdResults(stdout), + query: message, + })))() + : runBm25WithVariants(message, collectionName, qmdCfg); + const { results, query } = await queryResult; + logQmdDebug(agentId, qmdCfg, 'query-used', `mode=${mode.label} query=\"${query}\"`); if (results.length === 0) { + log('INFO', `Memory source for @${agentId}: none (${mode.label} no-hit)`); return message; } - const memoryBlock = formatMemoryPrompt(results, qmdCfg.maxChars); + const rankedResults = rerankAndHydrateResults(results, message, agentId); + + const memoryBlock = formatMemoryPrompt(rankedResults, qmdCfg.maxChars); if (!memoryBlock) { + log('INFO', `Memory source for @${agentId}: none (${mode.label} no-usable-snippet)`); return message; } - log('INFO', `Memory retrieval hit for @${agentId}: ${results.length} snippet(s)`); + log('INFO', `Memory retrieval hit for @${agentId}: ${rankedResults.length} snippet(s) via ${mode.label}`); + log('INFO', `Memory source for @${agentId}: ${mode.label}`); return `${message}${memoryBlock}`; } catch (error) { log('WARN', `Memory retrieval skipped for @${agentId}: ${(error as Error).message}`); + log('INFO', `Memory source for @${agentId}: none (qmd error)`); return message; } } diff --git a/src/lib/types.ts b/src/lib/types.ts index f167377..abef2f4 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -51,6 +51,13 @@ export interface Settings { max_chars?: number; update_interval_seconds?: number; use_semantic_search?: boolean; + disable_query_expansion?: boolean; + allow_unsafe_vsearch?: boolean; + quick_precheck_enabled?: boolean; + precheck_timeout_ms?: number; + search_timeout_ms?: number; + vector_search_timeout_ms?: number; + debug_logging?: boolean; }; }; } diff --git a/tinyclaw.sh b/tinyclaw.sh index 895cf43..738941b 100755 --- a/tinyclaw.sh +++ b/tinyclaw.sh @@ -21,6 +21,7 @@ if [ -f "$SCRIPT_DIR/.tinyclaw/settings.json" ]; then else SETTINGS_FILE="$HOME/.tinyclaw/settings.json" fi +RESET_FLAG_FILE="$(dirname "$SETTINGS_FILE")/reset_flag" mkdir -p "$LOG_DIR" @@ -63,7 +64,7 @@ case "${1:-}" in ;; reset) echo -e "${YELLOW}Resetting conversation...${NC}" - touch "$SCRIPT_DIR/.tinyclaw/reset_flag" + touch "$RESET_FLAG_FILE" echo -e "${GREEN}✓ Reset flag set${NC}" echo "" echo "The next message will start a fresh conversation (without -c)." From 4ae6ff4652d95a438e994b95a9641590f8cba67f Mon Sep 17 00:00:00 2001 From: MZ Date: Tue, 17 Feb 2026 23:34:27 -0700 Subject: [PATCH 4/6] feat(memory): schedule qmd embed when semantic search is enabled --- README.md | 2 ++ lib/setup-wizard.sh | 2 +- src/lib/memory.ts | 26 ++++++++++++++++++++++++++ src/lib/types.ts | 1 + 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d80b890..550752d 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,7 @@ Located at `.tinyclaw/settings.json`: "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 120, + "embed_interval_seconds": 120, "use_semantic_search": false, "disable_query_expansion": true, "allow_unsafe_vsearch": false, @@ -433,6 +434,7 @@ Safety note: - If `use_semantic_search` is enabled, TinyClaw defaults to safe mode and will fall back to BM25 unless disable-expansion support is detected. - Set `memory.qmd.allow_unsafe_vsearch: true` only if you explicitly want to allow unguarded `vsearch`. - Set `memory.qmd.debug_logging: true` to print QMD memory debug logs (command/mode/timeout) in `queue.log`. +- When semantic search is enabled, TinyClaw periodically runs `qmd embed` (interval controlled by `memory.qmd.embed_interval_seconds`). ### Heartbeat Configuration diff --git a/lib/setup-wizard.sh b/lib/setup-wizard.sh index 0200832..55e64c1 100755 --- a/lib/setup-wizard.sh +++ b/lib/setup-wizard.sh @@ -308,7 +308,7 @@ else fi if [ "$MEMORY_ENABLED" = true ]; then - MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' + MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "embed_interval_seconds": 300, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' else MEMORY_SECTION='"memory": { "enabled": false },' fi diff --git a/src/lib/memory.ts b/src/lib/memory.ts index 37f0502..365858e 100644 --- a/src/lib/memory.ts +++ b/src/lib/memory.ts @@ -27,6 +27,7 @@ let qmdUnsafeFallbackLogged = false; const collectionPrepared = new Set(); const lastCollectionUpdateMs = new Map(); +const lastCollectionEmbedMs = new Map(); const MEMORY_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); interface QmdResult { @@ -42,6 +43,7 @@ interface QmdConfig { minScore: number; maxChars: number; updateIntervalSeconds: number; + embedIntervalSeconds: number; useSemanticSearch: boolean; disableQueryExpansion: boolean; allowUnsafeVsearch: boolean; @@ -79,6 +81,11 @@ function getQmdConfig(settings: Settings): QmdConfig { updateIntervalSeconds: Number.isFinite(memoryCfg?.update_interval_seconds) ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) : DEFAULT_UPDATE_INTERVAL_SECONDS, + embedIntervalSeconds: Number.isFinite(memoryCfg?.embed_interval_seconds) + ? Math.max(10, Number(memoryCfg?.embed_interval_seconds)) + : (Number.isFinite(memoryCfg?.update_interval_seconds) + ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) + : DEFAULT_UPDATE_INTERVAL_SECONDS), useSemanticSearch: memoryCfg?.use_semantic_search === true, disableQueryExpansion: memoryCfg?.disable_query_expansion !== false, allowUnsafeVsearch: memoryCfg?.allow_unsafe_vsearch === true, @@ -268,6 +275,16 @@ async function maybeUpdateCollection(collectionName: string, updateIntervalSecon lastCollectionUpdateMs.set(collectionName, now); } +async function maybeEmbedCollection(collectionName: string, embedIntervalSeconds: number): Promise { + const now = Date.now(); + const last = lastCollectionEmbedMs.get(collectionName) || 0; + if (now - last < embedIntervalSeconds * 1000) { + return; + } + await runCommand(qmdCommandPath || 'qmd', ['embed', '--collections', collectionName], undefined, 30000); + lastCollectionEmbedMs.set(collectionName, now); +} + function buildLexicalQueryVariants(message: string): string[] { const variants: string[] = []; const push = (value: string) => { @@ -565,6 +582,15 @@ export async function enrichMessageWithMemory( } const mode = resolveRetrievalMode(qmdCfg); + if (mode.useVsearch) { + try { + await maybeEmbedCollection(collectionName, qmdCfg.embedIntervalSeconds); + logQmdDebug(agentId, qmdCfg, 'embed', `interval=${qmdCfg.embedIntervalSeconds}s`); + } catch (error) { + log('WARN', `Memory embed skipped for @${agentId}: ${(error as Error).message}`); + logQmdDebug(agentId, qmdCfg, 'embed', 'failed; continuing with existing vectors'); + } + } const queryArgs = mode.useVsearch ? ['vsearch', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)] : ['search', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; diff --git a/src/lib/types.ts b/src/lib/types.ts index abef2f4..4256528 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -50,6 +50,7 @@ export interface Settings { min_score?: number; max_chars?: number; update_interval_seconds?: number; + embed_interval_seconds?: number; use_semantic_search?: boolean; disable_query_expansion?: boolean; allow_unsafe_vsearch?: boolean; From ed15b79e47a5d4485c5a2f2a6a0597accd14c03e Mon Sep 17 00:00:00 2001 From: MZ Date: Wed, 18 Feb 2026 17:42:13 -0700 Subject: [PATCH 5/6] feat(memory): inject claude memory via .claude/MEMORY.md --- src/lib/invoke.ts | 49 ++++++++++++++++++++++++++++++++++++++++++----- src/lib/memory.ts | 30 +++++++++++++++++++---------- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/src/lib/invoke.ts b/src/lib/invoke.ts index b25370f..a657478 100644 --- a/src/lib/invoke.ts +++ b/src/lib/invoke.ts @@ -5,7 +5,34 @@ import { AgentConfig, Settings, TeamConfig } from './types'; import { SCRIPT_DIR, resolveClaudeModel, resolveCodexModel } from './config'; import { log } from './logging'; import { ensureAgentDirectory, updateAgentTeammates } from './agent-setup'; -import { enrichMessageWithMemory } from './memory'; +import { buildMemoryBlock } from './memory'; + +const CLAUDE_MEMORY_FILENAME = 'MEMORY.md'; + +function getClaudeMemoryFilePath(workingDir: string): string { + return path.join(workingDir, '.claude', CLAUDE_MEMORY_FILENAME); +} + +function deleteClaudeMemoryFile(memoryFilePath: string): void { + if (fs.existsSync(memoryFilePath)) { + fs.unlinkSync(memoryFilePath); + } +} + +function writeClaudeMemoryFile(memoryFilePath: string, memoryBlock: string): void { + const claudeDir = path.dirname(memoryFilePath); + fs.mkdirSync(claudeDir, { recursive: true }); + const body = memoryBlock.trim(); + const content = [ + '# Runtime Memory Context', + '', + 'Auto-generated for the current invocation. Do not persist manually.', + '', + body, + '', + ].join('\n'); + fs.writeFileSync(memoryFilePath, content, 'utf8'); +} export async function runCommand(command: string, args: string[], cwd?: string): Promise { return new Promise((resolve, reject) => { @@ -77,10 +104,11 @@ export async function invokeAgent( : path.join(workspacePath, agent.working_directory)) : agentDir; - const messageForModel = await enrichMessageWithMemory(agentId, message, settings, sourceChannel); + const memoryBlock = await buildMemoryBlock(agentId, message, settings, sourceChannel); const provider = agent.provider || 'anthropic'; if (provider === 'openai') { + const messageForModel = memoryBlock ? `${message}${memoryBlock}` : message; log('INFO', `Using Codex CLI (agent: ${agentId})`); const shouldResume = !shouldReset; @@ -134,8 +162,19 @@ export async function invokeAgent( if (continueConversation) { claudeArgs.push('-c'); } - claudeArgs.push('-p', messageForModel); - - return await runCommand('claude', claudeArgs, workingDir); + claudeArgs.push('-p', message); + + const memoryFilePath = getClaudeMemoryFilePath(workingDir); + // Defensive cleanup in case previous invocation crashed before deleting. + deleteClaudeMemoryFile(memoryFilePath); + try { + if (memoryBlock) { + writeClaudeMemoryFile(memoryFilePath, memoryBlock); + log('INFO', `Memory source injection for @${agentId}: .claude/${CLAUDE_MEMORY_FILENAME}`); + } + return await runCommand('claude', claudeArgs, workingDir); + } finally { + deleteClaudeMemoryFile(memoryFilePath); + } } } diff --git a/src/lib/memory.ts b/src/lib/memory.ts index 365858e..bca2f4f 100644 --- a/src/lib/memory.ts +++ b/src/lib/memory.ts @@ -530,7 +530,7 @@ async function runBm25WithVariants( return { results: [], query: lastQuery }; } -export async function enrichMessageWithMemory( +export async function buildMemoryBlock( agentId: string, message: string, settings: Settings, @@ -538,10 +538,10 @@ export async function enrichMessageWithMemory( ): Promise { const qmdCfg = getQmdConfig(settings); if (!qmdCfg.enabled) { - return message; + return ''; } if (!shouldUseMemoryForChannel(sourceChannel)) { - return message; + return ''; } const hasQmd = await isQmdAvailable(qmdCfg.command); @@ -551,7 +551,7 @@ export async function enrichMessageWithMemory( qmdUnavailableLogged = true; } log('INFO', `Memory source for @${agentId}: none (qmd unavailable)`); - return message; + return ''; } logQmdDebug(agentId, qmdCfg, 'qmd', `command=${qmdCommandPath || 'qmd'}`); @@ -572,12 +572,12 @@ export async function enrichMessageWithMemory( const hasQuickHit = await quickHasLexicalHit(message, collectionName, qmdCfg); if (!hasQuickHit) { log('INFO', `Memory source for @${agentId}: none (qmd precheck no-hit)`); - return message; + return ''; } } catch (error) { log('WARN', `Memory quick precheck skipped for @${agentId}: ${(error as Error).message}`); log('INFO', `Memory source for @${agentId}: none (qmd precheck error)`); - return message; + return ''; } } @@ -615,7 +615,7 @@ export async function enrichMessageWithMemory( logQmdDebug(agentId, qmdCfg, 'query-used', `mode=${mode.label} query=\"${query}\"`); if (results.length === 0) { log('INFO', `Memory source for @${agentId}: none (${mode.label} no-hit)`); - return message; + return ''; } const rankedResults = rerankAndHydrateResults(results, message, agentId); @@ -623,19 +623,29 @@ export async function enrichMessageWithMemory( const memoryBlock = formatMemoryPrompt(rankedResults, qmdCfg.maxChars); if (!memoryBlock) { log('INFO', `Memory source for @${agentId}: none (${mode.label} no-usable-snippet)`); - return message; + return ''; } log('INFO', `Memory retrieval hit for @${agentId}: ${rankedResults.length} snippet(s) via ${mode.label}`); log('INFO', `Memory source for @${agentId}: ${mode.label}`); - return `${message}${memoryBlock}`; + return memoryBlock; } catch (error) { log('WARN', `Memory retrieval skipped for @${agentId}: ${(error as Error).message}`); log('INFO', `Memory source for @${agentId}: none (qmd error)`); - return message; + return ''; } } +export async function enrichMessageWithMemory( + agentId: string, + message: string, + settings: Settings, + sourceChannel: string +): Promise { + const memoryBlock = await buildMemoryBlock(agentId, message, settings, sourceChannel); + return memoryBlock ? `${message}${memoryBlock}` : message; +} + function timestampFilename(ts: number): string { return new Date(ts).toISOString().replace(/[:.]/g, '-'); } From 6b53fd28d36a5cc1500d127f2e5dcaf0d787fc33 Mon Sep 17 00:00:00 2001 From: MZ Date: Wed, 18 Feb 2026 21:27:19 -0700 Subject: [PATCH 6/6] feat(memory): harden qmd retrieval and claude memory injection --- README.md | 18 ++- lib/setup-wizard.sh | 4 +- src/lib/invoke.ts | 92 ++++++++--- src/lib/memory-rerank.ts | 102 ++++++++++++ src/lib/memory.ts | 328 ++++++++++++++++++++++++++++++++------- src/lib/types.ts | 15 ++ src/queue-processor.ts | 9 +- 7 files changed, 484 insertions(+), 84 deletions(-) create mode 100644 src/lib/memory-rerank.ts diff --git a/README.md b/README.md index 550752d..bc586c6 100644 --- a/README.md +++ b/README.md @@ -388,6 +388,15 @@ Located at `.tinyclaw/settings.json`: }, "memory": { "enabled": true, + "rerank": { + "enabled": true + }, + "retention": { + "enabled": true, + "retain_days": 30, + "max_turn_files_per_agent": 2000, + "cleanup_interval_seconds": 300 + }, "qmd": { "enabled": true, "command": "/home/me/.bun/bin/qmd", @@ -395,7 +404,7 @@ Located at `.tinyclaw/settings.json`: "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 120, - "embed_interval_seconds": 120, + "embed_interval_seconds": 600, "use_semantic_search": false, "disable_query_expansion": true, "allow_unsafe_vsearch": false, @@ -413,6 +422,8 @@ TinyClaw can use external memory retrieval with `qmd` (BM25 by default). - Retrieval runs only for user chat channels (`telegram`, `discord`, `whatsapp`). - Heartbeat/system messages are excluded from retrieval and memory persistence. - Turns are stored in `~/.tinyclaw/memory/turns//`. +- Turn files are cleaned automatically (default: keep 30 days and max 2000 files per agent). +- Retrieval reranking heuristics are configurable via `memory.rerank` (can be disabled). Quick setup: @@ -431,10 +442,11 @@ tinyclaw restart ``` Safety note: -- If `use_semantic_search` is enabled, TinyClaw defaults to safe mode and will fall back to BM25 unless disable-expansion support is detected. +- `use_semantic_search` (`qmd-vsearch`) is currently **experimental**. +- If semantic search is enabled, TinyClaw defaults to safe mode and will fall back to BM25 unless disable-expansion support is detected. - Set `memory.qmd.allow_unsafe_vsearch: true` only if you explicitly want to allow unguarded `vsearch`. - Set `memory.qmd.debug_logging: true` to print QMD memory debug logs (command/mode/timeout) in `queue.log`. -- When semantic search is enabled, TinyClaw periodically runs `qmd embed` (interval controlled by `memory.qmd.embed_interval_seconds`). +- When semantic search is enabled, TinyClaw periodically runs `qmd embed` in background (non-blocking). Default `embed_interval_seconds` is `600`. ### Heartbeat Configuration diff --git a/lib/setup-wizard.sh b/lib/setup-wizard.sh index 55e64c1..6c1a272 100755 --- a/lib/setup-wizard.sh +++ b/lib/setup-wizard.sh @@ -200,7 +200,7 @@ if [[ "$ENABLE_MEMORY" =~ ^[yY] ]]; then echo -e "${YELLOW} You can install later with: bun install -g github:tobi/qmd${NC}" fi - read -rp "Use semantic search (vector)? [y/N]: " USE_SEMANTIC + read -rp "Use semantic search (vector, experimental)? [y/N]: " USE_SEMANTIC if [[ "$USE_SEMANTIC" =~ ^[yY] ]]; then USE_SEMANTIC_SEARCH=true fi @@ -308,7 +308,7 @@ else fi if [ "$MEMORY_ENABLED" = true ]; then - MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "embed_interval_seconds": 300, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' + MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "embed_interval_seconds": 600, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' else MEMORY_SECTION='"memory": { "enabled": false },' fi diff --git a/src/lib/invoke.ts b/src/lib/invoke.ts index a657478..fca0347 100644 --- a/src/lib/invoke.ts +++ b/src/lib/invoke.ts @@ -7,31 +7,78 @@ import { log } from './logging'; import { ensureAgentDirectory, updateAgentTeammates } from './agent-setup'; import { buildMemoryBlock } from './memory'; -const CLAUDE_MEMORY_FILENAME = 'MEMORY.md'; +const CLAUDE_SYSTEM_FILENAME = 'CLAUDE.md'; -function getClaudeMemoryFilePath(workingDir: string): string { - return path.join(workingDir, '.claude', CLAUDE_MEMORY_FILENAME); +function getClaudeSystemFilePath(workingDir: string): string { + return path.join(workingDir, '.claude', CLAUDE_SYSTEM_FILENAME); } -function deleteClaudeMemoryFile(memoryFilePath: string): void { - if (fs.existsSync(memoryFilePath)) { - fs.unlinkSync(memoryFilePath); - } -} - -function writeClaudeMemoryFile(memoryFilePath: string, memoryBlock: string): void { - const claudeDir = path.dirname(memoryFilePath); +function injectClaudeMemory(systemFilePath: string, memoryBlock: string, agentId: string): () => void { + const claudeDir = path.dirname(systemFilePath); fs.mkdirSync(claudeDir, { recursive: true }); + + const existed = fs.existsSync(systemFilePath); + const original = existed ? fs.readFileSync(systemFilePath, 'utf8') : ''; + const markerId = `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const startMarker = ``; + const endMarker = ``; const body = memoryBlock.trim(); - const content = [ - '# Runtime Memory Context', + const runtimeSection = [ + startMarker, + '## Runtime Memory Context', '', - 'Auto-generated for the current invocation. Do not persist manually.', + 'Auto-generated for current invocation only.', '', body, '', + endMarker, ].join('\n'); - fs.writeFileSync(memoryFilePath, content, 'utf8'); + const merged = original.trim().length > 0 + ? `${original.trimEnd()}\n\n${runtimeSection}` + : runtimeSection; + + fs.writeFileSync(systemFilePath, merged, 'utf8'); + + return () => { + if (!fs.existsSync(systemFilePath)) { + if (existed) { + log('WARN', `Memory cleanup marker missing for @${agentId}: .claude/${CLAUDE_SYSTEM_FILENAME} disappeared during invoke`); + } + return; + } + + const current = fs.readFileSync(systemFilePath, 'utf8'); + const startIdx = current.indexOf(startMarker); + const endIdx = current.indexOf(endMarker, startIdx >= 0 ? startIdx : 0); + if (startIdx < 0 || endIdx < 0) { + log('WARN', `Memory cleanup marker missing for @${agentId}: skipped restore to avoid overwriting .claude/${CLAUDE_SYSTEM_FILENAME}`); + return; + } + + const endExclusive = endIdx + endMarker.length; + const stripped = `${current.slice(0, startIdx)}${current.slice(endExclusive)}`; + const normalized = stripped + .replace(/\n{3,}/g, '\n\n') + .replace(/^\s+|\s+$/g, ''); + + if (!normalized) { + if (existed) { + fs.writeFileSync(systemFilePath, original, 'utf8'); + } else { + fs.unlinkSync(systemFilePath); + } + return; + } + + fs.writeFileSync(systemFilePath, `${normalized}\n`, 'utf8'); + }; +} + +function deleteLegacyClaudeMemoryFile(workingDir: string): void { + const legacyPath = path.join(workingDir, '.claude', 'MEMORY.md'); + if (fs.existsSync(legacyPath)) { + fs.unlinkSync(legacyPath); + } } export async function runCommand(command: string, args: string[], cwd?: string): Promise { @@ -164,17 +211,20 @@ export async function invokeAgent( } claudeArgs.push('-p', message); - const memoryFilePath = getClaudeMemoryFilePath(workingDir); - // Defensive cleanup in case previous invocation crashed before deleting. - deleteClaudeMemoryFile(memoryFilePath); + const systemFilePath = getClaudeSystemFilePath(workingDir); + let restoreSystemFile: (() => void) | null = null; + // Defensive cleanup from old MEMORY.md injection behavior. + deleteLegacyClaudeMemoryFile(workingDir); try { if (memoryBlock) { - writeClaudeMemoryFile(memoryFilePath, memoryBlock); - log('INFO', `Memory source injection for @${agentId}: .claude/${CLAUDE_MEMORY_FILENAME}`); + restoreSystemFile = injectClaudeMemory(systemFilePath, memoryBlock, agentId); + log('INFO', `Memory source injection for @${agentId}: .claude/${CLAUDE_SYSTEM_FILENAME} (runtime section)`); } return await runCommand('claude', claudeArgs, workingDir); } finally { - deleteClaudeMemoryFile(memoryFilePath); + if (restoreSystemFile) { + restoreSystemFile(); + } } } } diff --git a/src/lib/memory-rerank.ts b/src/lib/memory-rerank.ts new file mode 100644 index 0000000..595ed16 --- /dev/null +++ b/src/lib/memory-rerank.ts @@ -0,0 +1,102 @@ +import { Settings } from './types'; + +const DEFAULT_ANSWER_SIGNAL_PATTERNS = ['代号', 'key', 'code', '是', '喜欢', 'likes?']; +const DEFAULT_LOW_CONFIDENCE_PATTERNS = [ + '不知道', + '没有.*信息', + '无法', + '不清楚', + '没有足够.*上下文', + '缺少.*上下文', + 'need more context', + 'enough context', + "don't have enough context", + "don't have any information", + "i don't have", + 'not enough information', +]; + +const DEFAULT_CODE_PATTERN = /\b[A-Z]{3,}(?:-[A-Z0-9]+){2,}\b/; + +export interface MemoryRerankOptions { + enabled: boolean; + answerSignalPatterns: string[]; + lowConfidencePatterns: string[]; + answerSignalBonus: number; + lowConfidencePenalty: number; + codePatternBonus: number; + termHitBonus: number; +} + +export function resolveMemoryRerankOptions(settings?: Settings): MemoryRerankOptions { + const cfg = settings?.memory?.rerank; + return { + enabled: cfg?.enabled !== false, + answerSignalPatterns: Array.isArray(cfg?.answer_signal_patterns) && cfg.answer_signal_patterns.length > 0 + ? cfg.answer_signal_patterns + : DEFAULT_ANSWER_SIGNAL_PATTERNS, + lowConfidencePatterns: Array.isArray(cfg?.low_confidence_patterns) && cfg.low_confidence_patterns.length > 0 + ? cfg.low_confidence_patterns + : DEFAULT_LOW_CONFIDENCE_PATTERNS, + answerSignalBonus: Number.isFinite(cfg?.answer_signal_bonus) ? Number(cfg?.answer_signal_bonus) : 0.2, + lowConfidencePenalty: Number.isFinite(cfg?.low_confidence_penalty) ? Number(cfg?.low_confidence_penalty) : 0.5, + codePatternBonus: Number.isFinite(cfg?.code_pattern_bonus) ? Number(cfg?.code_pattern_bonus) : 0.5, + termHitBonus: Number.isFinite(cfg?.term_hit_bonus) ? Number(cfg?.term_hit_bonus) : 0.04, + }; +} + +function buildRegex(pattern: string): RegExp | null { + try { + return new RegExp(pattern, 'i'); + } catch { + return null; + } +} + +function matchesAnyPattern(text: string, patterns: string[]): boolean { + for (const pattern of patterns) { + const regex = buildRegex(pattern); + if (!regex) { + continue; + } + if (regex.test(text)) { + return true; + } + } + return false; +} + +export function isLowConfidenceText(text: string, options: MemoryRerankOptions): boolean { + return matchesAnyPattern(text, options.lowConfidencePatterns); +} + +export function computeHeuristicScoreDelta( + user: string, + assistant: string, + messageTerms: string[], + options: MemoryRerankOptions +): number { + if (!options.enabled) { + return 0; + } + + let delta = 0; + if (DEFAULT_CODE_PATTERN.test(assistant)) { + delta += options.codePatternBonus; + } + if (matchesAnyPattern(assistant, options.answerSignalPatterns)) { + delta += options.answerSignalBonus; + } + if (matchesAnyPattern(assistant, options.lowConfidencePatterns)) { + delta -= options.lowConfidencePenalty; + } + + const hay = `${user} ${assistant}`.toLowerCase(); + for (const term of messageTerms) { + if (hay.includes(term)) { + delta += options.termHitBonus; + } + } + + return delta; +} diff --git a/src/lib/memory.ts b/src/lib/memory.ts index bca2f4f..7feebd2 100644 --- a/src/lib/memory.ts +++ b/src/lib/memory.ts @@ -4,6 +4,7 @@ import path from 'path'; import { AgentConfig, Settings } from './types'; import { TINYCLAW_HOME } from './config'; import { log } from './logging'; +import { computeHeuristicScoreDelta, isLowConfidenceText, resolveMemoryRerankOptions } from './memory-rerank'; const MEMORY_ROOT = path.join(TINYCLAW_HOME, 'memory'); const MEMORY_TURNS_DIR = path.join(MEMORY_ROOT, 'turns'); @@ -12,23 +13,32 @@ const DEFAULT_TOP_K = 4; const DEFAULT_MIN_SCORE = 0.0; const DEFAULT_MAX_CHARS = 2500; const DEFAULT_UPDATE_INTERVAL_SECONDS = 120; +const DEFAULT_EMBED_INTERVAL_SECONDS = 600; +const DEFAULT_RETAIN_DAYS = 30; +const DEFAULT_MAX_TURN_FILES_PER_AGENT = 2000; +const DEFAULT_CLEANUP_INTERVAL_SECONDS = 300; const DEFAULT_PRECHECK_TIMEOUT_MS = 800; const DEFAULT_TEXT_SEARCH_TIMEOUT_MS = 3000; const DEFAULT_VECTOR_SEARCH_TIMEOUT_MS = 10000; +const QMD_FALLBACK_WARN_INTERVAL_MS = 10 * 60 * 1000; +const VSEARCH_EXPERIMENTAL_INFO_INTERVAL_MS = 10 * 60 * 1000; let qmdChecked = false; let qmdAvailable = false; -let qmdUnavailableLogged = false; +const qmdUnavailableLoggedByAgent = new Map(); let qmdCommandPath: string | null = null; let qmdCheckKey = ''; let qmdDisableExpansionCheckKey = ''; let qmdDisableExpansionSupported = false; -let qmdUnsafeFallbackLogged = false; const collectionPrepared = new Set(); const lastCollectionUpdateMs = new Map(); const lastCollectionEmbedMs = new Map(); -const MEMORY_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); +const embedInFlightByCollection = new Set(); +const lastTurnsCleanupMs = new Map(); +const lastUnsafeFallbackWarnMsByAgent = new Map(); +const lastVsearchExperimentalInfoMsByAgent = new Map(); +export const MEMORY_ELIGIBLE_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); interface QmdResult { score: number; @@ -69,6 +79,13 @@ interface TurnSections { assistant: string; } +interface RetentionConfig { + enabled: boolean; + retainDays: number; + maxTurnFilesPerAgent: number; + cleanupIntervalSeconds: number; +} + function getQmdConfig(settings: Settings): QmdConfig { const memoryCfg = settings.memory?.qmd; const command = typeof memoryCfg?.command === 'string' ? memoryCfg.command.trim() : ''; @@ -83,9 +100,7 @@ function getQmdConfig(settings: Settings): QmdConfig { : DEFAULT_UPDATE_INTERVAL_SECONDS, embedIntervalSeconds: Number.isFinite(memoryCfg?.embed_interval_seconds) ? Math.max(10, Number(memoryCfg?.embed_interval_seconds)) - : (Number.isFinite(memoryCfg?.update_interval_seconds) - ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) - : DEFAULT_UPDATE_INTERVAL_SECONDS), + : DEFAULT_EMBED_INTERVAL_SECONDS, useSemanticSearch: memoryCfg?.use_semantic_search === true, disableQueryExpansion: memoryCfg?.disable_query_expansion !== false, allowUnsafeVsearch: memoryCfg?.allow_unsafe_vsearch === true, @@ -103,6 +118,22 @@ function getQmdConfig(settings: Settings): QmdConfig { }; } +function getRetentionConfig(settings?: Settings): RetentionConfig { + const retentionCfg = settings?.memory?.retention; + return { + enabled: retentionCfg?.enabled !== false, + retainDays: Number.isFinite(retentionCfg?.retain_days) + ? Math.max(1, Number(retentionCfg?.retain_days)) + : DEFAULT_RETAIN_DAYS, + maxTurnFilesPerAgent: Number.isFinite(retentionCfg?.max_turn_files_per_agent) + ? Math.max(100, Number(retentionCfg?.max_turn_files_per_agent)) + : DEFAULT_MAX_TURN_FILES_PER_AGENT, + cleanupIntervalSeconds: Number.isFinite(retentionCfg?.cleanup_interval_seconds) + ? Math.max(30, Number(retentionCfg?.cleanup_interval_seconds)) + : DEFAULT_CLEANUP_INTERVAL_SECONDS, + }; +} + function logQmdDebug(agentId: string, qmdCfg: QmdConfig, stage: string, details: string): void { if (!qmdCfg.debugLogging) { return; @@ -110,6 +141,42 @@ function logQmdDebug(agentId: string, qmdCfg: QmdConfig, stage: string, details: log('INFO', `Memory debug @${agentId} [${stage}]: ${details}`); } +function logThrottled( + key: string, + cache: Map, + intervalMs: number, + level: 'INFO' | 'WARN', + message: string +): void { + const now = Date.now(); + const last = cache.get(key) || 0; + if (now - last < intervalMs) { + return; + } + cache.set(key, now); + log(level, message); +} + +function warnUnsafeFallback(agentId: string, reason: string): void { + logThrottled( + agentId, + lastUnsafeFallbackWarnMsByAgent, + QMD_FALLBACK_WARN_INTERVAL_MS, + 'WARN', + `QMD vsearch fallback for @${agentId}: ${reason}` + ); +} + +function infoVsearchExperimental(agentId: string): void { + logThrottled( + agentId, + lastVsearchExperimentalInfoMsByAgent, + VSEARCH_EXPERIMENTAL_INFO_INTERVAL_MS, + 'INFO', + `QMD vsearch is experimental for @${agentId}.` + ); +} + function sanitizeId(raw: string): string { return raw.toLowerCase().replace(/[^a-z0-9_-]/g, '-'); } @@ -239,7 +306,7 @@ function isDisableExpansionSupported(): boolean { } function shouldUseMemoryForChannel(channel: string): boolean { - return MEMORY_CHANNELS.has(channel); + return MEMORY_ELIGIBLE_CHANNELS.has(channel); } async function ensureCollection(agentId: string): Promise { @@ -281,8 +348,28 @@ async function maybeEmbedCollection(collectionName: string, embedIntervalSeconds if (now - last < embedIntervalSeconds * 1000) { return; } - await runCommand(qmdCommandPath || 'qmd', ['embed', '--collections', collectionName], undefined, 30000); + // Apply backoff from the trigger time to avoid tight retries on repeated failures. lastCollectionEmbedMs.set(collectionName, now); + await runCommand(qmdCommandPath || 'qmd', ['embed', '--collections', collectionName], undefined, 30000); +} + +function triggerEmbedCollectionAsync(agentId: string, collectionName: string, qmdCfg: QmdConfig): void { + if (embedInFlightByCollection.has(collectionName)) { + logQmdDebug(agentId, qmdCfg, 'embed', 'skip trigger (in-flight)'); + return; + } + embedInFlightByCollection.add(collectionName); + void maybeEmbedCollection(collectionName, qmdCfg.embedIntervalSeconds) + .then(() => { + logQmdDebug(agentId, qmdCfg, 'embed', `triggered interval=${qmdCfg.embedIntervalSeconds}s`); + }) + .catch((error) => { + log('WARN', `Memory embed skipped for @${agentId}: ${(error as Error).message}`); + logQmdDebug(agentId, qmdCfg, 'embed', 'failed; continuing with existing vectors'); + }) + .finally(() => { + embedInFlightByCollection.delete(collectionName); + }); } function buildLexicalQueryVariants(message: string): string[] { @@ -358,6 +445,18 @@ function normalizeInline(text: string): string { return text.replace(/\s+/g, ' ').trim(); } +function normalizeQueryKey(text: string): string { + return text + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeFilenameKey(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + function truncateInline(text: string, max: number): string { if (text.length <= max) { return text; @@ -365,6 +464,11 @@ function truncateInline(text: string, max: number): string { return `${text.slice(0, max)}...`; } +function summarizeSnippetForLog(text: string, max = 120): string { + const oneLine = normalizeInline(text).replace(/\n/g, ' '); + return oneLine.length <= max ? oneLine : `${oneLine.slice(0, max)}...`; +} + function parseTurnSections(content: string): TurnSections { const userMarker = '\n## User\n'; const assistantMarker = '\n## Assistant\n'; @@ -386,7 +490,21 @@ function loadTurnSectionsFromSource(source: string, agentId: string): TurnSectio return null; } const rel = decodeURIComponent(m[1]); - const fullPath = path.join(getAgentTurnsDir(agentId), rel); + const agentDir = getAgentTurnsDir(agentId); + let fullPath = path.join(agentDir, rel); + if (!fs.existsSync(fullPath)) { + // qmd may normalize source path (e.g. casing and punctuation like "_" -> "-"). + // Resolve with a tolerant fallback so we can hydrate turn sections reliably. + const wanted = normalizeFilenameKey(path.basename(rel)); + try { + const found = fs.readdirSync(agentDir).find(name => normalizeFilenameKey(name) === wanted); + if (found) { + fullPath = path.join(agentDir, found); + } + } catch { + return null; + } + } if (!fs.existsSync(fullPath)) { return null; } @@ -398,16 +516,13 @@ function loadTurnSectionsFromSource(source: string, agentId: string): TurnSectio } } -function isLowConfidenceAnswer(text: string): boolean { - return /不知道|没有.*信息|无法|不清楚|need more context|don't have any information|i don't have|not enough information/i.test(text); -} - -function rerankAndHydrateResults(results: QmdResult[], message: string, agentId: string): QmdResult[] { +function rerankAndHydrateResults(results: QmdResult[], message: string, agentId: string, settings?: Settings): QmdResult[] { if (results.length === 0) { return results; } const terms = Array.from(new Set((message.toLowerCase().match(/[a-z0-9_-]{2,}|[\u4e00-\u9fff]{1,3}/g) || []))); - const codePattern = /\b[A-Z]{3,}(?:-[A-Z0-9]+){2,}\b/; + const rerankOptions = resolveMemoryRerankOptions(settings); + const normalizedMessage = normalizeQueryKey(message); return results .map((result) => { @@ -422,23 +537,24 @@ function rerankAndHydrateResults(results: QmdResult[], message: string, agentId: snippet = `User: ${truncateInline(user, 180)}\nAssistant: ${truncateInline(assistant, 260)}`; } - if (codePattern.test(assistant)) score += 0.5; - if (/代号|key|code|是|喜欢|likes?/i.test(assistant)) score += 0.2; - if (isLowConfidenceAnswer(assistant)) score -= 0.5; - - const hay = `${user} ${assistant}`.toLowerCase(); - for (const t of terms) { - if (hay.includes(t)) score += 0.04; + score += computeHeuristicScoreDelta(user, assistant, terms, rerankOptions); + const lowConfidence = isLowConfidenceText(assistant, rerankOptions); + if (lowConfidence) { + return null; } } return { score, snippet, source: result.source }; }) + .filter((row): row is QmdResult => !!row) .sort((a, b) => b.score - a.score); } -async function quickHasLexicalHit(message: string, collectionName: string, qmdCfg: QmdConfig): Promise { - const variants = buildLexicalQueryVariants(message); +async function quickHasLexicalHit( + variants: string[], + collectionName: string, + qmdCfg: QmdConfig +): Promise { for (const query of variants) { const args = ['search', query, '--json', '-c', collectionName, '-n', '1', '--min-score', String(qmdCfg.minScore)]; const { stdout } = await runCommand(qmdCommandPath || 'qmd', args, undefined, qmdCfg.precheckTimeoutMs); @@ -487,23 +603,20 @@ function formatMemoryPrompt(results: QmdResult[], maxChars: number): string { ].join('\n'); } -function resolveRetrievalMode(qmdCfg: QmdConfig): { useVsearch: boolean; label: 'qmd-bm25' | 'qmd-vsearch' } { +function resolveRetrievalMode(agentId: string, qmdCfg: QmdConfig): { useVsearch: boolean; label: 'qmd-bm25' | 'qmd-vsearch' } { let useVsearch = qmdCfg.useSemanticSearch; if (useVsearch && !qmdCfg.allowUnsafeVsearch) { if (!qmdCfg.disableQueryExpansion) { useVsearch = false; - if (!qmdUnsafeFallbackLogged) { - log('WARN', 'QMD vsearch requested without disable_query_expansion; fallback to BM25 for safety. Set memory.qmd.allow_unsafe_vsearch=true to override.'); - qmdUnsafeFallbackLogged = true; - } + warnUnsafeFallback(agentId, 'disable_query_expansion is false; using BM25. Set memory.qmd.allow_unsafe_vsearch=true to override.'); } else if (!isDisableExpansionSupported()) { useVsearch = false; - if (!qmdUnsafeFallbackLogged) { - log('WARN', 'QMD disable-query-expansion support not detected; fallback to BM25 to avoid unexpected model downloads. Run scripts/patch-qmd-no-expansion.sh or set memory.qmd.allow_unsafe_vsearch=true.'); - qmdUnsafeFallbackLogged = true; - } + warnUnsafeFallback(agentId, 'safe vsearch support not detected; using BM25. Run scripts/patch-qmd-no-expansion.sh or set memory.qmd.allow_unsafe_vsearch=true.'); } } + if (useVsearch) { + infoVsearchExperimental(agentId); + } return { useVsearch, @@ -512,12 +625,12 @@ function resolveRetrievalMode(qmdCfg: QmdConfig): { useVsearch: boolean; label: } async function runBm25WithVariants( - message: string, + variants: string[], + fallbackQuery: string, collectionName: string, qmdCfg: QmdConfig ): Promise { - const variants = buildLexicalQueryVariants(message); - let lastQuery = message; + let lastQuery = fallbackQuery; for (const query of variants) { lastQuery = query; const args = ['search', query, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; @@ -546,13 +659,14 @@ export async function buildMemoryBlock( const hasQmd = await isQmdAvailable(qmdCfg.command); if (!hasQmd) { - if (!qmdUnavailableLogged) { - log('WARN', 'qmd not found in PATH, memory retrieval disabled'); - qmdUnavailableLogged = true; + if (!qmdUnavailableLoggedByAgent.get(agentId)) { + log('WARN', `qmd not found in PATH, memory retrieval disabled for @${agentId}`); + qmdUnavailableLoggedByAgent.set(agentId, true); } log('INFO', `Memory source for @${agentId}: none (qmd unavailable)`); return ''; } + qmdUnavailableLoggedByAgent.delete(agentId); logQmdDebug(agentId, qmdCfg, 'qmd', `command=${qmdCommandPath || 'qmd'}`); try { @@ -561,15 +675,19 @@ export async function buildMemoryBlock( await maybeUpdateCollection(collectionName, qmdCfg.updateIntervalSeconds); logQmdDebug(agentId, qmdCfg, 'update', `interval=${qmdCfg.updateIntervalSeconds}s`); - if (qmdCfg.quickPrecheckEnabled) { + const mode = resolveRetrievalMode(agentId, qmdCfg); + const queryVariants = buildLexicalQueryVariants(message); + + // Only keep precheck for vsearch path. For BM25 it duplicates work and adds latency. + if (qmdCfg.quickPrecheckEnabled && mode.useVsearch) { try { logQmdDebug( agentId, qmdCfg, 'precheck', - `cmd=search timeout=${qmdCfg.precheckTimeoutMs}ms min_score=${qmdCfg.minScore} variants=${buildLexicalQueryVariants(message).length}` + `cmd=search timeout=${qmdCfg.precheckTimeoutMs}ms min_score=${qmdCfg.minScore} variants=${queryVariants.length}` ); - const hasQuickHit = await quickHasLexicalHit(message, collectionName, qmdCfg); + const hasQuickHit = await quickHasLexicalHit(queryVariants, collectionName, qmdCfg); if (!hasQuickHit) { log('INFO', `Memory source for @${agentId}: none (qmd precheck no-hit)`); return ''; @@ -579,17 +697,12 @@ export async function buildMemoryBlock( log('INFO', `Memory source for @${agentId}: none (qmd precheck error)`); return ''; } + } else if (qmdCfg.quickPrecheckEnabled && !mode.useVsearch) { + logQmdDebug(agentId, qmdCfg, 'precheck', 'skipped for bm25 mode (avoid duplicate searches)'); } - const mode = resolveRetrievalMode(qmdCfg); if (mode.useVsearch) { - try { - await maybeEmbedCollection(collectionName, qmdCfg.embedIntervalSeconds); - logQmdDebug(agentId, qmdCfg, 'embed', `interval=${qmdCfg.embedIntervalSeconds}s`); - } catch (error) { - log('WARN', `Memory embed skipped for @${agentId}: ${(error as Error).message}`); - logQmdDebug(agentId, qmdCfg, 'embed', 'failed; continuing with existing vectors'); - } + triggerEmbedCollectionAsync(agentId, collectionName, qmdCfg); } const queryArgs = mode.useVsearch ? ['vsearch', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)] @@ -610,15 +723,49 @@ export async function buildMemoryBlock( results: parseQmdResults(stdout), query: message, })))() - : runBm25WithVariants(message, collectionName, qmdCfg); - const { results, query } = await queryResult; + : runBm25WithVariants(queryVariants, message, collectionName, qmdCfg); + let { results, query } = await queryResult; logQmdDebug(agentId, qmdCfg, 'query-used', `mode=${mode.label} query=\"${query}\"`); if (results.length === 0) { log('INFO', `Memory source for @${agentId}: none (${mode.label} no-hit)`); return ''; } - const rankedResults = rerankAndHydrateResults(results, message, agentId); + let rankedResults = rerankAndHydrateResults(results, message, agentId, settings); + logQmdDebug( + agentId, + qmdCfg, + 'rerank', + `raw=${results.length} ranked=${rankedResults.length} top=${rankedResults + .slice(0, 3) + .map((r, i) => `${i + 1}:${r.score.toFixed(3)}:${summarizeSnippetForLog(r.snippet, 90)}`) + .join(' | ')}` + ); + if (mode.useVsearch && rankedResults.length === 0) { + logQmdDebug(agentId, qmdCfg, 'query', 'vsearch results filtered out; fallback=bm25'); + const bm25Fallback = await runBm25WithVariants(queryVariants, message, collectionName, qmdCfg); + results = bm25Fallback.results; + query = bm25Fallback.query; + logQmdDebug(agentId, qmdCfg, 'query-used', `mode=qmd-bm25-fallback query=\"${query}\"`); + if (results.length === 0) { + log('INFO', `Memory source for @${agentId}: none (qmd-vsearch filtered + bm25 no-hit)`); + return ''; + } + rankedResults = rerankAndHydrateResults(results, message, agentId, settings); + logQmdDebug( + agentId, + qmdCfg, + 'rerank', + `fallback raw=${results.length} ranked=${rankedResults.length} top=${rankedResults + .slice(0, 3) + .map((r, i) => `${i + 1}:${r.score.toFixed(3)}:${summarizeSnippetForLog(r.snippet, 90)}`) + .join(' | ')}` + ); + if (rankedResults.length === 0) { + log('INFO', `Memory source for @${agentId}: none (qmd-vsearch filtered + bm25 filtered)`); + return ''; + } + } const memoryBlock = formatMemoryPrompt(rankedResults, qmdCfg.maxChars); if (!memoryBlock) { @@ -647,7 +794,7 @@ export async function enrichMessageWithMemory( } function timestampFilename(ts: number): string { - return new Date(ts).toISOString().replace(/[:.]/g, '-'); + return new Date(ts).toISOString().replace(/[:.]/g, '-').toLowerCase(); } function truncate(text: string, max = 16000): string { @@ -657,6 +804,77 @@ function truncate(text: string, max = 16000): string { return `${text.substring(0, max)}\n\n[truncated]`; } +function maybeCleanupTurns(agentId: string, settings?: Settings): void { + const retention = getRetentionConfig(settings); + if (!retention.enabled) { + return; + } + + const now = Date.now(); + const last = lastTurnsCleanupMs.get(agentId) || 0; + if (now - last < retention.cleanupIntervalSeconds * 1000) { + return; + } + lastTurnsCleanupMs.set(agentId, now); + + const dir = getAgentTurnsDir(agentId); + if (!fs.existsSync(dir)) { + return; + } + + let entries = fs.readdirSync(dir) + .filter(name => name.endsWith('.md')) + .map(name => { + const filePath = path.join(dir, name); + const stat = fs.statSync(filePath); + return { + name, + filePath, + mtimeMs: stat.mtimeMs, + }; + }) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + + if (entries.length === 0) { + return; + } + + const cutoffMs = now - retention.retainDays * 24 * 60 * 60 * 1000; + const keep: typeof entries = []; + const remove: typeof entries = []; + + for (const e of entries) { + if (e.mtimeMs < cutoffMs) { + remove.push(e); + } else { + keep.push(e); + } + } + + if (keep.length > retention.maxTurnFilesPerAgent) { + const overflow = keep.slice(retention.maxTurnFilesPerAgent); + remove.push(...overflow); + keep.splice(retention.maxTurnFilesPerAgent); + } + + if (remove.length === 0) { + return; + } + + let deleted = 0; + for (const e of remove) { + try { + fs.unlinkSync(e.filePath); + deleted++; + } catch { + // Keep going; best-effort cleanup. + } + } + if (deleted > 0) { + log('INFO', `Memory turns cleanup for @${agentId}: deleted ${deleted} file(s), kept ${keep.length}`); + } +} + export async function saveTurnToMemory(params: { agentId: string; agent: AgentConfig; @@ -666,6 +884,7 @@ export async function saveTurnToMemory(params: { userMessage: string; agentResponse: string; timestampMs?: number; + settings?: Settings; }): Promise { try { const timestampMs = params.timestampMs || Date.now(); @@ -693,6 +912,7 @@ export async function saveTurnToMemory(params: { ]; fs.writeFileSync(filePath, lines.join('\n')); + maybeCleanupTurns(params.agentId, params.settings); } catch (error) { log('WARN', `Failed to persist memory turn for @${params.agentId}: ${(error as Error).message}`); } diff --git a/src/lib/types.ts b/src/lib/types.ts index 4256528..0f3d67c 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -43,6 +43,21 @@ export interface Settings { }; memory?: { enabled?: boolean; + rerank?: { + enabled?: boolean; + answer_signal_patterns?: string[]; + low_confidence_patterns?: string[]; + answer_signal_bonus?: number; + low_confidence_penalty?: number; + code_pattern_bonus?: number; + term_hit_bonus?: number; + }; + retention?: { + enabled?: boolean; + retain_days?: number; + max_turn_files_per_agent?: number; + cleanup_interval_seconds?: number; + }; qmd?: { enabled?: boolean; command?: string; diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 2024382..70fd494 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -21,12 +21,10 @@ import { import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; import { invokeAgent } from './lib/invoke'; -import { saveTurnToMemory } from './lib/memory'; - -const MEMORY_PERSIST_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); +import { saveTurnToMemory, MEMORY_ELIGIBLE_CHANNELS } from './lib/memory'; function shouldPersistMemoryTurn(channel: string): boolean { - return MEMORY_PERSIST_CHANNELS.has(channel); + return MEMORY_ELIGIBLE_CHANNELS.has(channel); } // Ensure directories exist @@ -156,6 +154,7 @@ async function processMessage(messageFile: string): Promise { userMessage: message, agentResponse: finalResponse, timestampMs: Date.now(), + settings, }); } } catch (error) { @@ -206,6 +205,7 @@ async function processMessage(messageFile: string): Promise { userMessage: currentMessage, agentResponse: stepResponse, timestampMs: Date.now(), + settings, }); } } catch (error) { @@ -278,6 +278,7 @@ async function processMessage(messageFile: string): Promise { userMessage: mMessage, agentResponse: mResponse, timestampMs: Date.now(), + settings, }); } } catch (error) {