diff --git a/README.md b/README.md index 490fe90..bc586c6 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ The setup wizard will guide you through: 5. **AI provider** - Select Anthropic (Claude) or OpenAI 6. **Model selection** - Choose model (e.g., Sonnet, Opus, GPT-5.3) 7. **Heartbeat interval** - Set proactive check-in frequency +8. **Memory retrieval (optional)** - Enable/disable QMD-based memory at setup time
πŸ“± Channel Setup Guides @@ -384,10 +385,69 @@ Located at `.tinyclaw/settings.json`: }, "monitoring": { "heartbeat_interval": 3600 + }, + "memory": { + "enabled": true, + "rerank": { + "enabled": true + }, + "retention": { + "enabled": true, + "retain_days": 30, + "max_turn_files_per_agent": 2000, + "cleanup_interval_seconds": 300 + }, + "qmd": { + "enabled": true, + "command": "/home/me/.bun/bin/qmd", + "top_k": 4, + "min_score": 0.05, + "max_chars": 2500, + "update_interval_seconds": 120, + "embed_interval_seconds": 600, + "use_semantic_search": false, + "disable_query_expansion": true, + "allow_unsafe_vsearch": false, + "debug_logging": false + } } } ``` +### Optional Memory Retrieval (QMD) + +TinyClaw can use external memory retrieval with `qmd` (BM25 by default). + +- Memory retrieval is **disabled by default**. Set `"memory.enabled": true` to enable. +- Retrieval runs only for user chat channels (`telegram`, `discord`, `whatsapp`). +- Heartbeat/system messages are excluded from retrieval and memory persistence. +- Turns are stored in `~/.tinyclaw/memory/turns//`. +- Turn files are cleaned automatically (default: keep 30 days and max 2000 files per agent). +- Retrieval reranking heuristics are configurable via `memory.rerank` (can be disabled). + +Quick setup: + +```bash +# Install qmd (recommended by qmd project) +bun install -g github:tobi/qmd + +# Linux/WSL: install sqlite-vec extension package if qmd reports missing extension +bun add -g sqlite-vec-linux-x64 + +# Verify qmd works +qmd status + +# Optional: if qmd is not in PATH, set memory.qmd.command in settings.json +tinyclaw restart +``` + +Safety note: +- `use_semantic_search` (`qmd-vsearch`) is currently **experimental**. +- If semantic search is enabled, TinyClaw defaults to safe mode and will fall back to BM25 unless disable-expansion support is detected. +- Set `memory.qmd.allow_unsafe_vsearch: true` only if you explicitly want to allow unguarded `vsearch`. +- Set `memory.qmd.debug_logging: true` to print QMD memory debug logs (command/mode/timeout) in `queue.log`. +- When semantic search is enabled, TinyClaw periodically runs `qmd embed` in background (non-blocking). Default `embed_interval_seconds` is `600`. + ### Heartbeat Configuration Edit agent-specific heartbeat prompts: @@ -480,6 +540,7 @@ tinyclaw logs all - Bash version error β†’ Install bash 4.0+: `brew install bash` - WhatsApp not connecting β†’ Reset auth: `tinyclaw channels reset whatsapp` +- WhatsApp startup fails with `Could not find Chrome` β†’ Install browser: `npx puppeteer browsers install chrome` - Messages stuck β†’ Clear queue: `rm -rf .tinyclaw/queue/processing/*` - Agent not found β†’ Check: `tinyclaw agent list` diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 3cd1797..4cd2456 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -46,10 +46,18 @@ tinyclaw channels reset whatsapp tinyclaw restart ``` +If startup fails with `Could not find Chrome (...)`, install Puppeteer Chrome first: + +```bash +npx puppeteer browsers install chrome +tinyclaw restart +``` + **Common causes:** - QR code expired (scan within 60 seconds) - Session files corrupted - Multiple WhatsApp Web sessions active +- Chrome/Chromium missing for Puppeteer **Solution:** 1. Delete session: `rm -rf .tinyclaw/whatsapp-session/` @@ -375,6 +383,38 @@ ps aux | grep -E 'claude|codex|node' | awk '{print $4, $11}' tail -f .tinyclaw/logs/queue.log | grep "Processing completed" ``` +### QMD memory retrieval not working + +If you enabled memory retrieval and see warnings like `qmd not found` or `Memory retrieval skipped`: + +1. **Check qmd command:** + ```bash + command -v qmd + qmd status + ``` + +2. **If qmd is not in PATH, set explicit command in `~/.tinyclaw/settings.json`:** + ```json + { + "memory": { + "enabled": true, + "qmd": { + "command": "/home/you/.bun/bin/qmd" + } + } + } + ``` + +3. **Restart TinyClaw:** + ```bash + tinyclaw restart + ``` + +4. **Verify retrieval hits in logs:** + ```bash + grep "Memory retrieval hit" ~/.tinyclaw/logs/queue.log + ``` + ## Log Analysis ### Enable debug logging diff --git a/lib/daemon.sh b/lib/daemon.sh index 839b9cb..22a9d34 100644 --- a/lib/daemon.sh +++ b/lib/daemon.sh @@ -2,6 +2,40 @@ # Daemon lifecycle management for TinyClaw # Handles starting, stopping, restarting, and status checking +whatsapp_enabled() { + local ch + for ch in "${ACTIVE_CHANNELS[@]}"; do + if [ "$ch" = "whatsapp" ]; then + return 0 + fi + done + return 1 +} + +check_whatsapp_preflight() { + if ! whatsapp_enabled; then + return 0 + fi + + # whatsapp-web.js launches Puppeteer; fail fast if Chrome is missing. + local chrome_path + chrome_path=$(node -e "const p=require('puppeteer');process.stdout.write(p.executablePath() || '')" 2>/dev/null || true) + if [ -n "$chrome_path" ] && [ -x "$chrome_path" ]; then + return 0 + fi + + echo -e "${RED}WhatsApp preflight failed: Chrome/Chromium not found for Puppeteer.${NC}" + echo "" + echo "Install browser dependency and retry:" + echo -e " ${GREEN}npx puppeteer browsers install chrome${NC}" + echo -e " ${GREEN}./tinyclaw.sh start${NC}" + echo "" + echo "Tip: if using a custom HOME, run both commands with the same HOME value." + echo "" + log "WhatsApp preflight failed: missing Chrome/Chromium for Puppeteer" + return 1 +} + # Start daemon start_daemon() { if session_exists; then @@ -54,6 +88,10 @@ start_daemon() { return 1 fi + if ! check_whatsapp_preflight; then + return 1 + fi + # Validate tokens for channels that need them for ch in "${ACTIVE_CHANNELS[@]}"; do local token_key="${CHANNEL_TOKEN_KEY[$ch]:-}" diff --git a/lib/setup-wizard.sh b/lib/setup-wizard.sh index c7acfe2..6c1a272 100755 --- a/lib/setup-wizard.sh +++ b/lib/setup-wizard.sh @@ -174,6 +174,39 @@ DEFAULT_AGENT_NAME=$(echo "$DEFAULT_AGENT_NAME" | tr ' ' '-' | tr -cd 'a-zA-Z0-9 echo -e "${GREEN}βœ“ Default agent: $DEFAULT_AGENT_NAME${NC}" echo "" +# Optional memory retrieval setup +echo "Enable memory retrieval (QMD)?" +echo -e "${YELLOW}(Optional. If disabled, TinyClaw still works normally.)${NC}" +echo "" +read -rp "Enable memory retrieval? [y/N]: " ENABLE_MEMORY + +MEMORY_ENABLED=false +USE_SEMANTIC_SEARCH=false +QMD_COMMAND="" + +if [[ "$ENABLE_MEMORY" =~ ^[yY] ]]; then + MEMORY_ENABLED=true + + if [ -x "$HOME/.bun/bin/qmd" ]; then + QMD_COMMAND="$HOME/.bun/bin/qmd" + elif command -v qmd >/dev/null 2>&1; then + QMD_COMMAND="$(command -v qmd)" + fi + + if [ -n "$QMD_COMMAND" ]; then + echo -e "${GREEN}βœ“ Found qmd: $QMD_COMMAND${NC}" + else + echo -e "${YELLOW}⚠ qmd not found in PATH right now.${NC}" + echo -e "${YELLOW} You can install later with: bun install -g github:tobi/qmd${NC}" + fi + + read -rp "Use semantic search (vector, experimental)? [y/N]: " USE_SEMANTIC + if [[ "$USE_SEMANTIC" =~ ^[yY] ]]; then + USE_SEMANTIC_SEARCH=true + fi +fi +echo "" + # --- Additional Agents (optional) --- echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${GREEN} Additional Agents (Optional)${NC}" @@ -274,6 +307,12 @@ else MODELS_SECTION='"models": { "provider": "openai", "openai": { "model": "'"${MODEL}"'" } }' fi +if [ "$MEMORY_ENABLED" = true ]; then + MEMORY_SECTION='"memory": { "enabled": true, "qmd": { "enabled": true, "command": "'"${QMD_COMMAND}"'", "top_k": 4, "min_score": 0.05, "max_chars": 2500, "update_interval_seconds": 300, "embed_interval_seconds": 600, "use_semantic_search": '"${USE_SEMANTIC_SEARCH}"', "disable_query_expansion": true, "allow_unsafe_vsearch": false, "debug_logging": false } },' +else + MEMORY_SECTION='"memory": { "enabled": false },' +fi + cat > "$SETTINGS_FILE" < "$SETTINGS_FILE" <&2 + exit 1 +fi + +if grep -q "QMD_VSEARCH_DISABLE_EXPANSION" "$TARGET_PATH"; then + echo "Already patched: $TARGET_PATH" + exit 0 +fi + +node - "$TARGET_PATH" <<'NODE' +const fs = require('fs'); +const target = process.argv[2]; +const src = fs.readFileSync(target, 'utf8'); + +const marker = 'QMD_VSEARCH_DISABLE_EXPANSION'; +if (src.includes(marker)) { + console.log(`Already patched: ${target}`); + process.exit(0); +} + +const pattern = / \/\/ Expand query β€” filter to vec\/hyde only \(lex queries target FTS, not vector\)\n const allExpanded = await store\.expandQuery\(query\);\n const vecExpanded = allExpanded\.filter\(q => q\.type !== 'lex'\);\n options\?\.hooks\?\.onExpand\?\.\(query, vecExpanded\);/; + +const replacement = [ + " // Optional: disable query expansion to run embedding-only vector search.", + " const disableExpansion = process.env.QMD_VSEARCH_DISABLE_EXPANSION === '1';", + " const vecExpanded = disableExpansion", + " ? []", + " : (await store.expandQuery(query)).filter(q => q.type !== 'lex');", + " options?.hooks?.onExpand?.(query, vecExpanded);", +].join('\n'); + +if (!pattern.test(src)) { + console.error('Patch anchor not found. qmd source layout may have changed.'); + process.exit(2); +} + +const out = src.replace(pattern, replacement); +fs.writeFileSync(target, out); +console.log(`Patched: ${target}`); +NODE diff --git a/src/lib/invoke.ts b/src/lib/invoke.ts index 471eaf2..fca0347 100644 --- a/src/lib/invoke.ts +++ b/src/lib/invoke.ts @@ -1,10 +1,85 @@ import { spawn } from 'child_process'; import fs from 'fs'; import path from 'path'; -import { AgentConfig, TeamConfig } from './types'; +import { AgentConfig, Settings, TeamConfig } from './types'; import { SCRIPT_DIR, resolveClaudeModel, resolveCodexModel } from './config'; import { log } from './logging'; import { ensureAgentDirectory, updateAgentTeammates } from './agent-setup'; +import { buildMemoryBlock } from './memory'; + +const CLAUDE_SYSTEM_FILENAME = 'CLAUDE.md'; + +function getClaudeSystemFilePath(workingDir: string): string { + return path.join(workingDir, '.claude', CLAUDE_SYSTEM_FILENAME); +} + +function injectClaudeMemory(systemFilePath: string, memoryBlock: string, agentId: string): () => void { + const claudeDir = path.dirname(systemFilePath); + fs.mkdirSync(claudeDir, { recursive: true }); + + const existed = fs.existsSync(systemFilePath); + const original = existed ? fs.readFileSync(systemFilePath, 'utf8') : ''; + const markerId = `${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const startMarker = ``; + const endMarker = ``; + const body = memoryBlock.trim(); + const runtimeSection = [ + startMarker, + '## Runtime Memory Context', + '', + 'Auto-generated for current invocation only.', + '', + body, + '', + endMarker, + ].join('\n'); + const merged = original.trim().length > 0 + ? `${original.trimEnd()}\n\n${runtimeSection}` + : runtimeSection; + + fs.writeFileSync(systemFilePath, merged, 'utf8'); + + return () => { + if (!fs.existsSync(systemFilePath)) { + if (existed) { + log('WARN', `Memory cleanup marker missing for @${agentId}: .claude/${CLAUDE_SYSTEM_FILENAME} disappeared during invoke`); + } + return; + } + + const current = fs.readFileSync(systemFilePath, 'utf8'); + const startIdx = current.indexOf(startMarker); + const endIdx = current.indexOf(endMarker, startIdx >= 0 ? startIdx : 0); + if (startIdx < 0 || endIdx < 0) { + log('WARN', `Memory cleanup marker missing for @${agentId}: skipped restore to avoid overwriting .claude/${CLAUDE_SYSTEM_FILENAME}`); + return; + } + + const endExclusive = endIdx + endMarker.length; + const stripped = `${current.slice(0, startIdx)}${current.slice(endExclusive)}`; + const normalized = stripped + .replace(/\n{3,}/g, '\n\n') + .replace(/^\s+|\s+$/g, ''); + + if (!normalized) { + if (existed) { + fs.writeFileSync(systemFilePath, original, 'utf8'); + } else { + fs.unlinkSync(systemFilePath); + } + return; + } + + fs.writeFileSync(systemFilePath, `${normalized}\n`, 'utf8'); + }; +} + +function deleteLegacyClaudeMemoryFile(workingDir: string): void { + const legacyPath = path.join(workingDir, '.claude', 'MEMORY.md'); + if (fs.existsSync(legacyPath)) { + fs.unlinkSync(legacyPath); + } +} export async function runCommand(command: string, args: string[], cwd?: string): Promise { return new Promise((resolve, reject) => { @@ -51,8 +126,10 @@ export async function invokeAgent( agent: AgentConfig, agentId: string, message: string, + sourceChannel: string, workspacePath: string, shouldReset: boolean, + settings: Settings, agents: Record = {}, teams: Record = {} ): Promise { @@ -74,9 +151,11 @@ export async function invokeAgent( : path.join(workspacePath, agent.working_directory)) : agentDir; + const memoryBlock = await buildMemoryBlock(agentId, message, settings, sourceChannel); const provider = agent.provider || 'anthropic'; if (provider === 'openai') { + const messageForModel = memoryBlock ? `${message}${memoryBlock}` : message; log('INFO', `Using Codex CLI (agent: ${agentId})`); const shouldResume = !shouldReset; @@ -93,7 +172,7 @@ export async function invokeAgent( if (modelId) { codexArgs.push('--model', modelId); } - codexArgs.push('--skip-git-repo-check', '--dangerously-bypass-approvals-and-sandbox', '--json', message); + codexArgs.push('--skip-git-repo-check', '--dangerously-bypass-approvals-and-sandbox', '--json', messageForModel); const codexOutput = await runCommand('codex', codexArgs, workingDir); @@ -132,6 +211,20 @@ export async function invokeAgent( } claudeArgs.push('-p', message); - return await runCommand('claude', claudeArgs, workingDir); + const systemFilePath = getClaudeSystemFilePath(workingDir); + let restoreSystemFile: (() => void) | null = null; + // Defensive cleanup from old MEMORY.md injection behavior. + deleteLegacyClaudeMemoryFile(workingDir); + try { + if (memoryBlock) { + restoreSystemFile = injectClaudeMemory(systemFilePath, memoryBlock, agentId); + log('INFO', `Memory source injection for @${agentId}: .claude/${CLAUDE_SYSTEM_FILENAME} (runtime section)`); + } + return await runCommand('claude', claudeArgs, workingDir); + } finally { + if (restoreSystemFile) { + restoreSystemFile(); + } + } } } diff --git a/src/lib/memory-rerank.ts b/src/lib/memory-rerank.ts new file mode 100644 index 0000000..595ed16 --- /dev/null +++ b/src/lib/memory-rerank.ts @@ -0,0 +1,102 @@ +import { Settings } from './types'; + +const DEFAULT_ANSWER_SIGNAL_PATTERNS = ['代号', 'key', 'code', '是', 'ε–œζ¬’', 'likes?']; +const DEFAULT_LOW_CONFIDENCE_PATTERNS = [ + '不ηŸ₯道', + 'ζ²‘ζœ‰.*俑息', + '无法', + '不清ζ₯š', + 'ζ²‘ζœ‰θΆ³ε€Ÿ.*δΈŠδΈ‹ζ–‡', + 'ηΌΊε°‘.*δΈŠδΈ‹ζ–‡', + 'need more context', + 'enough context', + "don't have enough context", + "don't have any information", + "i don't have", + 'not enough information', +]; + +const DEFAULT_CODE_PATTERN = /\b[A-Z]{3,}(?:-[A-Z0-9]+){2,}\b/; + +export interface MemoryRerankOptions { + enabled: boolean; + answerSignalPatterns: string[]; + lowConfidencePatterns: string[]; + answerSignalBonus: number; + lowConfidencePenalty: number; + codePatternBonus: number; + termHitBonus: number; +} + +export function resolveMemoryRerankOptions(settings?: Settings): MemoryRerankOptions { + const cfg = settings?.memory?.rerank; + return { + enabled: cfg?.enabled !== false, + answerSignalPatterns: Array.isArray(cfg?.answer_signal_patterns) && cfg.answer_signal_patterns.length > 0 + ? cfg.answer_signal_patterns + : DEFAULT_ANSWER_SIGNAL_PATTERNS, + lowConfidencePatterns: Array.isArray(cfg?.low_confidence_patterns) && cfg.low_confidence_patterns.length > 0 + ? cfg.low_confidence_patterns + : DEFAULT_LOW_CONFIDENCE_PATTERNS, + answerSignalBonus: Number.isFinite(cfg?.answer_signal_bonus) ? Number(cfg?.answer_signal_bonus) : 0.2, + lowConfidencePenalty: Number.isFinite(cfg?.low_confidence_penalty) ? Number(cfg?.low_confidence_penalty) : 0.5, + codePatternBonus: Number.isFinite(cfg?.code_pattern_bonus) ? Number(cfg?.code_pattern_bonus) : 0.5, + termHitBonus: Number.isFinite(cfg?.term_hit_bonus) ? Number(cfg?.term_hit_bonus) : 0.04, + }; +} + +function buildRegex(pattern: string): RegExp | null { + try { + return new RegExp(pattern, 'i'); + } catch { + return null; + } +} + +function matchesAnyPattern(text: string, patterns: string[]): boolean { + for (const pattern of patterns) { + const regex = buildRegex(pattern); + if (!regex) { + continue; + } + if (regex.test(text)) { + return true; + } + } + return false; +} + +export function isLowConfidenceText(text: string, options: MemoryRerankOptions): boolean { + return matchesAnyPattern(text, options.lowConfidencePatterns); +} + +export function computeHeuristicScoreDelta( + user: string, + assistant: string, + messageTerms: string[], + options: MemoryRerankOptions +): number { + if (!options.enabled) { + return 0; + } + + let delta = 0; + if (DEFAULT_CODE_PATTERN.test(assistant)) { + delta += options.codePatternBonus; + } + if (matchesAnyPattern(assistant, options.answerSignalPatterns)) { + delta += options.answerSignalBonus; + } + if (matchesAnyPattern(assistant, options.lowConfidencePatterns)) { + delta -= options.lowConfidencePenalty; + } + + const hay = `${user} ${assistant}`.toLowerCase(); + for (const term of messageTerms) { + if (hay.includes(term)) { + delta += options.termHitBonus; + } + } + + return delta; +} diff --git a/src/lib/memory.ts b/src/lib/memory.ts new file mode 100644 index 0000000..7feebd2 --- /dev/null +++ b/src/lib/memory.ts @@ -0,0 +1,919 @@ +import { spawn } from 'child_process'; +import fs from 'fs'; +import path from 'path'; +import { AgentConfig, Settings } from './types'; +import { TINYCLAW_HOME } from './config'; +import { log } from './logging'; +import { computeHeuristicScoreDelta, isLowConfidenceText, resolveMemoryRerankOptions } from './memory-rerank'; + +const MEMORY_ROOT = path.join(TINYCLAW_HOME, 'memory'); +const MEMORY_TURNS_DIR = path.join(MEMORY_ROOT, 'turns'); + +const DEFAULT_TOP_K = 4; +const DEFAULT_MIN_SCORE = 0.0; +const DEFAULT_MAX_CHARS = 2500; +const DEFAULT_UPDATE_INTERVAL_SECONDS = 120; +const DEFAULT_EMBED_INTERVAL_SECONDS = 600; +const DEFAULT_RETAIN_DAYS = 30; +const DEFAULT_MAX_TURN_FILES_PER_AGENT = 2000; +const DEFAULT_CLEANUP_INTERVAL_SECONDS = 300; +const DEFAULT_PRECHECK_TIMEOUT_MS = 800; +const DEFAULT_TEXT_SEARCH_TIMEOUT_MS = 3000; +const DEFAULT_VECTOR_SEARCH_TIMEOUT_MS = 10000; +const QMD_FALLBACK_WARN_INTERVAL_MS = 10 * 60 * 1000; +const VSEARCH_EXPERIMENTAL_INFO_INTERVAL_MS = 10 * 60 * 1000; + +let qmdChecked = false; +let qmdAvailable = false; +const qmdUnavailableLoggedByAgent = new Map(); +let qmdCommandPath: string | null = null; +let qmdCheckKey = ''; +let qmdDisableExpansionCheckKey = ''; +let qmdDisableExpansionSupported = false; + +const collectionPrepared = new Set(); +const lastCollectionUpdateMs = new Map(); +const lastCollectionEmbedMs = new Map(); +const embedInFlightByCollection = new Set(); +const lastTurnsCleanupMs = new Map(); +const lastUnsafeFallbackWarnMsByAgent = new Map(); +const lastVsearchExperimentalInfoMsByAgent = new Map(); +export const MEMORY_ELIGIBLE_CHANNELS = new Set(['telegram', 'discord', 'whatsapp']); + +interface QmdResult { + score: number; + snippet: string; + source: string; +} + +interface QmdConfig { + enabled: boolean; + command?: string; + topK: number; + minScore: number; + maxChars: number; + updateIntervalSeconds: number; + embedIntervalSeconds: number; + useSemanticSearch: boolean; + disableQueryExpansion: boolean; + allowUnsafeVsearch: boolean; + quickPrecheckEnabled: boolean; + precheckTimeoutMs: number; + searchTimeoutMs: number; + vectorSearchTimeoutMs: number; + debugLogging: boolean; +} + +interface CommandResult { + stdout: string; + stderr: string; +} + +interface QueryResult { + results: QmdResult[]; + query: string; +} + +interface TurnSections { + user: string; + assistant: string; +} + +interface RetentionConfig { + enabled: boolean; + retainDays: number; + maxTurnFilesPerAgent: number; + cleanupIntervalSeconds: number; +} + +function getQmdConfig(settings: Settings): QmdConfig { + const memoryCfg = settings.memory?.qmd; + const command = typeof memoryCfg?.command === 'string' ? memoryCfg.command.trim() : ''; + return { + enabled: settings.memory?.enabled === true && memoryCfg?.enabled !== false, + command: command || undefined, + topK: Number.isFinite(memoryCfg?.top_k) ? Math.max(1, Number(memoryCfg?.top_k)) : DEFAULT_TOP_K, + minScore: Number.isFinite(memoryCfg?.min_score) ? Number(memoryCfg?.min_score) : DEFAULT_MIN_SCORE, + maxChars: Number.isFinite(memoryCfg?.max_chars) ? Math.max(500, Number(memoryCfg?.max_chars)) : DEFAULT_MAX_CHARS, + updateIntervalSeconds: Number.isFinite(memoryCfg?.update_interval_seconds) + ? Math.max(10, Number(memoryCfg?.update_interval_seconds)) + : DEFAULT_UPDATE_INTERVAL_SECONDS, + embedIntervalSeconds: Number.isFinite(memoryCfg?.embed_interval_seconds) + ? Math.max(10, Number(memoryCfg?.embed_interval_seconds)) + : DEFAULT_EMBED_INTERVAL_SECONDS, + useSemanticSearch: memoryCfg?.use_semantic_search === true, + disableQueryExpansion: memoryCfg?.disable_query_expansion !== false, + allowUnsafeVsearch: memoryCfg?.allow_unsafe_vsearch === true, + quickPrecheckEnabled: memoryCfg?.quick_precheck_enabled !== false, + precheckTimeoutMs: Number.isFinite(memoryCfg?.precheck_timeout_ms) + ? Math.max(100, Number(memoryCfg?.precheck_timeout_ms)) + : DEFAULT_PRECHECK_TIMEOUT_MS, + searchTimeoutMs: Number.isFinite(memoryCfg?.search_timeout_ms) + ? Math.max(500, Number(memoryCfg?.search_timeout_ms)) + : DEFAULT_TEXT_SEARCH_TIMEOUT_MS, + vectorSearchTimeoutMs: Number.isFinite(memoryCfg?.vector_search_timeout_ms) + ? Math.max(1000, Number(memoryCfg?.vector_search_timeout_ms)) + : DEFAULT_VECTOR_SEARCH_TIMEOUT_MS, + debugLogging: memoryCfg?.debug_logging === true, + }; +} + +function getRetentionConfig(settings?: Settings): RetentionConfig { + const retentionCfg = settings?.memory?.retention; + return { + enabled: retentionCfg?.enabled !== false, + retainDays: Number.isFinite(retentionCfg?.retain_days) + ? Math.max(1, Number(retentionCfg?.retain_days)) + : DEFAULT_RETAIN_DAYS, + maxTurnFilesPerAgent: Number.isFinite(retentionCfg?.max_turn_files_per_agent) + ? Math.max(100, Number(retentionCfg?.max_turn_files_per_agent)) + : DEFAULT_MAX_TURN_FILES_PER_AGENT, + cleanupIntervalSeconds: Number.isFinite(retentionCfg?.cleanup_interval_seconds) + ? Math.max(30, Number(retentionCfg?.cleanup_interval_seconds)) + : DEFAULT_CLEANUP_INTERVAL_SECONDS, + }; +} + +function logQmdDebug(agentId: string, qmdCfg: QmdConfig, stage: string, details: string): void { + if (!qmdCfg.debugLogging) { + return; + } + log('INFO', `Memory debug @${agentId} [${stage}]: ${details}`); +} + +function logThrottled( + key: string, + cache: Map, + intervalMs: number, + level: 'INFO' | 'WARN', + message: string +): void { + const now = Date.now(); + const last = cache.get(key) || 0; + if (now - last < intervalMs) { + return; + } + cache.set(key, now); + log(level, message); +} + +function warnUnsafeFallback(agentId: string, reason: string): void { + logThrottled( + agentId, + lastUnsafeFallbackWarnMsByAgent, + QMD_FALLBACK_WARN_INTERVAL_MS, + 'WARN', + `QMD vsearch fallback for @${agentId}: ${reason}` + ); +} + +function infoVsearchExperimental(agentId: string): void { + logThrottled( + agentId, + lastVsearchExperimentalInfoMsByAgent, + VSEARCH_EXPERIMENTAL_INFO_INTERVAL_MS, + 'INFO', + `QMD vsearch is experimental for @${agentId}.` + ); +} + +function sanitizeId(raw: string): string { + return raw.toLowerCase().replace(/[^a-z0-9_-]/g, '-'); +} + +function getAgentTurnsDir(agentId: string): string { + return path.join(MEMORY_TURNS_DIR, sanitizeId(agentId)); +} + +function getCollectionName(agentId: string): string { + return `tinyclaw-${sanitizeId(agentId)}`; +} + +function ensureDir(dir: string): void { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } +} + +function runCommand( + command: string, + args: string[], + cwd?: string, + timeoutMs = 12000, + env?: Record +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: env ? { ...process.env, ...env } : process.env, + }); + + let stdout = ''; + let stderr = ''; + let timedOut = false; + + const timer = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, timeoutMs); + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => { stdout += chunk; }); + child.stderr.on('data', (chunk: string) => { stderr += chunk; }); + child.on('error', (error) => { + clearTimeout(timer); + reject(error); + }); + child.on('close', (code) => { + clearTimeout(timer); + if (timedOut) { + reject(new Error(`Command timed out after ${timeoutMs}ms`)); + return; + } + if (code === 0) { + resolve({ stdout, stderr }); + return; + } + reject(new Error(stderr.trim() || `Command exited with code ${code}`)); + }); + }); +} + +async function isQmdAvailable(preferredCommand?: string): Promise { + const key = preferredCommand || '__auto__'; + if (qmdChecked && qmdCheckKey === key) { + return qmdAvailable; + } + + qmdChecked = true; + qmdCheckKey = key; + qmdAvailable = false; + qmdCommandPath = null; + + const bundledQmd = path.join(require('os').homedir(), '.bun/bin/qmd'); + const candidates = preferredCommand ? [preferredCommand] : [bundledQmd, 'qmd']; + + try { + for (const candidate of candidates) { + try { + await runCommand(candidate, ['--help'], undefined, 5000); + qmdCommandPath = candidate; + qmdAvailable = true; + break; + } catch { + // Try next candidate. + } + } + } finally { + if (!qmdAvailable) { + qmdCommandPath = null; + } + } + + return qmdAvailable; +} + +function isDisableExpansionPatchedQmd(commandPath: string | null): boolean { + if (!commandPath) { + return false; + } + + const bunGlobalQmd = path.join(require('os').homedir(), '.bun/bin/qmd'); + const patchedStoreTs = path.join(require('os').homedir(), '.bun/install/global/node_modules/qmd/src/store.ts'); + if (commandPath !== bunGlobalQmd || !fs.existsSync(patchedStoreTs)) { + return false; + } + + try { + const src = fs.readFileSync(patchedStoreTs, 'utf8'); + return src.includes('QMD_VSEARCH_DISABLE_EXPANSION'); + } catch { + return false; + } +} + +function isDisableExpansionSupported(): boolean { + const key = qmdCommandPath || '__unknown__'; + if (qmdDisableExpansionCheckKey === key) { + return qmdDisableExpansionSupported; + } + + qmdDisableExpansionCheckKey = key; + qmdDisableExpansionSupported = isDisableExpansionPatchedQmd(qmdCommandPath); + return qmdDisableExpansionSupported; +} + +function shouldUseMemoryForChannel(channel: string): boolean { + return MEMORY_ELIGIBLE_CHANNELS.has(channel); +} + +async function ensureCollection(agentId: string): Promise { + ensureDir(MEMORY_ROOT); + const agentTurnsDir = getAgentTurnsDir(agentId); + ensureDir(agentTurnsDir); + + const collectionName = getCollectionName(agentId); + if (!collectionPrepared.has(collectionName)) { + try { + await runCommand(qmdCommandPath || 'qmd', ['collection', 'add', agentTurnsDir, '--name', collectionName, '--mask', '**/*.md'], undefined, 10000); + collectionPrepared.add(collectionName); + } catch (error) { + const msg = (error as Error).message.toLowerCase(); + if (msg.includes('already') || msg.includes('exists')) { + collectionPrepared.add(collectionName); + } else { + throw error; + } + } + } + + return collectionName; +} + +async function maybeUpdateCollection(collectionName: string, updateIntervalSeconds: number): Promise { + const now = Date.now(); + const last = lastCollectionUpdateMs.get(collectionName) || 0; + if (now - last < updateIntervalSeconds * 1000) { + return; + } + await runCommand(qmdCommandPath || 'qmd', ['update', '--collections', collectionName], undefined, 15000); + lastCollectionUpdateMs.set(collectionName, now); +} + +async function maybeEmbedCollection(collectionName: string, embedIntervalSeconds: number): Promise { + const now = Date.now(); + const last = lastCollectionEmbedMs.get(collectionName) || 0; + if (now - last < embedIntervalSeconds * 1000) { + return; + } + // Apply backoff from the trigger time to avoid tight retries on repeated failures. + lastCollectionEmbedMs.set(collectionName, now); + await runCommand(qmdCommandPath || 'qmd', ['embed', '--collections', collectionName], undefined, 30000); +} + +function triggerEmbedCollectionAsync(agentId: string, collectionName: string, qmdCfg: QmdConfig): void { + if (embedInFlightByCollection.has(collectionName)) { + logQmdDebug(agentId, qmdCfg, 'embed', 'skip trigger (in-flight)'); + return; + } + embedInFlightByCollection.add(collectionName); + void maybeEmbedCollection(collectionName, qmdCfg.embedIntervalSeconds) + .then(() => { + logQmdDebug(agentId, qmdCfg, 'embed', `triggered interval=${qmdCfg.embedIntervalSeconds}s`); + }) + .catch((error) => { + log('WARN', `Memory embed skipped for @${agentId}: ${(error as Error).message}`); + logQmdDebug(agentId, qmdCfg, 'embed', 'failed; continuing with existing vectors'); + }) + .finally(() => { + embedInFlightByCollection.delete(collectionName); + }); +} + +function buildLexicalQueryVariants(message: string): string[] { + const variants: string[] = []; + const push = (value: string) => { + const cleaned = value.trim().replace(/\s+/g, ' '); + if (!cleaned) { + return; + } + if (!variants.includes(cleaned)) { + variants.push(cleaned); + } + }; + + push(message); + + const noPunct = message.replace(/[?？!！,，.。;οΌ›:：]/g, ' '); + push(noPunct); + + // Chinese question-particle normalization to reduce BM25 false negatives. + const zhSimplified = noPunct + .replace(/ζ˜―δ»€δΉˆ|是ε•₯|δ»€δΉˆ|ε€šε°‘|ε‡ η‚Ή|ε“ͺι‡Œ|ε“ͺε„Ώ|ε“ͺδΈͺ|ε“ͺ位|谁|吗|ε‘’|ζ₯着/g, ' ') + .replace(/\s+/g, ' '); + push(zhSimplified); + + // English question-word normalization. + const enSimplified = noPunct + .replace(/\b(what|which|who|where|when|why|how)\b/gi, ' ') + .replace(/\s+/g, ' '); + push(enSimplified); + + // Code-friendly variant: treat hyphen as delimiter. + push(noPunct.replace(/-/g, ' ')); + + return variants; +} + +function parseQmdResults(raw: string): QmdResult[] { + const trimmed = raw.trim(); + if (!trimmed) { + return []; + } + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return []; + } + + const rows = Array.isArray(parsed) + ? parsed + : (parsed as { results?: unknown[] }).results || []; + + const results: QmdResult[] = []; + for (const row of rows) { + if (!row || typeof row !== 'object') { + continue; + } + const r = row as Record; + const score = typeof r.score === 'number' ? r.score : 0; + const snippet = String(r.snippet || r.context || r.text || r.content || '').trim(); + const source = String(r.path || r.file || r.source || r.title || '').trim(); + if (!snippet) { + continue; + } + results.push({ score, snippet, source }); + } + return results; +} + +function normalizeInline(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +function normalizeQueryKey(text: string): string { + return text + .toLowerCase() + .replace(/[^\p{L}\p{N}\s]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeFilenameKey(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function truncateInline(text: string, max: number): string { + if (text.length <= max) { + return text; + } + return `${text.slice(0, max)}...`; +} + +function summarizeSnippetForLog(text: string, max = 120): string { + const oneLine = normalizeInline(text).replace(/\n/g, ' '); + return oneLine.length <= max ? oneLine : `${oneLine.slice(0, max)}...`; +} + +function parseTurnSections(content: string): TurnSections { + const userMarker = '\n## User\n'; + const assistantMarker = '\n## Assistant\n'; + const userPos = content.indexOf(userMarker); + const assistantPos = content.indexOf(assistantMarker); + if (userPos < 0 || assistantPos < 0 || assistantPos <= userPos) { + return { user: '', assistant: '' }; + } + const userStart = userPos + userMarker.length; + const userText = content.slice(userStart, assistantPos).trim(); + const assistantStart = assistantPos + assistantMarker.length; + const assistantText = content.slice(assistantStart).trim(); + return { user: userText, assistant: assistantText }; +} + +function loadTurnSectionsFromSource(source: string, agentId: string): TurnSections | null { + const m = source.match(/^qmd:\/\/[^/]+\/(.+)$/); + if (!m) { + return null; + } + const rel = decodeURIComponent(m[1]); + const agentDir = getAgentTurnsDir(agentId); + let fullPath = path.join(agentDir, rel); + if (!fs.existsSync(fullPath)) { + // qmd may normalize source path (e.g. casing and punctuation like "_" -> "-"). + // Resolve with a tolerant fallback so we can hydrate turn sections reliably. + const wanted = normalizeFilenameKey(path.basename(rel)); + try { + const found = fs.readdirSync(agentDir).find(name => normalizeFilenameKey(name) === wanted); + if (found) { + fullPath = path.join(agentDir, found); + } + } catch { + return null; + } + } + if (!fs.existsSync(fullPath)) { + return null; + } + try { + const content = fs.readFileSync(fullPath, 'utf8'); + return parseTurnSections(content); + } catch { + return null; + } +} + +function rerankAndHydrateResults(results: QmdResult[], message: string, agentId: string, settings?: Settings): QmdResult[] { + if (results.length === 0) { + return results; + } + const terms = Array.from(new Set((message.toLowerCase().match(/[a-z0-9_-]{2,}|[\u4e00-\u9fff]{1,3}/g) || []))); + const rerankOptions = resolveMemoryRerankOptions(settings); + const normalizedMessage = normalizeQueryKey(message); + + return results + .map((result) => { + let score = result.score; + let snippet = result.snippet; + + const sections = loadTurnSectionsFromSource(result.source, agentId); + if (sections && sections.assistant) { + const user = normalizeInline(sections.user); + const assistant = normalizeInline(sections.assistant); + if (assistant) { + snippet = `User: ${truncateInline(user, 180)}\nAssistant: ${truncateInline(assistant, 260)}`; + } + + score += computeHeuristicScoreDelta(user, assistant, terms, rerankOptions); + const lowConfidence = isLowConfidenceText(assistant, rerankOptions); + if (lowConfidence) { + return null; + } + } + + return { score, snippet, source: result.source }; + }) + .filter((row): row is QmdResult => !!row) + .sort((a, b) => b.score - a.score); +} + +async function quickHasLexicalHit( + variants: string[], + collectionName: string, + qmdCfg: QmdConfig +): Promise { + for (const query of variants) { + const args = ['search', query, '--json', '-c', collectionName, '-n', '1', '--min-score', String(qmdCfg.minScore)]; + const { stdout } = await runCommand(qmdCommandPath || 'qmd', args, undefined, qmdCfg.precheckTimeoutMs); + const hits = parseQmdResults(stdout); + if (hits.length > 0) { + return true; + } + } + return false; +} + +function formatMemoryPrompt(results: QmdResult[], maxChars: number): string { + if (results.length === 0) { + return ''; + } + + const blocks: string[] = []; + let usedChars = 0; + + for (let i = 0; i < results.length; i++) { + const result = results[i]; + const block = [ + `Snippet ${i + 1} (score=${result.score.toFixed(3)}):`, + result.source ? `Source: ${result.source}` : 'Source: unknown', + result.snippet, + ].join('\n'); + + if (usedChars + block.length > maxChars) { + break; + } + blocks.push(block); + usedChars += block.length; + } + + if (blocks.length === 0) { + return ''; + } + + return [ + '', + '---', + 'Retrieved memory snippets (from past conversations):', + 'Use only if relevant. Prioritize current user instructions over old memory.', + '', + blocks.join('\n\n'), + ].join('\n'); +} + +function resolveRetrievalMode(agentId: string, qmdCfg: QmdConfig): { useVsearch: boolean; label: 'qmd-bm25' | 'qmd-vsearch' } { + let useVsearch = qmdCfg.useSemanticSearch; + if (useVsearch && !qmdCfg.allowUnsafeVsearch) { + if (!qmdCfg.disableQueryExpansion) { + useVsearch = false; + warnUnsafeFallback(agentId, 'disable_query_expansion is false; using BM25. Set memory.qmd.allow_unsafe_vsearch=true to override.'); + } else if (!isDisableExpansionSupported()) { + useVsearch = false; + warnUnsafeFallback(agentId, 'safe vsearch support not detected; using BM25. Run scripts/patch-qmd-no-expansion.sh or set memory.qmd.allow_unsafe_vsearch=true.'); + } + } + if (useVsearch) { + infoVsearchExperimental(agentId); + } + + return { + useVsearch, + label: useVsearch ? 'qmd-vsearch' : 'qmd-bm25', + }; +} + +async function runBm25WithVariants( + variants: string[], + fallbackQuery: string, + collectionName: string, + qmdCfg: QmdConfig +): Promise { + let lastQuery = fallbackQuery; + for (const query of variants) { + lastQuery = query; + const args = ['search', query, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; + const { stdout } = await runCommand(qmdCommandPath || 'qmd', args, undefined, qmdCfg.searchTimeoutMs); + const results = parseQmdResults(stdout); + if (results.length > 0) { + return { results, query }; + } + } + return { results: [], query: lastQuery }; +} + +export async function buildMemoryBlock( + agentId: string, + message: string, + settings: Settings, + sourceChannel: string +): Promise { + const qmdCfg = getQmdConfig(settings); + if (!qmdCfg.enabled) { + return ''; + } + if (!shouldUseMemoryForChannel(sourceChannel)) { + return ''; + } + + const hasQmd = await isQmdAvailable(qmdCfg.command); + if (!hasQmd) { + if (!qmdUnavailableLoggedByAgent.get(agentId)) { + log('WARN', `qmd not found in PATH, memory retrieval disabled for @${agentId}`); + qmdUnavailableLoggedByAgent.set(agentId, true); + } + log('INFO', `Memory source for @${agentId}: none (qmd unavailable)`); + return ''; + } + qmdUnavailableLoggedByAgent.delete(agentId); + logQmdDebug(agentId, qmdCfg, 'qmd', `command=${qmdCommandPath || 'qmd'}`); + + try { + const collectionName = await ensureCollection(agentId); + logQmdDebug(agentId, qmdCfg, 'collection', `name=${collectionName}`); + await maybeUpdateCollection(collectionName, qmdCfg.updateIntervalSeconds); + logQmdDebug(agentId, qmdCfg, 'update', `interval=${qmdCfg.updateIntervalSeconds}s`); + + const mode = resolveRetrievalMode(agentId, qmdCfg); + const queryVariants = buildLexicalQueryVariants(message); + + // Only keep precheck for vsearch path. For BM25 it duplicates work and adds latency. + if (qmdCfg.quickPrecheckEnabled && mode.useVsearch) { + try { + logQmdDebug( + agentId, + qmdCfg, + 'precheck', + `cmd=search timeout=${qmdCfg.precheckTimeoutMs}ms min_score=${qmdCfg.minScore} variants=${queryVariants.length}` + ); + const hasQuickHit = await quickHasLexicalHit(queryVariants, collectionName, qmdCfg); + if (!hasQuickHit) { + log('INFO', `Memory source for @${agentId}: none (qmd precheck no-hit)`); + return ''; + } + } catch (error) { + log('WARN', `Memory quick precheck skipped for @${agentId}: ${(error as Error).message}`); + log('INFO', `Memory source for @${agentId}: none (qmd precheck error)`); + return ''; + } + } else if (qmdCfg.quickPrecheckEnabled && !mode.useVsearch) { + logQmdDebug(agentId, qmdCfg, 'precheck', 'skipped for bm25 mode (avoid duplicate searches)'); + } + + if (mode.useVsearch) { + triggerEmbedCollectionAsync(agentId, collectionName, qmdCfg); + } + const queryArgs = mode.useVsearch + ? ['vsearch', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)] + : ['search', message, '--json', '-c', collectionName, '-n', String(qmdCfg.topK), '--min-score', String(qmdCfg.minScore)]; + const queryEnv = mode.useVsearch && qmdCfg.disableQueryExpansion + ? { QMD_VSEARCH_DISABLE_EXPANSION: '1' } + : undefined; + const queryTimeoutMs = mode.useVsearch ? qmdCfg.vectorSearchTimeoutMs : qmdCfg.searchTimeoutMs; + logQmdDebug( + agentId, + qmdCfg, + 'query', + `mode=${mode.label} timeout=${queryTimeoutMs}ms top_k=${qmdCfg.topK} min_score=${qmdCfg.minScore} disable_expansion=${qmdCfg.disableQueryExpansion}` + ); + + const queryResult = mode.useVsearch + ? (() => runCommand(qmdCommandPath || 'qmd', queryArgs, undefined, queryTimeoutMs, queryEnv).then(({ stdout }) => ({ + results: parseQmdResults(stdout), + query: message, + })))() + : runBm25WithVariants(queryVariants, message, collectionName, qmdCfg); + let { results, query } = await queryResult; + logQmdDebug(agentId, qmdCfg, 'query-used', `mode=${mode.label} query=\"${query}\"`); + if (results.length === 0) { + log('INFO', `Memory source for @${agentId}: none (${mode.label} no-hit)`); + return ''; + } + + let rankedResults = rerankAndHydrateResults(results, message, agentId, settings); + logQmdDebug( + agentId, + qmdCfg, + 'rerank', + `raw=${results.length} ranked=${rankedResults.length} top=${rankedResults + .slice(0, 3) + .map((r, i) => `${i + 1}:${r.score.toFixed(3)}:${summarizeSnippetForLog(r.snippet, 90)}`) + .join(' | ')}` + ); + if (mode.useVsearch && rankedResults.length === 0) { + logQmdDebug(agentId, qmdCfg, 'query', 'vsearch results filtered out; fallback=bm25'); + const bm25Fallback = await runBm25WithVariants(queryVariants, message, collectionName, qmdCfg); + results = bm25Fallback.results; + query = bm25Fallback.query; + logQmdDebug(agentId, qmdCfg, 'query-used', `mode=qmd-bm25-fallback query=\"${query}\"`); + if (results.length === 0) { + log('INFO', `Memory source for @${agentId}: none (qmd-vsearch filtered + bm25 no-hit)`); + return ''; + } + rankedResults = rerankAndHydrateResults(results, message, agentId, settings); + logQmdDebug( + agentId, + qmdCfg, + 'rerank', + `fallback raw=${results.length} ranked=${rankedResults.length} top=${rankedResults + .slice(0, 3) + .map((r, i) => `${i + 1}:${r.score.toFixed(3)}:${summarizeSnippetForLog(r.snippet, 90)}`) + .join(' | ')}` + ); + if (rankedResults.length === 0) { + log('INFO', `Memory source for @${agentId}: none (qmd-vsearch filtered + bm25 filtered)`); + return ''; + } + } + + const memoryBlock = formatMemoryPrompt(rankedResults, qmdCfg.maxChars); + if (!memoryBlock) { + log('INFO', `Memory source for @${agentId}: none (${mode.label} no-usable-snippet)`); + return ''; + } + + log('INFO', `Memory retrieval hit for @${agentId}: ${rankedResults.length} snippet(s) via ${mode.label}`); + log('INFO', `Memory source for @${agentId}: ${mode.label}`); + return memoryBlock; + } catch (error) { + log('WARN', `Memory retrieval skipped for @${agentId}: ${(error as Error).message}`); + log('INFO', `Memory source for @${agentId}: none (qmd error)`); + return ''; + } +} + +export async function enrichMessageWithMemory( + agentId: string, + message: string, + settings: Settings, + sourceChannel: string +): Promise { + const memoryBlock = await buildMemoryBlock(agentId, message, settings, sourceChannel); + return memoryBlock ? `${message}${memoryBlock}` : message; +} + +function timestampFilename(ts: number): string { + return new Date(ts).toISOString().replace(/[:.]/g, '-').toLowerCase(); +} + +function truncate(text: string, max = 16000): string { + if (text.length <= max) { + return text; + } + return `${text.substring(0, max)}\n\n[truncated]`; +} + +function maybeCleanupTurns(agentId: string, settings?: Settings): void { + const retention = getRetentionConfig(settings); + if (!retention.enabled) { + return; + } + + const now = Date.now(); + const last = lastTurnsCleanupMs.get(agentId) || 0; + if (now - last < retention.cleanupIntervalSeconds * 1000) { + return; + } + lastTurnsCleanupMs.set(agentId, now); + + const dir = getAgentTurnsDir(agentId); + if (!fs.existsSync(dir)) { + return; + } + + let entries = fs.readdirSync(dir) + .filter(name => name.endsWith('.md')) + .map(name => { + const filePath = path.join(dir, name); + const stat = fs.statSync(filePath); + return { + name, + filePath, + mtimeMs: stat.mtimeMs, + }; + }) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + + if (entries.length === 0) { + return; + } + + const cutoffMs = now - retention.retainDays * 24 * 60 * 60 * 1000; + const keep: typeof entries = []; + const remove: typeof entries = []; + + for (const e of entries) { + if (e.mtimeMs < cutoffMs) { + remove.push(e); + } else { + keep.push(e); + } + } + + if (keep.length > retention.maxTurnFilesPerAgent) { + const overflow = keep.slice(retention.maxTurnFilesPerAgent); + remove.push(...overflow); + keep.splice(retention.maxTurnFilesPerAgent); + } + + if (remove.length === 0) { + return; + } + + let deleted = 0; + for (const e of remove) { + try { + fs.unlinkSync(e.filePath); + deleted++; + } catch { + // Keep going; best-effort cleanup. + } + } + if (deleted > 0) { + log('INFO', `Memory turns cleanup for @${agentId}: deleted ${deleted} file(s), kept ${keep.length}`); + } +} + +export async function saveTurnToMemory(params: { + agentId: string; + agent: AgentConfig; + channel: string; + sender: string; + messageId: string; + userMessage: string; + agentResponse: string; + timestampMs?: number; + settings?: Settings; +}): Promise { + try { + const timestampMs = params.timestampMs || Date.now(); + const dir = getAgentTurnsDir(params.agentId); + ensureDir(dir); + + const fileName = `${timestampFilename(timestampMs)}-${params.messageId}.md`; + const filePath = path.join(dir, fileName); + const lines = [ + `# Turn for @${params.agentId} (${params.agent.name})`, + '', + `- Timestamp: ${new Date(timestampMs).toISOString()}`, + `- Channel: ${params.channel}`, + `- Sender: ${params.sender}`, + `- Message ID: ${params.messageId}`, + '', + '## User', + '', + truncate(params.userMessage), + '', + '## Assistant', + '', + truncate(params.agentResponse), + '', + ]; + + fs.writeFileSync(filePath, lines.join('\n')); + maybeCleanupTurns(params.agentId, params.settings); + } catch (error) { + log('WARN', `Failed to persist memory turn for @${params.agentId}: ${(error as Error).message}`); + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 1a20267..0f3d67c 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -41,6 +41,41 @@ export interface Settings { monitoring?: { heartbeat_interval?: number; }; + memory?: { + enabled?: boolean; + rerank?: { + enabled?: boolean; + answer_signal_patterns?: string[]; + low_confidence_patterns?: string[]; + answer_signal_bonus?: number; + low_confidence_penalty?: number; + code_pattern_bonus?: number; + term_hit_bonus?: number; + }; + retention?: { + enabled?: boolean; + retain_days?: number; + max_turn_files_per_agent?: number; + cleanup_interval_seconds?: number; + }; + qmd?: { + enabled?: boolean; + command?: string; + top_k?: number; + min_score?: number; + max_chars?: number; + update_interval_seconds?: number; + embed_interval_seconds?: number; + use_semantic_search?: boolean; + disable_query_expansion?: boolean; + allow_unsafe_vsearch?: boolean; + quick_precheck_enabled?: boolean; + precheck_timeout_ms?: number; + search_timeout_ms?: number; + vector_search_timeout_ms?: number; + debug_logging?: boolean; + }; + }; } export interface MessageData { diff --git a/src/queue-processor.ts b/src/queue-processor.ts index 1844d6a..70fd494 100644 --- a/src/queue-processor.ts +++ b/src/queue-processor.ts @@ -21,6 +21,11 @@ import { import { log, emitEvent } from './lib/logging'; import { parseAgentRouting, findTeamForAgent, getAgentResetFlag, extractTeammateMentions } from './lib/routing'; import { invokeAgent } from './lib/invoke'; +import { saveTurnToMemory, MEMORY_ELIGIBLE_CHANNELS } from './lib/memory'; + +function shouldPersistMemoryTurn(channel: string): boolean { + return MEMORY_ELIGIBLE_CHANNELS.has(channel); +} // Ensure directories exist [QUEUE_INCOMING, QUEUE_OUTGOING, QUEUE_PROCESSING, path.dirname(LOG_FILE)].forEach(dir => { @@ -138,7 +143,20 @@ async function processMessage(messageFile: string): Promise { if (!teamContext) { // No team context β€” single agent invocation (backward compatible) try { - finalResponse = await invokeAgent(agent, agentId, message, workspacePath, shouldReset, agents, teams); + finalResponse = await invokeAgent(agent, agentId, message, channel, workspacePath, shouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId, + agent, + channel, + sender, + messageId, + userMessage: message, + agentResponse: finalResponse, + timestampMs: Date.now(), + settings, + }); + } } catch (error) { const provider = agent.provider || 'anthropic'; log('ERROR', `${provider === 'openai' ? 'Codex' : 'Claude'} error (agent: ${agentId}): ${(error as Error).message}`); @@ -176,7 +194,20 @@ async function processMessage(messageFile: string): Promise { let stepResponse: string; try { - stepResponse = await invokeAgent(currentAgent, currentAgentId, currentMessage, workspacePath, currentShouldReset, agents, teams); + stepResponse = await invokeAgent(currentAgent, currentAgentId, currentMessage, channel, workspacePath, currentShouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId: currentAgentId, + agent: currentAgent, + channel, + sender, + messageId, + userMessage: currentMessage, + agentResponse: stepResponse, + timestampMs: Date.now(), + settings, + }); + } } catch (error) { const provider = currentAgent.provider || 'anthropic'; log('ERROR', `${provider === 'openai' ? 'Codex' : 'Claude'} error (agent: ${currentAgentId}): ${(error as Error).message}`); @@ -236,7 +267,20 @@ async function processMessage(messageFile: string): Promise { let mResponse: string; try { const mMessage = `[Message from teammate @${currentAgentId}]:\n${mention.message}`; - mResponse = await invokeAgent(mAgent, mention.teammateId, mMessage, workspacePath, mShouldReset, agents, teams); + mResponse = await invokeAgent(mAgent, mention.teammateId, mMessage, channel, workspacePath, mShouldReset, settings, agents, teams); + if (shouldPersistMemoryTurn(channel)) { + await saveTurnToMemory({ + agentId: mention.teammateId, + agent: mAgent, + channel, + sender, + messageId, + userMessage: mMessage, + agentResponse: mResponse, + timestampMs: Date.now(), + settings, + }); + } } catch (error) { log('ERROR', `Fan-out error (agent: ${mention.teammateId}): ${(error as Error).message}`); mResponse = "Sorry, I encountered an error processing this request."; diff --git a/tinyclaw.sh b/tinyclaw.sh index 895cf43..738941b 100755 --- a/tinyclaw.sh +++ b/tinyclaw.sh @@ -21,6 +21,7 @@ if [ -f "$SCRIPT_DIR/.tinyclaw/settings.json" ]; then else SETTINGS_FILE="$HOME/.tinyclaw/settings.json" fi +RESET_FLAG_FILE="$(dirname "$SETTINGS_FILE")/reset_flag" mkdir -p "$LOG_DIR" @@ -63,7 +64,7 @@ case "${1:-}" in ;; reset) echo -e "${YELLOW}Resetting conversation...${NC}" - touch "$SCRIPT_DIR/.tinyclaw/reset_flag" + touch "$RESET_FLAG_FILE" echo -e "${GREEN}βœ“ Reset flag set${NC}" echo "" echo "The next message will start a fresh conversation (without -c)."