From da866110d4501e2ba4743ab5dc880c9c5f00b059 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:23:04 +0100 Subject: [PATCH 01/37] support lumo v2 content blocks --- src/api/tools/native-tool-call-processor.ts | 28 +++++++------- src/conversations/fallback/store.ts | 8 ++-- .../fallback/sync/sync-service.ts | 6 +-- src/conversations/store.ts | 6 +-- src/conversations/types.ts | 18 +++------ src/lumo-client/client.ts | 37 +++++++++++++------ src/lumo-client/types.ts | 8 ++-- tests/unit/conversation-store.test.ts | 15 +++++--- tests/unit/native-tool-call-processor.test.ts | 7 +++- tests/unit/primary-conversation-store.test.ts | 17 +++++---- 10 files changed, 83 insertions(+), 67 deletions(-) diff --git a/src/api/tools/native-tool-call-processor.ts b/src/api/tools/native-tool-call-processor.ts index db04acd..be1d660 100644 --- a/src/api/tools/native-tool-call-processor.ts +++ b/src/api/tools/native-tool-call-processor.ts @@ -8,6 +8,7 @@ * * This processor: * - Parses streaming JSON via JsonBraceTracker + * - Builds ContentBlock[] for interleaved tool calls/results * - Detects misrouted custom tools (custom tools Lumo mistakenly routed through native pipeline) * - Tracks success/failure metrics */ @@ -18,6 +19,8 @@ import { getCustomToolsConfig } from '../../app/config.js'; import { getMetrics } from '../../app/metrics.js'; import { logger } from '../../app/logger.js'; import type { ParsedToolCall } from './types.js'; +import type { ContentBlock } from '@lumo/types.js'; +import { setToolCallInBlocks, setToolResultInBlocks } from '@lumo/messageHelpers.js'; const KNOWN_NATIVE_TOOLS = new Set([ 'proton_info', 'web_search', 'weather', 'stock', 'cryptocurrency' @@ -74,9 +77,10 @@ function isErrorResult(json: string): boolean { // ── Exported types and class ───────────────────────────────────────── export interface NativeToolCallResult { + /** ContentBlocks for tool calls/results (may be empty) */ + blocks: ContentBlock[]; + /** First tool call detected (for bounce handling) */ toolCall: ParsedToolCall | undefined; - /** Raw tool_result JSON string from SSE (if any) */ - toolResult: string | undefined; failed: boolean; /** True if a misrouted custom tool was detected */ misrouted: boolean; @@ -84,13 +88,13 @@ export interface NativeToolCallResult { /** * Processes native tool calls from Lumo's SSE tool_call/tool_result targets. - * Detects misrouted custom tools and tracks metrics. + * Builds ContentBlocks and detects misrouted custom tools. */ export class NativeToolCallProcessor { private toolCallTracker = new JsonBraceTracker(); private toolResultTracker = new JsonBraceTracker(); + private blocks: ContentBlock[] = []; private firstToolCall: ParsedToolCall | null = null; - private firstToolResult: string | null = null; private failed = false; private _misrouted = false; @@ -105,7 +109,7 @@ export class NativeToolCallProcessor { const toolCall = parseToolCallJson(json); if (!toolCall) continue; - // Save first for result (used by bounce logic) + // Save first for bounce logic if (!this.firstToolCall) { this.firstToolCall = toolCall; } @@ -118,14 +122,13 @@ export class NativeToolCallProcessor { logger.debug({ tool: toolCall.name, isBounce: this.isBounce }, 'Misrouted tool call detected'); // Only abort on first misroute in non-bounce mode. - // Note: This means we may undercount if Lumo queues multiple misrouted calls - // in one response. The bounce response will count any subsequent retries. if (!this.isBounce && toolCall === this.firstToolCall) { this._misrouted = true; return true; } } else { - // Native tool - no success/failed distinction (unreliable) + // Native tool - add to blocks + this.blocks = setToolCallInBlocks(this.blocks, json); getMetrics()?.toolCallsTotal.inc({ type: 'native', status: 'detected', tool_name: toolCall.name }); @@ -139,10 +142,9 @@ export class NativeToolCallProcessor { feedToolResult(content: string): void { for (const json of this.toolResultTracker.feed(content)) { logger.debug({ raw: json }, 'Native SSE tool_result'); - // Store first tool result for persistence - if (!this.firstToolResult) { - this.firstToolResult = json; - } + // Add to blocks + this.blocks = setToolResultInBlocks(this.blocks, json); + // Track failure status if (this.firstToolCall && !this.failed && isErrorResult(json)) { this.failed = true; } @@ -157,8 +159,8 @@ export class NativeToolCallProcessor { /** Get the result after stream completes. */ getResult(): NativeToolCallResult { return { + blocks: this.blocks, toolCall: this.firstToolCall ?? undefined, - toolResult: this.firstToolResult ?? undefined, failed: this.failed, misrouted: this._misrouted, }; diff --git a/src/conversations/fallback/store.ts b/src/conversations/fallback/store.ts index a661667..f242529 100644 --- a/src/conversations/fallback/store.ts +++ b/src/conversations/fallback/store.ts @@ -187,7 +187,7 @@ export class FallbackStore { * Append an assistant response to a conversation. * * @param id - Conversation ID - * @param messageData - Assistant message data (content, optional toolCall/toolResult) + * @param messageData - Assistant message data (content, optional blocks) * @param status - Message status (default: succeeded) * @param semanticId - Optional semantic ID for deduplication * @returns The created message @@ -213,8 +213,7 @@ export class FallbackStore { parentId, status, content: messageData.content, - toolCall: messageData.toolCall, - toolResult: messageData.toolResult, + blocks: messageData.blocks, semanticId: semanticId ?? hashMessage(Role.Assistant, messageData.content).slice(0, 16), }; @@ -229,8 +228,7 @@ export class FallbackStore { conversationId: id, messageId: message.id, contentLength: messageData.content.length, - hasToolCall: !!messageData.toolCall, - hasToolResult: !!messageData.toolResult, + hasBlocks: !!messageData.blocks?.length, }, 'Appended assistant response'); return message; diff --git a/src/conversations/fallback/sync/sync-service.ts b/src/conversations/fallback/sync/sync-service.ts index 61eaa48..13b05bc 100644 --- a/src/conversations/fallback/sync/sync-service.ts +++ b/src/conversations/fallback/sync/sync-service.ts @@ -263,8 +263,7 @@ export class SyncService { const messagePrivate: MessagePrivate = { content: contentToStore, context: message.context, - toolCall: message.toolCall, - toolResult: message.toolResult, + blocks: message.blocks, }; const encryptedPrivate = await codec.encryptMessage(messagePrivate, message, effectiveParentId); @@ -375,8 +374,7 @@ export class SyncService { status: msg.status as Status | undefined, content: messagePrivate?.content, context: messagePrivate?.context, - toolCall: messagePrivate?.toolCall, - toolResult: messagePrivate?.toolResult, + blocks: messagePrivate?.blocks, }); } diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 12c88a0..6aaf9f7 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -368,8 +368,7 @@ export class ConversationStore { parentId, status, content: messageData.content, - toolCall: messageData.toolCall, - toolResult: messageData.toolResult, + blocks: messageData.blocks, semanticId: effectiveSemanticId, }; @@ -379,8 +378,7 @@ export class ConversationStore { conversationId: id, messageId, contentLength: messageData.content.length, - hasToolCall: !!messageData.toolCall, - hasToolResult: !!messageData.toolResult, + hasBlocks: !!messageData.blocks?.length, }, 'Appended assistant response'); return message; diff --git a/src/conversations/types.ts b/src/conversations/types.ts index 6ef320a..b03f759 100644 --- a/src/conversations/types.ts +++ b/src/conversations/types.ts @@ -4,12 +4,12 @@ */ // Import types from upstream @lumo -import type { ConversationId, MessageId, SpaceId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub, Role } from '@lumo/types.js'; +import type { ConversationId, MessageId, SpaceId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub, Role, ContentBlock } from '@lumo/types.js'; import { ConversationStatus } from '@lumo/types.js'; import type { RemoteId } from '@lumo/remote/types.ts'; // Re-export types for consumers -export type { ConversationId, MessageId, SpaceId, RemoteId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub }; +export type { ConversationId, MessageId, SpaceId, RemoteId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub, ContentBlock }; /** * Full conversation record @@ -24,25 +24,19 @@ export interface Conversation extends ConversationPub { * * Content is optional to match Proton's model where tool_call/tool_result * messages may have no content (just toolCall/toolResult fields). - * Currently we serialize everything to content, but this allows future - * parity with WebClient's native tool storage. * * WebClient also has: attachments?: ShallowAttachment[], contextFiles?: AttachmentId[] * We don't handle attachments yet. * * Tool calls (native tools like web_search, weather): - * - Legacy: Single tool call stored in `toolCall` (JSON string) and `toolResult` (JSON string), - * with the synthesized response in `content`. All in the same assistant message. - * - v2: Multiple/interleaved tool calls use `blocks?: ContentBlock[]` where ContentBlock is - * TextBlock | ToolCallBlock | ToolResultBlock. We don't support this yet - would need to - * check if the API returns this format or if it requires a different endpoint. + * Use `blocks?: ContentBlock[]` for interleaved text/tool_call/tool_result blocks. + * The upstream `getMessageBlocks()` helper reconstructs blocks from legacy + * toolCall/toolResult fields if blocks is not present (for old synced data). */ export interface MessagePrivate { content?: string; context?: string; - toolCall?: string; // JSON string of tool call (legacy, single tool) - toolResult?: string; // JSON string of tool result (legacy, single tool) - // blocks?: ContentBlock[]; // v2: interleaved text/tool_call/tool_result blocks (not yet supported) + blocks?: ContentBlock[]; // Interleaved text/tool_call/tool_result blocks semanticId?: string; // For deduplication (call_id for tools, hash for regular). Not synced. } diff --git a/src/lumo-client/client.ts b/src/lumo-client/client.ts index f1dac34..5d8d473 100644 --- a/src/lumo-client/client.ts +++ b/src/lumo-client/client.ts @@ -14,6 +14,7 @@ import { RequestEncryptionParams, } from '@lumo/lib/lumo-api-client/core/encryptionParams.js'; import { StreamProcessor } from '@lumo/lib/lumo-api-client/core/streaming.js'; +import { appendTextToBlocks } from '@lumo/messageHelpers.js'; import { logger } from '../app/logger.js'; import { Role, @@ -28,6 +29,7 @@ import { type AssistantMessageData, type LumoClientOptions, type ChatResult, + type ContentBlock, } from './types.js'; import { getInstructionsConfig, getLogConfig, getConfigMode, getCustomToolsConfig, getEnableWebSearch } from '../app/config.js'; import { injectInstructionsIntoTurns } from './instructions.js'; @@ -41,6 +43,20 @@ const DEFAULT_INTERNAL_TOOLS: ToolName[] = ['proton_info']; const DEFAULT_EXTERNAL_TOOLS: ToolName[] = ['web_search', 'weather', 'stock', 'cryptocurrency']; const DEFAULT_ENDPOINT = 'ai/v1/chat'; +/** + * Merge text blocks with tool blocks. + * Text blocks come first (accumulated during streaming), then tool blocks. + * If there are no tool blocks, returns text blocks as-is. + */ +function mergeBlocks(textBlocks: ContentBlock[], toolBlocks: ContentBlock[]): ContentBlock[] { + if (toolBlocks.length === 0) { + return textBlocks; + } + // For now, simple concatenation. The upstream helpers handle + // proper interleaving during streaming; here we just combine final results. + return [...textBlocks, ...toolBlocks]; +} + /** Build the bounce instruction: config text + the misrouted tool call as JSON example. * Includes the prefix in the example JSON so Lumo outputs it correctly. */ function buildBounceInstruction(toolCall: ParsedToolCall): string { @@ -105,6 +121,7 @@ export class LumoClient { const processor = new StreamProcessor(); let fullResponse = ''; let fullTitle = ''; + let blocks: ContentBlock[] = []; // Native tool call processing (SSE tool_call/tool_result targets) const nativeToolProcessor = new NativeToolCallProcessor(isBounce); @@ -137,6 +154,7 @@ export class LumoClient { if (msg.target === 'message') { fullResponse += content; + blocks = appendTextToBlocks(blocks, content); if (!suppressChunks) { onChunk?.(content); } @@ -186,18 +204,15 @@ export class LumoClient { nativeToolProcessor.finalize(); const nativeResult = nativeToolProcessor.getResult(); + // Merge text blocks with native tool blocks + // Native tool blocks come from processor, text blocks accumulated here + const finalBlocks = mergeBlocks(blocks, nativeResult.blocks); + // Build message data for persistence - // Only include native tool data if not misrouted (misrouted calls are bounced) - const message: AssistantMessageData = { content: fullResponse }; - if (nativeResult.toolCall && !nativeResult.misrouted) { - message.toolCall = JSON.stringify({ - name: nativeResult.toolCall.name, - arguments: nativeResult.toolCall.arguments, - }); - if (nativeResult.toolResult) { - message.toolResult = nativeResult.toolResult; - } - } + const message: AssistantMessageData = { + content: fullResponse, + blocks: finalBlocks.length > 0 ? finalBlocks : undefined, + }; return { message, diff --git a/src/lumo-client/types.ts b/src/lumo-client/types.ts index 35e4b81..13091f1 100644 --- a/src/lumo-client/types.ts +++ b/src/lumo-client/types.ts @@ -13,6 +13,8 @@ export type { Turn, } from '@lumo/lib/lumo-api-client/core/types.js'; +import type { ContentBlock } from '@lumo/types.js'; +export type { ContentBlock }; export { Role } from '@lumo/types-api.js'; // Local-only types @@ -91,10 +93,8 @@ export interface NativeToolData { */ export interface AssistantMessageData { content: string; - /** JSON string of tool call (native tools only) */ - toolCall?: string; - /** JSON string of tool result (native tools only) */ - toolResult?: string; + /** Interleaved text/tool_call/tool_result blocks (native tools) */ + blocks?: ContentBlock[]; } // LumoClient types diff --git a/tests/unit/conversation-store.test.ts b/tests/unit/conversation-store.test.ts index c28c908..1f15b7f 100644 --- a/tests/unit/conversation-store.test.ts +++ b/tests/unit/conversation-store.test.ts @@ -129,17 +129,22 @@ describe('FallbackStore', () => { expect(assistantMsg.parentId).toBe(userMsg.id); }); - it('stores native tool call data', () => { + it('stores native tool call data in blocks', () => { store.appendMessages('conv-1', [{ role: 'user', content: 'Search for news' }]); const msg = store.appendAssistantResponse('conv-1', { content: 'Here are the results...', - toolCall: '{"name":"web_search","arguments":{"query":"news"}}', - toolResult: '{"results":[]}', + blocks: [ + { type: 'tool_call', content: '{"name":"web_search","arguments":{"query":"news"}}' }, + { type: 'tool_result', content: '{"results":[]}' }, + { type: 'text', content: 'Here are the results...' }, + ], }); expect(msg.content).toBe('Here are the results...'); - expect(msg.toolCall).toBe('{"name":"web_search","arguments":{"query":"news"}}'); - expect(msg.toolResult).toBe('{"results":[]}'); + expect(msg.blocks).toHaveLength(3); + expect(msg.blocks![0].type).toBe('tool_call'); + expect(msg.blocks![1].type).toBe('tool_result'); + expect(msg.blocks![2].type).toBe('text'); }); }); diff --git a/tests/unit/native-tool-call-processor.test.ts b/tests/unit/native-tool-call-processor.test.ts index a4ee330..f139e8e 100644 --- a/tests/unit/native-tool-call-processor.test.ts +++ b/tests/unit/native-tool-call-processor.test.ts @@ -74,7 +74,7 @@ describe('NativeToolCallProcessor', () => { }); describe('feedToolResult', () => { - it('captures tool result JSON', () => { + it('captures tool result in blocks', () => { const processor = new NativeToolCallProcessor(); processor.feedToolCall('{"name":"web_search","parameters":{"query":"test"}}'); @@ -82,7 +82,10 @@ describe('NativeToolCallProcessor', () => { processor.finalize(); const result = processor.getResult(); - expect(result.toolResult).toBe('{"results":[{"title":"Test"}],"total_count":1}'); + expect(result.blocks).toHaveLength(2); + expect(result.blocks[0].type).toBe('tool_call'); + expect(result.blocks[1].type).toBe('tool_result'); + expect(result.blocks[1].content).toBe('{"results":[{"title":"Test"}],"total_count":1}'); }); it('detects error results', () => { diff --git a/tests/unit/primary-conversation-store.test.ts b/tests/unit/primary-conversation-store.test.ts index 0089b23..b95fe67 100644 --- a/tests/unit/primary-conversation-store.test.ts +++ b/tests/unit/primary-conversation-store.test.ts @@ -134,21 +134,24 @@ describe('ConversationStore', () => { expect(assistantMsg.parentId).toBe(userMsg.id); }); - it('stores native tool call data', () => { + it('stores native tool call data in blocks', () => { ctx.conversationStore.appendMessages('conv-1', [ { role: 'user', content: 'Search for news' }, ]); const msg = ctx.conversationStore.appendAssistantResponse('conv-1', { content: 'Here are the results...', - toolCall: '{"name":"web_search","arguments":{"query":"news"}}', - toolResult: '{"results":[]}', + blocks: [ + { type: 'tool_call', content: '{"name":"web_search","arguments":{"query":"news"}}' }, + { type: 'tool_result', content: '{"results":[]}' }, + { type: 'text', content: 'Here are the results...' }, + ], }); expect(msg.content).toBe('Here are the results...'); - expect(msg.toolCall).toBe( - '{"name":"web_search","arguments":{"query":"news"}}' - ); - expect(msg.toolResult).toBe('{"results":[]}'); + expect(msg.blocks).toHaveLength(3); + expect(msg.blocks![0].type).toBe('tool_call'); + expect(msg.blocks![1].type).toBe('tool_result'); + expect(msg.blocks![2].type).toBe('text'); }); }); From 8de9e440c013a97dee8ffece29a8569e89b9f752 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:40:02 +0100 Subject: [PATCH 02/37] use upstream type MessagePriv instead of own type MessagePrivate --- .../fallback/sync/encryption-codec.ts | 9 +++--- .../fallback/sync/sync-service.ts | 8 ++--- src/conversations/index.ts | 2 +- src/conversations/types.ts | 30 +++++-------------- 4 files changed, 16 insertions(+), 33 deletions(-) diff --git a/src/conversations/fallback/sync/encryption-codec.ts b/src/conversations/fallback/sync/encryption-codec.ts index 4277d8a..648bbc5 100644 --- a/src/conversations/fallback/sync/encryption-codec.ts +++ b/src/conversations/fallback/sync/encryption-codec.ts @@ -10,12 +10,11 @@ import stableStringify from 'json-stable-stringify'; import { logger } from '../../../app/logger.js'; import { encryptData, decryptData } from '@proton/crypto/lib/subtle/aesGcm'; -import { Role } from '@lumo/types.js'; +import { Role, type MessagePriv } from '@lumo/types.js'; import type { Message, ProjectSpace, ConversationPriv, - MessagePrivate, } from '../../types.js'; // Role mapping for AD construction @@ -142,7 +141,7 @@ export class EncryptionCodec { * the WebClient will reconstruct from the Role integer field, NOT our internal role names. */ async encryptMessage( - data: MessagePrivate, + data: MessagePriv, message: Message, effectiveParentId?: string ): Promise { @@ -171,8 +170,8 @@ export class EncryptionCodec { conversationId: string, role: string, parentId?: string - ): Promise { - return this.decrypt( + ): Promise { + return this.decrypt( encryptedBase64, { app: 'lumo', diff --git a/src/conversations/fallback/sync/sync-service.ts b/src/conversations/fallback/sync/sync-service.ts index 13b05bc..4df9e1e 100644 --- a/src/conversations/fallback/sync/sync-service.ts +++ b/src/conversations/fallback/sync/sync-service.ts @@ -10,8 +10,8 @@ import { LumoApi } from '@lumo/remote/api.js'; import { RoleInt, StatusInt } from '@lumo/remote/types.js'; import { getFallbackStore } from '../store.js'; import type { KeyManager } from '../../key-manager.js'; -import { Role, type Status } from '@lumo/types.js'; -import type { ConversationState, Message, SpaceId, RemoteId, MessagePrivate } from '../../types.js'; +import { Role, type Status, type MessagePriv } from '@lumo/types.js'; +import type { ConversationState, Message, SpaceId, RemoteId } from '../../types.js'; import { SpaceManager } from './space-manager.js'; // Role mapping: our internal roles to API integer values @@ -260,7 +260,7 @@ export class SyncService { this.messageIdMap ); - const messagePrivate: MessagePrivate = { + const messagePrivate: MessagePriv = { content: contentToStore, context: message.context, blocks: message.blocks, @@ -354,7 +354,7 @@ export class SyncService { remoteId ); - let messagePrivate: MessagePrivate | null = null; + let messagePrivate: MessagePriv | null = null; if (fullMsg?.encrypted && typeof fullMsg.encrypted === 'string') { messagePrivate = await codec.decryptMessage( fullMsg.encrypted, diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 3680b41..9f2e2b2 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -16,7 +16,7 @@ export type { ConversationState, Message, MessageId, - MessagePrivate, + MessagePriv, SpaceId, RemoteId, IdMapEntry, diff --git a/src/conversations/types.ts b/src/conversations/types.ts index b03f759..91af60c 100644 --- a/src/conversations/types.ts +++ b/src/conversations/types.ts @@ -4,12 +4,12 @@ */ // Import types from upstream @lumo -import type { ConversationId, MessageId, SpaceId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub, Role, ContentBlock } from '@lumo/types.js'; +import type { ConversationId, MessageId, SpaceId, ProjectSpace, ConversationPriv, MessagePub, MessagePriv, ConversationPub, Role, ContentBlock } from '@lumo/types.js'; import { ConversationStatus } from '@lumo/types.js'; import type { RemoteId } from '@lumo/remote/types.ts'; // Re-export types for consumers -export type { ConversationId, MessageId, SpaceId, RemoteId, ProjectSpace, ConversationPriv, MessagePub, ConversationPub, ContentBlock }; +export type { ConversationId, MessageId, SpaceId, RemoteId, ProjectSpace, ConversationPriv, MessagePub, MessagePriv, ConversationPub, ContentBlock }; /** * Full conversation record @@ -20,31 +20,15 @@ export interface Conversation extends ConversationPub { } /** - * Message private data (encrypted) - * - * Content is optional to match Proton's model where tool_call/tool_result - * messages may have no content (just toolCall/toolResult fields). - * - * WebClient also has: attachments?: ShallowAttachment[], contextFiles?: AttachmentId[] - * We don't handle attachments yet. + * Full message record * - * Tool calls (native tools like web_search, weather): - * Use `blocks?: ContentBlock[]` for interleaved text/tool_call/tool_result blocks. - * The upstream `getMessageBlocks()` helper reconstructs blocks from legacy - * toolCall/toolResult fields if blocks is not present (for old synced data). + * Extends upstream MessagePub (id, role, timestamps, etc.) and MessagePriv + * (content, blocks, attachments, reasoning, etc.) with runtime-only fields. */ -export interface MessagePrivate { - content?: string; - context?: string; - blocks?: ContentBlock[]; // Interleaved text/tool_call/tool_result blocks - semanticId?: string; // For deduplication (call_id for tools, hash for regular). Not synced. +export interface Message extends MessagePub, MessagePriv { + semanticId?: string; // Runtime-only, for deduplication (call_id for tools, hash for regular). Not synced. } -/** - * Full message record - */ -export interface Message extends MessagePub, MessagePrivate { } - /** * In-memory conversation state */ From 55c17838029050933410cfdfdd1125e3fe4fc7af Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Fri, 6 Mar 2026 14:41:50 +0100 Subject: [PATCH 03/37] remove obsolete type NativeToolData --- src/lumo-client/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lumo-client/index.ts b/src/lumo-client/index.ts index d98e560..995cc09 100644 --- a/src/lumo-client/index.ts +++ b/src/lumo-client/index.ts @@ -12,7 +12,6 @@ export type { CachedUserKey, CachedMasterKey, ParsedToolCall, - NativeToolData, AssistantMessageData, LumoClientOptions, ChatResult, From 6c2d190c056e52326c3e04bf056c30bd1c2b8fed Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Fri, 6 Mar 2026 16:01:51 +0100 Subject: [PATCH 04/37] browser authenticate: unconditionally fetch keys (ignore enableSync) --- src/auth/browser/authenticate.ts | 164 +++++++++++++++---------------- 1 file changed, 77 insertions(+), 87 deletions(-) diff --git a/src/auth/browser/authenticate.ts b/src/auth/browser/authenticate.ts index da91ade..a1c4c2b 100644 --- a/src/auth/browser/authenticate.ts +++ b/src/auth/browser/authenticate.ts @@ -11,7 +11,7 @@ import { chromium, type Page, type BrowserContext, type Browser } from 'playwrig import { promises as dns, ADDRCONFIG } from 'dns'; import type { PersistedSessionData } from '../../lumo-client/types.js'; import type { StoredTokens } from '../types.js'; -import { authConfig, getConversationsConfig } from '../../app/config.js'; +import { authConfig } from '../../app/config.js'; import { APP_VERSION_HEADER } from '@lumo/config.js'; import { PROTON_URLS } from '../../app/urls.js'; import { logger } from '../../app/logger.js'; @@ -24,8 +24,6 @@ export interface ExtractionOptions { cdpEndpoint: string; /** Target URL (Lumo) */ targetUrl: string; - /** Whether to fetch persistence keys (userKeys, masterKeys) */ - fetchPersistenceKeys: boolean; /** Proton app version for API calls */ appVersion: string; /** Timeout for waiting for login (ms) */ @@ -402,7 +400,7 @@ async function connectAndGetPage( */ export async function extractBrowserTokens(options: ExtractionOptions): Promise { const warnings: string[] = []; - const { cdpEndpoint, targetUrl, fetchPersistenceKeys, appVersion, loginTimeout = 120000 } = options; + const { cdpEndpoint, targetUrl, appVersion, loginTimeout = 120000 } = options; logger.info('=== Browser Token Extraction ==='); @@ -530,95 +528,91 @@ export async function extractBrowserTokens(options: ExtractionOptions): Promise< persistedSession = extractPersistedSession(accountLocalStorage); } - // Fetch persistence keys if requested + // Fetch encryption keys let userKeys: StoredTokens['userKeys']; let masterKeys: StoredTokens['masterKeys']; let keyPassword: string | undefined; - if (fetchPersistenceKeys) { - logger.info('Fetching encryption keys for persistence...'); + logger.info('Fetching encryption keys...'); - // Fetch ClientKey and decrypt blob to get keyPassword - if (persistedSession?.blob) { - const matchingAuthCookie = relevantCookies.find( - c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('account.proton.me') - ) || relevantCookies.find( - c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') - ); + // Fetch ClientKey and decrypt blob to get keyPassword + if (persistedSession?.blob) { + const matchingAuthCookie = relevantCookies.find( + c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('account.proton.me') + ) || relevantCookies.find( + c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') + ); - const authCookie = matchingAuthCookie || primaryAccountAuthCookie || primaryLumoAuthCookie; - if (authCookie) { - const uid = authCookie.name.replace('AUTH-', ''); - const accessToken = authCookie.value; - - logger.info({ uid: uid.slice(0, 8) + '...' }, 'Fetching ClientKey from API...'); - const clientKey = await fetchClientKey(page, uid, accessToken, appVersion); - - if (clientKey) { - // Temporarily set clientKey to decrypt blob - persistedSession.clientKey = clientKey; - - try { - const decrypted = await decryptPersistedSession(persistedSession); - keyPassword = decrypted.keyPassword; - logger.info({ type: decrypted.type }, 'Successfully extracted keyPassword'); - // Clear encryption artifacts - keyPassword is stored directly in vault - delete persistedSession.blob; - delete persistedSession.clientKey; - delete persistedSession.payloadVersion; - } catch (err) { - logger.error({ err }, 'ClientKey fetch succeeded but decryption failed'); - delete persistedSession.clientKey; - warnings.push('ClientKey fetch succeeded but decryption failed'); - } + const authCookie = matchingAuthCookie || primaryAccountAuthCookie || primaryLumoAuthCookie; + if (authCookie) { + const uid = authCookie.name.replace('AUTH-', ''); + const accessToken = authCookie.value; + + logger.info({ uid: uid.slice(0, 8) + '...' }, 'Fetching ClientKey from API...'); + const clientKey = await fetchClientKey(page, uid, accessToken, appVersion); + + if (clientKey) { + // Temporarily set clientKey to decrypt blob + persistedSession.clientKey = clientKey; + + try { + const decrypted = await decryptPersistedSession(persistedSession); + keyPassword = decrypted.keyPassword; + logger.info({ type: decrypted.type }, 'Successfully extracted keyPassword'); + // Clear encryption artifacts - keyPassword is stored directly in vault + delete persistedSession.blob; + delete persistedSession.clientKey; + delete persistedSession.payloadVersion; + } catch (err) { + logger.error({ err }, 'ClientKey fetch succeeded but decryption failed'); + delete persistedSession.clientKey; + warnings.push('ClientKey fetch succeeded but decryption failed'); } } } + } - // Fetch user keys (only if we got keyPassword) - // Note: keyPassword being set implies persistedSession exists (it came from the blob) - if (keyPassword && persistedSession) { - const matchingAuthCookie = relevantCookies.find( - c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('account.proton.me') - ) || relevantCookies.find( - c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') - ); - const authCookieForUserInfo = matchingAuthCookie || primaryAccountAuthCookie || primaryLumoAuthCookie; - - if (authCookieForUserInfo) { - const uid = authCookieForUserInfo.name.replace('AUTH-', ''); - const accessToken = authCookieForUserInfo.value; - const userInfo = await fetchUserInfo(page, uid, accessToken, appVersion); - if (userInfo?.User?.Keys) { - userKeys = userInfo.User.Keys.map(k => ({ - ID: k.ID, - PrivateKey: k.PrivateKey, - Primary: k.Primary, - Active: k.Active, - })); - logger.info({ keyCount: userKeys.length }, 'Cached user keys'); - } + // Fetch user keys (only if we got keyPassword) + // Note: keyPassword being set implies persistedSession exists (it came from the blob) + if (keyPassword && persistedSession) { + const matchingAuthCookie = relevantCookies.find( + c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('account.proton.me') + ) || relevantCookies.find( + c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') + ); + const authCookieForUserInfo = matchingAuthCookie || primaryAccountAuthCookie || primaryLumoAuthCookie; + + if (authCookieForUserInfo) { + const uid = authCookieForUserInfo.name.replace('AUTH-', ''); + const accessToken = authCookieForUserInfo.value; + const userInfo = await fetchUserInfo(page, uid, accessToken, appVersion); + if (userInfo?.User?.Keys) { + userKeys = userInfo.User.Keys.map(k => ({ + ID: k.ID, + PrivateKey: k.PrivateKey, + Primary: k.Primary, + Active: k.Active, + })); + logger.info({ keyCount: userKeys.length }, 'Cached user keys'); } } + } - // Fetch master keys (only if we got keyPassword) - if (keyPassword && persistedSession) { - const lumoAuthForMasterKeys = relevantCookies.find( - c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') - ) || primaryLumoAuthCookie; - - if (lumoAuthForMasterKeys) { - const uid = lumoAuthForMasterKeys.name.replace('AUTH-', ''); - const accessToken = lumoAuthForMasterKeys.value; - const fetchedMasterKeys = await fetchMasterKeys(page, uid, accessToken, appVersion); - if (fetchedMasterKeys && fetchedMasterKeys.length > 0) { - masterKeys = fetchedMasterKeys; - logger.info({ keyCount: masterKeys.length }, 'Cached master keys'); - } + // Fetch master keys (only if we got keyPassword) + if (keyPassword && persistedSession) { + const lumoAuthForMasterKeys = relevantCookies.find( + c => c.name === `AUTH-${persistedSession.UID}` && c.domain.includes('lumo.proton.me') + ) || primaryLumoAuthCookie; + + if (lumoAuthForMasterKeys) { + const uid = lumoAuthForMasterKeys.name.replace('AUTH-', ''); + const accessToken = lumoAuthForMasterKeys.value; + const fetchedMasterKeys = await fetchMasterKeys(page, uid, accessToken, appVersion); + if (fetchedMasterKeys && fetchedMasterKeys.length > 0) { + masterKeys = fetchedMasterKeys; + logger.info({ keyCount: masterKeys.length }, 'Cached master keys'); } } - } else { - logger.info('Skipping encryption key extraction (persistence disabled)'); } // Determine output uid/accessToken - use the primary (active session) auth @@ -676,8 +670,8 @@ export async function extractBrowserTokens(options: ExtractionOptions): Promise< masterKeys, }; - // Add warnings for missing data (only if persistence was requested) - if (fetchPersistenceKeys && !keyPassword) { + // Add warnings for missing data + if (!keyPassword) { warnings.push('No keyPassword available - local-only encryption will be used'); } @@ -690,14 +684,13 @@ export async function extractBrowserTokens(options: ExtractionOptions): Promise< /** * Prompt user for CDP endpoint */ -async function promptForCdpEndpoint(defaultEndpoint?: string): Promise { +async function promptForCdpEndpoint(defaultEndpoint: string): Promise { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); - const defaultValue = defaultEndpoint || 'http://localhost:9222'; return new Promise(resolve => { - rl.question(`CDP endpoint [${defaultValue}]: `, answer => { + rl.question(`CDP endpoint [${defaultEndpoint}]: `, answer => { rl.close(); - resolve(answer.trim() || defaultValue); + resolve(answer.trim() || defaultEndpoint); }); }); } @@ -714,12 +707,9 @@ export async function runBrowserAuthentication(): Promise { const configEndpoint = authConfig.browser?.cdpEndpoint; const cdpEndpoint = await promptForCdpEndpoint(configEndpoint); - const syncEnabled = getConversationsConfig().enableSync; - const result = await extractBrowserTokens({ cdpEndpoint, targetUrl: PROTON_URLS.LUMO_BASE, - fetchPersistenceKeys: syncEnabled, appVersion: APP_VERSION_HEADER, }); From 620535437eef7c332c2d0aa9ddcb0df400fb2987 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Sat, 7 Mar 2026 00:41:39 +0100 Subject: [PATCH 05/37] Goodbye FallbackStore! - removed all of FallbackStore and SyncService - added MinimalStore to be used as fallback in CLI (can we get rid of this as well?) - removed unused an no-op methods in store --- config.defaults.yaml | 5 - src/api/routes/auth.ts | 5 - src/api/types.ts | 4 +- src/app/commands.ts | 55 +- src/app/config.ts | 1 - src/app/index.ts | 24 +- src/auth/status.ts | 16 +- src/cli/client.ts | 6 +- src/conversations/fallback/index.ts | 38 -- src/conversations/fallback/store.ts | 540 ------------------ src/conversations/fallback/sync/auto-sync.ts | 270 --------- .../fallback/sync/encryption-codec.ts | 188 ------ src/conversations/fallback/sync/index.ts | 47 -- .../fallback/sync/space-manager.ts | 323 ----------- .../fallback/sync/sync-service.ts | 447 --------------- src/conversations/index.ts | 174 ++---- src/conversations/minimal-store.ts | 256 +++++++++ src/conversations/store-interface.ts | 46 ++ src/conversations/store.ts | 78 +-- tests/e2e/cli-smoke.test.ts | 8 +- tests/helpers/test-server.ts | 7 +- tests/integration/metrics.test.ts | 4 +- ...on-store.test.ts => minimal-store.test.ts} | 114 +--- tests/unit/primary-conversation-store.test.ts | 49 -- 24 files changed, 430 insertions(+), 2275 deletions(-) delete mode 100644 src/conversations/fallback/index.ts delete mode 100644 src/conversations/fallback/store.ts delete mode 100644 src/conversations/fallback/sync/auto-sync.ts delete mode 100644 src/conversations/fallback/sync/encryption-codec.ts delete mode 100644 src/conversations/fallback/sync/index.ts delete mode 100644 src/conversations/fallback/sync/space-manager.ts delete mode 100644 src/conversations/fallback/sync/sync-service.ts create mode 100644 src/conversations/minimal-store.ts create mode 100644 src/conversations/store-interface.ts rename tests/unit/{conversation-store.test.ts => minimal-store.test.ts} (61%) diff --git a/config.defaults.yaml b/config.defaults.yaml index b699515..1517c50 100644 --- a/config.defaults.yaml +++ b/config.defaults.yaml @@ -75,11 +75,6 @@ conversations: # WARNING: May incorrectly merge unrelated conversations with same "user" field. Ignored if `user` is absent. deriveIdFromUser: false - # Use fallback in-memory store instead of the primary store - # When true, uses the legacy in-memory store (will be removed in future). - # When false (default), uses the primary Redux + IndexedDB store. - useFallbackStore: true - # Enable syncing conversations to Proton servers (requires browser auth) enableSync: false # Project name for conversations (created if doesn't exist) diff --git a/src/api/routes/auth.ts b/src/api/routes/auth.ts index 0989a26..f036275 100644 --- a/src/api/routes/auth.ts +++ b/src/api/routes/auth.ts @@ -9,7 +9,6 @@ import { Router, Request, Response } from 'express'; import { EndpointDependencies } from '../types.js'; -import { getAutoSyncService } from '../../conversations/index.js'; import { logger } from '../../app/logger.js'; export function createAuthRouter(deps: EndpointDependencies): Router { @@ -36,10 +35,6 @@ export function createAuthRouter(deps: EndpointDependencies): Router { return; } - // Stop auto-sync if running - const autoSync = getAutoSyncService(); - autoSync?.stop(); - // Perform logout (stops refresh timer, revokes session, deletes tokens) await deps.authManager.logout(); diff --git a/src/api/types.ts b/src/api/types.ts index a85df1c..4faf47e 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -1,12 +1,12 @@ import { RequestQueue } from './queue.js'; import { LumoClient } from '../lumo-client/index.js'; -import type { ConversationStore, FallbackStore } from '../conversations/index.js'; +import type { IConversationStore } from '../conversations/index.js'; import type { AuthManager } from '../auth/index.js'; export interface EndpointDependencies { queue: RequestQueue; lumoClient: LumoClient; - conversationStore?: ConversationStore | FallbackStore; + conversationStore?: IConversationStore; syncInitialized?: boolean; authManager?: AuthManager; vaultPath?: string; diff --git a/src/app/commands.ts b/src/app/commands.ts index 0d57dc5..969d347 100644 --- a/src/app/commands.ts +++ b/src/app/commands.ts @@ -5,7 +5,7 @@ import { logger } from './logger.js'; import { getCommandsConfig } from './config.js'; -import { getSyncService, getConversationStore, getAutoSyncService } from '../conversations/index.js'; +import { getConversationStore } from '../conversations/index.js'; import type { AuthManager } from '../auth/index.js'; import type { Turn } from '../lumo-client/index.js'; @@ -171,11 +171,12 @@ function handleTitleCommand(params: string, context?: CommandContext): string { } /** - * Handle /save command - save current conversation only + * Handle /save command - save current conversation * Optionally set title first with /save * * For stateless requests (no conversationId), creates a new conversation - * from the provided messages and saves it. + * from the provided messages. Sync happens automatically via Redux sagas + * when using the primary store. */ async function handleSaveCommand(params: string, context?: CommandContext): Promise<string> { try { @@ -184,6 +185,10 @@ async function handleSaveCommand(params: string, context?: CommandContext): Prom } const store = getConversationStore(); + if (!store) { + return 'Conversation store not available.'; + } + let conversationId = context?.conversationId; let wasCreated = false; @@ -204,19 +209,15 @@ async function handleSaveCommand(params: string, context?: CommandContext): Prom } } - const syncService = getSyncService(); - const synced = await syncService.syncById(conversationId); - - if (!synced) { - return 'Conversation not found or could not be saved.'; + const conversation = store.get(conversationId); + if (!conversation) { + return 'Conversation not found.'; } - const conversation = store.get(conversationId); - const title = conversation?.title ?? 'Unknown'; + const title = conversation.title ?? 'Unknown'; - // Different message for newly created vs existing conversation if (wasCreated) { - return `Created and saved conversation: ${title}`; + return `Created conversation: ${title}`; } return `Saved conversation: ${title}`; } catch (error) { @@ -226,32 +227,32 @@ async function handleSaveCommand(params: string, context?: CommandContext): Prom } /** - * Handle /load command - load a conversation from server by local ID + * Handle /load command - load a conversation by ID + * + * With the primary store, conversations are loaded from IndexedDB automatically. + * This command provides info about an existing conversation. */ async function handleLoadCommand(params: string, context?: CommandContext): Promise<string> { try { - if (!context?.syncInitialized) { - return 'Sync not initialized. Persistence may be disabled or KeyManager not ready.'; - } - const localId = params.trim(); if (!localId) { return 'Usage: /load <id>\nExample: /load f0654976-d628-4516-8e80-a0599b6593ac'; } - const syncService = getSyncService(); - const conversationId = await syncService.loadExistingConversation(localId); + const store = getConversationStore(); + if (!store) { + return 'Conversation store not available.'; + } - if (!conversationId) { + const conversation = store.get(localId); + if (!conversation) { return `Conversation not found: ${localId}`; } - const store = getConversationStore(); - const conversation = store.get(conversationId); - const messageCount = conversation?.messages.length ?? 0; - const title = conversation?.title ?? 'Untitled'; + const messageCount = conversation.messages.length ?? 0; + const title = conversation.title ?? 'Untitled'; - return `Loaded conversation: ${title}\nLocal ID: ${conversationId}\nMessages: ${messageCount}`; + return `Loaded conversation: ${title}\nLocal ID: ${localId}\nMessages: ${messageCount}`; } catch (error) { logger.error({ error }, 'Failed to execute /load command'); return `Load failed: ${error instanceof Error ? error.message : 'Unknown error'}`; @@ -284,10 +285,6 @@ async function handleLogoutCommand(context?: CommandContext): Promise<string> { return 'Logout not available - missing auth context.'; } - // Stop auto-sync if running - const autoSync = getAutoSyncService(); - autoSync?.stop(); - // Perform logout (stops refresh timer, revokes session, deletes tokens) await context.authManager.logout(); diff --git a/src/app/config.ts b/src/app/config.ts index e982b21..4982639 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -27,7 +27,6 @@ const logConfigSchema = z.object({ const conversationsConfigSchema = z.object({ deriveIdFromUser: z.boolean(), databasePath: z.string(), - useFallbackStore: z.boolean(), enableSync: z.boolean(), projectName: z.string().min(1), }); diff --git a/src/app/index.ts b/src/app/index.ts index 2ae636f..e732cde 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -10,7 +10,7 @@ import { logger } from './logger.js'; import { resolveProjectPath } from './paths.js'; import { LumoClient } from '../lumo-client/index.js'; import { createAuthProvider, AuthManager, type AuthProvider, type ProtonApi } from '../auth/index.js'; -import { getConversationStore, getFallbackStore, setConversationStore, type ConversationStore, initializeSync, initializeConversationStore, FallbackStore } from '../conversations/index.js'; +import { getConversationStore, setConversationStore, initializeSync, initializeConversationStore, type IConversationStore } from '../conversations/index.js'; import { createMockProtonApi } from '../mock/mock-api.js'; import { installFetchAdapter } from '../shims/fetch-adapter.js'; import { suppressFullApiErrors } from '../shims/console.js'; @@ -43,25 +43,15 @@ export class Application { * Initialize mock mode - bypass auth, use simulated API responses */ private async initializeMock(): Promise<void> { - const conversationsConfig = getConversationsConfig(); - - if (!conversationsConfig.useFallbackStore) { - // Use primary store with fake-indexeddb - const { initializeMockStore } = await import('../mock/mock-store.js'); - const result = await initializeMockStore(); - setConversationStore(result.conversationStore); - } else { - // Use fallback in-memory store - getFallbackStore(); - } + // Use primary store with fake-indexeddb for mock mode + const { initializeMockStore } = await import('../mock/mock-store.js'); + const result = await initializeMockStore(); + setConversationStore(result.conversationStore); this.protonApi = createMockProtonApi(mockConfig.scenario); this.lumoClient = new LumoClient(this.protonApi, { enableEncryption: false }); - logger.info({ - scenario: mockConfig.scenario, - useFallbackStore: conversationsConfig.useFallbackStore, - }, 'Mock mode active - auth and sync bypassed'); + logger.info({ scenario: mockConfig.scenario }, 'Mock mode active - auth and sync bypassed'); } /** @@ -136,7 +126,7 @@ export class Application { return this.lumoClient; } - getConversationStore(): ConversationStore | FallbackStore { + getConversationStore(): IConversationStore | undefined { return getConversationStore(); } diff --git a/src/auth/status.ts b/src/auth/status.ts index 17a712c..e7cc588 100644 --- a/src/auth/status.ts +++ b/src/auth/status.ts @@ -47,15 +47,13 @@ export function printSummary(status: AuthProviderStatus, options: SummaryOptions if (status.valid) { print('\x1b[32mAuthentication is configured and valid.\x1b[0m'); - // Primary store status (local encryption) - if (!conversationsConfig.useFallbackStore) { - if (!supportsPersistence) { - print('Primary store: \x1b[33mdisabled\x1b[0m (no cached encryption keys)'); - } else if (!status.details.hasKeyPassword) { - print('Primary store: \x1b[33mdisabled\x1b[0m (no keyPassword)'); - } else { - print('Primary store: \x1b[32menabled\x1b[0m'); - } + // Conversation store status + if (!supportsPersistence) { + print('Conversation store: \x1b[33mminimal (in-memory)\x1b[0m - no encryption keys'); + } else if (!status.details.hasKeyPassword) { + print('Conversation store: \x1b[33mminimal (in-memory)\x1b[0m - no keyPassword'); + } else { + print('Conversation store: \x1b[32mpersistent\x1b[0m'); } // Sync status (Proton server sync) diff --git a/src/cli/client.ts b/src/cli/client.ts index 08b7042..68b8b40 100644 --- a/src/cli/client.ts +++ b/src/cli/client.ts @@ -19,6 +19,7 @@ import type { AssistantMessageData } from '../lumo-client/index.js'; import { blockHandlers, executeBlocks, formatResultsMessage } from './local-actions/block-handlers.js'; import { CodeBlockDetector, type CodeBlock } from './local-actions/code-block-detector.js'; import { buildCliInstructions } from './message-converter.js'; +import { MinimalStore, type IConversationStore } from '../conversations/index.js'; interface LumoResponse { /** Assistant message data ready for persistence */ @@ -29,11 +30,12 @@ interface LumoResponse { export class CLIClient { private conversationId: string; - private store; + private store: IConversationStore; constructor(private app: Application) { this.conversationId = randomUUID(); - this.store = app.getConversationStore(); + // Use app store if available, otherwise create minimal in-memory store + this.store = app.getConversationStore() ?? new MinimalStore(); } async run(): Promise<void> { diff --git a/src/conversations/fallback/index.ts b/src/conversations/fallback/index.ts deleted file mode 100644 index 00a6842..0000000 --- a/src/conversations/fallback/index.ts +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Fallback Storage Module - * - * Provides in-memory conversation storage with optional server sync. - * Used when the primary ConversationStore (Redux + IndexedDB) cannot be used. - * - * @deprecated This module will be removed in a future version. - * Use the primary ConversationStore when possible. - */ - -// Store -export { - FallbackStore, - getFallbackStore, - resetFallbackStore, -} from './store.js'; - -// Sync services -export { - SyncService, - getSyncService, - resetSyncService, - type SyncServiceConfig, -} from './sync/index.js'; - -export { - AutoSyncService, - getAutoSyncService, - resetAutoSyncService, -} from './sync/index.js'; - -export { - SpaceManager, - type SpaceManagerConfig, - type SpaceContext, -} from './sync/index.js'; - -export { EncryptionCodec } from './sync/index.js'; diff --git a/src/conversations/fallback/store.ts b/src/conversations/fallback/store.ts deleted file mode 100644 index f242529..0000000 --- a/src/conversations/fallback/store.ts +++ /dev/null @@ -1,540 +0,0 @@ -/** - * Fallback in-memory conversation store with LRU eviction - * - * This is the legacy store used when the primary ConversationStore - * (Redux + IndexedDB) cannot be used (e.g., missing encryption keys). - * - * Manages active conversations and provides methods for: - * - Creating/retrieving conversations - * - Appending messages with deduplication - * - Converting to Lumo Turn format - * - Tracking dirty state for sync - */ - -import { randomUUID } from 'crypto'; -import { logger } from '../../app/logger.js'; -import { deterministicUUID } from '../../app/id-generator.js'; -import { Role, ConversationStatus } from '@lumo/types.js'; -import type { Turn, AssistantMessageData } from '../../lumo-client/types.js'; -import { - findNewMessages, - hashMessage, - isValidContinuation, -} from '../deduplication.js'; -import { type MessageForStore } from '../types.js'; -import type { - ConversationId, - ConversationState, - Message, - MessageId, - SpaceId, -} from '../types.js'; -import { getLogConfig } from '../../app/config.js'; -import { getMetrics } from '../../app/metrics.js'; - -/** Max conversations to keep in memory (LRU eviction) */ -const MAX_CONVERSATIONS = 100; - -/** - * Fallback in-memory conversation store - * - * @deprecated Use ConversationStore (Redux + IndexedDB) when possible. - * This fallback will be removed in a future version. - */ -export class FallbackStore { - private conversations = new Map<ConversationId, ConversationState>(); - private accessOrder: ConversationId[] = []; // LRU tracking - private maxConversations = MAX_CONVERSATIONS; - private defaultSpaceId: SpaceId; - private onDirtyCallback?: () => void; - - constructor() { - this.defaultSpaceId = randomUUID(); - logger.info({ spaceId: this.defaultSpaceId }, 'FallbackStore initialized'); - } - - /** - * Set callback to be called when a conversation becomes dirty - * Used by AutoSyncService to trigger sync scheduling - */ - setOnDirtyCallback(callback: () => void): void { - this.onDirtyCallback = callback; - } - - /** - * Get or create a conversation by ID - */ - getOrCreate(id: ConversationId): ConversationState { - let state = this.conversations.get(id); - - if (!state) { - state = this.createEmptyState(id); - this.conversations.set(id, state); - getMetrics()?.conversationsCreatedTotal.inc(); - logger.debug({ conversationId: id }, 'Created new conversation'); - } - - this.touchLRU(id); - this.evictIfNeeded(); - - return state; - } - - /** - * Get a conversation by ID (returns undefined if not found) - */ - get(id: ConversationId): ConversationState | undefined { - const state = this.conversations.get(id); - if (state) { - this.touchLRU(id); - } - return state; - } - - /** - * Check if a conversation exists - */ - has(id: ConversationId): boolean { - return this.conversations.has(id); - } - - /** - * Append messages from API request (with deduplication) - * - * @param id - Conversation ID - * @param incoming - Messages from API request - * @returns Array of newly added messages - */ - appendMessages( - id: ConversationId, - incoming: MessageForStore[] - ): Message[] { - const state = this.getOrCreate(id); - - // Validate continuation - const validation = isValidContinuation(incoming, state.messages); - if (!validation.valid) { - getMetrics()?.invalidContinuationsTotal.inc(); - logger.warn({ - conversationId: id, - reason: validation.reason, - incomingCount: incoming.length, - storedCount: state.messages.length, - ...validation.debugInfo, - }, 'Invalid conversation continuation'); - // For now, we continue anyway but log the warning - } - - // Find new messages - const newMessages = findNewMessages(incoming, state.messages); - - if (newMessages.length === 0) { - logger.debug({ conversationId: id }, 'No new messages to append'); - return []; - } - - // Convert to Message format and append - const now = new Date().toISOString(); - const lastMessageId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const addedMessages: Message[] = []; - let parentId = lastMessageId; - - for (const msg of newMessages) { - // Use provided ID (for tool messages) or compute hash (for regular messages) - const semanticId = msg.id ?? hashMessage(msg.role, msg.content ?? '').slice(0, 16); - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now, - role: msg.role , - parentId, - status: 'succeeded', - content: msg.content, - semanticId, - }; - - state.messages.push(message); - addedMessages.push(message); - parentId = message.id; - } - - // Mark as dirty - this.markDirty(state); - state.metadata.updatedAt = new Date().toISOString(); - - // Track metrics for new messages only - const metrics = getMetrics(); - if (metrics) { - for (const msg of addedMessages) { - metrics.messagesTotal.inc({ role: msg.role }); - } - } - - logger.debug({ - conversationId: id, - addedCount: addedMessages.length, - totalCount: state.messages.length, - }, 'Appended messages'); - - return addedMessages; - } - - /** - * Append an assistant response to a conversation. - * - * @param id - Conversation ID - * @param messageData - Assistant message data (content, optional blocks) - * @param status - Message status (default: succeeded) - * @param semanticId - Optional semantic ID for deduplication - * @returns The created message - */ - appendAssistantResponse( - id: ConversationId, - messageData: AssistantMessageData, - status: 'succeeded' | 'failed' = 'succeeded', - semanticId?: string - ): Message { - const state = this.getOrCreate(id); - const now = new Date(); - - const parentId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now.toISOString(), - role: Role.Assistant, - parentId, - status, - content: messageData.content, - blocks: messageData.blocks, - semanticId: semanticId ?? hashMessage(Role.Assistant, messageData.content).slice(0, 16), - }; - - state.messages.push(message); - this.markDirty(state); - state.metadata.updatedAt = now.toISOString(); - state.status = ConversationStatus.COMPLETED; - - getMetrics()?.messagesTotal.inc({ role: Role.Assistant }); - - logger.debug({ - conversationId: id, - messageId: message.id, - contentLength: messageData.content.length, - hasBlocks: !!messageData.blocks?.length, - }, 'Appended assistant response'); - - return message; - } - - /** - * Append tool calls as assistant messages. - * Each tool call stored as separate message with JSON content. - * Arguments are expected to already be normalized (via streaming-processor). - * - * NOTE: Currently unused. persistAssistantTurn() skips persistence when tool calls - * are present, relying on the client returning the assistant message when responding with tool output. - * (More robust as order of tool_calls & text may change) - * Kept for potential future use if we change the persistence strategy. - * (streaming tool processor should then return text & tool call blocks in order) - */ - appendAssistantToolCalls( - id: ConversationId, - toolCalls: Array<{ name: string; arguments: string; call_id: string }> - ): void { - for (const tc of toolCalls) { - const content = JSON.stringify({ - type: 'function_call', - call_id: tc.call_id, - name: tc.name, - arguments: tc.arguments, - }); - this.appendAssistantResponse(id, { content }, 'succeeded', tc.call_id); - } - } - - /** - * Append a single user message (CLI mode - no deduplication needed) - */ - appendUserMessage(id: ConversationId, content: string): Message { - const state = this.getOrCreate(id); - const now = new Date(); - - const parentId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now.toISOString(), - role: Role.User, - parentId, - status: 'succeeded', - content, - semanticId: hashMessage(Role.User, content).slice(0, 16), - }; - - state.messages.push(message); - this.markDirty(state); - state.metadata.updatedAt = now.toISOString(); - - logger.debug({ - conversationId: id, - messageId: message.id, - contentLength: content.length, - }, 'Appended user message'); - - return message; - } - - /** - * Create a conversation from turns (for stateless /save commands). - * - * Generates a deterministic conversation ID from the title to allow - * re-saving the same conversation without creating duplicates. - * - * @param turns - Turns to populate the conversation - * @param title - Optional title (auto-generated if not provided) - * @returns The created conversation ID and title - */ - createFromTurns( - turns: Turn[], - title?: string - ): { conversationId: ConversationId; title: string } { - const effectiveTitle = title?.trim().substring(0, 100) || generateAutoTitle(turns); - const conversationId = deterministicUUID(`save:${effectiveTitle}`); - - this.getOrCreate(conversationId); - this.appendMessages(conversationId, turns); - this.setTitle(conversationId, effectiveTitle); - - logger.info({ conversationId, title: effectiveTitle, turnCount: turns.length }, 'Created conversation from turns'); - - return { conversationId, title: effectiveTitle }; - } - - /** - * Mark conversation as generating (for streaming) - */ - setGenerating(id: ConversationId): void { - const state = this.get(id); - if (state) { - state.status = ConversationStatus.GENERATING; - } - } - - /** - * Update conversation title - */ - setTitle(id: ConversationId, title: string): void { - const state = this.get(id); - if (state) { - state.title = title; - this.markDirty(state); - state.metadata.updatedAt = new Date().toISOString(); - } - logger.debug(`Set title for ${id}${getLogConfig().messageContent ? `: ${title}` : ''}`); - } - - /** - * Convert conversation to Lumo Turn[] format for API call - */ - toTurns(id: ConversationId): Turn[] { - return this.getMessages(id).map(({ role, content }) => ({ - role, - content, - })); - } - - /** - * Get all messages in a conversation - */ - getMessages(id: ConversationId): Message[] { - const state = this.conversations.get(id); - return state?.messages ?? []; - } - - /** - * Get message by ID - */ - getMessage(conversationId: ConversationId, messageId: MessageId): Message | undefined { - const state = this.conversations.get(conversationId); - return state?.messages.find(m => m.id === messageId); - } - - /** - * Delete a conversation - */ - delete(id: ConversationId): boolean { - const existed = this.conversations.delete(id); - if (existed) { - this.accessOrder = this.accessOrder.filter(cid => cid !== id); - logger.debug({ conversationId: id }, 'Deleted conversation'); - } - return existed; - } - - /** - * Get all conversations (for iteration) - */ - entries(): IterableIterator<[ConversationId, ConversationState]> { - return this.conversations.entries(); - } - - /** - * Get all dirty conversations (need sync) - */ - getDirty(): ConversationState[] { - return Array.from(this.conversations.values()).filter(c => c.dirty); - } - - /** - * Mark a conversation as synced - */ - markSynced(id: ConversationId): void { - const state = this.conversations.get(id); - if (state) { - state.dirty = false; - state.lastSyncedAt = Date.now(); - } - } - - /** - * Mark a conversation as dirty (needs sync) - */ - markDirtyById(id: ConversationId): void { - const state = this.conversations.get(id); - if (state) { - this.markDirty(state); - } - } - - /** - * Get store statistics - */ - getStats(): { - total: number; - dirty: number; - maxSize: number; - } { - return { - total: this.conversations.size, - dirty: this.getDirty().length, - maxSize: this.maxConversations, - }; - } - - // Private methods - - /** - * Mark a conversation as dirty and notify callback - */ - private markDirty(state: ConversationState): void { - state.dirty = true; - this.onDirtyCallback?.(); - } - - private createEmptyState(id: ConversationId): ConversationState { - const now = new Date().toISOString(); - return { - metadata: { - id, - spaceId: this.defaultSpaceId, - createdAt: now, - updatedAt: now, - starred: false, - }, - title: 'New Conversation', - status: ConversationStatus.COMPLETED, - messages: [], - dirty: true, // New conversations need sync - }; - } - - private touchLRU(id: ConversationId): void { - // Remove from current position - const index = this.accessOrder.indexOf(id); - if (index !== -1) { - this.accessOrder.splice(index, 1); - } - // Add to end (most recently used) - this.accessOrder.push(id); - } - - private evictIfNeeded(): void { - while (this.conversations.size > this.maxConversations) { - // Evict least recently used - const toEvict = this.accessOrder.shift(); - if (toEvict) { - const state = this.conversations.get(toEvict); - if (state?.dirty) { - // Don't evict dirty conversations, move to end - this.accessOrder.push(toEvict); - logger.warn({ - conversationId: toEvict, - size: this.conversations.size, - }, 'Skipping eviction of dirty conversation'); - - // If all are dirty, we have to evict anyway - if (this.accessOrder.every(id => this.conversations.get(id)?.dirty)) { - const forced = this.accessOrder.shift(); - if (forced) { - this.conversations.delete(forced); - logger.warn({ conversationId: forced }, 'Force-evicted dirty conversation'); - } - break; - } - } else { - this.conversations.delete(toEvict); - logger.debug({ conversationId: toEvict }, 'Evicted conversation from cache'); - } - } - } - } -} - -/** - * Generate an auto-title from turns. - * - * Unlike other auto-title generation (which uses Lumo to summarize), - * this uses the first user message truncated to 50 chars. - * Used for stateless /save where we don't have a Lumo-generated title. - */ -function generateAutoTitle(turns: Turn[]): string { - const firstUserTurn = turns.find(t => t.role === Role.User); - if (firstUserTurn?.content) { - const content = firstUserTurn.content.trim(); - return content.length > 50 ? content.slice(0, 47) + '...' : content; - } - // Fallback to timestamp if no user message - const timestamp = new Date().toLocaleString('en-US', { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', hour12: false }); - return `Chat (${timestamp})`; -} - -// Singleton instance -let fallbackStoreInstance: FallbackStore | null = null; - -/** - * Get the global FallbackStore instance - */ -export function getFallbackStore(): FallbackStore { - if (!fallbackStoreInstance) { - fallbackStoreInstance = new FallbackStore(); - } - return fallbackStoreInstance; -} - -/** - * Reset the store (for testing) - */ -export function resetFallbackStore(): void { - fallbackStoreInstance = null; -} diff --git a/src/conversations/fallback/sync/auto-sync.ts b/src/conversations/fallback/sync/auto-sync.ts deleted file mode 100644 index b68d632..0000000 --- a/src/conversations/fallback/sync/auto-sync.ts +++ /dev/null @@ -1,270 +0,0 @@ -/** - * Auto-Sync Service for conversation persistence - * - * Provides smart automatic synchronization with: - * - Debouncing: Waits for activity to settle before syncing - * - Throttling: Ensures minimum interval between syncs - * - Max delay: Forces sync after maximum time regardless of activity - * - * Inspired by Proton WebClient's saga-based sync with dirty flags - * (see ~/WebClients/applications/lumo/src/app/redux/sagas/conversations.ts) - */ - -import { logger } from '../../../app/logger.js'; -import { getSyncService, type SyncService } from './sync-service.js'; -import { getMetrics } from '../../../app/metrics.js'; - -// Timing constants (not configurable - sensible defaults) -const DEBOUNCE_MS = 5000; // Wait after last change before syncing -const MIN_INTERVAL_MS = 30000; // Minimum interval between syncs -const MAX_DELAY_MS = 60000; // Force sync after this delay regardless - -/** - * Auto-Sync Service - * - * Manages automatic synchronization of dirty conversations. - * Uses a smart scheduling approach: - * 1. When a conversation is marked dirty, schedule a sync - * 2. Debounce: If more changes come in, push the sync back - * 3. Throttle: Don't sync more often than minIntervalMs - * 4. Max delay: Force sync after maxDelayMs regardless of activity - */ -export class AutoSyncService { - private enabled: boolean; - private syncService: SyncService; - - // Scheduling state - private debounceTimer: ReturnType<typeof setTimeout> | null = null; - private maxDelayTimer: ReturnType<typeof setTimeout> | null = null; - private lastSyncTime = 0; - private pendingSync = false; - private isSyncing = false; - private firstDirtyTime = 0; - - // Stats - private syncCount = 0; - private lastError: Error | null = null; - - constructor(syncService: SyncService, enabled = false) { - this.enabled = enabled; - this.syncService = syncService; - - if (this.enabled) { - logger.info({ - debounceMs: DEBOUNCE_MS, - minIntervalMs: MIN_INTERVAL_MS, - maxDelayMs: MAX_DELAY_MS, - }, 'AutoSyncService initialized'); - } - } - - /** - * Notify that a conversation has been marked dirty - * Call this whenever conversations change - */ - notifyDirty(): void { - if (!this.enabled) { - return; - } - - // Record first dirty time for max delay calculation - if (this.firstDirtyTime === 0) { - this.firstDirtyTime = Date.now(); - this.startMaxDelayTimer(); - } - - this.scheduleSync(); - } - - /** - * Schedule a sync with debouncing - */ - private scheduleSync(): void { - // Clear existing debounce timer - if (this.debounceTimer) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; - } - - // Calculate delay respecting throttle - const now = Date.now(); - const timeSinceLastSync = now - this.lastSyncTime; - const throttleDelay = Math.max(0, MIN_INTERVAL_MS - timeSinceLastSync); - const delay = Math.max(DEBOUNCE_MS, throttleDelay); - - logger.trace({ - delay, - throttleDelay, - debounceMs: DEBOUNCE_MS, - timeSinceLastSync, - }, 'Scheduling auto-sync'); - - this.pendingSync = true; - this.debounceTimer = setTimeout(() => { - this.debounceTimer = null; - this.executeSync(); - }, delay); - } - - /** - * Start the max delay timer (force sync after maxDelayMs) - */ - private startMaxDelayTimer(): void { - if (this.maxDelayTimer) { - return; - } - - this.maxDelayTimer = setTimeout(() => { - this.maxDelayTimer = null; - if (this.pendingSync && !this.isSyncing) { - logger.info('Max delay reached, forcing sync'); - this.executeSync(); - } - }, MAX_DELAY_MS); - } - - /** - * Execute the sync operation - */ - private async executeSync(): Promise<void> { - if (this.isSyncing) { - return; - } - - this.isSyncing = true; - this.pendingSync = false; - - // Clear max delay timer since we're syncing - if (this.maxDelayTimer) { - clearTimeout(this.maxDelayTimer); - this.maxDelayTimer = null; - } - - try { - const startTime = Date.now(); - const syncedCount = await this.syncService.sync(); - const duration = Date.now() - startTime; - const durationSeconds = duration / 1000; - - this.lastSyncTime = Date.now(); - this.firstDirtyTime = 0; - this.syncCount++; - this.lastError = null; - - // Track metrics - getMetrics()?.syncOperationsTotal.inc({ status: 'success' }); - getMetrics()?.syncDuration.observe(durationSeconds); - - if (syncedCount > 0) { - logger.info({ - syncedCount, - duration, - totalSyncs: this.syncCount, - }, 'Auto-sync completed'); - } else { - logger.debug('Auto-sync: no dirty conversations'); - } - } catch (error) { - this.lastError = error instanceof Error ? error : new Error(String(error)); - getMetrics()?.syncOperationsTotal.inc({ status: 'failure' }); - logger.error({ - error: this.lastError.message, - }, 'Auto-sync failed'); - - // Reschedule on failure (with backoff via throttle) - this.scheduleSync(); - } finally { - this.isSyncing = false; - } - } - - /** - * Force an immediate sync (bypasses debounce/throttle) - */ - async syncNow(): Promise<number> { - // Clear pending timers - if (this.debounceTimer) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; - } - if (this.maxDelayTimer) { - clearTimeout(this.maxDelayTimer); - this.maxDelayTimer = null; - } - - this.pendingSync = false; - this.firstDirtyTime = 0; - - const syncedCount = await this.syncService.sync(); - this.lastSyncTime = Date.now(); - this.syncCount++; - - return syncedCount; - } - - /** - * Stop auto-sync (cleanup timers) - */ - stop(): void { - if (this.debounceTimer) { - clearTimeout(this.debounceTimer); - this.debounceTimer = null; - } - if (this.maxDelayTimer) { - clearTimeout(this.maxDelayTimer); - this.maxDelayTimer = null; - } - this.pendingSync = false; - logger.info('AutoSyncService stopped'); - } - - /** - * Get auto-sync statistics - */ - getStats(): { - enabled: boolean; - syncCount: number; - lastSyncTime: number; - pendingSync: boolean; - isSyncing: boolean; - lastError: string | null; - } { - return { - enabled: this.enabled, - syncCount: this.syncCount, - lastSyncTime: this.lastSyncTime, - pendingSync: this.pendingSync, - isSyncing: this.isSyncing, - lastError: this.lastError?.message ?? null, - }; - } -} - -// Singleton instance -let autoSyncInstance: AutoSyncService | null = null; - -/** - * Get or create the global AutoSyncService instance - */ -export function getAutoSyncService( - syncService?: SyncService, - enabled?: boolean -): AutoSyncService { - if (!autoSyncInstance) { - if (!syncService) { - syncService = getSyncService(); - } - autoSyncInstance = new AutoSyncService(syncService, enabled); - } - return autoSyncInstance; -} - -/** - * Reset the AutoSyncService (for testing) - */ -export function resetAutoSyncService(): void { - if (autoSyncInstance) { - autoSyncInstance.stop(); - autoSyncInstance = null; - } -} diff --git a/src/conversations/fallback/sync/encryption-codec.ts b/src/conversations/fallback/sync/encryption-codec.ts deleted file mode 100644 index 648bbc5..0000000 --- a/src/conversations/fallback/sync/encryption-codec.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Encryption Codec for Lumo data persistence - * - * Handles encryption/decryption of spaces, conversations, and messages - * using AES-GCM with Authenticated Data (AEAD). - * - * AD format must match Lumo WebClient (json-stable-stringify with alphabetically sorted keys). - */ - -import stableStringify from 'json-stable-stringify'; -import { logger } from '../../../app/logger.js'; -import { encryptData, decryptData } from '@proton/crypto/lib/subtle/aesGcm'; -import { Role, type MessagePriv } from '@lumo/types.js'; -import type { - Message, - ProjectSpace, - ConversationPriv, -} from '../../types.js'; - -// Role mapping for AD construction -// Maps internal roles to API-compatible roles (user/assistant) -const RoleToApiInt: Record<Role, number> = { - [Role.User]: 1, - [Role.Assistant]: 2, - [Role.System]: 1, // Treat system as user for storage - [Role.ToolCall]: 2, // Tool calls are assistant messages - [Role.ToolResult]: 1, // Tool results are user messages -}; - -/** - * Encryption Codec - * - * Provides type-safe encryption/decryption for Lumo data types. - * All methods use AEAD with alphabetically-sorted JSON AD strings. - */ -export class EncryptionCodec { - constructor(private dataEncryptionKey: CryptoKey) {} - - /** - * Generic encryption with Authenticated Data - */ - private async encrypt<T>(data: T, ad: Record<string, string | undefined>): Promise<string> { - const json = JSON.stringify(data); - const plaintext = new TextEncoder().encode(json); - const adString = stableStringify(ad); - logger.debug({ adString }, 'Encrypting with AD'); - const adBytes = new TextEncoder().encode(adString); - - const encrypted = await encryptData(this.dataEncryptionKey, plaintext, adBytes); - return encrypted.toBase64(); - } - - /** - * Generic decryption with Authenticated Data - * Returns null on decryption failure (graceful fallback) - */ - private async decrypt<T>( - encryptedBase64: string, - ad: Record<string, string | undefined>, - entityType: string, - entityId: string - ): Promise<T | null> { - try { - const encrypted = Buffer.from(encryptedBase64, 'base64'); - const adString = stableStringify(ad); - logger.debug({ adString, entityType, entityId }, 'Decrypting with AD'); - const adBytes = new TextEncoder().encode(adString); - - const decrypted = await decryptData(this.dataEncryptionKey, new Uint8Array(encrypted), adBytes); - const json = new TextDecoder().decode(decrypted); - logger.debug({ entityType, entityId, json }, 'Successfully decrypted'); - return JSON.parse(json) as T; - } catch (error) { - logger.warn({ entityType, entityId, error }, `Failed to decrypt ${entityType} private data`); - return null; - } - } - - // --- Space --- - - async encryptSpace(data: ProjectSpace, spaceId: string): Promise<string> { - return this.encrypt(data, { - app: 'lumo', - type: 'space', - id: spaceId, - }); - } - - async decryptSpace(encryptedBase64: string, spaceId: string): Promise<ProjectSpace | null> { - return this.decrypt<ProjectSpace>( - encryptedBase64, - { - app: 'lumo', - type: 'space', - id: spaceId, - }, - 'space', - spaceId - ); - } - - // --- Conversation --- - - async encryptConversation( - data: ConversationPriv, - conversationId: string, - spaceId: string - ): Promise<string> { - return this.encrypt(data, { - app: 'lumo', - type: 'conversation', - id: conversationId, - spaceId: spaceId, - }); - } - - async decryptConversation( - encryptedBase64: string, - conversationId: string, - spaceId: string - ): Promise<ConversationPriv | null> { - return this.decrypt<ConversationPriv>( - encryptedBase64, - { - app: 'lumo', - type: 'conversation', - id: conversationId, - spaceId: spaceId, - }, - 'conversation', - conversationId - ); - } - - // --- Message --- - - /** - * Encrypt message private data - * - * IMPORTANT: The role in AD must be the mapped role (user/assistant) that matches what - * the WebClient will reconstruct from the Role integer field, NOT our internal role names. - */ - async encryptMessage( - data: MessagePriv, - message: Message, - effectiveParentId?: string - ): Promise<string> { - // Map our internal role to the role the WebClient will use for AD reconstruction - const roleInt = RoleToApiInt[message.role] ?? 1; - const adRole = roleInt === 2 ? 'assistant' : 'user'; - - return this.encrypt(data, { - app: 'lumo', - type: 'message', - id: message.id, - role: adRole, - parentId: effectiveParentId, - conversationId: message.conversationId, - }); - } - - /** - * Decrypt message private data - * - * @param role - Role from upstream (already a string: 'user'/'assistant') - */ - async decryptMessage( - encryptedBase64: string, - messageId: string, - conversationId: string, - role: string, - parentId?: string - ): Promise<MessagePriv | null> { - return this.decrypt<MessagePriv>( - encryptedBase64, - { - app: 'lumo', - type: 'message', - id: messageId, - role: role, - parentId: parentId, - conversationId: conversationId, - }, - 'message', - messageId - ); - } -} diff --git a/src/conversations/fallback/sync/index.ts b/src/conversations/fallback/sync/index.ts deleted file mode 100644 index 45c6c6d..0000000 --- a/src/conversations/fallback/sync/index.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Sync module for conversation persistence - * - * Provides: - * - LumoApi via adapter for API communication (from upstream WebClient) - * - SyncService for on-demand synchronization - */ - -export { - SyncService, - getSyncService, - resetSyncService, - type SyncServiceConfig, -} from './sync-service.js'; - -// Encryption codec -export { EncryptionCodec } from './encryption-codec.js'; - -// Space manager (for external use if needed) -export { - SpaceManager, - type SpaceManagerConfig, - type SpaceContext, -} from './space-manager.js'; - -// Auto-sync -export { - AutoSyncService, - getAutoSyncService, - resetAutoSyncService, -} from './auto-sync.js'; - -// Re-export LumoApi and types from upstream -export { LumoApi } from '@lumo/remote/api.js'; -export type { Priority } from '@lumo/remote/scheduler.js'; -export { - RoleInt, - StatusInt, -} from '@lumo/remote/types.js'; -export type { - ListSpacesRemote, - GetSpaceRemote, - GetConversationRemote, - RemoteMessage, - RemoteSpace, - RemoteConversation, -} from '@lumo/remote/types.js'; diff --git a/src/conversations/fallback/sync/space-manager.ts b/src/conversations/fallback/sync/space-manager.ts deleted file mode 100644 index ad3afaa..0000000 --- a/src/conversations/fallback/sync/space-manager.ts +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Space Manager for Lumo sync - * - * Handles space lifecycle: - * - Lazy space initialization (find existing or create new) - * - Space key derivation - * - Conversation ID mapping (local <-> remote) - */ - -import { randomUUID } from 'crypto'; -import { logger } from '../../../app/logger.js'; -import { exportKey, deriveKey } from '@proton/crypto/lib/subtle/aesGcm'; -import type { LumoApi } from '@lumo/remote/api.js'; -import type { RemoteSpace } from '@lumo/remote/types.js'; -import type { KeyManager } from '../../key-manager.js'; -import type { SpaceId, RemoteId, ProjectSpace } from '../../types.js'; -import { EncryptionCodec } from './encryption-codec.js'; - -// HKDF parameters matching Proton's implementation -const SPACE_KEY_DERIVATION_SALT = Buffer.from('Xd6V94/+5BmLAfc67xIBZcjsBPimm9/j02kHPI7Vsuc=', 'base64'); -const SPACE_DEK_CONTEXT = new TextEncoder().encode('dek.space.lumo'); - -export interface SpaceManagerConfig { - lumoApi: LumoApi; - keyManager: KeyManager; - spaceName: string; -} - -export interface SpaceContext { - spaceId: SpaceId; - remoteId: RemoteId; -} - -/** - * Space Manager - * - * Manages space lifecycle and provides access to the encryption codec. - */ -export class SpaceManager { - private lumoApi: LumoApi; - private keyManager: KeyManager; - private spaceName: string; - - // Current space state - private _spaceId?: SpaceId; - private _spaceRemoteId?: RemoteId; - private spaceKey?: CryptoKey; - private dataEncryptionKey?: CryptoKey; - private _codec?: EncryptionCodec; - - // ID mappings for conversations (messages handled by SyncService) - private conversationIdMap = new Map<string, RemoteId>(); - private existingConversationsLoaded = false; - - constructor(config: SpaceManagerConfig) { - this.lumoApi = config.lumoApi; - this.keyManager = config.keyManager; - this.spaceName = config.spaceName; - } - - // --- Accessors --- - - get spaceId(): SpaceId | undefined { - return this._spaceId; - } - - get spaceRemoteId(): RemoteId | undefined { - return this._spaceRemoteId; - } - - get codec(): EncryptionCodec | undefined { - return this._codec; - } - - getConversationRemoteId(localId: string): RemoteId | undefined { - return this.conversationIdMap.get(localId); - } - - setConversationRemoteId(localId: string, remoteId: RemoteId): void { - this.conversationIdMap.set(localId, remoteId); - } - - // --- Space Lifecycle --- - - /** - * Ensure a space exists, creating one if needed - * Called lazily on first sync - * - * Finds space by projectName, creates if not found. - */ - async getOrCreateSpace(): Promise<SpaceContext> { - // Already have a space - if (this._spaceId && this._spaceRemoteId && this.spaceKey) { - return { spaceId: this._spaceId, remoteId: this._spaceRemoteId }; - } - - logger.info({ spaceName: this.spaceName }, 'Checking for existing project...'); - - const listResult = await this.lumoApi.listSpaces(); - const existingSpaces = Object.values(listResult.spaces); - - const spacesWithData = existingSpaces.filter(s => s.encrypted); - logger.info({ - totalSpaces: existingSpaces.length, - spacesWithEncryptedData: spacesWithData.length, - spaceTags: existingSpaces.map(s => s.id), - }, 'Available projects'); - - return this.findSpaceByName(existingSpaces); - } - - private async findSpaceByName(existingSpaces: RemoteSpace[]): Promise<SpaceContext> { - logger.info(`Looking up project by name "${this.spaceName}" (among ${existingSpaces.length} projects)`); - - for (const space of existingSpaces) { - if (!space.id) continue; - - try { - const spaceKey = await this.keyManager.getSpaceKey(space.id, space.wrappedSpaceKey); - const dataEncryptionKey = await this.deriveDataEncryptionKey(spaceKey); - const codec = new EncryptionCodec(dataEncryptionKey); - - const encryptedData = typeof space.encrypted === 'string' ? space.encrypted : undefined; - logger.debug({ - spaceTag: space.id, - hasEncrypted: !!encryptedData, - encryptedLength: encryptedData?.length ?? 0, - }, 'Checking space'); - - if (!encryptedData) continue; - - const projectSpace = await codec.decryptSpace(encryptedData, space.id); - - logger.debug({ - spaceTag: space.id, - projectName: projectSpace?.projectName, - lookingFor: this.spaceName, - decryptedOk: !!projectSpace, - }, 'Checking project name match'); - - if (projectSpace && projectSpace.projectName === this.spaceName) { - this._spaceId = space.id; - this._spaceRemoteId = space.remoteId; - this.spaceKey = spaceKey; - this.dataEncryptionKey = dataEncryptionKey; - this._codec = codec; - - logger.info({ - spaceId: space.id, - remoteId: space.remoteId, - projectName: projectSpace.projectName, - }, 'Found existing project'); - - return { spaceId: this._spaceId, remoteId: this._spaceRemoteId }; - } - } catch (error) { - logger.debug({ spaceTag: space.id, error }, 'Could not decrypt space'); - continue; - } - } - - // No matching space found, create a new one - if (!this.spaceName) { - throw new Error('Cannot create project: no projectName configured'); - } - logger.info({ spaceName: this.spaceName }, 'Creating new project...'); - return this.createSpace(); - } - - /** - * Create a new space on the server - */ - private async createSpace(): Promise<SpaceContext> { - const localId = randomUUID(); - - // Generate a new space key and get it cached in KeyManager - const spaceKey = await this.keyManager.getSpaceKey(localId); - const wrappedSpaceKey = await this.keyManager.wrapSpaceKey(localId); - const dataEncryptionKey = await this.deriveDataEncryptionKey(spaceKey); - const codec = new EncryptionCodec(dataEncryptionKey); - - const projectSpace: ProjectSpace = { - isProject: true, - projectName: this.spaceName, - }; - const encryptedPrivate = await codec.encryptSpace(projectSpace, localId); - - const remoteId = await this.lumoApi.postSpace({ - SpaceKey: wrappedSpaceKey, - SpaceTag: localId, - Encrypted: encryptedPrivate, - }, 'background'); - - if (!remoteId) { - throw new Error('Failed to create project - no remote ID returned'); - } - - // Cache state - this._spaceId = localId; - this._spaceRemoteId = remoteId; - this.spaceKey = spaceKey; - this.dataEncryptionKey = dataEncryptionKey; - this._codec = codec; - - logger.info({ - spaceId: localId, - remoteId, - projectName: this.spaceName, - }, 'Created new project'); - - return { spaceId: localId, remoteId }; - } - - private async initializeSpaceKeys(space: { - id: string; - remoteId: string; - wrappedSpaceKey: string; - }): Promise<void> { - const spaceKey = await this.keyManager.getSpaceKey(space.id, space.wrappedSpaceKey); - const dataEncryptionKey = await this.deriveDataEncryptionKey(spaceKey); - - this._spaceId = space.id; - this._spaceRemoteId = space.remoteId; - this.spaceKey = spaceKey; - this.dataEncryptionKey = dataEncryptionKey; - this._codec = new EncryptionCodec(dataEncryptionKey); - } - - /** - * Derive data encryption key from space key using HKDF - */ - private async deriveDataEncryptionKey(spaceKey: CryptoKey): Promise<CryptoKey> { - const keyBytes = await exportKey(spaceKey); - return deriveKey(keyBytes, new Uint8Array(SPACE_KEY_DERIVATION_SALT), SPACE_DEK_CONTEXT); - } - - // --- Conversation ID Loading --- - - /** - * Ensure existing conversations are loaded from server (lazy, called once) - * Populates conversationIdMap with conversation IDs to prevent 409 errors on sync - */ - async ensureExistingConversationsLoaded(): Promise<void> { - if (this.existingConversationsLoaded) return; - if (!this._spaceRemoteId || !this._spaceId) return; - - try { - const spaceData = await this.lumoApi.getSpace(this._spaceRemoteId); - if (!spaceData) { - logger.warn({ spaceRemoteId: this._spaceRemoteId }, 'Project not found on server'); - return; - } - - for (const conv of spaceData.conversations ?? []) { - try { - const convData = await this.lumoApi.getConversation(conv.remoteId, this._spaceId); - if (convData?.conversation) { - const localId = convData.conversation.id; - this.conversationIdMap.set(localId, conv.remoteId); - logger.debug({ localId, remoteId: conv.remoteId }, 'Mapped conversation'); - } - } catch (error) { - logger.warn({ remoteId: conv.remoteId, error }, 'Failed to fetch conversation'); - } - } - - this.existingConversationsLoaded = true; - - logger.info({ - conversations: this.conversationIdMap.size, - }, 'Loaded existing conversation IDs from server'); - } catch (error) { - logger.error({ error }, 'Failed to load existing conversations'); - } - } - - // --- Cleanup --- - - /** - * Delete ALL spaces from the server - * WARNING: This is destructive and cannot be undone! - */ - async deleteAllSpaces(): Promise<number> { - const listResult = await this.lumoApi.listSpaces(); - const spaces = Object.values(listResult.spaces); - logger.warn({ count: spaces.length }, 'Deleting ALL projects...'); - - let deleted = 0; - for (const space of spaces) { - try { - await this.lumoApi.deleteSpace(space.remoteId, 'background'); - deleted++; - logger.info({ spaceId: space.id, remoteId: space.remoteId }, 'Deleted project'); - } catch (error) { - logger.error({ spaceId: space.id, error }, 'Failed to delete project'); - } - } - - // Clear local state - this._spaceId = undefined; - this._spaceRemoteId = undefined; - this.spaceKey = undefined; - this.dataEncryptionKey = undefined; - this._codec = undefined; - this.conversationIdMap.clear(); - - logger.warn({ deleted, total: spaces.length }, 'Finished deleting projects'); - return deleted; - } - - /** - * Reset state (for testing) - */ - reset(): void { - this._spaceId = undefined; - this._spaceRemoteId = undefined; - this.spaceKey = undefined; - this.dataEncryptionKey = undefined; - this._codec = undefined; - this.conversationIdMap.clear(); - this.existingConversationsLoaded = false; - } -} diff --git a/src/conversations/fallback/sync/sync-service.ts b/src/conversations/fallback/sync/sync-service.ts deleted file mode 100644 index 4df9e1e..0000000 --- a/src/conversations/fallback/sync/sync-service.ts +++ /dev/null @@ -1,447 +0,0 @@ -/** - * Sync Service for conversation persistence - * - * Orchestrates syncing conversations to the Lumo server. - * Delegates to SpaceManager for space lifecycle and EncryptionCodec for encryption. - */ - -import { logger } from '../../../app/logger.js'; -import { LumoApi } from '@lumo/remote/api.js'; -import { RoleInt, StatusInt } from '@lumo/remote/types.js'; -import { getFallbackStore } from '../store.js'; -import type { KeyManager } from '../../key-manager.js'; -import { Role, type Status, type MessagePriv } from '@lumo/types.js'; -import type { ConversationState, Message, SpaceId, RemoteId } from '../../types.js'; -import { SpaceManager } from './space-manager.js'; - -// Role mapping: our internal roles to API integer values -const RoleToInt: Record<Role, number> = { - [Role.User]: RoleInt.User, - [Role.Assistant]: RoleInt.Assistant, - [Role.System]: RoleInt.User, - [Role.ToolCall]: RoleInt.Assistant, - [Role.ToolResult]: RoleInt.User, -}; - -// Status mapping: our internal status to API integer values -const StatusToInt: Record<Status, number> = { - failed: StatusInt.Failed, - succeeded: StatusInt.Succeeded, -}; - -export interface SyncServiceConfig { - keyManager: KeyManager; - uid: string; - spaceName: string; -} - -/** - * Find the synced parent message in the chain - * - * Walks up the parent chain until finding a message that was synced to the server. - * This handles filtered messages (e.g., system messages) by finding their synced ancestors. - */ -function findSyncedParent( - messageParentId: string | undefined, - messageMap: Map<string, Message>, - messageIdMap: Map<string, RemoteId> -): { effectiveParentId?: string; parentRemoteId?: RemoteId } { - let effectiveParentId = messageParentId; - while (effectiveParentId) { - const parentRemoteId = messageIdMap.get(effectiveParentId); - if (parentRemoteId) { - return { effectiveParentId, parentRemoteId }; - } - effectiveParentId = messageMap.get(effectiveParentId)?.parentId; - } - return {}; -} - -/** - * Sync Service - * - * Manages server-side persistence for conversations. - */ -export class SyncService { - private lumoApi: LumoApi; - private keyManager: KeyManager; - private spaceManager: SpaceManager; - - // Message ID mapping (local -> remote) - private messageIdMap = new Map<string, RemoteId>(); - - constructor(config: SyncServiceConfig) { - this.lumoApi = new LumoApi(config.uid); - this.keyManager = config.keyManager; - - this.spaceManager = new SpaceManager({ - lumoApi: this.lumoApi, - keyManager: config.keyManager, - spaceName: config.spaceName, - }); - } - - /** - * Ensure a space exists, creating one if needed - */ - async getOrCreateSpace(): Promise<{ spaceId: SpaceId; remoteId: RemoteId }> { - return this.spaceManager.getOrCreateSpace(); - } - - /** - * Ensure existing conversations are loaded from server - */ - async ensureExistingConversationsLoaded(): Promise<void> { - return this.spaceManager.ensureExistingConversationsLoaded(); - } - - /** - * Sync all dirty conversations to the server - */ - async sync(): Promise<number> { - if (!this.keyManager.isInitialized()) { - throw new Error('KeyManager not initialized - cannot sync without encryption keys'); - } - - const { remoteId: spaceRemoteId } = await this.getOrCreateSpace(); - - const store = getFallbackStore(); - const dirtyConversations = store.getDirty(); - - if (dirtyConversations.length === 0) { - logger.info('No dirty conversations to sync'); - return 0; - } - - logger.info({ count: dirtyConversations.length }, 'Syncing dirty conversations'); - - let syncedCount = 0; - for (const conversation of dirtyConversations) { - try { - await this.syncConversation(conversation, spaceRemoteId); - store.markSynced(conversation.metadata.id); - syncedCount++; - } catch (error) { - logger.error({ - conversationId: conversation.metadata.id, - error, - }, 'Failed to sync conversation'); - } - } - - logger.info({ syncedCount, total: dirtyConversations.length }, 'Sync complete'); - return syncedCount; - } - - /** - * Sync a single conversation by ID - */ - async syncById(conversationId: string): Promise<boolean> { - if (!this.keyManager.isInitialized()) { - throw new Error('KeyManager not initialized - cannot sync without encryption keys'); - } - - const store = getFallbackStore(); - const conversation = store.get(conversationId); - - if (!conversation) { - logger.warn({ conversationId }, 'Conversation not found for sync'); - return false; - } - - if (!conversation.dirty) { - logger.info({ conversationId }, 'Conversation already synced'); - return true; - } - - const { remoteId: spaceRemoteId } = await this.getOrCreateSpace(); - - // Mark as synced early to prevent auto-sync from picking it up concurrently - store.markSynced(conversationId); - - logger.info({ conversationId }, 'Syncing single conversation'); - - try { - await this.syncConversation(conversation, spaceRemoteId); - logger.info({ conversationId }, 'Conversation synced successfully'); - return true; - } catch (error) { - store.markDirtyById(conversationId); - logger.error({ conversationId, error }, 'Failed to sync conversation'); - throw error; - } - } - - /** - * Sync a single conversation to the server - */ - private async syncConversation( - conversation: ConversationState, - spaceRemoteId: RemoteId - ): Promise<void> { - const conversationId = conversation.metadata.id; - const spaceId = this.spaceManager.spaceId!; - const codec = this.spaceManager.codec!; - - let conversationRemoteId = this.spaceManager.getConversationRemoteId(conversationId); - - if (!conversationRemoteId) { - // Create new conversation - const encryptedPrivate = await codec.encryptConversation( - { title: conversation.title }, - conversationId, - spaceId - ); - - const newRemoteId = await this.lumoApi.postConversation({ - SpaceID: spaceRemoteId, - IsStarred: conversation.metadata.starred ?? false, - ConversationTag: conversationId, - Encrypted: encryptedPrivate, - }, 'background'); - - if (!newRemoteId) { - throw new Error(`Failed to create conversation ${conversationId}`); - } - conversationRemoteId = newRemoteId; - this.spaceManager.setConversationRemoteId(conversationId, conversationRemoteId); - logger.debug({ conversationId, remoteId: conversationRemoteId }, 'Created conversation on server'); - } else { - // Update existing conversation - const encryptedPrivate = await codec.encryptConversation( - { title: conversation.title }, - conversationId, - spaceId - ); - - await this.lumoApi.putConversation({ - ID: conversationRemoteId, - SpaceID: spaceRemoteId, - IsStarred: conversation.metadata.starred ?? false, - ConversationTag: conversationId, - Encrypted: encryptedPrivate, - }, 'background'); - logger.debug({ conversationId, remoteId: conversationRemoteId }, 'Updated conversation on server'); - } - - // Sync all messages - const messageMap = new Map(conversation.messages.map(m => [m.id, m])); - - for (const message of conversation.messages) { - await this.syncMessage(message, conversationRemoteId, messageMap); - } - } - - /** - * Sync a single message to the server - */ - private async syncMessage( - message: Message, - conversationRemoteId: RemoteId, - messageMap: Map<string, Message> - ): Promise<void> { - // Skip if already synced (messages are immutable) - if (this.messageIdMap.has(message.id)) { - return; - } - - const codec = this.spaceManager.codec!; - - // Prefix non-user/assistant content with role for clarity in Proton UI - let contentToStore = message.content; - if (message.role !== Role.User && message.role !== Role.Assistant) { - contentToStore = `[${message.role}]\n${message.content}`; - } - - // Find the synced parent (walk up chain if parent was filtered) - const { effectiveParentId, parentRemoteId } = findSyncedParent( - message.parentId, - messageMap, - this.messageIdMap - ); - - const messagePrivate: MessagePriv = { - content: contentToStore, - context: message.context, - blocks: message.blocks, - }; - - const encryptedPrivate = await codec.encryptMessage(messagePrivate, message, effectiveParentId); - - const remoteId = await this.lumoApi.postMessage({ - ConversationID: conversationRemoteId, - Role: RoleToInt[message.role] ?? RoleInt.User, - ParentID: parentRemoteId, - ParentId: parentRemoteId, // Duplicate for buggy backend - Status: StatusToInt[message.status ?? 'succeeded'], - MessageTag: message.id, - Encrypted: encryptedPrivate, - }, 'background'); - - if (!remoteId) { - throw new Error(`Failed to create message ${message.id}`); - } - - this.messageIdMap.set(message.id, remoteId); - logger.debug({ messageId: message.id, remoteId }, 'Created message on server'); - } - - /** - * Load a single conversation from the server by local ID - */ - async loadExistingConversation(localId: string): Promise<string | undefined> { - if (!this.keyManager.isInitialized()) { - throw new Error('KeyManager not initialized - cannot load without encryption keys'); - } - - await this.getOrCreateSpace(); - - const spaceId = this.spaceManager.spaceId; - const codec = this.spaceManager.codec; - if (!spaceId || !codec) { - throw new Error('Space not initialized - cannot decrypt conversation'); - } - - await this.ensureExistingConversationsLoaded(); - - const remoteId = this.spaceManager.getConversationRemoteId(localId); - if (!remoteId) { - logger.warn({ localId }, 'Conversation not found in project'); - return undefined; - } - - try { - const convData = await this.lumoApi.getConversation(remoteId, spaceId); - if (!convData?.conversation) { - logger.warn({ localId, remoteId }, 'Conversation not found on server'); - return undefined; - } - - const conv = convData.conversation; - if ('deleted' in conv && conv.deleted) { - logger.warn({ localId }, 'Conversation is deleted'); - return undefined; - } - - // Decrypt title - let title = 'Untitled'; - if (conv.encrypted && typeof conv.encrypted === 'string') { - const decryptedPrivate = await codec.decryptConversation(conv.encrypted, localId, spaceId); - if (decryptedPrivate?.title) { - title = decryptedPrivate.title; - } - } - - // Create/update in store - const store = getFallbackStore(); - const state = store.getOrCreate(localId); - state.title = title; - state.metadata.starred = conv.starred ?? false; - state.metadata.createdAt = conv.createdAt; - state.metadata.spaceId = spaceId; - state.remoteId = remoteId; - state.dirty = false; - state.messages = []; - - // Load messages - for (const msg of convData.messages ?? []) { - this.messageIdMap.set(msg.id, msg.remoteId); - - const fullMsg = await this.lumoApi.getMessage( - msg.remoteId, - localId, - msg.parentId, - remoteId - ); - - let messagePrivate: MessagePriv | null = null; - if (fullMsg?.encrypted && typeof fullMsg.encrypted === 'string') { - messagePrivate = await codec.decryptMessage( - fullMsg.encrypted, - msg.id, - localId, - msg.role, - msg.parentId - ); - } - - state.messages.push({ - id: msg.id, - conversationId: localId, - createdAt: msg.createdAt, - role: msg.role , - parentId: msg.parentId, - status: msg.status as Status | undefined, - content: messagePrivate?.content, - context: messagePrivate?.context, - blocks: messagePrivate?.blocks, - }); - } - - store.markSynced(localId); - - logger.info({ - localId, - remoteId, - title, - messageCount: state.messages.length, - }, 'Loaded conversation from server'); - - return localId; - } catch (error) { - logger.error({ localId, error }, 'Failed to load conversation'); - throw error; - } - } - - /** - * Get sync statistics - */ - getStats(): { - hasSpace: boolean; - spaceId?: SpaceId; - spaceRemoteId?: RemoteId; - mappedConversations: number; - mappedMessages: number; - } { - return { - hasSpace: !!this.spaceManager.spaceId, - spaceId: this.spaceManager.spaceId, - spaceRemoteId: this.spaceManager.spaceRemoteId, - mappedConversations: 0, // SpaceManager doesn't expose this count - mappedMessages: this.messageIdMap.size, - }; - } - - /** - * Delete ALL spaces from the server - */ - async deleteAllSpaces(): Promise<number> { - const deleted = await this.spaceManager.deleteAllSpaces(); - this.messageIdMap.clear(); - return deleted; - } -} - -// Singleton instance -let syncServiceInstance: SyncService | null = null; - -/** - * Get the global SyncService instance - */ -export function getSyncService(config?: SyncServiceConfig): SyncService { - if (!syncServiceInstance && config) { - syncServiceInstance = new SyncService(config); - } - if (!syncServiceInstance) { - throw new Error('SyncService not initialized - call with config first'); - } - return syncServiceInstance; -} - -/** - * Reset the SyncService (for testing) - */ -export function resetSyncService(): void { - syncServiceInstance = null; -} diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 9f2e2b2..7032980 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -3,7 +3,7 @@ * * Provides: * - ConversationStore: Primary storage using Redux + IndexedDB - * - FallbackStore: Legacy in-memory storage (deprecated) + * - MinimalStore: Lightweight in-memory storage for CLI/tests * - Message deduplication for OpenAI API format * - Types compatible with Proton Lumo webclient */ @@ -23,9 +23,15 @@ export type { MessageForStore, } from './types.js'; +// Store interface +export type { IConversationStore } from './store-interface.js'; + // Primary store export { ConversationStore } from './store.js'; +// Minimal store (fallback for CLI/tests) +export { MinimalStore } from './minimal-store.js'; + // Store initialization export { initializeStore, @@ -51,23 +57,6 @@ export { type KeyManagerConfig, } from './key-manager.js'; -// Fallback store and its sync (deprecated) -export { - FallbackStore, - getFallbackStore, - resetFallbackStore, -} from './fallback/index.js'; - -export { - SyncService, - getSyncService, - resetSyncService, - type SyncServiceConfig, - AutoSyncService, - getAutoSyncService, - resetAutoSyncService, -} from './fallback/index.js'; - // Re-export LumoApi types for consumers export { LumoApi } from '@lumo/remote/api.js'; export { RoleInt, StatusInt } from '@lumo/remote/types.js'; @@ -80,9 +69,8 @@ import { logger } from '../app/logger.js'; import type { AuthProvider, ProtonApi } from '../auth/index.js'; import type { ConversationsConfig } from '../app/config.js'; import { getKeyManager } from './key-manager.js'; -import { getSyncService, getAutoSyncService, getFallbackStore } from './fallback/index.js'; -import { ConversationStore } from './store.js'; import { initializeStore, type StoreResult } from './init.js'; +import type { IConversationStore } from './store-interface.js'; // ============================================================================ // Conversation Store Initialization @@ -105,39 +93,31 @@ export interface InitializeStoreResult { // Module-level state to track store result for sync initialization let primaryStoreResult: StoreResult | null = null; -// Singleton for the active store (either ConversationStore or FallbackStore) -let activeStore: ConversationStore | ReturnType<typeof getFallbackStore> | null = null; +// Singleton for the active store +let activeStore: IConversationStore | null = null; /** * Initialize the conversation store * - * Creates either the primary ConversationStore (Redux + IndexedDB) or - * the fallback in-memory FallbackStore. + * Creates the primary ConversationStore (Redux + IndexedDB) if possible. + * Returns undefined if initialization fails - callers should handle this + * gracefully (server works stateless, CLI creates MinimalStore). * - * Primary store is used when: - * - useFallbackStore config is false (default) - * - Auth provider supports persistence (browser auth) + * Primary store requires: + * - Auth provider supports persistence (has cached encryption keys) * - keyPassword is available (for master key decryption) */ export async function initializeConversationStore( options: InitializeStoreOptions ): Promise<InitializeStoreResult> { - const { protonApi, uid, authProvider, conversationsConfig } = options; - - // Check if fallback is explicitly requested - if (conversationsConfig.useFallbackStore) { - logger.info('Using fallback store (explicitly configured)'); - activeStore = getFallbackStore(); - return { isPrimary: false }; - } + const { authProvider } = options; - // Try to initialize primary store + // Check prerequisites for primary store if (!authProvider.supportsPersistence()) { logger.warn( { method: authProvider.method }, - 'Primary store requires cached encryption keys. Falling back to in-memory store.' + 'Primary store requires encryption keys. Continuing without store.' ); - activeStore = getFallbackStore(); return { isPrimary: false }; } @@ -145,9 +125,8 @@ export async function initializeConversationStore( if (!keyPassword) { logger.warn( { method: authProvider.method }, - 'Primary store requires keyPassword. Falling back to in-memory store.' + 'Primary store requires keyPassword. Continuing without store.' ); - activeStore = getFallbackStore(); return { isPrimary: false }; } @@ -162,11 +141,9 @@ export async function initializeConversationStore( } } catch (error) { const msg = error instanceof Error ? error.message : String(error); - logger.error({ error: msg }, 'Failed to initialize primary store. Falling back to in-memory store.'); + logger.error({ error: msg }, 'Failed to initialize primary store. Continuing without store.'); } - // Fallback - activeStore = getFallbackStore(); return { isPrimary: false }; } @@ -216,20 +193,17 @@ async function initializePrimaryStore( /** * Get the active conversation store * - * Returns whichever store was initialized (primary or fallback). - * Throws if no store has been initialized. + * Returns the initialized store, or undefined if no store is available. + * Callers should handle undefined gracefully (stateless mode). */ -export function getConversationStore(): ConversationStore | ReturnType<typeof getFallbackStore> { - if (!activeStore) { - throw new Error('ConversationStore not initialized - call initializeConversationStore() first'); - } - return activeStore; +export function getConversationStore(): IConversationStore | undefined { + return activeStore ?? undefined; } /** - * Set the active conversation store (for mock mode) + * Set the active conversation store (for mock mode or CLI fallback) */ -export function setConversationStore(store: ConversationStore | ReturnType<typeof getFallbackStore>): void { +export function setConversationStore(store: IConversationStore): void { activeStore = store; } @@ -261,102 +235,40 @@ export interface InitializeSyncResult { /** * Initialize sync services * - * For primary store: sync is handled automatically by Redux sagas. - * For fallback store: sets up SyncService and AutoSyncService. + * Sync is handled automatically by Redux sagas when primary store is active. + * Returns initialized: false if no primary store or sync is disabled. */ export async function initializeSync( options: InitializeSyncOptions ): Promise<InitializeSyncResult> { - const { protonApi, uid, authProvider, conversationsConfig } = options; + const { authProvider, conversationsConfig } = options; if (!conversationsConfig?.enableSync) { logger.info('Sync is disabled, skipping sync initialization'); return { initialized: false }; } - // Sync requires browser auth for lumo scope (spaces API access) - if (!authProvider.supportsFullApi()) { - logger.warn( - { method: authProvider.method }, - 'Conversation sync requires browser auth method' - ); + // Sync requires primary store (sagas handle sync) + if (!primaryStoreResult) { + logger.info('No primary store - sync not available'); return { initialized: false }; } - const keyPassword = authProvider.getKeyPassword(); - if (!keyPassword) { + // Sync requires browser auth for lumo scope (spaces API access) + if (!authProvider.supportsFullApi()) { logger.warn( { method: authProvider.method }, - 'No keyPassword available - sync will not be initialized' + 'Conversation sync requires browser auth method' ); return { initialized: false }; } - try { - // Primary store: sync is handled by sagas - if (primaryStoreResult) { - logger.info( - { method: authProvider.method }, - 'Sync initialized (handled by sagas)' - ); - return { - initialized: true, - storeResult: primaryStoreResult, - }; - } - - // Fallback store: use SyncService + AutoSyncService - const cachedUserKeys = authProvider.getCachedUserKeys?.(); - const cachedMasterKeys = authProvider.getCachedMasterKeys?.(); - - logger.info( - { - method: authProvider.method, - hasCachedUserKeys: !!cachedUserKeys, - hasCachedMasterKeys: !!cachedMasterKeys, - }, - 'Initializing KeyManager for fallback sync...' - ); - - // Initialize KeyManager (needed for fallback sync) - const keyManager = getKeyManager({ - protonApi, - cachedUserKeys, - cachedMasterKeys, - }); - - await keyManager.initialize(keyPassword); - - // Initialize SyncService - const syncService = getSyncService({ - uid, - keyManager, - spaceName: conversationsConfig.projectName, - }); - - // Eagerly fetch/create space - try { - await syncService.getOrCreateSpace(); - logger.info({ method: authProvider.method }, 'Fallback sync service initialized'); - } catch (spaceError) { - const msg = spaceError instanceof Error ? spaceError.message : String(spaceError); - logger.warn({ error: msg }, 'getOrCreateSpace failed'); - } - - // Initialize auto-sync - const autoSync = getAutoSyncService(syncService, true); - - // Connect to fallback store - const store = getFallbackStore(); - store.setOnDirtyCallback(() => autoSync.notifyDirty()); - - logger.info('Auto-sync enabled for fallback store'); - - return { initialized: true }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - const errorStack = error instanceof Error ? error.stack : undefined; - logger.error({ errorMessage, errorStack }, 'Failed to initialize sync service'); - return { initialized: false }; - } + logger.info( + { method: authProvider.method }, + 'Sync initialized (handled by sagas)' + ); + return { + initialized: true, + storeResult: primaryStoreResult, + }; } diff --git a/src/conversations/minimal-store.ts b/src/conversations/minimal-store.ts new file mode 100644 index 0000000..77d4630 --- /dev/null +++ b/src/conversations/minimal-store.ts @@ -0,0 +1,256 @@ +/** + * MinimalStore - Lightweight in-memory store for single-session use + * + * Used when primary ConversationStore cannot be initialized: + * - CLI mode: tracks one conversation per session + * - Tests: provides mock store without IndexedDB overhead + * + * Only stores messages and converts them to turns for Lumo. + * Many methods are no-ops since CLI doesn't need full store functionality. + */ + +import { randomUUID } from 'crypto'; +import { logger } from '../app/logger.js'; +import { Role, ConversationStatus } from '@lumo/types.js'; +import type { Turn, AssistantMessageData } from '../lumo-client/types.js'; +import { + findNewMessages, + hashMessage, + isValidContinuation, +} from './deduplication.js'; +import type { + ConversationId, + ConversationState, + Message, + MessageId, + MessageForStore, + SpaceId, +} from './types.js'; +import type { IConversationStore } from './store-interface.js'; +import { getMetrics } from '../app/metrics.js'; + +/** + * Lightweight in-memory conversation store + */ +export class MinimalStore implements IConversationStore { + private conversations = new Map<ConversationId, ConversationState>(); + private defaultSpaceId: SpaceId; + + constructor() { + this.defaultSpaceId = randomUUID(); + logger.debug({ spaceId: this.defaultSpaceId }, 'MinimalStore initialized'); + } + + getOrCreate(id: ConversationId): ConversationState { + let state = this.conversations.get(id); + + if (!state) { + state = this.createEmptyState(id); + this.conversations.set(id, state); + getMetrics()?.conversationsCreatedTotal.inc(); + logger.debug({ conversationId: id }, 'Created new conversation'); + } + + return state; + } + + get(id: ConversationId): ConversationState | undefined { + return this.conversations.get(id); + } + + has(id: ConversationId): boolean { + return this.conversations.has(id); + } + + appendMessages(id: ConversationId, incoming: MessageForStore[]): Message[] { + const state = this.getOrCreate(id); + + const validation = isValidContinuation(incoming, state.messages); + if (!validation.valid) { + getMetrics()?.invalidContinuationsTotal.inc(); + logger.warn({ + conversationId: id, + reason: validation.reason, + incomingCount: incoming.length, + storedCount: state.messages.length, + }, 'Invalid conversation continuation'); + } + + const newMessages = findNewMessages(incoming, state.messages); + + if (newMessages.length === 0) { + return []; + } + + const now = new Date().toISOString(); + const lastMessageId = state.messages.length > 0 + ? state.messages[state.messages.length - 1].id + : undefined; + + const addedMessages: Message[] = []; + let parentId = lastMessageId; + + for (const msg of newMessages) { + const semanticId = msg.id ?? hashMessage(msg.role, msg.content ?? '').slice(0, 16); + + const message: Message = { + id: randomUUID(), + conversationId: id, + createdAt: now, + role: msg.role, + parentId, + status: 'succeeded', + content: msg.content, + semanticId, + }; + + state.messages.push(message); + addedMessages.push(message); + parentId = message.id; + } + + state.metadata.updatedAt = new Date().toISOString(); + + const metrics = getMetrics(); + if (metrics) { + for (const msg of addedMessages) { + metrics.messagesTotal.inc({ role: msg.role }); + } + } + + return addedMessages; + } + + appendAssistantResponse( + id: ConversationId, + messageData: AssistantMessageData, + status: 'succeeded' | 'failed' = 'succeeded', + semanticId?: string + ): Message { + const state = this.getOrCreate(id); + const now = new Date(); + + const parentId = state.messages.length > 0 + ? state.messages[state.messages.length - 1].id + : undefined; + + const message: Message = { + id: randomUUID(), + conversationId: id, + createdAt: now.toISOString(), + role: Role.Assistant, + parentId, + status, + content: messageData.content, + blocks: messageData.blocks, + semanticId: semanticId ?? hashMessage(Role.Assistant, messageData.content).slice(0, 16), + }; + + state.messages.push(message); + state.metadata.updatedAt = now.toISOString(); + state.status = ConversationStatus.COMPLETED; + + getMetrics()?.messagesTotal.inc({ role: Role.Assistant }); + + return message; + } + + appendUserMessage(id: ConversationId, content: string): Message { + const state = this.getOrCreate(id); + const now = new Date(); + + const parentId = state.messages.length > 0 + ? state.messages[state.messages.length - 1].id + : undefined; + + const message: Message = { + id: randomUUID(), + conversationId: id, + createdAt: now.toISOString(), + role: Role.User, + parentId, + status: 'succeeded', + content, + semanticId: hashMessage(Role.User, content).slice(0, 16), + }; + + state.messages.push(message); + state.metadata.updatedAt = now.toISOString(); + + return message; + } + + toTurns(id: ConversationId): Turn[] { + return this.getMessages(id).map(({ role, content }) => ({ + role, + content, + })); + } + + getMessages(id: ConversationId): Message[] { + const state = this.conversations.get(id); + return state?.messages ?? []; + } + + // No-op methods (keep interface compatibility but not needed for CLI) + + setTitle(_id: ConversationId, _title: string): void { + // No-op: CLI never reads title back + } + + createFromTurns( + _turns: Turn[], + _title?: string + ): { conversationId: ConversationId; title: string } { + // No-op: requires sync which MinimalStore doesn't support + return { conversationId: '', title: '' }; + } + + appendAssistantToolCalls( + id: ConversationId, + toolCalls: Array<{ name: string; arguments: string; call_id: string }> + ): void { + for (const tc of toolCalls) { + const content = JSON.stringify({ + type: 'function_call', + call_id: tc.call_id, + name: tc.name, + arguments: tc.arguments, + }); + this.appendAssistantResponse(id, { content }, 'succeeded', tc.call_id); + } + } + + getMessage(_conversationId: ConversationId, _messageId: MessageId): Message | undefined { + // No-op + return undefined; + } + + delete(_id: ConversationId): boolean { + // No-op + return false; + } + + *entries(): IterableIterator<[ConversationId, ConversationState]> { + // No-op: yield nothing + } + + // Private methods + + private createEmptyState(id: ConversationId): ConversationState { + const now = new Date().toISOString(); + return { + metadata: { + id, + spaceId: this.defaultSpaceId, + createdAt: now, + updatedAt: now, + starred: false, + }, + title: 'New Conversation', + status: ConversationStatus.COMPLETED, + messages: [], + dirty: false, + }; + } +} diff --git a/src/conversations/store-interface.ts b/src/conversations/store-interface.ts new file mode 100644 index 0000000..357d0c1 --- /dev/null +++ b/src/conversations/store-interface.ts @@ -0,0 +1,46 @@ +/** + * Common interface for conversation stores + * + * Implemented by: + * - ConversationStore (Redux + IndexedDB) - primary, persistent + * - MinimalStore (in-memory) - fallback for CLI/tests + */ + +import type { Turn, AssistantMessageData } from '../lumo-client/types.js'; +import type { + ConversationId, + ConversationState, + Message, + MessageId, + MessageForStore, +} from './types.js'; + +export interface IConversationStore { + // Core CRUD + get(id: ConversationId): ConversationState | undefined; + getOrCreate(id: ConversationId): ConversationState; + has(id: ConversationId): boolean; + delete(id: ConversationId): boolean; + entries(): IterableIterator<[ConversationId, ConversationState]>; + + // Message operations + appendMessages(id: ConversationId, incoming: MessageForStore[]): Message[]; + appendAssistantResponse( + id: ConversationId, + messageData: AssistantMessageData, + status?: 'succeeded' | 'failed', + semanticId?: string + ): Message; + appendUserMessage(id: ConversationId, content: string): Message; + appendAssistantToolCalls( + id: ConversationId, + toolCalls: Array<{ name: string; arguments: string; call_id: string }> + ): void; + getMessages(id: ConversationId): Message[]; + getMessage(conversationId: ConversationId, messageId: MessageId): Message | undefined; + + // Conversation metadata + setTitle(id: ConversationId, title: string): void; + toTurns(id: ConversationId): Turn[]; + createFromTurns(turns: Turn[], title?: string): { conversationId: ConversationId; title: string }; +} diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 6aaf9f7..785d260 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -112,7 +112,6 @@ function toConversationState( export class ConversationStore { private store: LumoStore; private spaceId: SpaceId; - private onDirtyCallback?: () => void; /** * Map of messageId (UUID) -> semanticId for deduplication. @@ -138,13 +137,6 @@ export class ConversationStore { logger.info({ spaceId }, 'ConversationStore initialized'); } - /** - * Set callback to be called when a conversation becomes dirty - */ - setOnDirtyCallback(callback: () => void): void { - this.onDirtyCallback = callback; - } - /** * Get or create a conversation by ID */ @@ -291,8 +283,7 @@ export class ConversationStore { parentId = messageId; } - this.notifyDirty(); - + // Track metrics const metrics = getMetrics(); if (metrics) { @@ -352,8 +343,7 @@ export class ConversationStore { // Request push to server this.store.dispatch(pushMessageRequest({ id: messageId })); - this.notifyDirty(); - + // Update conversation status this.store.dispatch(updateConversationStatus({ id, @@ -439,8 +429,7 @@ export class ConversationStore { // Request push to server this.store.dispatch(pushMessageRequest({ id: messageId })); - this.notifyDirty(); - + const message: Message = { id: messageId, conversationId: id, @@ -480,19 +469,6 @@ export class ConversationStore { return { conversationId, title: effectiveTitle }; } - /** - * Mark conversation as generating - */ - setGenerating(id: ConversationId): void { - if (this.has(id)) { - this.store.dispatch(updateConversationStatus({ - id, - status: ConversationStatus.GENERATING, - })); - this.store.dispatch(pushConversationRequest({ id })); - } - } - /** * Update conversation title */ @@ -505,8 +481,7 @@ export class ConversationStore { persist: true, })); this.store.dispatch(pushConversationRequest({ id })); - this.notifyDirty(); - } + } logger.debug({ conversationId: id }, 'Set title'); } @@ -567,53 +542,8 @@ export class ConversationStore { } } - /** - * Get all dirty conversations - * - * Note: Upstream uses IDB dirty flag, not in-memory. This returns empty - * since sagas handle sync automatically. - */ - getDirty(): ConversationState[] { - // Upstream sagas handle dirty tracking via IDB - // Return empty array since we don't track dirty in-memory - return []; - } - - /** - * Mark a conversation as synced (no-op for upstream) - */ - markSynced(_id: ConversationId): void { - // Upstream sagas handle sync marking via IDB - } - - /** - * Mark a conversation as dirty - */ - markDirtyById(_id: ConversationId): void { - // Upstream sagas handle dirty tracking via IDB - this.notifyDirty(); - } - - /** - * Get store statistics - */ - getStats(): { - total: number; - dirty: number; - } { - const state = this.store.getState(); - return { - total: Object.keys(state.conversations).length, - dirty: 0, // Upstream uses IDB dirty flag - }; - } - // Private methods - private notifyDirty(): void { - this.onDirtyCallback?.(); - } - private generateAutoTitle(turns: Turn[]): string { const firstUserTurn = turns.find(t => t.role === Role.User); if (firstUserTurn?.content) { diff --git a/tests/e2e/cli-smoke.test.ts b/tests/e2e/cli-smoke.test.ts index cc8f84d..765fe71 100644 --- a/tests/e2e/cli-smoke.test.ts +++ b/tests/e2e/cli-smoke.test.ts @@ -8,8 +8,8 @@ import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { FallbackStore } from '../../src/conversations/fallback/store.js'; -import type { AppContext } from '../../src/app/types.js'; +import { MinimalStore } from '../../src/conversations/index.js'; +import type { Application } from '../../src/app/index.js'; describe('CLI single-query mode', () => { let stdoutChunks: string[]; @@ -44,9 +44,9 @@ describe('CLI single-query mode', () => { it('runs single query and produces output', async () => { const mockApi = createMockProtonApi('success'); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new FallbackStore(); + const store = new MinimalStore(); - const mockApp: AppContext = { + const mockApp: Application = { getLumoClient: () => lumoClient, getConversationStore: () => store, getAuthProvider: () => undefined, diff --git a/tests/helpers/test-server.ts b/tests/helpers/test-server.ts index 2d8c141..1fd9cda 100644 --- a/tests/helpers/test-server.ts +++ b/tests/helpers/test-server.ts @@ -15,13 +15,12 @@ import { createModelsRouter } from '../../src/api/routes/models.js'; import { RequestQueue } from '../../src/api/queue.js'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { FallbackStore } from '../../src/conversations/fallback/store.js'; +import { MinimalStore, type IConversationStore } from '../../src/conversations/index.js'; import { MetricsService, setMetrics } from '../../src/app/metrics.js'; import { setupMetricsMiddleware } from '../../src/api/middleware.js'; import { createMetricsRouter } from '../../src/api/routes/metrics.js'; import type { EndpointDependencies } from '../../src/api/types.js'; import type { MockConfig } from '../../src/app/config.js'; -import { ConversationStore } from '../../src/conversations/store.js'; type Scenario = MockConfig['scenario']; @@ -34,7 +33,7 @@ export interface TestServer { server: Server; baseUrl: string; deps: EndpointDependencies; - store: ConversationStore; + store: IConversationStore; /** MetricsService instance (only if metrics option was true) */ metrics?: MetricsService; close: () => Promise<void>; @@ -53,7 +52,7 @@ export async function createTestServer( ): Promise<TestServer> { const mockApi = createMockProtonApi(scenario); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new FallbackStore(); + const store = new MinimalStore(); const queue = new RequestQueue(1); const deps: EndpointDependencies = { diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 13f186d..565dd03 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -11,7 +11,7 @@ import { createHealthRouter } from '../../src/api/routes/health.js'; import { RequestQueue } from '../../src/api/queue.js'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { FallbackStore } from '../../src/conversations/fallback/store.js'; +import { MinimalStore } from '../../src/conversations/index.js'; import { MetricsService, setMetrics } from '../../src/app/metrics.js'; import { setupMetricsMiddleware } from '../../src/api/middleware.js'; import { createMetricsRouter } from '../../src/api/routes/metrics.js'; @@ -27,7 +27,7 @@ interface TestServer { async function createTestServerWithMetrics(): Promise<TestServer> { const mockApi = createMockProtonApi('success'); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new FallbackStore(); + const store = new MinimalStore(); const queue = new RequestQueue(1); const deps: EndpointDependencies = { diff --git a/tests/unit/conversation-store.test.ts b/tests/unit/minimal-store.test.ts similarity index 61% rename from tests/unit/conversation-store.test.ts rename to tests/unit/minimal-store.test.ts index 1f15b7f..0e5dba0 100644 --- a/tests/unit/conversation-store.test.ts +++ b/tests/unit/minimal-store.test.ts @@ -1,20 +1,20 @@ /** - * Unit tests for FallbackStore (in-memory conversation store) + * Unit tests for MinimalStore (in-memory conversation store) * - * Tests in-memory conversation management, LRU eviction, + * Tests in-memory conversation management, * message deduplication, and Turn conversion. */ import { describe, it, expect, beforeEach } from 'vitest'; -import { FallbackStore } from '../../src/conversations/fallback/store.js'; +import { MinimalStore } from '../../src/conversations/index.js'; -let store: FallbackStore; +let store: MinimalStore; beforeEach(() => { - store = new FallbackStore(); + store = new MinimalStore(); }); -describe('FallbackStore', () => { +describe('MinimalStore', () => { describe('getOrCreate', () => { it('creates new conversation when none exists', () => { const state = store.getOrCreate('conv-1'); @@ -30,11 +30,6 @@ describe('FallbackStore', () => { const second = store.getOrCreate('conv-1'); expect(second.title).toBe('Modified'); }); - - it('marks new conversations as dirty', () => { - const state = store.getOrCreate('conv-1'); - expect(state.dirty).toBe(true); - }); }); describe('get / has', () => { @@ -112,15 +107,6 @@ describe('FallbackStore', () => { expect(state.status).toBe('completed'); }); - it('marks conversation dirty', () => { - store.getOrCreate('conv-1'); - store.markSynced('conv-1'); - expect(store.get('conv-1')!.dirty).toBe(false); - - store.appendAssistantResponse('conv-1', { content: 'Response' }); - expect(store.get('conv-1')!.dirty).toBe(true); - }); - it('sets parentId to last message', () => { const [userMsg] = store.appendMessages('conv-1', [ { role: 'user', content: 'Hi' }, @@ -167,84 +153,36 @@ describe('FallbackStore', () => { }); }); - describe('LRU eviction', () => { - // Note: MAX_CONVERSATIONS is hardcoded to 100, so eviction tests would need 101+ conversations - // These tests verify the LRU tracking mechanism works, not the actual eviction threshold - - it('evicts least recently used when max exceeded', () => { - // Create 100 conversations (at max) - for (let i = 0; i < 100; i++) { - const state = store.getOrCreate(`conv-${i}`); - store.markSynced(`conv-${i}`); // Mark clean so they can be evicted - state.dirty = false; - } - - // Access conv-0 to make it most recent - store.get('conv-0'); - - // Add 101st conversation, should evict conv-1 (oldest non-recently-used) - store.getOrCreate('conv-100'); - store.markSynced('conv-100'); - - expect(store.has('conv-0')).toBe(true); // accessed recently - expect(store.has('conv-1')).toBe(false); // evicted (LRU) - expect(store.has('conv-100')).toBe(true); // just created - }); - - it('skips dirty conversations during eviction', () => { - // Create 100 conversations, all clean except conv-1 - for (let i = 0; i < 100; i++) { - store.getOrCreate(`conv-${i}`); - if (i !== 1) { - store.markSynced(`conv-${i}`); - store.get(`conv-${i}`)!.dirty = false; - } - } - - // Add 101st: conv-0 is LRU but let's check dirty skipping - // conv-0 should be evicted since it's clean - store.getOrCreate('conv-100'); - - expect(store.has('conv-1')).toBe(true); // dirty, not evicted + describe('no-op methods', () => { + it('setTitle is no-op', () => { + store.getOrCreate('conv-1'); + store.setTitle('conv-1', 'My Chat'); + // Title not stored - still default + expect(store.get('conv-1')!.title).toBe('New Conversation'); }); - }); - - describe('setTitle', () => { - it('updates title and marks dirty', () => { + it('delete returns false', () => { store.getOrCreate('conv-1'); - store.markSynced('conv-1'); - store.get('conv-1')!.dirty = false; - - store.setTitle('conv-1', 'My Chat'); - expect(store.get('conv-1')!.title).toBe('My Chat'); - expect(store.get('conv-1')!.dirty).toBe(true); + expect(store.delete('conv-1')).toBe(false); + // Conversation still exists + expect(store.has('conv-1')).toBe(true); }); - }); - describe('delete', () => { - it('removes conversation', () => { + it('entries yields nothing', () => { store.getOrCreate('conv-1'); - expect(store.delete('conv-1')).toBe(true); - expect(store.has('conv-1')).toBe(false); + const entries = [...store.entries()]; + expect(entries).toEqual([]); }); - it('returns false for non-existent conversation', () => { - expect(store.delete('nonexistent')).toBe(false); + it('getMessage returns undefined', () => { + store.appendMessages('conv-1', [{ role: 'user', content: 'Hi' }]); + expect(store.getMessage('conv-1', 'any-message-id')).toBeUndefined(); }); - }); - describe('getStats', () => { - it('returns correct statistics', () => { - store.getOrCreate('conv-1'); - store.getOrCreate('conv-2'); - store.markSynced('conv-1'); - store.get('conv-1')!.dirty = false; - - const stats = store.getStats(); - expect(stats.total).toBe(2); - expect(stats.dirty).toBe(1); // conv-2 is dirty - expect(stats.maxSize).toBe(100); // hardcoded MAX_CONVERSATIONS + it('createFromTurns returns empty', () => { + const result = store.createFromTurns([{ role: 'user', content: 'Hi' }]); + expect(result.conversationId).toBe(''); + expect(result.title).toBe(''); }); }); }); diff --git a/tests/unit/primary-conversation-store.test.ts b/tests/unit/primary-conversation-store.test.ts index b95fe67..4b68917 100644 --- a/tests/unit/primary-conversation-store.test.ts +++ b/tests/unit/primary-conversation-store.test.ts @@ -237,14 +237,6 @@ describe('ConversationStore', () => { }); }); - describe('setGenerating', () => { - it('sets conversation status to generating', () => { - ctx.conversationStore.getOrCreate('conv-1'); - ctx.conversationStore.setGenerating('conv-1'); - expect(ctx.conversationStore.get('conv-1')!.status).toBe('generating'); - }); - }); - describe('delete', () => { it('removes conversation', () => { ctx.conversationStore.getOrCreate('conv-1'); @@ -271,18 +263,6 @@ describe('ConversationStore', () => { }); }); - describe('getStats', () => { - it('returns correct total count', () => { - ctx.conversationStore.getOrCreate('conv-1'); - ctx.conversationStore.getOrCreate('conv-2'); - - const stats = ctx.conversationStore.getStats(); - expect(stats.total).toBeGreaterThanOrEqual(2); - // dirty is always 0 for ConversationStore (upstream uses IDB flags) - expect(stats.dirty).toBe(0); - }); - }); - describe('createFromTurns', () => { it('creates conversation from turns', () => { const turns = [ @@ -311,35 +291,6 @@ describe('ConversationStore', () => { }); }); - describe('onDirtyCallback', () => { - it('calls callback when messages are appended', () => { - let callCount = 0; - ctx.conversationStore.setOnDirtyCallback(() => { - callCount++; - }); - - // getOrCreate doesn't call notifyDirty (upstream uses IDB dirty flags) - ctx.conversationStore.getOrCreate('conv-1'); - expect(callCount).toBe(0); - - // appendMessages does call notifyDirty - ctx.conversationStore.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - expect(callCount).toBe(1); - - // appendAssistantResponse also calls notifyDirty - ctx.conversationStore.appendAssistantResponse('conv-1', { - content: 'Hi!', - }); - expect(callCount).toBe(2); - - // setTitle also calls notifyDirty - ctx.conversationStore.setTitle('conv-1', 'New Title'); - expect(callCount).toBe(3); - }); - }); - describe('semantic ID handling', () => { it('uses provided ID for tool messages', () => { const added = ctx.conversationStore.appendMessages('conv-1', [ From 394614d4580c7574d6788d2fc2ed2f8c725f95cd Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:35:54 +0100 Subject: [PATCH 06/37] remove lazily loaded import --- src/auth/vault/vault.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/auth/vault/vault.ts b/src/auth/vault/vault.ts index 7b716c3..600597d 100644 --- a/src/auth/vault/vault.ts +++ b/src/auth/vault/vault.ts @@ -9,7 +9,7 @@ * Inspired by proton-bridge vault implementation. */ -import { existsSync, readFileSync, writeFileSync, unlinkSync, mkdirSync, statSync } from 'fs'; +import { existsSync, readFileSync, writeFileSync, unlinkSync, mkdirSync, statSync, renameSync } from 'fs'; import { createCipheriv, createDecipheriv, randomBytes } from 'crypto'; import { dirname } from 'path'; import { logger } from '../../app/logger.js'; @@ -126,7 +126,6 @@ export async function writeVault(vaultPath: string, tokens: StoredTokens, keyCon writeFileSync(tempPath, ciphertext, { mode: 0o600 }); // Rename (atomic on most filesystems) - const { renameSync } = await import('fs'); renameSync(tempPath, vaultPath); logger.debug({ vaultPath }, 'Tokens written to vault'); From f6890aad0a1b91e979261061343ec83c15af1b07 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:36:45 +0100 Subject: [PATCH 07/37] remove stale type; add comments on AssistantMessageData --- src/lumo-client/types.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/lumo-client/types.ts b/src/lumo-client/types.ts index 13091f1..8d4d1eb 100644 --- a/src/lumo-client/types.ts +++ b/src/lumo-client/types.ts @@ -81,15 +81,12 @@ export interface ParsedToolCall { arguments: Record<string, unknown>; } -/** Native tool call with optional result (for persistence). */ -export interface NativeToolData { - toolCall: ParsedToolCall; - toolResult?: string; -} - /** * Assistant message data ready for persistence. - * Matches MessagePrivate fields for assistant messages. + * + * Subset of upstream MessagePriv with required content (LumoClient always provides it). + * Could be typed as `Required<Pick<MessagePriv, 'content'>> & Pick<MessagePriv, 'blocks'>` + * but a named interface is clearer for the LumoClient contract. */ export interface AssistantMessageData { content: string; From 83870e36409ac17c610c7c58579d2cf3333e2bd4 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:45:24 +0200 Subject: [PATCH 08/37] simplify initializeSync --- src/app/index.ts | 7 +++---- src/conversations/index.ts | 20 +++++--------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/app/index.ts b/src/app/index.ts index e732cde..52c0c3d 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -34,7 +34,7 @@ export class Application { } else { await app.initializeAuth(); await app.initializeStore(); - await app.initializeSync(); + app.initializeSync(); } return app; } @@ -109,15 +109,14 @@ export class Application { /** * Initialize sync service for conversation persistence */ - private async initializeSync(): Promise<void> { + private initializeSync(): void { const conversationsConfig = getConversationsConfig(); - const result = await initializeSync({ + this.syncInitialized = initializeSync({ protonApi: this.protonApi, uid: this.uid, authProvider: this.authProvider, conversationsConfig, }); - this.syncInitialized = result.initialized; } // AppContext implementation diff --git a/src/conversations/index.ts b/src/conversations/index.ts index cf82376..d70b410 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -218,39 +218,29 @@ export interface InitializeSyncOptions { conversationsConfig: ConversationsConfig; } -export interface InitializeSyncResult { - initialized: boolean; - /** Store result, only set when primary store is used */ - storeResult?: StoreResult; -} - /** * Initialize sync services * * Sync is handled automatically by Redux sagas when primary store is active. - * Returns initialized: false if no primary store or sync is disabled. + * Returns false if no primary store or sync is disabled. */ -export async function initializeSync( - options: InitializeSyncOptions -): Promise<InitializeSyncResult> { +export function initializeSync(options: InitializeSyncOptions): boolean { const { authProvider, conversationsConfig } = options; if (!conversationsConfig?.enableSync) { logger.info('Sync is disabled, skipping sync initialization'); - return { initialized: false }; + return false; } const syncWarning = authProvider.getSyncWarning(); if (syncWarning) { logger.warn({ method: authProvider.method }, syncWarning); - return { initialized: false }; + return false; } logger.info( { method: authProvider.method }, 'Sync initialized (handled by sagas)' ); - return { - initialized: true, - }; + return true; } From 003d3a654fe9d59f70ea0152ddfc8f744e48f615 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:45:50 +0200 Subject: [PATCH 09/37] remove obsolete useFallbackStore --- src/auth/providers/provider.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/auth/providers/provider.ts b/src/auth/providers/provider.ts index a63d605..81141b3 100644 --- a/src/auth/providers/provider.ts +++ b/src/auth/providers/provider.ts @@ -296,14 +296,9 @@ export class AuthProvider implements IAuthProvider { // === Persistence warnings === /** - * Get warning if ConversationStore is configured but will fall back to FallbackStore. - * Returns null if ConversationStore will work or if FallbackStore is explicitly configured. + * Get warning if ConversationStore is unavailable */ getConversationStoreWarning(): string | null { - const config = getConversationsConfig(); - if (config.useFallbackStore) { - return null; // Fallback explicitly configured - } if (!this.supportsPersistence()) { return 'ConversationStore disabled: no encryption keys. Using FallbackStore. Conversations will not be persisted. Re-authenticate to enable.'; From d452a2d274b3daa7c0053885aa04959ab43820f7 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:50:14 +0200 Subject: [PATCH 10/37] fetch all missing messages on start-up --- src/conversations/index.ts | 9 +++++++- src/conversations/init.ts | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/conversations/index.ts b/src/conversations/index.ts index d70b410..f43f454 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -69,7 +69,7 @@ import { logger } from '../app/logger.js'; import type { AuthProvider, ProtonApi } from '../auth/index.js'; import type { ConversationsConfig } from '../app/config.js'; import { getKeyManager } from './key-manager.js'; -import { initializeStore, type StoreResult } from './init.js'; +import { initializeStore, pullIncompleteConversations, type StoreResult } from './init.js'; import type { IConversationStore } from './store-interface.js'; // ============================================================================ @@ -129,6 +129,13 @@ export async function initializeConversationStore( activeStore = result.conversationStore; primaryStoreResult = result; logger.info('Using primary conversation store'); + + // Pull incomplete conversations in background when sync is enabled + if (options.conversationsConfig.enableSync) { + pullIncompleteConversations(result.store, result.spaceId) + .catch(err => logger.error({ error: err }, 'Failed to pull incomplete conversations')); + } + return { isPrimary: true, storeResult: result }; } } catch (error) { diff --git a/src/conversations/init.ts b/src/conversations/init.ts index 5dcd9eb..55bc52b 100644 --- a/src/conversations/init.ts +++ b/src/conversations/init.ts @@ -26,6 +26,11 @@ import { pullSpacesSuccess, pullSpacesFailure, } from '@lumo/redux/slices/core/spaces.js'; +import { pullConversationRequest } from '@lumo/redux/slices/core/conversations.js'; +import { + selectConversationsBySpaceId, + selectMessagesByConversationId, +} from '@lumo/redux/selectors.js'; import { setupStore, type LumoSagaContext, type LumoStore } from '@lumo/redux/store.js'; import { LumoApi } from '@lumo/remote/api.js'; import type { Space } from '@lumo/types.js'; @@ -262,3 +267,40 @@ function findOrCreateSpace( return spaceId; } + +/** + * Pull conversations that have no messages loaded. + * + * After pullSpaces, conversations exist in Redux but may have no messages yet. + * This dispatches pullConversationRequest for each empty conversation to fetch + * full content from the server. + * + * Note: This does NOT handle messages that fail to decrypt (e.g., key mismatch + * or corruption). Those remain in IDB but are skipped during loadReduxFromIdb. + */ +export async function pullIncompleteConversations( + store: LumoStore, + spaceId: SpaceId +): Promise<void> { + const state = store.getState(); + + // Get conversations for this space from Redux + const conversations = selectConversationsBySpaceId(spaceId)(state); + + // Find conversations with no messages in Redux + const emptyConversationIds = Object.values(conversations) + .filter(c => Object.keys(selectMessagesByConversationId(c.id)(state)).length === 0) + .map(c => c.id); + + if (emptyConversationIds.length === 0) { + return; + } + + logger.info({ count: emptyConversationIds.length }, 'Pulling incomplete conversations'); + + // Dispatch pulls with rate limiting to avoid request bursts + for (const id of emptyConversationIds) { + store.dispatch(pullConversationRequest({ id })); + await new Promise(resolve => setTimeout(resolve, 100)); + } +} From 7e8cf0e84931495b699cff9fd0de4570c9d2b5f1 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Mon, 30 Mar 2026 14:42:41 +0200 Subject: [PATCH 11/37] add super basic /search command, searching in messages + conv title, showing snippets --- src/app/commands.ts | 24 ++++ src/conversations/search.ts | 175 ++++++++++++++++++++++++++++ tests/unit/search.test.ts | 219 ++++++++++++++++++++++++++++++++++++ 3 files changed, 418 insertions(+) create mode 100644 src/conversations/search.ts create mode 100644 tests/unit/search.test.ts diff --git a/src/app/commands.ts b/src/app/commands.ts index 969d347..3d020a9 100644 --- a/src/app/commands.ts +++ b/src/app/commands.ts @@ -6,6 +6,7 @@ import { logger } from './logger.js'; import { getCommandsConfig } from './config.js'; import { getConversationStore } from '../conversations/index.js'; +import { searchConversations, formatSearchResults } from '../conversations/search.js'; import type { AuthManager } from '../auth/index.js'; import type { Turn } from '../lumo-client/index.js'; @@ -106,6 +107,9 @@ export async function executeCommand( case 'load': return await handleLoadCommand(params, context); + case 'search': + return handleSearchCommand(params, context); + case 'title': return handleTitleCommand(params, context); @@ -143,6 +147,7 @@ function getHelpText(): string { /title <text> - Set conversation title /save [title] - Save stateless request to conversation (optionally set title) /load <id> - Load a conversation from Proton by ID + /search <query> - Search conversation titles and messages /refreshtokens - Manually refresh auth tokens /logout - Revoke session and delete tokens /quit - Exit CLI (CLI mode only)${wakewordHint}`; @@ -276,6 +281,25 @@ async function handleRefreshTokensCommand(context?: CommandContext): Promise<str } } +/** + * Handle /search command - search conversations by title and content + */ +function handleSearchCommand(params: string, context?: CommandContext): string { + const query = params.trim(); + if (!query) { + return 'Usage: /search <query>\nSearches conversation titles and message content.'; + } + + const store = getConversationStore(); + if (!store) { + return 'Conversation store not available.'; + } + + // Exclude current conversation from results (it would always be at the top) + const results = searchConversations(store, query, 20, context?.conversationId); + return formatSearchResults(results, query); +} + /** * Handle /logout command - revoke session and delete tokens */ diff --git a/src/conversations/search.ts b/src/conversations/search.ts new file mode 100644 index 0000000..dcc18e8 --- /dev/null +++ b/src/conversations/search.ts @@ -0,0 +1,175 @@ +/** + * Conversation search utilities + * + * Simple .includes() based search for conversation titles and message content. + * Snippet extraction inspired by WebClients searchService.ts. + */ + +import type { IConversationStore } from './store-interface.js'; +import type { ConversationId, Message } from './types.js'; + +export interface SearchResult { + conversationId: ConversationId; + title: string; + /** Snippet around match in message content, if found */ + snippet?: string; + updatedAt: string; +} + +/** + * Search conversations by title and message content + * + * @param store - Conversation store to search + * @param query - Search query (case-insensitive) + * @param limit - Maximum results to return (default 20) + * @param excludeId - Conversation ID to exclude (e.g., current conversation) + * @returns Array of matching conversations, sorted by most recent first + */ +export function searchConversations( + store: IConversationStore, + query: string, + limit = 20, + excludeId?: ConversationId +): SearchResult[] { + const lowerQuery = query.toLowerCase(); + const results: SearchResult[] = []; + + for (const [id, conv] of store.entries()) { + // Skip excluded conversation (e.g., current one) + if (excludeId && id === excludeId) continue; + + // Check title + const titleMatch = conv.title?.toLowerCase().includes(lowerQuery); + + // Check messages for content match + let snippet: string | undefined; + if (!titleMatch) { + snippet = findMatchingSnippet(conv.messages, lowerQuery); + } + + if (titleMatch || snippet) { + results.push({ + conversationId: id, + title: conv.title ?? 'Untitled', + snippet, + updatedAt: conv.metadata.updatedAt, + }); + } + + if (results.length >= limit) break; + } + + // Sort by updatedAt descending (most recent first) + results.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)); + + return results; +} + +/** + * Find a matching snippet in messages + * + * Searches message content for the query and extracts a snippet + * with context around the match. + */ +function findMatchingSnippet(messages: Message[], lowerQuery: string): string | undefined { + for (const msg of messages) { + const content = msg.content; + if (!content) continue; + + const lowerContent = content.toLowerCase(); + const matchIndex = lowerContent.indexOf(lowerQuery); + + if (matchIndex !== -1) { + return extractSnippet(content, matchIndex, lowerQuery.length); + } + } + return undefined; +} + +/** + * Extract a snippet around a match position + * + * Takes ~80 characters on each side of the match, adds ellipsis + * if truncated, and normalizes whitespace. + * + * Based on WebClients searchService.ts snippet extraction. + */ +function extractSnippet(content: string, matchIndex: number, matchLength: number): string { + const radius = 80; + const start = Math.max(0, matchIndex - radius); + const end = Math.min(content.length, matchIndex + matchLength + radius); + + let snippet = content.slice(start, end); + + // Add ellipsis indicators + if (start > 0) snippet = '...' + snippet; + if (end < content.length) snippet = snippet + '...'; + + // Normalize whitespace (newlines, multiple spaces) + return snippet.replace(/\s+/g, ' ').trim(); +} + +/** + * Strip markdown formatting from text + */ +function stripMarkdown(text: string): string { + return text + // Code blocks (```...```) + .replace(/```[\s\S]*?```/g, '') + // Inline code (`...`) + .replace(/`([^`]+)`/g, '$1') + // Bold (**...** or __...__) + .replace(/\*\*([^*]+)\*\*/g, '$1') + .replace(/__([^_]+)__/g, '$1') + // Italic (*...* or _..._) + .replace(/\*([^*]+)\*/g, '$1') + .replace(/_([^_]+)_/g, '$1') + // Links [text](url) + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') + // Headers + .replace(/^#{1,6}\s+/gm, '') + // Normalize whitespace + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Format date for display (e.g., "Mar 30" or "Mar 30, 2024") + */ +function formatShortDate(dateStr: string): string { + const date = new Date(dateStr); + const now = new Date(); + const month = date.toLocaleDateString('en-US', { month: 'short' }); + const day = date.getDate(); + + if (date.getFullYear() === now.getFullYear()) { + return `${month} ${day}`; + } + return `${month} ${day}, ${date.getFullYear()}`; +} + +/** + * Format search results for CLI output + */ +export function formatSearchResults(results: SearchResult[], query: string): string { + if (results.length === 0) { + return `No results found for "${query}"`; + } + + const lines: string[] = []; + lines.push(`Found ${results.length} result${results.length === 1 ? '' : 's'}:`); + lines.push(''); + + for (const result of results) { + lines.push(`**${stripMarkdown(result.title)}** ${formatShortDate(result.updatedAt)}`); + if (result.snippet) { + lines.push(` ${stripMarkdown(result.snippet)}`); + } + lines.push(` ID: ${result.conversationId}`); + lines.push(''); + } + + lines.push('Use /load <id> to open a conversation'); + + return lines.join('\n'); +} diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts new file mode 100644 index 0000000..b5976c6 --- /dev/null +++ b/tests/unit/search.test.ts @@ -0,0 +1,219 @@ +/** + * Unit tests for conversation search + */ + +import { describe, it, expect } from 'vitest'; +import { + searchConversations, + formatSearchResults, + type SearchResult, +} from '../../src/conversations/search.js'; +import type { IConversationStore } from '../../src/conversations/store-interface.js'; +import type { ConversationId, ConversationState, Message } from '../../src/conversations/types.js'; + +/** + * Create a mock store with test conversations + */ +function createMockStore(conversations: Map<ConversationId, ConversationState>): IConversationStore { + return { + entries: () => conversations.entries(), + // Other methods not needed for search tests + get: () => undefined, + getOrCreate: () => { throw new Error('not implemented'); }, + has: () => false, + delete: () => false, + appendMessages: () => [], + appendAssistantResponse: () => { throw new Error('not implemented'); }, + appendUserMessage: () => { throw new Error('not implemented'); }, + appendAssistantToolCalls: () => {}, + getMessages: () => [], + getMessage: () => undefined, + setTitle: () => {}, + toTurns: () => [], + createFromTurns: () => { throw new Error('not implemented'); }, + }; +} + +function createConversation( + id: string, + title: string, + messages: Array<{ role: string; content: string }>, + updatedAt = new Date().toISOString() +): [ConversationId, ConversationState] { + return [ + id, + { + metadata: { + id, + spaceId: 'space-1', + createdAt: updatedAt, + updatedAt, + }, + title, + status: 'completed', + messages: messages.map((m, i) => ({ + id: `msg-${i}`, + conversationId: id, + createdAt: updatedAt, + role: m.role as Message['role'], + status: 'succeeded' as const, + content: m.content, + })), + dirty: false, + }, + ]; +} + +describe('searchConversations', () => { + it('returns empty array when store is empty', () => { + const store = createMockStore(new Map()); + const results = searchConversations(store, 'test'); + expect(results).toEqual([]); + }); + + it('matches conversation titles (case-insensitive)', () => { + const conversations = new Map([ + createConversation('conv-1', 'How to configure Caddy', []), + createConversation('conv-2', 'Nginx setup guide', []), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'caddy'); + expect(results).toHaveLength(1); + expect(results[0].title).toBe('How to configure Caddy'); + }); + + it('matches message content', () => { + const conversations = new Map([ + createConversation('conv-1', 'Web Server Setup', [ + { role: 'user', content: 'How do I install nginx?' }, + { role: 'assistant', content: 'You can install nginx using apt...' }, + ]), + createConversation('conv-2', 'Database Help', [ + { role: 'user', content: 'How do I backup postgres?' }, + ]), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'nginx'); + expect(results).toHaveLength(1); + expect(results[0].conversationId).toBe('conv-1'); + expect(results[0].snippet).toContain('nginx'); + }); + + it('prefers title match over message match', () => { + const conversations = new Map([ + createConversation('conv-1', 'Caddy Configuration', [ + { role: 'user', content: 'The caddy server is great' }, + ]), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'caddy'); + expect(results).toHaveLength(1); + // No snippet when title matches + expect(results[0].snippet).toBeUndefined(); + }); + + it('extracts snippet with context around match', () => { + const longContent = 'This is a long message about various topics. ' + + 'Eventually we discuss how to configure caddy as a reverse proxy. ' + + 'There are many more details after this point.'; + + const conversations = new Map([ + createConversation('conv-1', 'General Discussion', [ + { role: 'user', content: longContent }, + ]), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'caddy'); + expect(results).toHaveLength(1); + expect(results[0].snippet).toContain('caddy'); + expect(results[0].snippet).toContain('...'); + }); + + it('respects limit parameter', () => { + const conversations = new Map( + Array.from({ length: 30 }, (_, i) => + createConversation(`conv-${i}`, `Test conversation ${i}`, []) + ) + ); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'test', 5); + expect(results).toHaveLength(5); + }); + + it('sorts results by most recent first', () => { + const conversations = new Map([ + createConversation('conv-old', 'Test old', [], '2024-01-01T00:00:00Z'), + createConversation('conv-new', 'Test new', [], '2024-12-01T00:00:00Z'), + createConversation('conv-mid', 'Test mid', [], '2024-06-01T00:00:00Z'), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'test'); + expect(results).toHaveLength(3); + expect(results[0].conversationId).toBe('conv-new'); + expect(results[1].conversationId).toBe('conv-mid'); + expect(results[2].conversationId).toBe('conv-old'); + }); + + it('excludes specified conversation ID', () => { + const conversations = new Map([ + createConversation('conv-1', 'Test one', []), + createConversation('conv-2', 'Test two', []), + createConversation('conv-3', 'Test three', []), + ]); + const store = createMockStore(conversations); + + const results = searchConversations(store, 'test', 20, 'conv-2'); + expect(results).toHaveLength(2); + expect(results.find(r => r.conversationId === 'conv-2')).toBeUndefined(); + }); +}); + +describe('formatSearchResults', () => { + it('shows "no results" message for empty results', () => { + const output = formatSearchResults([], 'test'); + expect(output).toContain('No results found'); + expect(output).toContain('test'); + }); + + it('formats single result correctly', () => { + const results: SearchResult[] = [{ + conversationId: 'conv-123', + title: 'My Conversation', + updatedAt: new Date().toISOString(), + }]; + + const output = formatSearchResults(results, 'test'); + expect(output).toContain('Found 1 result'); + expect(output).toContain('My Conversation'); + expect(output).toContain('conv-123'); + expect(output).toContain('/load'); + }); + + it('formats multiple results with snippets', () => { + const results: SearchResult[] = [ + { + conversationId: 'conv-1', + title: 'First Conversation', + snippet: '...matching text here...', + updatedAt: new Date().toISOString(), + }, + { + conversationId: 'conv-2', + title: 'Second Conversation', + updatedAt: new Date().toISOString(), + }, + ]; + + const output = formatSearchResults(results, 'query'); + expect(output).toContain('Found 2 results'); + expect(output).toContain('First Conversation'); + expect(output).toContain('matching text here'); + expect(output).toContain('Second Conversation'); + }); +}); From f95df60dbd10ac67c551b3dd1a5c9349506ff955 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Tue, 31 Mar 2026 11:43:13 +0200 Subject: [PATCH 12/37] support server tools: register, detect and execute server-side tool calls first server tool: search --- config.defaults.yaml | 6 +- src/api/instructions.ts | 19 +- src/api/routes/chat-completions/index.ts | 54 +++-- src/api/routes/responses/request-handlers.ts | 49 +++-- src/api/routes/shared.ts | 12 +- src/api/server.ts | 9 +- src/api/tools/server-tools/executor.ts | 133 +++++++++++++ src/api/tools/server-tools/index.ts | 33 ++++ src/api/tools/server-tools/loop.ts | 143 ++++++++++++++ src/api/tools/server-tools/registry.ts | 88 +++++++++ src/api/tools/server-tools/search.ts | 54 +++++ src/app/config.ts | 8 +- src/mock/custom-scenarios.ts | 45 +++++ tests/helpers/test-server.ts | 10 + tests/integration/responses-api.test.ts | 51 ++++- tests/unit/server-tools.test.ts | 197 +++++++++++++++++++ tests/unit/shared.test.ts | 170 +++++++++++++++- 17 files changed, 1017 insertions(+), 64 deletions(-) create mode 100644 src/api/tools/server-tools/executor.ts create mode 100644 src/api/tools/server-tools/index.ts create mode 100644 src/api/tools/server-tools/loop.ts create mode 100644 src/api/tools/server-tools/registry.ts create mode 100644 src/api/tools/server-tools/search.ts create mode 100644 tests/unit/server-tools.test.ts diff --git a/config.defaults.yaml b/config.defaults.yaml index 1517c50..c6ee176 100644 --- a/config.defaults.yaml +++ b/config.defaults.yaml @@ -47,7 +47,7 @@ test: mock: # Enable mock mode: bypass authentication, return simulated Lumo responses enabled: false - # Scenario: success, error, timeout, rejected, toolCall, misroutedToolCall, weeklyLimit, cycle + # Scenario: success, error, timeout, rejected, toolCall, misroutedToolCall, weeklyLimit, cycle, serverToolCall scenario: "success" # Shared Logging Configuration (can be overridden in server/cli sections) @@ -114,6 +114,10 @@ server: # Enable Lumo's native web_search tool (and other external tools: weather, stock, cryptocurrency) enableWebSearch: false + # Server-side tools callable by Lumo (search, etc.) + # When enabled, Lumo can call these tools and the server executes them directly. + enableServerTools: false + # Custom tool detection for API clients # Enable detection of JSON tool calls in Lumo's responses # WARNING: When enabled, Lumo can trigger actions via API clients! diff --git a/src/api/instructions.ts b/src/api/instructions.ts index 798bdf9..16c25a8 100644 --- a/src/api/instructions.ts +++ b/src/api/instructions.ts @@ -7,9 +7,10 @@ */ import { logger } from '../app/logger.js'; -import { getServerInstructionsConfig, getCustomToolsConfig } from '../app/config.js'; +import { getServerInstructionsConfig, getCustomToolsConfig, getServerToolsEnabled } from '../app/config.js'; import { interpolateTemplate } from '../app/template.js'; import { applyToolPrefix, applyToolNamePrefix } from './tools/prefix.js'; +import { getAllServerToolDefinitions } from './tools/server-tools/index.js'; import type { OpenAITool } from './types.js'; // ── Template validation ─────────────────────────────────────────────── @@ -119,11 +120,22 @@ function extractToolNames(tools?: OpenAITool[]): string[] { export function buildInstructions(tools?: OpenAITool[], clientInstructions?: string): string { const instructionsConfig = getServerInstructionsConfig(); const toolsConfig = getCustomToolsConfig(); + const serverToolsEnabled = getServerToolsEnabled(); const { prefix } = toolsConfig; const { replacePatterns } = instructionsConfig; + // Merge TamerTool definitions if enabled + let allTools = tools ?? []; + if (serverToolsEnabled) { + const serverToolDefs = getAllServerToolDefinitions(); + allTools = [...serverToolDefs, ...allTools]; + } + // Determine if we should include tools - const includeTools = toolsConfig.enabled && tools && tools.length > 0; + // Include if either CustomTools are enabled with client tools, or ServerTools are enabled + const hasCustomTools = toolsConfig.enabled && tools && tools.length > 0; + const hasServerTools = serverToolsEnabled && getAllServerToolDefinitions().length > 0; + const includeTools = hasCustomTools || hasServerTools; // Pre-interpolate forTools block (it can use {{prefix}}) const forTools = interpolateTemplate(instructionsConfig.forTools, { prefix }); @@ -131,7 +143,7 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str // Prepare tools JSON if enabled and provided let toolsJson: string | undefined; if (includeTools) { - const prefixedTools = applyToolPrefix(tools, prefix); + const prefixedTools = applyToolPrefix(allTools, prefix); toolsJson = JSON.stringify(prefixedTools, null, 2); } @@ -140,6 +152,7 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str if (clientInstructions) { cleanedClientInstructions = applyReplacePatterns(clientInstructions, replacePatterns); if (includeTools) { + // Only prefix client tool names, not TamerTool names const toolNames = extractToolNames(tools); cleanedClientInstructions = applyToolNamePrefix(cleanedClientInstructions, toolNames, prefix); } diff --git a/src/api/routes/chat-completions/index.ts b/src/api/routes/chat-completions/index.ts index 395c035..aa991ba 100644 --- a/src/api/routes/chat-completions/index.ts +++ b/src/api/routes/chat-completions/index.ts @@ -1,5 +1,5 @@ import { Router, Request, Response } from 'express'; -import { EndpointDependencies, OpenAIChatRequest, OpenAIChatResponse } from '../../types.js'; +import { EndpointDependencies, OpenAIChatRequest, OpenAIChatResponse, OpenAIToolCall } from '../../types.js'; import { getServerConfig, getConversationsConfig, getLogConfig, getServerInstructionsConfig } from '../../../app/config.js'; import { logger } from '../../../app/logger.js'; import { convertOpenAIChatMessages, extractSystemMessage } from '../../message-converter.js'; @@ -9,7 +9,7 @@ import { ChatCompletionEventEmitter } from './events.js'; import type { Turn } from '../../../lumo-client/index.js'; import type { ConversationId } from '../../../conversations/types.js'; import { trackCustomToolCompletion } from '../../tools/call-id.js'; -import { createStreamingToolProcessor } from '../../tools/streaming-processor.js'; +import { runServerToolLoop } from '../../tools/server-tools/index.js'; import { buildRequestContext, persistTitle, @@ -141,7 +141,7 @@ async function handleChatRequest( const id = generateChatCompletionId(); const created = Math.floor(Date.now() / 1000); const model = request.model || getServerConfig().apiModelName; - const ctx = buildRequestContext(deps, conversationId, request.tools); + const context = buildRequestContext(deps, conversationId, request.tools); // Streaming setup const emitter = streaming ? new ChatCompletionEventEmitter(res, id, created, model) : null; @@ -150,44 +150,42 @@ async function handleChatRequest( } let accumulatedText = ''; - let toolCalls: typeof processor.toolCallsEmitted | undefined; - - const processor = createStreamingToolProcessor(ctx.hasCustomTools, { - emitTextDelta(text) { - accumulatedText += text; - emitter?.emitContentDelta(text); - }, - emitToolCall(callId, tc) { - emitter?.emitToolCallDelta(callId, tc.name, tc.arguments); - }, - }); + let toolCalls: OpenAIToolCall[] | undefined; // Check for command before calling Lumo - const commandResult = await tryExecuteCommand(turns, ctx.commandContext); + const commandResult = await tryExecuteCommand(turns, context.commandContext); if (commandResult) { accumulatedText = commandResult.response; emitter?.emitContentDelta(accumulatedText); } else { - // Normal flow: call Lumo + // Normal flow: call Lumo with TamerTool loop try { - const result = await deps.queue.add(async () => - deps.lumoClient.chatWithHistory(turns, processor.onChunk, { - requestTitle: ctx.requestTitle, - instructions, - injectInstructionsInto, - }) - ); + const loopResult = await runServerToolLoop({ + deps, + context, + turns, + conversationId, + instructions, + injectInstructionsInto, + onTextDelta(text) { + accumulatedText += text; + emitter?.emitContentDelta(text); + }, + onToolCall(callId, tc) { + // Only CustomTool calls reach here (ServerTools filtered by loop) + emitter?.emitToolCallDelta(callId, tc.name, tc.arguments); + }, + }); logger.debug('[Server] Stream completed'); - processor.finalize(); - persistTitle(result, deps, conversationId); - toolCalls = processor.toolCallsEmitted.length > 0 ? processor.toolCallsEmitted : undefined; + persistTitle(loopResult.chatResult, deps, conversationId); + toolCalls = loopResult.customToolCalls.length > 0 ? loopResult.customToolCalls : undefined; persistAssistantTurn( deps, conversationId, - result.message, - mapToolCallsForPersistence(processor.toolCallsEmitted) + loopResult.chatResult.message, + mapToolCallsForPersistence(loopResult.customToolCalls) ); } catch (error) { logger.error({ error: String(error) }, 'Chat completion error'); diff --git a/src/api/routes/responses/request-handlers.ts b/src/api/routes/responses/request-handlers.ts index d5dfb3b..ea79189 100644 --- a/src/api/routes/responses/request-handlers.ts +++ b/src/api/routes/responses/request-handlers.ts @@ -13,7 +13,7 @@ import { ResponseEventEmitter } from './events.js'; import type { Turn } from '../../../lumo-client/index.js'; import type { ConversationId } from '../../../conversations/index.js'; import { generateCallId } from '../../tools/call-id.js'; -import { createStreamingToolProcessor } from '../../tools/streaming-processor.js'; +import { runServerToolLoop } from '../../tools/server-tools/index.js'; import { buildRequestContext, persistTitle, @@ -142,7 +142,7 @@ export async function handleRequest( const itemId = generateItemId(); const createdAt = Math.floor(Date.now() / 1000); const model = request.model || getServerConfig().apiModelName; - const ctx = buildRequestContext(deps, conversationId, request.tools); + const context = buildRequestContext(deps, conversationId, request.tools); // Streaming setup const emitter = streaming ? new ResponseEventEmitter(res) : null; @@ -157,44 +157,43 @@ export async function handleRequest( emitter.emitContentPartAdded(itemId, 0, 0); } - logger.debug({ hasCustomTools: ctx.hasCustomTools, toolCount: request.tools?.length }, '[Server] Tool detector state'); + logger.debug({ hasCustomTools: context.hasCustomTools, toolCount: request.tools?.length }, '[Server] Tool detector state'); let accumulatedText = ''; let toolCallsForPersist: ToolCallForPersistence[] | undefined; // Check for command before calling Lumo - const commandResult = await tryExecuteCommand(turns, ctx.commandContext); + const commandResult = await tryExecuteCommand(turns, context.commandContext); if (commandResult) { accumulatedText = commandResult.response; emitter?.emitOutputTextDelta(itemId, 0, 0, accumulatedText); } else { - // Normal flow: call Lumo + // Normal flow: call Lumo with TamerTool loop let nextOutputIndex = 1; - const processor = createStreamingToolProcessor(ctx.hasCustomTools, { - emitTextDelta(text) { - accumulatedText += text; - emitter?.emitOutputTextDelta(itemId, 0, 0, text); - }, - emitToolCall(callId, tc) { - emitter?.emitFunctionCallEvents(id, callId, tc.name, JSON.stringify(tc.arguments), nextOutputIndex++); - }, - }); try { - const result = await deps.queue.add(async () => - deps.lumoClient.chatWithHistory(turns, processor.onChunk, { - requestTitle: ctx.requestTitle, - instructions, - injectInstructionsInto, - }) - ); + const loopResult = await runServerToolLoop({ + deps, + context, + turns, + conversationId, + instructions, + injectInstructionsInto, + onTextDelta(text) { + accumulatedText += text; + emitter?.emitOutputTextDelta(itemId, 0, 0, text); + }, + onToolCall(callId, tc) { + // Only CustomTool calls reach here (ServerTools filtered by loop) + emitter?.emitFunctionCallEvents(id, callId, tc.name, JSON.stringify(tc.arguments), nextOutputIndex++); + }, + }); logger.debug('[Server] Stream completed'); - processor.finalize(); - persistTitle(result, deps, conversationId); - toolCallsForPersist = mapToolCallsForPersistence(processor.toolCallsEmitted); + persistTitle(loopResult.chatResult, deps, conversationId); + toolCallsForPersist = mapToolCallsForPersistence(loopResult.customToolCalls); - persistAssistantTurn(deps, conversationId, result.message, toolCallsForPersist); + persistAssistantTurn(deps, conversationId, loopResult.chatResult.message, toolCallsForPersist); } catch (error) { logger.error({ error: String(error) }, 'Response error'); if (emitter) { diff --git a/src/api/routes/shared.ts b/src/api/routes/shared.ts index f73d41d..712c31d 100644 --- a/src/api/routes/shared.ts +++ b/src/api/routes/shared.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'crypto'; import type { Response } from 'express'; -import { getCustomToolsConfig } from '../../app/config.js'; +import { getCustomToolsConfig, getServerToolsEnabled } from '../../app/config.js'; import { getMetrics } from '../../app/metrics'; import type { CommandContext } from '../../app/commands.js'; import type { EndpointDependencies, OpenAITool, OpenAIToolCall } from '../types.js'; @@ -51,9 +51,15 @@ export function buildRequestContext( conversationId: ConversationId | undefined, tools?: OpenAITool[] ): RequestContext { - const serverToolsConfig = getCustomToolsConfig(); + const customToolsConfig = getCustomToolsConfig(); + const serverToolsEnabled = getServerToolsEnabled(); + + // Enable tool detection if either CustomTools or ServerTools are active + const hasClientTools = customToolsConfig.enabled && !!tools && tools.length > 0; + const hasServerTools = serverToolsEnabled; + return { - hasCustomTools: serverToolsConfig.enabled && !!tools && tools.length > 0, + hasCustomTools: hasClientTools || hasServerTools, commandContext: { syncInitialized: deps.syncInitialized ?? false, conversationId, diff --git a/src/api/server.ts b/src/api/server.ts index 1180654..eede24a 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1,5 +1,5 @@ import express from 'express'; -import { getServerConfig, getMetricsConfig, authConfig } from '../app/config.js'; +import { getServerConfig, getMetricsConfig, getServerToolsEnabled, authConfig } from '../app/config.js'; import { resolveProjectPath } from '../app/paths.js'; import { logger } from '../app/logger.js'; import { setupAuthMiddleware, setupLoggingMiddleware, setupMetricsMiddleware } from './middleware.js'; @@ -72,6 +72,13 @@ export class APIServer { } async start(): Promise<void> { + // Initialize ServerTools if enabled + if (getServerToolsEnabled()) { + const { initializeServerTools } = await import('./tools/server-tools/index.js'); + initializeServerTools(); + logger.info('ServerTools initialized'); + } + const { validateTemplateOnce } = await import('./instructions.js'); validateTemplateOnce(this.serverConfig.instructions.template); diff --git a/src/api/tools/server-tools/executor.ts b/src/api/tools/server-tools/executor.ts new file mode 100644 index 0000000..c930f9a --- /dev/null +++ b/src/api/tools/server-tools/executor.ts @@ -0,0 +1,133 @@ +/** + * ServerTool Execution + * + * Executes ServerTools with error handling and logging. + * Provides helpers for partitioning and building continuation turns. + */ + +import { logger } from '../../../app/logger.js'; +import { getServerTool, isServerTool, type ServerToolContext } from './registry.js'; +import type { OpenAIToolCall } from '../../types.js'; +import type { Turn } from '../../../lumo-client/types.js'; +import { Role } from '../../../lumo-client/types.js'; + +export interface ServerToolExecutionResult { + /** Whether the tool name matched a registered ServerTool */ + isServerTool: boolean; + /** Result string if execution succeeded */ + result?: string; + /** Error message if execution failed */ + error?: string; +} + +/** + * Execute a ServerTool by name. + * + * @param toolName Tool name (without prefix) + * @param args Arguments parsed from the tool call + * @param context ServerTool context + * @returns Execution result + */ +export async function executeServerTool( + toolName: string, + args: Record<string, unknown>, + context: ServerToolContext +): Promise<ServerToolExecutionResult> { + const tool = getServerTool(toolName); + if (!tool) { + return { isServerTool: false }; + } + + try { + logger.info({ tool: toolName, args }, 'Executing ServerTool'); + const result = await tool.handler(args, context); + logger.debug({ tool: toolName, resultLength: result.length }, 'ServerTool completed'); + return { isServerTool: true, result }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + logger.error({ error, tool: toolName }, 'ServerTool execution failed'); + return { isServerTool: true, error: errorMessage }; + } +} + +// ── Partitioning ────────────────────────────────────────────────────── + +export interface PartitionedToolCalls { + serverToolCalls: OpenAIToolCall[]; + customToolCalls: OpenAIToolCall[]; +} + +/** + * Partition tool calls into ServerTools and CustomTools. + * ServerTools are executed server-side, CustomTools are passed to API clients. + */ +export function partitionToolCalls(toolCalls: OpenAIToolCall[]): PartitionedToolCalls { + const serverToolCalls: OpenAIToolCall[] = []; + const customToolCalls: OpenAIToolCall[] = []; + + for (const tc of toolCalls) { + if (isServerTool(tc.function.name)) { + serverToolCalls.push(tc); + } else { + customToolCalls.push(tc); + } + } + + return { serverToolCalls, customToolCalls }; +} + +// ── Continuation ────────────────────────────────────────────────────── + +/** + * Execute ServerTools and build continuation turns for the next Lumo call. + * + * Creates: + * 1. An assistant turn with the iteration text (which includes tool call JSON) + * 2. User turns with tool results for each executed ServerTool + * + * @param serverToolCalls - ServerTool calls to execute + * @param assistantText - Text from the current iteration (includes tool call JSON) + * @param context - ServerTool execution context + * @param prefix - CustomTools prefix for tool result formatting + * @returns Turn[] ready to append for next Lumo call + */ +export async function buildServerToolContinuation( + serverToolCalls: OpenAIToolCall[], + assistantText: string, + context: ServerToolContext, + prefix: string +): Promise<Turn[]> { + const continuationTurns: Turn[] = []; + + // Add assistant turn with the text (which includes tool call JSON) + continuationTurns.push({ + role: Role.Assistant, + content: assistantText, + }); + + // Execute each ServerTool and add result as user turn + for (const tc of serverToolCalls) { + const args = JSON.parse(tc.function.arguments); + const execResult = await executeServerTool(tc.function.name, args, context); + + // Format result similar to CustomTool results + const resultContent = execResult.error + ? `Error executing ${tc.function.name}: ${execResult.error}` + : execResult.result ?? 'No result'; + + // Build user turn with tool result in JSON format (similar to function_call_output) + const toolResultJson = JSON.stringify({ + type: 'function_call_output', + call_id: tc.id, + tool_name: `${prefix}${tc.function.name}`, + output: resultContent, + }); + + continuationTurns.push({ + role: Role.User, + content: `\`\`\`json\n${toolResultJson}\n\`\`\``, + }); + } + + return continuationTurns; +} diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts new file mode 100644 index 0000000..53180de --- /dev/null +++ b/src/api/tools/server-tools/index.ts @@ -0,0 +1,33 @@ +/** + * ServerTools - Server-side tools callable by Lumo + * + * ServerTools are executed by the server, unlike CustomTools which are passed to API clients. + * They use the same instruction mechanism as CustomTools (JSON in code blocks). + */ + +import { registerServerTool } from './registry.js'; +import { searchServerTool } from './search.js'; + +// Re-export types and functions +export { + registerServerTool, + getServerTool, + isServerTool, + getAllServerToolDefinitions, + clearServerTools, + type ServerTool, + type ServerToolContext, + type ServerToolHandler, +} from './registry.js'; + +export { executeServerTool, type ServerToolExecutionResult } from './executor.js'; + +export { runServerToolLoop, type ServerToolLoopOptions, type ServerToolLoopResult } from './loop.js'; + +/** + * Initialize all built-in ServerTools. + * Called during server startup when enableServerTools is true. + */ +export function initializeServerTools(): void { + registerServerTool(searchServerTool); +} diff --git a/src/api/tools/server-tools/loop.ts b/src/api/tools/server-tools/loop.ts new file mode 100644 index 0000000..2f07901 --- /dev/null +++ b/src/api/tools/server-tools/loop.ts @@ -0,0 +1,143 @@ +/** + * ServerTool Execution Loop + * + * Shared loop for both /v1/responses and /v1/chat/completions endpoints. + * Handles ServerTool detection, execution, and continuation. + */ + +import { logger } from '../../../app/logger.js'; +import { getCustomToolsConfig } from '../../../app/config.js'; +import { createStreamingToolProcessor, type StreamingToolEmitter } from '../streaming-processor.js'; +import { isServerTool, type ServerToolContext } from './registry.js'; +import { partitionToolCalls, buildServerToolContinuation } from './executor.js'; +import type { EndpointDependencies, OpenAIToolCall } from '../../types.js'; +import type { RequestContext } from '../../routes/shared.js'; +import type { Turn, ChatResult } from '../../../lumo-client/types.js'; +import type { ConversationId } from '../../../conversations/types.js'; +import type { ParsedToolCall } from '../types.js'; + +// ── Types ───────────────────────────────────────────────────────────── + +export interface ServerToolLoopOptions { + deps: EndpointDependencies; + context: RequestContext; + turns: Turn[]; + conversationId?: ConversationId; + instructions?: string; + injectInstructionsInto: 'first' | 'last'; + /** Callback for text deltas during streaming */ + onTextDelta: (text: string) => void; + /** Callback for tool calls (only CustomTools, ServerTools filtered out) */ + onToolCall: (callId: string, tc: ParsedToolCall) => void; +} + +export interface ServerToolLoopResult { + /** Accumulated text from all iterations */ + accumulatedText: string; + /** CustomTool calls only (ServerTool calls filtered out) */ + customToolCalls: OpenAIToolCall[]; + /** Final chat result from last Lumo call */ + chatResult: ChatResult; +} + +const MAX_SERVER_TOOL_LOOPS = 5; + +// ── Loop implementation ─────────────────────────────────────────────── + +/** + * Run the ServerTool execution loop. + * + * This function: + * 1. Calls Lumo with streaming processor + * 2. Detects ServerTool calls in the response + * 3. Executes ServerTools server-side + * 4. Loops back to Lumo with results (up to MAX_SERVER_TOOL_LOOPS times) + * 5. Returns final text and any CustomTool calls + */ +export async function runServerToolLoop(options: ServerToolLoopOptions): Promise<ServerToolLoopResult> { + const { deps, context, instructions, injectInstructionsInto, onTextDelta, onToolCall } = options; + const { prefix } = getCustomToolsConfig(); + + let currentTurns = [...options.turns]; + let loopCount = 0; + let accumulatedText = ''; + const allCustomToolCalls: OpenAIToolCall[] = []; + let chatResult: ChatResult | undefined; + + // Build ServerTool context + const serverToolCtx: ServerToolContext = { + conversationStore: deps.conversationStore, + conversationId: options.conversationId, + }; + + while (loopCount < MAX_SERVER_TOOL_LOOPS) { + loopCount++; + logger.debug({ loopCount }, 'ServerTool loop iteration'); + + // Track text for this iteration + let iterationText = ''; + + // Create emitter that wraps the original callbacks + const emitter: StreamingToolEmitter = { + emitTextDelta(text) { + iterationText += text; + accumulatedText += text; + onTextDelta(text); + }, + emitToolCall(callId, tc) { + // Only emit CustomTool calls to the client + if (!isServerTool(tc.name)) { + onToolCall(callId, tc); + } + }, + }; + + // Create streaming processor + const processor = createStreamingToolProcessor(context.hasCustomTools, emitter); + + // Call Lumo + const result = await deps.queue.add(async () => + deps.lumoClient.chatWithHistory(currentTurns, processor.onChunk, { + requestTitle: context.requestTitle, + instructions, + injectInstructionsInto, + }) + ); + + processor.finalize(); + chatResult = result; + + // Partition tool calls into ServerTools and CustomTools + const { serverToolCalls, customToolCalls } = partitionToolCalls(processor.toolCallsEmitted); + allCustomToolCalls.push(...customToolCalls); + + // If no ServerTools, we're done + if (serverToolCalls.length === 0) { + logger.debug({ loopCount, customToolCalls: customToolCalls.length }, 'ServerTool loop complete (no ServerTools)'); + break; + } + + logger.info({ loopCount, serverToolCount: serverToolCalls.length }, 'Executing ServerTools'); + + // Execute ServerTools and build continuation turns + const continuationTurns = await buildServerToolContinuation( + serverToolCalls, + iterationText, + serverToolCtx, + prefix + ); + + // Update turns for next iteration + currentTurns = [...currentTurns, ...continuationTurns]; + } + + if (loopCount >= MAX_SERVER_TOOL_LOOPS) { + logger.warn({ maxLoops: MAX_SERVER_TOOL_LOOPS }, 'ServerTool loop reached maximum iterations'); + } + + return { + accumulatedText, + customToolCalls: allCustomToolCalls, + chatResult: chatResult!, + }; +} diff --git a/src/api/tools/server-tools/registry.ts b/src/api/tools/server-tools/registry.ts new file mode 100644 index 0000000..d4d4eae --- /dev/null +++ b/src/api/tools/server-tools/registry.ts @@ -0,0 +1,88 @@ +/** + * Server Tool Registry + * + * Manages server-side tools (ServerTools) that Lumo can call. + * ServerTools are executed by the server, unlike CustomTools which are passed to API clients. + */ + +import { OpenAITool } from 'src/api/types.js'; +import type { IConversationStore } from '../../../conversations/index.js'; +import type { ConversationId } from '../../../conversations/types.js'; + +// ── Types ───────────────────────────────────────────────────────────── + +/** Context passed to ServerTool handlers. */ +export interface ServerToolContext { + conversationStore?: IConversationStore; + conversationId?: ConversationId; +} + +/** ServerTool handler function. Returns result as string. */ +export type ServerToolHandler = ( + args: Record<string, unknown>, + context: ServerToolContext +) => Promise<string>; + +/** A ServerTool with its definition and handler. */ +export interface ServerTool { + definition: OpenAITool; + handler: ServerToolHandler; +} + +/** + * Prefix to avoid name collisions between server and client tool names, while not confusing Lumo about distinction between native and custom tools. + * NOTE: This prefix is used at server tool definition time and not appended/stripped dynamically, like customTools.prefix + * Final server tool function names look like: + * customTools.prefix + serverToolPrefix + name + * ie. user:lumo_search + */ +export const serverToolPrefix = "lumo_" + +// ── Registry ────────────────────────────────────────────────────────── + +const registry = new Map<string, ServerTool>(); + +/** + * Register a ServerTool. + * @param tool The ServerTool definition and handler + */ +export function registerServerTool(tool: ServerTool): void { + const name = tool.definition.function.name; + if (registry.has(name)) { + throw new Error(`ServerTool "${name}" is already registered`); + } + registry.set(name, tool); +} + +/** + * Get a ServerTool by name. + * @param name Tool name (without prefix) + * @returns The ServerTool or undefined if not found + */ +export function getServerTool(name: string): ServerTool | undefined { + return registry.get(name); +} + +/** + * Check if a tool name is a registered ServerTool. + * @param name Tool name (without prefix) + */ +export function isServerTool(name: string): boolean { + return registry.has(name); +} + +/** + * Get all registered ServerTool definitions. + * Used for merging into instructions. + */ +export function getAllServerToolDefinitions(): OpenAITool[] { + return Array.from(registry.values()).map(t => t.definition); +} + +/** + * Clear all registered ServerTools. + * Mainly for testing. + */ +export function clearServerTools(): void { + registry.clear(); +} diff --git a/src/api/tools/server-tools/search.ts b/src/api/tools/server-tools/search.ts new file mode 100644 index 0000000..7ab77a1 --- /dev/null +++ b/src/api/tools/server-tools/search.ts @@ -0,0 +1,54 @@ +/** + * Search ServerTool + * + * Allows Lumo to search through conversation history. + * Wraps the existing search logic from src/conversations/search.ts. + */ + +import { searchConversations, formatSearchResults } from '../../../conversations/search.js'; +import { serverToolPrefix, type ServerTool } from './registry.js'; + +export const searchServerTool: ServerTool = { + definition: { + type: 'function', + function: { + name: serverToolPrefix + 'search', + description: 'Search through conversation history by title and message content. Returns matching conversations with snippets.', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'The search query to find in conversation titles and message content', + }, + limit: { + type: 'number', + description: 'Maximum number of results to return (default 10)', + }, + }, + required: ['query'], + }, + }, + }, + handler: async (args, context) => { + const query = args.query; + if (typeof query !== 'string' || !query.trim()) { + return 'Error: query parameter is required and must be a non-empty string'; + } + + if (!context.conversationStore) { + return 'Search unavailable: conversation store not initialized'; + } + + const limit = typeof args.limit === 'number' ? Math.min(Math.max(1, args.limit), 50) : 10; + + const results = searchConversations( + context.conversationStore, + query.trim(), + limit, + context.conversationId // Exclude current conversation + ); + + return formatSearchResults(results, query.trim()); + }, +}; diff --git a/src/app/config.ts b/src/app/config.ts index 4982639..23d5c2d 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -119,6 +119,7 @@ const serverMergedConfigSchema = z.object({ conversations: conversationsConfigSchema, commands: z.object({ enabled: z.boolean(), wakeword: z.string() }), enableWebSearch: z.boolean(), + enableServerTools: z.boolean(), customTools: customToolsConfigSchema, instructions: serverInstructionsConfigSchema, metrics: metricsConfigSchema, @@ -236,6 +237,11 @@ export function getCustomToolsConfig() { return cfg.customTools; } +export function getServerToolsEnabled() { + const cfg = getServerConfig(); + return cfg.enableServerTools; +} + export function getServerInstructionsConfig() { const cfg = getServerConfig(); return cfg.instructions; @@ -286,7 +292,7 @@ export const authConfig = ((): z.infer<typeof authConfigSchema> => { // Mock config (eagerly loaded, needed before initConfig to decide auth vs mock) const mockConfigSchema = z.object({ enabled: z.boolean(), - scenario: z.enum(['success', 'error', 'timeout', 'rejected', 'toolCall', 'misroutedToolCall', 'weeklyLimit', 'cycle']), + scenario: z.enum(['success', 'error', 'timeout', 'rejected', 'toolCall', 'misroutedToolCall', 'serverToolCall', 'weeklyLimit', 'cycle']), }); export const mockConfig = ((): z.infer<typeof mockConfigSchema> => { diff --git a/src/mock/custom-scenarios.ts b/src/mock/custom-scenarios.ts index 941e109..6658dcc 100644 --- a/src/mock/custom-scenarios.ts +++ b/src/mock/custom-scenarios.ts @@ -8,6 +8,7 @@ import { Role, type Turn, type ProtonApiOptions } from '../lumo-client/types.js'; import { formatSSEMessage, delay, type ScenarioGenerator } from './mock-api.js'; import { getServerInstructionsConfig, getCustomToolsConfig } from '../app/config.js'; +import { serverToolPrefix } from '../api/tools/server-tools/registry.js'; /** Extract turns from the mock request payload (unencrypted only). */ function getTurns(options: ProtonApiOptions): Turn[] { @@ -113,4 +114,48 @@ export const customScenarios: Record<string, ScenarioGenerator> = { yield formatSSEMessage({ type: 'done' }); }, + + serverToolCall: async function* (options) { + // Simulates Lumo calling the `search` ServerTool. + // The server should execute the tool and loop back with results. + // + // Phase detection: + // Initial call: output a search tool call + // Follow-up (tool result in turns): respond using the search results + + const turns = getTurns(options); + const lastUserTurn = lastTurnWithRole(turns, Role.User); + const hasToolResult = lastUserTurn?.content?.includes('"type":"function_call_output"'); + + if (hasToolResult) { + // Follow-up: Lumo has received search results, respond naturally + yield formatSSEMessage({ type: 'ingesting', target: 'message' }); + await delay(100); + const tokens = [ + 'Based on the search results, ', + 'I found some relevant conversations. ', + 'Let me summarize what I found for you.', + ]; + for (let i = 0; i < tokens.length; i++) { + yield formatSSEMessage({ type: 'token_data', target: 'message', count: i, content: tokens[i] }); + await delay(20); + } + yield formatSSEMessage({ type: 'done' }); + return; + } + + // Initial call: output a search tool call as JSON text + yield formatSSEMessage({ type: 'ingesting', target: 'message' }); + await delay(100); + + // Include the prefix so the tool call is detected + const prefix = getCustomToolsConfig().prefix + serverToolPrefix; + const toolName = `${prefix}search`; + const json = `\`\`\`json\n{"name":"${toolName}","arguments":{"query":"weather forecast"}}\n\`\`\``; + const tokens = json.split(''); + for (let i = 0; i < tokens.length; i++) { + yield formatSSEMessage({ type: 'token_data', target: 'message', count: i, content: tokens[i] }); + } + yield formatSSEMessage({ type: 'done' }); + }, }; diff --git a/tests/helpers/test-server.ts b/tests/helpers/test-server.ts index 1fd9cda..122a50e 100644 --- a/tests/helpers/test-server.ts +++ b/tests/helpers/test-server.ts @@ -21,12 +21,15 @@ import { setupMetricsMiddleware } from '../../src/api/middleware.js'; import { createMetricsRouter } from '../../src/api/routes/metrics.js'; import type { EndpointDependencies } from '../../src/api/types.js'; import type { MockConfig } from '../../src/app/config.js'; +import { initializeServerTools, clearServerTools } from '../../src/api/tools/server-tools/index.js'; type Scenario = MockConfig['scenario']; export interface TestServerOptions { /** Enable metrics collection and /metrics endpoint */ metrics?: boolean; + /** Enable ServerTools (search, etc.) */ + serverTools?: boolean; } export interface TestServer { @@ -69,6 +72,12 @@ export async function createTestServer( setMetrics(metrics); } + // Set up ServerTools if requested + if (options.serverTools) { + clearServerTools(); // Ensure clean state + initializeServerTools(); + } + const app = express(); app.use(express.json()); // No auth middleware - tests focus on route logic @@ -96,6 +105,7 @@ export async function createTestServer( metrics, close: () => new Promise((resolve) => { if (metrics) setMetrics(null); + if (options.serverTools) clearServerTools(); server.close(() => resolve()); }), }; diff --git a/tests/integration/responses-api.test.ts b/tests/integration/responses-api.test.ts index ad81b69..c27ab5b 100644 --- a/tests/integration/responses-api.test.ts +++ b/tests/integration/responses-api.test.ts @@ -7,7 +7,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createTestServer, parseSSEEvents, type TestServer } from '../helpers/test-server.js'; -import { getCustomToolsConfig } from '../../src/app/config.js'; +import { getCustomToolsConfig, getServerConfig } from '../../src/app/config.js'; /** POST /v1/responses with JSON body, returning the raw Response. */ function postResponses(ts: TestServer, body: Record<string, unknown>): Promise<Response> { @@ -243,6 +243,55 @@ describe('/v1/responses', () => { }); }); + describe('serverToolCall scenario', () => { + let ts: TestServer; + let originalEnableServerTools: boolean; + + beforeAll(async () => { + // Enable ServerTools for this test + originalEnableServerTools = (getServerConfig() as any).enableServerTools; + (getServerConfig() as any).enableServerTools = true; + + ts = await createTestServer('serverToolCall', { serverTools: true }); + }); + afterAll(async () => { + (getServerConfig() as any).enableServerTools = originalEnableServerTools; + await ts.close(); + }); + + it('executes search ServerTool and returns final response', async () => { + const res = await postResponses(ts, { input: 'Search for weather', stream: false }); + + expect(res.status).toBe(200); + const body = await res.json(); + + expect(body.status).toBe('completed'); + + // Final response should contain text from the second Lumo call (after tool execution) + const messageItem = body.output.find((o: any) => o.type === 'message'); + expect(messageItem).toBeDefined(); + expect(messageItem.content[0].text).toContain('search results'); + }); + + it('streaming: executes search ServerTool and streams final response', async () => { + const res = await postResponses(ts, { input: 'Search for weather', stream: true }); + + expect(res.status).toBe(200); + const text = await res.text(); + const events = parseSSEEvents(text); + + // Should have response lifecycle events + const eventTypes = events.map(e => (e.data as any)?.type).filter(Boolean); + expect(eventTypes).toContain('response.created'); + expect(eventTypes).toContain('response.completed'); + + // Final text should be from second Lumo response (after tool execution) + const doneEvent = events.find(e => (e.data as any)?.type === 'response.output_text.done'); + expect(doneEvent).toBeDefined(); + expect((doneEvent!.data as any).text).toContain('search results'); + }); + }); + describe('error scenarios', () => { it('error scenario returns response with error', async () => { const ts = await createTestServer('error'); diff --git a/tests/unit/server-tools.test.ts b/tests/unit/server-tools.test.ts new file mode 100644 index 0000000..57181e3 --- /dev/null +++ b/tests/unit/server-tools.test.ts @@ -0,0 +1,197 @@ +/** + * Tests for ServerTools - server-side tools callable by Lumo + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { + registerServerTool, + getServerTool, + isServerTool, + getAllServerToolDefinitions, + clearServerTools, + type ServerTool, + type ServerToolContext, +} from '../../src/api/tools/server-tools/registry.js'; +import { executeServerTool } from '../../src/api/tools/server-tools/executor.js'; + +describe('ServerTool Registry', () => { + beforeEach(() => { + clearServerTools(); + }); + + describe('registerServerTool', () => { + it('registers a tool successfully', () => { + const tool: ServerTool = { + definition: { + type: 'function', + function: { + name: 'test_tool', + description: 'A test tool', + parameters: { type: 'object', properties: {} }, + }, + }, + handler: async () => 'result', + }; + + registerServerTool(tool); + expect(getServerTool('test_tool')).toBe(tool); + }); + + it('throws when registering duplicate tool', () => { + const tool: ServerTool = { + definition: { + type: 'function', + function: { + name: 'test_tool', + description: 'A test tool', + parameters: {}, + }, + }, + handler: async () => 'result', + }; + + registerServerTool(tool); + expect(() => registerServerTool(tool)).toThrow( + 'ServerTool "test_tool" is already registered' + ); + }); + }); + + describe('getServerTool', () => { + it('returns undefined for unregistered tool', () => { + expect(getServerTool('nonexistent')).toBeUndefined(); + }); + + it('returns registered tool', () => { + const tool: ServerTool = { + definition: { + type: 'function', + function: { + name: 'my_tool', + description: 'My tool', + parameters: {}, + }, + }, + handler: async () => 'ok', + }; + + registerServerTool(tool); + expect(getServerTool('my_tool')).toBe(tool); + }); + }); + + describe('isServerTool', () => { + it('returns false for unregistered tool', () => { + expect(isServerTool('nonexistent')).toBe(false); + }); + + it('returns true for registered tool', () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'exists', description: '', parameters: {} }, + }, + handler: async () => '', + }); + expect(isServerTool('exists')).toBe(true); + }); + }); + + describe('getAllServerToolDefinitions', () => { + it('returns empty array when no tools registered', () => { + expect(getAllServerToolDefinitions()).toEqual([]); + }); + + it('returns all registered tool definitions', () => { + const tool1: ServerTool = { + definition: { + type: 'function', + function: { name: 'tool1', description: 'Tool 1', parameters: {} }, + }, + handler: async () => '1', + }; + const tool2: ServerTool = { + definition: { + type: 'function', + function: { name: 'tool2', description: 'Tool 2', parameters: {} }, + }, + handler: async () => '2', + }; + + registerServerTool(tool1); + registerServerTool(tool2); + + const defs = getAllServerToolDefinitions(); + expect(defs).toHaveLength(2); + expect(defs.map(d => d.function.name)).toContain('tool1'); + expect(defs.map(d => d.function.name)).toContain('tool2'); + }); + }); +}); + +describe('ServerTool Executor', () => { + beforeEach(() => { + clearServerTools(); + }); + + it('returns isServerTool: false for unregistered tool', async () => { + const result = await executeServerTool('nonexistent', {}, {}); + expect(result.isServerTool).toBe(false); + expect(result.result).toBeUndefined(); + expect(result.error).toBeUndefined(); + }); + + it('executes tool and returns result', async () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'echo', description: '', parameters: {} }, + }, + handler: async (args: Record<string, unknown>) => `echoed: ${args.message}`, + }); + + const result = await executeServerTool('echo', { message: 'hello' }, {}); + expect(result.isServerTool).toBe(true); + expect(result.result).toBe('echoed: hello'); + expect(result.error).toBeUndefined(); + }); + + it('returns error when handler throws', async () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'failing', description: '', parameters: {} }, + }, + handler: async () => { + throw new Error('Something went wrong'); + }, + }); + + const result = await executeServerTool('failing', {}, {}); + expect(result.isServerTool).toBe(true); + expect(result.result).toBeUndefined(); + expect(result.error).toBe('Something went wrong'); + }); + + it('passes context to handler', async () => { + let receivedCtx: ServerToolContext | undefined; + + registerServerTool({ + definition: { + type: 'function', + function: { name: 'ctx_test', description: '', parameters: {} }, + }, + handler: async (_args: Record<string, unknown>, context: ServerToolContext) => { + receivedCtx = context; + return 'ok'; + }, + }); + + const context: ServerToolContext = { + conversationId: 'conv-123' as any, + }; + + await executeServerTool('ctx_test', {}, context); + expect(receivedCtx?.conversationId).toBe('conv-123'); + }); +}); diff --git a/tests/unit/shared.test.ts b/tests/unit/shared.test.ts index 0194919..062dc8c 100644 --- a/tests/unit/shared.test.ts +++ b/tests/unit/shared.test.ts @@ -4,7 +4,7 @@ * Tests ID generators, accumulating tool processor, and persistence helpers. */ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; import { generateResponseId, generateItemId, @@ -12,6 +12,16 @@ import { generateChatCompletionId, persistAssistantTurn, } from '../../src/api/routes/shared.js'; +import { + registerServerTool, + clearServerTools, + type ServerToolContext, +} from '../../src/api/tools/server-tools/registry.js'; +import { + partitionToolCalls, + buildServerToolContinuation, +} from '../../src/api/tools/server-tools/executor.js'; +import { Role } from '../../src/lumo-client/types.js'; import { generateCallId, extractToolNameFromCallId } from '../../src/api/tools/call-id.js'; import { createAccumulatingToolProcessor } from '../../src/api/tools/streaming-processor.js'; import type { EndpointDependencies } from '../../src/api/types.js'; @@ -203,3 +213,161 @@ describe('persistAssistantTurn', () => { expect(deps.persistedMessages).toEqual([]); }); }); + +describe('partitionToolCalls', () => { + beforeEach(() => { + clearServerTools(); + }); + + it('returns empty arrays when no tool calls', () => { + const result = partitionToolCalls([]); + expect(result.serverToolCalls).toEqual([]); + expect(result.customToolCalls).toEqual([]); + }); + + it('partitions tool calls into server and custom tools', () => { + // Register a server tool + registerServerTool({ + definition: { + type: 'function', + function: { name: 'lumo_search', description: 'Search', parameters: {} }, + }, + handler: async () => 'result', + }); + + const toolCalls = [ + { id: 'call-1', type: 'function' as const, function: { name: 'lumo_search', arguments: '{}' } }, + { id: 'call-2', type: 'function' as const, function: { name: 'custom_tool', arguments: '{}' } }, + { id: 'call-3', type: 'function' as const, function: { name: 'another_custom', arguments: '{}' } }, + ]; + + const result = partitionToolCalls(toolCalls); + + expect(result.serverToolCalls).toHaveLength(1); + expect(result.serverToolCalls[0].function.name).toBe('lumo_search'); + expect(result.customToolCalls).toHaveLength(2); + expect(result.customToolCalls.map(tc => tc.function.name)).toEqual(['custom_tool', 'another_custom']); + }); + + it('returns all as custom when no server tools registered', () => { + const toolCalls = [ + { id: 'call-1', type: 'function' as const, function: { name: 'tool1', arguments: '{}' } }, + { id: 'call-2', type: 'function' as const, function: { name: 'tool2', arguments: '{}' } }, + ]; + + const result = partitionToolCalls(toolCalls); + + expect(result.serverToolCalls).toEqual([]); + expect(result.customToolCalls).toHaveLength(2); + }); +}); + +describe('buildServerToolContinuation', () => { + beforeEach(() => { + clearServerTools(); + }); + + it('builds continuation turns with assistant message and tool results', async () => { + // Register a server tool + registerServerTool({ + definition: { + type: 'function', + function: { name: 'lumo_search', description: 'Search', parameters: {} }, + }, + handler: async (args) => `Found results for: ${args.query}`, + }); + + const serverToolCalls = [ + { id: 'call-1', type: 'function' as const, function: { name: 'lumo_search', arguments: '{"query":"test"}' } }, + ]; + + const context: ServerToolContext = {}; + const turns = await buildServerToolContinuation(serverToolCalls, 'Assistant text', context, 'user:'); + + expect(turns).toHaveLength(2); + + // First turn: assistant message + expect(turns[0].role).toBe(Role.Assistant); + expect(turns[0].content).toBe('Assistant text'); + + // Second turn: user message with tool result + expect(turns[1].role).toBe(Role.User); + expect(turns[1].content).toContain('function_call_output'); + expect(turns[1].content).toContain('call-1'); + expect(turns[1].content).toContain('Found results for: test'); + }); + + it('handles multiple server tool calls', async () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'tool_a', description: 'A', parameters: {} }, + }, + handler: async () => 'Result A', + }); + registerServerTool({ + definition: { + type: 'function', + function: { name: 'tool_b', description: 'B', parameters: {} }, + }, + handler: async () => 'Result B', + }); + + const serverToolCalls = [ + { id: 'call-a', type: 'function' as const, function: { name: 'tool_a', arguments: '{}' } }, + { id: 'call-b', type: 'function' as const, function: { name: 'tool_b', arguments: '{}' } }, + ]; + + const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'prefix:'); + + // 1 assistant + 2 user turns + expect(turns).toHaveLength(3); + expect(turns[0].role).toBe(Role.Assistant); + expect(turns[1].role).toBe(Role.User); + expect(turns[2].role).toBe(Role.User); + + expect(turns[1].content).toContain('Result A'); + expect(turns[2].content).toContain('Result B'); + }); + + it('includes error message when tool execution fails', async () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'failing_tool', description: 'Fails', parameters: {} }, + }, + handler: async () => { + throw new Error('Something went wrong'); + }, + }); + + const serverToolCalls = [ + { id: 'call-fail', type: 'function' as const, function: { name: 'failing_tool', arguments: '{}' } }, + ]; + + const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'user:'); + + expect(turns).toHaveLength(2); + expect(turns[1].content).toContain('Error executing failing_tool'); + expect(turns[1].content).toContain('Something went wrong'); + }); + + it('includes prefix in tool_name field', async () => { + registerServerTool({ + definition: { + type: 'function', + function: { name: 'my_tool', description: 'My tool', parameters: {} }, + }, + handler: async () => 'ok', + }); + + const serverToolCalls = [ + { id: 'call-1', type: 'function' as const, function: { name: 'my_tool', arguments: '{}' } }, + ]; + + const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'custom:'); + + const content = turns[1].content; + expect(content).toContain('"tool_name":"custom:my_tool"'); + }); +}); From 874de29070e91e98460cf51173bbd41a59c4cd8d Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:10:29 +0200 Subject: [PATCH 13/37] =?UTF-8?q?Tool=20terminology=20updates=20BREAKING:?= =?UTF-8?q?=20renamed/moved=20all=20tool-related=20config=20settings=20-?= =?UTF-8?q?=20enableWebSearch=20=E2=86=92=20native.enabled=20-=20server.cu?= =?UTF-8?q?stomTools=20=E2=86=92=20server.tools.client=20-=20cli.localActi?= =?UTF-8?q?ons=20=E2=86=92=20cli.tools.local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 52 +-- config.defaults.yaml | 113 +++--- docs/custom-tools.md | 237 ----------- docs/howto-home-assistant.md | 24 +- docs/local-actions.md | 205 ---------- docs/tools.md | 383 ++++++++++++++++++ src/api/instructions.ts | 14 +- src/api/routes/shared.ts | 8 +- src/api/tools/call-id.ts | 4 +- src/api/tools/native-tool-call-processor.ts | 4 +- src/api/tools/server-tools/loop.ts | 4 +- src/api/tools/streaming-tool-detector.ts | 8 +- src/app/config.ts | 44 +- src/cli/client.ts | 4 +- src/cli/local-actions/code-executor.ts | 6 +- src/cli/local-actions/file-reader.ts | 6 +- src/cli/message-converter.ts | 14 +- src/lumo-client/client.ts | 6 +- src/mock/custom-scenarios.ts | 6 +- .../integration/chat-completions-api.test.ts | 6 +- tests/integration/responses-api.test.ts | 14 +- tests/unit/file-reader.test.ts | 10 +- 22 files changed, 568 insertions(+), 604 deletions(-) delete mode 100644 docs/custom-tools.md delete mode 100644 docs/local-actions.md create mode 100644 docs/tools.md diff --git a/README.md b/README.md index 69a036d..546171e 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ tamer # use Lumo interactively tamer "make me laugh" # one-time prompt ``` -To give Lumo access to your files and let it execute commands locally, set `cli.localActions.enabled: true` in `config.yaml` (see [Local Actions](#local-actions-cli)). +To give Lumo access to your files and let it execute commands locally, set `cli.tools.local.enabled: true` in `config.yaml` (see [Local Tools](#local-tools-cli)). You can ask Lumo to give you a demo of its capabilities, or see this [demo chat](docs/demo-cli-chat.md). ### In-chat commands @@ -186,14 +186,13 @@ cli: ### Web Search -Enable Lumo's native web search (and other external tools: weather, stock, cryptocurrency): +Enable Lumo's web search (and other native tools: weather, stock, cryptocurrency): ```yaml -server: - enableWebSearch: true - -cli: - enableWebSearch: true +server: # or cli: + tools: + native: + enabled: true ``` ### Instructions @@ -205,37 +204,39 @@ Instructions from API clients will be inserted in the main template. If you can, > **Note:** Under the hood, lumo-tamer injects instructions into normal messages (the same way it is done in Lumo's webclient). Instructions set in the webclient's personal or project settings will be ignored and left unchanged. -### Custom Tools (Server) +### Client Tools (API) Let Lumo use tools provided by your OpenAI-compatible client. ```yaml server: - customTools: - enabled: true + tools: + client: + enabled: true ``` -> **Warning:** Custom tool support is experimental and can fail in various ways. Experiment with `server.instructions` settings to improve results. See [Custom Tools](docs/custom-tools.md) for details, tweaking, and troubleshooting. +> **Tip:** See [Tools](docs/tools.md) for details, tweaking, and troubleshooting. -### Local Actions (CLI) +### Local Tools (CLI) Let Lumo read, create and edit files, and execute commands on your machine: ```yaml cli: - localActions: - enabled: true - fileReads: + tools: + local: enabled: true - executors: - bash: ["bash", "-c"] - python: ["python", "-c"] + fileReads: + enabled: true + executors: + bash: ["bash", "-c"] + python: ["python", "-c"] ``` -The CLI always asks for confirmation before executing commands or applying file changes. File reads are automatic. -Configure available languages for your system in `executors`. By default, `bash`, `python`, and `sh` are enabled. -See [Local Actions](docs/local-actions.md) for further configuration and troubleshooting. +The CLI always asks for confirmation before executing commands or applying file changes. File reads are automatic. +Configure available languages for your system in `executors`. By default, `bash`, `python`, and `sh` are enabled. +See [Tools](docs/tools.md#local-tools-cli) for further configuration and troubleshooting. ### Conversation Sync @@ -269,7 +270,7 @@ See the [full guide](docs/howto-home-assistant.md). TLDR: - Pass the environment variable `OPENAI_BASE_URL=http://yourhost:3003/v1` to Home Assistant. - Add the OpenAI integration and create a new Voice Assistant that uses it. -- To let Lumo control your devices, set `server.customTools.enabled: true` in `config.yaml` (Experimental, see [Custom Tools](docs/custom-tools.md)). +- To let Lumo control your devices, set `server.tools.client.enabled: true` in `config.yaml` (Experimental, see [Tools](docs/tools.md#client-tools-api)). - Open HA Assist in your dashboard or phone and chat away. ### OpenClaw @@ -302,7 +303,7 @@ curl http://localhost:3003/v1/chat/completions \ ### Other API clients -Many clients are untested with lumo-tamer but should work if they only use the `/v1/responses` or `/v1/chat/completions` endpoints. As a rule of thumb: basic chatting will most likely work, but the more a client relies on custom tools, the more the experience is degraded. +Many clients are untested with lumo-tamer but should work if they only use the `/v1/responses` or `/v1/chat/completions` endpoints. As a rule of thumb: basic chatting will most likely work, but the more a client relies on tools, the more the experience is degraded. To test an API client, increase log levels on both the client and lumo-tamer: `server.log.level: debug` and check for errors. Please share your experiences with new API clients (both issues and successes) in [the project discussions](https://github.com/ZeroTricks/lumo-tamer/discussions/new?category=general)! @@ -367,7 +368,7 @@ docker compose run --rm -it -v ./some-dir:/dir/ tamer cli > **Note:** Running the CLI within Docker may not be very useful: > - Lumo will not have access to your files unless you mount a directory. -> - The image is Alpine-based, so your system may not have the commands Lumo tries to run. You can change config options `cli.localActions.executors` and `cli.instructions.forLocalActions` to be more explicit what commands Lumo should use, or you can rebase the `Dockerfile`. +> - The image is Alpine-based, so your system may not have the commands Lumo tries to run. You can change config options `cli.tools.local.executors` and `cli.instructions.forLocalTools` to be more explicit what commands Lumo should use, or you can rebase the `Dockerfile`. @@ -377,9 +378,8 @@ See [docs/](docs/) for detailed documentation: - [Authentication](docs/authentication.md): Auth methods, setup and troubleshooting - [Conversations](docs/conversations.md): Conversation persistence and sync -- [Custom Tools](docs/custom-tools.md): Tool support for API clients +- [Tools](docs/tools.md): Native, client, server, and local tools - [Home Assistant Guide](docs/howto-home-assistant.md): Use Lumo as your Voice Assistant -- [Local Actions](docs/local-actions.md): CLI file operations and code execution - [Development](docs/development.md): Development setup and workflow - [Upstream Files](docs/upstream.md): Proton WebClients files, shims and path aliases diff --git a/config.defaults.yaml b/config.defaults.yaml index c6ee176..d7f9b2a 100644 --- a/config.defaults.yaml +++ b/config.defaults.yaml @@ -111,24 +111,29 @@ server: # Prefix for all metric names prefix: "lumo_" - # Enable Lumo's native web_search tool (and other external tools: weather, stock, cryptocurrency) - enableWebSearch: false - - # Server-side tools callable by Lumo (search, etc.) - # When enabled, Lumo can call these tools and the server executes them directly. - enableServerTools: false - - # Custom tool detection for API clients - # Enable detection of JSON tool calls in Lumo's responses - # WARNING: When enabled, Lumo can trigger actions via API clients! - customTools: - enabled: false - - # Prefix added to custom tool names to distinguish from Lumo's native tools. + # Tool configuration + tools: + # Enable Lumo's native external tools (web_search, weather, stock, cryptocurrency) + # Internal native tools (proton_info) are always enabled. + native: + enabled: false + + # Prefix added to custom tool names (client + server tools) to distinguish from native tools. # Applied to tool definitions sent to Lumo, stripped from tool calls returned to client. # Set to "" to disable prefixing. prefix: "user:" + # Server-side tools callable by Lumo (search, etc.) + # When enabled, Lumo can call these tools and the server executes them directly. + server: + enabled: true + + # Client tool detection for API clients + # Enable detection of JSON tool calls in Lumo's responses + # WARNING: When enabled, Lumo can trigger actions via API clients! + client: + enabled: false + instructions: @@ -139,7 +144,7 @@ server: # - clientInstructions: system/developer message from request # - forTools: the forTools block below (pre-interpolated with {{prefix}}) # - fallback: the fallback block below - # - prefix: tool prefix from customTools.prefix + # - prefix: tool prefix from tools.prefix template: | {{#if tools}} {{forTools}} @@ -208,50 +213,54 @@ cli: target: "file" filePath: "lumo-tamer-cli.log" - # Enable Lumo's native web_search tool (and other external tools: weather, stock, cryptocurrency) - enableWebSearch: false - - # Local actions: code block detection and execution - localActions: - # Enable code block detection (```bash, ```read, ```edit, etc.) - # WARNING: When enabled, Lumo can trigger actions on your machine! - enabled: false - - # ```read blocks: Lumo can read local files without user confirmation - fileReads: - # Enable ```read blocks - # Note that if this is false, Lumo can still ask to read a file using shell tools (ie. cat) - enabled: true - # Max file size. Reading files larger than this fail with an error. - # 512kb translates to roughly 32K tokens (Lumo's "warning level" on context size). - maxFileSize: "360kb" - - # Executors for code block execution - # Maps language tag -> [command, ...args]. Code is appended as last arg. - # Override to restrict or extend - executors: - bash: ["bash", "-c"] - python: ["python", "-c"] - sh: ["sh", "-c"] - # zsh: ["zsh", "-c"] - # powershell: ["powershell", "-Command"] - # ps1: ["powershell", "-Command"] - # cmd: ["cmd", "/c"] - # node: ["node", "-e"] - # javascript: ["node", "-e"] - # js: ["node", "-e"] - # perl: ["perl", "-e"] + # Tool configuration + tools: + # Enable Lumo's native external tools (web_search, weather, stock, cryptocurrency) + # Internal native tools (proton_info) are always enabled. + native: + enabled: false + + # Local tools: code block detection and execution + local: + # Enable code block detection (```bash, ```read, ```edit, etc.) + # WARNING: When enabled, Lumo can trigger actions on your machine! + enabled: false + + # ```read blocks: Lumo can read local files without user confirmation + fileReads: + # Enable ```read blocks + # Note that if this is false, Lumo can still ask to read a file using shell tools (ie. cat) + enabled: true + # Max file size. Reading files larger than this fail with an error. + # 512kb translates to roughly 32K tokens (Lumo's "warning level" on context size). + maxFileSize: "360kb" + + # Executors for code block execution + # Maps language tag -> [command, ...args]. Code is appended as last arg. + # Override to restrict or extend + executors: + bash: ["bash", "-c"] + python: ["python", "-c"] + sh: ["sh", "-c"] + # zsh: ["zsh", "-c"] + # powershell: ["powershell", "-Command"] + # ps1: ["powershell", "-Command"] + # cmd: ["cmd", "/c"] + # node: ["node", "-e"] + # javascript: ["node", "-e"] + # js: ["node", "-e"] + # perl: ["perl", "-e"] instructions: template: | You are a command line assistant. Your output will be read in a terminal. Keep the formatting to a minimum and be concise. - {{#if localActions}} - {{forLocalActions}} + {{#if localTools}} + {{forLocalTools}} {{/if}} - # Injected as {{forLocalActions}} when localActions.enabled=true - forLocalActions: | + # Injected as {{forLocalTools}} when tools.local.enabled=true + forLocalTools: | You can read, edit and create files, and you can execute {{executors}} commands on the user's machine. To execute code, use a code block like ```python. The user will be prompted to execute it and the result will be returned to you. Don't explain the user can execute commands, it will be handled automatically. Keep commands simple and safe. To read files, use a ```read block with one file path per line. Contents will be returned to you automatically without prompting the user. Example: diff --git a/docs/custom-tools.md b/docs/custom-tools.md deleted file mode 100644 index 26a1d9e..0000000 --- a/docs/custom-tools.md +++ /dev/null @@ -1,237 +0,0 @@ -# Custom Tools (API) - -This document covers custom tool integration for API clients (e.g., Home Assistant, Open WebUI). - -For CLI local actions (file operations, code execution), see [local-actions.md](local-actions.md). - ---- - -## Warning - -Custom tool support is experimental. Tool calls can fail because of: - -- **Too many tools**: Lumo gets confused when the client provides many tools or (very) long instructions. -- **Misrouted calls**: Lumo routes custom tools through its native pipeline, which fails server-side. lumo-tamer bounces these back, but this adds latency and isn't always reliable. -- **Wrong tool/arguments**: Lumo sets the wrong tool name or arguments. -- **Detection failures**: JSON code blocks are not properly detected or parsed. - -This requires trial and error. Experiment with `server.instructions` settings to improve results. - -**Privacy note**: When Lumo misroutes a tool call, the tool name and arguments are sent to Proton's servers via the native tool pipeline. This data may be processed unencrypted server-side and could appear in Proton's logs. If your tools handle sensitive data, be aware of this risk. Tool results are not affected: they flow through the normal message pipeline with encryption. - - ---- - -## Quick Start - -1. Enable custom tools in `config.yaml`: - ```yaml - server: - customTools: - enabled: true - ``` - -2. Configure your API client (Home Assistant, Open WebUI, etc.) to use tools as normal. - -3. lumo-tamer intercepts Lumo's responses, detects tool calls, and returns them in OpenAI format for your client to execute. - ---- - ---- - -## Configuration - -### Enable Custom Tools - -```yaml -server: - customTools: - # Enable detection of JSON tool calls in Lumo's responses - enabled: true - - # Prefix added to custom tool names to distinguish from Lumo's native tools. - # Applied to tool definitions sent to Lumo, stripped from tool calls returned to client. - # Set to "" to disable prefixing. - prefix: "user:" -``` - -### Instructions Template - -The instructions sent to Lumo are assembled from a template: - -```yaml -server: - instructions: - # Template for assembling instructions. - # Uses Handlebars-like syntax: {{varName}}, {{#if varName}}...{{/if}} - # Available variables: - # - tools: JSON-stringified tool definitions (truthy when tools provided) - # - clientInstructions: system/developer message from request - # - forTools: the forTools block below (pre-interpolated with {{prefix}}) - # - fallback: the fallback block below - # - prefix: tool prefix from customTools.prefix - template: | - {{#if tools}} - {{forTools}} - {{/if}} - - {{#if clientInstructions}} - {{clientInstructions}} - {{else}} - {{fallback}} - {{/if}} - - {{#if tools}} - Below are all the custom tools you can use. Remember, all tools prefixed with `{{prefix}}` are custom tools and must be called by outputting the JSON to the user. - - {{tools}} - {{/if}} - - # Fallback instructions when no system/developer message is provided - fallback: | - Always answer in plain text. Don't use tables, quote blocks, lists, etc. Be concise. - - # Instructions prepended when tools are provided in the request. - # Can use {{prefix}} variable. - forTools: | - === CUSTOM TOOL PROTOCOL === - The tools below are CUSTOM tools, prefixed with `{{prefix}}`. - - IMPORTANT: Custom tools are NOT part of your built-in tool system. - You MUST call them by outputting JSON as text in a code block to the user, like this: - ```json - {"name": "{{prefix}}example_tool", "arguments": {"param": "value"}} - ``` - DO NOT try to call custom tools through your internal tool mechanism, it will fail with error:true. - DO NOT remove the `{{prefix}}` prefix when calling these tools. - - The user's system will execute them and return results. - === END PROTOCOL === - - # Bounce instruction sent when Lumo routes a custom tool through its native pipeline. - forToolBounce: | - You tried to call a custom tool using your built-in tool system, but custom tools must be called by outputting JSON text within a code block. Please output the tool call as JSON, like this: -``` - -### Instruction Replace Patterns - -Clean up client instructions that might confuse Lumo: - -```yaml -server: - instructions: - # Search/replace patterns applied to client instructions (case-insensitive regex). - # Useful for removing or rewriting phrases that may confuse Lumo about tool calling. - # Each entry: { pattern: "regex", replacement: "text" } - omit replacement to strip. - replacePatterns: - - pattern: "(?<=(?:(?:native|custom|internal|external)\\s)?)(?=tool)" - replacement: "custom " -``` - - -## Troubleshooting - -**Tool calls not detected** -- Ensure `customTools.enabled: true` -- Check that Lumo is outputting valid JSON in code fences -- Review `instructions.forTools` - Lumo may need clearer instructions - -**Wrong tool names** -- Check `customTools.prefix` - it's added to definitions and stripped from responses -- If prefix is causing issues, set to `""` to disable - -**Lumo says "I don't have access to that tool"** -- This is a misrouted call being bounced - should resolve automatically -- If persistent, check logs for bounce failures - ---- - -## Native Tools - -Lumo has built-in tools executed server-side by Proton: - -| Tool | Description | -|------|-------------| -| `proton_info` | Proton product information (always enabled) | -| `web_search` | Web search via Proton's backend | -| `weather` | Weather data | -| `stock` | Stock prices | -| `cryptocurrency` | Cryptocurrency prices | - -Enable/disable external native tools: - -```yaml -server: - # Enable Lumo's native web_search tool (and other external tools) - enableWebSearch: true -``` - -Native and custom tools work together: native tools execute server-side, custom tools are detected client-side. - ---- - -## How Custom Tools Work - -1. **Tool definitions are prefixed** with `customTools.prefix` (e.g., `get_weather` becomes `user:get_weather`) -2. **Instructions are assembled** from `instructions.template` with tool definitions as JSON -3. **Instructions are prepended** to a user message as `[Project instructions: ...]` - - `instructions.injectInto: "first"` (default): inject into first user message (less token usage in multi-turn) - - `instructions.injectInto: "last"`: inject into last user message each request (matches WebClient) -4. **Lumo outputs tool calls** as JSON in code fences: - ```` - I'll check the weather for you. - ```json - {"name": "user:get_weather", "arguments": {"city": "Paris"}} - ``` - ```` - *If Lumo misroutes the tool call through its native pipeline, lumo-tamer bounces it, after which Lumo will output JSON (hopefully). See [Misrouted Tool Calls](#misrouted-tool-calls).* -5. **lumo-tamer detects and extracts** tool calls, strips the prefix, and returns in OpenAI format -6. **Your client executes** the tool and sends results back - -### Response Format - -```json -{ - "choices": [{ - "message": { - "role": "assistant", - "content": "I'll check the weather for you.", - "tool_calls": [{ - "id": "call_abc123", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"city\": \"Paris\"}" - } - }] - } - }] -} -``` - ---- - -## Misrouted Tool Calls - -Sometimes Lumo routes a custom tool through its native SSE pipeline instead of outputting JSON text. This always fails because Proton's backend doesn't know the tool. - -### What Happens - -1. lumo-tamer detects the misrouted call (tool name not in `KNOWN_NATIVE_TOOLS`) -2. Suppresses Lumo's error response ("I don't have access to that tool...") -3. Bounces the call back with `instructions.forToolBounce` -4. Lumo re-outputs the tool call as JSON text -5. Normal detection extracts the tool call - -This is transparent to API clients - the bounce happens internally. - ---- - -## Key Code - -| File | Purpose | -|------|---------| -| `src/api/instructions.ts` | Instruction template assembly | -| `src/api/routes/responses/tool-processor.ts` | `StreamingToolDetector` for streaming detection | -| `src/api/tool-parser.ts` | Non-streaming tool call extraction | -| `src/lumo-client/client.ts` | Misrouted tool bounce logic | diff --git a/docs/howto-home-assistant.md b/docs/howto-home-assistant.md index ba30c85..5f68968 100644 --- a/docs/howto-home-assistant.md +++ b/docs/howto-home-assistant.md @@ -60,9 +60,11 @@ Create `config.yaml`: ```yaml server: apiKey: "your-secret-api-key-here" - customTools: - enabled: true # allows Lumo to control your devices - enableWebSearch: true # optionally, enable Lumo's own websearch + tools: + client: + enabled: true # allows Lumo to control your devices + native: + enabled: true # optionally, enable Lumo's own websearch ``` > **Security:** Keep your API key private and make sure lumo-tamer is only accessible from your local network, not the internet. @@ -144,9 +146,11 @@ Create `config.yaml`: ```yaml server: apiKey: "your-secret-api-key-here" - customTools: - enabled: true # allows Lumo to control your devices - enableWebSearch: true # optionally, enable Lumo's own websearch + tools: + client: + enabled: true # allows Lumo to control your devices + native: + enabled: true # optionally, enable Lumo's own websearch ``` > **Security:** Keep your API key private and make sure lumo-tamer is only accessible from your local network, not the internet. @@ -315,7 +319,7 @@ Follow the [HACS installation guide](https://hacs.xyz/docs/setup/download). 1. Go to **Settings** > **Voice Assistants** > **Expose** tab 2. Select which entities Lumo can access -> **Tip:** Start with a few entities to test, add more later. Custom tool support is experimental, see [Custom Tools](custom-tools.md) for troubleshooting. +> **Tip:** Start with a few entities to test, add more later. Client tool support is experimental, see [Tools](tools.md#client-tools-api) for troubleshooting. --- @@ -352,14 +356,14 @@ Try: Lumo taking a few seconds to answer is to be expected. If you encounter larger response times when calling tools: - Reduce the number of exposed entities. - Enable Home Assistant's built-in intent recognition to handle simple commands locally. -- Lumo might [misroute](custom-tools.md#misrouted-tool-calls) tool calls, which lumo-tamer needs to redirect, adding to the latency. Enable debug logging for lumo-tamer (`server.log.level: debug`), look for "misrouted tool calls" and experiment with settings `server.instructions` to get better results. +- Lumo might [misroute](tools.md#misrouted-tool-calls) tool calls, which lumo-tamer needs to redirect, adding to the latency. Enable debug logging for lumo-tamer (`server.log.level: debug`), look for "misrouted tool calls" and experiment with settings `server.instructions` to get better results. ### Device control not working or Lumo saying "I can't do that" This usually indicates Lumo has trouble understanding the exposed entities and tools. - Ask Lumo "What devices do you know about?" or "What Home Assistant tools can you use?" to see what it can access. -- Ensure `customTools.enabled: true` in config.yaml +- Ensure `tools.client.enabled: true` in config.yaml - Check that entities are exposed in HA (**Settings** > **Voice Assistants** > **Expose**) and reduce the number of aliases per entity. - Enable debug logging for lumo-tamer (`server.log.level: debug`) and check logs for errors @@ -380,6 +384,6 @@ conversations: - [lumo-tamer README](../README.md) - [lumo-tamer Authentication](authentication.md) -- [lumo-tamer Custom Tools](custom-tools.md) +- [lumo-tamer Tools](tools.md) - [Home Assistant OpenAI integration](https://www.home-assistant.io/integrations/openai_conversation/) - [Home Assistant Extended OpenAI Conversation](https://github.com/jekalmin/extended_openai_conversation) diff --git a/docs/local-actions.md b/docs/local-actions.md deleted file mode 100644 index 1c50e4f..0000000 --- a/docs/local-actions.md +++ /dev/null @@ -1,205 +0,0 @@ -# Local Actions (CLI) - -This document covers CLI local actions: file operations and code execution. - -For API custom tool integration, see [custom-tools.md](custom-tools.md). - ---- - -## Status - -The CLI is a proof of concept. Local actions work, but the UI is basic and it's hard to keep track of conversations and actions. Development focus is on making API custom tools more reliable, enabling third-party clients like [Nanocoder](https://github.com/AbanteAI/nanocoder) that offer better local actions, richer instructions, and a cleaner interface. - ---- - -## Quick Start - -1. Enable local actions in `config.yaml`: - ```yaml - cli: - localActions: - enabled: true - ``` - -2. Start the CLI: - ```bash - tamer - ``` - -3. Lumo can now read files, make edits, and execute code on your machine. You can ask Lumo to give you a demo of its CLI capabilities, or see this [demo chat](demo-cli-chat.md) for inspiration. - ---- - -## Configuration - -### Enable Local Actions - -```yaml -cli: - localActions: - # Enable code block detection (```bash, ```read, ```edit, etc.) - # WARNING: When enabled, Lumo can trigger actions on your machine! - enabled: true -``` - -### File Reads - -```yaml -cli: - localActions: - # ```read blocks: Lumo can read local files without user confirmation - fileReads: - # Enable ```read blocks - # Note: if disabled, Lumo can still ask to read files using shell tools (e.g., cat) - enabled: true - # Max file size. Files larger than this are skipped with an error. - maxFileSize: "512kb" -``` - -### Code Executors - -```yaml -cli: - localActions: - # Maps language tag -> [command, ...args]. Code is appended as last arg. - executors: - bash: ["bash", "-c"] - python: ["python", "-c"] - sh: ["sh", "-c"] - # Uncomment to enable more: - # zsh: ["zsh", "-c"] - # powershell: ["powershell", "-Command"] - # node: ["node", "-e"] - # perl: ["perl", "-e"] -``` - -### Instructions - -```yaml -cli: - instructions: - template: | - You are a command line assistant. Your output will be read in a terminal. Keep the formatting to a minimum and be concise. - - {{#if localActions}} - {{forLocalActions}} - {{/if}} - - # Injected as {{forLocalActions}} when localActions.enabled=true - forLocalActions: | - You can read, edit and create files, and you can execute {{executors}} commands on the user's machine. - To execute code, use a code block like ```python. The user will be prompted to execute it and the result will be returned to you. - To read files, use a ```read block with one file path per line. Contents will be returned automatically. - To create a new file, use a ```create block. The user will be prompted to confirm. - To edit an existing file, use a ```edit block (one file per block). Read the file first if needed. -``` - -## User Confirmation - -| Action | Confirmation Required | -|------------|----------------------| -| `read` | No - automatic | -| `edit` | Yes - shows diff | -| `create` | Yes - shows content | -| Code execution | Yes - shows command | - ---- - -## Troubleshooting - -**"Command not found" for code execution** -- Check that the executor is configured in `cli.localActions.executors` -- Verify the command exists on your system (e.g., `python` vs `python3`) - -**File reads failing** -- Check `fileReads.maxFileSize` - large files are rejected -- Verify file path is correct (relative to working directory) - -**Edits not applying** -- Lumo must match the exact text in `<<<<<<< SEARCH` -- Read the file first so Lumo sees current content - ---- - -## How It Works - -Lumo outputs code blocks with specific language tags. The CLI detects these and executes them locally: - -1. `CodeBlockDetector` detects triple-backtick code blocks in Lumo's response -2. `BlockHandler.matches()` checks the language tag to find the right handler -3. Handler executes (with confirmation if required) -4. Results are sent back to Lumo as follow-up messages - -The language tag is the dispatch mechanism - no JSON parsing involved. - -### Read Files - -Lumo reads files without prompting: - -```` -```read -README.md -src/config.ts -``` -```` - -File contents are returned to Lumo automatically. - -### Edit Files - -Lumo proposes edits, you confirm: - -```` -```edit -=== FILE: src/config.ts -<<<<<<< SEARCH -const timeout = 5000; -======= -const timeout = 10000; ->>>>>>> REPLACE -``` -```` - -You'll see a diff and be prompted to accept or reject. - -### Create Files - -Lumo proposes new files, you confirm: - -```` -```create -=== FILE: src/new-feature.ts -export function newFeature() { - return "Hello!"; -} -``` -```` - -### Execute Code - -Lumo runs commands, you confirm: - -```` -```bash -ls -la -``` -```` - -```` -```python -print("Hello from Python!") -``` -```` - -Only languages configured in `executors` are allowed. - -### Key Files - -| File | Purpose | -|------|---------| -| `src/cli/code-block-detector.ts` | Detects code blocks in streaming response | -| `src/cli/block-handlers.ts` | Handler registry and base class | -| `src/cli/handlers/file-reader.ts` | `read` block handler | -| `src/cli/handlers/edit-applier.ts` | `edit` block handler | -| `src/cli/handlers/file-creator.ts` | `create` block handler | -| `src/cli/handlers/code-executor.ts` | Code execution handler | diff --git a/docs/tools.md b/docs/tools.md new file mode 100644 index 0000000..4eccfe0 --- /dev/null +++ b/docs/tools.md @@ -0,0 +1,383 @@ +# Tools + +This document covers all tool-related features in lumo-tamer. + +--- + +## Terminology + +| Term | Description | +|------|-------------| +| **Native tools** | Tools executed by Lumo/Proton via SSE. Internal (`proton_info`) are always enabled; external (`web_search`, `weather`, `stock`, `cryptocurrency`) are configurable. | +| **Custom tools** | Non-native tools called via JSON text output. Prefixed with `tools.prefix` (default: `user:`). | +| **Server tools** | Custom tools executed by lumo-tamer itself (e.g., `user:lumo_search`). | +| **Client tools** | Custom tools defined by API clients (e.g., Home Assistant), returned for client-side execution. | +| **Local tools** | CLI-only code block handlers for file operations and code execution. | +| **Misrouted** | When Lumo incorrectly routes a custom tool through its native pipeline. | + +### Tool Flow + +``` +API Request + | + v ++-- Native tools --> Proton executes server-side +| ++-- Custom tools (JSON in response) + | + +-- Server tools --> lumo-tamer executes, loops back + | + +-- Client tools --> Returned to API client for execution + +CLI Request + | + +-- Native tools --> Proton executes server-side + | + +-- Local tools --> CLI detects code blocks, executes locally +``` + +--- + +## Configuration + +### Server Tools Config + +```yaml +server: + tools: + # Enable Lumo's native external tools (web_search, weather, stock, cryptocurrency) + # Internal native tools (proton_info) are always enabled. + native: + enabled: false + + # Prefix for custom tool names (client + server tools) + # Applied to definitions sent to Lumo, stripped from responses. + prefix: "user:" + + # Server-side tools (search, etc.) executed by lumo-tamer + server: + enabled: false + + # Client tool detection - returns tool calls to API clients + client: + enabled: false +``` + +### CLI Tools Config + +```yaml +cli: + tools: + # Enable Lumo's native external tools + native: + enabled: false + + # Local tools: code block detection and execution + local: + enabled: false + fileReads: + enabled: true + maxFileSize: "360kb" + executors: + bash: ["bash", "-c"] + python: ["python", "-c"] + sh: ["sh", "-c"] +``` + +--- + +## Native Tools + +Lumo has built-in tools executed server-side by Proton: + +| Tool | Description | +|------|-------------| +| `proton_info` | Proton product information (always enabled) | +| `web_search` | Web search via Proton's backend | +| `weather` | Weather data | +| `stock` | Stock prices | +| `cryptocurrency` | Cryptocurrency prices | + +Enable external native tools: + +```yaml +server: + tools: + native: + enabled: true + +cli: + tools: + native: + enabled: true +``` + +--- + +## Client Tools (API) + +Client tools allow API clients (Home Assistant, Open WebUI, etc.) to provide tools that Lumo can call. + +### Warning + +Client tool support is experimental. Tool calls can fail because of: + +- **Too many tools**: Lumo gets confused with many tools or long instructions. +- **Misrouted calls**: Lumo routes custom tools through its native pipeline, which fails. lumo-tamer bounces these back, adding latency. +- **Wrong tool/arguments**: Lumo sets wrong tool names or arguments. +- **Detection failures**: JSON code blocks not properly detected. + +**Privacy note**: When Lumo misroutes a tool call, the tool name and arguments are sent to Proton's servers unencrypted. + +### Quick Start + +1. Enable client tools: + ```yaml + server: + tools: + client: + enabled: true + ``` + +2. Configure your API client to use tools as normal. + +3. lumo-tamer intercepts Lumo's responses, detects tool calls, and returns them in OpenAI format. + +### How It Works + +1. **Tool definitions are prefixed** with `tools.prefix` (e.g., `get_weather` becomes `user:get_weather`) +2. **Instructions are assembled** from template with tool definitions as JSON +3. **Instructions are injected** into a user message +4. **Lumo outputs tool calls** as JSON in code fences: + ```` + I'll check the weather for you. + ```json + {"name": "user:get_weather", "arguments": {"city": "Paris"}} + ``` + ```` +5. **lumo-tamer detects and extracts** tool calls, strips the prefix, returns in OpenAI format +6. **Your client executes** the tool and sends results back + +### Misrouted Tool Calls + +Sometimes Lumo routes a custom tool through its native SSE pipeline instead of outputting JSON text. + +1. lumo-tamer detects the misrouted call (tool name not in known native tools) +2. Suppresses Lumo's error response +3. Bounces the call back with `instructions.forToolBounce` +4. Lumo re-outputs the tool call as JSON text +5. Normal detection extracts the tool call + +This is transparent to API clients. + +### Instructions Template + +Customize how instructions are sent to Lumo: + +```yaml +server: + instructions: + # Template variables: tools, clientInstructions, forTools, fallback, prefix + template: | + {{#if tools}} + {{forTools}} + {{/if}} + + {{#if clientInstructions}} + {{clientInstructions}} + {{else}} + {{fallback}} + {{/if}} + + {{#if tools}} + Below are all the custom tools you can use... + {{tools}} + {{/if}} + + forTools: | + === CUSTOM TOOL PROTOCOL === + The tools below are CUSTOM tools, prefixed with `{{prefix}}`. + You MUST call them by outputting JSON in a code block... + === END PROTOCOL === + + # Where to inject: "first" or "last" user message + injectInto: "first" + + # Sent when Lumo misroutes a tool call + forToolBounce: | + You tried to call a custom tool using your built-in tool system... +``` + +### Troubleshooting + +**Tool calls not detected** +- Ensure `tools.client.enabled: true` +- Check that Lumo outputs valid JSON in code fences +- Review `instructions.forTools` + +**Wrong tool names** +- Check `tools.prefix` - it's added to definitions and stripped from responses + +**Lumo says "I don't have access to that tool"** +- This is a misrouted call being bounced - should resolve automatically + +--- + +## Server Tools + +Server tools are custom tools executed by lumo-tamer itself, not passed to API clients. + +### Available Server Tools + +| Tool | Description | +|------|-------------| +| `lumo_search` | Search conversation history by title and message content | + +### Enable + +```yaml +server: + tools: + server: + enabled: true +``` + +Server tools are prefixed with both `tools.prefix` and an internal `lumo_` prefix (e.g., `user:lumo_search`). + +--- + +## Local Tools (CLI) + +Local tools allow the CLI to execute code blocks and file operations on your machine. + +### Status + +The CLI is a proof of concept. Local tools work, but the UI is basic. Development focus is on API custom tools and third-party clients like [Nanocoder](https://github.com/AbanteAI/nanocoder). + +### Quick Start + +1. Enable local tools: + ```yaml + cli: + tools: + local: + enabled: true + ``` + +2. Start the CLI: + ```bash + tamer + ``` + +3. Lumo can now read files, make edits, and execute code. See [demo chat](demo-cli-chat.md) for examples. + +### Configuration + +```yaml +cli: + tools: + local: + enabled: true + fileReads: + enabled: true + maxFileSize: "512kb" + executors: + bash: ["bash", "-c"] + python: ["python", "-c"] + sh: ["sh", "-c"] +``` + +### User Confirmation + +| Action | Confirmation Required | +|--------|----------------------| +| `read` | No - automatic | +| `edit` | Yes - shows diff | +| `create` | Yes - shows content | +| Code execution | Yes - shows command | + +### How It Works + +Lumo outputs code blocks with specific language tags. The CLI detects these and executes them locally: + +1. `CodeBlockDetector` detects triple-backtick code blocks +2. `BlockHandler.matches()` checks the language tag +3. Handler executes (with confirmation if required) +4. Results are sent back to Lumo + +### Read Files + +```` +```read +README.md +src/config.ts +``` +```` + +File contents are returned automatically. + +### Edit Files + +```` +```edit +=== FILE: src/config.ts +<<<<<<< SEARCH +const timeout = 5000; +======= +const timeout = 10000; +>>>>>>> REPLACE +``` +```` + +### Create Files + +```` +```create +=== FILE: src/new-feature.ts +export function newFeature() { + return "Hello!"; +} +``` +```` + +### Execute Code + +```` +```bash +ls -la +``` +```` + +```` +```python +print("Hello from Python!") +``` +```` + +Only languages configured in `executors` are allowed. + +### Troubleshooting + +**"Command not found" for code execution** +- Check that the executor is configured in `cli.tools.local.executors` +- Verify the command exists on your system (e.g., `python` vs `python3`) + +**File reads failing** +- Check `fileReads.maxFileSize` +- Verify file path is correct + +**Edits not applying** +- Lumo must match the exact text in `<<<<<<< SEARCH` +- Read the file first so Lumo sees current content + +--- + +## Key Code + +| File | Purpose | +|------|---------| +| `src/api/instructions.ts` | Instruction template assembly | +| `src/api/tools/streaming-tool-detector.ts` | JSON tool call detection in streams | +| `src/api/tools/server-tools/` | Server tool registry, executor, loop | +| `src/lumo-client/client.ts` | Misrouted tool bounce logic | +| `src/cli/local-actions/` | Local tool handlers (read, edit, create, execute) | diff --git a/src/api/instructions.ts b/src/api/instructions.ts index 16c25a8..23060a0 100644 --- a/src/api/instructions.ts +++ b/src/api/instructions.ts @@ -7,7 +7,7 @@ */ import { logger } from '../app/logger.js'; -import { getServerInstructionsConfig, getCustomToolsConfig, getServerToolsEnabled } from '../app/config.js'; +import { getServerInstructionsConfig, getClientToolsConfig, getCustomToolPrefix, getServerToolsEnabled } from '../app/config.js'; import { interpolateTemplate } from '../app/template.js'; import { applyToolPrefix, applyToolNamePrefix } from './tools/prefix.js'; import { getAllServerToolDefinitions } from './tools/server-tools/index.js'; @@ -119,9 +119,9 @@ function extractToolNames(tools?: OpenAITool[]): string[] { */ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: string): string { const instructionsConfig = getServerInstructionsConfig(); - const toolsConfig = getCustomToolsConfig(); + const clientToolsConfig = getClientToolsConfig(); const serverToolsEnabled = getServerToolsEnabled(); - const { prefix } = toolsConfig; + const prefix = getCustomToolPrefix(); const { replacePatterns } = instructionsConfig; // Merge TamerTool definitions if enabled @@ -132,10 +132,10 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str } // Determine if we should include tools - // Include if either CustomTools are enabled with client tools, or ServerTools are enabled - const hasCustomTools = toolsConfig.enabled && tools && tools.length > 0; + // Include if either client tools are enabled with tools, or server tools are enabled + const hasClientTools = clientToolsConfig.enabled && tools && tools.length > 0; const hasServerTools = serverToolsEnabled && getAllServerToolDefinitions().length > 0; - const includeTools = hasCustomTools || hasServerTools; + const includeTools = hasClientTools || hasServerTools; // Pre-interpolate forTools block (it can use {{prefix}}) const forTools = interpolateTemplate(instructionsConfig.forTools, { prefix }); @@ -152,7 +152,7 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str if (clientInstructions) { cleanedClientInstructions = applyReplacePatterns(clientInstructions, replacePatterns); if (includeTools) { - // Only prefix client tool names, not TamerTool names + // Only prefix client tool names, not server tool names const toolNames = extractToolNames(tools); cleanedClientInstructions = applyToolNamePrefix(cleanedClientInstructions, toolNames, prefix); } diff --git a/src/api/routes/shared.ts b/src/api/routes/shared.ts index 712c31d..0869667 100644 --- a/src/api/routes/shared.ts +++ b/src/api/routes/shared.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'crypto'; import type { Response } from 'express'; -import { getCustomToolsConfig, getServerToolsEnabled } from '../../app/config.js'; +import { getClientToolsConfig, getServerToolsEnabled } from '../../app/config.js'; import { getMetrics } from '../../app/metrics'; import type { CommandContext } from '../../app/commands.js'; import type { EndpointDependencies, OpenAITool, OpenAIToolCall } from '../types.js'; @@ -51,11 +51,11 @@ export function buildRequestContext( conversationId: ConversationId | undefined, tools?: OpenAITool[] ): RequestContext { - const customToolsConfig = getCustomToolsConfig(); + const clientToolsConfig = getClientToolsConfig(); const serverToolsEnabled = getServerToolsEnabled(); - // Enable tool detection if either CustomTools or ServerTools are active - const hasClientTools = customToolsConfig.enabled && !!tools && tools.length > 0; + // Enable tool detection if either client tools or server tools are active + const hasClientTools = clientToolsConfig.enabled && !!tools && tools.length > 0; const hasServerTools = serverToolsEnabled; return { diff --git a/src/api/tools/call-id.ts b/src/api/tools/call-id.ts index 9864885..855f0ba 100644 --- a/src/api/tools/call-id.ts +++ b/src/api/tools/call-id.ts @@ -7,7 +7,7 @@ */ import { randomUUID } from 'crypto'; -import { getCustomToolsConfig } from '../../app/config.js'; +import { getCustomToolPrefix } from '../../app/config.js'; import { logger } from '../../app/logger.js'; import { getMetrics } from '../../app/metrics.js'; @@ -70,7 +70,7 @@ export function addToolNameToFunctionOutput(content: string): string { if (parsed.type === 'function_call_output' && parsed.call_id) { const toolName = extractToolNameFromCallId(String(parsed.call_id)); if (toolName) { - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); const prefixedToolName = prefix ? `${prefix}${toolName}` : toolName; return JSON.stringify({ ...parsed, diff --git a/src/api/tools/native-tool-call-processor.ts b/src/api/tools/native-tool-call-processor.ts index be1d660..89bbcf3 100644 --- a/src/api/tools/native-tool-call-processor.ts +++ b/src/api/tools/native-tool-call-processor.ts @@ -15,7 +15,7 @@ import { JsonBraceTracker } from './json-brace-tracker.js'; import { stripToolPrefix } from './prefix.js'; -import { getCustomToolsConfig } from '../../app/config.js'; +import { getCustomToolPrefix } from '../../app/config.js'; import { getMetrics } from '../../app/metrics.js'; import { logger } from '../../app/logger.js'; import type { ParsedToolCall } from './types.js'; @@ -115,7 +115,7 @@ export class NativeToolCallProcessor { } if (this.isMisrouted(toolCall)) { - const strippedName = stripToolPrefix(toolCall.name, getCustomToolsConfig().prefix); + const strippedName = stripToolPrefix(toolCall.name, getCustomToolPrefix()); getMetrics()?.toolCallsTotal.inc({ type: 'custom', status: 'misrouted', tool_name: strippedName }); diff --git a/src/api/tools/server-tools/loop.ts b/src/api/tools/server-tools/loop.ts index 2f07901..6917d7b 100644 --- a/src/api/tools/server-tools/loop.ts +++ b/src/api/tools/server-tools/loop.ts @@ -6,7 +6,7 @@ */ import { logger } from '../../../app/logger.js'; -import { getCustomToolsConfig } from '../../../app/config.js'; +import { getCustomToolPrefix } from '../../../app/config.js'; import { createStreamingToolProcessor, type StreamingToolEmitter } from '../streaming-processor.js'; import { isServerTool, type ServerToolContext } from './registry.js'; import { partitionToolCalls, buildServerToolContinuation } from './executor.js'; @@ -56,7 +56,7 @@ const MAX_SERVER_TOOL_LOOPS = 5; */ export async function runServerToolLoop(options: ServerToolLoopOptions): Promise<ServerToolLoopResult> { const { deps, context, instructions, injectInstructionsInto, onTextDelta, onToolCall } = options; - const { prefix } = getCustomToolsConfig(); + const prefix = getCustomToolPrefix(); let currentTurns = [...options.turns]; let loopCount = 0; diff --git a/src/api/tools/streaming-tool-detector.ts b/src/api/tools/streaming-tool-detector.ts index 1833f58..2f36b56 100644 --- a/src/api/tools/streaming-tool-detector.ts +++ b/src/api/tools/streaming-tool-detector.ts @@ -13,7 +13,7 @@ import { JsonBraceTracker } from './json-brace-tracker.js'; import { isToolCallJson, parseToolCallJson, type ParsedToolCall } from './types.js'; import { logger } from '../../app/logger.js'; -import { getCustomToolsConfig } from '../../app/config.js'; +import { getCustomToolPrefix } from '../../app/config.js'; import { stripToolPrefix } from './prefix.js'; import { getMetrics } from '../../app/metrics.js'; @@ -224,7 +224,7 @@ export class StreamingToolDetector { private extractToolName(content: string): string | null { const match = content.match(/"name"\s*:\s*"([^"]+)"/); if (match) { - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); return stripToolPrefix(match[1], prefix); } return null; @@ -250,7 +250,7 @@ export class StreamingToolDetector { if (isToolCallJson(parsed)) { const normalized = parseToolCallJson(parsed); if (!normalized) return null; - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); const toolName = stripToolPrefix(normalized.name, prefix); logger.info(`Tool call detected: ${content.replace(/\n/g, ' ').substring(0, 100)}...`); return { @@ -260,7 +260,7 @@ export class StreamingToolDetector { } // JSON parsed but schema invalid - only track if it has a name (looks like attempted tool call) if ('name' in parsed && typeof parsed.name === 'string') { - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); const toolName = stripToolPrefix(parsed.name, prefix); this.trackInvalidToolCall('missing arguments', content, toolName); } diff --git a/src/app/config.ts b/src/app/config.ts index 23d5c2d..7a2750e 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -37,10 +37,12 @@ const replacePatternSchema = z.object({ replacement: z.string().optional(), }); -// Server-specific custom tools config -const customToolsConfigSchema = z.object({ - enabled: z.boolean(), +// Server tools config (native, custom prefix, server, client) +const serverToolsConfigSchema = z.object({ + native: z.object({ enabled: z.boolean() }), prefix: z.string(), + server: z.object({ enabled: z.boolean() }), + client: z.object({ enabled: z.boolean() }), }); // Metrics config @@ -63,7 +65,7 @@ const injectIntoSchema = z.enum(['first', 'last']); const cliInstructionsConfigSchema = z.object({ injectInto: injectIntoSchema, template: z.string(), - forLocalActions: z.string(), + forLocalTools: z.string(), forToolBounce: z.string(), }); const serverInstructionsConfigSchema = z.object({ @@ -75,8 +77,8 @@ const serverInstructionsConfigSchema = z.object({ replacePatterns: z.array(replacePatternSchema), }); -// CLI local actions config -const localActionsConfigSchema = z.object({ +// CLI local tools config +const localToolsConfigSchema = z.object({ enabled: z.boolean(), fileReads: z.object({ enabled: z.boolean(), @@ -85,6 +87,12 @@ const localActionsConfigSchema = z.object({ executors: z.record(z.string(), z.array(z.string())), }); +// CLI tools config (native, local) +const cliToolsConfigSchema = z.object({ + native: z.object({ enabled: z.boolean() }), + local: localToolsConfigSchema, +}); + export const authMethodSchema = z.enum(['login', 'browser', 'rclone']); const authConfigSchema = z.object({ @@ -118,9 +126,7 @@ const serverMergedConfigSchema = z.object({ log: logConfigSchema, conversations: conversationsConfigSchema, commands: z.object({ enabled: z.boolean(), wakeword: z.string() }), - enableWebSearch: z.boolean(), - enableServerTools: z.boolean(), - customTools: customToolsConfigSchema, + tools: serverToolsConfigSchema, instructions: serverInstructionsConfigSchema, metrics: metricsConfigSchema, bodyLimit: byteSizeSchema, @@ -135,8 +141,7 @@ const cliMergedConfigSchema = z.object({ log: logConfigSchema, conversations: conversationsConfigSchema, commands: z.object({ enabled: z.boolean(), wakeword: z.string() }), - enableWebSearch: z.boolean(), - localActions: localActionsConfigSchema, + tools: cliToolsConfigSchema, instructions: cliInstructionsConfigSchema, }); @@ -224,7 +229,7 @@ function getConfig(): MergedConfig { export const getLogConfig = () => getConfig().log; export const getConversationsConfig = () => getConfig().conversations; export const getCommandsConfig = () => getConfig().commands; -export const getEnableWebSearch = () => getConfig().enableWebSearch; +export const getNativeToolsEnabled = () => getConfig().tools.native.enabled; // Server-specific getters export function getServerConfig(): ServerMergedConfig { @@ -232,14 +237,19 @@ export function getServerConfig(): ServerMergedConfig { return config as ServerMergedConfig; } -export function getCustomToolsConfig() { +export function getClientToolsConfig() { + const cfg = getServerConfig(); + return cfg.tools.client; +} + +export function getCustomToolPrefix() { const cfg = getServerConfig(); - return cfg.customTools; + return cfg.tools.prefix; } export function getServerToolsEnabled() { const cfg = getServerConfig(); - return cfg.enableServerTools; + return cfg.tools.server.enabled; } export function getServerInstructionsConfig() { @@ -258,9 +268,9 @@ export function getCliConfig(): CliMergedConfig { return config as CliMergedConfig; } -export function getLocalActionsConfig() { +export function getLocalToolsConfig() { const cfg = getCliConfig(); - return cfg.localActions; + return cfg.tools.local; } export function getCliInstructionsConfig() { diff --git a/src/cli/client.ts b/src/cli/client.ts index 68b8b40..74950da 100644 --- a/src/cli/client.ts +++ b/src/cli/client.ts @@ -9,7 +9,7 @@ */ import { executeCommand, isCommand, type CommandContext } from '../app/commands.js'; -import { getCliInstructionsConfig, getCommandsConfig, getLocalActionsConfig } from '../app/config.js'; +import { getCliInstructionsConfig, getCommandsConfig, getLocalToolsConfig } from '../app/config.js'; import logger from '../app/logger.js'; import { BUSY_INDICATOR, clearBusyIndicator, print } from '../app/terminal.js'; import type { Application } from '../app/index.js'; @@ -53,7 +53,7 @@ export class CLIClient { * Handles streaming, detection, and display. */ private async sendToLumo(options: { requestTitle?: boolean } = {}): Promise<LumoResponse> { - const localActionsConfig = getLocalActionsConfig(); + const localActionsConfig = getLocalToolsConfig(); const detector = localActionsConfig.enabled ? new CodeBlockDetector((lang) => blockHandlers.some(h => h.matches({ language: lang, content: '' })) diff --git a/src/cli/local-actions/code-executor.ts b/src/cli/local-actions/code-executor.ts index f189ae6..59d6c7f 100644 --- a/src/cli/local-actions/code-executor.ts +++ b/src/cli/local-actions/code-executor.ts @@ -6,7 +6,7 @@ import { spawn } from 'child_process'; import type { CodeBlock, BlockHandler } from './types.js'; -import { getLocalActionsConfig } from '../../app/config.js'; +import { getLocalToolsConfig } from '../../app/config.js'; export interface ExecutionResult { type: 'execution'; @@ -32,7 +32,7 @@ export function summarizeExecutableBlock(language: string | null, content: strin */ export function isExecutable(language: string | null): boolean { if (!language) return false; - const { executors } = getLocalActionsConfig(); + const { executors } = getLocalToolsConfig(); return language in executors; } @@ -43,7 +43,7 @@ export async function executeBlock( block: CodeBlock, onOutput: (chunk: string) => void ): Promise<ExecutionResult> { - const { executors } = getLocalActionsConfig(); + const { executors } = getLocalToolsConfig(); const executor = block.language ? executors[block.language] : undefined; if (!executor) { diff --git a/src/cli/local-actions/file-reader.ts b/src/cli/local-actions/file-reader.ts index c01b9a8..d004de0 100644 --- a/src/cli/local-actions/file-reader.ts +++ b/src/cli/local-actions/file-reader.ts @@ -12,7 +12,7 @@ import bytes from 'bytes'; import { closeSync, openSync, readFileSync, readSync, statSync } from 'fs'; -import { getLocalActionsConfig } from '../../app/config.js'; +import { getLocalToolsConfig } from '../../app/config.js'; import { FILE_PREFIX, type BlockHandler, type CodeBlock } from './types.js'; export interface ReadResult { @@ -66,7 +66,7 @@ export function getFileSize(filePath: string): number { * Read files listed in a read block and return their contents. */ export async function applyReadBlock(block: CodeBlock): Promise<ReadResult> { - const { fileReads } = getLocalActionsConfig(); + const { fileReads } = getLocalToolsConfig(); const maxFileSize = bytes.parse(fileReads.maxFileSize); const paths = block.content.split('\n').map(l => l.trim()).filter(Boolean); @@ -124,7 +124,7 @@ export const readHandler: BlockHandler = { requiresConfirmation: false, confirmOptions: () => ({ label: '', prompt: '', verb: '', errorLabel: '' }), apply: (block) => { - if (!getLocalActionsConfig().fileReads.enabled) { + if (!getLocalToolsConfig().fileReads.enabled) { return Promise.resolve({ type: 'read', success: false, diff --git a/src/cli/message-converter.ts b/src/cli/message-converter.ts index 8d9026f..f26262d 100644 --- a/src/cli/message-converter.ts +++ b/src/cli/message-converter.ts @@ -5,7 +5,7 @@ * not persisted in the conversation store. */ -import { getCliInstructionsConfig, getLocalActionsConfig } from '../app/config.js'; +import { getCliInstructionsConfig, getLocalToolsConfig } from '../app/config.js'; import { interpolateTemplate } from '../app/template.js'; /** @@ -13,19 +13,19 @@ import { interpolateTemplate } from '../app/template.js'; */ export function buildCliInstructions(): string | undefined { const instructionsConfig = getCliInstructionsConfig(); - const localActionsConfig = getLocalActionsConfig(); + const localToolsConfig = getLocalToolsConfig(); // Build executor list (comma-separated language tags) - const executorKeys = Object.keys(localActionsConfig.executors || {}); + const executorKeys = Object.keys(localToolsConfig.executors || {}); const executors = executorKeys.join(', '); - // Pre-interpolate forLocalActions with executors - const forLocalActions = interpolateTemplate(instructionsConfig.forLocalActions, { executors }); + // Pre-interpolate forLocalTools with executors + const forLocalTools = interpolateTemplate(instructionsConfig.forLocalTools, { executors }); // Interpolate main template const result = interpolateTemplate(instructionsConfig.template, { - localActions: localActionsConfig.enabled ? 'true' : undefined, - forLocalActions, + localTools: localToolsConfig.enabled ? 'true' : undefined, + forLocalTools, executors, }); diff --git a/src/lumo-client/client.ts b/src/lumo-client/client.ts index 5d8d473..37f600e 100644 --- a/src/lumo-client/client.ts +++ b/src/lumo-client/client.ts @@ -31,7 +31,7 @@ import { type ChatResult, type ContentBlock, } from './types.js'; -import { getInstructionsConfig, getLogConfig, getConfigMode, getCustomToolsConfig, getEnableWebSearch } from '../app/config.js'; +import { getInstructionsConfig, getLogConfig, getConfigMode, getCustomToolPrefix, getNativeToolsEnabled } from '../app/config.js'; import { injectInstructionsIntoTurns } from './instructions.js'; import { NativeToolCallProcessor } from '../api/tools/native-tool-call-processor.js'; import { postProcessTitle } from '@lumo/lib/lumo-api-client/utils.js'; @@ -66,7 +66,7 @@ function buildBounceInstruction(toolCall: ParsedToolCall): string { // (the tool name in toolCall has already been stripped, so we re-add it) let toolName = toolCall.name; if (getConfigMode() === 'server') { - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); if (prefix && !toolName.startsWith(prefix)) { toolName = `${prefix}${toolName}`; } @@ -258,7 +258,7 @@ export class LumoClient { } // Read from config - applies to both server and CLI modes - const tools: ToolName[] = getEnableWebSearch() + const tools: ToolName[] = getNativeToolsEnabled() ? [...DEFAULT_INTERNAL_TOOLS, ...DEFAULT_EXTERNAL_TOOLS] : DEFAULT_INTERNAL_TOOLS; diff --git a/src/mock/custom-scenarios.ts b/src/mock/custom-scenarios.ts index 6658dcc..14bfbf9 100644 --- a/src/mock/custom-scenarios.ts +++ b/src/mock/custom-scenarios.ts @@ -7,7 +7,7 @@ import { Role, type Turn, type ProtonApiOptions } from '../lumo-client/types.js'; import { formatSSEMessage, delay, type ScenarioGenerator } from './mock-api.js'; -import { getServerInstructionsConfig, getCustomToolsConfig } from '../app/config.js'; +import { getServerInstructionsConfig, getCustomToolPrefix } from '../app/config.js'; import { serverToolPrefix } from '../api/tools/server-tools/registry.js'; /** Extract turns from the mock request payload (unencrypted only). */ @@ -44,7 +44,7 @@ export const customScenarios: Record<string, ScenarioGenerator> = { // Include the prefix so the tool call matches what we instructed Lumo to output yield formatSSEMessage({ type: 'ingesting', target: 'message' }); await delay(200); - const prefix = getCustomToolsConfig().prefix; + const prefix = getCustomToolPrefix(); const toolName = prefix ? `${prefix}GetLiveContext` : 'GetLiveContext'; const json = `\`\`\`json\n{"name":"${toolName}","arguments":{}}\n\`\`\``; const tokens = json.split(''); @@ -149,7 +149,7 @@ export const customScenarios: Record<string, ScenarioGenerator> = { await delay(100); // Include the prefix so the tool call is detected - const prefix = getCustomToolsConfig().prefix + serverToolPrefix; + const prefix = getCustomToolPrefix() + serverToolPrefix; const toolName = `${prefix}search`; const json = `\`\`\`json\n{"name":"${toolName}","arguments":{"query":"weather forecast"}}\n\`\`\``; const tokens = json.split(''); diff --git a/tests/integration/chat-completions-api.test.ts b/tests/integration/chat-completions-api.test.ts index 49140ed..c80b2e6 100644 --- a/tests/integration/chat-completions-api.test.ts +++ b/tests/integration/chat-completions-api.test.ts @@ -4,7 +4,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createTestServer, parseSSEEvents, type TestServer } from '../helpers/test-server.js'; -import { getCustomToolsConfig } from '../../src/app/config.js'; +import { getClientToolsConfig } from '../../src/app/config.js'; /** POST /v1/chat/completions with JSON body, returning the raw Response. */ function postChat(ts: TestServer, body: Record<string, unknown>): Promise<Response> { @@ -139,10 +139,10 @@ describe('/v1/chat/completions', () => { beforeAll(async () => { nativeTs = await createTestServer('misroutedToolCall'); - (getCustomToolsConfig() as any).enabled = true; + (getClientToolsConfig() as any).enabled = true; }); afterAll(async () => { - (getCustomToolsConfig() as any).enabled = false; + (getClientToolsConfig() as any).enabled = false; await nativeTs.close(); }); diff --git a/tests/integration/responses-api.test.ts b/tests/integration/responses-api.test.ts index c27ab5b..6a32d3d 100644 --- a/tests/integration/responses-api.test.ts +++ b/tests/integration/responses-api.test.ts @@ -7,7 +7,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createTestServer, parseSSEEvents, type TestServer } from '../helpers/test-server.js'; -import { getCustomToolsConfig, getServerConfig } from '../../src/app/config.js'; +import { getClientToolsConfig, getServerConfig } from '../../src/app/config.js'; /** POST /v1/responses with JSON body, returning the raw Response. */ function postResponses(ts: TestServer, body: Record<string, unknown>): Promise<Response> { @@ -188,11 +188,11 @@ describe('/v1/responses', () => { beforeAll(async () => { ts = await createTestServer('misroutedToolCall', { metrics: true }); - // Enable custom tool detection so the bounce response JSON is parsed - (getCustomToolsConfig() as any).enabled = true; + // Enable client tool detection so the bounce response JSON is parsed + (getClientToolsConfig() as any).enabled = true; }); afterAll(async () => { - (getCustomToolsConfig() as any).enabled = false; + (getClientToolsConfig() as any).enabled = false; await ts.close(); }); @@ -249,13 +249,13 @@ describe('/v1/responses', () => { beforeAll(async () => { // Enable ServerTools for this test - originalEnableServerTools = (getServerConfig() as any).enableServerTools; - (getServerConfig() as any).enableServerTools = true; + originalEnableServerTools = (getServerConfig() as any).tools.server.enabled; + (getServerConfig() as any).tools.server.enabled = true; ts = await createTestServer('serverToolCall', { serverTools: true }); }); afterAll(async () => { - (getServerConfig() as any).enableServerTools = originalEnableServerTools; + (getServerConfig() as any).tools.server.enabled = originalEnableServerTools; await ts.close(); }); diff --git a/tests/unit/file-reader.test.ts b/tests/unit/file-reader.test.ts index d915954..43fecf8 100644 --- a/tests/unit/file-reader.test.ts +++ b/tests/unit/file-reader.test.ts @@ -9,7 +9,7 @@ import { mkdtempSync, writeFileSync, rmSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { applyReadBlock, isBinaryFile, getFileSize } from '../../src/cli/local-actions/file-reader.js'; -import { initConfig, getLocalActionsConfig } from '../../src/app/config.js'; +import { initConfig, getLocalToolsConfig } from '../../src/app/config.js'; import type { CodeBlock } from '../../src/cli/local-actions/types.js'; // CLI tests need CLI mode config @@ -108,15 +108,15 @@ describe('applyReadBlock', () => { let originalMaxSize: string | number; beforeAll(() => { - originalMaxSize = getLocalActionsConfig().fileReads.maxFileSize; + originalMaxSize = getLocalToolsConfig().fileReads.maxFileSize; }); afterAll(() => { - (getLocalActionsConfig().fileReads as any).maxFileSize = originalMaxSize; + (getLocalToolsConfig().fileReads as any).maxFileSize = originalMaxSize; }); it('rejects a file exceeding maxFileSize', async () => { - (getLocalActionsConfig().fileReads as any).maxFileSize = '1kb'; + (getLocalToolsConfig().fileReads as any).maxFileSize = '1kb'; const result = await applyReadBlock(readBlock(largeFile)); expect(result.success).toBe(false); expect(result.output).toContain('File too large'); @@ -124,7 +124,7 @@ describe('applyReadBlock', () => { }); it('accepts a file within the size limit', async () => { - (getLocalActionsConfig().fileReads as any).maxFileSize = '1kb'; + (getLocalToolsConfig().fileReads as any).maxFileSize = '1kb'; const result = await applyReadBlock(readBlock(textFile)); // 14 bytes expect(result.success).toBe(true); }); From b200326490202a60819940ff7d0839eb4090d1be Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:40:42 +0200 Subject: [PATCH 14/37] =?UTF-8?q?rename=20runServerToolLoop=20=E2=86=92=20?= =?UTF-8?q?chatAndExecute?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/routes/chat-completions/index.ts | 4 ++-- src/api/routes/responses/request-handlers.ts | 4 ++-- src/api/routes/shared.ts | 11 +---------- src/api/tools/server-tools/{loop.ts => handler.ts} | 8 ++++---- src/api/tools/server-tools/index.ts | 2 +- src/api/types.ts | 8 ++++++++ 6 files changed, 18 insertions(+), 19 deletions(-) rename src/api/tools/server-tools/{loop.ts => handler.ts} (95%) diff --git a/src/api/routes/chat-completions/index.ts b/src/api/routes/chat-completions/index.ts index aa991ba..827742c 100644 --- a/src/api/routes/chat-completions/index.ts +++ b/src/api/routes/chat-completions/index.ts @@ -9,7 +9,7 @@ import { ChatCompletionEventEmitter } from './events.js'; import type { Turn } from '../../../lumo-client/index.js'; import type { ConversationId } from '../../../conversations/types.js'; import { trackCustomToolCompletion } from '../../tools/call-id.js'; -import { runServerToolLoop } from '../../tools/server-tools/index.js'; +import { chatAndExecute } from '../../tools/server-tools/index.js'; import { buildRequestContext, persistTitle, @@ -160,7 +160,7 @@ async function handleChatRequest( } else { // Normal flow: call Lumo with TamerTool loop try { - const loopResult = await runServerToolLoop({ + const loopResult = await chatAndExecute({ deps, context, turns, diff --git a/src/api/routes/responses/request-handlers.ts b/src/api/routes/responses/request-handlers.ts index ea79189..f202bd5 100644 --- a/src/api/routes/responses/request-handlers.ts +++ b/src/api/routes/responses/request-handlers.ts @@ -13,7 +13,7 @@ import { ResponseEventEmitter } from './events.js'; import type { Turn } from '../../../lumo-client/index.js'; import type { ConversationId } from '../../../conversations/index.js'; import { generateCallId } from '../../tools/call-id.js'; -import { runServerToolLoop } from '../../tools/server-tools/index.js'; +import { chatAndExecute } from '../../tools/server-tools/index.js'; import { buildRequestContext, persistTitle, @@ -172,7 +172,7 @@ export async function handleRequest( let nextOutputIndex = 1; try { - const loopResult = await runServerToolLoop({ + const loopResult = await chatAndExecute({ deps, context, turns, diff --git a/src/api/routes/shared.ts b/src/api/routes/shared.ts index 0869667..52fe90a 100644 --- a/src/api/routes/shared.ts +++ b/src/api/routes/shared.ts @@ -2,8 +2,7 @@ import { randomUUID } from 'crypto'; import type { Response } from 'express'; import { getClientToolsConfig, getServerToolsEnabled } from '../../app/config.js'; import { getMetrics } from '../../app/metrics'; -import type { CommandContext } from '../../app/commands.js'; -import type { EndpointDependencies, OpenAITool, OpenAIToolCall } from '../types.js'; +import type { EndpointDependencies, OpenAITool, OpenAIToolCall, RequestContext } from '../types.js'; import type { ConversationId } from '../../conversations/types.js'; import type { ChatResult, AssistantMessageData } from '../../lumo-client/index.js'; @@ -34,14 +33,6 @@ export function mapToolCallsForPersistence( })); } -// ── Request context ──────────────────────────────────────────────── - -export interface RequestContext { - hasCustomTools: boolean; - commandContext: CommandContext; - requestTitle: boolean; -} - /** * Build the common request context shared by all handler variants. * When conversationId is undefined (stateless request), requestTitle is false. diff --git a/src/api/tools/server-tools/loop.ts b/src/api/tools/server-tools/handler.ts similarity index 95% rename from src/api/tools/server-tools/loop.ts rename to src/api/tools/server-tools/handler.ts index 6917d7b..857c758 100644 --- a/src/api/tools/server-tools/loop.ts +++ b/src/api/tools/server-tools/handler.ts @@ -11,14 +11,14 @@ import { createStreamingToolProcessor, type StreamingToolEmitter } from '../stre import { isServerTool, type ServerToolContext } from './registry.js'; import { partitionToolCalls, buildServerToolContinuation } from './executor.js'; import type { EndpointDependencies, OpenAIToolCall } from '../../types.js'; -import type { RequestContext } from '../../routes/shared.js'; +import type { RequestContext } from 'src/api/types.js'; import type { Turn, ChatResult } from '../../../lumo-client/types.js'; import type { ConversationId } from '../../../conversations/types.js'; import type { ParsedToolCall } from '../types.js'; // ── Types ───────────────────────────────────────────────────────────── -export interface ServerToolLoopOptions { +export interface ChatAndExecuteOptions { deps: EndpointDependencies; context: RequestContext; turns: Turn[]; @@ -31,7 +31,7 @@ export interface ServerToolLoopOptions { onToolCall: (callId: string, tc: ParsedToolCall) => void; } -export interface ServerToolLoopResult { +export interface ChatAndExecuteResult { /** Accumulated text from all iterations */ accumulatedText: string; /** CustomTool calls only (ServerTool calls filtered out) */ @@ -54,7 +54,7 @@ const MAX_SERVER_TOOL_LOOPS = 5; * 4. Loops back to Lumo with results (up to MAX_SERVER_TOOL_LOOPS times) * 5. Returns final text and any CustomTool calls */ -export async function runServerToolLoop(options: ServerToolLoopOptions): Promise<ServerToolLoopResult> { +export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<ChatAndExecuteResult> { const { deps, context, instructions, injectInstructionsInto, onTextDelta, onToolCall } = options; const prefix = getCustomToolPrefix(); diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts index 53180de..a40b0a1 100644 --- a/src/api/tools/server-tools/index.ts +++ b/src/api/tools/server-tools/index.ts @@ -22,7 +22,7 @@ export { export { executeServerTool, type ServerToolExecutionResult } from './executor.js'; -export { runServerToolLoop, type ServerToolLoopOptions, type ServerToolLoopResult } from './loop.js'; +export { chatAndExecute, type ChatAndExecuteOptions, type ChatAndExecuteResult } from './handler.js'; /** * Initialize all built-in ServerTools. diff --git a/src/api/types.ts b/src/api/types.ts index 4faf47e..1ba3075 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -2,6 +2,7 @@ import { RequestQueue } from './queue.js'; import { LumoClient } from '../lumo-client/index.js'; import type { IConversationStore } from '../conversations/index.js'; import type { AuthManager } from '../auth/index.js'; +import type { CommandContext } from 'src/app/commands.js'; export interface EndpointDependencies { queue: RequestQueue; @@ -240,3 +241,10 @@ export type ResponseStreamEvent = | { type: 'response.function_call_arguments.delta'; item_id: string; output_index: number; delta: string; sequence_number: number } | { type: 'response.function_call_arguments.done'; item_id: string; output_index: number; arguments: string; name: string; sequence_number: number } | { type: 'error'; code: string; message: string; param: string | null; sequence_number: number }; +// ── Request context ──────────────────────────────────────────────── + +export interface RequestContext { + hasCustomTools: boolean; + commandContext: CommandContext; + requestTitle: boolean; +} From 76d9eabc5df1e99323b534d045a8c5215e1a46ed Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:48:24 +0200 Subject: [PATCH 15/37] persist server tool call and outputs --- src/api/routes/chat-completions/index.ts | 11 -- src/api/routes/responses/request-handlers.ts | 21 ++-- src/api/routes/shared.ts | 82 ++----------- src/api/tools/server-tools/executor.ts | 17 +-- src/api/tools/server-tools/handler.ts | 69 +++++++++-- src/conversations/store-interface.ts | 2 +- src/conversations/store.ts | 51 ++++---- tests/unit/shared.test.ts | 116 +------------------ 8 files changed, 121 insertions(+), 248 deletions(-) diff --git a/src/api/routes/chat-completions/index.ts b/src/api/routes/chat-completions/index.ts index 827742c..38a589c 100644 --- a/src/api/routes/chat-completions/index.ts +++ b/src/api/routes/chat-completions/index.ts @@ -12,10 +12,7 @@ import { trackCustomToolCompletion } from '../../tools/call-id.js'; import { chatAndExecute } from '../../tools/server-tools/index.js'; import { buildRequestContext, - persistTitle, - persistAssistantTurn, generateChatCompletionId, - mapToolCallsForPersistence, tryExecuteCommand, setSSEHeaders, } from '../shared.js'; @@ -178,15 +175,7 @@ async function handleChatRequest( }); logger.debug('[Server] Stream completed'); - persistTitle(loopResult.chatResult, deps, conversationId); toolCalls = loopResult.customToolCalls.length > 0 ? loopResult.customToolCalls : undefined; - - persistAssistantTurn( - deps, - conversationId, - loopResult.chatResult.message, - mapToolCallsForPersistence(loopResult.customToolCalls) - ); } catch (error) { logger.error({ error: String(error) }, 'Chat completion error'); if (emitter) { diff --git a/src/api/routes/responses/request-handlers.ts b/src/api/routes/responses/request-handlers.ts index f202bd5..c727f4e 100644 --- a/src/api/routes/responses/request-handlers.ts +++ b/src/api/routes/responses/request-handlers.ts @@ -16,15 +16,11 @@ import { generateCallId } from '../../tools/call-id.js'; import { chatAndExecute } from '../../tools/server-tools/index.js'; import { buildRequestContext, - persistTitle, - persistAssistantTurn, generateResponseId, generateItemId, generateFunctionCallId, - mapToolCallsForPersistence, tryExecuteCommand, setSSEHeaders, - type ToolCallForPersistence, } from '../shared.js'; import { sendServerError } from '../../error-handler.js'; @@ -33,6 +29,7 @@ import { sendServerError } from '../../error-handler.js'; interface ToolCall { name: string; arguments: string | object; + id?: string; // call_id for tool calls from chatAndExecute } interface BuildOutputOptions { @@ -67,7 +64,7 @@ function buildOutputItems(options: BuildOutputOptions): OutputItem[] { : JSON.stringify(toolCall.arguments); // Use pre-generated call_id if available, otherwise generate new one - const callId = 'call_id' in toolCall ? (toolCall as ToolCallForPersistence).call_id : generateCallId(toolCall.name); + const callId = 'id' in toolCall ? (toolCall as { id: string }).id : generateCallId(toolCall.name); output.push({ type: 'function_call', @@ -160,7 +157,7 @@ export async function handleRequest( logger.debug({ hasCustomTools: context.hasCustomTools, toolCount: request.tools?.length }, '[Server] Tool detector state'); let accumulatedText = ''; - let toolCallsForPersist: ToolCallForPersistence[] | undefined; + let toolCalls: ToolCall[] | undefined; // Check for command before calling Lumo const commandResult = await tryExecuteCommand(turns, context.commandContext); @@ -190,10 +187,12 @@ export async function handleRequest( }); logger.debug('[Server] Stream completed'); - persistTitle(loopResult.chatResult, deps, conversationId); - toolCallsForPersist = mapToolCallsForPersistence(loopResult.customToolCalls); - - persistAssistantTurn(deps, conversationId, loopResult.chatResult.message, toolCallsForPersist); + // Map custom tool calls to format needed for response building + toolCalls = loopResult.customToolCalls.map(tc => ({ + name: tc.function.name, + arguments: tc.function.arguments, + id: tc.id, + })); } catch (error) { logger.error({ error: String(error) }, 'Response error'); if (emitter) { @@ -208,7 +207,7 @@ export async function handleRequest( // Build and send response (shared for both command and normal flow) try { - const output = buildOutputItems({ text: accumulatedText, itemId, toolCalls: toolCallsForPersist }); + const output = buildOutputItems({ text: accumulatedText, itemId, toolCalls }); const response = createCompletedResponse(id, createdAt, request, output); if (emitter) { diff --git a/src/api/routes/shared.ts b/src/api/routes/shared.ts index 52fe90a..495959b 100644 --- a/src/api/routes/shared.ts +++ b/src/api/routes/shared.ts @@ -1,38 +1,12 @@ import { randomUUID } from 'crypto'; import type { Response } from 'express'; -import { getClientToolsConfig, getServerToolsEnabled } from '../../app/config.js'; -import { getMetrics } from '../../app/metrics'; -import type { EndpointDependencies, OpenAITool, OpenAIToolCall, RequestContext } from '../types.js'; +import { getServerConfig } from '../../app/config.js'; +import type { EndpointDependencies, OpenAITool, RequestContext } from '../types.js'; import type { ConversationId } from '../../conversations/types.js'; -import type { ChatResult, AssistantMessageData } from '../../lumo-client/index.js'; // Re-export for convenience export { tryExecuteCommand, type CommandResult } from '../../app/commands.js'; -// ── Tool call type for persistence ───────────────────────────────── - -/** Tool call with call_id for persistence and response building. */ -export interface ToolCallForPersistence { - name: string; - arguments: string; - call_id: string; -} - -/** - * Map emitted tool calls to format needed for persistence. - * Returns undefined if no tool calls were emitted. - */ -export function mapToolCallsForPersistence( - toolCallsEmitted: OpenAIToolCall[] -): ToolCallForPersistence[] | undefined { - if (toolCallsEmitted.length === 0) return undefined; - return toolCallsEmitted.map(tc => ({ - name: tc.function.name, - arguments: tc.function.arguments, - call_id: tc.id, - })); -} - /** * Build the common request context shared by all handler variants. * When conversationId is undefined (stateless request), requestTitle is false. @@ -42,15 +16,17 @@ export function buildRequestContext( conversationId: ConversationId | undefined, tools?: OpenAITool[] ): RequestContext { - const clientToolsConfig = getClientToolsConfig(); - const serverToolsEnabled = getServerToolsEnabled(); + + const { tools: { + server: { enabled: serverToolsEnabled }, + client: { enabled: clientToolsEnabled } + } } = getServerConfig(); // Enable tool detection if either client tools or server tools are active - const hasClientTools = clientToolsConfig.enabled && !!tools && tools.length > 0; - const hasServerTools = serverToolsEnabled; + const hasClientTools = clientToolsEnabled && !!tools && tools?.length > 0; return { - hasCustomTools: hasClientTools || hasServerTools, + hasCustomTools: hasClientTools || serverToolsEnabled, commandContext: { syncInitialized: deps.syncInitialized ?? false, conversationId, @@ -63,46 +39,6 @@ export function buildRequestContext( }; } -// ── Persistence helpers ──────────────────────────────────────────── - -/** Persist title if Lumo generated one. No-op for stateless requests. */ -export function persistTitle(result: ChatResult, deps: EndpointDependencies, conversationId: ConversationId | undefined): void { - if (!conversationId || !result.title || !deps.conversationStore) return; - deps.conversationStore.setTitle(conversationId, result.title); // Already processed by LumoClient -} - -/** - * Persist an assistant turn. - * - * When custom tool calls are present, we skip persistence entirely. The client (e.g. Home Assistant) - * will send the assistant message back with the tool output in the next request, and - * appendMessages() will handle it via ID-based deduplication. This avoids order mismatches - * between what we persist and what the client sends back. - * - * Native tool calls (web_search, weather, etc.) are handled differently - they are executed - * server-side by Lumo, so we persist them immediately with the tool call/result data. - * The message data (including JSON-serialized tool call) comes from ChatResult.message. - */ -export function persistAssistantTurn( - deps: EndpointDependencies, - conversationId: ConversationId | undefined, - message: AssistantMessageData, - customToolCalls?: Array<{ name: string; arguments: string; call_id: string }> -): void { - if (conversationId && deps.conversationStore) { - // Custom tool calls: skip persistence (client will send back) - if (customToolCalls && customToolCalls.length > 0) { - return; - } - - // Persist message (with or without native tool data) - deps.conversationStore.appendAssistantResponse(conversationId, message); - } else { - // Stateless: track metric only (no persistence) - getMetrics()?.messagesTotal.inc({ role: 'assistant' }); - } -} - // ── ID generation ───────────────────────────────────────────────── /** Generate a response ID (`resp-xxx`). */ diff --git a/src/api/tools/server-tools/executor.ts b/src/api/tools/server-tools/executor.ts index c930f9a..9f14c4b 100644 --- a/src/api/tools/server-tools/executor.ts +++ b/src/api/tools/server-tools/executor.ts @@ -8,8 +8,8 @@ import { logger } from '../../../app/logger.js'; import { getServerTool, isServerTool, type ServerToolContext } from './registry.js'; import type { OpenAIToolCall } from '../../types.js'; -import type { Turn } from '../../../lumo-client/types.js'; import { Role } from '../../../lumo-client/types.js'; +import type { MessageForStore } from 'src/conversations/types.js'; export interface ServerToolExecutionResult { /** Whether the tool name matched a registered ServerTool */ @@ -54,7 +54,7 @@ export async function executeServerTool( export interface PartitionedToolCalls { serverToolCalls: OpenAIToolCall[]; - customToolCalls: OpenAIToolCall[]; + clientToolCalls: OpenAIToolCall[]; } /** @@ -63,17 +63,17 @@ export interface PartitionedToolCalls { */ export function partitionToolCalls(toolCalls: OpenAIToolCall[]): PartitionedToolCalls { const serverToolCalls: OpenAIToolCall[] = []; - const customToolCalls: OpenAIToolCall[] = []; + const clientToolCalls: OpenAIToolCall[] = []; for (const tc of toolCalls) { if (isServerTool(tc.function.name)) { serverToolCalls.push(tc); } else { - customToolCalls.push(tc); + clientToolCalls.push(tc); } } - return { serverToolCalls, customToolCalls }; + return { serverToolCalls, clientToolCalls }; } // ── Continuation ────────────────────────────────────────────────────── @@ -89,15 +89,15 @@ export function partitionToolCalls(toolCalls: OpenAIToolCall[]): PartitionedTool * @param assistantText - Text from the current iteration (includes tool call JSON) * @param context - ServerTool execution context * @param prefix - CustomTools prefix for tool result formatting - * @returns Turn[] ready to append for next Lumo call + * @returns MessageForStore[] ready to append for next Lumo call */ export async function buildServerToolContinuation( serverToolCalls: OpenAIToolCall[], assistantText: string, context: ServerToolContext, prefix: string -): Promise<Turn[]> { - const continuationTurns: Turn[] = []; +): Promise<MessageForStore[]> { + const continuationTurns: MessageForStore[] = []; // Add assistant turn with the text (which includes tool call JSON) continuationTurns.push({ @@ -126,6 +126,7 @@ export async function buildServerToolContinuation( continuationTurns.push({ role: Role.User, content: `\`\`\`json\n${toolResultJson}\n\`\`\``, + id: tc.id, }); } diff --git a/src/api/tools/server-tools/handler.ts b/src/api/tools/server-tools/handler.ts index 857c758..bb24688 100644 --- a/src/api/tools/server-tools/handler.ts +++ b/src/api/tools/server-tools/handler.ts @@ -10,11 +10,12 @@ import { getCustomToolPrefix } from '../../../app/config.js'; import { createStreamingToolProcessor, type StreamingToolEmitter } from '../streaming-processor.js'; import { isServerTool, type ServerToolContext } from './registry.js'; import { partitionToolCalls, buildServerToolContinuation } from './executor.js'; -import type { EndpointDependencies, OpenAIToolCall } from '../../types.js'; +import type { ChatMessageWithTools, EndpointDependencies, OpenAIToolCall } from '../../types.js'; import type { RequestContext } from 'src/api/types.js'; import type { Turn, ChatResult } from '../../../lumo-client/types.js'; -import type { ConversationId } from '../../../conversations/types.js'; +import type { ConversationId, MessageForStore } from '../../../conversations/types.js'; import type { ParsedToolCall } from '../types.js'; +import { convertToolMessage } from 'src/api/message-converter.js'; // ── Types ───────────────────────────────────────────────────────────── @@ -61,7 +62,7 @@ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<Ch let currentTurns = [...options.turns]; let loopCount = 0; let accumulatedText = ''; - const allCustomToolCalls: OpenAIToolCall[] = []; + const allClientToolCalls: OpenAIToolCall[] = []; let chatResult: ChatResult | undefined; // Build ServerTool context @@ -108,12 +109,12 @@ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<Ch chatResult = result; // Partition tool calls into ServerTools and CustomTools - const { serverToolCalls, customToolCalls } = partitionToolCalls(processor.toolCallsEmitted); - allCustomToolCalls.push(...customToolCalls); + const { serverToolCalls, clientToolCalls } = partitionToolCalls(processor.toolCallsEmitted); + allClientToolCalls.push(...clientToolCalls); // If no ServerTools, we're done if (serverToolCalls.length === 0) { - logger.debug({ loopCount, customToolCalls: customToolCalls.length }, 'ServerTool loop complete (no ServerTools)'); + logger.debug({ loopCount, clientToolCalls: clientToolCalls.length }, 'ServerTool loop complete (no ServerTools)'); break; } @@ -129,15 +130,69 @@ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<Ch // Update turns for next iteration currentTurns = [...currentTurns, ...continuationTurns]; + + // Persist intermediate turns for stateful requests + if (options.conversationId && deps.conversationStore) { + + // Persist title if generated + if (chatResult.title) { + deps.conversationStore.setTitle(options.conversationId, chatResult.title); + } + + // First continuation turn is assistant with tool call JSON + const assistantTurn = continuationTurns[0]; + if (assistantTurn.content) + deps.conversationStore.appendAssistantResponse( + options.conversationId, + { content: assistantTurn.content } + ); + const serverToolCallMessages = convertToolMessage({ + role: 'assistant', + tool_calls: serverToolCalls, + } as ChatMessageWithTools) as MessageForStore[]; + + for (const serverToolCall of serverToolCallMessages) { + deps.conversationStore.appendAssistantResponse( + options.conversationId, + { content: serverToolCall.content! }, + 'succeeded', + serverToolCall.id + ); + } + + // Remaining turns are tool results + const toolResultTurns = continuationTurns.slice(1); + if (toolResultTurns.length > 0) { + deps.conversationStore.appendMessages( + options.conversationId, + toolResultTurns, + false + ); + } + + logger.debug({ conversationId: options.conversationId, loopCount }, 'Persisted server tool iteration'); + } } if (loopCount >= MAX_SERVER_TOOL_LOOPS) { logger.warn({ maxLoops: MAX_SERVER_TOOL_LOOPS }, 'ServerTool loop reached maximum iterations'); } + // Persist final assistant message and title + if (options.conversationId && deps.conversationStore) { + // Skip if custom tools present (client will send back with results) + if (allClientToolCalls.length === 0) { + deps.conversationStore.appendAssistantResponse( + options.conversationId, + chatResult!.message + ); + } + + } + return { accumulatedText, - customToolCalls: allCustomToolCalls, + customToolCalls: allClientToolCalls, chatResult: chatResult!, }; } diff --git a/src/conversations/store-interface.ts b/src/conversations/store-interface.ts index 357d0c1..730936d 100644 --- a/src/conversations/store-interface.ts +++ b/src/conversations/store-interface.ts @@ -24,7 +24,7 @@ export interface IConversationStore { entries(): IterableIterator<[ConversationId, ConversationState]>; // Message operations - appendMessages(id: ConversationId, incoming: MessageForStore[]): Message[]; + appendMessages(id: ConversationId, incoming: MessageForStore[], deduplicate?: boolean): Message[]; appendAssistantResponse( id: ConversationId, messageData: AssistantMessageData, diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 785d260..add7a94 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -204,25 +204,32 @@ export class ConversationStore { */ appendMessages( id: ConversationId, - incoming: MessageForStore[] + incoming: MessageForStore[], + deduplicate = true ): Message[] { const convState = this.getOrCreate(id); + let newMessages: MessageForStore[]; + + if(deduplicate){ + // Validate continuation + const validation = isValidContinuation(incoming, convState.messages); + if (!validation.valid) { + getMetrics()?.invalidContinuationsTotal.inc(); + logger.warn({ + conversationId: id, + reason: validation.reason, + incomingCount: incoming.length, + storedCount: convState.messages.length, + ...validation.debugInfo, + }, 'Invalid conversation continuation'); + } - // Validate continuation - const validation = isValidContinuation(incoming, convState.messages); - if (!validation.valid) { - getMetrics()?.invalidContinuationsTotal.inc(); - logger.warn({ - conversationId: id, - reason: validation.reason, - incomingCount: incoming.length, - storedCount: convState.messages.length, - ...validation.debugInfo, - }, 'Invalid conversation continuation'); + // Find new messages + newMessages = findNewMessages(incoming, convState.messages); + } + else{ + newMessages = incoming; } - - // Find new messages - const newMessages = findNewMessages(incoming, convState.messages); if (newMessages.length === 0) { logger.debug({ conversationId: id }, 'No new messages to append'); @@ -248,7 +255,7 @@ export class ConversationStore { id: messageId, conversationId: id, createdAt: now.toISOString(), - role: msg.role , + role: msg.role, parentId, status: 'succeeded', })); @@ -261,7 +268,7 @@ export class ConversationStore { spaceId: this.spaceId, content: msg.content, status: 'succeeded', - role: msg.role , + role: msg.role, })); } @@ -272,7 +279,7 @@ export class ConversationStore { id: messageId, conversationId: id, createdAt: now.toISOString(), - role: msg.role , + role: msg.role, parentId, status: 'succeeded', content: msg.content, @@ -283,7 +290,7 @@ export class ConversationStore { parentId = messageId; } - + // Track metrics const metrics = getMetrics(); if (metrics) { @@ -343,7 +350,7 @@ export class ConversationStore { // Request push to server this.store.dispatch(pushMessageRequest({ id: messageId })); - + // Update conversation status this.store.dispatch(updateConversationStatus({ id, @@ -429,7 +436,7 @@ export class ConversationStore { // Request push to server this.store.dispatch(pushMessageRequest({ id: messageId })); - + const message: Message = { id: messageId, conversationId: id, @@ -481,7 +488,7 @@ export class ConversationStore { persist: true, })); this.store.dispatch(pushConversationRequest({ id })); - } + } logger.debug({ conversationId: id }, 'Set title'); } diff --git a/tests/unit/shared.test.ts b/tests/unit/shared.test.ts index 062dc8c..0dcc2bb 100644 --- a/tests/unit/shared.test.ts +++ b/tests/unit/shared.test.ts @@ -4,13 +4,12 @@ * Tests ID generators, accumulating tool processor, and persistence helpers. */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, beforeEach } from 'vitest'; import { generateResponseId, generateItemId, generateFunctionCallId, generateChatCompletionId, - persistAssistantTurn, } from '../../src/api/routes/shared.js'; import { registerServerTool, @@ -24,7 +23,6 @@ import { import { Role } from '../../src/lumo-client/types.js'; import { generateCallId, extractToolNameFromCallId } from '../../src/api/tools/call-id.js'; import { createAccumulatingToolProcessor } from '../../src/api/tools/streaming-processor.js'; -import type { EndpointDependencies } from '../../src/api/types.js'; describe('ID generators', () => { it('generateResponseId returns resp-{uuid} format', () => { @@ -102,118 +100,6 @@ describe('createAccumulatingToolProcessor', () => { }); }); -describe('persistAssistantTurn', () => { - interface PersistedMessage { - content: string; - toolCall?: string; - toolResult?: string; - } - - function createMockDeps(): EndpointDependencies & { - persistedMessages: PersistedMessage[]; - } { - const persistedMessages: PersistedMessage[] = []; - return { - persistedMessages, - queue: {} as any, - lumoClient: {} as any, - conversationStore: { - appendAssistantResponse: vi.fn( - (_id: string, messageData: { content: string; toolCall?: string; toolResult?: string }) => { - persistedMessages.push(messageData); - } - ), - } as any, - }; - } - - it('persists content when no tool calls', () => { - const deps = createMockDeps(); - persistAssistantTurn(deps, 'conv-123', { content: 'Hello world' }, undefined); - - expect(deps.persistedMessages).toHaveLength(1); - expect(deps.persistedMessages[0].content).toBe('Hello world'); - expect(deps.persistedMessages[0].toolCall).toBeUndefined(); - }); - - it('skips persistence when custom tool calls are present', () => { - const deps = createMockDeps(); - const toolCalls = [ - { name: 'search', arguments: '{}', call_id: 'call-123' }, - ]; - - persistAssistantTurn(deps, 'conv-123', { content: 'Some text' }, toolCalls); - - // Should NOT persist anything - client will send it back - expect(deps.persistedMessages).toEqual([]); - }); - - it('skips persistence when multiple custom tool calls are present', () => { - const deps = createMockDeps(); - const toolCalls = [ - { name: 'search', arguments: '{"q":"test"}', call_id: 'call-1' }, - { name: 'weather', arguments: '{"loc":"Paris"}', call_id: 'call-2' }, - ]; - - persistAssistantTurn(deps, 'conv-123', { content: 'Let me check that' }, toolCalls); - - expect(deps.persistedMessages).toEqual([]); - }); - - it('does nothing for stateless requests (no conversationId)', () => { - const deps = createMockDeps(); - persistAssistantTurn(deps, undefined, { content: 'Hello' }, undefined); - - expect(deps.persistedMessages).toEqual([]); - }); - - it('persists native tool call with tool data', () => { - const deps = createMockDeps(); - const message = { - content: 'Based on search results...', - toolCall: '{"name":"web_search","arguments":{"query":"test search"}}', - toolResult: '{"results":[{"title":"Result"}]}', - }; - - persistAssistantTurn(deps, 'conv-123', message, undefined); - - expect(deps.persistedMessages).toHaveLength(1); - expect(deps.persistedMessages[0].content).toBe('Based on search results...'); - expect(deps.persistedMessages[0].toolCall).toBe('{"name":"web_search","arguments":{"query":"test search"}}'); - expect(deps.persistedMessages[0].toolResult).toBe('{"results":[{"title":"Result"}]}'); - }); - - it('persists native tool call without tool result', () => { - const deps = createMockDeps(); - const message = { - content: 'Weather info...', - toolCall: '{"name":"weather","arguments":{"location":{"city":"Paris"}}}', - toolResult: undefined, - }; - - persistAssistantTurn(deps, 'conv-123', message, undefined); - - expect(deps.persistedMessages).toHaveLength(1); - expect(deps.persistedMessages[0].toolCall).toBe('{"name":"weather","arguments":{"location":{"city":"Paris"}}}'); - expect(deps.persistedMessages[0].toolResult).toBeUndefined(); - }); - - it('prioritizes custom tool calls over native tool calls', () => { - const deps = createMockDeps(); - const customToolCalls = [{ name: 'custom_tool', arguments: '{}', call_id: 'call-1' }]; - const message = { - content: 'Text', - toolCall: '{"name":"web_search","arguments":{"query":"test"}}', - toolResult: '{}', - }; - - // Both custom and native present - custom takes precedence (skip persistence) - persistAssistantTurn(deps, 'conv-123', message, customToolCalls); - - expect(deps.persistedMessages).toEqual([]); - }); -}); - describe('partitionToolCalls', () => { beforeEach(() => { clearServerTools(); From 64ff6c6e50686833e32d9e2fc1eb3e7056ac22af Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:49:45 +0200 Subject: [PATCH 16/37] simplify erver config getters --- src/api/instructions.ts | 19 +++++++++++-------- src/api/server.ts | 4 ++-- src/app/config.ts | 10 ---------- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/api/instructions.ts b/src/api/instructions.ts index 23060a0..9009279 100644 --- a/src/api/instructions.ts +++ b/src/api/instructions.ts @@ -7,7 +7,7 @@ */ import { logger } from '../app/logger.js'; -import { getServerInstructionsConfig, getClientToolsConfig, getCustomToolPrefix, getServerToolsEnabled } from '../app/config.js'; +import { getServerInstructionsConfig, getCustomToolPrefix, getServerConfig } from '../app/config.js'; import { interpolateTemplate } from '../app/template.js'; import { applyToolPrefix, applyToolNamePrefix } from './tools/prefix.js'; import { getAllServerToolDefinitions } from './tools/server-tools/index.js'; @@ -118,11 +118,14 @@ function extractToolNames(tools?: OpenAITool[]): string[] { * @returns Formatted instruction string */ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: string): string { - const instructionsConfig = getServerInstructionsConfig(); - const clientToolsConfig = getClientToolsConfig(); - const serverToolsEnabled = getServerToolsEnabled(); - const prefix = getCustomToolPrefix(); - const { replacePatterns } = instructionsConfig; + const { + instructions: instructionsConfig, + tools: { + prefix, + server: { enabled: serverToolsEnabled }, + client: { enabled: clientToolsEnabled } + } + } = getServerConfig(); // Merge TamerTool definitions if enabled let allTools = tools ?? []; @@ -133,7 +136,7 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str // Determine if we should include tools // Include if either client tools are enabled with tools, or server tools are enabled - const hasClientTools = clientToolsConfig.enabled && tools && tools.length > 0; + const hasClientTools = clientToolsEnabled && tools && tools.length > 0; const hasServerTools = serverToolsEnabled && getAllServerToolDefinitions().length > 0; const includeTools = hasClientTools || hasServerTools; @@ -150,7 +153,7 @@ export function buildInstructions(tools?: OpenAITool[], clientInstructions?: str // Clean and prefix client instructions let cleanedClientInstructions: string | undefined; if (clientInstructions) { - cleanedClientInstructions = applyReplacePatterns(clientInstructions, replacePatterns); + cleanedClientInstructions = applyReplacePatterns(clientInstructions, instructionsConfig.replacePatterns); if (includeTools) { // Only prefix client tool names, not server tool names const toolNames = extractToolNames(tools); diff --git a/src/api/server.ts b/src/api/server.ts index eede24a..36f4d21 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -1,5 +1,5 @@ import express from 'express'; -import { getServerConfig, getMetricsConfig, getServerToolsEnabled, authConfig } from '../app/config.js'; +import { getServerConfig, getMetricsConfig, authConfig } from '../app/config.js'; import { resolveProjectPath } from '../app/paths.js'; import { logger } from '../app/logger.js'; import { setupAuthMiddleware, setupLoggingMiddleware, setupMetricsMiddleware } from './middleware.js'; @@ -73,7 +73,7 @@ export class APIServer { async start(): Promise<void> { // Initialize ServerTools if enabled - if (getServerToolsEnabled()) { + if (this.serverConfig.tools.server) { const { initializeServerTools } = await import('./tools/server-tools/index.js'); initializeServerTools(); logger.info('ServerTools initialized'); diff --git a/src/app/config.ts b/src/app/config.ts index 7a2750e..c4ea29f 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -237,21 +237,11 @@ export function getServerConfig(): ServerMergedConfig { return config as ServerMergedConfig; } -export function getClientToolsConfig() { - const cfg = getServerConfig(); - return cfg.tools.client; -} - export function getCustomToolPrefix() { const cfg = getServerConfig(); return cfg.tools.prefix; } -export function getServerToolsEnabled() { - const cfg = getServerConfig(); - return cfg.tools.server.enabled; -} - export function getServerInstructionsConfig() { const cfg = getServerConfig(); return cfg.instructions; From 8b8e5e6f4f072d75434df1de03672490096c84fc Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:51:41 +0200 Subject: [PATCH 17/37] fix title not being set when first Lumo message is bounced --- src/lumo-client/client.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lumo-client/client.ts b/src/lumo-client/client.ts index 37f600e..bb723e5 100644 --- a/src/lumo-client/client.ts +++ b/src/lumo-client/client.ts @@ -334,7 +334,10 @@ export class LumoClient { { role: Role.User, content: bounceInstruction }, ]; - return this.chatWithHistory(bounceTurns, onChunk, options, true); + return { + ...this.chatWithHistory(bounceTurns, onChunk, options, true), + title: result.title ? postProcessTitle(result.title) : undefined, + }; } // Post-process title (remove quotes, trim, limit length) From 1381b9d1928333c47e1a669cb42d6bb2212b07f1 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 17:52:34 +0200 Subject: [PATCH 18/37] add server tool to return current time&date (as hello world basically) --- src/api/tools/server-tools/date.ts | 14 ++++++++++++++ src/api/tools/server-tools/index.ts | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 src/api/tools/server-tools/date.ts diff --git a/src/api/tools/server-tools/date.ts b/src/api/tools/server-tools/date.ts new file mode 100644 index 0000000..061f8fb --- /dev/null +++ b/src/api/tools/server-tools/date.ts @@ -0,0 +1,14 @@ +import { serverToolPrefix, type ServerTool } from './registry.js'; + +export const dateServerTool: ServerTool = { + definition: { + type: 'function', + function: { + name: serverToolPrefix + 'get_date', + description: 'Get current time and date', + }, + }, + handler: async () => { + return new Date().toISOString(); + }, +}; diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts index a40b0a1..5346312 100644 --- a/src/api/tools/server-tools/index.ts +++ b/src/api/tools/server-tools/index.ts @@ -6,6 +6,7 @@ */ import { registerServerTool } from './registry.js'; +import { dateServerTool } from './date.js'; import { searchServerTool } from './search.js'; // Re-export types and functions @@ -29,5 +30,6 @@ export { chatAndExecute, type ChatAndExecuteOptions, type ChatAndExecuteResult } * Called during server startup when enableServerTools is true. */ export function initializeServerTools(): void { + registerServerTool(dateServerTool); registerServerTool(searchServerTool); } From 16ece97b76d7484ea30c75af47fdbcc340d2c000 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:19:00 +0200 Subject: [PATCH 19/37] emit server tool calls and results to client --- src/api/routes/chat-completions/events.ts | 31 +++++++- src/api/routes/chat-completions/index.ts | 10 ++- src/api/routes/responses/events.ts | 35 +++++++++ src/api/routes/responses/request-handlers.ts | 17 ++++- src/api/tools/server-tools/executor.ts | 79 ++++++++++++++------ src/api/tools/server-tools/handler.ts | 38 ++++++---- src/api/tools/server-tools/index.ts | 8 +- tests/unit/shared.test.ts | 8 +- 8 files changed, 178 insertions(+), 48 deletions(-) diff --git a/src/api/routes/chat-completions/events.ts b/src/api/routes/chat-completions/events.ts index 436e14b..9af2986 100644 --- a/src/api/routes/chat-completions/events.ts +++ b/src/api/routes/chat-completions/events.ts @@ -27,7 +27,7 @@ export class ChatCompletionEventEmitter { this.res.write(`data: ${JSON.stringify(chunk)}\n\n`); } - emitToolCallDelta(callId: string, name: string, args: Record<string, unknown>): void { + emitToolCallDelta(callId: string, name: string, args: string): void { const chunk: OpenAIStreamChunk = { id: this.id, object: 'chat.completion.chunk', @@ -40,7 +40,7 @@ export class ChatCompletionEventEmitter { index: this.toolCallIndex++, id: callId, type: 'function', - function: { name, arguments: JSON.stringify(args) }, + function: { name, arguments: args }, }], }, finish_reason: null, @@ -49,6 +49,33 @@ export class ChatCompletionEventEmitter { this.res.write(`data: ${JSON.stringify(chunk)}\n\n`); } + /** + * Emit events for a server-executed tool call and its result. + * Emits both the tool_call and a tool result message so clients + * see the complete execution cycle. + */ + emitServerToolExecution(callId: string, toolName: string, args: string, output: string): void { + this.emitToolCallDelta(callId, toolName, args); + + // Emit tool result message + const toolResultChunk = { + id: this.id, + object: 'chat.completion.chunk', + created: this.created, + model: this.model, + choices: [{ + index: 0, + delta: { + role: 'tool', + tool_call_id: callId, + content: output, + }, + finish_reason: null, + }], + }; + this.res.write(`data: ${JSON.stringify(toolResultChunk)}\n\n`); + } + emitDone(toolCalls: OpenAIToolCall[] | undefined): void { const finalChunk: OpenAIStreamChunk = { id: this.id, diff --git a/src/api/routes/chat-completions/index.ts b/src/api/routes/chat-completions/index.ts index 38a589c..193efc3 100644 --- a/src/api/routes/chat-completions/index.ts +++ b/src/api/routes/chat-completions/index.ts @@ -168,9 +168,13 @@ async function handleChatRequest( accumulatedText += text; emitter?.emitContentDelta(text); }, - onToolCall(callId, tc) { - // Only CustomTool calls reach here (ServerTools filtered by loop) - emitter?.emitToolCallDelta(callId, tc.name, tc.arguments); + onClientToolCall(callId, tc) { + // Client tool calls - client must execute + emitter?.emitToolCallDelta(callId, tc.name, JSON.stringify(tc.arguments)); + }, + onServerToolResult(result) { + // Server tool results - emit tool_call + tool result + emitter?.emitServerToolExecution(result.callId, result.toolName, result.args, result.output); }, }); diff --git a/src/api/routes/responses/events.ts b/src/api/routes/responses/events.ts index 1ab627f..6e74621 100644 --- a/src/api/routes/responses/events.ts +++ b/src/api/routes/responses/events.ts @@ -172,6 +172,41 @@ export class ResponseEventEmitter { }); } + /** + * Emit events for a server-executed tool call and its result. + * Unlike client tool calls, these are emitted with status: 'completed' immediately + * and include the function_call_output with the result. + */ + emitServerToolExecution( + callId: string, + toolName: string, + args: string, + output: string, + outputIndex: number + ): { nextOutputIndex: number } { + const functionCallItem = { + type: 'function_call', + id: `fc-${randomUUID()}`, + call_id: callId, + status: 'completed', + name: toolName, + arguments: args, + }; + this.emitOutputItemAdded(functionCallItem, outputIndex); + this.emitOutputItemDone(functionCallItem, outputIndex); + + const outputItem = { + type: 'function_call_output', + id: `item-${randomUUID()}`, + call_id: callId, + output, + }; + this.emitOutputItemAdded(outputItem, outputIndex + 1); + this.emitOutputItemDone(outputItem, outputIndex + 1); + + return { nextOutputIndex: outputIndex + 2 }; + } + emitResponseCompleted(response: OpenAIResponse): void { this.emit({ type: 'response.completed', diff --git a/src/api/routes/responses/request-handlers.ts b/src/api/routes/responses/request-handlers.ts index c727f4e..ba5a494 100644 --- a/src/api/routes/responses/request-handlers.ts +++ b/src/api/routes/responses/request-handlers.ts @@ -180,10 +180,23 @@ export async function handleRequest( accumulatedText += text; emitter?.emitOutputTextDelta(itemId, 0, 0, text); }, - onToolCall(callId, tc) { - // Only CustomTool calls reach here (ServerTools filtered by loop) + onClientToolCall(callId, tc) { + // Client tool calls - client must execute emitter?.emitFunctionCallEvents(id, callId, tc.name, JSON.stringify(tc.arguments), nextOutputIndex++); }, + onServerToolResult(result) { + // Server tool results - emit completed function_call + function_call_output + if (emitter) { + const { nextOutputIndex: newIndex } = emitter.emitServerToolExecution( + result.callId, + result.toolName, + result.args, + result.output, + nextOutputIndex + ); + nextOutputIndex = newIndex; + } + }, }); logger.debug('[Server] Stream completed'); diff --git a/src/api/tools/server-tools/executor.ts b/src/api/tools/server-tools/executor.ts index 9f14c4b..90a21e1 100644 --- a/src/api/tools/server-tools/executor.ts +++ b/src/api/tools/server-tools/executor.ts @@ -20,6 +20,15 @@ export interface ServerToolExecutionResult { error?: string; } +/** Result from executing a server tool (for emission to clients) */ +export interface ServerToolResult { + callId: string; + toolName: string; + args: string; + output: string; + success: boolean; +} + /** * Execute a ServerTool by name. * @@ -76,27 +85,60 @@ export function partitionToolCalls(toolCalls: OpenAIToolCall[]): PartitionedTool return { serverToolCalls, clientToolCalls }; } +// ── Execution ────────────────────────────────────────────────────── + +/** + * Execute multiple ServerTools and return results. + * + * @param serverToolCalls - ServerTool calls to execute + * @param context - ServerTool execution context + * @returns Array of execution results + */ +export async function executeServerTools( + serverToolCalls: OpenAIToolCall[], + context: ServerToolContext +): Promise<ServerToolResult[]> { + const results: ServerToolResult[] = []; + + for (const tc of serverToolCalls) { + const args = JSON.parse(tc.function.arguments); + const execResult = await executeServerTool(tc.function.name, args, context); + + const output = execResult.error + ? `Error executing ${tc.function.name}: ${execResult.error}` + : execResult.result ?? 'No result'; + + results.push({ + callId: tc.id, + toolName: tc.function.name, + args: tc.function.arguments, + output, + success: !execResult.error, + }); + } + + return results; +} + // ── Continuation ────────────────────────────────────────────────────── /** - * Execute ServerTools and build continuation turns for the next Lumo call. + * Build continuation turns from execution results for the next Lumo call. * * Creates: * 1. An assistant turn with the iteration text (which includes tool call JSON) * 2. User turns with tool results for each executed ServerTool * - * @param serverToolCalls - ServerTool calls to execute * @param assistantText - Text from the current iteration (includes tool call JSON) - * @param context - ServerTool execution context + * @param results - Execution results from executeServerTools * @param prefix - CustomTools prefix for tool result formatting * @returns MessageForStore[] ready to append for next Lumo call */ -export async function buildServerToolContinuation( - serverToolCalls: OpenAIToolCall[], +export function buildContinuationTurns( assistantText: string, - context: ServerToolContext, + results: ServerToolResult[], prefix: string -): Promise<MessageForStore[]> { +): MessageForStore[] { const continuationTurns: MessageForStore[] = []; // Add assistant turn with the text (which includes tool call JSON) @@ -105,30 +147,21 @@ export async function buildServerToolContinuation( content: assistantText, }); - // Execute each ServerTool and add result as user turn - for (const tc of serverToolCalls) { - const args = JSON.parse(tc.function.arguments); - const execResult = await executeServerTool(tc.function.name, args, context); - - // Format result similar to CustomTool results - const resultContent = execResult.error - ? `Error executing ${tc.function.name}: ${execResult.error}` - : execResult.result ?? 'No result'; - - // Build user turn with tool result in JSON format (similar to function_call_output) + // Add user turn for each tool result + for (const result of results) { const toolResultJson = JSON.stringify({ type: 'function_call_output', - call_id: tc.id, - tool_name: `${prefix}${tc.function.name}`, - output: resultContent, + call_id: result.callId, + tool_name: `${prefix}${result.toolName}`, + output: result.output, }); continuationTurns.push({ role: Role.User, content: `\`\`\`json\n${toolResultJson}\n\`\`\``, - id: tc.id, + id: result.callId, }); } return continuationTurns; -} +} \ No newline at end of file diff --git a/src/api/tools/server-tools/handler.ts b/src/api/tools/server-tools/handler.ts index bb24688..c50b328 100644 --- a/src/api/tools/server-tools/handler.ts +++ b/src/api/tools/server-tools/handler.ts @@ -9,13 +9,13 @@ import { logger } from '../../../app/logger.js'; import { getCustomToolPrefix } from '../../../app/config.js'; import { createStreamingToolProcessor, type StreamingToolEmitter } from '../streaming-processor.js'; import { isServerTool, type ServerToolContext } from './registry.js'; -import { partitionToolCalls, buildServerToolContinuation } from './executor.js'; +import { partitionToolCalls, executeServerTools, buildContinuationTurns, type ServerToolResult } from './executor.js'; import type { ChatMessageWithTools, EndpointDependencies, OpenAIToolCall } from '../../types.js'; import type { RequestContext } from 'src/api/types.js'; import type { Turn, ChatResult } from '../../../lumo-client/types.js'; import type { ConversationId, MessageForStore } from '../../../conversations/types.js'; import type { ParsedToolCall } from '../types.js'; -import { convertToolMessage } from 'src/api/message-converter.js'; +import { convertToolMessage } from '../../message-converter.js'; // ── Types ───────────────────────────────────────────────────────────── @@ -28,8 +28,12 @@ export interface ChatAndExecuteOptions { injectInstructionsInto: 'first' | 'last'; /** Callback for text deltas during streaming */ onTextDelta: (text: string) => void; - /** Callback for tool calls (only CustomTools, ServerTools filtered out) */ - onToolCall: (callId: string, tc: ParsedToolCall) => void; + /** Callback for client tool calls (client must execute) */ + onClientToolCall: (callId: string, tc: ParsedToolCall) => void; + /** Callback for server tool calls (informational, server executes) */ + onServerToolCall?: (tc: OpenAIToolCall) => void; + /** Callback for server tool results (after execution) */ + onServerToolResult?: (result: ServerToolResult) => void; } export interface ChatAndExecuteResult { @@ -56,7 +60,7 @@ const MAX_SERVER_TOOL_LOOPS = 5; * 5. Returns final text and any CustomTool calls */ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<ChatAndExecuteResult> { - const { deps, context, instructions, injectInstructionsInto, onTextDelta, onToolCall } = options; + const { deps, context, instructions, injectInstructionsInto, onTextDelta, onClientToolCall } = options; const prefix = getCustomToolPrefix(); let currentTurns = [...options.turns]; @@ -88,7 +92,7 @@ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<Ch emitToolCall(callId, tc) { // Only emit CustomTool calls to the client if (!isServerTool(tc.name)) { - onToolCall(callId, tc); + onClientToolCall(callId, tc); } }, }; @@ -120,13 +124,21 @@ export async function chatAndExecute(options: ChatAndExecuteOptions): Promise<Ch logger.info({ loopCount, serverToolCount: serverToolCalls.length }, 'Executing ServerTools'); - // Execute ServerTools and build continuation turns - const continuationTurns = await buildServerToolContinuation( - serverToolCalls, - iterationText, - serverToolCtx, - prefix - ); + // Emit server tool calls before execution + for (const tc of serverToolCalls) { + options.onServerToolCall?.(tc); + } + + // Execute ServerTools and get results + const results = await executeServerTools(serverToolCalls, serverToolCtx); + + // Emit server tool results after execution + for (const result of results) { + options.onServerToolResult?.(result); + } + + // Build continuation turns for next Lumo call + const continuationTurns = buildContinuationTurns(iterationText, results, prefix); // Update turns for next iteration currentTurns = [...currentTurns, ...continuationTurns]; diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts index 5346312..6c45ac7 100644 --- a/src/api/tools/server-tools/index.ts +++ b/src/api/tools/server-tools/index.ts @@ -21,7 +21,13 @@ export { type ServerToolHandler, } from './registry.js'; -export { executeServerTool, type ServerToolExecutionResult } from './executor.js'; +export { + executeServerTool, + executeServerTools, + buildContinuationTurns, + type ServerToolExecutionResult, + type ServerToolResult, +} from './executor.js'; export { chatAndExecute, type ChatAndExecuteOptions, type ChatAndExecuteResult } from './handler.js'; diff --git a/tests/unit/shared.test.ts b/tests/unit/shared.test.ts index 0dcc2bb..a962505 100644 --- a/tests/unit/shared.test.ts +++ b/tests/unit/shared.test.ts @@ -108,7 +108,7 @@ describe('partitionToolCalls', () => { it('returns empty arrays when no tool calls', () => { const result = partitionToolCalls([]); expect(result.serverToolCalls).toEqual([]); - expect(result.customToolCalls).toEqual([]); + expect(result.clientToolCalls).toEqual([]); }); it('partitions tool calls into server and custom tools', () => { @@ -131,8 +131,8 @@ describe('partitionToolCalls', () => { expect(result.serverToolCalls).toHaveLength(1); expect(result.serverToolCalls[0].function.name).toBe('lumo_search'); - expect(result.customToolCalls).toHaveLength(2); - expect(result.customToolCalls.map(tc => tc.function.name)).toEqual(['custom_tool', 'another_custom']); + expect(result.clientToolCalls).toHaveLength(2); + expect(result.clientToolCalls.map(tc => tc.function.name)).toEqual(['custom_tool', 'another_custom']); }); it('returns all as custom when no server tools registered', () => { @@ -144,7 +144,7 @@ describe('partitionToolCalls', () => { const result = partitionToolCalls(toolCalls); expect(result.serverToolCalls).toEqual([]); - expect(result.customToolCalls).toHaveLength(2); + expect(result.clientToolCalls).toHaveLength(2); }); }); From a2ff39c11ec896b77a04f78ebc99f1cb0dc6ba5b Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:20:44 +0200 Subject: [PATCH 20/37] message deduplication: fix continuations after server tool calls; remove isValidContinuation() and unused detectBranching() --- docs/conversations.md | 1 - src/conversations/deduplication.ts | 92 +++-------------------- src/conversations/index.ts | 2 - src/conversations/minimal-store.ts | 12 --- src/conversations/store.ts | 15 ---- tests/unit/deduplication.test.ts | 116 +---------------------------- 6 files changed, 12 insertions(+), 226 deletions(-) diff --git a/docs/conversations.md b/docs/conversations.md index 645d94b..f3a9658 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -78,7 +78,6 @@ When authenticated via browser, sync is automatic via Redux sagas: - **semanticId**: Call ID for tool messages, hash(role+content) for regular messages - `findNewMessages()`: Compares incoming messages against stored messages -- `isValidContinuation()`: Validates no branching in conversation tree ### Manual save Call `/save [optional title]` to save stateless conversations. See [troubleshooting](#i-enabled-sync-but-my-chats-dont-appear-in-lumo). diff --git a/src/conversations/deduplication.ts b/src/conversations/deduplication.ts index d8d1d7e..ce68082 100644 --- a/src/conversations/deduplication.ts +++ b/src/conversations/deduplication.ts @@ -8,6 +8,9 @@ import { createHash } from 'crypto'; import { Role } from '@lumo/types.js'; import type { Message, MessageForStore } from './types.js'; +import { getMetrics } from 'src/app/metrics.js'; +import logger from 'src/app/logger.js'; +import { serverToolPrefix } from 'src/api/tools/server-tools/registry.js'; /** * Compute hash for a message (role + content) @@ -83,10 +86,15 @@ export function findNewMessages( // Compute semantic ID for incoming message const incomingSemanticId = incomingMsg.id ?? hashMessage(incomingMsg.role, incomingMsg.content ?? '').slice(0, 16); - if (storedIds.has(incomingSemanticId)) { + // semantic id is stored or assistant response differs after a server tool call + if (storedIds.has(incomingSemanticId) || stored[i + 1]?.semanticId?.startsWith(serverToolPrefix)) { matchedCount++; } else { // Divergence found - stop matching + getMetrics()?.invalidContinuationsTotal.inc(); + logger.warn({ + index: i, + }, 'Conversation message divergence'); break; } } @@ -94,85 +102,3 @@ export function findNewMessages( // Return messages after the matched prefix return incoming.slice(matchedCount); } - -/** - * Check if incoming messages are a valid continuation of stored conversation - * - * Valid if: - * - Incoming starts with the same messages as stored (prefix match by semantic ID) - * - Or incoming is entirely new (stored is empty) - */ -export function isValidContinuation( - incoming: MessageForStore[], - stored: Message[] -): { valid: boolean; reason?: string; debugInfo?: { storedMsg?: string; incomingMsg?: string } } { - if (stored.length === 0) { - return { valid: true }; - } - - if (incoming.length < stored.length) { - return { - valid: false, - reason: 'Incoming has fewer messages than stored - possible history truncation' - }; - } - - // Check if incoming starts with the same messages (using semantic IDs) - for (let i = 0; i < stored.length; i++) { - const storedSemanticId = stored[i].semanticId; - const incomingSemanticId = incoming[i].id ?? hashMessage(incoming[i].role, incoming[i].content ?? '').slice(0, 16); - - if (storedSemanticId !== incomingSemanticId) { - return { - valid: false, - reason: `Message mismatch at index ${i} - history may have been modified`, - debugInfo: { - storedMsg: `${stored[i].role}: ${stored[i].content ?? ''}`, - incomingMsg: `${incoming[i].role}: ${incoming[i].content ?? ''}`, - } - }; - } - } - - return { valid: true }; -} - -/** - * Detect if this is a branching request (user is continuing from a different point) - * - * Branching is detected when: - * - Incoming has some matching prefix with stored - * - But then diverges (different message at some index) - */ -export function detectBranching( - incoming: MessageForStore[], - stored: Message[] -): { isBranching: boolean; branchPoint?: number } { - if (stored.length === 0 || incoming.length === 0) { - return { isBranching: false }; - } - - // Find the point where they diverge - let divergePoint = -1; - const minLength = Math.min(stored.length, incoming.length); - - for (let i = 0; i < minLength; i++) { - const storedSemanticId = stored[i].semanticId; - const incomingSemanticId = incoming[i].id ?? hashMessage(incoming[i].role, incoming[i].content ?? '').slice(0, 16); - - if (storedSemanticId !== incomingSemanticId) { - divergePoint = i; - break; - } - } - - // If they diverge before the end of stored messages, it's a branch - if (divergePoint >= 0 && divergePoint < stored.length) { - return { - isBranching: true, - branchPoint: divergePoint - }; - } - - return { isBranching: false }; -} diff --git a/src/conversations/index.ts b/src/conversations/index.ts index f43f454..f3aec67 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -45,8 +45,6 @@ export { createFingerprint, fingerprintMessages, findNewMessages, - isValidContinuation, - detectBranching, } from './deduplication.js'; // Key management diff --git a/src/conversations/minimal-store.ts b/src/conversations/minimal-store.ts index 77d4630..2b71b2b 100644 --- a/src/conversations/minimal-store.ts +++ b/src/conversations/minimal-store.ts @@ -16,7 +16,6 @@ import type { Turn, AssistantMessageData } from '../lumo-client/types.js'; import { findNewMessages, hashMessage, - isValidContinuation, } from './deduplication.js'; import type { ConversationId, @@ -65,17 +64,6 @@ export class MinimalStore implements IConversationStore { appendMessages(id: ConversationId, incoming: MessageForStore[]): Message[] { const state = this.getOrCreate(id); - const validation = isValidContinuation(incoming, state.messages); - if (!validation.valid) { - getMetrics()?.invalidContinuationsTotal.inc(); - logger.warn({ - conversationId: id, - reason: validation.reason, - incomingCount: incoming.length, - storedCount: state.messages.length, - }, 'Invalid conversation continuation'); - } - const newMessages = findNewMessages(incoming, state.messages); if (newMessages.length === 0) { diff --git a/src/conversations/store.ts b/src/conversations/store.ts index add7a94..ee1829c 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -22,7 +22,6 @@ import type { AssistantMessageData, Turn } from '../lumo-client/types.js'; import { findNewMessages, hashMessage, - isValidContinuation, } from './deduplication.js'; import type { ConversationId, @@ -211,20 +210,6 @@ export class ConversationStore { let newMessages: MessageForStore[]; if(deduplicate){ - // Validate continuation - const validation = isValidContinuation(incoming, convState.messages); - if (!validation.valid) { - getMetrics()?.invalidContinuationsTotal.inc(); - logger.warn({ - conversationId: id, - reason: validation.reason, - incomingCount: incoming.length, - storedCount: convState.messages.length, - ...validation.debugInfo, - }, 'Invalid conversation continuation'); - } - - // Find new messages newMessages = findNewMessages(incoming, convState.messages); } else{ diff --git a/tests/unit/deduplication.test.ts b/tests/unit/deduplication.test.ts index 5537fd8..bd6ec49 100644 --- a/tests/unit/deduplication.test.ts +++ b/tests/unit/deduplication.test.ts @@ -6,18 +6,15 @@ import { describe, it, expect } from 'vitest'; import { hashMessage, findNewMessages, - isValidContinuation, - detectBranching, - type MessageForStore, } from '../../src/conversations/deduplication.js'; -import type { Message } from '../../src/conversations/types.js'; +import type { Message, MessageForStore } from '../../src/conversations/types.js'; function createStoredMessage( role: string, content: string, index: number, semanticId?: string, -): Message { +): MessageForStore { return { id: `msg-${index}`, conversationId: 'conv-1', @@ -109,60 +106,7 @@ describe('findNewMessages', () => { }); }); -describe('isValidContinuation', () => { - it('should be valid when stored is empty', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello' }, - ]; - - const result = isValidContinuation(incoming, []); - expect(result.valid).toBe(true); - }); - - it('should be valid when incoming continues stored', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi!' }, - { role: 'user', content: 'New message' }, - ]; - const stored: Message[] = [ - createStoredMessage('user', 'Hello', 0), - createStoredMessage('assistant', 'Hi!', 1), - ]; - - const result = isValidContinuation(incoming, stored); - expect(result.valid).toBe(true); - }); - - it('should be invalid when incoming has fewer messages', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello' }, - ]; - const stored: Message[] = [ - createStoredMessage('user', 'Hello', 0), - createStoredMessage('assistant', 'Hi!', 1), - ]; - - const result = isValidContinuation(incoming, stored); - expect(result.valid).toBe(false); - expect(result.reason).toContain('fewer messages'); - }); - - it('should be invalid when history is modified', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello MODIFIED' }, - { role: 'assistant', content: 'Hi!' }, - ]; - const stored: Message[] = [ - createStoredMessage('user', 'Hello', 0), - createStoredMessage('assistant', 'Hi!', 1), - ]; - const result = isValidContinuation(incoming, stored); - expect(result.valid).toBe(false); - expect(result.reason).toContain('mismatch'); - }); -}); describe('ID-based deduplication', () => { it('should deduplicate tool messages by call_id even when content changes', () => { @@ -227,58 +171,4 @@ describe('ID-based deduplication', () => { expect(result[0].content).toBe('New message'); }); - it('should validate continuation with ID-based matching', () => { - const callId = 'tool-call-789'; - - const stored: Message[] = [ - createStoredMessage('user', 'Original content', 0, callId), - ]; - - // Different content but same ID - should be valid - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Modified content', id: callId }, - { role: 'user', content: 'New message' }, - ]; - - const result = isValidContinuation(incoming, stored); - expect(result.valid).toBe(true); - }); -}); - -describe('detectBranching', () => { - it('should not detect branching when empty', () => { - const result = detectBranching([], []); - expect(result.isBranching).toBe(false); - }); - - it('should not detect branching for simple continuation', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi!' }, - { role: 'user', content: 'New' }, - ]; - const stored: Message[] = [ - createStoredMessage('user', 'Hello', 0), - createStoredMessage('assistant', 'Hi!', 1), - ]; - - const result = detectBranching(incoming, stored); - expect(result.isBranching).toBe(false); - }); - - it('should detect branching when history diverges', () => { - const incoming: MessageForStore[] = [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Different response' }, - ]; - const stored: Message[] = [ - createStoredMessage('user', 'Hello', 0), - createStoredMessage('assistant', 'Original response', 1), - createStoredMessage('user', 'Follow up', 2), - ]; - - const result = detectBranching(incoming, stored); - expect(result.isBranching).toBe(true); - expect(result.branchPoint).toBe(1); - }); -}); +}); \ No newline at end of file From 1ad6660778cb3ac38a5be628bd8f8f380e52eaed Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:23:20 +0200 Subject: [PATCH 21/37] mock mode: install mock fetch adapter to intercept store api calls --- src/app/index.ts | 7 +++++++ src/mock/custom-scenarios.ts | 2 +- src/shims/fetch-adapter.ts | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/app/index.ts b/src/app/index.ts index 52c0c3d..69e6739 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -43,6 +43,13 @@ export class Application { * Initialize mock mode - bypass auth, use simulated API responses */ private async initializeMock(): Promise<void> { + // Install mock fetch adapter BEFORE store init (sagas make API calls) + const { installMockFetchAdapter } = await import('../shims/fetch-adapter.js'); + this.cleanupFetchAdapter = installMockFetchAdapter(); + + // Suppress API errors in logs (same as local-only mode) + suppressFullApiErrors(); + // Use primary store with fake-indexeddb for mock mode const { initializeMockStore } = await import('../mock/mock-store.js'); const result = await initializeMockStore(); diff --git a/src/mock/custom-scenarios.ts b/src/mock/custom-scenarios.ts index 14bfbf9..e65279b 100644 --- a/src/mock/custom-scenarios.ts +++ b/src/mock/custom-scenarios.ts @@ -150,7 +150,7 @@ export const customScenarios: Record<string, ScenarioGenerator> = { // Include the prefix so the tool call is detected const prefix = getCustomToolPrefix() + serverToolPrefix; - const toolName = `${prefix}search`; + const toolName = `${prefix}get_date`; const json = `\`\`\`json\n{"name":"${toolName}","arguments":{"query":"weather forecast"}}\n\`\`\``; const tokens = json.split(''); for (let i = 0; i < tokens.length; i++) { diff --git a/src/shims/fetch-adapter.ts b/src/shims/fetch-adapter.ts index 59ccbfb..52f07e7 100644 --- a/src/shims/fetch-adapter.ts +++ b/src/shims/fetch-adapter.ts @@ -139,3 +139,38 @@ export function installFetchAdapter( globalThis.fetch = originalFetch; }; } + +/** + * Installs a mock fetch adapter for test/mock mode. + * + * Returns 418 for all /api/lumo/v1/ calls, same as local-only mode. + * Does not require authentication - for use before store initialization. + * + * @returns A cleanup function to restore the original fetch + */ +export function installMockFetchAdapter(): () => void { + globalThis.fetch = async function mockFetch( + input: RequestInfo | URL, + init?: RequestInit + ): Promise<Response> { + const url = typeof input === 'string' ? input : input.toString(); + + // Intercept /api/lumo/v1/ calls - return 418 like local-only mode + if (url.startsWith('/api/lumo/v1/')) { + return new Response(JSON.stringify({ + Code: 418, + Error: 'Mock mode - API calls disabled', + }), { + status: 418, + statusText: "I'm a teapot", + headers: { 'content-type': 'application/json' }, + }); + } + + return originalFetch(input, init); + }; + + return () => { + globalThis.fetch = originalFetch; + }; +} From d9d5c6f87dcef451b93e025ad34a9b8548d56e09 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 00:13:06 +0200 Subject: [PATCH 22/37] =?UTF-8?q?add=20metrics=20for=20server=20tools=20ca?= =?UTF-8?q?lls;=20rename=20metric=20for=20custom-completed=20=E2=86=92=20c?= =?UTF-8?q?lient-completed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/api/tools/call-id.ts | 2 +- src/api/tools/server-tools/executor.ts | 3 +++ tests/unit/metrics.test.ts | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/api/tools/call-id.ts b/src/api/tools/call-id.ts index 855f0ba..aa6db58 100644 --- a/src/api/tools/call-id.ts +++ b/src/api/tools/call-id.ts @@ -52,7 +52,7 @@ export function trackCustomToolCompletion(callId: string): void { logger.info({ toolName, call_id: callId }, 'Custom tool call completed'); getMetrics()?.toolCallsTotal.inc({ - type: 'custom', + type: 'client', status: 'completed', tool_name: toolName, }); diff --git a/src/api/tools/server-tools/executor.ts b/src/api/tools/server-tools/executor.ts index 90a21e1..50fece0 100644 --- a/src/api/tools/server-tools/executor.ts +++ b/src/api/tools/server-tools/executor.ts @@ -6,6 +6,7 @@ */ import { logger } from '../../../app/logger.js'; +import { getMetrics } from '../../../app/metrics.js'; import { getServerTool, isServerTool, type ServerToolContext } from './registry.js'; import type { OpenAIToolCall } from '../../types.js'; import { Role } from '../../../lumo-client/types.js'; @@ -51,10 +52,12 @@ export async function executeServerTool( logger.info({ tool: toolName, args }, 'Executing ServerTool'); const result = await tool.handler(args, context); logger.debug({ tool: toolName, resultLength: result.length }, 'ServerTool completed'); + getMetrics()?.toolCallsTotal.inc({ type: 'server', status: 'success', tool_name: toolName }); return { isServerTool: true, result }; } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; logger.error({ error, tool: toolName }, 'ServerTool execution failed'); + getMetrics()?.toolCallsTotal.inc({ type: 'server', status: 'failed', tool_name: toolName }); return { isServerTool: true, error: errorMessage }; } } diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts index a8279ee..acb19bf 100644 --- a/tests/unit/metrics.test.ts +++ b/tests/unit/metrics.test.ts @@ -98,23 +98,30 @@ describe('MetricsService', () => { metrics.toolCallsTotal.inc({ type: 'native', status: 'detected', tool_name: 'web_search' }); metrics.toolCallsTotal.inc({ type: 'native', status: 'detected', tool_name: 'proton_info' }); // Custom tools: completed (tracked on function_call_output) - metrics.toolCallsTotal.inc({ type: 'custom', status: 'completed', tool_name: 'my_tool' }); + metrics.toolCallsTotal.inc({ type: 'client', status: 'completed', tool_name: 'my_tool' }); // Custom tools: invalid (malformed JSON) metrics.toolCallsTotal.inc({ type: 'custom', status: 'invalid', tool_name: 'unknown' }); // Custom tools: misrouted (incorrectly routed through native pipeline) metrics.toolCallsTotal.inc({ type: 'custom', status: 'misrouted', tool_name: 'computer' }); + metrics.toolCallsTotal.inc({ type: 'server', status: 'success', tool_name: 'computer' }); + metrics.toolCallsTotal.inc({ type: 'server', status: 'failed', tool_name: 'computer' }); const output = await metrics.getMetrics(); expect(output).toContain('test_tool_calls_total'); expect(output).toContain('type="native"'); expect(output).toContain('type="custom"'); + expect(output).toContain('type="client"'); + expect(output).toContain('type="server"'); expect(output).toContain('status="detected"'); expect(output).toContain('status="completed"'); expect(output).toContain('status="invalid"'); expect(output).toContain('status="misrouted"'); + expect(output).toContain('status="success"'); + expect(output).toContain('status="failed"'); expect(output).toContain('tool_name="web_search"'); expect(output).toContain('tool_name="proton_info"'); expect(output).toContain('tool_name="my_tool"'); + expect(output).toContain('tool_name="current_time"'); }); }); From aeb27dca4c045739068b2642f59018ae0a1d49b9 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 00:19:36 +0200 Subject: [PATCH 23/37] fix alwas registering server tools --- src/api/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server.ts b/src/api/server.ts index 36f4d21..6158a44 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -73,7 +73,7 @@ export class APIServer { async start(): Promise<void> { // Initialize ServerTools if enabled - if (this.serverConfig.tools.server) { + if (this.serverConfig.tools.server.enabled) { const { initializeServerTools } = await import('./tools/server-tools/index.js'); initializeServerTools(); logger.info('ServerTools initialized'); From 2e0034c1b39e8ebcf9bd6d2fd531a63a7b16be82 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 01:18:00 +0200 Subject: [PATCH 24/37] git rid of MinimalStore and IConversationStore B-) --- docs/conversations.md | 52 +---- src/api/routes/chat-completions/index.ts | 2 + src/api/routes/responses/index.ts | 2 + src/api/tools/server-tools/registry.ts | 4 +- src/api/types.ts | 4 +- src/app/index.ts | 4 +- src/cli/client.ts | 38 ++-- src/conversations/index.ts | 17 +- src/conversations/minimal-store.ts | 244 ----------------------- src/conversations/search.ts | 4 +- src/conversations/store-interface.ts | 46 ----- tests/e2e/cli-smoke.test.ts | 4 +- tests/helpers/test-server.ts | 6 +- tests/integration/metrics.test.ts | 4 +- tests/unit/minimal-store.test.ts | 188 ----------------- tests/unit/search.test.ts | 18 +- 16 files changed, 43 insertions(+), 594 deletions(-) delete mode 100644 src/conversations/minimal-store.ts delete mode 100644 src/conversations/store-interface.ts delete mode 100644 tests/unit/minimal-store.test.ts diff --git a/docs/conversations.md b/docs/conversations.md index f3a9658..c4b7202 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -5,14 +5,7 @@ This document is for developers working on conversation persistence. ## Overview -lumo-tamer supports two conversation stores: -- **ConversationStore**: encrypted offline persistence and full sync reusing WebClient's code -- **FallbackStore**: in-memory, optional one-way sync - -ConversationStore is the way forward, but still new. It will allow future lumo-tamer versions to make Lumo remember and search past converations. -However, FallbackStore is the default for now (`useFallbackStore: true`) because: -- ConversationStore needs more testing (general performance and performance with `login`and `rclone` authentications) -- Persistence is not required for the core functionality of chatting with Lumo +lumo-tamer uses **ConversationStore** for encrypted offline persistence and server sync, reusing Proton's WebClient code. When the store is unavailable (missing encryption keys), the API operates stateless and the CLI uses a local turn array. To sync conversations with other Lumo instances (web- or mobile apps), **browser authentication** is required. @@ -22,7 +15,6 @@ To sync conversations with other Lumo instances (web- or mobile apps), **browser ```yaml conversations: - useFallbackStore: true # true = fallback, false = ConversationStore (default: true) enableSync: false # Enable server sync (requires browser auth) projectName: lumo-tamer # Project name (created if doesn't exist) deriveIdFromUser: false # For stateless clients (Home Assistant) @@ -95,44 +87,6 @@ Call `/save [optional title]` to save stateless conversations. See [troubleshoot | Redux sagas | [packages/lumo/src/redux/sagas/](../packages/lumo/src/redux/sagas/) | Async sync operations (push/pull) | | IndexedDB layer | [packages/lumo/src/indexedDb/db.ts](../packages/lumo/src/indexedDb/db.ts) | DbApi for local SQLite storage | ---- - -## FallbackStore - -Legacy in-memory cache for environments without full persistence support. - -### Architecture - -``` -FallbackStore (in-memory LRU) - → SyncService (manual sync to server) - → SpaceManager (space lifecycle) - → EncryptionCodec (AEAD encryption) - → AutoSyncService -``` - -### Auto-Sync - -When authenticated via browser and `enableSync: true`: - -1. `FallbackStore.markDirtyById()` notifies `AutoSyncService` -2. **Debounce**: Waits 5s for activity to settle -3. **Throttle**: Respects 30s minimum interval -4. **Max delay**: Forces sync after 60s -5. Auto syncs on exit. - -### Key Components - -| Component | Location | Purpose | -|-----------|----------|---------| -| FallbackStore | [src/conversations/fallback/store.ts](../src/conversations/fallback/store.ts) | In-memory Map with LRU eviction | -| SyncService | [src/conversations/fallback/sync/sync-service.ts](../src/conversations/fallback/sync/sync-service.ts) | Orchestrates server sync | -| SpaceManager | [src/conversations/fallback/sync/space-manager.ts](../src/conversations/fallback/sync/space-manager.ts) | Space lifecycle and key management | -| EncryptionCodec | [src/conversations/fallback/sync/encryption-codec.ts](../src/conversations/fallback/sync/encryption-codec.ts) | AEAD encryption/decryption | -| AutoSyncService | [src/conversations/fallback/sync/auto-sync.ts](../src/conversations/fallback/sync/auto-sync.ts) | Debounced/throttled sync | - - - --- ## Known Limitations @@ -148,9 +102,9 @@ Proton's backend enforces a per-project conversation limit. Deleted conversation ## Troubleshooting -### "I set useFallbackStore: false but it's still using FallbackStore" +### "ConversationStore disabled" warning -**Solution:** ConversationStore requires cached encryption keys. Re-authenticate to save/generate them. +**Cause:** ConversationStore requires cached encryption keys. Re-authenticate to save/generate them. ### "I enabled sync but my chats don't appear in Lumo" diff --git a/src/api/routes/chat-completions/index.ts b/src/api/routes/chat-completions/index.ts index 193efc3..e85f06d 100644 --- a/src/api/routes/chat-completions/index.ts +++ b/src/api/routes/chat-completions/index.ts @@ -108,6 +108,8 @@ export function createChatCompletionsRouter(deps: EndpointDependencies): Router if (conversationId && deps.conversationStore && turns.length > 0) { deps.conversationStore.appendMessages(conversationId, turns); logger.debug({ conversationId, messageCount: turns.length }, 'Persisted conversation messages'); + } else if (conversationId && !deps.conversationStore) { + logger.warn({ conversationId }, 'Stateful request but no conversation store available'); } else if (!conversationId) { // Stateless request - track +1 user message (not deduplicated) getMetrics()?.messagesTotal.inc({ role: 'user' }); diff --git a/src/api/routes/responses/index.ts b/src/api/routes/responses/index.ts index d5814f6..310da95 100644 --- a/src/api/routes/responses/index.ts +++ b/src/api/routes/responses/index.ts @@ -118,6 +118,8 @@ export function createResponsesRouter(deps: EndpointDependencies): Router { if (conversationId && deps.conversationStore && turns.length > 0) { deps.conversationStore.appendMessages(conversationId, turns); logger.debug({ conversationId, messageCount: turns.length }, 'Persisted conversation messages'); + } else if (conversationId && !deps.conversationStore) { + logger.warn({ conversationId }, 'Stateful request but no conversation store available'); } else if (!conversationId) { // Stateless request - track +1 user message (not deduplicated) getMetrics()?.messagesTotal.inc({ role: 'user' }); diff --git a/src/api/tools/server-tools/registry.ts b/src/api/tools/server-tools/registry.ts index d4d4eae..ab5a102 100644 --- a/src/api/tools/server-tools/registry.ts +++ b/src/api/tools/server-tools/registry.ts @@ -6,14 +6,14 @@ */ import { OpenAITool } from 'src/api/types.js'; -import type { IConversationStore } from '../../../conversations/index.js'; +import type { ConversationStore } from '../../../conversations/index.js'; import type { ConversationId } from '../../../conversations/types.js'; // ── Types ───────────────────────────────────────────────────────────── /** Context passed to ServerTool handlers. */ export interface ServerToolContext { - conversationStore?: IConversationStore; + conversationStore?: ConversationStore; conversationId?: ConversationId; } diff --git a/src/api/types.ts b/src/api/types.ts index 1ba3075..ae248f5 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -1,13 +1,13 @@ import { RequestQueue } from './queue.js'; import { LumoClient } from '../lumo-client/index.js'; -import type { IConversationStore } from '../conversations/index.js'; +import type { ConversationStore } from '../conversations/index.js'; import type { AuthManager } from '../auth/index.js'; import type { CommandContext } from 'src/app/commands.js'; export interface EndpointDependencies { queue: RequestQueue; lumoClient: LumoClient; - conversationStore?: IConversationStore; + conversationStore?: ConversationStore; syncInitialized?: boolean; authManager?: AuthManager; vaultPath?: string; diff --git a/src/app/index.ts b/src/app/index.ts index 69e6739..743fbe8 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -10,7 +10,7 @@ import { logger } from './logger.js'; import { resolveProjectPath } from './paths.js'; import { LumoClient } from '../lumo-client/index.js'; import { createAuthProvider, AuthManager, type AuthProvider, type ProtonApi } from '../auth/index.js'; -import { getConversationStore, setConversationStore, initializeSync, initializeConversationStore, type IConversationStore } from '../conversations/index.js'; +import { getConversationStore, setConversationStore, initializeSync, initializeConversationStore, type ConversationStore } from '../conversations/index.js'; import { createMockProtonApi } from '../mock/mock-api.js'; import { installFetchAdapter } from '../shims/fetch-adapter.js'; import { suppressFullApiErrors } from '../shims/console.js'; @@ -132,7 +132,7 @@ export class Application { return this.lumoClient; } - getConversationStore(): IConversationStore | undefined { + getConversationStore(): ConversationStore | undefined { return getConversationStore(); } diff --git a/src/cli/client.ts b/src/cli/client.ts index 74950da..cdd4167 100644 --- a/src/cli/client.ts +++ b/src/cli/client.ts @@ -15,11 +15,11 @@ import { BUSY_INDICATOR, clearBusyIndicator, print } from '../app/terminal.js'; import type { Application } from '../app/index.js'; import { randomUUID } from 'crypto'; import * as readline from 'readline'; -import type { AssistantMessageData } from '../lumo-client/index.js'; +import type { AssistantMessageData, Turn } from '../lumo-client/index.js'; +import type { ConversationStore } from '../conversations/index.js'; import { blockHandlers, executeBlocks, formatResultsMessage } from './local-actions/block-handlers.js'; import { CodeBlockDetector, type CodeBlock } from './local-actions/code-block-detector.js'; import { buildCliInstructions } from './message-converter.js'; -import { MinimalStore, type IConversationStore } from '../conversations/index.js'; interface LumoResponse { /** Assistant message data ready for persistence */ @@ -29,13 +29,13 @@ interface LumoResponse { } export class CLIClient { - private conversationId: string; - private store: IConversationStore; + private conversationId = randomUUID(); + private turns: Turn[] = []; + private store?: ConversationStore; constructor(private app: Application) { - this.conversationId = randomUUID(); - // Use app store if available, otherwise create minimal in-memory store - this.store = app.getConversationStore() ?? new MinimalStore(); + this.store = app.getConversationStore(); + } async run(): Promise<void> { @@ -64,7 +64,7 @@ export class CLIClient { print('Lumo: ' + BUSY_INDICATOR, false); - const turns = this.store.toTurns(this.conversationId); + const turns = this.store?.toTurns(this.conversationId) ?? this.turns; const instructions = buildCliInstructions(); const { injectInto } = getCliInstructionsConfig(); const result = await this.app.getLumoClient().chatWithHistory( @@ -94,9 +94,8 @@ export class CLIClient { } print('\n'); - // Handle title (already processed by LumoClient) if (result.title) { - this.store.setTitle(this.conversationId, result.title); + this.store?.setTitle(this.conversationId, result.title); } return { @@ -217,15 +216,12 @@ export class CLIClient { } try { - // Append user message and get response - this.store.appendUserMessage(this.conversationId, input); - - // Request title for new conversations (first message) - const existingConv = this.store.get(this.conversationId); - const requestTitle = existingConv?.title === 'New Conversation'; + this.turns.push({ role: 'user', content: input }); + this.store?.appendUserMessage(this.conversationId, input); - let lumoResponse = await this.sendToLumo({ requestTitle }); - this.store.appendAssistantResponse(this.conversationId, lumoResponse.message); + let lumoResponse = await this.sendToLumo(); + this.turns.push({ role: 'assistant', content: lumoResponse.message.content }); + this.store?.appendAssistantResponse(this.conversationId, lumoResponse.message); // Execute blocks until none remain (or user skips all) while (lumoResponse.blocks.length > 0) { @@ -236,10 +232,12 @@ export class CLIClient { // Send batch results back to Lumo print('─── Sending results to Lumo ───\n'); const batchMessage = formatResultsMessage(results); - this.store.appendUserMessage(this.conversationId, batchMessage); + this.turns.push({ role: 'user', content: batchMessage }); + this.store?.appendUserMessage(this.conversationId, batchMessage); lumoResponse = await this.sendToLumo(); - this.store.appendAssistantResponse(this.conversationId, lumoResponse.message); + this.turns.push({ role: 'assistant', content: lumoResponse.message.content }); + this.store?.appendAssistantResponse(this.conversationId, lumoResponse.message); } } catch (error) { clearBusyIndicator(); diff --git a/src/conversations/index.ts b/src/conversations/index.ts index f3aec67..54aac77 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -3,7 +3,6 @@ * * Provides: * - ConversationStore: Primary storage using Redux + IndexedDB - * - MinimalStore: Lightweight in-memory storage for CLI/tests * - Message deduplication for OpenAI API format * - Types compatible with Proton Lumo webclient */ @@ -23,15 +22,9 @@ export type { MessageForStore, } from './types.js'; -// Store interface -export type { IConversationStore } from './store-interface.js'; - // Primary store export { ConversationStore } from './store.js'; -// Minimal store (fallback for CLI/tests) -export { MinimalStore } from './minimal-store.js'; - // Store initialization export { initializeStore, @@ -68,7 +61,7 @@ import type { AuthProvider, ProtonApi } from '../auth/index.js'; import type { ConversationsConfig } from '../app/config.js'; import { getKeyManager } from './key-manager.js'; import { initializeStore, pullIncompleteConversations, type StoreResult } from './init.js'; -import type { IConversationStore } from './store-interface.js'; +import { ConversationStore } from './store.js'; // ============================================================================ // Conversation Store Initialization @@ -92,14 +85,14 @@ export interface InitializeStoreResult { let primaryStoreResult: StoreResult | null = null; // Singleton for the active store -let activeStore: IConversationStore | null = null; +let activeStore: ConversationStore | null = null; /** * Initialize the conversation store * * Creates the primary ConversationStore (Redux + IndexedDB) if possible. * Returns undefined if initialization fails - callers should handle this - * gracefully (server works stateless, CLI creates MinimalStore). + * gracefully (server works stateless, CLI uses local Turn array). * * Primary store requires: * - Auth provider supports persistence (has cached encryption keys) @@ -193,14 +186,14 @@ async function initializePrimaryStore( * Returns the initialized store, or undefined if no store is available. * Callers should handle undefined gracefully (stateless mode). */ -export function getConversationStore(): IConversationStore | undefined { +export function getConversationStore(): ConversationStore | undefined { return activeStore ?? undefined; } /** * Set the active conversation store (for mock mode or CLI fallback) */ -export function setConversationStore(store: IConversationStore): void { +export function setConversationStore(store: ConversationStore): void { activeStore = store; } diff --git a/src/conversations/minimal-store.ts b/src/conversations/minimal-store.ts deleted file mode 100644 index 2b71b2b..0000000 --- a/src/conversations/minimal-store.ts +++ /dev/null @@ -1,244 +0,0 @@ -/** - * MinimalStore - Lightweight in-memory store for single-session use - * - * Used when primary ConversationStore cannot be initialized: - * - CLI mode: tracks one conversation per session - * - Tests: provides mock store without IndexedDB overhead - * - * Only stores messages and converts them to turns for Lumo. - * Many methods are no-ops since CLI doesn't need full store functionality. - */ - -import { randomUUID } from 'crypto'; -import { logger } from '../app/logger.js'; -import { Role, ConversationStatus } from '@lumo/types.js'; -import type { Turn, AssistantMessageData } from '../lumo-client/types.js'; -import { - findNewMessages, - hashMessage, -} from './deduplication.js'; -import type { - ConversationId, - ConversationState, - Message, - MessageId, - MessageForStore, - SpaceId, -} from './types.js'; -import type { IConversationStore } from './store-interface.js'; -import { getMetrics } from '../app/metrics.js'; - -/** - * Lightweight in-memory conversation store - */ -export class MinimalStore implements IConversationStore { - private conversations = new Map<ConversationId, ConversationState>(); - private defaultSpaceId: SpaceId; - - constructor() { - this.defaultSpaceId = randomUUID(); - logger.debug({ spaceId: this.defaultSpaceId }, 'MinimalStore initialized'); - } - - getOrCreate(id: ConversationId): ConversationState { - let state = this.conversations.get(id); - - if (!state) { - state = this.createEmptyState(id); - this.conversations.set(id, state); - getMetrics()?.conversationsCreatedTotal.inc(); - logger.debug({ conversationId: id }, 'Created new conversation'); - } - - return state; - } - - get(id: ConversationId): ConversationState | undefined { - return this.conversations.get(id); - } - - has(id: ConversationId): boolean { - return this.conversations.has(id); - } - - appendMessages(id: ConversationId, incoming: MessageForStore[]): Message[] { - const state = this.getOrCreate(id); - - const newMessages = findNewMessages(incoming, state.messages); - - if (newMessages.length === 0) { - return []; - } - - const now = new Date().toISOString(); - const lastMessageId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const addedMessages: Message[] = []; - let parentId = lastMessageId; - - for (const msg of newMessages) { - const semanticId = msg.id ?? hashMessage(msg.role, msg.content ?? '').slice(0, 16); - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now, - role: msg.role, - parentId, - status: 'succeeded', - content: msg.content, - semanticId, - }; - - state.messages.push(message); - addedMessages.push(message); - parentId = message.id; - } - - state.metadata.updatedAt = new Date().toISOString(); - - const metrics = getMetrics(); - if (metrics) { - for (const msg of addedMessages) { - metrics.messagesTotal.inc({ role: msg.role }); - } - } - - return addedMessages; - } - - appendAssistantResponse( - id: ConversationId, - messageData: AssistantMessageData, - status: 'succeeded' | 'failed' = 'succeeded', - semanticId?: string - ): Message { - const state = this.getOrCreate(id); - const now = new Date(); - - const parentId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now.toISOString(), - role: Role.Assistant, - parentId, - status, - content: messageData.content, - blocks: messageData.blocks, - semanticId: semanticId ?? hashMessage(Role.Assistant, messageData.content).slice(0, 16), - }; - - state.messages.push(message); - state.metadata.updatedAt = now.toISOString(); - state.status = ConversationStatus.COMPLETED; - - getMetrics()?.messagesTotal.inc({ role: Role.Assistant }); - - return message; - } - - appendUserMessage(id: ConversationId, content: string): Message { - const state = this.getOrCreate(id); - const now = new Date(); - - const parentId = state.messages.length > 0 - ? state.messages[state.messages.length - 1].id - : undefined; - - const message: Message = { - id: randomUUID(), - conversationId: id, - createdAt: now.toISOString(), - role: Role.User, - parentId, - status: 'succeeded', - content, - semanticId: hashMessage(Role.User, content).slice(0, 16), - }; - - state.messages.push(message); - state.metadata.updatedAt = now.toISOString(); - - return message; - } - - toTurns(id: ConversationId): Turn[] { - return this.getMessages(id).map(({ role, content }) => ({ - role, - content, - })); - } - - getMessages(id: ConversationId): Message[] { - const state = this.conversations.get(id); - return state?.messages ?? []; - } - - // No-op methods (keep interface compatibility but not needed for CLI) - - setTitle(_id: ConversationId, _title: string): void { - // No-op: CLI never reads title back - } - - createFromTurns( - _turns: Turn[], - _title?: string - ): { conversationId: ConversationId; title: string } { - // No-op: requires sync which MinimalStore doesn't support - return { conversationId: '', title: '' }; - } - - appendAssistantToolCalls( - id: ConversationId, - toolCalls: Array<{ name: string; arguments: string; call_id: string }> - ): void { - for (const tc of toolCalls) { - const content = JSON.stringify({ - type: 'function_call', - call_id: tc.call_id, - name: tc.name, - arguments: tc.arguments, - }); - this.appendAssistantResponse(id, { content }, 'succeeded', tc.call_id); - } - } - - getMessage(_conversationId: ConversationId, _messageId: MessageId): Message | undefined { - // No-op - return undefined; - } - - delete(_id: ConversationId): boolean { - // No-op - return false; - } - - *entries(): IterableIterator<[ConversationId, ConversationState]> { - // No-op: yield nothing - } - - // Private methods - - private createEmptyState(id: ConversationId): ConversationState { - const now = new Date().toISOString(); - return { - metadata: { - id, - spaceId: this.defaultSpaceId, - createdAt: now, - updatedAt: now, - starred: false, - }, - title: 'New Conversation', - status: ConversationStatus.COMPLETED, - messages: [], - dirty: false, - }; - } -} diff --git a/src/conversations/search.ts b/src/conversations/search.ts index dcc18e8..5ab82a1 100644 --- a/src/conversations/search.ts +++ b/src/conversations/search.ts @@ -5,7 +5,7 @@ * Snippet extraction inspired by WebClients searchService.ts. */ -import type { IConversationStore } from './store-interface.js'; +import type { ConversationStore } from './store.js'; import type { ConversationId, Message } from './types.js'; export interface SearchResult { @@ -26,7 +26,7 @@ export interface SearchResult { * @returns Array of matching conversations, sorted by most recent first */ export function searchConversations( - store: IConversationStore, + store: Pick<ConversationStore, 'entries'>, query: string, limit = 20, excludeId?: ConversationId diff --git a/src/conversations/store-interface.ts b/src/conversations/store-interface.ts deleted file mode 100644 index 730936d..0000000 --- a/src/conversations/store-interface.ts +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Common interface for conversation stores - * - * Implemented by: - * - ConversationStore (Redux + IndexedDB) - primary, persistent - * - MinimalStore (in-memory) - fallback for CLI/tests - */ - -import type { Turn, AssistantMessageData } from '../lumo-client/types.js'; -import type { - ConversationId, - ConversationState, - Message, - MessageId, - MessageForStore, -} from './types.js'; - -export interface IConversationStore { - // Core CRUD - get(id: ConversationId): ConversationState | undefined; - getOrCreate(id: ConversationId): ConversationState; - has(id: ConversationId): boolean; - delete(id: ConversationId): boolean; - entries(): IterableIterator<[ConversationId, ConversationState]>; - - // Message operations - appendMessages(id: ConversationId, incoming: MessageForStore[], deduplicate?: boolean): Message[]; - appendAssistantResponse( - id: ConversationId, - messageData: AssistantMessageData, - status?: 'succeeded' | 'failed', - semanticId?: string - ): Message; - appendUserMessage(id: ConversationId, content: string): Message; - appendAssistantToolCalls( - id: ConversationId, - toolCalls: Array<{ name: string; arguments: string; call_id: string }> - ): void; - getMessages(id: ConversationId): Message[]; - getMessage(conversationId: ConversationId, messageId: MessageId): Message | undefined; - - // Conversation metadata - setTitle(id: ConversationId, title: string): void; - toTurns(id: ConversationId): Turn[]; - createFromTurns(turns: Turn[], title?: string): { conversationId: ConversationId; title: string }; -} diff --git a/tests/e2e/cli-smoke.test.ts b/tests/e2e/cli-smoke.test.ts index 765fe71..d0d8c69 100644 --- a/tests/e2e/cli-smoke.test.ts +++ b/tests/e2e/cli-smoke.test.ts @@ -8,7 +8,6 @@ import { describe, it, expect, vi, beforeAll, afterAll } from 'vitest'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { MinimalStore } from '../../src/conversations/index.js'; import type { Application } from '../../src/app/index.js'; describe('CLI single-query mode', () => { @@ -44,11 +43,10 @@ describe('CLI single-query mode', () => { it('runs single query and produces output', async () => { const mockApi = createMockProtonApi('success'); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new MinimalStore(); const mockApp: Application = { getLumoClient: () => lumoClient, - getConversationStore: () => store, + getConversationStore: () => undefined, getAuthProvider: () => undefined, getAuthManager: () => undefined, isSyncInitialized: () => false, diff --git a/tests/helpers/test-server.ts b/tests/helpers/test-server.ts index 122a50e..4b4daff 100644 --- a/tests/helpers/test-server.ts +++ b/tests/helpers/test-server.ts @@ -15,7 +15,6 @@ import { createModelsRouter } from '../../src/api/routes/models.js'; import { RequestQueue } from '../../src/api/queue.js'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { MinimalStore, type IConversationStore } from '../../src/conversations/index.js'; import { MetricsService, setMetrics } from '../../src/app/metrics.js'; import { setupMetricsMiddleware } from '../../src/api/middleware.js'; import { createMetricsRouter } from '../../src/api/routes/metrics.js'; @@ -36,7 +35,6 @@ export interface TestServer { server: Server; baseUrl: string; deps: EndpointDependencies; - store: IConversationStore; /** MetricsService instance (only if metrics option was true) */ metrics?: MetricsService; close: () => Promise<void>; @@ -55,13 +53,12 @@ export async function createTestServer( ): Promise<TestServer> { const mockApi = createMockProtonApi(scenario); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new MinimalStore(); const queue = new RequestQueue(1); const deps: EndpointDependencies = { queue, lumoClient, - conversationStore: store, + conversationStore: undefined, syncInitialized: false, }; @@ -101,7 +98,6 @@ export async function createTestServer( server, baseUrl, deps, - store, metrics, close: () => new Promise((resolve) => { if (metrics) setMetrics(null); diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 565dd03..de6448b 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -11,7 +11,6 @@ import { createHealthRouter } from '../../src/api/routes/health.js'; import { RequestQueue } from '../../src/api/queue.js'; import { LumoClient } from '../../src/lumo-client/index.js'; import { createMockProtonApi } from '../../src/mock/mock-api.js'; -import { MinimalStore } from '../../src/conversations/index.js'; import { MetricsService, setMetrics } from '../../src/app/metrics.js'; import { setupMetricsMiddleware } from '../../src/api/middleware.js'; import { createMetricsRouter } from '../../src/api/routes/metrics.js'; @@ -27,13 +26,12 @@ interface TestServer { async function createTestServerWithMetrics(): Promise<TestServer> { const mockApi = createMockProtonApi('success'); const lumoClient = new LumoClient(mockApi, { enableEncryption: false }); - const store = new MinimalStore(); const queue = new RequestQueue(1); const deps: EndpointDependencies = { queue, lumoClient, - conversationStore: store, + conversationStore: undefined, syncInitialized: false, }; diff --git a/tests/unit/minimal-store.test.ts b/tests/unit/minimal-store.test.ts deleted file mode 100644 index 0e5dba0..0000000 --- a/tests/unit/minimal-store.test.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Unit tests for MinimalStore (in-memory conversation store) - * - * Tests in-memory conversation management, - * message deduplication, and Turn conversion. - */ - -import { describe, it, expect, beforeEach } from 'vitest'; -import { MinimalStore } from '../../src/conversations/index.js'; - -let store: MinimalStore; - -beforeEach(() => { - store = new MinimalStore(); -}); - -describe('MinimalStore', () => { - describe('getOrCreate', () => { - it('creates new conversation when none exists', () => { - const state = store.getOrCreate('conv-1'); - expect(state).toBeDefined(); - expect(state.title).toBe('New Conversation'); - expect(state.messages).toEqual([]); - expect(state.status).toBe('completed'); - }); - - it('returns existing conversation on second call', () => { - const first = store.getOrCreate('conv-1'); - first.title = 'Modified'; - const second = store.getOrCreate('conv-1'); - expect(second.title).toBe('Modified'); - }); - }); - - describe('get / has', () => { - it('returns undefined for non-existent conversation', () => { - expect(store.get('nonexistent')).toBeUndefined(); - }); - - it('returns state for existing conversation', () => { - store.getOrCreate('conv-1'); - expect(store.get('conv-1')).toBeDefined(); - }); - - it('has returns true for existing conversation', () => { - store.getOrCreate('conv-1'); - expect(store.has('conv-1')).toBe(true); - expect(store.has('nonexistent')).toBe(false); - }); - }); - - describe('appendMessages', () => { - it('appends messages to empty conversation', () => { - const added = store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - expect(added).toHaveLength(1); - expect(added[0].role).toBe('user'); - expect(added[0].content).toBe('Hello'); - }); - - it('deduplicates previously stored messages', () => { - store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - // Send same message again (typical API re-send pattern) - const added = store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - expect(added).toHaveLength(0); - }); - - it('returns only newly added messages', () => { - store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - const added = store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi!' }, - { role: 'user', content: 'New message' }, - ]); - expect(added).toHaveLength(2); - expect(added[0].content).toBe('Hi!'); - expect(added[1].content).toBe('New message'); - }); - - it('sets parentId chain correctly', () => { - const added = store.appendMessages('conv-1', [ - { role: 'user', content: 'First' }, - { role: 'assistant', content: 'Second' }, - ]); - expect(added[0].parentId).toBeUndefined(); - expect(added[1].parentId).toBe(added[0].id); - }); - }); - - describe('appendAssistantResponse', () => { - it('appends response and marks conversation completed', () => { - store.appendMessages('conv-1', [{ role: 'user', content: 'Hi' }]); - const msg = store.appendAssistantResponse('conv-1', { content: 'Hello there!' }); - - expect(msg.role).toBe('assistant'); - expect(msg.content).toBe('Hello there!'); - expect(msg.status).toBe('succeeded'); - - const state = store.get('conv-1')!; - expect(state.status).toBe('completed'); - }); - - it('sets parentId to last message', () => { - const [userMsg] = store.appendMessages('conv-1', [ - { role: 'user', content: 'Hi' }, - ]); - const assistantMsg = store.appendAssistantResponse('conv-1', { content: 'Hello' }); - expect(assistantMsg.parentId).toBe(userMsg.id); - }); - - it('stores native tool call data in blocks', () => { - store.appendMessages('conv-1', [{ role: 'user', content: 'Search for news' }]); - const msg = store.appendAssistantResponse('conv-1', { - content: 'Here are the results...', - blocks: [ - { type: 'tool_call', content: '{"name":"web_search","arguments":{"query":"news"}}' }, - { type: 'tool_result', content: '{"results":[]}' }, - { type: 'text', content: 'Here are the results...' }, - ], - }); - - expect(msg.content).toBe('Here are the results...'); - expect(msg.blocks).toHaveLength(3); - expect(msg.blocks![0].type).toBe('tool_call'); - expect(msg.blocks![1].type).toBe('tool_result'); - expect(msg.blocks![2].type).toBe('text'); - }); - }); - - describe('toTurns', () => { - it('converts messages to Turn[] format', () => { - store.appendMessages('conv-1', [ - { role: 'user', content: 'Hello' }, - ]); - store.appendAssistantResponse('conv-1', { content: 'Hi!' }); - - const turns = store.toTurns('conv-1'); - expect(turns).toEqual([ - { role: 'user', content: 'Hello' }, - { role: 'assistant', content: 'Hi!' }, - ]); - }); - - it('returns empty array for non-existent conversation', () => { - expect(store.toTurns('nonexistent')).toEqual([]); - }); - }); - - describe('no-op methods', () => { - it('setTitle is no-op', () => { - store.getOrCreate('conv-1'); - store.setTitle('conv-1', 'My Chat'); - // Title not stored - still default - expect(store.get('conv-1')!.title).toBe('New Conversation'); - }); - - it('delete returns false', () => { - store.getOrCreate('conv-1'); - expect(store.delete('conv-1')).toBe(false); - // Conversation still exists - expect(store.has('conv-1')).toBe(true); - }); - - it('entries yields nothing', () => { - store.getOrCreate('conv-1'); - const entries = [...store.entries()]; - expect(entries).toEqual([]); - }); - - it('getMessage returns undefined', () => { - store.appendMessages('conv-1', [{ role: 'user', content: 'Hi' }]); - expect(store.getMessage('conv-1', 'any-message-id')).toBeUndefined(); - }); - - it('createFromTurns returns empty', () => { - const result = store.createFromTurns([{ role: 'user', content: 'Hi' }]); - expect(result.conversationId).toBe(''); - expect(result.title).toBe(''); - }); - }); -}); diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index b5976c6..d382502 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -8,29 +8,15 @@ import { formatSearchResults, type SearchResult, } from '../../src/conversations/search.js'; -import type { IConversationStore } from '../../src/conversations/store-interface.js'; +import type { ConversationStore } from '../../src/conversations/index.js'; import type { ConversationId, ConversationState, Message } from '../../src/conversations/types.js'; /** * Create a mock store with test conversations */ -function createMockStore(conversations: Map<ConversationId, ConversationState>): IConversationStore { +function createMockStore(conversations: Map<ConversationId, ConversationState>): Pick<ConversationStore, 'entries'> { return { entries: () => conversations.entries(), - // Other methods not needed for search tests - get: () => undefined, - getOrCreate: () => { throw new Error('not implemented'); }, - has: () => false, - delete: () => false, - appendMessages: () => [], - appendAssistantResponse: () => { throw new Error('not implemented'); }, - appendUserMessage: () => { throw new Error('not implemented'); }, - appendAssistantToolCalls: () => {}, - getMessages: () => [], - getMessage: () => undefined, - setTitle: () => {}, - toTurns: () => [], - createFromTurns: () => { throw new Error('not implemented'); }, }; } From 6a54d1af4182ae27926ada2c9c8cd623254f1a4d Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Wed, 1 Apr 2026 22:53:28 +0200 Subject: [PATCH 25/37] suppress softDeleteSpaceCascade warnings --- src/shims/console.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/shims/console.ts b/src/shims/console.ts index fdcd3da..8033d2b 100644 --- a/src/shims/console.ts +++ b/src/shims/console.ts @@ -32,6 +32,12 @@ const suppressLogs = [ ]; const suppressLogRegex = new RegExp(`^(?:${suppressLogs.join('|')})`); +const suppressErrors = [ + 'softDeleteSpaceCascade: [a-f0-9-]+ not found', +]; + +const suppressErrorRegex = new RegExp(`^(?:${suppressErrors.join('|')})`); + const suppressApiErrors = [ // Sync-disabled errors (login/rclone auth without lumo scope) 'list spaces failure', @@ -93,6 +99,7 @@ function createLogFunction(logger: Logger) { ee.shift() if ( (levelOrLog == 'log' && suppressLogRegex.test(first)) || (fullApiErrorsSuppressed && levelOrLog == 'error' && suppressApiErrorRegex.test(first)) + || ((levelOrLog == 'error' || levelOrLog == 'warn') && suppressErrorRegex.test(first)) ) level = 'trace'; logger[level](minimal(ee), first); From 0a9b9b72014b9a1c7a109b5e33c24f17486ec5e8 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:45:46 +0200 Subject: [PATCH 26/37] fix type mismatches and import paths --- src/cli/client.ts | 10 +++++----- src/conversations/deduplication.ts | 6 +++--- tests/integration/chat-completions-api.test.ts | 6 +++--- tests/integration/responses-api.test.ts | 6 +++--- tests/unit/metrics.test.ts | 1 - 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/cli/client.ts b/src/cli/client.ts index cdd4167..cf68bdf 100644 --- a/src/cli/client.ts +++ b/src/cli/client.ts @@ -15,7 +15,7 @@ import { BUSY_INDICATOR, clearBusyIndicator, print } from '../app/terminal.js'; import type { Application } from '../app/index.js'; import { randomUUID } from 'crypto'; import * as readline from 'readline'; -import type { AssistantMessageData, Turn } from '../lumo-client/index.js'; +import { Role, type AssistantMessageData, type Turn } from '../lumo-client/index.js'; import type { ConversationStore } from '../conversations/index.js'; import { blockHandlers, executeBlocks, formatResultsMessage } from './local-actions/block-handlers.js'; import { CodeBlockDetector, type CodeBlock } from './local-actions/code-block-detector.js'; @@ -216,11 +216,11 @@ export class CLIClient { } try { - this.turns.push({ role: 'user', content: input }); + this.turns.push({ role: Role.User, content: input }); this.store?.appendUserMessage(this.conversationId, input); let lumoResponse = await this.sendToLumo(); - this.turns.push({ role: 'assistant', content: lumoResponse.message.content }); + this.turns.push({ role: Role.Assistant, content: lumoResponse.message.content }); this.store?.appendAssistantResponse(this.conversationId, lumoResponse.message); // Execute blocks until none remain (or user skips all) @@ -232,11 +232,11 @@ export class CLIClient { // Send batch results back to Lumo print('─── Sending results to Lumo ───\n'); const batchMessage = formatResultsMessage(results); - this.turns.push({ role: 'user', content: batchMessage }); + this.turns.push({ role: Role.User, content: batchMessage }); this.store?.appendUserMessage(this.conversationId, batchMessage); lumoResponse = await this.sendToLumo(); - this.turns.push({ role: 'assistant', content: lumoResponse.message.content }); + this.turns.push({ role: Role.Assistant, content: lumoResponse.message.content }); this.store?.appendAssistantResponse(this.conversationId, lumoResponse.message); } } catch (error) { diff --git a/src/conversations/deduplication.ts b/src/conversations/deduplication.ts index ce68082..ff05ce1 100644 --- a/src/conversations/deduplication.ts +++ b/src/conversations/deduplication.ts @@ -8,9 +8,9 @@ import { createHash } from 'crypto'; import { Role } from '@lumo/types.js'; import type { Message, MessageForStore } from './types.js'; -import { getMetrics } from 'src/app/metrics.js'; -import logger from 'src/app/logger.js'; -import { serverToolPrefix } from 'src/api/tools/server-tools/registry.js'; +import { getMetrics } from '../app/metrics.js'; +import logger from '../app/logger.js'; +import { serverToolPrefix } from '../api/tools/server-tools/registry.js'; /** * Compute hash for a message (role + content) diff --git a/tests/integration/chat-completions-api.test.ts b/tests/integration/chat-completions-api.test.ts index c80b2e6..14b11a7 100644 --- a/tests/integration/chat-completions-api.test.ts +++ b/tests/integration/chat-completions-api.test.ts @@ -4,7 +4,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createTestServer, parseSSEEvents, type TestServer } from '../helpers/test-server.js'; -import { getClientToolsConfig } from '../../src/app/config.js'; +import { getServerConfig } from '../../src/app/config.js'; /** POST /v1/chat/completions with JSON body, returning the raw Response. */ function postChat(ts: TestServer, body: Record<string, unknown>): Promise<Response> { @@ -139,10 +139,10 @@ describe('/v1/chat/completions', () => { beforeAll(async () => { nativeTs = await createTestServer('misroutedToolCall'); - (getClientToolsConfig() as any).enabled = true; + getServerConfig().tools.client.enabled = true; }); afterAll(async () => { - (getClientToolsConfig() as any).enabled = false; + getServerConfig().tools.client.enabled = false; await nativeTs.close(); }); diff --git a/tests/integration/responses-api.test.ts b/tests/integration/responses-api.test.ts index 6a32d3d..f396352 100644 --- a/tests/integration/responses-api.test.ts +++ b/tests/integration/responses-api.test.ts @@ -7,7 +7,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { createTestServer, parseSSEEvents, type TestServer } from '../helpers/test-server.js'; -import { getClientToolsConfig, getServerConfig } from '../../src/app/config.js'; +import { getServerConfig } from '../../src/app/config.js'; /** POST /v1/responses with JSON body, returning the raw Response. */ function postResponses(ts: TestServer, body: Record<string, unknown>): Promise<Response> { @@ -189,10 +189,10 @@ describe('/v1/responses', () => { beforeAll(async () => { ts = await createTestServer('misroutedToolCall', { metrics: true }); // Enable client tool detection so the bounce response JSON is parsed - (getClientToolsConfig() as any).enabled = true; + getServerConfig().tools.client.enabled = true; }); afterAll(async () => { - (getClientToolsConfig() as any).enabled = false; + getServerConfig().tools.client.enabled = false; await ts.close(); }); diff --git a/tests/unit/metrics.test.ts b/tests/unit/metrics.test.ts index acb19bf..caeaa40 100644 --- a/tests/unit/metrics.test.ts +++ b/tests/unit/metrics.test.ts @@ -121,7 +121,6 @@ describe('MetricsService', () => { expect(output).toContain('tool_name="web_search"'); expect(output).toContain('tool_name="proton_info"'); expect(output).toContain('tool_name="my_tool"'); - expect(output).toContain('tool_name="current_time"'); }); }); From dbc25820114a5c4be61de07976ce0dac7ce9739e Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:52:49 +0200 Subject: [PATCH 27/37] fix not awating bounced tool call --- src/lumo-client/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lumo-client/client.ts b/src/lumo-client/client.ts index bb723e5..c51b742 100644 --- a/src/lumo-client/client.ts +++ b/src/lumo-client/client.ts @@ -335,7 +335,7 @@ export class LumoClient { ]; return { - ...this.chatWithHistory(bounceTurns, onChunk, options, true), + ...await this.chatWithHistory(bounceTurns, onChunk, options, true), title: result.title ? postProcessTitle(result.title) : undefined, }; } From d7e6e8911b3472004ab7fc4c0e735ff3abd637c7 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 10:53:11 +0200 Subject: [PATCH 28/37] fix tests covering executeServerTools --- tests/unit/shared.test.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/unit/shared.test.ts b/tests/unit/shared.test.ts index a962505..0f24062 100644 --- a/tests/unit/shared.test.ts +++ b/tests/unit/shared.test.ts @@ -18,7 +18,8 @@ import { } from '../../src/api/tools/server-tools/registry.js'; import { partitionToolCalls, - buildServerToolContinuation, + executeServerTools, + buildContinuationTurns, } from '../../src/api/tools/server-tools/executor.js'; import { Role } from '../../src/lumo-client/types.js'; import { generateCallId, extractToolNameFromCallId } from '../../src/api/tools/call-id.js'; @@ -148,7 +149,7 @@ describe('partitionToolCalls', () => { }); }); -describe('buildServerToolContinuation', () => { +describe('executeServerTools + buildContinuationTurns', () => { beforeEach(() => { clearServerTools(); }); @@ -168,7 +169,8 @@ describe('buildServerToolContinuation', () => { ]; const context: ServerToolContext = {}; - const turns = await buildServerToolContinuation(serverToolCalls, 'Assistant text', context, 'user:'); + const results = await executeServerTools(serverToolCalls, context); + const turns = buildContinuationTurns('Assistant text', results, 'user:'); expect(turns).toHaveLength(2); @@ -204,7 +206,8 @@ describe('buildServerToolContinuation', () => { { id: 'call-b', type: 'function' as const, function: { name: 'tool_b', arguments: '{}' } }, ]; - const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'prefix:'); + const results = await executeServerTools(serverToolCalls, {}); + const turns = buildContinuationTurns('Text', results, 'prefix:'); // 1 assistant + 2 user turns expect(turns).toHaveLength(3); @@ -231,7 +234,8 @@ describe('buildServerToolContinuation', () => { { id: 'call-fail', type: 'function' as const, function: { name: 'failing_tool', arguments: '{}' } }, ]; - const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'user:'); + const results = await executeServerTools(serverToolCalls, {}); + const turns = buildContinuationTurns('Text', results, 'user:'); expect(turns).toHaveLength(2); expect(turns[1].content).toContain('Error executing failing_tool'); @@ -251,7 +255,8 @@ describe('buildServerToolContinuation', () => { { id: 'call-1', type: 'function' as const, function: { name: 'my_tool', arguments: '{}' } }, ]; - const turns = await buildServerToolContinuation(serverToolCalls, 'Text', {}, 'custom:'); + const results = await executeServerTools(serverToolCalls, {}); + const turns = buildContinuationTurns('Text', results, 'custom:'); const content = turns[1].content; expect(content).toContain('"tool_name":"custom:my_tool"'); From dba36ec06a3b6b360a71d6348de93a85796adf59 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 11:26:37 +0200 Subject: [PATCH 29/37] fix CLI error upon misrouted tool call --- src/api/tools/native-tool-call-processor.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/tools/native-tool-call-processor.ts b/src/api/tools/native-tool-call-processor.ts index 89bbcf3..7010246 100644 --- a/src/api/tools/native-tool-call-processor.ts +++ b/src/api/tools/native-tool-call-processor.ts @@ -15,7 +15,7 @@ import { JsonBraceTracker } from './json-brace-tracker.js'; import { stripToolPrefix } from './prefix.js'; -import { getCustomToolPrefix } from '../../app/config.js'; +import { getConfigMode, getCustomToolPrefix } from '../../app/config.js'; import { getMetrics } from '../../app/metrics.js'; import { logger } from '../../app/logger.js'; import type { ParsedToolCall } from './types.js'; @@ -115,7 +115,9 @@ export class NativeToolCallProcessor { } if (this.isMisrouted(toolCall)) { - const strippedName = stripToolPrefix(toolCall.name, getCustomToolPrefix()); + // Only strip prefix in server mode (CLI has no tool prefix concept) + const prefix = getConfigMode() === 'server' ? getCustomToolPrefix() : ''; + const strippedName = stripToolPrefix(toolCall.name, prefix); getMetrics()?.toolCallsTotal.inc({ type: 'custom', status: 'misrouted', tool_name: strippedName }); From af6774b60de5283bae76642eaf1925ae4f4857ef Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 11:53:46 +0200 Subject: [PATCH 30/37] Added conversations.enableStore config option (default true) --- config.defaults.yaml | 11 ++++++----- src/app/config.ts | 1 + src/conversations/index.ts | 8 +++++++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/config.defaults.yaml b/config.defaults.yaml index d7f9b2a..d883e49 100644 --- a/config.defaults.yaml +++ b/config.defaults.yaml @@ -62,14 +62,11 @@ log: messageContent: false # Shared Conversations Configuration (can be overridden in server/cli sections) -# Note: In-memory conversation storage is always active regardless of sync settings -# - do they still work/make sense with upstream store? -# - can they still be overwritten for cli/server? (ie. overwriting dbpath doesn't make sense if sync enabled, does it otherwise?) conversations: - # Path for IndexedDB SQLite files (used when useUpstreamStorage is true) + # Path for IndexedDB SQLite files databasePath: "sessions/" - # WORKAROUND for clients without conversation_id support (e.g., Home Assistant). + # WORKAROUND for API clients without conversation_id support (e.g., Home Assistant). # Derives conversation ID from the `user` field in the request. # Home Assistant sends its internal conversation_id as user, making it unique per chat session. # WARNING: May incorrectly merge unrelated conversations with same "user" field. Ignored if `user` is absent. @@ -80,6 +77,10 @@ conversations: # Project name for conversations (created if doesn't exist) projectName: lumo-tamer + # Enable ConversationStore for conversation persistence + # When disabled, conversations are not persisted (same as when encryption keys unavailable) + enableStore: true + # Shared Commands Configuration (can be overridden in server/cli sections) commands: # Enable slash commands (/save, /help, /logout, etc.) diff --git a/src/app/config.ts b/src/app/config.ts index c4ea29f..dfcfd4d 100644 --- a/src/app/config.ts +++ b/src/app/config.ts @@ -25,6 +25,7 @@ const logConfigSchema = z.object({ const conversationsConfigSchema = z.object({ + enableStore: z.boolean(), deriveIdFromUser: z.boolean(), databasePath: z.string(), enableSync: z.boolean(), diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 54aac77..4b3f812 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -101,7 +101,13 @@ let activeStore: ConversationStore | null = null; export async function initializeConversationStore( options: InitializeStoreOptions ): Promise<InitializeStoreResult> { - const { authProvider } = options; + const { authProvider, conversationsConfig } = options; + + // Check if store is disabled via config + if (!conversationsConfig.enableStore) { + logger.info('ConversationStore disabled via config'); + return { isPrimary: false }; + } // Check if ConversationStore can be used const storeWarning = authProvider.getConversationStoreWarning(); From 9cda8f2097908652788d5eacb688acf3582028e3 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:01:22 +0200 Subject: [PATCH 31/37] ConversationStore: simplify init return types, drop "primary" from names --- src/conversations/index.ts | 106 +++++++----------- src/conversations/store.ts | 2 +- ...ore.test.ts => conversation-store.test.ts} | 3 +- 3 files changed, 42 insertions(+), 69 deletions(-) rename tests/unit/{primary-conversation-store.test.ts => conversation-store.test.ts} (98%) diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 4b3f812..962d3f4 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -2,7 +2,7 @@ * Conversation persistence module * * Provides: - * - ConversationStore: Primary storage using Redux + IndexedDB + * - ConversationStore: Redux + IndexedDB storage * - Message deduplication for OpenAI API format * - Types compatible with Proton Lumo webclient */ @@ -22,7 +22,7 @@ export type { MessageForStore, } from './types.js'; -// Primary store +// Store export { ConversationStore } from './store.js'; // Store initialization @@ -74,15 +74,8 @@ export interface InitializeStoreOptions { conversationsConfig: ConversationsConfig; } -export interface InitializeStoreResult { - /** Whether the primary store is being used (vs fallback) */ - isPrimary: boolean; - /** Store result, only set when primary store is used */ - storeResult?: StoreResult; -} - // Module-level state to track store result for sync initialization -let primaryStoreResult: StoreResult | null = null; +let storeResult: StoreResult | null = null; // Singleton for the active store let activeStore: ConversationStore | null = null; @@ -90,68 +83,35 @@ let activeStore: ConversationStore | null = null; /** * Initialize the conversation store * - * Creates the primary ConversationStore (Redux + IndexedDB) if possible. - * Returns undefined if initialization fails - callers should handle this + * Creates the ConversationStore (Redux + IndexedDB) if possible. + * Logs warnings if initialization fails - callers should handle this * gracefully (server works stateless, CLI uses local Turn array). * - * Primary store requires: + * Requires: * - Auth provider supports persistence (has cached encryption keys) * - keyPassword is available (for master key decryption) */ export async function initializeConversationStore( options: InitializeStoreOptions -): Promise<InitializeStoreResult> { +): Promise<void> { const { authProvider, conversationsConfig } = options; // Check if store is disabled via config if (!conversationsConfig.enableStore) { logger.info('ConversationStore disabled via config'); - return { isPrimary: false }; + return; } // Check if ConversationStore can be used const storeWarning = authProvider.getConversationStoreWarning(); if (storeWarning) { logger.warn({ method: authProvider.method }, storeWarning); - return { isPrimary: false }; + return; } // If we get here, getConversationStoreWarning() confirmed keyPassword exists const keyPassword = authProvider.getKeyPassword()!; - // All conditions met - initialize primary store - try { - const result = await initializePrimaryStore(options, keyPassword); - if (result) { - activeStore = result.conversationStore; - primaryStoreResult = result; - logger.info('Using primary conversation store'); - - // Pull incomplete conversations in background when sync is enabled - if (options.conversationsConfig.enableSync) { - pullIncompleteConversations(result.store, result.spaceId) - .catch(err => logger.error({ error: err }, 'Failed to pull incomplete conversations')); - } - - return { isPrimary: true, storeResult: result }; - } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - logger.error({ error: msg }, 'Failed to initialize primary store. Continuing without store.'); - } - - return { isPrimary: false }; -} - -/** - * Initialize the primary conversation store - */ -async function initializePrimaryStore( - options: InitializeStoreOptions, - keyPassword: string -): Promise<StoreResult | null> { - const { protonApi, uid, authProvider, conversationsConfig } = options; - // Get cached keys from browser provider if available const cachedUserKeys = authProvider.getCachedUserKeys?.(); const cachedMasterKeys = authProvider.getCachedMasterKeys?.(); @@ -162,28 +122,42 @@ async function initializePrimaryStore( hasCachedUserKeys: !!cachedUserKeys, hasCachedMasterKeys: !!cachedMasterKeys, }, - 'Initializing KeyManager for primary store...' + 'Initializing KeyManager...' ); // Initialize KeyManager const keyManager = getKeyManager({ - protonApi, + protonApi: options.protonApi, cachedUserKeys, cachedMasterKeys, }); - await keyManager.initialize(keyPassword); - - // Get master key as base64 for crypto layer - const masterKeyBase64 = keyManager.getMasterKeyBase64(); - - const result = await initializeStore({ - sessionUid: uid, - userId: authProvider.getUserId() ?? uid, - masterKey: masterKeyBase64, - projectName: conversationsConfig.projectName, - }); - return result; + try { + await keyManager.initialize(keyPassword); + + // Get master key as base64 for crypto layer + const masterKeyBase64 = keyManager.getMasterKeyBase64(); + + const result = await initializeStore({ + sessionUid: options.uid, + userId: authProvider.getUserId() ?? options.uid, + masterKey: masterKeyBase64, + projectName: conversationsConfig.projectName, + }); + + activeStore = result.conversationStore; + storeResult = result; + logger.info('ConversationStore initialized'); + + // Pull incomplete conversations in background when sync is enabled + if (conversationsConfig.enableSync) { + pullIncompleteConversations(result.store, result.spaceId) + .catch(err => logger.error({ error: err }, 'Failed to pull incomplete conversations')); + } + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + logger.error({ error: msg }, 'Failed to initialize store. Continuing without store.'); + } } /** @@ -208,7 +182,7 @@ export function setConversationStore(store: ConversationStore): void { */ export function resetConversationStore(): void { activeStore = null; - primaryStoreResult = null; + storeResult = null; } // ============================================================================ @@ -225,8 +199,8 @@ export interface InitializeSyncOptions { /** * Initialize sync services * - * Sync is handled automatically by Redux sagas when primary store is active. - * Returns false if no primary store or sync is disabled. + * Sync is handled automatically by Redux sagas when the store is active. + * Returns false if no store or sync is disabled. */ export function initializeSync(options: InitializeSyncOptions): boolean { const { authProvider, conversationsConfig } = options; diff --git a/src/conversations/store.ts b/src/conversations/store.ts index ee1829c..93f3eb8 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -1,7 +1,7 @@ /** * Conversation Store * - * Primary conversation storage using Redux + IndexedDB for persistence. + * Conversation storage using Redux + IndexedDB for persistence. * * Architecture: * - Redux store holds in-memory state (conversations, messages) diff --git a/tests/unit/primary-conversation-store.test.ts b/tests/unit/conversation-store.test.ts similarity index 98% rename from tests/unit/primary-conversation-store.test.ts rename to tests/unit/conversation-store.test.ts index 4b68917..1ea1d4f 100644 --- a/tests/unit/primary-conversation-store.test.ts +++ b/tests/unit/conversation-store.test.ts @@ -1,8 +1,7 @@ /** * Unit tests for ConversationStore (Redux + IndexedDB) * - * Tests the primary conversation store implementation using fake-indexeddb. - * Mirrors FallbackStore tests for consistent behavior verification. + * Tests the conversation store implementation using fake-indexeddb. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; From a03db05755acb42efbe49c90cfbcb7f90ac9362d Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:09:49 +0200 Subject: [PATCH 32/37] add ServerTool.isAvailable(), use it to make search dependent on ConversationStore --- src/api/tools/server-tools/index.ts | 8 ++++++-- src/api/tools/server-tools/registry.ts | 2 ++ src/api/tools/server-tools/search.ts | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts index 6c45ac7..2235922 100644 --- a/src/api/tools/server-tools/index.ts +++ b/src/api/tools/server-tools/index.ts @@ -36,6 +36,10 @@ export { chatAndExecute, type ChatAndExecuteOptions, type ChatAndExecuteResult } * Called during server startup when enableServerTools is true. */ export function initializeServerTools(): void { - registerServerTool(dateServerTool); - registerServerTool(searchServerTool); + const tools = [dateServerTool, searchServerTool]; + for (const tool of tools) { + if (tool.isAvailable === undefined || tool.isAvailable()) { + registerServerTool(tool); + } + } } diff --git a/src/api/tools/server-tools/registry.ts b/src/api/tools/server-tools/registry.ts index ab5a102..952dca3 100644 --- a/src/api/tools/server-tools/registry.ts +++ b/src/api/tools/server-tools/registry.ts @@ -27,6 +27,8 @@ export type ServerToolHandler = ( export interface ServerTool { definition: OpenAITool; handler: ServerToolHandler; + /** Optional check if tool should be registered. If undefined, tool is always available. */ + isAvailable?: () => boolean; } /** diff --git a/src/api/tools/server-tools/search.ts b/src/api/tools/server-tools/search.ts index 7ab77a1..29b242b 100644 --- a/src/api/tools/server-tools/search.ts +++ b/src/api/tools/server-tools/search.ts @@ -7,6 +7,7 @@ import { searchConversations, formatSearchResults } from '../../../conversations/search.js'; import { serverToolPrefix, type ServerTool } from './registry.js'; +import { getConversationStore } from '../../../conversations/index.js'; export const searchServerTool: ServerTool = { definition: { @@ -51,4 +52,5 @@ export const searchServerTool: ServerTool = { return formatSearchResults(results, query.trim()); }, + isAvailable: () => getConversationStore() !== undefined, }; From b1582e1187156038747ad914bc778e962c5e01b1 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 12:36:00 +0200 Subject: [PATCH 33/37] add ServerTool to read previous conversations B-) --- src/api/tools/server-tools/index.ts | 3 +- .../tools/server-tools/read-conversation.ts | 90 +++++++++++++++++++ src/conversations/search.ts | 2 +- 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 src/api/tools/server-tools/read-conversation.ts diff --git a/src/api/tools/server-tools/index.ts b/src/api/tools/server-tools/index.ts index 2235922..e4455f0 100644 --- a/src/api/tools/server-tools/index.ts +++ b/src/api/tools/server-tools/index.ts @@ -8,6 +8,7 @@ import { registerServerTool } from './registry.js'; import { dateServerTool } from './date.js'; import { searchServerTool } from './search.js'; +import { readConversationServerTool } from './read-conversation.js'; // Re-export types and functions export { @@ -36,7 +37,7 @@ export { chatAndExecute, type ChatAndExecuteOptions, type ChatAndExecuteResult } * Called during server startup when enableServerTools is true. */ export function initializeServerTools(): void { - const tools = [dateServerTool, searchServerTool]; + const tools = [dateServerTool, searchServerTool, readConversationServerTool]; for (const tool of tools) { if (tool.isAvailable === undefined || tool.isAvailable()) { registerServerTool(tool); diff --git a/src/api/tools/server-tools/read-conversation.ts b/src/api/tools/server-tools/read-conversation.ts new file mode 100644 index 0000000..8d14182 --- /dev/null +++ b/src/api/tools/server-tools/read-conversation.ts @@ -0,0 +1,90 @@ +/** + * Read Conversation ServerTool + * + * Reads a conversation by ID and returns formatted markdown text. + */ + +import { serverToolPrefix, type ServerTool } from './registry.js'; +import { getConversationStore, type ConversationState } from '../../../conversations/index.js'; + +export const readConversationServerTool: ServerTool = { + definition: { + type: 'function', + function: { + name: serverToolPrefix + 'read_conversation', + description: 'Read the full text of a conversation by its ID. Returns formatted markdown.', + parameters: { + type: 'object', + properties: { + conversation_id: { + type: 'string', + description: 'The conversation ID to read', + }, + }, + required: ['conversation_id'], + }, + }, + }, + handler: async (args, context) => { + const conversationId = args.conversation_id; + if (typeof conversationId !== 'string' || !conversationId.trim()) { + return 'Error: conversation_id is required'; + } + + if (!context.conversationStore) { + return 'Error: conversation store not available'; + } + + const conversation = context.conversationStore.get(conversationId); + if (!conversation) { + return `Error: conversation not found: ${conversationId}`; + } + + return formatConversation(conversation); + }, + isAvailable: () => getConversationStore() !== undefined, +}; + +function formatConversation(conversation: ConversationState): string { + const lines: string[] = []; + lines.push(`# ${conversation.title || 'Untitled'}\n`); + + for (const message of conversation.messages) { + if (isToolMessage(message.content)) continue; + + const roleHeader = message.role === 'user' ? '## User' : '## Assistant'; + lines.push(roleHeader); + lines.push(message.content || ''); + lines.push(''); + } + + return lines.join('\n'); +} + +function isToolMessage(content: string | undefined): boolean { + if (!content) return false; + const trimmed = content.trim(); + + // Raw JSON (function_call from assistant) + if (trimmed.startsWith('{')) { + try { + const parsed = JSON.parse(trimmed); + return parsed.type === 'function_call' || parsed.type === 'function_call_output'; + } catch { + return false; + } + } + + // Fenced JSON (function_call_output from user) + if (trimmed.startsWith('```json\n{')) { + const jsonContent = trimmed.slice(8, -4); // Remove ```json\n and \n``` + try { + const parsed = JSON.parse(jsonContent); + return parsed.type === 'function_call_output'; + } catch { + return false; + } + } + + return false; +} diff --git a/src/conversations/search.ts b/src/conversations/search.ts index 5ab82a1..77b697b 100644 --- a/src/conversations/search.ts +++ b/src/conversations/search.ts @@ -169,7 +169,7 @@ export function formatSearchResults(results: SearchResult[], query: string): str lines.push(''); } - lines.push('Use /load <id> to open a conversation'); + // lines.push('Use /load <id> to open a conversation'); return lines.join('\n'); } From a404b32acaa29c2ac4afef80b0dd15ad9439131e Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 14:24:35 +0200 Subject: [PATCH 34/37] mention server tools shortly in docs --- README.md | 2 +- docs/tools.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 546171e..9d139b7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ lumo-tamer is a lightweight local proxy that talks to Proton's Lumo API using th - OpenAI-compatible API server with experimental tool support. - Interactive CLI, let Lumo help you execute commands, read, create and edit files. -- Sync your conversations with Proton to access them on https://lumo.proton.me or in mobile apps. +- Sync your conversations with Proton to access them on https://lumo.proton.me or in mobile apps. Let Lumo search through and read your past conversations. ## Project Status diff --git a/docs/tools.md b/docs/tools.md index 4eccfe0..743efa7 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -232,6 +232,7 @@ Server tools are custom tools executed by lumo-tamer itself, not passed to API c | Tool | Description | |------|-------------| | `lumo_search` | Search conversation history by title and message content | +| `lumo_read_conversation` | Read the full text of a conversation by ID | ### Enable From a8e4ee052b57a2163e9142fe71367d01fcb8c19d Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 15:03:02 +0200 Subject: [PATCH 35/37] Clean up conversations/ dead code and unnecessary exports --- src/conversations/index.ts | 55 ++++---------------------------------- src/conversations/store.ts | 18 ------------- src/conversations/types.ts | 25 ++++++++++++++++- tests/unit/search.test.ts | 1 - 4 files changed, 29 insertions(+), 70 deletions(-) diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 962d3f4..789eb44 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -20,27 +20,14 @@ export type { RemoteId, IdMapEntry, MessageForStore, + InitializeStoreOptions, + InitializeSyncOptions, } from './types.js'; // Store export { ConversationStore } from './store.js'; -// Store initialization -export { - initializeStore, - type StoreConfig, - type StoreResult, -} from './init.js'; - -// Deduplication utilities -export { - hashMessage, - createFingerprint, - fingerprintMessages, - findNewMessages, -} from './deduplication.js'; - -// Key management +// Key management (exported for testing) export { KeyManager, getKeyManager, @@ -48,34 +35,15 @@ export { type KeyManagerConfig, } from './key-manager.js'; -// Re-export LumoApi types for consumers -export { LumoApi } from '@lumo/remote/api.js'; -export { RoleInt, StatusInt } from '@lumo/remote/types.js'; - // ============================================================================ // Persistence initialization // ============================================================================ import { logger } from '../app/logger.js'; -import type { AuthProvider, ProtonApi } from '../auth/index.js'; -import type { ConversationsConfig } from '../app/config.js'; import { getKeyManager } from './key-manager.js'; -import { initializeStore, pullIncompleteConversations, type StoreResult } from './init.js'; +import { initializeStore, pullIncompleteConversations } from './init.js'; import { ConversationStore } from './store.js'; - -// ============================================================================ -// Conversation Store Initialization -// ============================================================================ - -export interface InitializeStoreOptions { - protonApi: ProtonApi; - uid: string; - authProvider: AuthProvider; - conversationsConfig: ConversationsConfig; -} - -// Module-level state to track store result for sync initialization -let storeResult: StoreResult | null = null; +import type { InitializeStoreOptions, InitializeSyncOptions } from './types.js'; // Singleton for the active store let activeStore: ConversationStore | null = null; @@ -146,7 +114,6 @@ export async function initializeConversationStore( }); activeStore = result.conversationStore; - storeResult = result; logger.info('ConversationStore initialized'); // Pull incomplete conversations in background when sync is enabled @@ -182,18 +149,6 @@ export function setConversationStore(store: ConversationStore): void { */ export function resetConversationStore(): void { activeStore = null; - storeResult = null; -} - -// ============================================================================ -// Sync Initialization -// ============================================================================ - -export interface InitializeSyncOptions { - protonApi: ProtonApi; - uid: string; - authProvider: AuthProvider; - conversationsConfig: ConversationsConfig; } /** diff --git a/src/conversations/store.ts b/src/conversations/store.ts index 93f3eb8..f967a6b 100644 --- a/src/conversations/store.ts +++ b/src/conversations/store.ts @@ -366,24 +366,6 @@ export class ConversationStore { return message; } - /** - * Append tool calls as assistant messages (currently unused) - */ - appendAssistantToolCalls( - id: ConversationId, - toolCalls: Array<{ name: string; arguments: string; call_id: string }> - ): void { - for (const tc of toolCalls) { - const content = JSON.stringify({ - type: 'function_call', - call_id: tc.call_id, - name: tc.name, - arguments: tc.arguments, - }); - this.appendAssistantResponse(id, { content }, 'succeeded', tc.call_id); - } - } - /** * Append a single user message (CLI mode) */ diff --git a/src/conversations/types.ts b/src/conversations/types.ts index 91af60c..c5e9065 100644 --- a/src/conversations/types.ts +++ b/src/conversations/types.ts @@ -54,9 +54,32 @@ export interface IdMapEntry { /** * Incoming message format from API */ - export interface MessageForStore { role: Role; content?: string; id?: string; // Semantic ID for deduplication (call_id for tools) } + +// Re-export auth types for initialization interfaces +import type { AuthProvider, ProtonApi } from '../auth/index.js'; +import type { ConversationsConfig } from '../app/config.js'; + +/** + * Options for initializing the conversation store + */ +export interface InitializeStoreOptions { + protonApi: ProtonApi; + uid: string; + authProvider: AuthProvider; + conversationsConfig: ConversationsConfig; +} + +/** + * Options for initializing sync services + */ +export interface InitializeSyncOptions { + protonApi: ProtonApi; + uid: string; + authProvider: AuthProvider; + conversationsConfig: ConversationsConfig; +} diff --git a/tests/unit/search.test.ts b/tests/unit/search.test.ts index d382502..4de49b9 100644 --- a/tests/unit/search.test.ts +++ b/tests/unit/search.test.ts @@ -178,7 +178,6 @@ describe('formatSearchResults', () => { expect(output).toContain('Found 1 result'); expect(output).toContain('My Conversation'); expect(output).toContain('conv-123'); - expect(output).toContain('/load'); }); it('formats multiple results with snippets', () => { From 6da79019b750f7572ebbdf28257c348ebbd5ccc2 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 15:10:33 +0200 Subject: [PATCH 36/37] move store initialization to conversations/init.ts --- src/app/index.ts | 17 +---- src/conversations/index.ts | 152 ++----------------------------------- src/conversations/init.ts | 140 +++++++++++++++++++++++++++++++--- src/conversations/types.ts | 10 --- 4 files changed, 140 insertions(+), 179 deletions(-) diff --git a/src/app/index.ts b/src/app/index.ts index 743fbe8..245ae48 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -10,7 +10,7 @@ import { logger } from './logger.js'; import { resolveProjectPath } from './paths.js'; import { LumoClient } from '../lumo-client/index.js'; import { createAuthProvider, AuthManager, type AuthProvider, type ProtonApi } from '../auth/index.js'; -import { getConversationStore, setConversationStore, initializeSync, initializeConversationStore, type ConversationStore } from '../conversations/index.js'; +import { getConversationStore, setConversationStore, initializeConversationStore, type ConversationStore } from '../conversations/index.js'; import { createMockProtonApi } from '../mock/mock-api.js'; import { installFetchAdapter } from '../shims/fetch-adapter.js'; import { suppressFullApiErrors } from '../shims/console.js'; @@ -34,7 +34,6 @@ export class Application { } else { await app.initializeAuth(); await app.initializeStore(); - app.initializeSync(); } return app; } @@ -111,19 +110,9 @@ export class Application { authProvider: this.authProvider, conversationsConfig, }); - } - /** - * Initialize sync service for conversation persistence - */ - private initializeSync(): void { - const conversationsConfig = getConversationsConfig(); - this.syncInitialized = initializeSync({ - protonApi: this.protonApi, - uid: this.uid, - authProvider: this.authProvider, - conversationsConfig, - }); + // Sync is enabled if config allows and auth provider supports it + this.syncInitialized = conversationsConfig.enableSync && !this.authProvider.getSyncWarning(); } // AppContext implementation diff --git a/src/conversations/index.ts b/src/conversations/index.ts index 789eb44..ba857ec 100644 --- a/src/conversations/index.ts +++ b/src/conversations/index.ts @@ -21,12 +21,19 @@ export type { IdMapEntry, MessageForStore, InitializeStoreOptions, - InitializeSyncOptions, } from './types.js'; // Store export { ConversationStore } from './store.js'; +// Initialization and singleton management +export { + initializeConversationStore, + getConversationStore, + setConversationStore, + resetConversationStore, +} from './init.js'; + // Key management (exported for testing) export { KeyManager, @@ -34,146 +41,3 @@ export { resetKeyManager, type KeyManagerConfig, } from './key-manager.js'; - -// ============================================================================ -// Persistence initialization -// ============================================================================ - -import { logger } from '../app/logger.js'; -import { getKeyManager } from './key-manager.js'; -import { initializeStore, pullIncompleteConversations } from './init.js'; -import { ConversationStore } from './store.js'; -import type { InitializeStoreOptions, InitializeSyncOptions } from './types.js'; - -// Singleton for the active store -let activeStore: ConversationStore | null = null; - -/** - * Initialize the conversation store - * - * Creates the ConversationStore (Redux + IndexedDB) if possible. - * Logs warnings if initialization fails - callers should handle this - * gracefully (server works stateless, CLI uses local Turn array). - * - * Requires: - * - Auth provider supports persistence (has cached encryption keys) - * - keyPassword is available (for master key decryption) - */ -export async function initializeConversationStore( - options: InitializeStoreOptions -): Promise<void> { - const { authProvider, conversationsConfig } = options; - - // Check if store is disabled via config - if (!conversationsConfig.enableStore) { - logger.info('ConversationStore disabled via config'); - return; - } - - // Check if ConversationStore can be used - const storeWarning = authProvider.getConversationStoreWarning(); - if (storeWarning) { - logger.warn({ method: authProvider.method }, storeWarning); - return; - } - - // If we get here, getConversationStoreWarning() confirmed keyPassword exists - const keyPassword = authProvider.getKeyPassword()!; - - // Get cached keys from browser provider if available - const cachedUserKeys = authProvider.getCachedUserKeys?.(); - const cachedMasterKeys = authProvider.getCachedMasterKeys?.(); - - logger.info( - { - method: authProvider.method, - hasCachedUserKeys: !!cachedUserKeys, - hasCachedMasterKeys: !!cachedMasterKeys, - }, - 'Initializing KeyManager...' - ); - - // Initialize KeyManager - const keyManager = getKeyManager({ - protonApi: options.protonApi, - cachedUserKeys, - cachedMasterKeys, - }); - - try { - await keyManager.initialize(keyPassword); - - // Get master key as base64 for crypto layer - const masterKeyBase64 = keyManager.getMasterKeyBase64(); - - const result = await initializeStore({ - sessionUid: options.uid, - userId: authProvider.getUserId() ?? options.uid, - masterKey: masterKeyBase64, - projectName: conversationsConfig.projectName, - }); - - activeStore = result.conversationStore; - logger.info('ConversationStore initialized'); - - // Pull incomplete conversations in background when sync is enabled - if (conversationsConfig.enableSync) { - pullIncompleteConversations(result.store, result.spaceId) - .catch(err => logger.error({ error: err }, 'Failed to pull incomplete conversations')); - } - } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - logger.error({ error: msg }, 'Failed to initialize store. Continuing without store.'); - } -} - -/** - * Get the active conversation store - * - * Returns the initialized store, or undefined if no store is available. - * Callers should handle undefined gracefully (stateless mode). - */ -export function getConversationStore(): ConversationStore | undefined { - return activeStore ?? undefined; -} - -/** - * Set the active conversation store (for mock mode or CLI fallback) - */ -export function setConversationStore(store: ConversationStore): void { - activeStore = store; -} - -/** - * Reset the conversation store (for testing) - */ -export function resetConversationStore(): void { - activeStore = null; -} - -/** - * Initialize sync services - * - * Sync is handled automatically by Redux sagas when the store is active. - * Returns false if no store or sync is disabled. - */ -export function initializeSync(options: InitializeSyncOptions): boolean { - const { authProvider, conversationsConfig } = options; - - if (!conversationsConfig?.enableSync) { - logger.info('Sync is disabled, skipping sync initialization'); - return false; - } - - const syncWarning = authProvider.getSyncWarning(); - if (syncWarning) { - logger.warn({ method: authProvider.method }, syncWarning); - return false; - } - - logger.info( - { method: authProvider.method }, - 'Sync initialized (handled by sagas)' - ); - return true; -} diff --git a/src/conversations/init.ts b/src/conversations/init.ts index 55bc52b..07960a6 100644 --- a/src/conversations/init.ts +++ b/src/conversations/init.ts @@ -2,7 +2,7 @@ * Conversation Store Initialization * * Sets up the Redux store with saga middleware, IndexedDB persistence, - * and returns a ConversationStore. + * and provides singleton management for ConversationStore. * * This module handles: * 1. IndexedDB polyfill initialization (must happen first) @@ -10,12 +10,16 @@ * 3. Redux store setup with saga middleware * 4. Root saga startup * 5. Waiting for IDB data to load into Redux + * 6. KeyManager initialization + * 7. Singleton management */ import createSagaMiddleware from 'redux-saga'; import { logger } from '../app/logger.js'; -import type { SpaceId } from './types.js'; +import type { SpaceId, InitializeStoreOptions } from './types.js'; +import { getKeyManager } from './key-manager.js'; +import { ConversationStore } from './store.js'; import { DbApi } from '@lumo/indexedDb/db.js'; import { generateSpaceKeyBase64 } from '@lumo/crypto/index.js'; @@ -35,9 +39,124 @@ import { setupStore, type LumoSagaContext, type LumoStore } from '@lumo/redux/st import { LumoApi } from '@lumo/remote/api.js'; import type { Space } from '@lumo/types.js'; -import { ConversationStore } from './store.js'; +// ============================================================================ +// Singleton Management +// ============================================================================ + +let activeStore: ConversationStore | null = null; + +/** + * Get the active conversation store + * + * Returns the initialized store, or undefined if no store is available. + * Callers should handle undefined gracefully (stateless mode). + */ +export function getConversationStore(): ConversationStore | undefined { + return activeStore ?? undefined; +} + +/** + * Set the active conversation store (for mock mode or CLI fallback) + */ +export function setConversationStore(store: ConversationStore): void { + activeStore = store; +} + +/** + * Reset the conversation store (for testing) + */ +export function resetConversationStore(): void { + activeStore = null; +} + +// ============================================================================ +// High-Level Initialization +// ============================================================================ + +/** + * Initialize the conversation store + * + * Creates the ConversationStore (Redux + IndexedDB) if possible. + * Logs warnings if initialization fails - callers should handle this + * gracefully (server works stateless, CLI uses local Turn array). + * + * Requires: + * - Auth provider supports persistence (has cached encryption keys) + * - keyPassword is available (for master key decryption) + */ +export async function initializeConversationStore( + options: InitializeStoreOptions +): Promise<void> { + const { authProvider, conversationsConfig } = options; + + // Check if store is disabled via config + if (!conversationsConfig.enableStore) { + logger.info('ConversationStore disabled via config'); + return; + } + + // Check if ConversationStore can be used + const storeWarning = authProvider.getConversationStoreWarning(); + if (storeWarning) { + logger.warn({ method: authProvider.method }, storeWarning); + return; + } + + // If we get here, getConversationStoreWarning() confirmed keyPassword exists + const keyPassword = authProvider.getKeyPassword()!; + + // Get cached keys from browser provider if available + const cachedUserKeys = authProvider.getCachedUserKeys?.(); + const cachedMasterKeys = authProvider.getCachedMasterKeys?.(); + + logger.info( + { + method: authProvider.method, + hasCachedUserKeys: !!cachedUserKeys, + hasCachedMasterKeys: !!cachedMasterKeys, + }, + 'Initializing KeyManager...' + ); + + // Initialize KeyManager + const keyManager = getKeyManager({ + protonApi: options.protonApi, + cachedUserKeys, + cachedMasterKeys, + }); + + try { + await keyManager.initialize(keyPassword); + + // Get master key as base64 for crypto layer + const masterKeyBase64 = keyManager.getMasterKeyBase64(); + + const result = await createReduxStore({ + sessionUid: options.uid, + userId: authProvider.getUserId() ?? options.uid, + masterKey: masterKeyBase64, + projectName: conversationsConfig.projectName, + }); + + activeStore = result.conversationStore; + logger.info('ConversationStore initialized'); + + // Pull incomplete conversations in background when sync is enabled + if (conversationsConfig.enableSync) { + pullIncompleteConversations(result.store, result.spaceId) + .catch(err => logger.error({ error: err }, 'Failed to pull incomplete conversations')); + } + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + logger.error({ error: msg }, 'Failed to initialize store. Continuing without store.'); + } +} + +// ============================================================================ +// Redux Store Setup (Internal) +// ============================================================================ -export interface StoreConfig { +interface ReduxStoreConfig { /** Session UID for API authentication (x-pm-uid header) */ sessionUid: string; /** Stable user ID for database naming (userKeys[0].ID) */ @@ -47,7 +166,7 @@ export interface StoreConfig { projectName: string; } -export interface StoreResult { +interface ReduxStoreResult { store: LumoStore; conversationStore: ConversationStore; dbApi: DbApi; @@ -55,7 +174,7 @@ export interface StoreResult { } /** - * Initialize the upstream storage system + * Create the Redux-backed store infrastructure * * This sets up: * - IndexedDB (via indexeddbshim) for local persistence @@ -67,9 +186,9 @@ export interface StoreResult { * 1. Find existing space by projectName in Redux state * 2. Create new space with projectName if no match */ -export async function initializeStore( - config: StoreConfig -): Promise<StoreResult> { +async function createReduxStore( + config: ReduxStoreConfig +): Promise<ReduxStoreResult> { const { sessionUid, userId, masterKey, projectName } = config; logger.info({ userId: userId.slice(0, 8) + '...' }, 'Initializing upstream storage'); @@ -164,7 +283,6 @@ async function waitForReduxLoaded( * The initAppSaga triggers pullSpacesRequest after loading from IDB. * We need to wait for that to complete before checking if our space exists, * otherwise we might create a local space that conflicts with a remote one. - * TODO: this looks like a good thing to have on a generic level */ async function waitForRemoteSpaces( store: LumoStore, @@ -278,7 +396,7 @@ function findOrCreateSpace( * Note: This does NOT handle messages that fail to decrypt (e.g., key mismatch * or corruption). Those remain in IDB but are skipped during loadReduxFromIdb. */ -export async function pullIncompleteConversations( +async function pullIncompleteConversations( store: LumoStore, spaceId: SpaceId ): Promise<void> { diff --git a/src/conversations/types.ts b/src/conversations/types.ts index c5e9065..6ff7188 100644 --- a/src/conversations/types.ts +++ b/src/conversations/types.ts @@ -73,13 +73,3 @@ export interface InitializeStoreOptions { authProvider: AuthProvider; conversationsConfig: ConversationsConfig; } - -/** - * Options for initializing sync services - */ -export interface InitializeSyncOptions { - protonApi: ProtonApi; - uid: string; - authProvider: AuthProvider; - conversationsConfig: ConversationsConfig; -} From 4ba7c54d2b9ff7161932e4113e096f7296ad63e2 Mon Sep 17 00:00:00 2001 From: ZeroTricks <11061483+ZeroTricks@users.noreply.github.com> Date: Thu, 2 Apr 2026 15:16:10 +0200 Subject: [PATCH 37/37] Remove impossible and incomplte in-chat /load command --- src/app/commands.ts | 37 ------------------------------------- src/conversations/search.ts | 2 -- 2 files changed, 39 deletions(-) diff --git a/src/app/commands.ts b/src/app/commands.ts index 3d020a9..6ffb77c 100644 --- a/src/app/commands.ts +++ b/src/app/commands.ts @@ -104,9 +104,6 @@ export async function executeCommand( case 'save': return await handleSaveCommand(params, context); - case 'load': - return await handleLoadCommand(params, context); - case 'search': return handleSearchCommand(params, context); @@ -146,7 +143,6 @@ function getHelpText(): string { /help - Show this help message /title <text> - Set conversation title /save [title] - Save stateless request to conversation (optionally set title) - /load <id> - Load a conversation from Proton by ID /search <query> - Search conversation titles and messages /refreshtokens - Manually refresh auth tokens /logout - Revoke session and delete tokens @@ -231,39 +227,6 @@ async function handleSaveCommand(params: string, context?: CommandContext): Prom } } -/** - * Handle /load command - load a conversation by ID - * - * With the primary store, conversations are loaded from IndexedDB automatically. - * This command provides info about an existing conversation. - */ -async function handleLoadCommand(params: string, context?: CommandContext): Promise<string> { - try { - const localId = params.trim(); - if (!localId) { - return 'Usage: /load <id>\nExample: /load f0654976-d628-4516-8e80-a0599b6593ac'; - } - - const store = getConversationStore(); - if (!store) { - return 'Conversation store not available.'; - } - - const conversation = store.get(localId); - if (!conversation) { - return `Conversation not found: ${localId}`; - } - - const messageCount = conversation.messages.length ?? 0; - const title = conversation.title ?? 'Untitled'; - - return `Loaded conversation: ${title}\nLocal ID: ${localId}\nMessages: ${messageCount}`; - } catch (error) { - logger.error({ error }, 'Failed to execute /load command'); - return `Load failed: ${error instanceof Error ? error.message : 'Unknown error'}`; - } -} - /** * Handle /refreshtokens command - manually trigger token refresh */ diff --git a/src/conversations/search.ts b/src/conversations/search.ts index 77b697b..8805a51 100644 --- a/src/conversations/search.ts +++ b/src/conversations/search.ts @@ -169,7 +169,5 @@ export function formatSearchResults(results: SearchResult[], query: string): str lines.push(''); } - // lines.push('Use /load <id> to open a conversation'); - return lines.join('\n'); }