From 8653d0bb29b8e9b0091fa67fd43dc924c5b1524f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 23 Mar 2026 08:36:56 +0000 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20add=20deja-edge=20=E2=80=94=20FTS5-?= =?UTF-8?q?powered=20memory=20for=20Cloudflare=20Durable=20Objects?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero external dependencies (no Vectorize, no embeddings). Pure SQLite FTS5 full-text search with BM25 ranking, confidence scoring, dedup/conflict resolution, and audit logging. Exports: - createEdgeMemory(ctx) — core memory store for use inside any DO - DejaEdgeDO — ready-to-use DO class with HTTP routes https://claude.ai/code/session_01Fb57JFi8VM36x8pCM1BFTw --- packages/deja-edge/package.json | 50 +++ packages/deja-edge/src/do.ts | 109 +++++ packages/deja-edge/src/index.ts | 421 ++++++++++++++++++++ packages/deja-edge/test/edge-memory.test.ts | 250 ++++++++++++ packages/deja-edge/tsconfig.json | 16 + 5 files changed, 846 insertions(+) create mode 100644 packages/deja-edge/package.json create mode 100644 packages/deja-edge/src/do.ts create mode 100644 packages/deja-edge/src/index.ts create mode 100644 packages/deja-edge/test/edge-memory.test.ts create mode 100644 packages/deja-edge/tsconfig.json diff --git a/packages/deja-edge/package.json b/packages/deja-edge/package.json new file mode 100644 index 0000000..137ffb8 --- /dev/null +++ b/packages/deja-edge/package.json @@ -0,0 +1,50 @@ +{ + "name": "deja-edge", + "version": "0.1.0", + "description": "Edge memory for Cloudflare Durable Objects. FTS5-powered full-text search, zero external dependencies.", + "main": "dist/index.js", + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "./do": { + "types": "./dist/do.d.ts", + "import": "./dist/do.mjs", + "require": "./dist/do.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsup src/index.ts src/do.ts --format cjs,esm --dts --clean", + "test": "bun test", + "lint": "tsc --noEmit", + "prepublishOnly": "bun run lint && bun run test && bun run build" + }, + "keywords": [ + "ai", + "agents", + "memory", + "fts5", + "edge", + "cloudflare", + "durable-objects", + "deja" + ], + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/acoyfellow/deja" + }, + "devDependencies": { + "tsup": "^8.0.0", + "typescript": "^5.4.0", + "@cloudflare/workers-types": "^4.20250312.0" + } +} diff --git a/packages/deja-edge/src/do.ts b/packages/deja-edge/src/do.ts new file mode 100644 index 0000000..62ac635 --- /dev/null +++ b/packages/deja-edge/src/do.ts @@ -0,0 +1,109 @@ +/** + * Ready-to-use Durable Object class with HTTP routes. + * + * ```ts + * // wrangler.json: + * // { "durable_objects": { "bindings": [{ "name": "MEMORY", "class_name": "DejaEdgeDO" }] } } + * // { "migrations": [{ "tag": "v1", "new_sqlite_classes": ["DejaEdgeDO"] }] } + * + * // worker.ts: + * export { DejaEdgeDO } from 'deja-edge/do' + * ``` + */ + +import { createEdgeMemory, type EdgeMemoryStore, type CreateEdgeMemoryOptions } from './index' + +export class DejaEdgeDO extends DurableObject { + private memory: EdgeMemoryStore + + constructor(ctx: DurableObjectState, env: unknown) { + super(ctx, env) + this.memory = createEdgeMemory(ctx) + } + + async fetch(request: Request): Promise { + const url = new URL(request.url) + const path = url.pathname + const method = request.method + + try { + // POST /remember — store a memory + if (method === 'POST' && path === '/remember') { + const body = await request.json<{ text: string }>() + if (!body.text) return json({ error: 'text is required' }, 400) + const result = this.memory.remember(body.text) + return json(result, 201) + } + + // POST /recall — search memories + if (method === 'POST' && path === '/recall') { + const body = await request.json<{ context: string; limit?: number; threshold?: number; minConfidence?: number }>() + if (!body.context) return json({ error: 'context is required' }, 400) + const results = this.memory.recall(body.context, { + limit: body.limit, + threshold: body.threshold, + minConfidence: body.minConfidence, + }) + return json(results) + } + + // POST /confirm/:id + if (method === 'POST' && path.startsWith('/confirm/')) { + const id = path.slice('/confirm/'.length) + const ok = this.memory.confirm(id) + return ok ? json({ ok: true }) : json({ error: 'not found' }, 404) + } + + // POST /reject/:id + if (method === 'POST' && path.startsWith('/reject/')) { + const id = path.slice('/reject/'.length) + const ok = this.memory.reject(id) + return ok ? json({ ok: true }) : json({ error: 'not found' }, 404) + } + + // DELETE /forget/:id + if (method === 'DELETE' && path.startsWith('/forget/')) { + const id = path.slice('/forget/'.length) + const ok = this.memory.forget(id) + return ok ? json({ ok: true }) : json({ error: 'not found' }, 404) + } + + // GET /list + if (method === 'GET' && path === '/list') { + const limit = parseInt(url.searchParams.get('limit') ?? '100') + const offset = parseInt(url.searchParams.get('offset') ?? '0') + return json(this.memory.list({ limit, offset })) + } + + // GET /recall-log + if (method === 'GET' && path === '/recall-log') { + const limit = parseInt(url.searchParams.get('limit') ?? '50') + return json(this.memory.recallLog({ limit })) + } + + // GET /size + if (method === 'GET' && path === '/size') { + return json({ size: this.memory.size }) + } + + // GET / — health + if (method === 'GET' && path === '/') { + return json({ status: 'ok', service: 'deja-edge', size: this.memory.size }) + } + + return json({ error: 'not found' }, 404) + } catch (err) { + const message = err instanceof Error ? err.message : 'internal error' + return json({ error: message }, 500) + } + } +} + +function json(data: unknown, status = 200): Response { + return new Response(JSON.stringify(data), { + status, + headers: { 'Content-Type': 'application/json' }, + }) +} + +export default DejaEdgeDO diff --git a/packages/deja-edge/src/index.ts b/packages/deja-edge/src/index.ts new file mode 100644 index 0000000..62d9829 --- /dev/null +++ b/packages/deja-edge/src/index.ts @@ -0,0 +1,421 @@ +/** + * deja-edge — Edge memory for Cloudflare Durable Objects. + * + * FTS5-powered full-text search. No Vectorize. No embeddings. Pure text matching at the edge. + * + * ```ts + * // In your Durable Object constructor: + * import { createEdgeMemory } from 'deja-edge' + * + * export class MyDO extends DurableObject { + * private memory: EdgeMemoryStore + * constructor(ctx: DurableObjectState, env: Env) { + * super(ctx, env) + * this.memory = createEdgeMemory(ctx) + * } + * + * async remember(text: string) { return this.memory.remember(text) } + * async recall(context: string) { return this.memory.recall(context) } + * } + * ``` + */ + +// ============================================================================ +// Types +// ============================================================================ + +export interface Memory { + id: string + text: string + confidence: number + supersedes?: string + createdAt: string +} + +export interface RecallResult { + id: string + text: string + score: number + confidence: number + createdAt: string +} + +export interface RecallLogEntry { + id: number + context: string + results: Array<{ memoryId: string; score: number }> + timestamp: string +} + +export interface EdgeMemoryStore { + /** Store a memory. Deduplicates automatically via FTS5 similarity. */ + remember(text: string): Memory + + /** Find relevant memories via FTS5 full-text search. */ + recall(context: string, options?: RecallOptions): RecallResult[] + + /** Signal that a recalled memory was useful. Boosts its confidence. */ + confirm(id: string): boolean + + /** Signal that a recalled memory was wrong or outdated. Drops its confidence. */ + reject(id: string): boolean + + /** Remove a memory by id. */ + forget(id: string): boolean + + /** All memories, newest first. */ + list(options?: { limit?: number; offset?: number }): Memory[] + + /** View the recall audit log. */ + recallLog(options?: { limit?: number }): RecallLogEntry[] + + /** How many memories are stored. */ + readonly size: number +} + +export interface RecallOptions { + limit?: number + /** Minimum BM25 score (lower is better in raw BM25; we negate so higher = better). Default: 0 (return all matches) */ + threshold?: number + minConfidence?: number +} + +export interface CreateEdgeMemoryOptions { + /** Minimum confidence to return in recall. Default: 0 */ + minConfidence?: number + /** Similarity threshold for dedup (0-1 via trigram Jaccard). Default: 0.85 */ + dedupeThreshold?: number + /** Similarity range for conflict detection. Default: 0.5 */ + conflictThreshold?: number +} + +// ============================================================================ +// Stop words — filtered from FTS5 queries for better matching +// ============================================================================ + +const STOP_WORDS = new Set([ + 'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', + 'of', 'with', 'by', 'from', 'is', 'it', 'as', 'be', 'was', 'are', + 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', + 'would', 'could', 'should', 'may', 'might', 'can', 'this', 'that', + 'these', 'those', 'i', 'you', 'he', 'she', 'we', 'they', 'my', 'me', + 'what', 'how', 'when', 'where', 'which', 'who', 'all', 'about', + 'up', 'out', 'if', 'not', 'no', 'so', 'just', 'get', 'make', + 'full', 'before', 'after', 'every', 'any', 'some', +]) + +// ============================================================================ +// Text utilities +// ============================================================================ + +/** Extract meaningful keywords from text, filtering stop words */ +function extractKeywords(text: string): string[] { + return text + .toLowerCase() + .replace(/[^a-z0-9\s-]/g, '') + .split(/\s+/) + .filter(w => w.length > 1 && !STOP_WORDS.has(w)) +} + +/** Build an FTS5 query from natural language context */ +function buildFtsQuery(context: string): string { + const keywords = extractKeywords(context) + if (keywords.length === 0) return '' + + // Use OR to match any keyword — FTS5 ranks by relevance automatically + return keywords.map(k => `"${k}"`).join(' OR ') +} + +/** Simple trigram-based Jaccard similarity for dedup/conflict detection */ +function trigramSimilarity(a: string, b: string): number { + const trigramsOf = (s: string): Set => { + const t = new Set() + const lower = s.toLowerCase() + for (let i = 0; i <= lower.length - 3; i++) t.add(lower.slice(i, i + 3)) + return t + } + const ta = trigramsOf(a) + const tb = trigramsOf(b) + if (ta.size === 0 && tb.size === 0) return 1 + let intersection = 0 + for (const t of ta) if (tb.has(t)) intersection++ + return intersection / (ta.size + tb.size - intersection) +} + +// ============================================================================ +// Confidence scoring +// ============================================================================ + +const CONFIDENCE_DEFAULT = 0.5 +const CONFIDENCE_BOOST = 0.1 +const CONFIDENCE_DECAY = 0.15 +const CONFIDENCE_MIN = 0.01 +const CONFIDENCE_MAX = 1.0 + +function clampConfidence(c: number): number { + return Math.min(CONFIDENCE_MAX, Math.max(CONFIDENCE_MIN, Math.round(c * 1000) / 1000)) +} + +// ============================================================================ +// ID generation +// ============================================================================ + +function createId(): string { + return `${Date.now()}-${Math.random().toString(36).slice(2, 11)}` +} + +// ============================================================================ +// Schema +// ============================================================================ + +function initSchema(sql: DurableObjectState['storage']['sql']) { + sql.exec(` + CREATE TABLE IF NOT EXISTS memories ( + id TEXT PRIMARY KEY, + text TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.5, + supersedes TEXT, + created_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at); + CREATE INDEX IF NOT EXISTS idx_memories_confidence ON memories(confidence); + + CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( + text, + content='memories', + content_rowid='rowid', + tokenize='porter unicode61' + ); + + CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN + INSERT INTO memories_fts(rowid, text) VALUES (new.rowid, new.text); + END; + CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, text) VALUES('delete', old.rowid, old.text); + END; + CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN + INSERT INTO memories_fts(memories_fts, rowid, text) VALUES('delete', old.rowid, old.text); + INSERT INTO memories_fts(rowid, text) VALUES (new.rowid, new.text); + END; + + CREATE TABLE IF NOT EXISTS recall_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + context TEXT NOT NULL, + results TEXT NOT NULL, + timestamp TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_recall_log_ts ON recall_log(timestamp); + `) +} + +// ============================================================================ +// createEdgeMemory +// ============================================================================ + +export function createEdgeMemory( + ctx: DurableObjectState, + opts: CreateEdgeMemoryOptions = {}, +): EdgeMemoryStore { + const dedupeThreshold = opts.dedupeThreshold ?? 0.85 + const conflictThreshold = opts.conflictThreshold ?? 0.5 + + const sql = ctx.storage.sql + + // Initialize schema (idempotent) + initSchema(sql) + + function remember(text: string): Memory { + const trimmed = text.trim() + if (!trimmed) throw new Error('Memory text cannot be empty') + + // Check for dedup/conflict against existing memories via FTS5 + const keywords = extractKeywords(trimmed) + if (keywords.length > 0) { + const ftsQuery = keywords.map(k => `"${k}"`).join(' OR ') + const candidates = [ + ...sql.exec<{ id: string; text: string; confidence: number; supersedes: string | null; created_at: string }>( + `SELECT m.id, m.text, m.confidence, m.supersedes, m.created_at + FROM memories m + JOIN memories_fts ON memories_fts.rowid = m.rowid + WHERE memories_fts MATCH ? + LIMIT 10`, + ftsQuery, + ), + ] + + let bestSim = 0 + let bestCandidate: typeof candidates[0] | null = null + for (const c of candidates) { + const sim = trigramSimilarity(trimmed, c.text) + if (sim > bestSim) { + bestSim = sim + bestCandidate = c + } + } + + // Dedup: near-identical + if (bestCandidate && bestSim >= dedupeThreshold) { + return { + id: bestCandidate.id, + text: bestCandidate.text, + confidence: bestCandidate.confidence, + createdAt: bestCandidate.created_at, + supersedes: bestCandidate.supersedes ?? undefined, + } + } + + // Conflict: same topic, different content — supersede + if (bestCandidate && bestSim >= conflictThreshold) { + const newConf = clampConfidence(bestCandidate.confidence * 0.3) + sql.exec(`UPDATE memories SET confidence = ? WHERE id = ?`, newConf, bestCandidate.id) + + const id = createId() + const createdAt = new Date().toISOString() + sql.exec( + `INSERT INTO memories (id, text, confidence, supersedes, created_at) VALUES (?, ?, ?, ?, ?)`, + id, trimmed, CONFIDENCE_DEFAULT, bestCandidate.id, createdAt, + ) + return { id, text: trimmed, confidence: CONFIDENCE_DEFAULT, supersedes: bestCandidate.id, createdAt } + } + } + + // New memory — no dedup or conflict + const id = createId() + const createdAt = new Date().toISOString() + sql.exec( + `INSERT INTO memories (id, text, confidence, supersedes, created_at) VALUES (?, ?, ?, ?, ?)`, + id, trimmed, CONFIDENCE_DEFAULT, null, createdAt, + ) + return { id, text: trimmed, confidence: CONFIDENCE_DEFAULT, createdAt } + } + + function recall(context: string, options: RecallOptions = {}): RecallResult[] { + const limit = options.limit ?? 5 + const minConf = options.minConfidence ?? 0 + const threshold = options.threshold ?? 0 + + const ftsQuery = buildFtsQuery(context) + if (!ftsQuery) return [] + + // FTS5 bm25() returns negative scores (lower = more relevant) + // We negate to make higher = better, then blend with confidence + const rows = [ + ...sql.exec<{ + id: string + text: string + confidence: number + created_at: string + rank: number + }>( + `SELECT m.id, m.text, m.confidence, m.created_at, bm25(memories_fts) as rank + FROM memories_fts + JOIN memories m ON memories_fts.rowid = m.rowid + WHERE memories_fts MATCH ? + AND m.confidence >= ? + ORDER BY rank + LIMIT ?`, + ftsQuery, minConf, limit * 3, // over-fetch then re-rank with confidence + ), + ] + + if (rows.length === 0) return [] + + // Normalize BM25 scores to 0-1 range, then blend with confidence + const maxRank = Math.max(...rows.map(r => -r.rank)) + const minRank = Math.min(...rows.map(r => -r.rank)) + const range = maxRank - minRank || 1 + + const results: RecallResult[] = rows.map(r => { + const normalizedRelevance = (-r.rank - minRank) / range + const blended = normalizedRelevance * 0.7 + r.confidence * 0.3 + return { + id: r.id, + text: r.text, + score: Math.round(blended * 1000) / 1000, + confidence: r.confidence, + createdAt: r.created_at, + } + }) + + results.sort((a, b) => b.score - a.score) + const topResults = results.filter(r => r.score >= threshold).slice(0, limit) + + // Audit log + const now = new Date().toISOString() + const logData = topResults.map(r => ({ memoryId: r.id, score: r.score })) + sql.exec( + `INSERT INTO recall_log (context, results, timestamp) VALUES (?, ?, ?)`, + context, JSON.stringify(logData), now, + ) + + return topResults + } + + const store: EdgeMemoryStore = { + get size(): number { + const row = [...sql.exec<{ count: number }>('SELECT COUNT(*) as count FROM memories')] + return row[0]?.count ?? 0 + }, + + remember, + recall, + + confirm(id: string): boolean { + const rows = [...sql.exec<{ confidence: number }>('SELECT confidence FROM memories WHERE id = ?', id)] + if (rows.length === 0) return false + const newConf = clampConfidence(rows[0].confidence + CONFIDENCE_BOOST) + sql.exec('UPDATE memories SET confidence = ? WHERE id = ?', newConf, id) + return true + }, + + reject(id: string): boolean { + const rows = [...sql.exec<{ confidence: number }>('SELECT confidence FROM memories WHERE id = ?', id)] + if (rows.length === 0) return false + const newConf = clampConfidence(rows[0].confidence - CONFIDENCE_DECAY) + sql.exec('UPDATE memories SET confidence = ? WHERE id = ?', newConf, id) + return true + }, + + forget(id: string): boolean { + const before = store.size + sql.exec('DELETE FROM memories WHERE id = ?', id) + return store.size < before + }, + + list(options = {}): Memory[] { + const limit = options.limit ?? 1000 + const offset = options.offset ?? 0 + return [ + ...sql.exec<{ id: string; text: string; confidence: number; supersedes: string | null; created_at: string }>( + 'SELECT id, text, confidence, supersedes, created_at FROM memories ORDER BY created_at DESC LIMIT ? OFFSET ?', + limit, offset, + ), + ].map(r => ({ + id: r.id, + text: r.text, + confidence: r.confidence, + supersedes: r.supersedes ?? undefined, + createdAt: r.created_at, + })) + }, + + recallLog(options = {}): RecallLogEntry[] { + const limit = options.limit ?? 50 + return [ + ...sql.exec<{ id: number; context: string; results: string; timestamp: string }>( + 'SELECT id, context, results, timestamp FROM recall_log ORDER BY timestamp DESC LIMIT ?', + limit, + ), + ].map(r => ({ + id: r.id, + context: r.context, + results: JSON.parse(r.results), + timestamp: r.timestamp, + })) + }, + } + + return store +} + +export default createEdgeMemory diff --git a/packages/deja-edge/test/edge-memory.test.ts b/packages/deja-edge/test/edge-memory.test.ts new file mode 100644 index 0000000..85ff519 --- /dev/null +++ b/packages/deja-edge/test/edge-memory.test.ts @@ -0,0 +1,250 @@ +import { describe, test, expect } from 'bun:test' +import { Database } from 'bun:sqlite' +import { createEdgeMemory, type EdgeMemoryStore } from '../src/index' + +/** + * Create a mock DurableObjectState backed by bun:sqlite. + * The DO sql.exec() returns an iterable cursor — we simulate that + * by returning the array directly (which is iterable). + */ +function createMockCtx() { + const db = new Database(':memory:') + db.exec('PRAGMA journal_mode = WAL') + + const mockSql = { + exec>(query: string, ...bindings: unknown[]): T[] { + const trimmed = query.trim() + + // Multi-statement DDL (CREATE TABLE, triggers, etc.) + if (isMultiStatement(trimmed)) { + db.exec(trimmed) + return [] as T[] + } + + // Single statements + if ( + trimmed.toUpperCase().startsWith('INSERT') || + trimmed.toUpperCase().startsWith('UPDATE') || + trimmed.toUpperCase().startsWith('DELETE') || + trimmed.toUpperCase().startsWith('CREATE') || + trimmed.toUpperCase().startsWith('DROP') + ) { + const stmt = db.prepare(trimmed) + stmt.run(...(bindings as any[])) + return [] as T[] + } + + // SELECT + const stmt = db.prepare(trimmed) + return stmt.all(...(bindings as any[])) as T[] + }, + } + + const state = { + blockConcurrencyWhile: async (fn: () => Promise) => fn(), + storage: { + sql: mockSql, + }, + } + + return { state: state as unknown as DurableObjectState, db } +} + +function isMultiStatement(sql: string): boolean { + // Strip string literals and comments, then check for multiple semicolons + const stripped = sql.replace(/'[^']*'/g, '').replace(/--[^\n]*/g, '') + const statements = stripped.split(';').filter(s => s.trim().length > 0) + return statements.length > 1 +} + +describe('deja-edge: createEdgeMemory', () => { + let memory: EdgeMemoryStore + let ctx: ReturnType + + function freshMemory(opts = {}) { + ctx = createMockCtx() + memory = createEdgeMemory(ctx.state, opts) + return memory + } + + test('starts empty', () => { + freshMemory() + expect(memory.size).toBe(0) + expect(memory.list()).toEqual([]) + }) + + test('remember stores a memory', () => { + freshMemory() + const m = memory.remember('always check wrangler.toml before deploying') + expect(m.id).toBeTruthy() + expect(m.text).toBe('always check wrangler.toml before deploying') + expect(m.confidence).toBe(0.5) + expect(m.createdAt).toBeTruthy() + expect(memory.size).toBe(1) + }) + + test('remember rejects empty text', () => { + freshMemory() + expect(() => memory.remember('')).toThrow('empty') + expect(() => memory.remember(' ')).toThrow('empty') + }) + + test('list returns memories newest first', () => { + freshMemory() + memory.remember('first memory') + memory.remember('second memory') + memory.remember('third memory') + const list = memory.list() + expect(list.length).toBe(3) + expect(list[0].text).toBe('third memory') + expect(list[2].text).toBe('first memory') + }) + + test('recall finds relevant memories via FTS5', () => { + freshMemory() + memory.remember('always check wrangler.toml before deploying') + memory.remember('use npm run test before pushing code') + memory.remember('database migrations need review by senior dev') + + const results = memory.recall('deploying to production') + expect(results.length).toBeGreaterThan(0) + expect(results[0].text).toContain('deploying') + }) + + test('recall returns empty for no matches', () => { + freshMemory() + memory.remember('check wrangler config') + const results = memory.recall('quantum physics experiments') + expect(results.length).toBe(0) + }) + + test('recall respects limit', () => { + freshMemory() + for (let i = 0; i < 10; i++) { + memory.remember(`memory about testing approach number ${i}`) + } + const results = memory.recall('testing approach', { limit: 3 }) + expect(results.length).toBeLessThanOrEqual(3) + }) + + test('recall filters by minConfidence', () => { + freshMemory() + memory.remember('high confidence item about deployment') + const m = memory.remember('low confidence item about deployment scripts') + // Reject it twice to drop confidence + memory.reject(m.id) + memory.reject(m.id) + + const results = memory.recall('deployment', { minConfidence: 0.4 }) + for (const r of results) { + expect(r.confidence).toBeGreaterThanOrEqual(0.4) + } + }) + + test('confirm boosts confidence', () => { + freshMemory() + const m = memory.remember('test memory for confidence boost') + expect(m.confidence).toBe(0.5) + + memory.confirm(m.id) + const list = memory.list() + expect(list[0].confidence).toBe(0.6) + + memory.confirm(m.id) + const list2 = memory.list() + expect(list2[0].confidence).toBe(0.7) + }) + + test('reject drops confidence', () => { + freshMemory() + const m = memory.remember('test memory for confidence drop') + memory.reject(m.id) + const list = memory.list() + expect(list[0].confidence).toBe(0.35) + }) + + test('confidence is clamped to [0.01, 1.0]', () => { + freshMemory() + const m = memory.remember('clamp test memory') + + // Boost many times — should cap at 1.0 + for (let i = 0; i < 20; i++) memory.confirm(m.id) + let list = memory.list() + expect(list[0].confidence).toBe(1.0) + + // Reject many times — should floor at 0.01 + for (let i = 0; i < 30; i++) memory.reject(m.id) + list = memory.list() + expect(list[0].confidence).toBe(0.01) + }) + + test('confirm/reject return false for unknown id', () => { + freshMemory() + expect(memory.confirm('nonexistent')).toBe(false) + expect(memory.reject('nonexistent')).toBe(false) + }) + + test('forget removes a memory', () => { + freshMemory() + const m = memory.remember('memory to forget') + expect(memory.size).toBe(1) + const ok = memory.forget(m.id) + expect(ok).toBe(true) + expect(memory.size).toBe(0) + }) + + test('forget returns false for unknown id', () => { + freshMemory() + expect(memory.forget('nonexistent')).toBe(false) + }) + + test('recallLog tracks recall queries', () => { + freshMemory() + memory.remember('log test memory about deploying') + memory.recall('deploying') + + const log = memory.recallLog() + expect(log.length).toBe(1) + expect(log[0].context).toBe('deploying') + expect(Array.isArray(log[0].results)).toBe(true) + }) + + test('dedup: near-identical memories return existing', () => { + freshMemory() + const m1 = memory.remember('check wrangler.toml before deploying') + const m2 = memory.remember('check wrangler.toml before deploying') + expect(m2.id).toBe(m1.id) // same memory returned + expect(memory.size).toBe(1) // no duplicate + }) + + test('conflict: similar but different text supersedes old memory', () => { + freshMemory({ conflictThreshold: 0.4 }) + const m1 = memory.remember('check wrangler.toml before deploying to staging') + const m2 = memory.remember('check wrangler.toml before deploying to production') + + // m2 should supersede m1 + if (m2.supersedes) { + expect(m2.supersedes).toBe(m1.id) + // Old memory confidence should be reduced + const list = memory.list() + const old = list.find(m => m.id === m1.id) + expect(old!.confidence).toBeLessThan(0.5) + } + // Either way, we should have at most 2 memories + expect(memory.size).toBeLessThanOrEqual(2) + }) + + test('list supports pagination', () => { + freshMemory() + memory.remember('alpha memory about cats') + memory.remember('beta memory about dogs') + memory.remember('gamma memory about birds') + memory.remember('delta memory about fish') + expect(memory.size).toBe(4) + const page1 = memory.list({ limit: 2, offset: 0 }) + const page2 = memory.list({ limit: 2, offset: 2 }) + expect(page1.length).toBe(2) + expect(page2.length).toBe(2) + expect(page1[0].id).not.toBe(page2[0].id) + }) +}) diff --git a/packages/deja-edge/tsconfig.json b/packages/deja-edge/tsconfig.json new file mode 100644 index 0000000..9f1eabf --- /dev/null +++ b/packages/deja-edge/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "declaration": true, + "resolveJsonModule": true, + "types": ["@cloudflare/workers-types"] + }, + "include": ["src"], + "exclude": ["node_modules", "dist", "test"] +} From a1e8ec8114141e50e231d84ce9df62fa2f934b97 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 23 Mar 2026 08:47:23 +0000 Subject: [PATCH 2/4] fix(deja-local): address PR #9 review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Mark package as Bun-only (engines field + description) since bun:sqlite cannot be resolved by Node.js consumers. 2. Upgrade synchronous pragma from NORMAL to FULL to prevent data loss on host crash with WAL mode — honoring the durability promise. https://claude.ai/code/session_01Fb57JFi8VM36x8pCM1BFTw --- packages/deja-local/package.json | 5 ++++- packages/deja-local/src/index.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/deja-local/package.json b/packages/deja-local/package.json index b09e7de..a644c80 100644 --- a/packages/deja-local/package.json +++ b/packages/deja-local/package.json @@ -1,7 +1,7 @@ { "name": "deja-local", "version": "0.1.0", - "description": "Local in-process vector memory for agents. Zero latency, zero eventual consistency.", + "description": "Local in-process vector memory for agents. Zero latency, zero eventual consistency. Requires Bun runtime.", "main": "dist/index.js", "module": "dist/index.mjs", "types": "dist/index.d.ts", @@ -30,6 +30,9 @@ "embedding", "deja" ], + "engines": { + "bun": ">=1.0.0" + }, "author": "", "license": "MIT", "repository": { diff --git a/packages/deja-local/src/index.ts b/packages/deja-local/src/index.ts index 5a35e76..68bceb4 100644 --- a/packages/deja-local/src/index.ts +++ b/packages/deja-local/src/index.ts @@ -234,7 +234,7 @@ export function createMemory(opts: CreateMemoryOptions): MemoryStore { // Open database, enable WAL mode, create schema const db = new Database(opts.path) db.exec('PRAGMA journal_mode = WAL') - db.exec('PRAGMA synchronous = NORMAL') + db.exec('PRAGMA synchronous = FULL') db.exec(SCHEMA) migrateSchema(db) From 21506049716276a5e9fdf549d1c0023cd8bad339 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 23 Mar 2026 08:53:02 +0000 Subject: [PATCH 3/4] docs: rewrite documentation for three memory systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root README now leads with a comparison table of deja-local, deja-edge, and deja (hosted) — what each is, when to use it, and how to install. - Add deja-edge README - Trim deja-local README (add Bun requirement, remove redundancy) - Trim deja-client README (remove type definitions that TypeScript provides) - Cut total doc volume — clarity over completeness https://claude.ai/code/session_01Fb57JFi8VM36x8pCM1BFTw --- README.md | 126 +++++++++++++++++----------- packages/deja-client/README.md | 147 ++++----------------------------- packages/deja-edge/README.md | 132 +++++++++++++++++++++++++++++ packages/deja-local/README.md | 125 +++++++--------------------- 4 files changed, 254 insertions(+), 276 deletions(-) create mode 100644 packages/deja-edge/README.md diff --git a/README.md b/README.md index a3bf59e..3001c74 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,73 @@ # deja -> Persistent memory for agents. Store learnings, recall context. +Persistent memory for AI agents. Agents learn from runs — deja remembers across them. -[Docs](https://deja.coey.dev/docs) · [Quickstart](https://deja.coey.dev/guides/quickstart) +## Three ways to use deja -## What Deja does +| | **deja-local** | **deja-edge** | **deja (hosted)** | +|---|---|---|---| +| **What it is** | In-process SQLite memory | Cloudflare Durable Object memory | Hosted Cloudflare Worker service | +| **Search** | Vector embeddings (all-MiniLM-L6-v2) | FTS5 full-text search | Vectorize + Workers AI embeddings | +| **Runtime** | Bun | Cloudflare Workers | Cloudflare Workers | +| **Dependencies** | None (runs locally) | None (runs in your DO) | Vectorize index + Workers AI | +| **Latency** | Zero (in-process) | Zero (in-DO) | Network round-trip | +| **Install** | `npm install deja-local` | `npm install deja-edge` | Clone + `wrangler deploy` | +| **Best for** | Local agents, scripts, CLI tools | Edge agents running in Workers | Multi-tenant, team-shared memory | -Agents learn from runs. Deja remembers across them. +### deja-local -- **Learn** — store a learning with trigger, context, and confidence -- **Recall** — semantic search returns relevant learnings before the next run -- **Working state** — live snapshots and event streams for in-progress work -- **Scoped** — learnings are isolated by scope (`shared`, `agent:`, `session:`, or custom) +Memory lives in a local SQLite file. Vector search via ONNX embeddings on CPU. No network, no API keys. Requires Bun. -## Install +```ts +import { createMemory } from 'deja-local' -### As part of filepath (recommended) +const mem = createMemory({ path: './agent.db' }) +await mem.remember('always run migrations before deploying') +const results = await mem.recall('deploying to staging') +``` -Deja is an npm-installable Cloudflare Worker. When used with [filepath](https://github.com/acoyfellow/filepath), add it to `alchemy.run.ts` and it deploys alongside your filepath instance. See the [filepath README](https://github.com/acoyfellow/filepath#how-to-enable-memory-deja) for setup. +[Full docs →](./packages/deja-local/) -### Standalone +### deja-edge -```bash -git clone https://github.com/acoyfellow/deja -cd deja -bun install -wrangler login -wrangler vectorize create deja-embeddings --dimensions 384 --metric cosine -wrangler secret put API_KEY -bun run deploy -``` +Memory lives in a Cloudflare Durable Object's SQLite. FTS5 full-text search with BM25 ranking. No Vectorize, no Workers AI — just text matching. Zero external dependencies. -## Connect +```ts +import { createEdgeMemory } from 'deja-edge' -### REST +export class MyDO extends DurableObject { + private memory = createEdgeMemory(this.ctx) + async onRemember(text: string) { + return this.memory.remember(text) + } + async onRecall(context: string) { + return this.memory.recall(context) + } +} ``` -POST /learn — store a learning -POST /inject — recall relevant learnings for a context -POST /inject/trace — recall with debug info -POST /query — search learnings -GET /learnings — list learnings (filterable by scope) -GET /stats — counts by scope + +Or use the drop-in DO class with HTTP routes: + +```ts +export { DejaEdgeDO } from 'deja-edge/do' ``` -### MCP +[Full docs →](./packages/deja-edge/) + +### deja (hosted) -Any MCP-capable agent can connect: +Full-featured hosted service with Vectorize semantic search, scoped memory, working state, secrets, and MCP support. Deploy your own instance or use as part of [filepath](https://github.com/acoyfellow/filepath). + +```bash +git clone https://github.com/acoyfellow/deja && cd deja +bun install && wrangler login +wrangler vectorize create deja-embeddings --dimensions 384 --metric cosine +wrangler secret put API_KEY +bun run deploy +``` + +Connect via REST, the [client package](./packages/deja-client/), or MCP: ```json { @@ -54,37 +75,42 @@ Any MCP-capable agent can connect: "deja": { "type": "http", "url": "https://your-deja-instance.workers.dev/mcp", - "headers": { - "Authorization": "Bearer ${DEJA_API_KEY}" - } + "headers": { "Authorization": "Bearer ${DEJA_API_KEY}" } } } } ``` -Integration guides: [Cursor](https://deja.coey.dev/integrations/cursor) · [Claude Code](https://deja.coey.dev/integrations/claude-code) · [GitHub Actions](https://deja.coey.dev/integrations/github-actions) +## Core concepts + +All three systems share the same mental model: + +- **Remember** — store a memory (text, optionally with trigger/context/confidence) +- **Recall** — search for relevant memories given a context +- **Confirm / Reject** — feedback loop. Confirmed memories rise, rejected ones fade +- **Forget** — permanently delete a memory + +Memories are deduplicated at write time. Conflicting memories (same topic, different content) are automatically resolved — the newer one supersedes the older one. + +## Hosted service API -## API surface +The hosted service adds features beyond basic memory: -- **Memory**: `/learn`, `/inject`, `/inject/trace`, `/query`, `/learnings`, `/learning/:id`, `/learning/:id/neighbors`, `/stats`, `DELETE /learning/:id`, `DELETE /learnings` -- **Working state**: `/state/:runId`, `/state/:runId/events`, `/state/:runId/resolve` -- **Secrets**: `/secret`, `/secret/:name`, `/secrets` +- **Scoped memory** — `shared`, `agent:`, `session:`, or custom scopes +- **Working state** — live snapshots + event streams for in-progress work +- **Secrets** — scoped key-value storage +- **Loop runs** — track optimization loops with auto-learning from outcomes -Learnings track `lastRecalledAt` and `recallCount`. Bulk delete supports filters: `?confidence_lt=0.5`, `?not_recalled_in_days=90`, `?scope=shared`. +REST endpoints: `/learn`, `/inject`, `/query`, `/learnings`, `/stats`, `/state/:runId`, `/secret`, `/run` Full reference: https://deja.coey.dev/docs ## Architecture -- **Runtime**: Cloudflare Worker + Durable Object (per-user SQLite) -- **Embeddings**: Workers AI (`@cf/baai/bge-small-en-v1.5`, 384 dimensions) -- **Search**: Cloudflare Vectorize (cosine similarity) -- **Auth**: optional `API_KEY` secret; open access if not set +- **deja-local**: Bun SQLite + ONNX embeddings (all-MiniLM-L6-v2). In-memory vector index loaded at startup. +- **deja-edge**: Cloudflare DO SQLite + FTS5. Porter stemming tokenizer. BM25 ranking blended with confidence. +- **deja (hosted)**: Cloudflare Worker + Durable Object + Vectorize + Workers AI. Per-user isolation via API key. -## Reference +## License -- Schema: `src/schema.ts` -- Worker entry: `src/index.ts` -- Durable Object: `src/do/DejaDO.ts` -- Client package: `packages/deja-client/` -- Architecture guide: https://deja.coey.dev/guides/architecture-and-self-hosting +MIT diff --git a/packages/deja-client/README.md b/packages/deja-client/README.md index d6f26f9..045729f 100644 --- a/packages/deja-client/README.md +++ b/packages/deja-client/README.md @@ -1,152 +1,39 @@ # deja-client -Thin client for [deja](https://github.com/acoyfellow/deja) — persistent memory for agents. - -## Install - -```bash -npm install deja-client -# or -bun add deja-client -``` - -## Usage +HTTP client for a hosted [deja](https://github.com/acoyfellow/deja) instance. ```ts import deja from 'deja-client' -const mem = deja('https://deja.coey.dev') +const mem = deja('https://your-deja-instance.workers.dev', { apiKey: '...' }) -// Store a learning await mem.learn('deploy failed', 'check wrangler.toml first') - -// Get relevant memories before a task const { prompt, learnings } = await mem.inject('deploying to production') - -// Search memories const results = await mem.query('wrangler config') - -// List all memories -const all = await mem.list() - -// Delete a memory -await mem.forget('1234567890-abc123def') - -// Get stats -const stats = await mem.stats() -``` - -## Types - -All types are exported for use in your application: - -```ts -import deja, { type Learning, type InjectResult, type QueryResult, type Stats } from 'deja-client' -``` - -### `Learning` - -The core memory type returned by most methods: - -```ts -interface Learning { - id: string // Unique identifier - trigger: string // When this memory applies - learning: string // What was learned - reason?: string // Why this was learned - confidence: number // 0-1 confidence score - source?: string // Source identifier - scope: string // "shared", "agent:", or "session:" - createdAt: string // ISO timestamp -} -``` - -### `InjectResult` - -Returned by `mem.inject()`: - -```ts -interface InjectResult { - prompt: string // Formatted prompt text - learnings: Learning[] // Raw learnings -} ``` -### `QueryResult` - -Returned by `mem.query()`: +## Install -```ts -interface QueryResult { - learnings: Learning[] - hits: Record // Hits per scope -} +```bash +npm install deja-client ``` -### `Stats` - -Returned by `mem.stats()`: +## API ```ts -interface Stats { - totalLearnings: number - totalSecrets: number - scopes: Record -} +const mem = deja(url, { apiKey?, fetch? }) + +await mem.learn(trigger, learning, { confidence?, scope?, reason?, source? }) +await mem.inject(context, { scopes?, limit?, format? }) +await mem.query(text, { scopes?, limit? }) +await mem.list({ scope?, limit? }) +await mem.forget(id) +await mem.stats() +await mem.recordRun(outcome, attempts, { scope?, code?, error? }) +await mem.getRuns({ scope?, limit? }) ``` -## API - -### `deja(url, options?)` - -Create a client instance. - -- `url` — Your deja instance URL -- `options.apiKey` — API key for authenticated endpoints -- `options.fetch` — Custom fetch implementation - -### `mem.learn(trigger, learning, options?)` - -Store a learning for future recall. - -- `trigger` — When this learning applies -- `learning` — What was learned -- `options.confidence` — 0-1 (default: 0.8) -- `options.scope` — `shared`, `agent:`, or `session:` (default: `shared`) -- `options.reason` — Why this was learned -- `options.source` — Source identifier - -### `mem.inject(context, options?)` - -Get relevant memories for current context. - -- `context` — Current task or situation -- `options.scopes` — Scopes to search (default: `['shared']`) -- `options.limit` — Max memories (default: 5) -- `options.format` — `'prompt'` or `'learnings'` (default: `'prompt'`) - -### `mem.query(text, options?)` - -Search memories semantically. - -- `text` — Search query -- `options.scopes` — Scopes to search (default: `['shared']`) -- `options.limit` — Max results (default: 10) - -### `mem.list(options?)` - -List all memories. - -- `options.scope` — Filter by scope -- `options.limit` — Max results - -### `mem.forget(id)` - -Delete a specific memory by ID. - -### `mem.stats()` - -Get memory statistics. +All types (`Learning`, `InjectResult`, `QueryResult`, `Stats`) are exported for TypeScript users. ## License diff --git a/packages/deja-edge/README.md b/packages/deja-edge/README.md new file mode 100644 index 0000000..0c97e2e --- /dev/null +++ b/packages/deja-edge/README.md @@ -0,0 +1,132 @@ +# deja-edge + +Edge memory for Cloudflare Durable Objects. FTS5 full-text search, zero external dependencies. + +```ts +import { createEdgeMemory } from 'deja-edge' + +export class MyDO extends DurableObject { + private memory = createEdgeMemory(this.ctx) + + async fetch(request: Request) { + const context = await request.text() + const results = this.memory.recall(context) + return Response.json(results) + } +} +``` + +## Why + +You have agents running in Cloudflare Workers. You want them to remember things across requests without adding Vectorize, Workers AI, or any external service. deja-edge gives you full-text search memory inside your Durable Object's built-in SQLite. + +## Install + +```bash +npm install deja-edge +``` + +Your wrangler config needs a DO with SQLite: + +```json +{ + "durable_objects": { + "bindings": [{ "name": "MEMORY", "class_name": "MyDO" }] + }, + "migrations": [{ "tag": "v1", "new_sqlite_classes": ["MyDO"] }] +} +``` + +## API + +All methods are synchronous — no async needed, no network calls. Everything runs in the DO's SQLite. + +### `remember(text)` + +Store a memory. + +```ts +memory.remember('Redis must be running before starting the API server') +memory.remember('Use pnpm, not npm') +``` + +Dedup: near-identical text is detected and skipped. +Conflict: same topic but different content — the new memory supersedes the old one (old confidence drops). + +### `recall(context, opts?)` + +Search memories. Returns results ranked by FTS5 relevance blended with confidence. + +```ts +const results = memory.recall('setting up the dev environment') +// [{ id, text, score, confidence, createdAt }] +``` + +Options: `{ limit, threshold, minConfidence }` + +### `confirm(id)` / `reject(id)` + +Feedback loop. Confirm boosts confidence (+0.1), reject drops it (-0.15). + +```ts +memory.confirm(results[0].id) // useful +memory.reject(results[1].id) // outdated +``` + +### `forget(id)` + +Delete a memory. + +### `list(opts?)` / `recallLog(opts?)` + +Inspect memories and the audit trail. + +```ts +memory.list() // all memories, newest first +memory.list({ limit: 10 }) // paginated +memory.recallLog() // what was recalled and when +``` + +### `size` + +Number of stored memories. + +## Drop-in DO class + +If you don't need a custom DO, use `DejaEdgeDO` directly: + +```ts +// worker.ts +export { DejaEdgeDO } from 'deja-edge/do' + +export default { + async fetch(request, env) { + const id = env.MEMORY.idFromName('default') + const stub = env.MEMORY.get(id) + return stub.fetch(request) + } +} +``` + +Routes: `POST /remember`, `POST /recall`, `POST /confirm/:id`, `POST /reject/:id`, `DELETE /forget/:id`, `GET /list`, `GET /recall-log`, `GET /size`, `GET /` + +## How it works + +**Search**: SQLite FTS5 with Porter stemming tokenizer. Queries are decomposed into keywords, matched with `OR`, and ranked by BM25. Scores are normalized to 0-1 and blended with confidence (70% relevance, 30% confidence). + +**Dedup**: Trigram Jaccard similarity. Memories above 0.85 similarity are considered duplicates. + +**Conflict resolution**: Memories between 0.5-0.85 similarity are about the same topic but say different things. The old memory's confidence drops to 30%, the new one takes priority. + +## Configuration + +```ts +const memory = createEdgeMemory(ctx, { + dedupeThreshold: 0.85, // similarity to skip as duplicate (default 0.85) + conflictThreshold: 0.5, // similarity to detect conflict (default 0.5) +}) +``` + +## License + +MIT diff --git a/packages/deja-local/README.md b/packages/deja-local/README.md index 590cc18..9440709 100644 --- a/packages/deja-local/README.md +++ b/packages/deja-local/README.md @@ -1,147 +1,80 @@ # deja-local -Cross-session memory for AI agents. One function to remember, one function to recall. +Local in-process memory for AI agents. SQLite-backed, vector search, no network. **Requires Bun.** ```ts -import { createMemory } from "deja-local"; +import { createMemory } from 'deja-local' -const mem = await createMemory({ path: "./agent-memory.db" }); +const mem = createMemory({ path: './agent.db' }) -// Store a learning -await mem.remember("Always run migrations before deploying to staging"); - -// Recall relevant memories later, in any session -const results = await mem.recall("deploying to staging"); -// [{ text: "Always run migrations before deploying to staging", score: 0.82, confidence: 0.5 }] +await mem.remember('always run migrations before deploying to staging') +const results = await mem.recall('deploying to staging') +// [{ text: "always run migrations before deploying to staging", score: 0.82, confidence: 0.5 }] ``` -## Why - -AI agents are amnesiacs. Every session starts from zero. Your agent figures out that the test suite needs `NODE_ENV=test` on Monday, then wastes 10 minutes rediscovering it on Tuesday. - -deja-local gives agents a durable memory that gets smarter over time. - -- **SQLite-backed** -- no external services, no API keys, no network -- **Real embeddings** -- all-MiniLM-L6-v2 via ONNX, runs on CPU -- **ACID durable** -- memory is persisted before `remember()` returns -- **Gets smarter** -- confirm/reject feedback makes good memories rise and bad ones fade - ## Install ```bash -npm install deja-local +bun add deja-local ``` +This package uses `bun:sqlite` and requires the Bun runtime. It will not work in Node.js. + ## API ### `remember(text)` -Store a memory. Deja handles dedup and conflict resolution automatically. - -```ts -await mem.remember("The Stripe webhook secret is in 1Password, not .env"); -await mem.remember("Use pnpm, not npm -- the lockfile breaks otherwise"); -await mem.remember("Redis must be running before starting the API server"); -``` - -If a new memory contradicts an existing one, Deja detects the conflict and supersedes it: +Store a memory. Dedup and conflict resolution happen automatically. ```ts -await mem.remember("Deploy target is us-east-1"); -// ... weeks later ... -await mem.remember("Deploy target moved to eu-west-1"); -// Old memory is superseded -- its confidence drops, new one takes priority +await mem.remember('Use pnpm, not npm -- the lockfile breaks otherwise') +await mem.remember('Redis must be running before starting the API server') ``` -Identical or near-identical memories are deduplicated at write time. +If a new memory contradicts an existing one, the old memory's confidence drops and the new one takes priority. ### `recall(query, opts?)` -Retrieve relevant memories, ranked by relevance and confidence. - -```ts -const results = await mem.recall("setting up the dev environment"); -// [ -// { text: "Redis must be running before starting the API server", score: 0.81, confidence: 0.7 }, -// { text: "Use pnpm, not npm -- the lockfile breaks otherwise", score: 0.74, confidence: 0.5 }, -// ] -``` - -Complex queries are automatically decomposed into sub-queries for broader recall: +Search memories by semantic similarity, ranked by relevance and confidence. ```ts -const results = await mem.recall("full deploy checklist for production"); -// Internally searches: deploy, checklist, production, deploy checklist, checklist production -// Returns merged, deduplicated results +const results = await mem.recall('setting up the dev environment') ``` -Options: - -```ts -await mem.recall("deploy steps", { limit: 3 }); // max results -await mem.recall("deploy steps", { threshold: 0.5 }); // min relevance -await mem.recall("deploy steps", { minConfidence: 0.4 }); // skip low-confidence memories -``` +Complex queries are decomposed into sub-queries for broader recall. Options: `{ limit, threshold, minConfidence }` ### `confirm(id)` / `reject(id)` -Give feedback on recalled memories. This is the ratchet -- memories that help get promoted, memories that mislead get demoted. - -```ts -const results = await mem.recall("database connection string format"); - -// This one was useful -await mem.confirm(results[0].id); - -// This one was outdated -await mem.reject(results[1].id); -``` - -Over time, high-signal memories surface first. Low-signal memories fade. +Feedback loop. Confirm boosts confidence (+0.1), reject drops it (-0.15). Over time, useful memories surface first. ### `forget(id)` -Permanently remove a memory. - -```ts -await mem.forget(memory.id); -``` +Delete a memory. ### `list(opts?)` / `recallLog(opts?)` -Inspect stored memories and the audit trail. - -```ts -const all = mem.list(); // all memories, newest first -const recent = mem.list({ limit: 10 }); // paginated - -const log = mem.recallLog(); // what was recalled and when -``` +Inspect memories and the audit trail. ## Configuration ```ts const mem = createMemory({ - path: "./memory.db", // required -- SQLite file path - model: "Xenova/all-MiniLM-L6-v2", // HuggingFace model (default) - embed: customEmbedFn, // or bring your own embed function - threshold: 0.3, // min similarity for recall (default 0.3) - dedupeThreshold: 0.95, // similarity to consider duplicate (default 0.95) - conflictThreshold: 0.6, // similarity to detect conflict (default 0.6) -}); + path: './memory.db', // required -- SQLite file path + model: 'Xenova/all-MiniLM-L6-v2', // HuggingFace model (default) + embed: customEmbedFn, // or bring your own embed function + threshold: 0.3, // min similarity for recall (default 0.3) + dedupeThreshold: 0.95, // similarity to skip as duplicate (default 0.95) + conflictThreshold: 0.6, // similarity to detect conflict (default 0.6) +}) ``` ## How it works -**Dedup:** At write time, if a new memory's embedding is >= 0.95 similar to an existing one, it's a duplicate and skipped. - -**Conflict resolution:** If similarity is between 0.6 and 0.95, the memories are about the same topic but say different things. The new memory supersedes the old one -- the old memory's confidence drops to 30% of its current value, so it naturally sinks in recall rankings. - -**The ratchet:** `confirm()` boosts confidence by 0.1, `reject()` drops it by 0.15. Recall ranks by `relevance * 0.7 + confidence * 0.3`. Over thousands of memories, this is the difference between useful recall and noise. +**Embeddings**: all-MiniLM-L6-v2 via ONNX, runs on CPU (~23MB model, cached locally). In-memory vector index loaded at startup. -**Recall decomposition:** Complex queries are split into keyword pairs and individual terms. Each sub-query is embedded and scored independently. The best score per memory wins. This catches memories that match part of your intent even if they don't match the full query. +**Scoring**: `relevance * 0.7 + confidence * 0.3`. Confirm/reject adjusts confidence. Over thousands of memories, this separates signal from noise. -**Audit trail:** Every `recall()` is logged with the query, matched memory IDs, scores, and timestamp. Inspect with `recallLog()`. +**Dedup**: >= 0.95 similarity at write time is skipped. 0.6-0.95 similarity triggers conflict resolution — old memory's confidence drops to 30%. ## License From b49ea4a7ab03066ebd7f761dc8ffe6fe6e434548 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 23 Mar 2026 08:55:50 +0000 Subject: [PATCH 4/4] fix(deja-edge): fix BM25 normalization bug and wire constructor minConfidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: When all BM25 ranks are identical (single result or tied scores), normalizedRelevance was 0, capping scores at confidence*0.3. Now treats equal ranks as full relevance (1.0). P2: Constructor minConfidence option was ignored in recall() — always defaulted to 0. Now used as the fallback when callers don't pass it. https://claude.ai/code/session_01Fb57JFi8VM36x8pCM1BFTw --- packages/deja-edge/src/index.ts | 8 +++-- packages/deja-edge/test/edge-memory.test.ts | 37 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/packages/deja-edge/src/index.ts b/packages/deja-edge/src/index.ts index 62d9829..28dffe7 100644 --- a/packages/deja-edge/src/index.ts +++ b/packages/deja-edge/src/index.ts @@ -218,6 +218,7 @@ export function createEdgeMemory( ): EdgeMemoryStore { const dedupeThreshold = opts.dedupeThreshold ?? 0.85 const conflictThreshold = opts.conflictThreshold ?? 0.5 + const defaultMinConfidence = opts.minConfidence ?? 0 const sql = ctx.storage.sql @@ -291,7 +292,7 @@ export function createEdgeMemory( function recall(context: string, options: RecallOptions = {}): RecallResult[] { const limit = options.limit ?? 5 - const minConf = options.minConfidence ?? 0 + const minConf = options.minConfidence ?? defaultMinConfidence const threshold = options.threshold ?? 0 const ftsQuery = buildFtsQuery(context) @@ -323,10 +324,11 @@ export function createEdgeMemory( // Normalize BM25 scores to 0-1 range, then blend with confidence const maxRank = Math.max(...rows.map(r => -r.rank)) const minRank = Math.min(...rows.map(r => -r.rank)) - const range = maxRank - minRank || 1 + const range = maxRank - minRank const results: RecallResult[] = rows.map(r => { - const normalizedRelevance = (-r.rank - minRank) / range + // When all ranks are identical (single result or tied scores), treat as full relevance + const normalizedRelevance = range === 0 ? 1.0 : (-r.rank - minRank) / range const blended = normalizedRelevance * 0.7 + r.confidence * 0.3 return { id: r.id, diff --git a/packages/deja-edge/test/edge-memory.test.ts b/packages/deja-edge/test/edge-memory.test.ts index 85ff519..2fb62b3 100644 --- a/packages/deja-edge/test/edge-memory.test.ts +++ b/packages/deja-edge/test/edge-memory.test.ts @@ -247,4 +247,41 @@ describe('deja-edge: createEdgeMemory', () => { expect(page2.length).toBe(2) expect(page1[0].id).not.toBe(page2[0].id) }) + + test('recall returns meaningful scores when only one result matches', () => { + freshMemory() + memory.remember('always check wrangler.toml before deploying') + memory.remember('quantum physics is fascinating') + + const results = memory.recall('wrangler deploy') + expect(results.length).toBeGreaterThan(0) + // With the normalization fix, a single match should get full relevance (0.7) + confidence (0.15) + expect(results[0].score).toBeGreaterThan(0.5) + }) + + test('recall scores are meaningful when all results have identical BM25 rank', () => { + freshMemory() + memory.remember('deploy step one check config') + memory.remember('deploy step two run tests') + + // Both match "deploy" equally — scores should still be > 0.15 + const results = memory.recall('deploy') + for (const r of results) { + expect(r.score).toBeGreaterThan(0.15) + } + }) + + test('constructor minConfidence is used as default in recall', () => { + freshMemory({ minConfidence: 0.4 }) + const m = memory.remember('high confidence item about deployment') + const low = memory.remember('low confidence item about deployment scripts') + memory.reject(low.id) + memory.reject(low.id) + + // Should filter by constructor minConfidence without passing it per-call + const results = memory.recall('deployment') + for (const r of results) { + expect(r.confidence).toBeGreaterThanOrEqual(0.4) + } + }) })