Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"type": "number",
"minimum": 0.01,
"maximum": 0.5,
"default": 0.10,
"default": 0.1,
"description": "Angular distance threshold for cluster assignment"
},
"minClusterSize": {
Expand Down
12 changes: 9 additions & 3 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,16 @@ services:
# - CAUSANTIC_ANTHROPIC_KEY=${CAUSANTIC_ANTHROPIC_KEY}
ports:
# HTTP port for web dashboard (when implemented)
- "3000:3000"
- '3000:3000'
restart: unless-stopped
healthcheck:
test: ["CMD", "node", "-e", "require('./dist/storage/db.js').getDatabase().prepare('SELECT 1').get()"]
test:
[
'CMD',
'node',
'-e',
"require('./dist/storage/db.js').getDatabase().prepare('SELECT 1').get()",
]
interval: 30s
timeout: 10s
retries: 3
Expand All @@ -39,7 +45,7 @@ services:
environment:
- CAUSANTIC_STORAGE_DB_PATH=/data/causantic/memory.db
- CAUSANTIC_STORAGE_VECTOR_PATH=/data/causantic/vectors
command: ["node", "dist/cli/index.js", "maintenance", "daemon"]
command: ['node', 'dist/cli/index.js', 'maintenance', 'daemon']
restart: unless-stopped
depends_on:
- causantic
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@
"test:watch": "vitest",
"lint": "eslint src/ test/",
"lint:fix": "eslint src/ test/ --fix",
"format": "prettier --write 'src/**/*.ts' 'test/**/*.ts'",
"format:check": "prettier --check 'src/**/*.ts' 'test/**/*.ts'",
"format": "prettier --write 'src/**/*.{ts,tsx,css}' 'test/**/*.ts' 'scripts/**/*.ts' config.schema.json docker-compose.yml",
"format:check": "prettier --check 'src/**/*.{ts,tsx,css}' 'test/**/*.ts' 'scripts/**/*.ts' config.schema.json docker-compose.yml",
"ingest": "tsx scripts/ops/ingest.ts",
"batch-ingest": "tsx scripts/ops/batch-ingest.ts",
"recall": "tsx scripts/ops/recall.ts",
Expand Down
6 changes: 4 additions & 2 deletions scripts/benchmarks/hdbscan-benchmark.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ async function runBenchmarks() {
}

console.log(
`| ${size.toString().padEnd(4)} | ${String(nativeParallel.ms).padEnd(17)}ms | ${String(nativeSingle.ms).padEnd(15)}ms | ${String(oldLib.ms).padEnd(10)}ms |`
`| ${size.toString().padEnd(4)} | ${String(nativeParallel.ms).padEnd(17)}ms | ${String(nativeSingle.ms).padEnd(15)}ms | ${String(oldLib.ms).padEnd(10)}ms |`,
);
}

Expand Down Expand Up @@ -155,7 +155,9 @@ async function runBenchmarks() {
console.log(`Approximate k-NN: ${approxResult.ms}ms`);

console.log('\n--- Cluster Quality Comparison ---');
console.log(`Native clusters: ${defaultResult.value.numClusters}, noise: ${defaultResult.value.noiseCount}`);
console.log(
`Native clusters: ${defaultResult.value.numClusters}, noise: ${defaultResult.value.noiseCount}`,
);

try {
const oldH = new OldHDBSCAN({ minClusterSize: 4, minSamples: 4 });
Expand Down
42 changes: 32 additions & 10 deletions scripts/debug/compare-retrieval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@ async function main() {
const chunk = getChunkById(r.id);
const preview = chunk?.content.slice(0, 100).replace(/\n/g, ' ') || '(no content)';
const similarity = (1 - r.distance).toFixed(3);
console.log((i + 1) + '. [sim=' + similarity + '] ' + (chunk?.sessionSlug || 'unknown'));
console.log(i + 1 + '. [sim=' + similarity + '] ' + (chunk?.sessionSlug || 'unknown'));
console.log(' ' + preview + '...');
}

console.log('\n' + '='.repeat(80));
console.log('\n## GRAPH TRAVERSAL ADDITIONS\n');

const startIds = vectorResults.map(r => r.id);
const startIds = vectorResults.map((r) => r.id);

// Uses config defaults: maxDepth=50, minWeight=0.01
const backwardResult = traverseMultiple(startIds, {
Expand All @@ -47,19 +47,33 @@ async function main() {
direction: 'forward',
});

const backwardAdditions = backwardResult.chunks.filter(c => !vectorChunkIds.has(c.chunkId));
const forwardAdditions = forwardResult.chunks.filter(c => !vectorChunkIds.has(c.chunkId));

console.log('Backward traversal: ' + backwardResult.chunks.length + ' total, ' + backwardAdditions.length + ' NEW');
console.log('Forward traversal: ' + forwardResult.chunks.length + ' total, ' + forwardAdditions.length + ' NEW');
const backwardAdditions = backwardResult.chunks.filter((c) => !vectorChunkIds.has(c.chunkId));
const forwardAdditions = forwardResult.chunks.filter((c) => !vectorChunkIds.has(c.chunkId));

console.log(
'Backward traversal: ' +
backwardResult.chunks.length +
' total, ' +
backwardAdditions.length +
' NEW',
);
console.log(
'Forward traversal: ' +
forwardResult.chunks.length +
' total, ' +
forwardAdditions.length +
' NEW',
);

if (backwardAdditions.length > 0) {
console.log('\n### Top 5 Backward Additions (context that LED TO matches):\n');
for (let i = 0; i < Math.min(5, backwardAdditions.length); i++) {
const c = backwardAdditions[i];
const chunk = getChunkById(c.chunkId);
const preview = chunk?.content.slice(0, 120).replace(/\n/g, ' ') || '';
console.log((i + 1) + '. [w=' + c.weight.toFixed(3) + ' d=' + c.depth + '] ' + (chunk?.sessionSlug || ''));
console.log(
i + 1 + '. [w=' + c.weight.toFixed(3) + ' d=' + c.depth + '] ' + (chunk?.sessionSlug || ''),
);
console.log(' ' + preview + '...');
}
}
Expand All @@ -70,7 +84,9 @@ async function main() {
const c = forwardAdditions[i];
const chunk = getChunkById(c.chunkId);
const preview = chunk?.content.slice(0, 120).replace(/\n/g, ' ') || '';
console.log((i + 1) + '. [w=' + c.weight.toFixed(3) + ' d=' + c.depth + '] ' + (chunk?.sessionSlug || ''));
console.log(
i + 1 + '. [w=' + c.weight.toFixed(3) + ' d=' + c.depth + '] ' + (chunk?.sessionSlug || ''),
);
console.log(' ' + preview + '...');
}
}
Expand All @@ -81,7 +97,13 @@ async function main() {
console.log('Vector search alone: ' + vectorResults.length + ' chunks');
console.log('+ Backward traversal: ' + backwardAdditions.length + ' additional');
console.log('+ Forward traversal: ' + forwardAdditions.length + ' additional');
console.log('Graph added: ' + totalAdded + ' chunks (' + Math.round(totalAdded / vectorResults.length * 100) + '% increase)');
console.log(
'Graph added: ' +
totalAdded +
' chunks (' +
Math.round((totalAdded / vectorResults.length) * 100) +
'% increase)',
);

await embedder.dispose();
closeDb();
Expand Down
16 changes: 10 additions & 6 deletions scripts/debug/debug-forward.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ async function debug(query: string) {

// Vector search
const vectorResults = await vectorStore.search(embedding, 20);
const vectorIds = new Set(vectorResults.map(r => r.id));
const vectorIds = new Set(vectorResults.map((r) => r.id));

console.log(`\n=== Query: "${query}" ===\n`);
console.log(`Vector search found: ${vectorResults.length} chunks\n`);
Expand All @@ -35,15 +35,17 @@ async function debug(query: string) {
totalForwardTargets++;
if (vectorIds.has(edge.targetChunkId)) {
targetsAlreadyInVectorSearch++;
console.log(` -> Target ${edge.targetChunkId.slice(0,30)}... ALREADY in vector search`);
console.log(` -> Target ${edge.targetChunkId.slice(0, 30)}... ALREADY in vector search`);
} else {
// Check if it was filtered by decay
const weighted = weightedForward.find(w => w.targetChunkId === edge.targetChunkId);
const weighted = weightedForward.find((w) => w.targetChunkId === edge.targetChunkId);
if (!weighted) {
targetsWithZeroWeight++;
console.log(` -> Target ${edge.targetChunkId.slice(0,30)}... FILTERED (zero weight)`);
console.log(` -> Target ${edge.targetChunkId.slice(0, 30)}... FILTERED (zero weight)`);
} else {
console.log(` -> Target ${edge.targetChunkId.slice(0,30)}... weight=${weighted.weight.toFixed(3)} SHOULD BE ADDED`);
console.log(
` -> Target ${edge.targetChunkId.slice(0, 30)}... weight=${weighted.weight.toFixed(3)} SHOULD BE ADDED`,
);
}
}
}
Expand All @@ -54,7 +56,9 @@ async function debug(query: string) {
console.log(`Total forward edge targets: ${totalForwardTargets}`);
console.log(`Already in vector search: ${targetsAlreadyInVectorSearch}`);
console.log(`Filtered by decay: ${targetsWithZeroWeight}`);
console.log(`Should be added: ${totalForwardTargets - targetsAlreadyInVectorSearch - targetsWithZeroWeight}`);
console.log(
`Should be added: ${totalForwardTargets - targetsAlreadyInVectorSearch - targetsWithZeroWeight}`,
);

await embedder.dispose();
}
Expand Down
40 changes: 28 additions & 12 deletions scripts/debug/debug-traversal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,50 +9,66 @@ async function debug() {
const query = process.argv[2] || 'git commit workflow';
const embedder = new Embedder();
await embedder.load(getModel('jina-small'));

const { embedding } = await embedder.embed(query, true);
const results = await vectorStore.search(embedding, 10);

console.log('Query: "' + query + '"\n');
console.log('Vector search results and their edges:\n');

const allTargets = new Set<string>();
const vectorIds = new Set(results.map(r => r.id));
const vectorIds = new Set(results.map((r) => r.id));

for (const r of results) {
const chunk = getChunkById(r.id);
const backEdges = getOutgoingEdges(r.id, 'backward');
const fwdEdges = getOutgoingEdges(r.id, 'forward');

console.log('Chunk: ' + r.id.slice(-25));
console.log(' Session: ' + chunk?.sessionSlug);
console.log(' Sim: ' + (1 - r.distance).toFixed(3));
console.log(' Backward edges: ' + backEdges.length);
console.log(' Forward edges: ' + fwdEdges.length);

if (backEdges.length > 0 || fwdEdges.length > 0) {
for (const e of backEdges) {
allTargets.add(e.targetChunkId);
const inVector = vectorIds.has(e.targetChunkId) ? ' (OVERLAP)' : '';
console.log(' BACK -> ' + e.targetChunkId.slice(-20) + ' w=' + e.initialWeight.toFixed(2) + ' links=' + e.linkCount + inVector);
console.log(
' BACK -> ' +
e.targetChunkId.slice(-20) +
' w=' +
e.initialWeight.toFixed(2) +
' links=' +
e.linkCount +
inVector,
);
}

for (const e of fwdEdges) {
allTargets.add(e.targetChunkId);
const inVector = vectorIds.has(e.targetChunkId) ? ' (OVERLAP)' : '';
console.log(' FWD -> ' + e.targetChunkId.slice(-20) + ' w=' + e.initialWeight.toFixed(2) + ' links=' + e.linkCount + inVector);
console.log(
' FWD -> ' +
e.targetChunkId.slice(-20) +
' w=' +
e.initialWeight.toFixed(2) +
' links=' +
e.linkCount +
inVector,
);
}
}
console.log('');
}
const overlapping = [...allTargets].filter(t => vectorIds.has(t)).length;

const overlapping = [...allTargets].filter((t) => vectorIds.has(t)).length;
const newTargets = allTargets.size - overlapping;
console.log('='.repeat(60));
console.log('Total edge targets: ' + allTargets.size);
console.log(' Already in vector results: ' + overlapping);
console.log(' NEW (added by traversal): ' + newTargets);

await embedder.dispose();
closeDb();
}
Expand Down
Loading