Skip to content

Commit f603add

Browse files
committed
refactor: remove last JSON.stringify from query-do, update architecture diagram
- Replace JSON.stringify-based cache key hashing with direct FNV-1a feed over query components — no serialization at all in DOs - Remove bigIntReplacer import from query-do (no longer needed) - Update excalidraw: NDJSON → columnar binary streaming, PageProcessor → streaming operator pipeline, JSON return → RPC zero serialization
1 parent 9393fb4 commit f603add

File tree

2 files changed

+19
-22
lines changed

2 files changed

+19
-22
lines changed

docs/architecture/querymode-architecture.excalidraw

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -672,7 +672,7 @@
672672
"y": 395.1260756922146,
673673
"width": 149.15985107421875,
674674
"height": 140,
675-
"text": "Query DO — SJC\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
675+
"text": "Query DO — SJC\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
676676
"fontSize": 11,
677677
"fontFamily": 1,
678678
"textAlign": "center",
@@ -698,7 +698,7 @@
698698
"roundness": null,
699699
"updated": 1772498220235,
700700
"link": null,
701-
"originalText": "Query DO — SJC\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
701+
"originalText": "Query DO — SJC\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
702702
"autoResize": true,
703703
"lineHeight": 1.4141414141414141
704704
},
@@ -760,7 +760,7 @@
760760
"y": 395,
761761
"width": 180,
762762
"height": 140,
763-
"text": "Query DO — NRT\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
763+
"text": "Query DO — NRT\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
764764
"fontSize": 11,
765765
"fontFamily": 1,
766766
"textAlign": "center",
@@ -786,7 +786,7 @@
786786
"roundness": null,
787787
"updated": 1772496885793,
788788
"link": null,
789-
"originalText": "Query DO — NRT\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
789+
"originalText": "Query DO — NRT\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
790790
"autoResize": true,
791791
"lineHeight": 1.4141414141414141
792792
},
@@ -844,7 +844,7 @@
844844
"y": 395,
845845
"width": 149.15985107421875,
846846
"height": 140,
847-
"text": "Query DO — AMS\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
847+
"text": "Query DO — AMS\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
848848
"fontSize": 11,
849849
"fontFamily": 1,
850850
"textAlign": "center",
@@ -870,7 +870,7 @@
870870
"roundness": null,
871871
"updated": 1772497096583,
872872
"link": null,
873-
"originalText": "Query DO — AMS\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nPageProcessor bridge\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nNDJSON /query/stream",
873+
"originalText": "Query DO — AMS\n\nFooter cache (~4KB/table)\nWASM Engine (Zig SIMD)\nStreaming operator pipeline\nVIP eviction (zell)\nCoalesced Range reads\nrefreshStaleTables()\nColumnar binary streaming",
874874
"autoResize": true,
875875
"lineHeight": 1.4141414141414141
876876
},
@@ -2421,7 +2421,7 @@
24212421
"y": 833,
24222422
"width": 90,
24232423
"height": 34,
2424-
"text": "Return JSON\nor NDJSON",
2424+
"text": "Return RPC result\n(zero serialization)",
24252425
"fontSize": 12,
24262426
"fontFamily": 1,
24272427
"textAlign": "center",
@@ -2447,7 +2447,7 @@
24472447
"roundness": null,
24482448
"updated": 1772496885793,
24492449
"link": null,
2450-
"originalText": "Return JSON\nor NDJSON",
2450+
"originalText": "Return RPC result\n(zero serialization)",
24512451
"autoResize": true,
24522452
"lineHeight": 1.4166666666666667
24532453
},

src/query-do.ts

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { parseFooter, parseColumnMetaFromProtobuf } from "./footer.js";
44
import { parseManifest, logicalTypeToDataType } from "./manifest.js";
55
import { detectFormat, getParquetFooterLength, parseParquetFooter, parquetMetaToTableMeta } from "./parquet.js";
66
import { parseIcebergMetadata, extractParquetPathsFromManifest } from "./iceberg.js";
7-
import { canSkipPage, bigIntReplacer } from "./decode.js";
7+
import { canSkipPage } from "./decode.js";
88
import { decodeParquetColumnChunk } from "./parquet-decode.js";
99
import { instantiateWasm, type WasmEngine } from "./wasm-engine.js";
1010
import { mergeQueryResults } from "./merge.js";
@@ -591,21 +591,18 @@ export class QueryDO extends DurableObject<Env> {
591591
}
592592

593593
private queryKey(query: QueryDescriptor): string {
594-
const normalized = {
595-
t: query.table,
596-
f: [...query.filters].sort((a, b) => a.column.localeCompare(b.column) || a.op.localeCompare(b.op)),
597-
p: [...query.projections].sort(),
598-
s: query.sortColumn, sd: query.sortDirection,
599-
l: query.limit,
600-
a: query.aggregates, g: query.groupBy,
601-
};
602-
const str = JSON.stringify(normalized, bigIntReplacer);
603-
// FNV-1a hash
594+
// FNV-1a hash over query components — no JSON serialization
604595
let h = 0x811c9dc5;
605-
for (let i = 0; i < str.length; i++) {
606-
h ^= str.charCodeAt(i);
607-
h = Math.imul(h, 0x01000193);
596+
const feed = (s: string) => { for (let i = 0; i < s.length; i++) { h ^= s.charCodeAt(i); h = Math.imul(h, 0x01000193); } };
597+
feed(query.table);
598+
for (const f of [...query.filters].sort((a, b) => a.column.localeCompare(b.column) || a.op.localeCompare(b.op))) {
599+
feed(f.column); feed(f.op); feed(String(f.value));
608600
}
601+
for (const p of [...query.projections].sort()) feed(p);
602+
if (query.sortColumn) { feed(query.sortColumn); feed(query.sortDirection ?? "asc"); }
603+
if (query.limit !== undefined) feed(String(query.limit));
604+
if (query.aggregates) for (const a of query.aggregates) { feed(a.fn); feed(a.column); if (a.alias) feed(a.alias); }
605+
if (query.groupBy) for (const g of query.groupBy) feed(g);
609606
return `qr:${query.table}:${(h >>> 0).toString(36)}`;
610607
}
611608

0 commit comments

Comments
 (0)