feat: implement lazy DataFrame with full query engine capabilities

Transform QueryMode from basic SQL into a composable lazy DataFrame API
with 10 new capabilities:

- Phase 0: Distributed execution layer (WorkerPool + R2Partitioner tree fan-out)
- Phase 1: Immutable lazy DataFrame with computed columns, replacing mutable TableQuery
- Phase 2: Window functions (row_number, rank, dense_rank, lag, lead, aggregates)
- Phase 3: Set operations (union/intersect/except) and DISTINCT operator
- Phase 4: Extended aggregates (stddev, variance, median, percentile, count_distinct)
         + right/full/cross join types in HashJoinOperator
- Phase 5: Subqueries (filterIn) and materialized intermediates
- Phase 6: Pure TypeScript HNSW index for approximate nearest neighbor search
- Phase 7: Pluggable format readers (CSV, JSON/NDJSON, Arrow IPC) via ReaderRegistry
- Phase 8: MaterializationCache with page-level LRU and byte-budget eviction
- Phase 9: Time travel (version queries, diff, version listing)

All operators honor the pull-based pipeline contract (bounded memory, spill-to-disk).
Backward compatible: TableQuery kept as type alias, .exec() as alias for .collect().

Author: teamchong

src/client.ts

@@ -1,15 +1,25 @@
-import { TableQuery } from "./client.js";
+import { DataFrame, TableQuery } from "./client.js";
 import type { QueryDescriptor, QueryExecutor } from "./client.js";
 import type { AppendResult, ExplainResult, QueryResult, Row, QueryDORpc, MasterDORpc } from "./types.js";
 import { LocalExecutor } from "./local-executor.js";
 
 export { MasterDO } from "./master-do.js";
 export { QueryDO } from "./query-do.js";
 export { FragmentDO } from "./fragment-do.js";
-export { TableQuery } from "./client.js";
+export { WorkerPool, R2Partitioner } from "./worker-pool.js";
+export { WorkerDO } from "./worker-do.js";
+export type { R2Partition, WorkerDORpc } from "./worker-pool.js";
+export { ReaderRegistry, FileDataSource, UrlDataSource } from "./reader.js";
+export type { FormatReader, DataSource } from "./reader.js";
+export { DataFrame, TableQuery } from "./client.js";
+export { LazyResultHandle } from "./client.js";
 export { QueryModeError } from "./errors.js";
 export { LocalExecutor } from "./local-executor.js";
 export { bigIntReplacer } from "./decode.js";
+export { HnswIndex, cosineDistance, l2DistanceSq, dotDistance } from "./hnsw.js";
+export type { HnswOptions } from "./hnsw.js";
+export { MaterializationCache, queryHashKey } from "./lazy.js";
+export type { MaterializationCacheOptions } from "./lazy.js";
 export type { QueryExecutor, QueryDescriptor } from "./client.js";
 export type {
   Env,
@@ -25,6 +35,10 @@ export type {
   FilterOp,
   AggregateOp,
   JoinDescriptor,
+  JoinKeys,
+  JoinType,
+  WindowSpec,
+  VectorOpts,
   QueryResult,
   Row,
   VectorSearchParams,
@@ -33,6 +47,8 @@ export type {
   AppendResult,
   ExplainResult,
   VectorIndexInfo,
+  VersionInfo,
+  DiffResult,
   QueryDORpc,
   MasterDORpc,
 } from "./types.js";
@@ -81,9 +97,9 @@ export class QueryMode {
     return new QueryMode(executor);
   }
 
-  /** Start building a query against a table. */
-  table(name: string): TableQuery {
-    return new TableQuery(name, this.executor);
+  /** Start building a query against a table. Returns a lazy DataFrame. */
+  table(name: string): DataFrame {
+    return new DataFrame(name, this.executor);
   }
 
   /**

src/hnsw.ts

@@ -0,0 +1,224 @@
+import type { Row } from "./types.js";
+import type { QueryDescriptor } from "./client.js";
+
+export interface MaterializationCacheOptions {
+  /** Maximum bytes for the cache (default 64MB). */
+  maxBytes?: number;
+}
+
+interface CacheEntry {
+  rows: Row[];
+  byteSize: number;
+  lastAccess: number;
+}
+
+const DEFAULT_MAX_BYTES = 64 * 1024 * 1024; // 64MB
+
+/**
+ * Estimate the byte size of a single Row.
+ * 64 bytes base overhead + 8 bytes per key + value sizes.
+ */
+function estimateRowBytes(row: Row): number {
+  let bytes = 64; // base object overhead
+  for (const key in row) {
+    bytes += 8; // key pointer/overhead
+    const val = row[key];
+    if (val === null || val === undefined) {
+      bytes += 0;
+    } else if (typeof val === "number") {
+      bytes += 8;
+    } else if (typeof val === "bigint") {
+      bytes += 8;
+    } else if (typeof val === "boolean") {
+      bytes += 4;
+    } else if (typeof val === "string") {
+      bytes += val.length * 2;
+    } else if (val instanceof Float32Array) {
+      bytes += val.byteLength;
+    }
+  }
+  return bytes;
+}
+
+/**
+ * Estimate total byte size for a page of rows.
+ */
+function estimatePageBytes(rows: Row[]): number {
+  let total = 0;
+  for (let i = 0; i < rows.length; i++) {
+    total += estimateRowBytes(rows[i]);
+  }
+  return total;
+}
+
+/**
+ * Page-level LRU cache for materialized query results.
+ *
+ * Keyed by query hash + page offset. Evicts least-recently-accessed
+ * entries when the byte budget is exceeded.
+ */
+export class MaterializationCache {
+  private readonly maxBytes: number;
+  private readonly entries: Map<string, CacheEntry> = new Map();
+  private _bytesUsed = 0;
+  private _hits = 0;
+  private _misses = 0;
+
+  constructor(opts?: MaterializationCacheOptions) {
+    this.maxBytes = opts?.maxBytes ?? DEFAULT_MAX_BYTES;
+  }
+
+  /** Store a page of rows under a query hash + offset. */
+  set(queryHash: string, offset: number, rows: Row[]): void {
+    const key = `${queryHash}:${offset}`;
+    // Remove existing entry if present (to update size accounting)
+    const existing = this.entries.get(key);
+    if (existing) {
+      this._bytesUsed -= existing.byteSize;
+      this.entries.delete(key);
+    }
+
+    const byteSize = estimatePageBytes(rows);
+
+    // If single entry exceeds budget, don't cache it
+    if (byteSize > this.maxBytes) return;
+
+    // Evict LRU entries until there's room
+    while (this._bytesUsed + byteSize > this.maxBytes && this.entries.size > 0) {
+      this.evictLRU();
+    }
+
+    this.entries.set(key, {
+      rows,
+      byteSize,
+      lastAccess: performance.now(),
+    });
+    this._bytesUsed += byteSize;
+  }
+
+  /**
+   * Retrieve a cached page. Returns null on miss.
+   *
+   * A get for offset=100, limit=50 succeeds if there is a cached page
+   * starting at offset=100 that contains at least 50 rows.
+   */
+  get(queryHash: string, offset: number, limit: number): Row[] | null {
+    const key = `${queryHash}:${offset}`;
+    const entry = this.entries.get(key);
+    if (!entry || entry.rows.length < limit) {
+      this._misses++;
+      return null;
+    }
+    // Update last access for LRU
+    entry.lastAccess = performance.now();
+    this._hits++;
+    // Return exactly `limit` rows if the cached page has more
+    return entry.rows.length === limit ? entry.rows : entry.rows.slice(0, limit);
+  }
+
+  /** Invalidate all cached pages for a query. */
+  invalidate(queryHash: string): void {
+    const prefix = `${queryHash}:`;
+    for (const [key, entry] of this.entries) {
+      if (key.startsWith(prefix)) {
+        this._bytesUsed -= entry.byteSize;
+        this.entries.delete(key);
+      }
+    }
+  }
+
+  /** Clear the entire cache. */
+  clear(): void {
+    this.entries.clear();
+    this._bytesUsed = 0;
+  }
+
+  /** Cache statistics. */
+  get stats(): { hits: number; misses: number; bytesUsed: number; entries: number } {
+    return {
+      hits: this._hits,
+      misses: this._misses,
+      bytesUsed: this._bytesUsed,
+      entries: this.entries.size,
+    };
+  }
+
+  /** Evict the least-recently-accessed entry. */
+  private evictLRU(): void {
+    let oldestKey: string | undefined;
+    let oldestTime = Infinity;
+    for (const [key, entry] of this.entries) {
+      if (entry.lastAccess < oldestTime) {
+        oldestTime = entry.lastAccess;
+        oldestKey = key;
+      }
+    }
+    if (oldestKey !== undefined) {
+      const entry = this.entries.get(oldestKey)!;
+      this._bytesUsed -= entry.byteSize;
+      this.entries.delete(oldestKey);
+    }
+  }
+}
+
+/**
+ * Create a deterministic hash key from a QueryDescriptor, excluding
+ * offset and limit (since those define the page, not the query identity).
+ */
+export function queryHashKey(desc: QueryDescriptor): string {
+  const parts: string[] = [desc.table];
+
+  // Filters — sorted by column for determinism
+  if (desc.filters.length > 0) {
+    const sorted = [...desc.filters].sort((a, b) => a.column.localeCompare(b.column));
+    for (const f of sorted) {
+      parts.push(`f:${f.column}:${f.op}:${stringifyValue(f.value)}`);
+    }
+  }
+
+  // Projections — sorted
+  if (desc.projections.length > 0) {
+    parts.push(`p:${[...desc.projections].sort().join(",")}`);
+  }
+
+  // Sort
+  if (desc.sortColumn) {
+    parts.push(`s:${desc.sortColumn}:${desc.sortDirection ?? "asc"}`);
+  }
+
+  // Aggregates
+  if (desc.aggregates && desc.aggregates.length > 0) {
+    for (const agg of desc.aggregates) {
+      parts.push(`a:${agg.fn}:${agg.column}${agg.alias ? `:${agg.alias}` : ""}`);
+    }
+  }
+
+  // Group by
+  if (desc.groupBy && desc.groupBy.length > 0) {
+    parts.push(`g:${[...desc.groupBy].sort().join(",")}`);
+  }
+
+  // Vector search
+  if (desc.vectorSearch) {
+    const vs = desc.vectorSearch;
+    parts.push(`v:${vs.column}:${vs.topK}:${Array.from(vs.queryVector).join(",")}`);
+  }
+
+  // Join
+  if (desc.join) {
+    const j = desc.join;
+    parts.push(`j:${j.leftKey}:${j.rightKey}:${j.type ?? "inner"}:${queryHashKey(j.right as QueryDescriptor)}`);
+  }
+
+  return parts.join("|");
+}
+
+function stringifyValue(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map(stringifyValue).join(",")}]`;
+  }
+  if (typeof value === "bigint") {
+    return `${value}n`;
+  }
+  return String(value);
+}