fix: Lance v2 nullable columns, column validation, cursor batching, BigInt export

- Rewrite Lance v2 parser to handle nullable columns with 64-byte aligned
  bitmap+data layout (all row slots stored, null bitmap as mask)
- Add column validation in dataset queries (filter, projection, sort)
- Fix cursor to respect batchSize via row-level chunking
- Export bigIntReplacer from both entry points for JSON serialization
- Fix CLAUDE.md API examples to match actual filter(col, op, val) syntax
- Add dataOffsetInPage to PageInfo for Lance v2 alignment padding
- Copy WASM binary in build:ts script

Author: teamchong

.claude/CLAUDE.md

@@ -162,14 +162,14 @@ import { QueryMode } from "querymode"
 const local = QueryMode.local()
 const results = await local
   .table("./data/users.lance")
-  .filter(r => r.age > 25)
+  .filter("age", "gt", 25)
   .exec()
 
 // Edge mode (Durable Objects)
 const edge = QueryMode.remote(env.QUERY_DO)
 const results = await edge
   .table("users")
-  .filter(r => r.age > 25)
+  .filter("age", "gt", 25)
   .select("name", "email")
   .sort("age", "desc")
   .limit(100)
@@ -184,15 +184,13 @@ const similar = await edge
 // JOIN (code, not SQL)
 const orders = edge.table("orders")
 const users = edge.table("users")
-const results = await edge.query(async () => {
-  const topOrders = await orders.filter(r => r.amount > 100).exec()
-  const userIds = topOrders.map(r => r.user_id)
-  const orderUsers = await users.filter(r => userIds.includes(r.id)).exec()
-  return topOrders.map(o => ({
-    ...o,
-    user: orderUsers.find(u => u.id === o.user_id)
-  }))
-})
+const topOrders = await orders.filter("amount", "gt", 100).exec()
+const userIds = topOrders.rows.map(r => r.user_id)
+const orderUsers = await users.filter("id", "in", userIds).exec()
+const joined = topOrders.rows.map(o => ({
+  ...o,
+  user: orderUsers.rows.find(u => u.id === o.user_id)
+}))
 
 // Same API works everywhere:
 // - Browser (WASM + fetch)
@@ -202,7 +200,7 @@ const results = await edge.query(async () => {
 
 API surface:
 - `.table(name)` — typed handle (schema from cached footer)
-- `.filter(predicate)` — pushdown to page-level skipping via min/max stats
+- `.filter(column, op, value)` — pushdown to page-level skipping via min/max stats
 - `.select(...columns)` — column projection (only fetch needed byte ranges)
 - `.sort(column, dir)` — use index if available, otherwise top-K heap
 - `.limit(n)` — early termination
@@ -309,7 +307,7 @@ tools: [{
 
 // Agent generates code directly — no SQL to learn
 await query_data({
-  code: `table.filter(r => r.embedding.similarity(queryVec) > 0.8)`
+  code: `table.vector("embedding", queryVec, 10)`
 })
 ```
 

package.json

@@ -18,9 +18,9 @@
   "scripts": {
     "dev": "wrangler dev",
     "prepublishOnly": "pnpm run build",
-    "build": "pnpm run build:wasm && tsc",
+    "build": "pnpm run build:wasm && pnpm run build:ts",
     "build:wasm": "cd wasm && zig build wasm && mkdir -p ../src/wasm && cp zig-out/bin/querymode.wasm ../src/wasm/",
-    "build:ts": "tsc",
+    "build:ts": "tsc && mkdir -p dist/wasm && cp src/wasm/querymode.wasm dist/wasm/",
     "test": "vitest run",
     "test:watch": "vitest",
     "deploy": "pnpm run build && wrangler deploy",

src/client.ts

@@ -14,7 +14,7 @@ import type {
  * Usage:
  *   const results = await querymode
  *     .table("users")
- *     .filter(r => r.age > 25)
+ *     .filter("age", "gt", 25)
  *     .select("name", "email")
  *     .sort("age", "desc")
  *     .limit(100)

src/decode.ts

@@ -2,6 +2,7 @@ import type { ColumnMeta, PageInfo, Row, DataType } from "./types.js";
 import type { QueryDescriptor } from "./client.js";
 import type { WasmEngine } from "./wasm-engine.js";
 import { decodeParquetColumnChunk, decodePlainValues } from "./parquet-decode.js";
+import { decodeLanceV2Utf8 } from "./lance-v2.js";
 
 /** Check if a page can be skipped via min/max stats. */
 export function canSkipPage(page: PageInfo, filters: QueryDescriptor["filters"], columnName: string): boolean {
@@ -94,7 +95,7 @@ function decodeAllColumns(
         const pi = col.pages[i];
         const decoded = pi?.encoding
           ? decodeParquetColumnChunk(pages[i], pi.encoding, col.dtype, pi.rowCount, wasm)
-          : decodePage(pages[i], col.dtype, pi?.nullCount ?? 0, pi?.rowCount ?? 0);
+          : decodePage(pages[i], col.dtype, pi?.nullCount ?? 0, pi?.rowCount ?? 0, pi?.dataOffsetInPage);
         for (const v of decoded) {
           values.push(v);
         }
@@ -121,12 +122,15 @@ function decodeFixedSizeListPages(pages: ArrayBuffer[], dim: number): Float32Arr
 }
 
 
-/** Decode a raw page buffer into typed values. Handles null bitmaps when nullCount > 0. */
+/** Decode a raw page buffer into typed values. Handles null bitmaps when nullCount > 0.
+ * @param dataOffsetInPage - For Lance v2 nullable pages: byte offset where data starts (after bitmap + alignment padding)
+ */
 export function decodePage(
   buf: ArrayBuffer,
   dtype: string,
   nullCount = 0,
   rowCount = 0,
+  dataOffsetInPage?: number,
 ): (number | bigint | string | null)[] {
   let nulls: Set<number> | null = null;
   if (nullCount > 0 && rowCount > 0) {
@@ -140,14 +144,55 @@ export function decodePage(
         if (((bytes[b] >> bit) & 1) === 0) nulls.add(idx);
       }
     }
-    buf = buf.slice(bitmapBytes);
+    // Lance v2 uses alignment padding between bitmap and data.
+    // dataOffsetInPage gives the exact data start; otherwise strip only bitmap bytes.
+    const stripBytes = dataOffsetInPage ?? bitmapBytes;
+    buf = buf.slice(stripBytes);
   }
 
   const bytes = new Uint8Array(buf);
-  const numValues = rowCount > 0 ? rowCount - (nulls?.size ?? 0) : Number.MAX_SAFE_INTEGER;
+  // Lance v2 stores ALL row slots (including zeros at null positions).
+  // When dataOffsetInPage is set, decode all rowCount values and mask nulls.
+  // Parquet-style packs only non-null values, so decode rowCount - nulls.size.
+  const isLanceV2Nullable = dataOffsetInPage !== undefined;
+  const numValues = rowCount > 0
+    ? (isLanceV2Nullable ? rowCount : rowCount - (nulls?.size ?? 0))
+    : Number.MAX_SAFE_INTEGER;
+
+  // For utf8/binary with rowCount known and buffer large enough to contain an offsets array,
+  // try Lance v2 format (i64 offsets + string data) first.
+  if ((dtype === "utf8" || dtype === "binary") && rowCount > 0 && buf.byteLength >= rowCount * 8) {
+    const decodeCount = isLanceV2Nullable ? rowCount : numValues;
+    const v2Strings = decodeLanceV2Utf8(buf, decodeCount);
+    if (v2Strings.length === decodeCount && v2Strings.every(s => typeof s === "string")) {
+      const looksValid = v2Strings.every(s => s.length < buf.byteLength);
+      if (looksValid) {
+        if (nulls && nulls.size > 0) {
+          if (isLanceV2Nullable) {
+            // Lance v2: all slots present, just null-mask them
+            return v2Strings.map((s, i) => nulls!.has(i) ? null : s);
+          }
+          // Parquet-style: packed non-null values, interleave with nulls
+          const withNulls: (string | null)[] = [];
+          let vi = 0;
+          for (let i = 0; i < rowCount; i++) {
+            withNulls.push(nulls.has(i) ? null : (vi < v2Strings.length ? v2Strings[vi++] : null));
+          }
+          return withNulls;
+        }
+        return v2Strings;
+      }
+    }
+  }
+
   const values = decodePlainValues(bytes, dtype as DataType, numValues) as (number | bigint | string | null)[];
 
   if (nulls && nulls.size > 0) {
+    if (isLanceV2Nullable) {
+      // Lance v2: all row slots present, replace null positions with null
+      return values.map((v, i) => nulls!.has(i) ? null : v);
+    }
+    // Parquet-style: packed non-null values, interleave with nulls
     const withNulls: (number | bigint | string | null)[] = [];
     let vi = 0;
     for (let i = 0; i < rowCount; i++) {

src/index.ts

@@ -10,6 +10,7 @@ export { FragmentDO } from "./fragment-do.js";
 export { TableQuery } from "./client.js";
 export { QueryModeError } from "./errors.js";
 export { LocalExecutor } from "./local-executor.js";
+export { bigIntReplacer } from "./decode.js";
 export type { QueryExecutor, QueryDescriptor } from "./client.js";
 export type {
   Env,