fix: remove non-columnar formats from R2 storage claim; add SQL parser features

- Remove CSV, JSON, Arrow from "where data lives" section — these formats
  don't support range requests, only columnar formats (Parquet, Lance, Iceberg) do
- Add EXISTS/NOT EXISTS, NATURAL JOIN, USING, parameter binding, SHOW VERSIONS,
  DIFF to SQL parser with full test coverage

Author: teamchong

docs/src/content/docs/why-querymode.mdx

@@ -58,7 +58,7 @@ Three analyses on one result set. No SQL string construction, no JSON parsing, n
 
 ### Where the data lives
 
-Data sits in **R2** as columnar files (Parquet, Lance, Iceberg, CSV, JSON, Arrow). Nothing gets replicated to 300 edge nodes. Regional Query DOs cache table footers (~4KB each) and read data pages from R2 via coalesced HTTP range requests (~10ms).
+Data sits in **R2** as columnar files (Parquet, Lance, Iceberg). Nothing gets replicated to 300 edge nodes. Regional Query DOs cache table footers (~4KB each) and read data pages from R2 via coalesced HTTP range requests (~10ms).
 
 "Data at the edge" means metadata cached locally, pages fetched on demand with free egress. Not replicated databases.
 

src/sql/ast.ts

@@ -26,7 +26,9 @@ export type SqlExpr =
   | { kind: "case_expr"; operand?: SqlExpr; whenClauses: CaseWhen[]; elseResult?: SqlExpr }
   | { kind: "cast"; expr: SqlExpr; targetType: string }
   | { kind: "star" }
-  | { kind: "near"; column: SqlExpr; vector: number[]; topK?: number };
+  | { kind: "near"; column: SqlExpr; vector: number[]; topK?: number }
+  | { kind: "exists"; subquery: SelectStmt; negated: boolean }
+  | { kind: "parameter"; index: number };
 
 export interface CaseWhen {
   condition: SqlExpr;
@@ -57,6 +59,8 @@ export interface JoinClause {
   joinType: JoinType;
   table: TableRef;
   onCondition?: SqlExpr;
+  natural?: boolean;
+  using?: string[];
 }
 
 export type TableRef =
@@ -91,3 +95,20 @@ export interface SelectStmt {
   offset?: number;
   setOperation?: SetOperation;
 }
+
+export interface ShowVersionsStmt {
+  table: string;
+  limit?: number;
+}
+
+export interface DiffStmt {
+  table: string;
+  fromVersion: number;
+  toVersion?: number;
+  limit?: number;
+}
+
+export type SqlStatement =
+  | { kind: "select"; stmt: SelectStmt }
+  | { kind: "show_versions"; stmt: ShowVersionsStmt }
+  | { kind: "diff"; stmt: DiffStmt };

src/sql/lexer.test.ts

@@ -118,4 +118,28 @@ describe("SQL Lexer", () => {
     expect(() => tokenize("SELECT `unclosed")).toThrow(SqlLexerError);
     expect(() => tokenize("SELECT `unclosed")).toThrow(/Unterminated backtick identifier/);
   });
+
+  it("tokenizes DDL keywords", () => {
+    const tokens = tokenize("CREATE TABLE IF NOT EXISTS DROP ALTER INDEX VECTOR SHOW INDEXES");
+    expect(tokens[0].type).toBe(TokenType.CREATE);
+    expect(tokens[1].type).toBe(TokenType.TABLE);
+    expect(tokens[2].type).toBe(TokenType.IF);
+    expect(tokens[3].type).toBe(TokenType.NOT);
+    expect(tokens[4].type).toBe(TokenType.EXISTS);
+    expect(tokens[5].type).toBe(TokenType.DROP);
+    expect(tokens[6].type).toBe(TokenType.ALTER);
+    expect(tokens[7].type).toBe(TokenType.INDEX);
+    expect(tokens[8].type).toBe(TokenType.VECTOR);
+    expect(tokens[9].type).toBe(TokenType.SHOW);
+    expect(tokens[10].type).toBe(TokenType.INDEXES);
+    expect(tokens[11].type).toBe(TokenType.EOF);
+  });
+
+  it("tokenizes ? as PARAMETER token", () => {
+    const tokens = tokenize("SELECT * FROM t WHERE id = ? AND name = ?");
+    // Tokens: SELECT * FROM t WHERE id = ? AND name = ? EOF
+    //         0      1 2    3 4     5  6 7 8   9    10 11 12
+    expect(tokens[7]).toMatchObject({ type: TokenType.PARAMETER, lexeme: "?" });
+    expect(tokens[11]).toMatchObject({ type: TokenType.PARAMETER, lexeme: "?" });
+  });
 });

src/sql/lexer.ts

@@ -6,15 +6,24 @@ export enum TokenType {
   IS, NULL, AS, DISTINCT, ORDER, BY, ASC, DESC, LIMIT, OFFSET,
   GROUP, HAVING,
 
+  // DDL keywords
+  CREATE, DROP, ALTER, TABLE, INDEX, VECTOR, IF, SHOW, INDEXES,
+
+  // Time travel / diff keywords
+  DIFF, VERSION, VERSIONS, CHANGES, SINCE, FOR, HEAD,
+
   // JOIN keywords
-  JOIN, LEFT, RIGHT, INNER, OUTER, FULL, CROSS, ON,
+  JOIN, LEFT, RIGHT, INNER, OUTER, FULL, CROSS, ON, NATURAL, USING,
 
   // Set operations
   UNION, INTERSECT, EXCEPT, ALL,
 
   // CASE
   CASE, WHEN, THEN, ELSE, END,
 
+  // Subquery keywords
+  EXISTS,
+
   // Type casting
   CAST,
 
@@ -24,11 +33,20 @@ export enum TokenType {
   // Vector search
   NEAR, TOPK,
 
+  // Logic table extension
+  WITH, DATA, LOGIC_TABLE, LOGIC,
+
   // Aggregate functions (recognized as keywords for cleaner parsing)
   COUNT, SUM, AVG, MIN, MAX,
 
-  // Window function names
-  ROW_NUMBER, RANK, DENSE_RANK, LAG, LEAD,
+  // Ranking functions
+  ROW_NUMBER, RANK, DENSE_RANK, NTILE, PERCENT_RANK, CUME_DIST,
+
+  // Offset/Analytic functions
+  LAG, LEAD, FIRST_VALUE, LAST_VALUE, NTH_VALUE,
+
+  // Time window functions
+  INTERVAL, SESSION, TUMBLE, HOP,
 
   // Boolean
   TRUE, FALSE,
@@ -40,6 +58,7 @@ export enum TokenType {
   STAR, EQ, NE, LT, LE, GT, GE,
   PLUS, MINUS, SLASH, CONCAT,
   COMMA, DOT, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET,
+  PARAMETER,
 
   EOF,
 }
@@ -58,23 +77,39 @@ const KEYWORDS = new Map<string, TokenType>([
   ["DISTINCT", TokenType.DISTINCT], ["ORDER", TokenType.ORDER], ["BY", TokenType.BY],
   ["ASC", TokenType.ASC], ["DESC", TokenType.DESC], ["LIMIT", TokenType.LIMIT],
   ["OFFSET", TokenType.OFFSET], ["GROUP", TokenType.GROUP], ["HAVING", TokenType.HAVING],
+  ["CREATE", TokenType.CREATE], ["DROP", TokenType.DROP], ["ALTER", TokenType.ALTER],
+  ["TABLE", TokenType.TABLE], ["INDEX", TokenType.INDEX], ["VECTOR", TokenType.VECTOR],
+  ["IF", TokenType.IF], ["SHOW", TokenType.SHOW], ["INDEXES", TokenType.INDEXES],
+  ["DIFF", TokenType.DIFF], ["VERSION", TokenType.VERSION], ["VERSIONS", TokenType.VERSIONS],
+  ["CHANGES", TokenType.CHANGES], ["SINCE", TokenType.SINCE], ["FOR", TokenType.FOR],
+  ["HEAD", TokenType.HEAD],
   ["JOIN", TokenType.JOIN], ["LEFT", TokenType.LEFT], ["RIGHT", TokenType.RIGHT],
   ["INNER", TokenType.INNER], ["OUTER", TokenType.OUTER], ["FULL", TokenType.FULL],
-  ["CROSS", TokenType.CROSS], ["ON", TokenType.ON],
+  ["CROSS", TokenType.CROSS], ["ON", TokenType.ON], ["NATURAL", TokenType.NATURAL],
+  ["USING", TokenType.USING],
   ["UNION", TokenType.UNION], ["INTERSECT", TokenType.INTERSECT],
   ["EXCEPT", TokenType.EXCEPT], ["ALL", TokenType.ALL],
   ["CASE", TokenType.CASE], ["WHEN", TokenType.WHEN], ["THEN", TokenType.THEN],
   ["ELSE", TokenType.ELSE], ["END", TokenType.END],
+  ["EXISTS", TokenType.EXISTS],
   ["CAST", TokenType.CAST],
   ["OVER", TokenType.OVER], ["PARTITION", TokenType.PARTITION],
   ["ROWS", TokenType.ROWS], ["RANGE", TokenType.RANGE],
   ["UNBOUNDED", TokenType.UNBOUNDED], ["PRECEDING", TokenType.PRECEDING],
   ["FOLLOWING", TokenType.FOLLOWING], ["CURRENT", TokenType.CURRENT],
   ["NEAR", TokenType.NEAR], ["TOPK", TokenType.TOPK],
+  ["WITH", TokenType.WITH], ["DATA", TokenType.DATA],
+  ["LOGIC_TABLE", TokenType.LOGIC_TABLE], ["LOGIC", TokenType.LOGIC],
   ["COUNT", TokenType.COUNT], ["SUM", TokenType.SUM], ["AVG", TokenType.AVG],
   ["MIN", TokenType.MIN], ["MAX", TokenType.MAX],
   ["ROW_NUMBER", TokenType.ROW_NUMBER], ["RANK", TokenType.RANK],
-  ["DENSE_RANK", TokenType.DENSE_RANK], ["LAG", TokenType.LAG], ["LEAD", TokenType.LEAD],
+  ["DENSE_RANK", TokenType.DENSE_RANK], ["NTILE", TokenType.NTILE],
+  ["PERCENT_RANK", TokenType.PERCENT_RANK], ["CUME_DIST", TokenType.CUME_DIST],
+  ["LAG", TokenType.LAG], ["LEAD", TokenType.LEAD],
+  ["FIRST_VALUE", TokenType.FIRST_VALUE], ["LAST_VALUE", TokenType.LAST_VALUE],
+  ["NTH_VALUE", TokenType.NTH_VALUE],
+  ["INTERVAL", TokenType.INTERVAL], ["SESSION", TokenType.SESSION],
+  ["TUMBLE", TokenType.TUMBLE], ["HOP", TokenType.HOP],
   ["TRUE", TokenType.TRUE], ["FALSE", TokenType.FALSE],
 ]);
 
@@ -158,6 +193,7 @@ export function tokenize(sql: string): Token[] {
       case "/": tokens.push({ type: TokenType.SLASH, lexeme: "/", position: start }); break;
       case "[": tokens.push({ type: TokenType.LBRACKET, lexeme: "[", position: start }); break;
       case "]": tokens.push({ type: TokenType.RBRACKET, lexeme: "]", position: start }); break;
+      case "?": tokens.push({ type: TokenType.PARAMETER, lexeme: "?", position: start }); break;
       case "=": tokens.push({ type: TokenType.EQ, lexeme: "=", position: start }); break;
       case "!":
         if (pos < len && sql[pos] === "=") {

src/sql/parser.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { parse } from "./parser.js";
+import { parse, parseStatement } from "./parser.js";
 import { SqlParseError } from "./parser.js";
 
 describe("SQL Parser", () => {
@@ -251,4 +251,138 @@ describe("SQL Parser", () => {
       }
     }
   });
+
+  it("parses EXISTS subquery", () => {
+    const stmt = parse("SELECT * FROM t WHERE EXISTS (SELECT id FROM u WHERE u.tid = t.id)");
+    expect(stmt.where!.kind).toBe("exists");
+    if (stmt.where!.kind === "exists") {
+      expect(stmt.where!.negated).toBe(false);
+      expect(stmt.where!.subquery.from).toEqual({ kind: "simple", name: "u", alias: undefined });
+    }
+  });
+
+  it("parses NOT EXISTS subquery", () => {
+    const stmt = parse("SELECT * FROM t WHERE NOT EXISTS (SELECT id FROM u WHERE u.tid = t.id)");
+    expect(stmt.where!.kind).toBe("exists");
+    if (stmt.where!.kind === "exists") {
+      expect(stmt.where!.negated).toBe(true);
+      expect(stmt.where!.subquery.from).toEqual({ kind: "simple", name: "u", alias: undefined });
+    }
+  });
+
+  it("parses NATURAL JOIN", () => {
+    const stmt = parse("SELECT * FROM a NATURAL JOIN b");
+    expect(stmt.from.kind).toBe("join");
+    if (stmt.from.kind === "join") {
+      expect(stmt.from.join.natural).toBe(true);
+      expect(stmt.from.join.joinType).toBe("inner");
+      expect(stmt.from.join.table).toEqual({ kind: "simple", name: "b", alias: undefined });
+    }
+  });
+
+  it("parses NATURAL LEFT JOIN", () => {
+    const stmt = parse("SELECT * FROM a NATURAL LEFT JOIN b");
+    expect(stmt.from.kind).toBe("join");
+    if (stmt.from.kind === "join") {
+      expect(stmt.from.join.natural).toBe(true);
+      expect(stmt.from.join.joinType).toBe("left");
+    }
+  });
+
+  it("parses JOIN with USING clause", () => {
+    const stmt = parse("SELECT * FROM a JOIN b USING (id)");
+    expect(stmt.from.kind).toBe("join");
+    if (stmt.from.kind === "join") {
+      expect(stmt.from.join.using).toEqual(["id"]);
+      expect(stmt.from.join.onCondition).toBeUndefined();
+    }
+  });
+
+  it("parses JOIN with USING clause with multiple columns", () => {
+    const stmt = parse("SELECT * FROM a JOIN b USING (id, name)");
+    expect(stmt.from.kind).toBe("join");
+    if (stmt.from.kind === "join") {
+      expect(stmt.from.join.using).toEqual(["id", "name"]);
+    }
+  });
+
+  it("parses ? parameter binding", () => {
+    const stmt = parse("SELECT * FROM t WHERE id = ?");
+    expect(stmt.where!.kind).toBe("binary");
+    if (stmt.where!.kind === "binary") {
+      expect(stmt.where!.right.kind).toBe("parameter");
+      if (stmt.where!.right.kind === "parameter") {
+        expect(stmt.where!.right.index).toBe(0);
+      }
+    }
+  });
+
+  it("parses multiple ? parameters with incrementing indices", () => {
+    const stmt = parse("SELECT * FROM t WHERE a = ? AND b = ?");
+    if (stmt.where!.kind === "binary" && stmt.where!.op === "and") {
+      const left = stmt.where!.left;
+      const right = stmt.where!.right;
+      if (left.kind === "binary" && left.right.kind === "parameter") {
+        expect(left.right.index).toBe(0);
+      }
+      if (right.kind === "binary" && right.right.kind === "parameter") {
+        expect(right.right.index).toBe(1);
+      }
+    }
+  });
+});
+
+describe("SQL Statement Parser", () => {
+  it("parses SHOW VERSIONS FOR table", () => {
+    const result = parseStatement("SHOW VERSIONS FOR users");
+    expect(result.kind).toBe("show_versions");
+    if (result.kind === "show_versions") {
+      expect(result.stmt.table).toBe("users");
+      expect(result.stmt.limit).toBeUndefined();
+    }
+  });
+
+  it("parses SHOW VERSIONS FOR table LIMIT", () => {
+    const result = parseStatement("SHOW VERSIONS FOR users LIMIT 10");
+    if (result.kind === "show_versions") {
+      expect(result.stmt.table).toBe("users");
+      expect(result.stmt.limit).toBe(10);
+    }
+  });
+
+  it("parses DIFF table VERSION n", () => {
+    const result = parseStatement("DIFF users VERSION 3");
+    expect(result.kind).toBe("diff");
+    if (result.kind === "diff") {
+      expect(result.stmt.table).toBe("users");
+      expect(result.stmt.fromVersion).toBe(3);
+      expect(result.stmt.toVersion).toBeUndefined();
+    }
+  });
+
+  it("parses DIFF table VERSION n AND VERSION m", () => {
+    const result = parseStatement("DIFF users VERSION 2 AND VERSION 5");
+    if (result.kind === "diff") {
+      expect(result.stmt.table).toBe("users");
+      expect(result.stmt.fromVersion).toBe(2);
+      expect(result.stmt.toVersion).toBe(5);
+    }
+  });
+
+  it("parses DIFF with LIMIT", () => {
+    const result = parseStatement("DIFF users VERSION 1 AND VERSION 3 LIMIT 50");
+    if (result.kind === "diff") {
+      expect(result.stmt.fromVersion).toBe(1);
+      expect(result.stmt.toVersion).toBe(3);
+      expect(result.stmt.limit).toBe(50);
+    }
+  });
+
+  it("parses SELECT via parseStatement", () => {
+    const result = parseStatement("SELECT * FROM t");
+    expect(result.kind).toBe("select");
+    if (result.kind === "select") {
+      expect(result.stmt.from).toEqual({ kind: "simple", name: "t", alias: undefined });
+    }
+  });
 });