Skip to content

Commit 93e7362

Browse files
teamchongteamchong
committed
ci: add conformance + operator benchmark job with runner summary
- Add Vitest bench file comparing QueryMode (Miniflare full DO stack) vs DuckDB (native Node FFI) across 7 query patterns - Add `conformance` CI job: runs operator conformance tests, then head-to-head benchmarks with wrangler dev - Post benchmark results to GitHub Actions job summary for both the operator and E2E benchmark jobs - Add `bench:operators` package.json script
1 parent 61edde0 commit 93e7362

File tree

3 files changed

+311
-2
lines changed

3 files changed

+311
-2
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,61 @@ jobs:
3232
- name: Unit tests
3333
run: pnpm test
3434

35+
conformance:
36+
runs-on: ubuntu-latest
37+
needs: test
38+
steps:
39+
- uses: actions/checkout@v4
40+
41+
- uses: pnpm/action-setup@v4
42+
43+
- uses: actions/setup-node@v4
44+
with:
45+
node-version: 22
46+
cache: pnpm
47+
48+
- run: pnpm install
49+
50+
- name: Install Zig
51+
uses: mlugg/setup-zig@v2
52+
with:
53+
version: 0.15.2
54+
55+
- name: Build (WASM + TypeScript)
56+
run: pnpm build
57+
58+
- name: Conformance tests
59+
run: npx vitest run src/operators-conformance.test.ts
60+
61+
- name: Generate benchmark data
62+
run: npx tsx scripts/generate-bench-data.ts
63+
64+
- name: Start wrangler dev
65+
run: |
66+
pnpm dev &
67+
for i in $(seq 1 30); do
68+
if curl -sf http://localhost:8787/health > /dev/null 2>&1; then
69+
echo "Worker ready"
70+
break
71+
fi
72+
sleep 1
73+
done
74+
75+
- name: Seed local R2
76+
run: npx tsx scripts/seed-local-r2.ts
77+
78+
- name: Operator benchmarks (QueryMode vs DuckDB)
79+
run: npx vitest bench src/bench-vs-duckdb.bench.ts 2>&1 | tee /tmp/bench-output.txt
80+
81+
- name: Post benchmark results to summary
82+
if: always()
83+
run: |
84+
echo "## Operator Benchmarks — QueryMode (Miniflare) vs DuckDB (native)" >> $GITHUB_STEP_SUMMARY
85+
echo "" >> $GITHUB_STEP_SUMMARY
86+
echo '```' >> $GITHUB_STEP_SUMMARY
87+
cat /tmp/bench-output.txt >> $GITHUB_STEP_SUMMARY
88+
echo '```' >> $GITHUB_STEP_SUMMARY
89+
3590
bench:
3691
runs-on: ubuntu-latest
3792
needs: test
@@ -73,4 +128,13 @@ jobs:
73128
run: npx tsx scripts/seed-local-r2.ts
74129

75130
- name: Run benchmarks
76-
run: npx tsx scripts/bench.ts
131+
run: npx tsx scripts/bench.ts 2>&1 | tee /tmp/bench-output.txt
132+
133+
- name: Post benchmark results to summary
134+
if: always()
135+
run: |
136+
echo "## E2E Benchmarks — Full DO Stack" >> $GITHUB_STEP_SUMMARY
137+
echo "" >> $GITHUB_STEP_SUMMARY
138+
echo '```' >> $GITHUB_STEP_SUMMARY
139+
cat /tmp/bench-output.txt >> $GITHUB_STEP_SUMMARY
140+
echo '```' >> $GITHUB_STEP_SUMMARY

‎package.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@
2626
"deploy": "pnpm run build && wrangler deploy",
2727
"wasm": "pnpm run build:wasm",
2828
"bench:seed": "npx tsx scripts/seed-local-r2.ts",
29-
"bench": "npx tsx scripts/bench.ts"
29+
"bench": "npx tsx scripts/bench.ts",
30+
"bench:operators": "vitest bench src/bench-vs-duckdb.bench.ts"
3031
},
3132
"devDependencies": {
3233
"@cloudflare/workers-types": "^4.20250224.0",
3334
"@types/node": "^25.3.3",
35+
"duckdb": "^1.4.4",
3436
"typescript": "^5.7.0",
3537
"vitest": "^3.0.0",
3638
"wrangler": "^4.0.0"
@@ -43,5 +45,8 @@
4345
"packageManager": "pnpm@10.21.0",
4446
"dependencies": {
4547
"zod": "^4.3.6"
48+
},
49+
"pnpm": {
50+
"onlyBuiltDependencies": ["duckdb"]
4651
}
4752
}

‎src/bench-vs-duckdb.bench.ts‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
/**
2+
* Head-to-head benchmarks: QueryMode (Miniflare, full DO stack) vs DuckDB (native Node).
3+
*
4+
* QueryMode runs on the real CF Worker runtime via wrangler dev:
5+
* HTTP → Worker → Query DO → R2 → WASM decode → operators → response
6+
*
7+
* DuckDB runs natively in Node.js — no serialization, no HTTP, no Worker overhead.
8+
*
9+
* Prerequisites:
10+
* 1. `pnpm dev` running on localhost:8787
11+
* 2. `npx tsx scripts/generate-bench-data.ts`
12+
* 3. `npx tsx scripts/seed-local-r2.ts`
13+
*
14+
* Usage: pnpm bench:operators
15+
*/
16+
17+
import { describe, bench, beforeAll, afterAll } from "vitest";
18+
import duckdb from "duckdb";
19+
20+
// ---------------------------------------------------------------------------
21+
// QueryMode (Miniflare) helpers
22+
// ---------------------------------------------------------------------------
23+
24+
const BASE_URL = process.env.WORKER_URL ?? "http://localhost:8787";
25+
26+
async function qmQuery(body: unknown): Promise<Record<string, unknown>> {
27+
const resp = await fetch(`${BASE_URL}/query`, {
28+
method: "POST",
29+
headers: { "content-type": "application/json" },
30+
body: JSON.stringify(body),
31+
});
32+
if (!resp.ok) throw new Error(`QueryMode ${resp.status}: ${(await resp.text()).slice(0, 200)}`);
33+
return resp.json() as Promise<Record<string, unknown>>;
34+
}
35+
36+
// ---------------------------------------------------------------------------
37+
// DuckDB helpers
38+
// ---------------------------------------------------------------------------
39+
40+
let db: duckdb.Database;
41+
let con: duckdb.Connection;
42+
43+
function duckRun(sql: string): Promise<void> {
44+
return new Promise((resolve, reject) => {
45+
con.run(sql, (err: Error | null) => {
46+
if (err) reject(err);
47+
else resolve();
48+
});
49+
});
50+
}
51+
52+
function duckQuery(sql: string): Promise<Record<string, unknown>[]> {
53+
return new Promise((resolve, reject) => {
54+
con.all(sql, (err: Error | null, rows: Record<string, unknown>[]) => {
55+
if (err) reject(err);
56+
else resolve(rows);
57+
});
58+
});
59+
}
60+
61+
// ---------------------------------------------------------------------------
62+
// Setup: DuckDB in-memory tables matching the seeded R2 Parquet data
63+
// ---------------------------------------------------------------------------
64+
65+
beforeAll(async () => {
66+
// Verify worker is reachable
67+
const health = await fetch(`${BASE_URL}/health`);
68+
if (!health.ok) throw new Error(`Worker not reachable at ${BASE_URL}. Is 'pnpm dev' running?`);
69+
70+
// Warm up Query DO registration
71+
await fetch(`${BASE_URL}/tables`);
72+
73+
db = new duckdb.Database(":memory:");
74+
con = new duckdb.Connection(db);
75+
76+
// bench_1m_numeric: id (BIGINT), value (DOUBLE) — 1M rows, deterministic
77+
// Matches generate-bench-data.ts: ids 0..999999, values random * 100000
78+
// For fair comparison, use the same deterministic pattern
79+
await duckRun(`CREATE TABLE bench_1m AS SELECT i AS id, (i * 7 + 13) % 10000 AS value FROM generate_series(0, 999999) t(i)`);
80+
81+
// bench_100k_3col: id (BIGINT), value (DOUBLE), category (VARCHAR) — 100K rows
82+
const cats = ["alpha", "beta", "gamma", "delta", "epsilon"];
83+
await duckRun(`
84+
CREATE TABLE bench_100k AS
85+
SELECT i AS id,
86+
(i * 7 + 13) % 1000 AS value,
87+
CASE i % 5
88+
WHEN 0 THEN '${cats[0]}'
89+
WHEN 1 THEN '${cats[1]}'
90+
WHEN 2 THEN '${cats[2]}'
91+
WHEN 3 THEN '${cats[3]}'
92+
ELSE '${cats[4]}'
93+
END AS category
94+
FROM generate_series(0, 99999) t(i)
95+
`);
96+
}, 60_000);
97+
98+
afterAll(() => {
99+
con?.close();
100+
db?.close();
101+
});
102+
103+
// ===================================================================
104+
// 1. Full scan — 1M rows, 2 numeric columns
105+
// ===================================================================
106+
107+
describe("Full scan 1M×2col numeric", () => {
108+
bench("QueryMode (Miniflare)", async () => {
109+
await qmQuery({
110+
table: "bench_1m_numeric",
111+
filters: [],
112+
projections: [],
113+
});
114+
}, { time: 30_000, warmupIterations: 2 });
115+
116+
bench("DuckDB (native)", async () => {
117+
await duckQuery(`SELECT * FROM bench_1m`);
118+
}, { time: 30_000, warmupIterations: 2 });
119+
});
120+
121+
// ===================================================================
122+
// 2. Filter scan — 1M rows, filter id > 900000 (~10% selectivity)
123+
// ===================================================================
124+
125+
describe("Filter scan 1M id>900000", () => {
126+
bench("QueryMode (Miniflare)", async () => {
127+
await qmQuery({
128+
table: "bench_1m_numeric",
129+
filters: [{ column: "id", op: "gt", value: 900000 }],
130+
projections: [],
131+
});
132+
}, { time: 30_000, warmupIterations: 2 });
133+
134+
bench("DuckDB (native)", async () => {
135+
await duckQuery(`SELECT * FROM bench_1m WHERE id > 900000`);
136+
}, { time: 30_000, warmupIterations: 2 });
137+
});
138+
139+
// ===================================================================
140+
// 3. Aggregate SUM — 1M rows
141+
// ===================================================================
142+
143+
describe("Aggregate SUM 1M", () => {
144+
bench("QueryMode (Miniflare)", async () => {
145+
await qmQuery({
146+
table: "bench_1m_numeric",
147+
filters: [],
148+
projections: [],
149+
aggregates: [{ fn: "sum", column: "value", alias: "total" }],
150+
});
151+
}, { time: 30_000, warmupIterations: 2 });
152+
153+
bench("DuckDB (native)", async () => {
154+
await duckQuery(`SELECT SUM(value) as total FROM bench_1m`);
155+
}, { time: 30_000, warmupIterations: 2 });
156+
});
157+
158+
// ===================================================================
159+
// 4. Aggregate group by category — 100K×3col
160+
// ===================================================================
161+
162+
describe("Aggregate group by category 100K", () => {
163+
bench("QueryMode (Miniflare)", async () => {
164+
await qmQuery({
165+
table: "bench_100k_3col",
166+
filters: [],
167+
projections: [],
168+
aggregates: [
169+
{ fn: "sum", column: "value", alias: "sum_value" },
170+
{ fn: "count", column: "id", alias: "cnt" },
171+
],
172+
groupBy: ["category"],
173+
});
174+
}, { time: 30_000, warmupIterations: 2 });
175+
176+
bench("DuckDB (native)", async () => {
177+
await duckQuery(
178+
`SELECT category, SUM(value) as sum_value, COUNT(id) as cnt
179+
FROM bench_100k GROUP BY category`,
180+
);
181+
}, { time: 30_000, warmupIterations: 2 });
182+
});
183+
184+
// ===================================================================
185+
// 5. Sort + Limit (TopK) — 1M rows, top 100
186+
// ===================================================================
187+
188+
describe("TopK 100 from 1M", () => {
189+
bench("QueryMode (Miniflare)", async () => {
190+
await qmQuery({
191+
table: "bench_1m_numeric",
192+
filters: [],
193+
projections: [],
194+
sortColumn: "value",
195+
sortDirection: "desc",
196+
limit: 100,
197+
});
198+
}, { time: 30_000, warmupIterations: 2 });
199+
200+
bench("DuckDB (native)", async () => {
201+
await duckQuery(`SELECT * FROM bench_1m ORDER BY value DESC LIMIT 100`);
202+
}, { time: 30_000, warmupIterations: 2 });
203+
});
204+
205+
// ===================================================================
206+
// 6. Column projection — 100K, select 1 of 3 columns
207+
// ===================================================================
208+
209+
describe("Projection 100K select 1 col", () => {
210+
bench("QueryMode (Miniflare)", async () => {
211+
await qmQuery({
212+
table: "bench_100k_3col",
213+
filters: [],
214+
projections: ["id"],
215+
});
216+
}, { time: 30_000, warmupIterations: 2 });
217+
218+
bench("DuckDB (native)", async () => {
219+
await duckQuery(`SELECT id FROM bench_100k`);
220+
}, { time: 30_000, warmupIterations: 2 });
221+
});
222+
223+
// ===================================================================
224+
// 7. Filter + Aggregate — 100K, filter + count
225+
// ===================================================================
226+
227+
describe("Filter + Count 100K id>50000", () => {
228+
bench("QueryMode (Miniflare)", async () => {
229+
await qmQuery({
230+
table: "bench_100k_3col",
231+
filters: [{ column: "id", op: "gt", value: 50000 }],
232+
projections: [],
233+
aggregates: [{ fn: "count", column: "id", alias: "cnt" }],
234+
});
235+
}, { time: 30_000, warmupIterations: 2 });
236+
237+
bench("DuckDB (native)", async () => {
238+
await duckQuery(`SELECT COUNT(id) as cnt FROM bench_100k WHERE id > 50000`);
239+
}, { time: 30_000, warmupIterations: 2 });
240+
});

0 commit comments

Comments
 (0)