Skip to content

Commit a49c7d1

Browse files
committed
fix: manifest/metadata numeric sort + cache key fixes in query-do/master-do
Same class of bugs as the local-executor fixes: 1. query-do and master-do sorted manifest/metadata R2 keys lexicographically, picking "9.manifest" as latest instead of "10.manifest" for datasets with 10+ versions. Now extracts version number and sorts numerically. 2. query-do cache key used `w.column ?? ""` instead of NULL_SENTINEL and omitted w.args.default_ for lag/lead window functions.
1 parent ef463dd commit a49c7d1

File tree

2 files changed

+6
-5
lines changed

2 files changed

+6
-5
lines changed

src/master-do.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ export class MasterDO extends DurableObject<Env> {
7070
const listed = await resolveBucket(this.env, r2Prefix).list({ prefix: `${r2Prefix}_versions/`, limit: 100 });
7171
const manifestKeys = listed.objects
7272
.filter(o => o.key.endsWith(".manifest"))
73-
.sort((a, b) => a.key.localeCompare(b.key));
73+
.sort((a, b) => { const na = parseInt(a.key.split("/").pop()!); const nb = parseInt(b.key.split("/").pop()!); return na - nb; });
7474
if (manifestKeys.length === 0) throw new Error("No manifests found");
7575

7676
const latestKey = manifestKeys[manifestKeys.length - 1].key;

src/query-do.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { DurableObject } from "cloudflare:workers";
22
import type { ColumnMeta, DataType, Env, ExplainResult, FilterOp, Footer, Row, TableMeta, DatasetMeta, IcebergDatasetMeta, QueryResult } from "./types.js";
3-
import { queryReferencedColumns } from "./types.js";
3+
import { queryReferencedColumns, NULL_SENTINEL } from "./types.js";
44
import type { QueryDescriptor } from "./client.js";
55
import { parseFooter, parseColumnMetaFromProtobuf } from "./footer.js";
66
import { parseManifest, logicalTypeToDataType } from "./manifest.js";
@@ -677,11 +677,12 @@ export class QueryDO extends DurableObject<Env> {
677677
if (query.groupBy) for (const g of query.groupBy) { feed(g); feed("\0"); }
678678
if (query.distinct) for (const d of query.distinct) { feed(d); feed("\0"); }
679679
if (query.windows) for (const w of query.windows) {
680-
feed(w.fn); feed("\0"); feed(w.alias); feed("\0"); feed(w.column ?? ""); feed("\0");
680+
feed(w.fn); feed("\0"); feed(w.alias); feed("\0"); feed(w.column ?? NULL_SENTINEL); feed("\0");
681681
if (w.partitionBy) for (const p of w.partitionBy) { feed(p); feed("\0"); }
682682
if (w.orderBy) for (const o of w.orderBy) { feed(o.column); feed(o.direction); feed("\0"); }
683683
if (w.frame) { feed(w.frame.type); feed(String(w.frame.start)); feed(String(w.frame.end)); feed("\0"); }
684684
if (w.args?.offset !== undefined) { feed(String(w.args.offset)); feed("\0"); }
685+
if (w.args?.default_ !== undefined) { feed(String(w.args.default_)); feed("\0"); }
685686
}
686687
if (query.computedColumns) for (const cc of query.computedColumns) { feed(cc.alias); feed("\0"); }
687688
if (query.setOperation) { feed(query.setOperation.mode); feed("\0"); feed(this.queryKey(query.setOperation.right)); feed("\0"); }
@@ -1523,7 +1524,7 @@ export class QueryDO extends DurableObject<Env> {
15231524
const listed = await this.r2(prefix).list({ prefix: `${prefix}_versions/`, limit: 100 });
15241525
const manifestKeys = listed.objects
15251526
.filter(o => o.key.endsWith(".manifest"))
1526-
.sort((a, b) => a.key.localeCompare(b.key));
1527+
.sort((a, b) => { const na = parseInt(a.key.split("/").pop()!); const nb = parseInt(b.key.split("/").pop()!); return na - nb; });
15271528
if (manifestKeys.length === 0) continue;
15281529

15291530
// Read latest manifest
@@ -1675,7 +1676,7 @@ export class QueryDO extends DurableObject<Env> {
16751676
const listed = await this.r2(prefix).list({ prefix: `${prefix}metadata/`, limit: 100 });
16761677
const metadataKeys = listed.objects
16771678
.filter(o => o.key.endsWith(".metadata.json"))
1678-
.sort((a, b) => a.key.localeCompare(b.key));
1679+
.sort((a, b) => { const na = parseInt(a.key.split("/").pop()!); const nb = parseInt(b.key.split("/").pop()!); return na - nb; });
16791680
if (metadataKeys.length === 0) continue;
16801681

16811682
const latestKey = metadataKeys[metadataKeys.length - 1].key;

0 commit comments

Comments
 (0)