Skip to content

Commit 4a9d018

Browse files
committed
perf: avoid per-value Int8Array allocation and filter().length in parquet decode
- int8 decode: replace `new Int8Array(buf, off, 1)[0]` with `view.getInt8(i)` - DATA_PAGE_V2 null count: replace `defLevels.filter(d => d > 0).length` (allocates filtered array) with counting loop
1 parent 80df545 commit 4a9d018

File tree

1 file changed

+6
-4
lines changed

1 file changed

+6
-4
lines changed

src/parquet-decode.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ export function decodePlainValues(
281281
switch (dtype) {
282282
case "int8": {
283283
const n = Math.min(numValues, bytes.length);
284-
for (let i = 0; i < n; i++) values.push(new Int8Array(bytes.buffer, bytes.byteOffset + i, 1)[0]);
284+
for (let i = 0; i < n; i++) values.push(view.getInt8(i));
285285
break;
286286
}
287287
case "uint8": {
@@ -502,9 +502,11 @@ export function decodeParquetColumnChunk(
502502
}
503503
}
504504
} else {
505-
const nonNullCount = defLevels
506-
? defLevels.filter(d => d > 0).length
507-
: header.numValues;
505+
let nonNullCount = header.numValues;
506+
if (defLevels) {
507+
nonNullCount = 0;
508+
for (let di = 0; di < defLevels.length; di++) if (defLevels[di] > 0) nonNullCount++;
509+
}
508510
const { values: indices } = decodeRleBitPacked(dataPayload, payloadOffset, bitWidth, nonNullCount);
509511
let idxPtr = 0;
510512
for (let i = 0; i < header.numValues && values.length < numValues; i++) {

0 commit comments

Comments
 (0)