Inline thrift readFieldBegin (#159)

Author: platypii

package.json

@@ -10,6 +10,8 @@
     "dataset",
     "hyperparam",
     "hyparquet",
+    "geoparquet",
+    "llm",
     "ml",
     "parquet",
     "parquetjs",
@@ -55,10 +57,10 @@
     "test": "vitest run"
   },
   "devDependencies": {
-    "@types/node": "25.2.3",
+    "@types/node": "25.3.0",
     "@vitest/coverage-v8": "4.0.18",
     "eslint": "9.39.2",
-    "eslint-plugin-jsdoc": "62.5.4",
+    "eslint-plugin-jsdoc": "62.7.0",
     "hyparquet-compressors": "1.1.1",
     "typescript": "5.9.3",
     "vitest": "4.0.18"

src/thrift.js

@@ -1,20 +1,19 @@
+/**
+ * @import {DataReader, ThriftObject, ThriftType} from '../src/types.d.ts'
+ */
+
 // TCompactProtocol types
-export const CompactType = {
-  STOP: 0,
-  TRUE: 1,
-  FALSE: 2,
-  BYTE: 3,
-  I16: 4,
-  I32: 5,
-  I64: 6,
-  DOUBLE: 7,
-  BINARY: 8,
-  LIST: 9,
-  SET: 10,
-  MAP: 11,
-  STRUCT: 12,
-  UUID: 13,
-}
+const STOP = 0
+const TRUE = 1
+const FALSE = 2
+const BYTE = 3
+const I16 = 4
+const I32 = 5
+const I64 = 6
+const DOUBLE = 7
+const BINARY = 8
+const LIST = 9
+const STRUCT = 12
 
 /**
  * Parse TCompactProtocol
@@ -23,20 +22,17 @@ export const CompactType = {
  * @returns {{ [key: `field_${number}`]: any }}
  */
 export function deserializeTCompactProtocol(reader) {
-  let lastFid = 0
   /** @type {ThriftObject} */
   const value = {}
+  let fid = 0
 
   while (reader.offset < reader.view.byteLength) {
     // Parse each field based on its type and add to the result object
-    const [type, fid, newLastFid] = readFieldBegin(reader, lastFid)
-    lastFid = newLastFid
-
-    if (type === CompactType.STOP) {
-      break
-    }
-
-    // Handle the field based on its type
+    const byte = reader.view.getUint8(reader.offset++)
+    const type = byte & 0x0f
+    if (type === STOP) break
+    const delta = byte >> 4
+    fid = delta ? fid + delta : readZigZag(reader)
     value[`field_${fid}`] = readElement(reader, type)
   }
 
@@ -46,73 +42,59 @@ export function deserializeTCompactProtocol(reader) {
 /**
  * Read a single element based on its type
  *
- * @import {DataReader, ThriftObject, ThriftType} from '../src/types.d.ts'
  * @param {DataReader} reader
  * @param {number} type
  * @returns {ThriftType}
  */
 function readElement(reader, type) {
   switch (type) {
-  case CompactType.TRUE:
+  case TRUE:
     return true
-  case CompactType.FALSE:
+  case FALSE:
     return false
-  case CompactType.BYTE:
-    // read byte directly
+  case BYTE:
     return reader.view.getInt8(reader.offset++)
-  case CompactType.I16:
-  case CompactType.I32:
+  case I16:
+  case I32:
     return readZigZag(reader)
-  case CompactType.I64:
+  case I64:
     return readZigZagBigInt(reader)
-  case CompactType.DOUBLE: {
+  case DOUBLE: {
     const value = reader.view.getFloat64(reader.offset, true)
     reader.offset += 8
     return value
   }
-  case CompactType.BINARY: {
+  case BINARY: {
     const stringLength = readVarInt(reader)
     const strBytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, stringLength)
     reader.offset += stringLength
     return strBytes
   }
-  case CompactType.LIST: {
+  case LIST: {
     const byte = reader.view.getUint8(reader.offset++)
     const elemType = byte & 0x0f
     let listSize = byte >> 4
     if (listSize === 15) {
       listSize = readVarInt(reader)
     }
-    const boolType = elemType === CompactType.TRUE || elemType === CompactType.FALSE
+    const boolType = elemType === TRUE || elemType === FALSE
     const values = new Array(listSize)
     for (let i = 0; i < listSize; i++) {
-      values[i] = boolType ? readElement(reader, CompactType.BYTE) === 1 : readElement(reader, elemType)
+      values[i] = boolType ? readElement(reader, BYTE) === 1 : readElement(reader, elemType)
     }
     return values
   }
-  case CompactType.STRUCT: {
-    /** @type {ThriftObject} */
-    const structValues = {}
-    let lastFid = 0
-    while (true) {
-      const [fieldType, fid, newLastFid] = readFieldBegin(reader, lastFid)
-      lastFid = newLastFid
-      if (fieldType === CompactType.STOP) {
-        break
-      }
-      structValues[`field_${fid}`] = readElement(reader, fieldType)
-    }
-    return structValues
-  }
-  // TODO: MAP, SET, UUID
+  case STRUCT:
+    // main function handles struct parsing
+    return deserializeTCompactProtocol(reader)
   default:
+    // MAP, SET, UUID not used by parquet
     throw new Error(`thrift unhandled type: ${type}`)
   }
 }
 
 /**
- * Var int aka Unsigned LEB128.
- * Reads groups of 7 low bits until high bit is 0.
+ * Read varint aka Unsigned LEB128.
  *
  * @param {DataReader} reader
  * @returns {number}
@@ -121,6 +103,7 @@ export function readVarInt(reader) {
   let result = 0
   let shift = 0
   while (true) {
+    // Read groups of 7 low bits until high bit is 0
     const byte = reader.view.getUint8(reader.offset++)
     result |= (byte & 0x7f) << shift
     if (!(byte & 0x80)) {
@@ -150,46 +133,24 @@ function readVarBigInt(reader) {
 }
 
 /**
- * Values of type int32 and int64 are transformed to a zigzag int.
- * A zigzag int folds positive and negative numbers into the positive number space.
+ * Read a zigzag number.
+ * Zigzag folds positive and negative numbers into the positive number space.
  *
  * @param {DataReader} reader
  * @returns {number}
  */
 export function readZigZag(reader) {
   const zigzag = readVarInt(reader)
-  // convert zigzag to int
   return zigzag >>> 1 ^ -(zigzag & 1)
 }
 
 /**
- * A zigzag int folds positive and negative numbers into the positive number space.
- * This version returns a BigInt.
+ * Read a zigzag bigint.
  *
  * @param {DataReader} reader
  * @returns {bigint}
  */
 export function readZigZagBigInt(reader) {
   const zigzag = readVarBigInt(reader)
-  // convert zigzag to int
   return zigzag >> 1n ^ -(zigzag & 1n)
 }
-
-/**
- * Read field type and field id
- *
- * @param {DataReader} reader
- * @param {number} lastFid
- * @returns {[number, number, number]} [type, fid, newLastFid]
- */
-function readFieldBegin(reader, lastFid) {
-  const byte = reader.view.getUint8(reader.offset++)
-  const type = byte & 0x0f
-  if (type === CompactType.STOP) {
-    // STOP also ends a struct
-    return [0, 0, lastFid]
-  }
-  const delta = byte >> 4
-  const fid = delta ? lastFid + delta : readZigZag(reader)
-  return [type, fid, fid]
-}

src/types.d.ts

@@ -408,8 +408,8 @@ interface PageLocation {
 
 export interface ColumnIndex {
   null_pages: boolean[]
-  min_values: MinMaxType[]
-  max_values: MinMaxType[]
+  min_values: Uint8Array[]
+  max_values: Uint8Array[]
   boundary_order: BoundaryOrder
   null_counts?: bigint[]
   repetition_level_histograms?: bigint[]

test/snappy.test.js

@@ -34,7 +34,7 @@ describe('snappy uncompress', () => {
       },
       // from datapage_v2.snappy.parquet
       { compressed: [2, 4, 0, 3], expected: new Uint8Array([0, 3]) },
-      { compressed: [ 6, 20, 2, 0, 0, 0, 3, 23], expected: new Uint8Array([2, 0, 0, 0, 3, 23]) },
+      { compressed: [6, 20, 2, 0, 0, 0, 3, 23], expected: new Uint8Array([2, 0, 0, 0, 3, 23]) },
     ]
 
     for (const { compressed, expected } of testCases) {