Optimize dict decode and view writes

- Optimize OffsetBuffer dict decode by inlining extend + offset push
- Use raw ptr writes in ByteViewArray dictionary decoder for better perf
- Remove unused append_raw_view_unchecked from ViewBuffer

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: lyang24

parquet/src/arrow/array_reader/builder.rs

@@ -33,6 +33,7 @@ use crate::arrow::array_reader::{
     NullArrayReader, PrimitiveArrayReader, RowGroups, StructArrayReader,
     make_byte_array_dictionary_reader, make_byte_array_reader,
 };
+use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE;
 use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics;
 use crate::arrow::schema::{ParquetField, ParquetFieldType, VirtualColumnType};
 use crate::basic::Type as PhysicalType;
@@ -102,23 +103,24 @@ pub struct ArrayReaderBuilder<'a> {
 
 impl<'a> ArrayReaderBuilder<'a> {
     /// Create a new `ArrayReaderBuilder`
-    ///
-    /// `batch_size` is used to pre-allocate internal buffers with the expected capacity,
-    /// avoiding reallocations when reading the first batch of data.
-    pub fn new(
-        row_groups: &'a dyn RowGroups,
-        metrics: &'a ArrowReaderMetrics,
-        batch_size: usize,
-    ) -> Self {
+    pub fn new(row_groups: &'a dyn RowGroups, metrics: &'a ArrowReaderMetrics) -> Self {
         Self {
             row_groups,
             cache_options: None,
             parquet_metadata: None,
             metrics,
-            batch_size,
+            batch_size: DEFAULT_BATCH_SIZE,
         }
     }
 
+    /// Set the batch size used to pre-allocate internal buffers.
+    ///
+    /// This avoids reallocations when reading the first batch of data.
+    pub fn with_batch_size(mut self, batch_size: usize) -> Self {
+        self.batch_size = batch_size;
+        self
+    }
+
     /// Add cache options to the builder
     pub fn with_cache_options(mut self, cache_options: Option<&'a CacheOptions<'a>>) -> Self {
         self.cache_options = cache_options;
@@ -566,7 +568,8 @@ mod tests {
         .unwrap();
 
         let metrics = ArrowReaderMetrics::disabled();
-        let array_reader = ArrayReaderBuilder::new(&file_reader, &metrics, 1024)
+        let array_reader = ArrayReaderBuilder::new(&file_reader, &metrics)
+            .with_batch_size(DEFAULT_BATCH_SIZE)
             .build_array_reader(fields.as_ref(), &mask)
             .unwrap();
 
@@ -599,7 +602,8 @@ mod tests {
         .unwrap();
 
         let metrics = ArrowReaderMetrics::disabled();
-        let array_reader = ArrayReaderBuilder::new(&file_reader, &metrics, 1024)
+        let array_reader = ArrayReaderBuilder::new(&file_reader, &metrics)
+            .with_batch_size(DEFAULT_BATCH_SIZE)
             .with_parquet_metadata(file_reader.metadata())
             .build_array_reader(fields.as_ref(), &mask)
             .unwrap();

parquet/src/arrow/array_reader/byte_array.rs

@@ -485,24 +485,28 @@ impl ByteArrayDecoderDeltaLength {
         let initial_values_length = output.values.len();
 
         let to_read = len.min(self.lengths.len() - self.length_offset);
-        output.offsets.reserve(to_read);
-
         let src_lengths = &self.lengths[self.length_offset..self.length_offset + to_read];
-
         let total_bytes: usize = src_lengths.iter().map(|x| *x as usize).sum();
-        output.values.reserve(total_bytes);
 
-        let mut current_offset = self.data_offset;
-        for length in src_lengths {
-            let end_offset = current_offset + *length as usize;
-            output.try_push(
-                &self.data.as_ref()[current_offset..end_offset],
-                self.validate_utf8,
-            )?;
-            current_offset = end_offset;
-        }
+        // Reserve capacity for both offsets and values upfront
+        output.offsets.reserve(to_read);
+        output.values.reserve(total_bytes);
 
-        self.data_offset = current_offset;
+        // Delta length data is contiguous — copy all value bytes at once
+        let data_end = self.data_offset + total_bytes;
+        output
+            .values
+            .extend_from_slice(&self.data.as_ref()[self.data_offset..data_end]);
+
+        // Compute and extend offsets in batch using extend
+        let base_offset = initial_values_length;
+        let mut running = base_offset;
+        output.offsets.extend(src_lengths.iter().map(|length| {
+            running += *length as usize;
+            I::from_usize(running).expect("index overflow decoding byte array")
+        }));
+
+        self.data_offset = data_end;
         self.length_offset += to_read;
 
         if self.validate_utf8 {

parquet/src/arrow/array_reader/byte_view_array.rs

@@ -675,13 +675,20 @@ impl ByteViewArrayDecoderDelta {
     // <https://parquet.apache.org/docs/file-format/data-pages/encodings/#delta-strings-delta_byte_array--7>
 
     fn read(&mut self, output: &mut ViewBuffer, len: usize) -> Result<usize> {
-        output.views.reserve(len.min(self.decoder.remaining()));
+        let to_reserve = len.min(self.decoder.remaining());
+        output.views.reserve(to_reserve);
 
         // array buffer only have long strings
         let mut array_buffer: Vec<u8> = Vec::with_capacity(4096);
 
         let buffer_id = output.buffers.len() as u32;
 
+        // Use unsafe ptr writes instead of per-element push to avoid
+        // repeated length checks. Safety: we reserved enough space above.
+        let views_ptr = output.views.as_mut_ptr();
+        let initial_len = output.views.len();
+        let mut write_count = 0;
+
         let read = if !self.validate_utf8 {
             self.decoder.read(len, |bytes| {
                 let offset = array_buffer.len();
@@ -691,18 +698,18 @@ impl ByteViewArrayDecoderDelta {
                     array_buffer.extend_from_slice(bytes);
                 }
 
-                // # Safety
-                // The buffer_id is the last buffer in the output buffers
-                // The offset is calculated from the buffer, so it is valid
+                // Safety: views_ptr is valid for writes, we reserved enough space,
+                // and write_count < to_reserve.
                 unsafe {
-                    output.append_raw_view_unchecked(view);
+                    views_ptr.add(initial_len + write_count).write(view);
                 }
+                write_count += 1;
                 Ok(())
             })?
         } else {
             // utf8 validation buffer has only short strings. These short
             // strings are inlined into the views but we copy them into a
-            // contiguous buffer to accelerate validation.®
+            // contiguous buffer to accelerate validation.
             let mut utf8_validation_buffer = Vec::with_capacity(4096);
 
             let v = self.decoder.read(len, |bytes| {
@@ -715,20 +722,24 @@ impl ByteViewArrayDecoderDelta {
                     utf8_validation_buffer.extend_from_slice(bytes);
                 }
 
-                // # Safety
-                // The buffer_id is the last buffer in the output buffers
-                // The offset is calculated from the buffer, so it is valid
-                // Utf-8 validation is done later
+                // Safety: views_ptr is valid for writes, we reserved enough space,
+                // and write_count < to_reserve. Utf-8 validation is done later.
                 unsafe {
-                    output.append_raw_view_unchecked(view);
+                    views_ptr.add(initial_len + write_count).write(view);
                 }
+                write_count += 1;
                 Ok(())
             })?;
             check_valid_utf8(&array_buffer)?;
             check_valid_utf8(&utf8_validation_buffer)?;
             v
         };
 
+        // Safety: we wrote exactly `read` views via ptr writes above
+        unsafe {
+            output.views.set_len(initial_len + read);
+        }
+
         let actual_block_id = output.append_block(Buffer::from_vec(array_buffer));
         assert_eq!(actual_block_id, buffer_id);
         Ok(read)

parquet/src/arrow/array_reader/list_array.rs

@@ -249,6 +249,7 @@ mod tests {
     use crate::arrow::array_reader::ArrayReaderBuilder;
     use crate::arrow::array_reader::list_array::ListArrayReader;
     use crate::arrow::array_reader::test_util::InMemoryArrayReader;
+    use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE;
     use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics;
     use crate::arrow::schema::parquet_to_arrow_schema_and_fields;
     use crate::arrow::{ArrowWriter, ProjectionMask, parquet_to_arrow_schema};
@@ -566,7 +567,8 @@ mod tests {
         .unwrap();
 
         let metrics = ArrowReaderMetrics::disabled();
-        let mut array_reader = ArrayReaderBuilder::new(&file_reader, &metrics, 1024)
+        let mut array_reader = ArrayReaderBuilder::new(&file_reader, &metrics)
+            .with_batch_size(DEFAULT_BATCH_SIZE)
             .build_array_reader(fields.as_ref(), &mask)
             .unwrap();
 

parquet/src/arrow/array_reader/primitive_array.rs

@@ -439,6 +439,7 @@ fn pack_dictionary_impl<K: ArrowDictionaryKeyType, V: ArrowPrimitiveType>(
 mod tests {
     use super::*;
     use crate::arrow::array_reader::test_util::EmptyPageIterator;
+    use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE;
     use crate::basic::Encoding;
     use crate::column::page::Page;
     use crate::data_type::{Int32Type, Int64Type};
@@ -513,7 +514,7 @@ mod tests {
             Box::<EmptyPageIterator>::default(),
             schema.column(0),
             None,
-            1024,
+            DEFAULT_BATCH_SIZE,
         )
         .unwrap();
 
@@ -560,7 +561,7 @@ mod tests {
                 Box::new(page_iterator),
                 column_desc,
                 None,
-                1024,
+                DEFAULT_BATCH_SIZE,
             )
             .unwrap();
 
@@ -631,7 +632,7 @@ mod tests {
                     Box::new(page_iterator),
                     column_desc.clone(),
                     None,
-                    1024,
+                    DEFAULT_BATCH_SIZE,
                 )
                 .expect("Unable to get array reader");
 
@@ -771,7 +772,7 @@ mod tests {
                 Box::new(page_iterator),
                 column_desc,
                 None,
-                1024,
+                DEFAULT_BATCH_SIZE,
             )
             .unwrap();
 
@@ -851,7 +852,7 @@ mod tests {
                 Box::new(page_iterator),
                 column_desc,
                 None,
-                1024,
+                DEFAULT_BATCH_SIZE,
             )
             .unwrap();
 
@@ -914,7 +915,7 @@ mod tests {
                 Box::new(page_iterator),
                 column_desc,
                 None,
-                1024,
+                DEFAULT_BATCH_SIZE,
             )
             .unwrap();
 
@@ -980,7 +981,7 @@ mod tests {
                 Box::new(page_iterator),
                 column_desc,
                 None,
-                1024,
+                DEFAULT_BATCH_SIZE,
             )
             .unwrap();
 

parquet/src/arrow/arrow_reader/mod.rs

@@ -1219,15 +1219,17 @@ impl<T: ChunkReader + 'static> ParquetRecordBatchReaderBuilder<T> {
                 let mut cache_projection = predicate.projection().clone();
                 cache_projection.intersect(&projection);
 
-                let array_reader = ArrayReaderBuilder::new(&reader, &metrics, batch_size)
+                let array_reader = ArrayReaderBuilder::new(&reader, &metrics)
+                    .with_batch_size(batch_size)
                     .with_parquet_metadata(&reader.metadata)
                     .build_array_reader(fields.as_deref(), predicate.projection())?;
 
                 plan_builder = plan_builder.with_predicate(array_reader, predicate.as_mut())?;
             }
         }
 
-        let array_reader = ArrayReaderBuilder::new(&reader, &metrics, batch_size)
+        let array_reader = ArrayReaderBuilder::new(&reader, &metrics)
+            .with_batch_size(batch_size)
             .with_parquet_metadata(&reader.metadata)
             .build_array_reader(fields.as_deref(), &projection)?;
 
@@ -1534,7 +1536,8 @@ impl ParquetRecordBatchReader {
     ) -> Result<Self> {
         // note metrics are not supported in this API
         let metrics = ArrowReaderMetrics::disabled();
-        let array_reader = ArrayReaderBuilder::new(row_groups, &metrics, batch_size)
+        let array_reader = ArrayReaderBuilder::new(row_groups, &metrics)
+            .with_batch_size(batch_size)
             .with_parquet_metadata(row_groups.metadata())
             .build_array_reader(levels.levels.as_ref(), &ProjectionMask::all())?;
 

parquet/src/arrow/buffer/offset_buffer.rs

@@ -95,6 +95,8 @@ impl<I: OffsetSizeTrait> OffsetBuffer<I> {
         dict_offsets: &[V],
         dict_values: &[u8],
     ) -> Result<()> {
+        self.offsets.reserve(keys.len());
+
         for key in keys {
             let index = key.as_usize();
             if index + 1 >= dict_offsets.len() {
@@ -107,7 +109,11 @@ impl<I: OffsetSizeTrait> OffsetBuffer<I> {
             let end_offset = dict_offsets[index + 1].as_usize();
 
             // Dictionary values are verified when decoding dictionary page
-            self.try_push(&dict_values[start_offset..end_offset], false)?;
+            self.values
+                .extend_from_slice(&dict_values[start_offset..end_offset]);
+            let index_offset = I::from_usize(self.values.len())
+                .ok_or_else(|| general_err!("index overflow decoding byte array"))?;
+            self.offsets.push(index_offset);
         }
         Ok(())
     }

parquet/src/arrow/buffer/view_buffer.rs

@@ -51,15 +51,6 @@ impl ViewBuffer {
         block_id
     }
 
-    /// Directly append a view to the view array.
-    /// This is used when we create a StringViewArray from a dictionary whose values are StringViewArray.
-    ///
-    /// # Safety
-    /// The `view` must be a valid view as per the ByteView spec.
-    pub unsafe fn append_raw_view_unchecked(&mut self, view: u128) {
-        self.views.push(view);
-    }
-
     /// Converts this into an [`ArrayRef`] with the provided `data_type` and `null_buffer`
     pub fn into_array(self, null_buffer: Option<Buffer>, data_type: &ArrowType) -> ArrayRef {
         let len = self.views.len();
@@ -122,11 +113,9 @@ mod tests {
         let string_buffer = Buffer::from(data);
         let block_id = buffer.append_block(string_buffer);
 
-        unsafe {
-            buffer.append_raw_view_unchecked(make_view(&data[0..1], block_id, 0));
-            buffer.append_raw_view_unchecked(make_view(&data[1..10], block_id, 1));
-            buffer.append_raw_view_unchecked(make_view(&data[10..41], block_id, 10));
-        }
+        buffer.views.push(make_view(&data[0..1], block_id, 0));
+        buffer.views.push(make_view(&data[1..10], block_id, 1));
+        buffer.views.push(make_view(&data[10..41], block_id, 10));
 
         let array = buffer.into_array(None, &ArrowType::Utf8View);
         let string_array = array
@@ -150,11 +139,9 @@ mod tests {
         let string_buffer = Buffer::from(data);
         let block_id = buffer.append_block(string_buffer);
 
-        unsafe {
-            buffer.append_raw_view_unchecked(make_view(&data[0..1], block_id, 0));
-            buffer.append_raw_view_unchecked(make_view(&data[1..10], block_id, 1));
-            buffer.append_raw_view_unchecked(make_view(&data[10..41], block_id, 10));
-        }
+        buffer.views.push(make_view(&data[0..1], block_id, 0));
+        buffer.views.push(make_view(&data[1..10], block_id, 1));
+        buffer.views.push(make_view(&data[10..41], block_id, 10));
 
         let valid = [true, false, false, true, false, false, true];
         let valid_mask = Buffer::from_iter(valid.iter().copied());

parquet/src/arrow/push_decoder/reader_builder/mod.rs

@@ -432,11 +432,11 @@ impl RowGroupReaderBuilder {
 
                 let cache_options = filter_info.cache_builder().producer();
 
-                let array_reader =
-                    ArrayReaderBuilder::new(&row_group, &self.metrics, self.batch_size)
-                        .with_cache_options(Some(&cache_options))
-                        .with_parquet_metadata(&self.metadata)
-                        .build_array_reader(self.fields.as_deref(), predicate.projection())?;
+                let array_reader = ArrayReaderBuilder::new(&row_group, &self.metrics)
+                    .with_batch_size(self.batch_size)
+                    .with_cache_options(Some(&cache_options))
+                    .with_parquet_metadata(&self.metadata)
+                    .build_array_reader(self.fields.as_deref(), predicate.projection())?;
 
                 // Reset to original policy before each predicate so the override
                 // can detect page skipping for THIS predicate's columns.
@@ -609,9 +609,9 @@ impl RowGroupReaderBuilder {
                 let plan = plan_builder.build();
 
                 // if we have any cached results, connect them up
-                let array_reader_builder =
-                    ArrayReaderBuilder::new(&row_group, &self.metrics, self.batch_size)
-                        .with_parquet_metadata(&self.metadata);
+                let array_reader_builder = ArrayReaderBuilder::new(&row_group, &self.metrics)
+                    .with_batch_size(self.batch_size)
+                    .with_parquet_metadata(&self.metadata);
                 let array_reader = if let Some(cache_info) = cache_info.as_ref() {
                     let cache_options: CacheOptions = cache_info.builder().consumer();
                     array_reader_builder