Merge remote-tracking branch 'apache/main' into fix/ipc-null-dictionary

Author: alamb

.github/workflows/miri.sh

@@ -12,9 +12,8 @@ cargo miri setup
 cargo clean
 
 echo "Starting Arrow MIRI run..."
-cargo miri test -p arrow-buffer
-cargo miri test -p arrow-data --features ffi
-cargo miri test -p arrow-schema --features ffi
-cargo miri test -p arrow-ord
-cargo miri test -p arrow-array
-cargo miri test -p arrow-arith
+cargo miri nextest run \
+    -p arrow-buffer -p arrow-data \
+    -p arrow-schema -p arrow-ord \
+    -p arrow-array -p arrow-arith \
+    --features ffi --no-fail-fast

.github/workflows/miri.yaml

@@ -55,6 +55,9 @@ jobs:
           rustup toolchain install nightly --component miri
           rustup override set nightly
           cargo miri setup
+      - name: Set up nextest
+        run: |
+          cargo install cargo-nextest --version 0.9.132 --locked
       - name: Run Miri Checks
         env:
           RUST_BACKTRACE: full

arrow-arith/src/aggregate.rs

@@ -20,7 +20,7 @@
 use arrow_array::cast::*;
 use arrow_array::iterator::ArrayIter;
 use arrow_array::*;
-use arrow_buffer::{ArrowNativeType, NullBuffer};
+use arrow_buffer::NullBuffer;
 use arrow_data::bit_iterator::try_for_each_valid_idx;
 use arrow_schema::*;
 use std::borrow::BorrowMut;
@@ -541,11 +541,9 @@ pub fn min_string_view(array: &StringViewArray) -> Option<&str> {
 ///
 /// This doesn't detect overflow. Once overflowing, the result will wrap around.
 /// For an overflow-checking variant, use [`sum_array_checked`] instead.
-pub fn sum_array<T, A: ArrayAccessor<Item = T::Native>>(array: A) -> Option<T::Native>
-where
-    T: ArrowNumericType,
-    T::Native: ArrowNativeTypeOp,
-{
+pub fn sum_array<T: ArrowNumericType, A: ArrayAccessor<Item = T::Native>>(
+    array: A,
+) -> Option<T::Native> {
     match array.data_type() {
         DataType::Dictionary(_, _) => {
             let null_count = array.null_count();
@@ -583,13 +581,9 @@ where
 /// use [`sum_array`] instead.
 /// Additionally returns an `Err` on run-end-encoded arrays with a provided
 /// values type parameter that is incorrect.
-pub fn sum_array_checked<T, A: ArrayAccessor<Item = T::Native>>(
+pub fn sum_array_checked<T: ArrowNumericType, A: ArrayAccessor<Item = T::Native>>(
     array: A,
-) -> Result<Option<T::Native>, ArrowError>
-where
-    T: ArrowNumericType,
-    T::Native: ArrowNativeTypeOp,
-{
+) -> Result<Option<T::Native>, ArrowError> {
     match array.data_type() {
         DataType::Dictionary(_, _) => {
             let null_count = array.null_count();
@@ -717,21 +711,17 @@ mod ree {
 
 /// Returns the min of values in the array of `ArrowNumericType` type, or dictionary
 /// array with value of `ArrowNumericType` type.
-pub fn min_array<T, A: ArrayAccessor<Item = T::Native>>(array: A) -> Option<T::Native>
-where
-    T: ArrowNumericType,
-    T::Native: ArrowNativeType,
-{
+pub fn min_array<T: ArrowNumericType, A: ArrayAccessor<Item = T::Native>>(
+    array: A,
+) -> Option<T::Native> {
     min_max_array_helper::<T, A, _, _>(array, |a, b| a.is_gt(*b), min)
 }
 
 /// Returns the max of values in the array of `ArrowNumericType` type, or dictionary
 /// array with value of `ArrowNumericType` type.
-pub fn max_array<T, A: ArrayAccessor<Item = T::Native>>(array: A) -> Option<T::Native>
-where
-    T: ArrowNumericType,
-    T::Native: ArrowNativeTypeOp,
-{
+pub fn max_array<T: ArrowNumericType, A: ArrayAccessor<Item = T::Native>>(
+    array: A,
+) -> Option<T::Native> {
     min_max_array_helper::<T, A, _, _>(array, |a, b| a.is_lt(*b), max)
 }
 
@@ -874,11 +864,9 @@ pub fn bool_or(array: &BooleanArray) -> Option<bool> {
 ///
 /// This detects overflow and returns an `Err` for that. For an non-overflow-checking variant,
 /// use [`sum`] instead.
-pub fn sum_checked<T>(array: &PrimitiveArray<T>) -> Result<Option<T::Native>, ArrowError>
-where
-    T: ArrowNumericType,
-    T::Native: ArrowNativeTypeOp,
-{
+pub fn sum_checked<T: ArrowNumericType>(
+    array: &PrimitiveArray<T>,
+) -> Result<Option<T::Native>, ArrowError> {
     let null_count = array.null_count();
 
     if null_count == array.len() {
@@ -922,10 +910,7 @@ where
 ///
 /// This doesn't detect overflow in release mode by default. Once overflowing, the result will
 /// wrap around. For an overflow-checking variant, use [`sum_checked`] instead.
-pub fn sum<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
-where
-    T::Native: ArrowNativeTypeOp,
-{
+pub fn sum<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native> {
     aggregate::<T::Native, T, SumAccumulator<T::Native>>(array)
 }
 
@@ -940,10 +925,7 @@ where
 /// let result = min(&array);
 /// assert_eq!(result, Some(2));
 /// ```
-pub fn min<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
-where
-    T::Native: PartialOrd,
-{
+pub fn min<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native> {
     aggregate::<T::Native, T, MinAccumulator<T::Native>>(array)
 }
 
@@ -958,10 +940,7 @@ where
 /// let result = max(&array);
 /// assert_eq!(result, Some(8));
 /// ```
-pub fn max<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
-where
-    T::Native: PartialOrd,
-{
+pub fn max<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native> {
     aggregate::<T::Native, T, MaxAccumulator<T::Native>>(array)
 }
 

arrow-arith/src/arithmetic.rs

@@ -170,11 +170,7 @@ pub fn multiply_fixed_point(
 }
 
 /// Divide a decimal native value by given divisor and round the result.
-fn divide_and_round<I>(input: I::Native, div: I::Native) -> I::Native
-where
-    I: DecimalType,
-    I::Native: ArrowNativeTypeOp,
-{
+fn divide_and_round<I: DecimalType>(input: I::Native, div: I::Native) -> I::Native {
     let d = input.div_wrapping(div);
     let r = input.mod_wrapping(div);
 

arrow-array/src/array/byte_view_array.rs

@@ -670,6 +670,37 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
         }
     }
 
+    /// Returns the total number of bytes of all non-null values in this array.
+    ///
+    /// Unlike [`Self::total_buffer_bytes_used`], this method includes inlined strings
+    /// (those with length ≤ [`MAX_INLINE_VIEW_LEN`]), making it suitable as a
+    /// capacity hint when pre-allocating output buffers.
+    ///
+    /// Null values are excluded from the sum.
+    ///
+    /// # Example
+    ///
+    /// ```rust
+    /// # use arrow_array::StringViewArray;
+    /// let array = StringViewArray::from_iter(vec![
+    ///     Some("hello"),   // 5 bytes, inlined
+    ///     None,            // excluded
+    ///     Some("large payload over 12 bytes"),  // 27 bytes, non-inlined
+    /// ]);
+    /// assert_eq!(array.total_bytes_len(), 5 + 27);
+    /// ```
+    pub fn total_bytes_len(&self) -> usize {
+        match self.nulls() {
+            None => self.views().iter().map(|v| (*v as u32) as usize).sum(),
+            Some(nulls) => self
+                .views()
+                .iter()
+                .zip(nulls.iter())
+                .map(|(v, is_valid)| if is_valid { (*v as u32) as usize } else { 0 })
+                .sum(),
+        }
+    }
+
     /// Returns the total number of bytes used by all non inlined views in all
     /// buffers.
     ///
@@ -1809,4 +1840,41 @@ mod tests {
         assert!(from_utf8(array.value(2)).is_ok());
         array
     }
+
+    #[test]
+    fn test_total_bytes_len() {
+        // inlined: "hello"=5, "world"=5, "lulu"=4 → 14
+        // non-inlined: "large payload over 12 bytes"=27
+        // null: should not count
+        let mut builder = StringViewBuilder::new();
+        builder.append_value("hello");
+        builder.append_value("world");
+        builder.append_value("lulu");
+        builder.append_null();
+        builder.append_value("large payload over 12 bytes");
+        let array = builder.finish();
+        assert_eq!(array.total_bytes_len(), 5 + 5 + 4 + 27);
+    }
+
+    #[test]
+    fn test_total_bytes_len_empty() {
+        let array = StringViewArray::from_iter::<Vec<Option<&str>>>(vec![]);
+        assert_eq!(array.total_bytes_len(), 0);
+    }
+
+    #[test]
+    fn test_total_bytes_len_all_nulls() {
+        let array = StringViewArray::new_null(5);
+        assert_eq!(array.total_bytes_len(), 0);
+    }
+
+    #[test]
+    fn test_total_bytes_len_binary_view() {
+        let array = BinaryViewArray::from_iter(vec![
+            Some(b"hi".as_ref()),
+            None,
+            Some(b"large payload over 12 bytes".as_ref()),
+        ]);
+        assert_eq!(array.total_bytes_len(), 2 + 27);
+    }
 }

arrow-buffer/src/buffer/mutable.rs

@@ -450,7 +450,13 @@ impl MutableBuffer {
 
     /// Clear all existing data from this buffer.
     pub fn clear(&mut self) {
-        self.len = 0
+        self.len = 0;
+        #[cfg(feature = "pool")]
+        {
+            if let Some(reservation) = self.reservation.lock().unwrap().as_mut() {
+                reservation.resize(self.len);
+            }
+        }
     }
 
     /// Returns the data stored in this buffer as a slice.
@@ -1371,7 +1377,7 @@ mod tests {
             assert_eq!(pool.used(), 40);
 
             // Truncate to zero
-            buffer.truncate(0);
+            buffer.clear();
             assert_eq!(buffer.len(), 0);
             assert_eq!(pool.used(), 0);
         }

arrow-cast/src/cast/decimal.rs

@@ -812,7 +812,7 @@ where
     T: ArrowPrimitiveType,
     <T as ArrowPrimitiveType>::Native: NumCast,
     D: DecimalType + ArrowPrimitiveType,
-    <D as ArrowPrimitiveType>::Native: ArrowNativeTypeOp + ToPrimitive,
+    <D as ArrowPrimitiveType>::Native: ToPrimitive,
 {
     let array = array.as_primitive::<D>();