Merge branch 'main' into friendlymatthew/track-bytes-during-union-row-decoding

Author: friendlymatthew

.github/workflows/integration.yml

@@ -78,58 +78,112 @@ jobs:
       run:
         shell: bash
     steps:
+      - name: Monitor disk usage - Initial
+        run: |
+          echo "=== Initial Disk Usage ==="
+          df -h /
+          echo ""
+
+      - name: Remove unnecessary preinstalled software
+        run: |
+          echo "=== Cleaning up host disk space ==="
+          echo "Disk space before cleanup:"
+          df -h /
+
+          # Clean apt cache
+          apt-get clean || true
+
+          # Remove GitHub Actions tool cache
+          rm -rf /__t/* || true
+
+          # Remove large packages from host filesystem (mounted at /host/)
+          rm -rf /host/usr/share/dotnet || true
+          rm -rf /host/usr/local/lib/android || true
+          rm -rf /host/usr/local/.ghcup || true
+          rm -rf /host/opt/hostedtoolcache/CodeQL || true
+
+          echo ""
+          echo "Disk space after cleanup:"
+          df -h /
+          echo ""
+
       # This is necessary so that actions/checkout can find git
       - name: Export conda path
         run: echo "/opt/conda/envs/arrow/bin" >> $GITHUB_PATH
       # This is necessary so that Rust can find cargo
       - name: Export cargo path
         run: echo "/root/.cargo/bin" >> $GITHUB_PATH
-      - name: Check rustup
-        run: which rustup
-      - name: Check cmake
-        run: which cmake
+
+      # Checkout repos (using shallow clones with fetch-depth: 1)
       - name: Checkout Arrow
         uses: actions/checkout@v6
         with:
           repository: apache/arrow
           submodules: true
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Checkout Arrow Rust
         uses: actions/checkout@v6
         with:
           path: rust
           submodules: true
-          fetch-depth: 0
+          fetch-depth: 1
       - name: Checkout Arrow .NET
         uses: actions/checkout@v6
         with:
           repository: apache/arrow-dotnet
           path: dotnet
+          fetch-depth: 1
       - name: Checkout Arrow Go
         uses: actions/checkout@v6
         with:
           repository: apache/arrow-go
           path: go
+          fetch-depth: 1
       - name: Checkout Arrow Java
         uses: actions/checkout@v6
         with:
           repository: apache/arrow-java
           path: java
+          fetch-depth: 1
       - name: Checkout Arrow JavaScript
         uses: actions/checkout@v6
         with:
           repository: apache/arrow-js
           path: js
+          fetch-depth: 1
       - name: Checkout Arrow nanoarrow
         uses: actions/checkout@v6
         with:
           repository: apache/arrow-nanoarrow
           path: nanoarrow
+          fetch-depth: 1
+
+      - name: Monitor disk usage - After checkouts
+        run: |
+          echo "=== After Checkouts ==="
+          df -h /
+          echo ""
+
       - name: Build
         run: conda run --no-capture-output ci/scripts/integration_arrow_build.sh $PWD /build
+
+      - name: Monitor disk usage - After build
+        if: always()
+        run: |
+          echo "=== After Build ==="
+          df -h /
+          echo ""
+
       - name: Run
         run: conda run --no-capture-output ci/scripts/integration_arrow.sh $PWD /build
 
+      - name: Monitor disk usage - After tests
+        if: always()
+        run: |
+          echo "=== After Tests ==="
+          df -h /
+          echo ""
+
   # test FFI against the C-Data interface exposed by pyarrow
   pyarrow-integration-test:
     name: Pyarrow C Data Interface

arrow-arith/src/aggregate.rs

@@ -332,10 +332,10 @@ fn aggregate<T: ArrowNativeTypeOp, P: ArrowPrimitiveType<Native = T>, A: Numeric
 
 /// Returns the minimum value in the boolean array.
 ///
+/// # Example
 /// ```
 /// # use arrow_array::BooleanArray;
 /// # use arrow_arith::aggregate::min_boolean;
-///
 /// let a = BooleanArray::from(vec![Some(true), None, Some(false)]);
 /// assert_eq!(min_boolean(&a), Some(false))
 /// ```
@@ -390,10 +390,10 @@ pub fn min_boolean(array: &BooleanArray) -> Option<bool> {
 
 /// Returns the maximum value in the boolean array
 ///
+/// # Example
 /// ```
 /// # use arrow_array::BooleanArray;
 /// # use arrow_arith::aggregate::max_boolean;
-///
 /// let a = BooleanArray::from(vec![Some(true), None, Some(false)]);
 /// assert_eq!(max_boolean(&a), Some(true))
 /// ```
@@ -809,6 +809,15 @@ where
 
 /// Returns the minimum value in the array, according to the natural order.
 /// For floating point arrays any NaN values are considered to be greater than any other non-null value
+///
+/// # Example
+/// ```rust
+/// # use arrow_array::Int32Array;
+/// # use arrow_arith::aggregate::min;
+/// let array = Int32Array::from(vec![8, 2, 4]);
+/// let result = min(&array);
+/// assert_eq!(result, Some(2));
+/// ```
 pub fn min<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
 where
     T::Native: PartialOrd,
@@ -818,6 +827,15 @@ where
 
 /// Returns the maximum value in the array, according to the natural order.
 /// For floating point arrays any NaN values are considered to be greater than any other non-null value
+///
+/// # Example
+/// ```rust
+/// # use arrow_array::Int32Array;
+/// # use arrow_arith::aggregate::max;
+/// let array = Int32Array::from(vec![4, 8, 2]);
+/// let result = max(&array);
+/// assert_eq!(result, Some(8));
+/// ```
 pub fn max<T: ArrowNumericType>(array: &PrimitiveArray<T>) -> Option<T::Native>
 where
     T::Native: PartialOrd,

arrow-array/src/builder/generic_bytes_view_builder.rs

@@ -430,6 +430,53 @@ impl<T: ByteViewType + ?Sized> GenericByteViewBuilder<T> {
         };
     }
 
+    /// Append the same value `n` times into the builder
+    ///
+    /// This is more efficient than calling [`Self::try_append_value`] `n` times,
+    /// especially when deduplication is enabled, as it only hashes the value once.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if
+    /// - String buffer count exceeds `u32::MAX`
+    /// - String length exceeds `u32::MAX`
+    ///
+    /// # Example
+    /// ```
+    /// # use arrow_array::builder::StringViewBuilder;
+    /// # use arrow_array::Array;
+    /// let mut builder = StringViewBuilder::new().with_deduplicate_strings();
+    ///
+    /// // Append "hello" 1000 times efficiently
+    /// builder.try_append_value_n("hello", 1000)?;
+    ///
+    /// let array = builder.finish();
+    /// assert_eq!(array.len(), 1000);
+    ///
+    /// // All values are "hello"
+    /// for value in array.iter() {
+    ///     assert_eq!(value, Some("hello"));
+    /// }
+    /// # Ok::<(), arrow_schema::ArrowError>(())
+    /// ```
+    #[inline]
+    pub fn try_append_value_n(
+        &mut self,
+        value: impl AsRef<T::Native>,
+        n: usize,
+    ) -> Result<(), ArrowError> {
+        if n == 0 {
+            return Ok(());
+        }
+        // Process value once (handles deduplication, buffer management, view creation)
+        self.try_append_value(value)?;
+        // Reuse the view (n-1) times
+        let view = *self.views_buffer.last().unwrap();
+        self.views_buffer.extend(std::iter::repeat_n(view, n - 1));
+        self.null_buffer_builder.append_n_non_nulls(n - 1);
+        Ok(())
+    }
+
     /// Append a null value into the builder
     #[inline]
     pub fn append_null(&mut self) {
@@ -884,4 +931,76 @@ mod tests {
             MAX_BLOCK_SIZE as usize
         );
     }
+
+    #[test]
+    fn test_append_value_n() {
+        // Test with inline strings (<=12 bytes)
+        let mut builder = StringViewBuilder::new();
+
+        builder.try_append_value_n("hello", 100).unwrap();
+        builder.append_value("world");
+        builder.try_append_value_n("foo", 50).unwrap();
+
+        let array = builder.finish();
+        assert_eq!(array.len(), 151);
+        assert_eq!(array.null_count(), 0);
+
+        // Verify the values
+        for i in 0..100 {
+            assert_eq!(array.value(i), "hello");
+        }
+        assert_eq!(array.value(100), "world");
+        for i in 101..151 {
+            assert_eq!(array.value(i), "foo");
+        }
+
+        // All inline strings should have no data buffers
+        assert_eq!(array.data_buffers().len(), 0);
+    }
+
+    #[test]
+    fn test_append_value_n_with_deduplication() {
+        let long_string = "This is a very long string that exceeds the inline length";
+
+        // Test with deduplication enabled
+        let mut builder = StringViewBuilder::new().with_deduplicate_strings();
+
+        // First append the string once to add it to the hash map
+        builder.append_value(long_string);
+
+        // Then append_n the same string - should deduplicate and reuse the existing value
+        builder.try_append_value_n(long_string, 999).unwrap();
+
+        let array = builder.finish();
+        assert_eq!(array.len(), 1000);
+        assert_eq!(array.null_count(), 0);
+
+        // Verify all values are the same
+        for i in 0..1000 {
+            assert_eq!(array.value(i), long_string);
+        }
+
+        // With deduplication, should only have 1 data buffer containing the string once
+        assert_eq!(array.data_buffers().len(), 1);
+
+        // All views should be identical
+        let first_view = array.views()[0];
+        for view in array.views().iter() {
+            assert_eq!(*view, first_view);
+        }
+    }
+
+    #[test]
+    fn test_append_value_n_zero() {
+        let mut builder = StringViewBuilder::new();
+
+        builder.append_value("first");
+        builder.try_append_value_n("should not appear", 0).unwrap();
+        builder.append_value("second");
+
+        let array = builder.finish();
+        assert_eq!(array.len(), 2);
+        assert_eq!(array.value(0), "first");
+        assert_eq!(array.value(1), "second");
+    }
 }