Merge remote-tracking branch 'apache/main' into alamb/boolean_kernel

alamb · alamb · commit a110e2c68d53 · 2026-01-07T12:42:52.000-05:00
diff --git a/arrow-array/src/array/boolean_array.rs b/arrow-array/src/array/boolean_array.rs
@@ -286,6 +286,8 @@ impl BooleanArray {
     }
 }
 
+impl super::private::Sealed for BooleanArray {}
+
 impl Array for BooleanArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/byte_array.rs b/arrow-array/src/array/byte_array.rs
@@ -462,6 +462,8 @@ impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {
     }
 }
 
+impl<T: ByteArrayType> super::private::Sealed for GenericByteArray<T> {}
+
 impl<T: ByteArrayType> Array for GenericByteArray<T> {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/byte_view_array.rs b/arrow-array/src/array/byte_view_array.rs
@@ -854,6 +854,8 @@ impl<T: ByteViewType + ?Sized> Debug for GenericByteViewArray<T> {
     }
 }
 
+impl<T: ByteViewType + ?Sized> super::private::Sealed for GenericByteViewArray<T> {}
+
 impl<T: ByteViewType + ?Sized> Array for GenericByteViewArray<T> {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/dictionary_array.rs b/arrow-array/src/array/dictionary_array.rs
@@ -697,6 +697,8 @@ impl<'a, T: ArrowDictionaryKeyType> FromIterator<&'a str> for DictionaryArray<T>
     }
 }
 
+impl<T: ArrowDictionaryKeyType> super::private::Sealed for DictionaryArray<T> {}
+
 impl<T: ArrowDictionaryKeyType> Array for DictionaryArray<T> {
     fn as_any(&self) -> &dyn Any {
         self
@@ -856,6 +858,8 @@ impl<'a, K: ArrowDictionaryKeyType, V> TypedDictionaryArray<'a, K, V> {
     }
 }
 
+impl<K: ArrowDictionaryKeyType, V: Sync> super::private::Sealed for TypedDictionaryArray<'_, K, V> {}
+
 impl<K: ArrowDictionaryKeyType, V: Sync> Array for TypedDictionaryArray<'_, K, V> {
     fn as_any(&self) -> &dyn Any {
         self.dictionary
diff --git a/arrow-array/src/array/fixed_size_binary_array.rs b/arrow-array/src/array/fixed_size_binary_array.rs
@@ -602,6 +602,8 @@ impl std::fmt::Debug for FixedSizeBinaryArray {
     }
 }
 
+impl super::private::Sealed for FixedSizeBinaryArray {}
+
 impl Array for FixedSizeBinaryArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/fixed_size_list_array.rs b/arrow-array/src/array/fixed_size_list_array.rs
@@ -462,6 +462,8 @@ impl From<FixedSizeListArray> for ArrayData {
     }
 }
 
+impl super::private::Sealed for FixedSizeListArray {}
+
 impl Array for FixedSizeListArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/list_array.rs b/arrow-array/src/array/list_array.rs
@@ -525,6 +525,8 @@ impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {
     }
 }
 
+impl<OffsetSize: OffsetSizeTrait> super::private::Sealed for GenericListArray<OffsetSize> {}
+
 impl<OffsetSize: OffsetSizeTrait> Array for GenericListArray<OffsetSize> {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/list_view_array.rs b/arrow-array/src/array/list_view_array.rs
@@ -415,6 +415,8 @@ impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<Offset
     }
 }
 
+impl<OffsetSize: OffsetSizeTrait> super::private::Sealed for GenericListViewArray<OffsetSize> {}
+
 impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize> {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/map_array.rs b/arrow-array/src/array/map_array.rs
@@ -361,6 +361,8 @@ impl MapArray {
     }
 }
 
+impl super::private::Sealed for MapArray {}
+
 impl Array for MapArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/mod.rs b/arrow-array/src/array/mod.rs
@@ -78,8 +78,18 @@ pub use list_view_array::*;
 
 use crate::iterator::ArrayIter;
 
+mod private {
+    /// Private marker trait to ensure [`super::Array`] can not be implemented outside this crate
+    pub trait Sealed {}
+
+    impl<T: Sealed> Sealed for &T {}
+}
+
 /// An array in the [arrow columnar format](https://arrow.apache.org/docs/format/Columnar.html)
-pub trait Array: std::fmt::Debug + Send + Sync {
+///
+/// This trait is sealed as it is not intended for custom array types, rather only
+/// those defined in this crate.
+pub trait Array: std::fmt::Debug + Send + Sync + private::Sealed {
     /// Returns the array as [`Any`] so that it can be
     /// downcasted to a specific implementation.
     ///
@@ -341,6 +351,8 @@ pub trait Array: std::fmt::Debug + Send + Sync {
 /// A reference-counted reference to a generic `Array`
 pub type ArrayRef = Arc<dyn Array>;
 
+impl private::Sealed for ArrayRef {}
+
 /// Ergonomics: Allow use of an ArrayRef as an `&dyn Array`
 impl Array for ArrayRef {
     fn as_any(&self) -> &dyn Any {
diff --git a/arrow-array/src/array/null_array.rs b/arrow-array/src/array/null_array.rs
@@ -76,6 +76,8 @@ impl NullArray {
     }
 }
 
+impl super::private::Sealed for NullArray {}
+
 impl Array for NullArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/primitive_array.rs b/arrow-array/src/array/primitive_array.rs
@@ -1190,6 +1190,8 @@ impl<T: ArrowPrimitiveType> From<PrimitiveArray<T>> for ArrayData {
     }
 }
 
+impl<T: ArrowPrimitiveType> super::private::Sealed for PrimitiveArray<T> {}
+
 impl<T: ArrowPrimitiveType> Array for PrimitiveArray<T> {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/run_array.rs b/arrow-array/src/array/run_array.rs
@@ -260,6 +260,8 @@ impl<R: RunEndIndexType> From<RunArray<R>> for ArrayData {
     }
 }
 
+impl<T: RunEndIndexType> super::private::Sealed for RunArray<T> {}
+
 impl<T: RunEndIndexType> Array for RunArray<T> {
     fn as_any(&self) -> &dyn Any {
         self
@@ -519,6 +521,8 @@ impl<'a, R: RunEndIndexType, V> TypedRunArray<'a, R, V> {
     }
 }
 
+impl<R: RunEndIndexType, V: Sync> super::private::Sealed for TypedRunArray<'_, R, V> {}
+
 impl<R: RunEndIndexType, V: Sync> Array for TypedRunArray<'_, R, V> {
     fn as_any(&self) -> &dyn Any {
         self.run_array
diff --git a/arrow-array/src/array/struct_array.rs b/arrow-array/src/array/struct_array.rs
@@ -401,6 +401,8 @@ impl TryFrom<Vec<(&str, ArrayRef)>> for StructArray {
     }
 }
 
+impl super::private::Sealed for StructArray {}
+
 impl Array for StructArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-array/src/array/union_array.rs b/arrow-array/src/array/union_array.rs
@@ -738,6 +738,8 @@ impl From<UnionArray> for ArrayData {
     }
 }
 
+impl super::private::Sealed for UnionArray {}
+
 impl Array for UnionArray {
     fn as_any(&self) -> &dyn Any {
         self
diff --git a/arrow-flight/src/lib.rs b/arrow-flight/src/lib.rs
@@ -603,6 +603,12 @@ impl FlightInfo {
         self
     }
 
+    /// Add endpoints for fetching all data
+    pub fn with_endpoints(mut self, endpoints: Vec<FlightEndpoint>) -> Self {
+        self.endpoint = endpoints;
+        self
+    }
+
     /// Add a [`FlightDescriptor`] describing what this data is
     pub fn with_descriptor(mut self, flight_descriptor: FlightDescriptor) -> Self {
         self.flight_descriptor = Some(flight_descriptor);
diff --git a/arrow-json/benches/serde.rs b/arrow-json/benches/serde.rs
@@ -22,12 +22,14 @@ use rand::{Rng, rng};
 use serde::Serialize;
 use std::sync::Arc;
 
+const ROWS: usize = 1 << 18;
+
 #[allow(deprecated)]
 fn do_bench<R: Serialize>(c: &mut Criterion, name: &str, rows: &[R], schema: &Schema) {
     let schema = Arc::new(schema.clone());
     c.bench_function(name, |b| {
         b.iter(|| {
-            let builder = ReaderBuilder::new(schema.clone()).with_batch_size(64);
+            let builder = ReaderBuilder::new(schema.clone()).with_batch_size(8192);
             let mut decoder = builder.build_decoder().unwrap();
             decoder.serialize(rows)
         })
@@ -37,26 +39,26 @@ fn do_bench<R: Serialize>(c: &mut Criterion, name: &str, rows: &[R], schema: &Sc
 fn criterion_benchmark(c: &mut Criterion) {
     let mut rng = rng();
     let schema = Schema::new(vec![Field::new("i32", DataType::Int32, false)]);
-    let v: Vec<i32> = (0..2048).map(|_| rng.random_range(0..10000)).collect();
+    let v: Vec<i32> = (0..ROWS).map(|_| rng.random_range(0..10000)).collect();
 
     do_bench(c, "small_i32", &v, &schema);
-    let v: Vec<i32> = (0..2048).map(|_| rng.random()).collect();
+    let v: Vec<i32> = (0..ROWS).map(|_| rng.random()).collect();
     do_bench(c, "large_i32", &v, &schema);
 
     let schema = Schema::new(vec![Field::new("i64", DataType::Int64, false)]);
-    let v: Vec<i64> = (0..2048).map(|_| rng.random_range(0..10000)).collect();
+    let v: Vec<i64> = (0..ROWS).map(|_| rng.random_range(0..10000)).collect();
     do_bench(c, "small_i64", &v, &schema);
-    let v: Vec<i64> = (0..2048)
+    let v: Vec<i64> = (0..ROWS)
         .map(|_| rng.random_range(0..i32::MAX as _))
         .collect();
     do_bench(c, "medium_i64", &v, &schema);
-    let v: Vec<i64> = (0..2048).map(|_| rng.random()).collect();
+    let v: Vec<i64> = (0..ROWS).map(|_| rng.random()).collect();
     do_bench(c, "large_i64", &v, &schema);
 
     let schema = Schema::new(vec![Field::new("f32", DataType::Float32, false)]);
-    let v: Vec<f32> = (0..2048).map(|_| rng.random_range(0.0..10000.)).collect();
+    let v: Vec<f32> = (0..ROWS).map(|_| rng.random_range(0.0..10000.)).collect();
     do_bench(c, "small_f32", &v, &schema);
-    let v: Vec<f32> = (0..2048).map(|_| rng.random_range(0.0..f32::MAX)).collect();
+    let v: Vec<f32> = (0..ROWS).map(|_| rng.random_range(0.0..f32::MAX)).collect();
     do_bench(c, "large_f32", &v, &schema);
 }
 
diff --git a/arrow-row/src/lib.rs b/arrow-row/src/lib.rs
diff --git a/arrow-schema/src/datatype.rs b/arrow-schema/src/datatype.rs
diff --git a/parquet-variant-compute/src/shred_variant.rs b/parquet-variant-compute/src/shred_variant.rs
diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs
diff --git a/parquet/src/arrow/array_reader/cached_array_reader.rs b/parquet/src/arrow/array_reader/cached_array_reader.rs
diff --git a/parquet/src/arrow/arrow_reader/read_plan.rs b/parquet/src/arrow/arrow_reader/read_plan.rs
diff --git a/parquet/src/arrow/arrow_writer/levels.rs b/parquet/src/arrow/arrow_writer/levels.rs
diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs

Original file line number	Diff line number	Diff line change
`@@ -286,6 +286,8 @@ impl BooleanArray {`
`286`	`286`	`}`
`287`	`287`	`}`
`288`	`288`
	`289`	`+impl super::private::Sealed for BooleanArray {}`
	`290`	`+`
`289`	`291`	`impl Array for BooleanArray {`
`290`	`292`	`fn as_any(&self) -> &dyn Any {`
`291`	`293`	`self`
Original file line number	Diff line number	Diff line change
`@@ -462,6 +462,8 @@ impl<T: ByteArrayType> std::fmt::Debug for GenericByteArray<T> {`
`462`	`462`	`}`
`463`	`463`	`}`
`464`	`464`
	`465`	`+impl<T: ByteArrayType> super::private::Sealed for GenericByteArray<T> {}`
	`466`	`+`
`465`	`467`	`impl<T: ByteArrayType> Array for GenericByteArray<T> {`
`466`	`468`	`fn as_any(&self) -> &dyn Any {`
`467`	`469`	`self`
Original file line number	Diff line number	Diff line change
`@@ -854,6 +854,8 @@ impl<T: ByteViewType + ?Sized> Debug for GenericByteViewArray<T> {`
`854`	`854`	`}`
`855`	`855`	`}`
`856`	`856`
	`857`	`+impl<T: ByteViewType + ?Sized> super::private::Sealed for GenericByteViewArray<T> {}`
	`858`	`+`
`857`	`859`	`impl<T: ByteViewType + ?Sized> Array for GenericByteViewArray<T> {`
`858`	`860`	`fn as_any(&self) -> &dyn Any {`
`859`	`861`	`self`
Original file line number	Diff line number	Diff line change
`@@ -697,6 +697,8 @@ impl<'a, T: ArrowDictionaryKeyType> FromIterator<&'a str> for DictionaryArray<T>`
`697`	`697`	`}`
`698`	`698`	`}`
`699`	`699`
	`700`	`+impl<T: ArrowDictionaryKeyType> super::private::Sealed for DictionaryArray<T> {}`
	`701`	`+`
`700`	`702`	`impl<T: ArrowDictionaryKeyType> Array for DictionaryArray<T> {`
`701`	`703`	`fn as_any(&self) -> &dyn Any {`
`702`	`704`	`self`
`@@ -856,6 +858,8 @@ impl<'a, K: ArrowDictionaryKeyType, V> TypedDictionaryArray<'a, K, V> {`
`856`	`858`	`}`
`857`	`859`	`}`
`858`	`860`
	`861`	`+impl<K: ArrowDictionaryKeyType, V: Sync> super::private::Sealed for TypedDictionaryArray<'_, K, V> {}`
	`862`	`+`
`859`	`863`	`impl<K: ArrowDictionaryKeyType, V: Sync> Array for TypedDictionaryArray<'_, K, V> {`
`860`	`864`	`fn as_any(&self) -> &dyn Any {`
`861`	`865`	`self.dictionary`
Original file line number	Diff line number	Diff line change
`@@ -602,6 +602,8 @@ impl std::fmt::Debug for FixedSizeBinaryArray {`
`602`	`602`	`}`
`603`	`603`	`}`
`604`	`604`
	`605`	`+impl super::private::Sealed for FixedSizeBinaryArray {}`
	`606`	`+`
`605`	`607`	`impl Array for FixedSizeBinaryArray {`
`606`	`608`	`fn as_any(&self) -> &dyn Any {`
`607`	`609`	`self`
Original file line number	Diff line number	Diff line change
`@@ -462,6 +462,8 @@ impl From<FixedSizeListArray> for ArrayData {`
`462`	`462`	`}`
`463`	`463`	`}`
`464`	`464`
	`465`	`+impl super::private::Sealed for FixedSizeListArray {}`
	`466`	`+`
`465`	`467`	`impl Array for FixedSizeListArray {`
`466`	`468`	`fn as_any(&self) -> &dyn Any {`
`467`	`469`	`self`
Original file line number	Diff line number	Diff line change
`@@ -525,6 +525,8 @@ impl<OffsetSize: OffsetSizeTrait> GenericListArray<OffsetSize> {`
`525`	`525`	`}`
`526`	`526`	`}`
`527`	`527`
	`528`	`+impl<OffsetSize: OffsetSizeTrait> super::private::Sealed for GenericListArray<OffsetSize> {}`
	`529`	`+`
`528`	`530`	`impl<OffsetSize: OffsetSizeTrait> Array for GenericListArray<OffsetSize> {`
`529`	`531`	`fn as_any(&self) -> &dyn Any {`
`530`	`532`	`self`
Original file line number	Diff line number	Diff line change
`@@ -415,6 +415,8 @@ impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<Offset`
`415`	`415`	`}`
`416`	`416`	`}`
`417`	`417`
	`418`	`+impl<OffsetSize: OffsetSizeTrait> super::private::Sealed for GenericListViewArray<OffsetSize> {}`
	`419`	`+`
`418`	`420`	`impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize> {`
`419`	`421`	`fn as_any(&self) -> &dyn Any {`
`420`	`422`	`self`
Original file line number	Diff line number	Diff line change
`@@ -361,6 +361,8 @@ impl MapArray {`
`361`	`361`	`}`
`362`	`362`	`}`
`363`	`363`
	`364`	`+impl super::private::Sealed for MapArray {}`
	`365`	`+`
`364`	`366`	`impl Array for MapArray {`
`365`	`367`	`fn as_any(&self) -> &dyn Any {`
`366`	`368`	`self`