Skip to content

Commit d0b59a3

Browse files
heilheadheilhead
authored
Merge pull request #5 from WalletConnect/fix/update-from-upstream-111225
fix: update from upstream 11.12.25
2 parents e862f32 + c615973 commit d0b59a3

File tree

7 files changed

+205
-46
lines changed

7 files changed

+205
-46
lines changed

‎arrow-array/src/array/list_view_array.rs‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,12 @@ use std::ops::Add;
2323
use std::sync::Arc;
2424

2525
use crate::array::{make_array, print_long_array};
26+
use crate::builder::{GenericListViewBuilder, PrimitiveBuilder};
2627
use crate::iterator::GenericListViewArrayIter;
27-
use crate::{Array, ArrayAccessor, ArrayRef, FixedSizeListArray, OffsetSizeTrait, new_empty_array};
28+
use crate::{
29+
Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, FixedSizeListArray, OffsetSizeTrait,
30+
new_empty_array,
31+
};
2832

2933
/// A [`GenericListViewArray`] of variable size lists, storing offsets as `i32`.
3034
pub type ListViewArray = GenericListViewArray<i32>;
@@ -357,6 +361,46 @@ impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
357361
value_sizes: self.value_sizes.slice(offset, length),
358362
}
359363
}
364+
365+
/// Creates a [`GenericListViewArray`] from an iterator of primitive values
366+
/// # Example
367+
/// ```
368+
/// # use arrow_array::ListViewArray;
369+
/// # use arrow_array::types::Int32Type;
370+
///
371+
/// let data = vec![
372+
/// Some(vec![Some(0), Some(1), Some(2)]),
373+
/// None,
374+
/// Some(vec![Some(3), None, Some(5)]),
375+
/// Some(vec![Some(6), Some(7)]),
376+
/// ];
377+
/// let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
378+
/// println!("{:?}", list_array);
379+
/// ```
380+
pub fn from_iter_primitive<T, P, I>(iter: I) -> Self
381+
where
382+
T: ArrowPrimitiveType,
383+
P: IntoIterator<Item = Option<<T as ArrowPrimitiveType>::Native>>,
384+
I: IntoIterator<Item = Option<P>>,
385+
{
386+
let iter = iter.into_iter();
387+
let size_hint = iter.size_hint().0;
388+
let mut builder =
389+
GenericListViewBuilder::with_capacity(PrimitiveBuilder::<T>::new(), size_hint);
390+
391+
for i in iter {
392+
match i {
393+
Some(p) => {
394+
for t in p {
395+
builder.values().append_option(t);
396+
}
397+
builder.append(true);
398+
}
399+
None => builder.append(false),
400+
}
401+
}
402+
builder.finish()
403+
}
360404
}
361405

362406
impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<OffsetSize> {
@@ -559,7 +603,7 @@ impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize> {
559603

560604
#[cfg(test)]
561605
mod tests {
562-
use arrow_buffer::{BooleanBuffer, Buffer, ScalarBuffer, bit_util};
606+
use arrow_buffer::{BooleanBuffer, Buffer, NullBufferBuilder, ScalarBuffer, bit_util};
563607
use arrow_schema::Field;
564608

565609
use crate::builder::{FixedSizeListBuilder, Int32Builder};
@@ -1127,4 +1171,29 @@ mod tests {
11271171
let array = ListViewArray::new_null(field, 5);
11281172
assert_eq!(array.len(), 5);
11291173
}
1174+
1175+
#[test]
1176+
fn test_from_iter_primitive() {
1177+
let data = vec![
1178+
Some(vec![Some(0), Some(1), Some(2)]),
1179+
None,
1180+
Some(vec![Some(3), Some(4), Some(5)]),
1181+
Some(vec![Some(6), Some(7)]),
1182+
];
1183+
let list_array = ListViewArray::from_iter_primitive::<Int32Type, _, _>(data);
1184+
1185+
// [[0, 1, 2], NULL, [3, 4, 5], [6, 7]]
1186+
let values = Int32Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7]);
1187+
let offsets = ScalarBuffer::from(vec![0, 3, 3, 6]);
1188+
let sizes = ScalarBuffer::from(vec![3, 0, 3, 2]);
1189+
let field = Arc::new(Field::new_list_field(DataType::Int32, true));
1190+
1191+
let mut nulls = NullBufferBuilder::new(4);
1192+
nulls.append(true);
1193+
nulls.append(false);
1194+
nulls.append_n_non_nulls(2);
1195+
let another = ListViewArray::new(field, offsets, sizes, Arc::new(values), nulls.finish());
1196+
1197+
assert_eq!(list_array, another)
1198+
}
11301199
}

‎arrow-array/src/ffi_stream.rs‎

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,9 @@ impl Iterator for ArrowArrayStreamReader {
364364
let result = unsafe {
365365
from_ffi_and_data_type(array, DataType::Struct(self.schema().fields().clone()))
366366
};
367-
Some(result.map(|data| RecordBatch::from(StructArray::from(data))))
367+
Some(result.and_then(|data| {
368+
RecordBatch::try_new(self.schema.clone(), StructArray::from(data).into_parts().1)
369+
}))
368370
} else {
369371
let last_error = self.get_stream_last_error();
370372
let err = ArrowError::CDataInterface(last_error.unwrap());
@@ -382,6 +384,7 @@ impl RecordBatchReader for ArrowArrayStreamReader {
382384
#[cfg(test)]
383385
mod tests {
384386
use super::*;
387+
use std::collections::HashMap;
385388

386389
use arrow_schema::Field;
387390

@@ -417,11 +420,18 @@ mod tests {
417420
}
418421

419422
fn _test_round_trip_export(arrays: Vec<Arc<dyn Array>>) -> Result<()> {
420-
let schema = Arc::new(Schema::new(vec![
421-
Field::new("a", arrays[0].data_type().clone(), true),
422-
Field::new("b", arrays[1].data_type().clone(), true),
423-
Field::new("c", arrays[2].data_type().clone(), true),
424-
]));
423+
let metadata = HashMap::from([("foo".to_owned(), "bar".to_owned())]);
424+
let schema = Arc::new(Schema::new_with_metadata(
425+
vec![
426+
Field::new("a", arrays[0].data_type().clone(), true)
427+
.with_metadata(metadata.clone()),
428+
Field::new("b", arrays[1].data_type().clone(), true)
429+
.with_metadata(metadata.clone()),
430+
Field::new("c", arrays[2].data_type().clone(), true)
431+
.with_metadata(metadata.clone()),
432+
],
433+
metadata,
434+
));
425435
let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap();
426436
let iter = Box::new(vec![batch.clone(), batch.clone()].into_iter().map(Ok)) as _;
427437

@@ -452,7 +462,11 @@ mod tests {
452462

453463
let array = unsafe { from_ffi(ffi_array, &ffi_schema) }.unwrap();
454464

455-
let record_batch = RecordBatch::from(StructArray::from(array));
465+
let record_batch = RecordBatch::try_new(
466+
SchemaRef::from(exported_schema.clone()),
467+
StructArray::from(array).into_parts().1,
468+
)
469+
.unwrap();
456470
produced_batches.push(record_batch);
457471
}
458472

@@ -462,11 +476,18 @@ mod tests {
462476
}
463477

464478
fn _test_round_trip_import(arrays: Vec<Arc<dyn Array>>) -> Result<()> {
465-
let schema = Arc::new(Schema::new(vec![
466-
Field::new("a", arrays[0].data_type().clone(), true),
467-
Field::new("b", arrays[1].data_type().clone(), true),
468-
Field::new("c", arrays[2].data_type().clone(), true),
469-
]));
479+
let metadata = HashMap::from([("foo".to_owned(), "bar".to_owned())]);
480+
let schema = Arc::new(Schema::new_with_metadata(
481+
vec![
482+
Field::new("a", arrays[0].data_type().clone(), true)
483+
.with_metadata(metadata.clone()),
484+
Field::new("b", arrays[1].data_type().clone(), true)
485+
.with_metadata(metadata.clone()),
486+
Field::new("c", arrays[2].data_type().clone(), true)
487+
.with_metadata(metadata.clone()),
488+
],
489+
metadata,
490+
));
470491
let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap();
471492
let iter = Box::new(vec![batch.clone(), batch.clone()].into_iter().map(Ok)) as _;
472493

‎arrow-buffer/src/bigint/mod.rs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -583,13 +583,22 @@ impl i256 {
583583
self.high.is_positive() || self.high == 0 && self.low != 0
584584
}
585585

586-
fn leading_zeros(&self) -> u32 {
586+
/// Returns the number of leading zeros in the binary representation of this [`i256`].
587+
pub const fn leading_zeros(&self) -> u32 {
587588
match self.high {
588589
0 => u128::BITS + self.low.leading_zeros(),
589590
_ => self.high.leading_zeros(),
590591
}
591592
}
592593

594+
/// Returns the number of trailing zeros in the binary representation of this [`i256`].
595+
pub const fn trailing_zeros(&self) -> u32 {
596+
match self.low {
597+
0 => u128::BITS + self.high.trailing_zeros(),
598+
_ => self.low.trailing_zeros(),
599+
}
600+
}
601+
593602
fn redundant_leading_sign_bits_i256(n: i256) -> u8 {
594603
let mask = n >> 255; // all ones or all zeros
595604
((n ^ mask).leading_zeros() - 1) as u8 // we only need one sign bit
@@ -1327,4 +1336,36 @@ mod tests {
13271336
let out = big_neg.to_f64().unwrap();
13281337
assert!(out.is_finite() && out.is_sign_negative());
13291338
}
1339+
1340+
#[test]
1341+
fn test_leading_zeros() {
1342+
// Without high part
1343+
assert_eq!(i256::from(0).leading_zeros(), 256);
1344+
assert_eq!(i256::from(1).leading_zeros(), 256 - 1);
1345+
assert_eq!(i256::from(16).leading_zeros(), 256 - 5);
1346+
assert_eq!(i256::from(17).leading_zeros(), 256 - 5);
1347+
1348+
// With high part
1349+
assert_eq!(i256::from_parts(2, 16).leading_zeros(), 128 - 5);
1350+
assert_eq!(i256::from_parts(2, i128::MAX).leading_zeros(), 1);
1351+
1352+
assert_eq!(i256::MAX.leading_zeros(), 1);
1353+
assert_eq!(i256::from(-1).leading_zeros(), 0);
1354+
}
1355+
1356+
#[test]
1357+
fn test_trailing_zeros() {
1358+
// Without high part
1359+
assert_eq!(i256::from(0).trailing_zeros(), 256);
1360+
assert_eq!(i256::from(2).trailing_zeros(), 1);
1361+
assert_eq!(i256::from(16).trailing_zeros(), 4);
1362+
assert_eq!(i256::from(17).trailing_zeros(), 0);
1363+
// With high part
1364+
assert_eq!(i256::from_parts(0, i128::MAX).trailing_zeros(), 128);
1365+
assert_eq!(i256::from_parts(0, 16).trailing_zeros(), 128 + 4);
1366+
assert_eq!(i256::from_parts(2, i128::MAX).trailing_zeros(), 1);
1367+
1368+
assert_eq!(i256::MAX.trailing_zeros(), 0);
1369+
assert_eq!(i256::from(-1).trailing_zeros(), 0);
1370+
}
13301371
}

‎arrow-data/src/transform/run.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,7 @@ fn get_last_run_end<T: ArrowNativeType>(run_ends_data: &super::MutableArrayData)
2525
if run_ends_data.data.len == 0 {
2626
T::default()
2727
} else {
28-
// Convert buffer to typed slice and get the last element
29-
let buffer = Buffer::from(run_ends_data.data.buffer1.as_slice());
30-
let typed_slice: &[T] = buffer.typed_data();
28+
let typed_slice: &[T] = run_ends_data.data.buffer1.typed_data();
3129
if typed_slice.len() >= run_ends_data.data.len {
3230
typed_slice[run_ends_data.data.len - 1]
3331
} else {

‎arrow-pyarrow-integration-testing/tests/test_sql.py‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -527,7 +527,7 @@ def test_empty_recordbatch_with_row_count():
527527
"""
528528

529529
# Create an empty schema with no fields
530-
batch = pa.RecordBatch.from_pydict({"a": [1, 2, 3, 4]}).select([])
530+
batch = pa.RecordBatch.from_pydict({"a": [1, 2, 3, 4]}, metadata={b'key1': b'value1'}).select([])
531531
num_rows = 4
532532
assert batch.num_rows == num_rows
533533
assert batch.num_columns == 0
@@ -545,7 +545,7 @@ def test_record_batch_reader():
545545
"""
546546
Python -> Rust -> Python
547547
"""
548-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
548+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
549549
batches = [
550550
pa.record_batch([[[1], [2, 42]]], schema),
551551
pa.record_batch([[None, [], [5, 6]]], schema),
@@ -571,7 +571,7 @@ def test_record_batch_reader_pycapsule():
571571
"""
572572
Python -> Rust -> Python
573573
"""
574-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
574+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
575575
batches = [
576576
pa.record_batch([[[1], [2, 42]]], schema),
577577
pa.record_batch([[None, [], [5, 6]]], schema),
@@ -621,7 +621,7 @@ def test_record_batch_pycapsule():
621621
"""
622622
Python -> Rust -> Python
623623
"""
624-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
624+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
625625
batch = pa.record_batch([[[1], [2, 42]]], schema)
626626
wrapped = StreamWrapper(batch)
627627
b = rust.round_trip_record_batch_reader(wrapped)
@@ -640,7 +640,7 @@ def test_table_pycapsule():
640640
"""
641641
Python -> Rust -> Python
642642
"""
643-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
643+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
644644
batches = [
645645
pa.record_batch([[[1], [2, 42]]], schema),
646646
pa.record_batch([[None, [], [5, 6]]], schema),
@@ -650,55 +650,59 @@ def test_table_pycapsule():
650650
b = rust.round_trip_record_batch_reader(wrapped)
651651
new_table = b.read_all()
652652

653-
assert table.schema == new_table.schema
654653
assert table == new_table
654+
assert table.schema == new_table.schema
655+
assert table.schema.metadata == new_table.schema.metadata
655656
assert len(table.to_batches()) == len(new_table.to_batches())
656657

657658

658659
def test_table_empty():
659660
"""
660661
Python -> Rust -> Python
661662
"""
662-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
663+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
663664
table = pa.Table.from_batches([], schema=schema)
664665
new_table = rust.build_table([], schema=schema)
665666

666-
assert table.schema == new_table.schema
667667
assert table == new_table
668+
assert table.schema == new_table.schema
669+
assert table.schema.metadata == new_table.schema.metadata
668670
assert len(table.to_batches()) == len(new_table.to_batches())
669671

670672

671673
def test_table_roundtrip():
672674
"""
673675
Python -> Rust -> Python
674676
"""
675-
schema = pa.schema([('ints', pa.list_(pa.int32()))])
677+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
676678
batches = [
677679
pa.record_batch([[[1], [2, 42]]], schema),
678680
pa.record_batch([[None, [], [5, 6]]], schema),
679681
]
680682
table = pa.Table.from_batches(batches, schema=schema)
681683
new_table = rust.round_trip_table(table)
682684

683-
assert table.schema == new_table.schema
684685
assert table == new_table
686+
assert table.schema == new_table.schema
687+
assert table.schema.metadata == new_table.schema.metadata
685688
assert len(table.to_batches()) == len(new_table.to_batches())
686689

687690

688691
def test_table_from_batches():
689692
"""
690693
Python -> Rust -> Python
691694
"""
692-
schema = pa.schema([('ints', pa.list_(pa.int32()))], metadata={b'key1': b'value1'})
695+
schema = pa.schema([pa.field(name='ints', type=pa.list_(pa.int32()), metadata={b'key1': b'value1'})], metadata={b'key1': b'value1'})
693696
batches = [
694697
pa.record_batch([[[1], [2, 42]]], schema),
695698
pa.record_batch([[None, [], [5, 6]]], schema),
696699
]
697700
table = pa.Table.from_batches(batches)
698701
new_table = rust.build_table(batches, schema)
699702

700-
assert table.schema == new_table.schema
701703
assert table == new_table
704+
assert table.schema == new_table.schema
705+
assert table.schema.metadata == new_table.schema.metadata
702706
assert len(table.to_batches()) == len(new_table.to_batches())
703707

704708

0 commit comments

Comments
 (0)