From 65e1ac1014f4d6425c1627a7e8ba5a931ca5cd15 Mon Sep 17 00:00:00 2001 From: Wilson Marcilio-Jr Date: Wed, 26 Mar 2025 18:06:03 -0300 Subject: [PATCH] ensure overflow batches have the same schema as written tables --- quadfeather/tiler.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/quadfeather/tiler.py b/quadfeather/tiler.py index 873d14c..a875b82 100644 --- a/quadfeather/tiler.py +++ b/quadfeather/tiler.py @@ -1528,8 +1528,14 @@ def insert_table(self, table: pa.Table): else: for sidecar, tb in self.keyed_batches(table).items(): for tb in rebatch(tb.to_batches(), 50e6): + writer_schema = self.overflow_buffers[sidecar].schema for batch in tb.to_batches(): - self.overflow_buffers[sidecar].write_batch(batch) + # ensure the new batch has the exact schema instance from the writer + aligned_batch = pa.RecordBatch.from_arrays( + [batch[field.name] for field in writer_schema], + schema=writer_schema + ) + self.overflow_buffers[sidecar].write_batch(aligned_batch) def keyed_batches(self, table: pa.Table): """