latest parquet spec with geometry don't seems to be suported

[djouallah]

it seems parquet with geometry does not works

import duckdb
sd = sedona.db.connect()
url = "https://raw.githubusercontent.com/datasets/geo-countries/master/data/countries.geojson"
local_file = "/lakehouse/default/Files/xxxx.geojson"
urllib.request.urlretrieve(url, local_file)
duckdb.sql(f""" copy(   FROM ST_Read('{local_file}')) TO 'countries.parquet' (FORMAT PARQUET); """)
result = sd.read_parquet('countries.parquet')
result.show()

error

SedonaError: Parquet error: Could not parse metadata: bad data

[paleolimbot]

Thank you for opening!

We don't currently support the new Parquet GEOMETRY/GEOGRAPHY types because the underlying arrow-rs reader doesn't support it yet (we're working on that!). In the meantime, you can read or write using pyarrow >= 21.0.0 and use SedonaDB's Arrow integration:

import fsspec
import geoarrow.pyarrow
import sedona.db
from pyarrow import parquet

sd = sedona.db.connect()

url = "https://raw.githubusercontent.com/geoarrow/geoarrow-data/v0.2.0/natural-earth/files/natural-earth_cities.parquet"
with fsspec.open(url) as f:
    df = sd.create_data_frame(parquet.read_table(f)).to_memtable()
df.show(4)
#> ┌──────────────┬───────────────────────────────┐
#> │     name     ┆            geometry           │
#> │     utf8     ┆            geometry           │
#> ╞══════════════╪═══════════════════════════════╡
#> │ Vatican City ┆ POINT(12.4533865 41.9032822)  │
#> ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
#> │ San Marino   ┆ POINT(12.4417702 43.9360958)  │
#> ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
#> │ Vaduz        ┆ POINT(9.5166695 47.1337238)   │
#> ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤
#> │ Lobamba      ┆ POINT(31.1999971 -26.4666675) │
#> └──────────────┴───────────────────────────────┘

parquet.write_table(df.to_arrow_table(), "foofy.parquet")
parquet.ParquetFile("foofy.parquet").schema
#> <pyarrow._parquet.ParquetSchema object at 0x114e91d80>
#> required group field_id=-1 schema {
#>   optional binary field_id=-1 name (String);
#>   optional binary field_id=-1 geometry (Geometry(crs=));
#> }