Nested shading can be generated, but not read back

[rouault]

Zarr version

3.1.5

Numcodecs version

0.15.0

Python Version

3.12.0

Operating System

Linux

Installation

pip install zarr-python

Description

A 2-level nested sharded array can be written fine, but on reading back its values, an exception is thown. The example is a bit dummy, but I wanted to report just in case

Steps to reproduce

# /// script
# requires-python = ">=3.11"
# dependencies = [
#   "zarr@git+https://github.com/zarr-developers/zarr-python.git@main",
# ]
# ///
#
# This script automatically imports the development branch of zarr to check for issues

import zarr
from zarr.codecs import ShardingCodec
inner_sharding = zarr.codecs.ShardingCodec(chunk_shape=(1, 2), codecs=[zarr.codecs.BytesCodec()], index_codecs=[zarr.codecs.BytesCodec()])
outer_sharding = zarr.codecs.ShardingCodec(chunk_shape=(2, 4), codecs=[inner_sharding], index_codecs=[zarr.codecs.BytesCodec()])
z = zarr.array([[0,1,2,3,4,5,6,7,8,9],[10,11,12,13,14,15,16,17,18,19],[20,21,22,23,24,25,26,27,28,29],[30,31,32,33,34,35,36,37,38,39],[40,41,42,43,44,45,46,47,48,49]], chunk_shape=(4,8), codecs=[outer_sharding], store='nested_sharding.zarr', dtype='uint16')
z[:]

Additional output

>>> z[:]
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/array.py", line 2868, in __getitem__
    return self.get_orthogonal_selection(pure_selection, fields=fields)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/array.py", line 3339, in get_orthogonal_selection
    return sync(
           ^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/sync.py", line 159, in sync
    raise return_result
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/sync.py", line 119, in _runner
    return await coro
           ^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/array.py", line 1565, in _get_selection
    await self.codec_pipeline.read(
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 473, in read
    await concurrent_map(
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 116, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 114, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 256, in read_batch
    chunk_array_batch = await self.decode_partial_batch(
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 212, in decode_partial_batch
    return await self.array_bytes_codec.decode_partial(batch_info)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/abc/codec.py", line 228, in decode_partial
    return await concurrent_map(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 116, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 114, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/codecs/sharding.py", line 431, in _decode_partial_single
    await self.codec_pipeline.read(
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 473, in read
    await concurrent_map(
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 116, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 114, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 256, in read_batch
    chunk_array_batch = await self.decode_partial_batch(
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/codec_pipeline.py", line 212, in decode_partial_batch
    return await self.array_bytes_codec.decode_partial(batch_info)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/abc/codec.py", line 228, in decode_partial
    return await concurrent_map(
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 116, in concurrent_map
    return await asyncio.gather(*[asyncio.ensure_future(run(item)) for item in items])
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/core/common.py", line 114, in run
    return await func(*item)
           ^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/codecs/sharding.py", line 416, in _decode_partial_single
    shard_index = await self._load_shard_index_maybe(byte_getter, chunks_per_shard)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/codecs/sharding.py", line 671, in _load_shard_index_maybe
    index_bytes = await byte_getter.get(
                  ^^^^^^^^^^^^^^^^^^^^^^
  File "/home/even/gdal/gdal/build_cmake/doc_venv/lib/python3.12/site-packages/zarr/codecs/sharding.py", line 89, in get
    assert byte_range is None, "byte_range is not supported within shards"
AssertionError: byte_range is not supported within shards

[d-v-b]

thanks for the bug report @rouault, I tracked down the cause of the error and I will open a PR fixing it soon.