diff --git a/.gitignore b/.gitignore index b383699..fcafa91 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,9 @@ target/ !**/src/test/**/target/ ### IntelliJ IDEA ### -.idea/ +.idea/* +!.idea/codeStyles/ +!.idea/codeStyles/** *.iws *.iml *.ipr @@ -43,4 +45,4 @@ build/ /main.py /pyproject.toml /uv.lock -**/__pycache__ +**/__pycache__ \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/README.md b/README.md index 82c4520..5357548 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ This repository contains a Java implementation of Zarr version 2 and 3. ## Usage + ```java import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.HttpStore; @@ -11,36 +12,42 @@ import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.Group; Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") ); Group color = (Group) hierarchy.get("color"); Array array = (Array) color.get("1"); ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape ); Array array = Array.create( - new FilesystemStore("/path/to/zarr").resolve("array"), - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() + new FilesystemStore("/path/to/zarr").resolve("array"), + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() ); ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024}); -array.write( - new long[]{0, 0, 0, 0}, // offset - data +array. + +write( + new long[] { + 0, 0, 0, 0 +}, // offset +data ); ``` + ## Development Start-Guide ### Run Tests Locally -To be able to run the tests locally, make sure to have `python3.11` and `uv` installed. + +To be able to run the tests locally, make sure to have `python3.11` and `uv` installed. Furthermore, you will need the `l4_sample` test data: @@ -48,3 +55,11 @@ Furthermore, you will need the `l4_sample` test data: && cd testdata && unzip l4_sample.zip ` + +### Code Style & Formatting + +This project uses IntelliJ IDEA default Java formatting + +Before submitting changes, please run: + +- IntelliJ: `Reformat Code` and `Optimize Imports` diff --git a/pom.xml b/pom.xml index f4c1091..3df9b78 100644 --- a/pom.xml +++ b/pom.xml @@ -194,10 +194,10 @@ 3.5.0 - attach-javadoc - - jar - + attach-javadoc + + jar + @@ -210,14 +210,14 @@ 1.6 - sign-artifacts - verify - - sign - - - 9F88D86AD9A0D91E - + sign-artifacts + verify + + sign + + + 9F88D86AD9A0D91E + diff --git a/src/main/java/dev/zarr/zarrjava/ZarrException.java b/src/main/java/dev/zarr/zarrjava/ZarrException.java index 9553a68..aa82bc5 100644 --- a/src/main/java/dev/zarr/zarrjava/ZarrException.java +++ b/src/main/java/dev/zarr/zarrjava/ZarrException.java @@ -2,11 +2,11 @@ public class ZarrException extends Exception { - public ZarrException(String message, Throwable cause) { - super(message, cause); - } + public ZarrException(String message, Throwable cause) { + super(message, cause); + } - public ZarrException(String message) { - super(message); - } + public ZarrException(String message) { + super(message); + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 44daa75..a8efaeb 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -1,12 +1,12 @@ package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.core.codec.CodecPipeline; import ucar.ma2.InvalidRangeException; import javax.annotation.Nonnull; @@ -21,7 +21,6 @@ public abstract class Array extends AbstractNode { protected CodecPipeline codecPipeline; - public abstract ArrayMetadata metadata(); protected Array(StoreHandle storeHandle) throws ZarrException { super(storeHandle); @@ -47,7 +46,8 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept throw new ZarrException("No Zarr array found at the specified location."); } } - /** + + /** * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. * * @param path the storage location of the Zarr array @@ -69,6 +69,8 @@ public static Array open(String path) throws IOException, ZarrException { return open(Paths.get(path)); } + public abstract ArrayMetadata metadata(); + /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array * needs be large enough for the write. @@ -94,32 +96,32 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { chunkStream = chunkStream.parallel(); } chunkStream.forEach( - chunkCoords -> { - try { - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape - ); - - ucar.ma2.Array chunkArray; - if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape, - chunkShape - )) { - chunkArray = array.sectionNoReduce(chunkProjection.outOffset, - chunkProjection.shape, - null - ); - } else { - chunkArray = readChunk(chunkCoords); - MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray, - chunkProjection.chunkOffset, chunkProjection.shape - ); + chunkCoords -> { + try { + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, + shape + ); + + ucar.ma2.Array chunkArray; + if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape, + chunkShape + )) { + chunkArray = array.sectionNoReduce(chunkProjection.outOffset, + chunkProjection.shape, + null + ); + } else { + chunkArray = readChunk(chunkCoords); + MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray, + chunkProjection.chunkOffset, chunkProjection.shape + ); + } + writeChunk(chunkCoords, chunkArray); + } catch (ZarrException | InvalidRangeException e) { + throw new RuntimeException(e); } - writeChunk(chunkCoords, chunkArray); - } catch (ZarrException | InvalidRangeException e) { - throw new RuntimeException(e); - } - }); + }); } @@ -246,7 +248,7 @@ boolean chunkIsInArray(long[] chunkCoords) { final int[] chunkShape = metadata().chunkShape(); for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { if (chunkCoords[dimIdx] < 0 - || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) { + || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) { return false; } } @@ -282,47 +284,47 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean } final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), - shape); + shape); Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); } chunkStream.forEach( - chunkCoords -> { - try { - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape - ); - - if (chunkIsInArray(chunkCoords)) { - MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(), - chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset, - chunkProjection.shape - ); - } - - final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - if (!chunkHandle.exists()) { - return; + chunkCoords -> { + try { + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, + shape + ); + + if (chunkIsInArray(chunkCoords)) { + MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(), + chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset, + chunkProjection.shape + ); + } + + final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); + final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); + if (!chunkHandle.exists()) { + return; + } + if (codecPipeline.supportsPartialDecode()) { + final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, + Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); + MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, + chunkProjection.outOffset, chunkProjection.shape + ); + } else { + MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset, + outputArray, chunkProjection.outOffset, chunkProjection.shape + ); + } + + } catch (ZarrException e) { + throw new RuntimeException(e); } - if (codecPipeline.supportsPartialDecode()) { - final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, - Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); - MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, - chunkProjection.outOffset, chunkProjection.shape - ); - } else { - MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset, - outputArray, chunkProjection.outOffset, chunkProjection.shape - ); - } - - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }); + }); return outputArray; } diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index 154d669..4a1a1ad 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -4,9 +4,9 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import ucar.ma2.Array; import javax.annotation.Nonnull; @@ -29,6 +29,111 @@ public ArrayMetadata(long[] shape, Object fillValue, DataType dataType) throws Z this.parsedFillValue = parseFillValue(fillValue, dataType); } + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) + throws ZarrException { + if (fillValue == null) { + return null; + } + boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL; + boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8; + boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16; + boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32; + boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64; + boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32; + boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64; + + if (fillValue instanceof Boolean) { + Boolean fillValueBool = (Boolean) fillValue; + if (dataTypeIsBool) { + return fillValueBool; + } + } + if (fillValue instanceof Number) { + Number fillValueNumber = (Number) fillValue; + if (dataTypeIsBool) { + return fillValueNumber.byteValue() != 0; + } else if (dataTypeIsByte) { + return fillValueNumber.byteValue(); + } else if (dataTypeIsShort) { + return fillValueNumber.shortValue(); + } else if (dataTypeIsInt) { + return fillValueNumber.intValue(); + } else if (dataTypeIsLong) { + return fillValueNumber.longValue(); + } else if (dataTypeIsFloat) { + return fillValueNumber.floatValue(); + } else if (dataTypeIsDouble) { + return fillValueNumber.doubleValue(); + } + // Fallback to throwing below + } else if (fillValue instanceof String) { + String fillValueString = (String) fillValue; + if (fillValueString.equals("NaN")) { + if (dataTypeIsFloat) { + return Float.NaN; + } else if (dataTypeIsDouble) { + return Double.NaN; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } else if (fillValueString.equals("+Infinity")) { + if (dataTypeIsFloat) { + return Float.POSITIVE_INFINITY; + } else if (dataTypeIsDouble) { + return Double.POSITIVE_INFINITY; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } else if (fillValueString.equals("-Infinity")) { + if (dataTypeIsFloat) { + return Float.NEGATIVE_INFINITY; + } else if (dataTypeIsDouble) { + return Double.NEGATIVE_INFINITY; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { + ByteBuffer buf = null; + if (fillValueString.startsWith("0b")) { + buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { + for (int i = 0; i < dataType.getByteCount(); i++) { + b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 8, 2 + (i + 1) * 8), + 2)); + } + return b; + }); + } else if (fillValueString.startsWith("0x")) { + buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { + for (int i = 0; i < dataType.getByteCount(); i++) { + b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 2, 2 + (i + 1) * 2), + 16)); + } + return b; + }); + } + if (buf != null) { + if (dataTypeIsBool) { + return buf.get() != 0; + } else if (dataTypeIsByte) { + return buf.get(); + } else if (dataTypeIsShort) { + return buf.getShort(); + } else if (dataTypeIsInt) { + return buf.getInt(); + } else if (dataTypeIsLong) { + return buf.getLong(); + } else if (dataTypeIsFloat) { + return buf.getFloat(); + } else if (dataTypeIsDouble) { + return buf.getDouble(); + // Fallback to throwing below + } + } + } + } + throw new ZarrException("Invalid fill value '" + fillValue + "'."); + } + public int ndim() { return shape.length; } @@ -43,148 +148,43 @@ public int ndim() { public abstract Object parsedFillValue(); - public @Nonnull abstract Attributes attributes() throws ZarrException; + public @Nonnull + abstract Attributes attributes() throws ZarrException; - public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) - throws ZarrException { - if (fillValue == null) { - return null; - } - boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL; - boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8; - boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16; - boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32; - boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64; - boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32; - boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64; - - if (fillValue instanceof Boolean) { - Boolean fillValueBool = (Boolean) fillValue; - if (dataTypeIsBool) { - return fillValueBool; - } - } - if (fillValue instanceof Number) { - Number fillValueNumber = (Number) fillValue; - if (dataTypeIsBool) { - return fillValueNumber.byteValue() != 0; - } else if (dataTypeIsByte) { - return fillValueNumber.byteValue(); - } else if (dataTypeIsShort) { - return fillValueNumber.shortValue(); - } else if (dataTypeIsInt) { - return fillValueNumber.intValue(); - } else if (dataTypeIsLong) { - return fillValueNumber.longValue(); - } else if (dataTypeIsFloat) { - return fillValueNumber.floatValue(); - } else if (dataTypeIsDouble) { - return fillValueNumber.doubleValue(); - } - // Fallback to throwing below - } else if (fillValue instanceof String) { - String fillValueString = (String) fillValue; - if (fillValueString.equals("NaN")) { - if (dataTypeIsFloat) { - return Float.NaN; - } else if (dataTypeIsDouble) { - return Double.NaN; - } - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } else if (fillValueString.equals("+Infinity")) { - if (dataTypeIsFloat) { - return Float.POSITIVE_INFINITY; - } else if (dataTypeIsDouble) { - return Double.POSITIVE_INFINITY; - } - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } else if (fillValueString.equals("-Infinity")) { - if (dataTypeIsFloat) { - return Float.NEGATIVE_INFINITY; - } else if (dataTypeIsDouble) { - return Double.NEGATIVE_INFINITY; - } - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } - else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { - ByteBuffer buf = null; - if (fillValueString.startsWith("0b")) { - buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { - for (int i = 0; i < dataType.getByteCount(); i++) { - b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 8, 2 + (i + 1) * 8), - 2)); - } - return b; - }); - } else if (fillValueString.startsWith("0x")) { - buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { - for (int i = 0; i < dataType.getByteCount(); i++) { - b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 2, 2 + (i + 1) * 2), - 16)); - } - return b; - }); - } - if (buf != null) { - if (dataTypeIsBool) { - return buf.get() != 0; - } else if (dataTypeIsByte) { - return buf.get(); - } else if (dataTypeIsShort) { - return buf.getShort(); - } else if (dataTypeIsInt) { - return buf.getInt(); - } else if (dataTypeIsLong) { - return buf.getLong(); - } else if (dataTypeIsFloat) { - return buf.getFloat(); - } else if (dataTypeIsDouble) { - return buf.getDouble(); - // Fallback to throwing below - } - } - } - } - throw new ZarrException("Invalid fill value '" + fillValue + "'."); - } + public static final class CoreArrayMetadata { - public static final class CoreArrayMetadata { + public final long[] shape; + public final int[] chunkShape; + public final DataType dataType; + public final Object parsedFillValue; - public final long[] shape; - public final int[] chunkShape; - public final DataType dataType; - public final Object parsedFillValue; - - public CoreArrayMetadata(long[] shape, int[] chunkShape, DataType dataType, - Object parsedFillValue) { - this.shape = shape; - this.chunkShape = chunkShape; - this.dataType = dataType; - this.parsedFillValue = parsedFillValue; - } + public CoreArrayMetadata(long[] shape, int[] chunkShape, DataType dataType, + Object parsedFillValue) { + this.shape = shape; + this.chunkShape = chunkShape; + this.dataType = dataType; + this.parsedFillValue = parsedFillValue; + } - public int ndim() { - return shape.length; - } + public int ndim() { + return shape.length; + } - public int chunkSize() { - return Arrays.stream(chunkShape) - .reduce(1, (acc, a) -> acc * a); - } + public int chunkSize() { + return Arrays.stream(chunkShape) + .reduce(1, (acc, a) -> acc * a); + } - public int chunkByteLength() { - return this.dataType.getByteCount() * chunkSize(); - } + public int chunkByteLength() { + return this.dataType.getByteCount() * chunkSize(); + } - public ucar.ma2.Array allocateFillValueChunk() { - ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunkShape); - MultiArrayUtils.fill(outputArray, parsedFillValue); - return outputArray; + public ucar.ma2.Array allocateFillValueChunk() { + ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunkShape); + MultiArrayUtils.fill(outputArray, parsedFillValue); + return outputArray; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java index bc1cfe2..bc14292 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Attributes.java +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -14,7 +14,7 @@ public Attributes() { super(); } - public Attributes (Function attributeMapper) { + public Attributes(Function attributeMapper) { super(); attributeMapper.apply(this); } @@ -24,12 +24,12 @@ public Attributes(Map attributes) { super(attributes); } - public Attributes set(String s, Object o){ + public Attributes set(String s, Object o) { this.put(s, o); return this; } - public Attributes delete(String s){ + public Attributes delete(String s) { this.remove(s); return this; } @@ -96,43 +96,43 @@ public Attributes getAttributes(String key) { } public T[] getArray(String key, Class clazz) throws ZarrException { - Object value = this.get(key); - if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { - return (T[]) value; - } - if (value instanceof List) { - List list = (List) value; - @SuppressWarnings("unchecked") - T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); - for (int i = 0; i < list.size(); i++) { - Object elem = list.get(i); - if (clazz.isInstance(elem)) { - array[i] = clazz.cast(elem); - } else { - // Try to find a constructor that takes the element's class - java.lang.reflect.Constructor matched = null; - for (java.lang.reflect.Constructor c : clazz.getConstructors()) { - Class[] params = c.getParameterTypes(); - if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) { - matched = c; - break; + Object value = this.get(key); + if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]))) { + return (T[]) value; + } + if (value instanceof List) { + List list = (List) value; + @SuppressWarnings("unchecked") + T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (clazz.isInstance(elem)) { + array[i] = clazz.cast(elem); + } else { + // Try to find a constructor that takes the element's class + java.lang.reflect.Constructor matched = null; + for (java.lang.reflect.Constructor c : clazz.getConstructors()) { + Class[] params = c.getParameterTypes(); + if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) { + matched = c; + break; + } } - } - if (matched != null) { - try { - array[i] = (T) matched.newInstance(elem); - } catch (Exception e) { - throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e); + if (matched != null) { + try { + array[i] = (T) matched.newInstance(elem); + } catch (Exception e) { + throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e); + } + } else { + throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName()); } - } else { - throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName()); } } + return array; } - return array; + throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName()); } - throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName()); -} public int[] getIntArray(String key) { Object value = this.get(key); diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index d8b9a6b..c89617a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -68,14 +68,14 @@ public static Group open(String path) throws IOException, ZarrException { public Stream list() { return storeHandle.list() - .map(key -> { - try { - return get(key); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) - .filter(Objects::nonNull); + .map(key -> { + try { + return get(key); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(Objects::nonNull); } public Node[] listAsArray() { diff --git a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java index e7bd6eb..361d694 100644 --- a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java @@ -1,11 +1,12 @@ package dev.zarr.zarrjava.core; -import javax.annotation.Nonnull; - import dev.zarr.zarrjava.ZarrException; +import javax.annotation.Nonnull; + public abstract class GroupMetadata { - public @Nonnull abstract Attributes attributes() throws ZarrException; + public @Nonnull + abstract Attributes attributes() throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java index c82cb75..1c56f2a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java @@ -2,6 +2,6 @@ public interface ChunkKeyEncoding { - String[] encodeChunkKey(long[] chunkCoords); + String[] encodeChunkKey(long[] chunkCoords); } diff --git a/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java index 5074cd5..6a9526a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java +++ b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java @@ -3,17 +3,17 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum Separator { - SLASH("/"), - DOT("."); + SLASH("/"), + DOT("."); - private final String value; + private final String value; - Separator(String value) { - this.value = value; - } + Separator(String value) { + this.value = value; + } - @JsonValue - public String getValue() { - return value; - } + @JsonValue + public String getValue() { + return value; + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java index d4c634f..71e21b5 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java @@ -3,7 +3,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.ArrayMetadata; -public abstract class AbstractCodec implements Codec{ +public abstract class AbstractCodec implements Codec { protected ArrayMetadata.CoreArrayMetadata arrayMetadata; public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { @@ -13,7 +13,7 @@ public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrExcepti return this.arrayMetadata; } - public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { this.arrayMetadata = arrayMetadata; } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java index be7a7bd..151f5ba 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java @@ -5,10 +5,10 @@ public abstract class ArrayArrayCodec extends AbstractCodec { - public abstract Array encode(Array chunkArray) - throws ZarrException; + public abstract Array encode(Array chunkArray) + throws ZarrException; - public abstract Array decode(Array chunkArray) - throws ZarrException; + public abstract Array decode(Array chunkArray) + throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java index cf4ab64..f8505be 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java @@ -2,25 +2,27 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import java.nio.ByteBuffer; import ucar.ma2.Array; +import java.nio.ByteBuffer; + public abstract class ArrayBytesCodec extends AbstractCodec { - public abstract ByteBuffer encode(Array chunkArray) - throws ZarrException; + public abstract ByteBuffer encode(Array chunkArray) + throws ZarrException; + + public abstract Array decode(ByteBuffer chunkBytes) + throws ZarrException; - public abstract Array decode(ByteBuffer chunkBytes) - throws ZarrException; + public abstract static class WithPartialDecode extends ArrayBytesCodec { - public abstract static class WithPartialDecode extends ArrayBytesCodec { + public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; - public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; - public abstract ByteBuffer encode(Array shardArray) throws ZarrException; + public abstract ByteBuffer encode(Array shardArray) throws ZarrException; - protected abstract Array decodePartial( - StoreHandle handle, long[] offset, int[] shape - ) throws ZarrException; - } + protected abstract Array decodePartial( + StoreHandle handle, long[] offset, int[] shape + ) throws ZarrException; + } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java index 0574f2f..99502aa 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java @@ -2,9 +2,6 @@ import dev.zarr.zarrjava.ZarrException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; public abstract class BytesBytesCodec extends AbstractCodec { diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java index 04bab9a..37a7e09 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java @@ -8,7 +8,7 @@ public interface Codec { ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException; - default long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + default long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { throw new ZarrException("Not implemented for " + this.getClass()); } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java index f4f1706..0498025 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java @@ -4,11 +4,11 @@ public abstract class CodecBuilder { - final protected DataType dataType; + final protected DataType dataType; - public CodecBuilder(DataType dataType) { - this.dataType = dataType; - } + public CodecBuilder(DataType dataType) { + this.dataType = dataType; + } - public abstract Codec[] build(); + public abstract Codec[] build(); } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java index 9d83836..2c27d20 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java @@ -1,174 +1,175 @@ package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.store.StoreHandle; +import ucar.ma2.Array; + +import javax.annotation.Nonnull; import java.nio.ByteBuffer; import java.util.Arrays; -import javax.annotation.Nonnull; -import ucar.ma2.Array; public class CodecPipeline { - @Nonnull - final Codec[] codecs; - public final CoreArrayMetadata arrayMetadata; - - public CodecPipeline(@Nonnull Codec[] codecs, CoreArrayMetadata arrayMetadata) throws ZarrException { - this.arrayMetadata = arrayMetadata; - long arrayBytesCodecCount = Arrays.stream(codecs).filter(c -> c instanceof ArrayBytesCodec) - .count(); - if (arrayBytesCodecCount != 1) { - throw new ZarrException( - "Exactly 1 ArrayBytesCodec is required. Found " + arrayBytesCodecCount + "."); - } - Codec prevCodec = null; - CoreArrayMetadata codecArrayMetadata = arrayMetadata; - for (Codec codec : codecs) { - if (prevCodec != null) { - if (codec instanceof ArrayBytesCodec && prevCodec instanceof ArrayBytesCodec) { - throw new ZarrException( - "ArrayBytesCodec '" + codec.getClass() + "' cannot follow after ArrayBytesCodec '" + - prevCodec.getClass() + "' because only 1 ArrayBytesCodec is allowed."); - } - if (codec instanceof ArrayBytesCodec && prevCodec instanceof BytesBytesCodec) { - throw new ZarrException( - "ArrayBytesCodec '" + codec.getClass() + "' cannot follow after BytesBytesCodec '" + - prevCodec.getClass() + "'."); - } - if (codec instanceof ArrayArrayCodec && prevCodec instanceof ArrayBytesCodec) { - throw new ZarrException( - "ArrayArrayCodec '" + codec.getClass() + "' cannot follow after ArrayBytesCodec '" + - prevCodec.getClass() + "'."); + public final CoreArrayMetadata arrayMetadata; + @Nonnull + final Codec[] codecs; + + public CodecPipeline(@Nonnull Codec[] codecs, CoreArrayMetadata arrayMetadata) throws ZarrException { + this.arrayMetadata = arrayMetadata; + long arrayBytesCodecCount = Arrays.stream(codecs).filter(c -> c instanceof ArrayBytesCodec) + .count(); + if (arrayBytesCodecCount != 1) { + throw new ZarrException( + "Exactly 1 ArrayBytesCodec is required. Found " + arrayBytesCodecCount + "."); } - if (codec instanceof ArrayArrayCodec && prevCodec instanceof BytesBytesCodec) { - throw new ZarrException( - "ArrayArrayCodec '" + codec.getClass() + "' cannot follow after BytesBytesCodec '" + - prevCodec.getClass() + "'."); + Codec prevCodec = null; + CoreArrayMetadata codecArrayMetadata = arrayMetadata; + for (Codec codec : codecs) { + if (prevCodec != null) { + if (codec instanceof ArrayBytesCodec && prevCodec instanceof ArrayBytesCodec) { + throw new ZarrException( + "ArrayBytesCodec '" + codec.getClass() + "' cannot follow after ArrayBytesCodec '" + + prevCodec.getClass() + "' because only 1 ArrayBytesCodec is allowed."); + } + if (codec instanceof ArrayBytesCodec && prevCodec instanceof BytesBytesCodec) { + throw new ZarrException( + "ArrayBytesCodec '" + codec.getClass() + "' cannot follow after BytesBytesCodec '" + + prevCodec.getClass() + "'."); + } + if (codec instanceof ArrayArrayCodec && prevCodec instanceof ArrayBytesCodec) { + throw new ZarrException( + "ArrayArrayCodec '" + codec.getClass() + "' cannot follow after ArrayBytesCodec '" + + prevCodec.getClass() + "'."); + } + if (codec instanceof ArrayArrayCodec && prevCodec instanceof BytesBytesCodec) { + throw new ZarrException( + "ArrayArrayCodec '" + codec.getClass() + "' cannot follow after BytesBytesCodec '" + + prevCodec.getClass() + "'."); + } + } + codec.setCoreArrayMetadata(codecArrayMetadata); + codecArrayMetadata = codec.resolveArrayMetadata(); + prevCodec = codec; } - } - codec.setCoreArrayMetadata(codecArrayMetadata); - codecArrayMetadata = codec.resolveArrayMetadata(); - prevCodec = codec; - } - - this.codecs = codecs; - } - - ArrayArrayCodec[] getArrayArrayCodecs() { - return Arrays.stream(codecs) - .filter(c -> c instanceof ArrayArrayCodec) - .toArray(ArrayArrayCodec[]::new); - } - ArrayBytesCodec getArrayBytesCodec() { - for (Codec codec : codecs) { - if (codec instanceof ArrayBytesCodec) { - return (ArrayBytesCodec) codec; - } - } - throw new RuntimeException( - "Unreachable because the existence of exactly 1 ArrayBytes codec is asserted upon construction."); - } - - BytesBytesCodec[] getBytesBytesCodecs() { - return Arrays.stream(codecs) - .filter(c -> c instanceof BytesBytesCodec) - .toArray(BytesBytesCodec[]::new); - } - - public boolean supportsPartialDecode() { - return codecs.length == 1 && codecs[0] instanceof ArrayBytesCodec.WithPartialDecode; - } - - @Nonnull - public Array decodePartial( - @Nonnull StoreHandle storeHandle, - long[] offset, int[] shape - ) throws ZarrException { - if (!supportsPartialDecode()) { - throw new ZarrException( - "Partial decode is not supported for these codecs. " + Arrays.toString(codecs)); - } - Array chunkArray = ((ArrayBytesCodec.WithPartialDecode) getArrayBytesCodec()).decodePartial( - storeHandle, offset, shape); - if (chunkArray == null) { - throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + this.codecs = codecs; } - return chunkArray; - } - - @Nonnull - public Array decode( - @Nonnull ByteBuffer chunkBytes - ) throws ZarrException { - if (chunkBytes == null) { - throw new ZarrException("chunkBytes is null. Ohh nooo."); + + ArrayArrayCodec[] getArrayArrayCodecs() { + return Arrays.stream(codecs) + .filter(c -> c instanceof ArrayArrayCodec) + .toArray(ArrayArrayCodec[]::new); } - BytesBytesCodec[] bytesBytesCodecs = getBytesBytesCodecs(); - for (int i = bytesBytesCodecs.length - 1; i >= 0; --i) { - BytesBytesCodec codec = bytesBytesCodecs[i]; - chunkBytes = codec.decode(chunkBytes); + ArrayBytesCodec getArrayBytesCodec() { + for (Codec codec : codecs) { + if (codec instanceof ArrayBytesCodec) { + return (ArrayBytesCodec) codec; + } + } + throw new RuntimeException( + "Unreachable because the existence of exactly 1 ArrayBytes codec is asserted upon construction."); } - if (chunkBytes == null) { - throw new ZarrException( - "chunkBytes is null. This is likely a bug in one of the codecs. " + Arrays.toString( - getBytesBytesCodecs())); + BytesBytesCodec[] getBytesBytesCodecs() { + return Arrays.stream(codecs) + .filter(c -> c instanceof BytesBytesCodec) + .toArray(BytesBytesCodec[]::new); } - Array chunkArray = getArrayBytesCodec().decode(chunkBytes); - if (chunkArray == null) { - throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + + public boolean supportsPartialDecode() { + return codecs.length == 1 && codecs[0] instanceof ArrayBytesCodec.WithPartialDecode; } - ArrayArrayCodec[] arrayArrayCodecs = getArrayArrayCodecs(); - for (int i = arrayArrayCodecs.length - 1; i >= 0; --i) { - ArrayArrayCodec codec = arrayArrayCodecs[i]; - chunkArray = codec.decode(chunkArray); + @Nonnull + public Array decodePartial( + @Nonnull StoreHandle storeHandle, + long[] offset, int[] shape + ) throws ZarrException { + if (!supportsPartialDecode()) { + throw new ZarrException( + "Partial decode is not supported for these codecs. " + Arrays.toString(codecs)); + } + Array chunkArray = ((ArrayBytesCodec.WithPartialDecode) getArrayBytesCodec()).decodePartial( + storeHandle, offset, shape); + if (chunkArray == null) { + throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + } + return chunkArray; } - if (chunkArray == null) { - throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + @Nonnull + public Array decode( + @Nonnull ByteBuffer chunkBytes + ) throws ZarrException { + if (chunkBytes == null) { + throw new ZarrException("chunkBytes is null. Ohh nooo."); + } + + BytesBytesCodec[] bytesBytesCodecs = getBytesBytesCodecs(); + for (int i = bytesBytesCodecs.length - 1; i >= 0; --i) { + BytesBytesCodec codec = bytesBytesCodecs[i]; + chunkBytes = codec.decode(chunkBytes); + } + + if (chunkBytes == null) { + throw new ZarrException( + "chunkBytes is null. This is likely a bug in one of the codecs. " + Arrays.toString( + getBytesBytesCodecs())); + } + Array chunkArray = getArrayBytesCodec().decode(chunkBytes); + if (chunkArray == null) { + throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + } + + ArrayArrayCodec[] arrayArrayCodecs = getArrayArrayCodecs(); + for (int i = arrayArrayCodecs.length - 1; i >= 0; --i) { + ArrayArrayCodec codec = arrayArrayCodecs[i]; + chunkArray = codec.decode(chunkArray); + } + + if (chunkArray == null) { + throw new ZarrException("chunkArray is null. This is likely a bug in one of the codecs."); + } + return chunkArray; } - return chunkArray; - } - - @Nonnull - public ByteBuffer encode( - @Nonnull Array chunkArray - ) throws ZarrException { - for (ArrayArrayCodec codec : getArrayArrayCodecs()) { - chunkArray = codec.encode(chunkArray); + + @Nonnull + public ByteBuffer encode( + @Nonnull Array chunkArray + ) throws ZarrException { + for (ArrayArrayCodec codec : getArrayArrayCodecs()) { + chunkArray = codec.encode(chunkArray); + } + + ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray); + + for (BytesBytesCodec codec : getBytesBytesCodecs()) { + chunkBytes = codec.encode(chunkBytes); + } + return chunkBytes; } - ByteBuffer chunkBytes = getArrayBytesCodec().encode(chunkArray); + public long computeEncodedSize(long inputByteLength, CoreArrayMetadata arrayMetadata) + throws ZarrException { + for (Codec codec : codecs) { + inputByteLength = codec.computeEncodedSize(inputByteLength, arrayMetadata); + } + return inputByteLength; + } - for (BytesBytesCodec codec : getBytesBytesCodecs()) { - chunkBytes = codec.encode(chunkBytes); + public Array partialDecode( + StoreHandle valueHandle, long[] offset, int[] shape, + CoreArrayMetadata arrayMetadata + ) { + return null; // TODO } - return chunkBytes; - } - public long computeEncodedSize(long inputByteLength, CoreArrayMetadata arrayMetadata) - throws ZarrException { - for (Codec codec : codecs) { - inputByteLength = codec.computeEncodedSize(inputByteLength, arrayMetadata); + public ByteBuffer partialEncode( + StoreHandle oldValueHandle, Array array, long[] offset, int[] shape, + CoreArrayMetadata arrayMetadata + ) { + return null; // TODO } - return inputByteLength; - } - - public Array partialDecode( - StoreHandle valueHandle, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata - ) { - return null; // TODO - } - - public ByteBuffer partialEncode( - StoreHandle oldValueHandle, Array array, long[] offset, int[] shape, - CoreArrayMetadata arrayMetadata - ) { - return null; // TODO - } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java index bdbe3f1..18dddda 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java @@ -17,57 +17,57 @@ public abstract class BloscCodec extends BytesBytesCodec { - @Override - public ByteBuffer decode(ByteBuffer chunkBytes) - throws ZarrException { - try { - return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); - } catch (Exception ex) { - throw new ZarrException("Error in decoding blosc.", ex); + @Override + public ByteBuffer decode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); + } catch (Exception ex) { + throw new ZarrException("Error in decoding blosc.", ex); + } } - } - public static final class CustomCompressorDeserializer extends StdDeserializer { + public static final class CustomCompressorDeserializer extends StdDeserializer { - public CustomCompressorDeserializer() { - this(null); - } + public CustomCompressorDeserializer() { + this(null); + } - public CustomCompressorDeserializer(Class vc) { - super(vc); - } + public CustomCompressorDeserializer(Class vc) { + super(vc); + } - @Override - public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - String cname = jsonParser.getCodec() - .readValue(jsonParser, String.class); - Blosc.Compressor compressor = Blosc.Compressor.fromString(cname); - if (compressor == null) { - throw new JsonParseException( - jsonParser, - String.format("Could not parse the Blosc.Compressor. Got '%s'", cname) - ); - } - return compressor; + @Override + public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContext ctxt) + throws IOException { + String cname = jsonParser.getCodec() + .readValue(jsonParser, String.class); + Blosc.Compressor compressor = Blosc.Compressor.fromString(cname); + if (compressor == null) { + throw new JsonParseException( + jsonParser, + String.format("Could not parse the Blosc.Compressor. Got '%s'", cname) + ); + } + return compressor; + } } - } - public static final class CustomCompressorSerializer extends StdSerializer { + public static final class CustomCompressorSerializer extends StdSerializer { - public CustomCompressorSerializer() { - super(Blosc.Compressor.class); - } + public CustomCompressorSerializer() { + super(Blosc.Compressor.class); + } - public CustomCompressorSerializer(Class t) { - super(t); - } + public CustomCompressorSerializer(Class t) { + super(t); + } - @Override - public void serialize(Blosc.Compressor compressor, JsonGenerator generator, - SerializerProvider provider) - throws IOException { - generator.writeString(compressor.getValue()); + @Override + public void serialize(Blosc.Compressor compressor, JsonGenerator generator, + SerializerProvider provider) + throws IOException { + generator.writeString(compressor.getValue()); + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 9f86530..3faf97a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -73,9 +73,10 @@ public ByteBuffer encode(Array chunkArray) throws ZarrException { return bb; } - // All other primitive types (byte, short, int, long, char) - return chunkArray.getDataAsByteBuffer(order); -} + // All other primitive types (byte, short, int, long, char) + return chunkArray.getDataAsByteBuffer(order); + } + public enum Endian { LITTLE("little"), BIG("big"); @@ -85,6 +86,10 @@ public enum Endian { this.endian = endian; } + public static Endian nativeOrder() { + return ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? LITTLE : BIG; + } + @JsonValue public String getValue() { return endian; @@ -100,10 +105,6 @@ public ByteOrder getByteOrder() { throw new RuntimeException("Unreachable"); } } - - public static Endian nativeOrder() { - return ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? LITTLE : BIG; - } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index f0b01cc..3d2e447 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -1,143 +1,140 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; +import java.nio.file.*; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class FilesystemStore implements Store, Store.ListableStore { - @Nonnull - private final Path path; + @Nonnull + private final Path path; + + public FilesystemStore(@Nonnull Path path) { + this.path = path; + } - public FilesystemStore(@Nonnull Path path) { - this.path = path; - } + public FilesystemStore(@Nonnull String path) { + this.path = Paths.get(path); + } - public FilesystemStore(@Nonnull String path) { - this.path = Paths.get(path); - } + Path resolveKeys(String[] keys) { + Path newPath = path; + for (String key : keys) { + newPath = newPath.resolve(key); + } + return newPath; + } - Path resolveKeys(String[] keys) { - Path newPath = path; - for (String key : keys) { - newPath = newPath.resolve(key); + @Override + public boolean exists(String[] keys) { + return Files.exists(resolveKeys(keys)); } - return newPath; - } - - @Override - public boolean exists(String[] keys) { - return Files.exists(resolveKeys(keys)); - } - - @Nullable - @Override - public ByteBuffer get(String[] keys) { - try { - return ByteBuffer.wrap(Files.readAllBytes(resolveKeys(keys))); - } catch (IOException e) { - return null; + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + try { + return ByteBuffer.wrap(Files.readAllBytes(resolveKeys(keys))); + } catch (IOException e) { + return null; + } } - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start) { - try (SeekableByteChannel byteChannel = Files.newByteChannel(resolveKeys(keys))) { - long startOffset = 0; - if (start >= 0) { - startOffset = start; - } else { - startOffset = byteChannel.size() + start; - } - long endOffset = byteChannel.size(); - ByteBuffer bytes = Utils.allocateNative((int) (endOffset - startOffset)); - byteChannel.position(startOffset); - byteChannel.read(bytes); - bytes.rewind(); - return bytes; - } catch (IOException e) { - return null; + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + try (SeekableByteChannel byteChannel = Files.newByteChannel(resolveKeys(keys))) { + long startOffset = 0; + if (start >= 0) { + startOffset = start; + } else { + startOffset = byteChannel.size() + start; + } + long endOffset = byteChannel.size(); + ByteBuffer bytes = Utils.allocateNative((int) (endOffset - startOffset)); + byteChannel.position(startOffset); + byteChannel.read(bytes); + bytes.rewind(); + return bytes; + } catch (IOException e) { + return null; + } } - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start, long end) { - try (SeekableByteChannel byteChannel = Files.newByteChannel(resolveKeys(keys))) { - long startOffset = 0; - if (start >= 0) { - startOffset = start; - } else { - startOffset = byteChannel.size() + start; - } - ByteBuffer bytes = Utils.allocateNative((int) (end - startOffset)); - byteChannel.position(startOffset); - byteChannel.read(bytes); - bytes.rewind(); - return bytes; - } catch (IOException e) { - return null; + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + try (SeekableByteChannel byteChannel = Files.newByteChannel(resolveKeys(keys))) { + long startOffset = 0; + if (start >= 0) { + startOffset = start; + } else { + startOffset = byteChannel.size() + start; + } + ByteBuffer bytes = Utils.allocateNative((int) (end - startOffset)); + byteChannel.position(startOffset); + byteChannel.read(bytes); + bytes.rewind(); + return bytes; + } catch (IOException e) { + return null; + } } - } - @Override - public void set(String[] keys, ByteBuffer bytes) { - Path keyPath = resolveKeys(keys); - try { - Files.createDirectories(keyPath.getParent()); - } catch (IOException e) { - throw new RuntimeException(e); + @Override + public void set(String[] keys, ByteBuffer bytes) { + Path keyPath = resolveKeys(keys); + try { + Files.createDirectories(keyPath.getParent()); + } catch (IOException e) { + throw new RuntimeException(e); + } + try (SeekableByteChannel channel = Files.newByteChannel(keyPath.toAbsolutePath(), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE + )) { + channel.write(bytes); + } catch (IOException e) { + throw new RuntimeException(e); + } } - try (SeekableByteChannel channel = Files.newByteChannel(keyPath.toAbsolutePath(), - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE - )) { - channel.write(bytes); - } catch (IOException e) { - throw new RuntimeException(e); + + @Override + public void delete(String[] keys) { + try { + Files.delete(resolveKeys(keys)); + } catch (NoSuchFileException e) { + // ignore + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Stream list(String[] keys) { + try { + return Files.list(resolveKeys(keys)).map(p -> p.toFile().getName()); + } catch (IOException e) { + throw new RuntimeException(e); + } } - } - - @Override - public void delete(String[] keys) { - try { - Files.delete(resolveKeys(keys)); - } catch (NoSuchFileException e) { - // ignore - } catch (IOException e) { - throw new RuntimeException(e); + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); } - } - public Stream list(String[] keys) { - try { - return Files.list(resolveKeys(keys)).map(p -> p.toFile().getName()); - } catch (IOException e) { - throw new RuntimeException(e); + @Override + public String toString() { + return this.path.toUri().toString().replaceAll("\\/$", ""); } - } - - @Nonnull - @Override - public StoreHandle resolve(String... keys) { - return new StoreHandle(this, keys); - } - - @Override - public String toString() { - return this.path.toUri().toString().replaceAll("\\/$", ""); - } } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 343d251..cb7ddc2 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -1,106 +1,103 @@ package dev.zarr.zarrjava.store; -import com.squareup.okhttp.Call; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import com.squareup.okhttp.ResponseBody; -import java.io.IOException; -import java.nio.ByteBuffer; +import com.squareup.okhttp.*; + import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; public class HttpStore implements Store { - @Nonnull - private final OkHttpClient httpClient; - @Nonnull - private final String uri; + @Nonnull + private final OkHttpClient httpClient; + @Nonnull + private final String uri; + + public HttpStore(@Nonnull String uri) { + this.httpClient = new OkHttpClient(); + this.uri = uri; + } + + String resolveKeys(String[] keys) { + StringBuilder newUri = new StringBuilder(uri.replaceAll("\\/+$", "")); + for (String key : keys) { + newUri.append("/").append(key); + } + return newUri.toString(); + } + + @Nullable + ByteBuffer get(Request request) { + Call call = httpClient.newCall(request); + try { + Response response = call.execute(); + try (ResponseBody body = response.body()) { + return ByteBuffer.wrap(body.bytes()); + } + } catch (IOException e) { + return null; + } + } + + @Override + public boolean exists(String[] keys) { + Request request = new Request.Builder().head().url(resolveKeys(keys)).build(); + Call call = httpClient.newCall(request); + try { + Response response = call.execute(); + return response.isSuccessful(); + } catch (IOException e) { + return false; + } + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + Request request = new Request.Builder().url(resolveKeys(keys)).build(); + return get(request); + } - public HttpStore(@Nonnull String uri) { - this.httpClient = new OkHttpClient(); - this.uri = uri; - } + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + Request request = new Request.Builder().url(resolveKeys(keys)).header( + "Range", start < 0 ? String.format("Bytes=%d", start) : String.format("Bytes=%d-", start)) + .build(); - String resolveKeys(String[] keys) { - StringBuilder newUri = new StringBuilder(uri.replaceAll("\\/+$", "")); - for (String key : keys) { - newUri.append("/").append(key); + return get(request); } - return newUri.toString(); - } - - @Nullable - ByteBuffer get(Request request) { - Call call = httpClient.newCall(request); - try { - Response response = call.execute(); - try (ResponseBody body = response.body()) { - return ByteBuffer.wrap(body.bytes()); - } - } catch (IOException e) { - return null; + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + if (start < 0) { + throw new IllegalArgumentException("Argument 'start' needs to be non-negative."); + } + Request request = new Request.Builder().url(resolveKeys(keys)).header( + "Range", String.format("Bytes=%d-%d", start, end - 1)).build(); + return get(request); } - } - - @Override - public boolean exists(String[] keys) { - Request request = new Request.Builder().head().url(resolveKeys(keys)).build(); - Call call = httpClient.newCall(request); - try { - Response response = call.execute(); - return response.isSuccessful(); - } catch (IOException e) { - return false; + + @Override + public void set(String[] keys, ByteBuffer bytes) { + throw new UnsupportedOperationException("Not implemented"); } - } - - @Nullable - @Override - public ByteBuffer get(String[] keys) { - Request request = new Request.Builder().url(resolveKeys(keys)).build(); - return get(request); - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start) { - Request request = new Request.Builder().url(resolveKeys(keys)).header( - "Range", start < 0 ? String.format("Bytes=%d", start) : String.format("Bytes=%d-", start)) - .build(); - - return get(request); - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start, long end) { - if (start < 0) { - throw new IllegalArgumentException("Argument 'start' needs to be non-negative."); + + @Override + public void delete(String[] keys) { + throw new UnsupportedOperationException("Not implemented"); + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return uri; } - Request request = new Request.Builder().url(resolveKeys(keys)).header( - "Range", String.format("Bytes=%d-%d", start, end - 1)).build(); - return get(request); - } - - @Override - public void set(String[] keys, ByteBuffer bytes) { - throw new UnsupportedOperationException("Not implemented"); - } - - @Override - public void delete(String[] keys) { - throw new UnsupportedOperationException("Not implemented"); - } - - @Nonnull - @Override - public StoreHandle resolve(String... keys) { - return new StoreHandle(this, keys); - } - - @Override - public String toString() { - return uri; - } } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index c1bbb9d..42ac6ff 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -8,81 +8,81 @@ import java.util.stream.Stream; public class MemoryStore implements Store, Store.ListableStore { - private final Map, byte[]> map = new ConcurrentHashMap<>(); + private final Map, byte[]> map = new ConcurrentHashMap<>(); - List resolveKeys(String[] keys) { - ArrayList resolvedKeys = new ArrayList<>(); - for(String key:keys){ - if(key.startsWith("/")){ - key = key.substring(1); - } - resolvedKeys.addAll(Arrays.asList(key.split("/"))); + List resolveKeys(String[] keys) { + ArrayList resolvedKeys = new ArrayList<>(); + for (String key : keys) { + if (key.startsWith("/")) { + key = key.substring(1); + } + resolvedKeys.addAll(Arrays.asList(key.split("/"))); + } + return resolvedKeys; } - return resolvedKeys; - } - @Override - public boolean exists(String[] keys) { - return map.containsKey(resolveKeys(keys)); - } + @Override + public boolean exists(String[] keys) { + return map.containsKey(resolveKeys(keys)); + } - @Nullable - @Override - public ByteBuffer get(String[] keys) { - return get(keys, 0); - } + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return get(keys, 0); + } - @Nullable - @Override - public ByteBuffer get(String[] keys, long start) { - return get(keys, start, -1); - } + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return get(keys, start, -1); + } - @Nullable - @Override - public ByteBuffer get(String[] keys, long start, long end) { - byte[] bytes = map.get(resolveKeys(keys)); - if (bytes == null) return null; - if (end < 0) end = bytes.length; - if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); - return ByteBuffer.wrap(bytes, (int) start, (int) end); - } + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + byte[] bytes = map.get(resolveKeys(keys)); + if (bytes == null) return null; + if (end < 0) end = bytes.length; + if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); + return ByteBuffer.wrap(bytes, (int) start, (int) end); + } - @Override - public void set(String[] keys, ByteBuffer bytes) { - map.put(resolveKeys(keys), bytes.array()); - } + @Override + public void set(String[] keys, ByteBuffer bytes) { + map.put(resolveKeys(keys), bytes.array()); + } - @Override - public void delete(String[] keys) { - map.remove(resolveKeys(keys)); - } + @Override + public void delete(String[] keys) { + map.remove(resolveKeys(keys)); + } - public Stream list(String[] keys) { - List prefix = resolveKeys(keys); - Set allKeys = new HashSet<>(); + public Stream list(String[] keys) { + List prefix = resolveKeys(keys); + Set allKeys = new HashSet<>(); - for (List k : map.keySet()) { - if (k.size() <= prefix.size() || ! k.subList(0, prefix.size()).equals(prefix)) - continue; - for (int i = 0; i < k.size(); i++) { - List subKey = k.subList(0, i+1); - allKeys.add(String.join("/", subKey)); - } + for (List k : map.keySet()) { + if (k.size() <= prefix.size() || !k.subList(0, prefix.size()).equals(prefix)) + continue; + for (int i = 0; i < k.size(); i++) { + List subKey = k.subList(0, i + 1); + allKeys.add(String.join("/", subKey)); + } + } + return allKeys.stream(); } - return allKeys.stream(); - } - @Nonnull - @Override - public StoreHandle resolve(String... keys) { - return new StoreHandle(this, keys); - } + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } - @Override - public String toString() { - return String.format("", hashCode()); - } + @Override + public String toString() { + return String.format("", hashCode()); + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 27aef77..ff1ad7d 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -6,123 +6,123 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class S3Store implements Store, Store.ListableStore { - @Nonnull - private final S3Client s3client; - @Nonnull - private final String bucketName; - @Nullable - private final String prefix; - - public S3Store(@Nonnull S3Client s3client, @Nonnull String bucketName, @Nullable String prefix) { - this.s3client = s3client; - this.bucketName = bucketName; - this.prefix = prefix; - } - - String resolveKeys(String[] keys) { - if (prefix == null) { - return String.join("/", keys); + @Nonnull + private final S3Client s3client; + @Nonnull + private final String bucketName; + @Nullable + private final String prefix; + + public S3Store(@Nonnull S3Client s3client, @Nonnull String bucketName, @Nullable String prefix) { + this.s3client = s3client; + this.bucketName = bucketName; + this.prefix = prefix; + } + + String resolveKeys(String[] keys) { + if (prefix == null) { + return String.join("/", keys); + } + if (keys == null || keys.length == 0) { + return prefix; + } + return prefix + "/" + String.join("/", keys); + } + + @Nullable + ByteBuffer get(GetObjectRequest getObjectRequest) { + try (ResponseInputStream inputStream = s3client.getObject(getObjectRequest)) { + return Utils.asByteBuffer(inputStream); + } catch (IOException e) { + return null; + } + } + + @Override + public boolean exists(String[] keys) { + HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(); + try { + return s3client.headObject(req).sdkHttpResponse().statusCode() == 200; + } catch (NoSuchKeyException e) { + return false; + } + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return get(GetObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) + .build()); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(String.valueOf(start)) + .build(); + return get(req); } - if (keys == null || keys.length == 0) { - return prefix; + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(start + "-" + end) + .build(); + return get(req); } - return prefix + "/" + String.join("/", keys); - } - - @Nullable - ByteBuffer get(GetObjectRequest getObjectRequest) { - try (ResponseInputStream inputStream = s3client.getObject(getObjectRequest)) { - return Utils.asByteBuffer(inputStream); - } catch (IOException e) { - return null; + + @Override + public void set(String[] keys, ByteBuffer bytes) { + try (InputStream byteStream = new ByteArrayInputStream(Utils.toArray(bytes))) { + /*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/ + s3client.putObject(PutObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(), RequestBody.fromContentProvider(() -> byteStream, "application/octet-stream")); + } catch (IOException e) { + throw new RuntimeException(e); + } } - } - - @Override - public boolean exists(String[] keys) { - HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(); - try { - return s3client.headObject(req).sdkHttpResponse().statusCode() == 200; - } catch (NoSuchKeyException e) { - return false; + + @Override + public void delete(String[] keys) { + s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) + .build()); } - } - - @Nullable - @Override - public ByteBuffer get(String[] keys) { - return get(GetObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) - .build()); - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start) { - GetObjectRequest req = GetObjectRequest.builder() - .bucket(bucketName) - .key(resolveKeys(keys)) - .range(String.valueOf(start)) - .build(); - return get(req); - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start, long end) { - GetObjectRequest req = GetObjectRequest.builder() - .bucket(bucketName) - .key(resolveKeys(keys)) - .range(start +"-"+ end) - .build(); - return get(req); - } - - @Override - public void set(String[] keys, ByteBuffer bytes) { - try (InputStream byteStream = new ByteArrayInputStream(Utils.toArray(bytes))) { - /*AWS SDK for Java v2 migration: When using InputStream to upload with S3Client, Content-Length should be specified and used with RequestBody.fromInputStream(). Otherwise, the entire stream will be buffered in memory. If content length must be unknown, we recommend using the CRT-based S3 client - https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html*/ - s3client.putObject(PutObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(), RequestBody.fromContentProvider(() -> byteStream, "application/octet-stream")); - } catch (IOException e) { - throw new RuntimeException(e); + + @Override + public Stream list(String[] keys) { + final String fullKey = resolveKeys(keys); + ListObjectsRequest req = ListObjectsRequest.builder() + .bucket(bucketName).prefix(fullKey) + .build(); + ListObjectsResponse res = s3client.listObjects(req); + return res.contents() + .stream() + .map(p -> p.key().substring(fullKey.length() + 1)); + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return "s3://" + bucketName + "/" + prefix; } - } - - @Override - public void delete(String[] keys) { - s3client.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)) - .build()); - } - - @Override - public Stream list(String[] keys) { - final String fullKey = resolveKeys(keys); - ListObjectsRequest req = ListObjectsRequest.builder() - .bucket(bucketName).prefix(fullKey) - .build(); - ListObjectsResponse res = s3client.listObjects(req); - return res.contents() - .stream() - .map(p -> p.key().substring(fullKey.length() + 1)); - } - - @Nonnull - @Override - public StoreHandle resolve(String... keys) { - return new StoreHandle(this, keys); - } - - @Override - public String toString() { - return "s3://" + bucketName + "/" + prefix; - } } diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index c92906d..41996a6 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -1,32 +1,32 @@ package dev.zarr.zarrjava.store; -import java.nio.ByteBuffer; -import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.stream.Stream; public interface Store { - boolean exists(String[] keys); + boolean exists(String[] keys); - @Nullable - ByteBuffer get(String[] keys); + @Nullable + ByteBuffer get(String[] keys); - @Nullable - ByteBuffer get(String[] keys, long start); + @Nullable + ByteBuffer get(String[] keys, long start); - @Nullable - ByteBuffer get(String[] keys, long start, long end); + @Nullable + ByteBuffer get(String[] keys, long start, long end); - void set(String[] keys, ByteBuffer bytes); + void set(String[] keys, ByteBuffer bytes); - void delete(String[] keys); + void delete(String[] keys); - @Nonnull - StoreHandle resolve(String... keys); + @Nonnull + StoreHandle resolve(String... keys); - interface ListableStore extends Store { + interface ListableStore extends Store { - Stream list(String[] keys); - } + Stream list(String[] keys); + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index b82424f..d435646 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -1,81 +1,82 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; public class StoreHandle { - @Nonnull - final Store store; - @Nonnull - final String[] keys; - - public StoreHandle(@Nonnull Store store, @Nonnull String... keys) { - this.store = store; - this.keys = keys; - } - - @Nullable - public ByteBuffer read() { - return store.get(keys); - } - - @Nonnull - public ByteBuffer readNonNull() throws NoSuchFileException { - ByteBuffer bytes = read(); - if (bytes == null) { - throw new NoSuchFileException(this.toString()); + @Nonnull + final Store store; + @Nonnull + final String[] keys; + + public StoreHandle(@Nonnull Store store, @Nonnull String... keys) { + this.store = store; + this.keys = keys; } - return bytes; - } - - @Nullable - public ByteBuffer read(long start) { - return store.get(keys, start); - } - - @Nullable - public ByteBuffer read(long start, long end) { - return store.get(keys, start, end); - } - - public void set(ByteBuffer bytes) { - store.set(keys, bytes); - } - - public void delete() { - store.delete(keys); - } - - public boolean exists() { - return store.exists(keys); - } - - public Stream list() { - if (!(store instanceof Store.ListableStore)) { - throw new UnsupportedOperationException("The underlying store does not support listing."); + + @Nullable + public ByteBuffer read() { + return store.get(keys); + } + + @Nonnull + public ByteBuffer readNonNull() throws NoSuchFileException { + ByteBuffer bytes = read(); + if (bytes == null) { + throw new NoSuchFileException(this.toString()); + } + return bytes; } - return ((Store.ListableStore) store).list(keys); - } - @Override - public String toString() { - return store + "/" + String.join("/", keys); - } + @Nullable + public ByteBuffer read(long start) { + return store.get(keys, start); + } + + @Nullable + public ByteBuffer read(long start, long end) { + return store.get(keys, start, end); + } - public StoreHandle resolve(String... subKeys) { - return new StoreHandle(store, Utils.concatArrays(keys, subKeys)); - } + public void set(ByteBuffer bytes) { + store.set(keys, bytes); + } + + public void delete() { + store.delete(keys); + } + + public boolean exists() { + return store.exists(keys); + } + + public Stream list() { + if (!(store instanceof Store.ListableStore)) { + throw new UnsupportedOperationException("The underlying store does not support listing."); + } + return ((Store.ListableStore) store).list(keys); + } + + @Override + public String toString() { + return store + "/" + String.join("/", keys); + } + + public StoreHandle resolve(String... subKeys) { + return new StoreHandle(store, Utils.concatArrays(keys, subKeys)); + } - public Path toPath() { - if (!(store instanceof FilesystemStore)) { - throw new UnsupportedOperationException("The underlying store is not a filesystem store."); + public Path toPath() { + if (!(store instanceof FilesystemStore)) { + throw new UnsupportedOperationException("The underlying store is not a filesystem store."); + } + return ((FilesystemStore) store).resolveKeys(keys); } - return ((FilesystemStore) store).resolveKeys(keys); - } } diff --git a/src/main/java/dev/zarr/zarrjava/utils/CRC32C.java b/src/main/java/dev/zarr/zarrjava/utils/CRC32C.java index 3a3e262..c85a5ed 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/CRC32C.java +++ b/src/main/java/dev/zarr/zarrjava/utils/CRC32C.java @@ -7,159 +7,159 @@ public class CRC32C implements Checksum { - /** - * This class generates a CRC32C checksum, defined by rfc3720 section B.4. - */ + /** + * This class generates a CRC32C checksum, defined by rfc3720 section B.4. + */ - private static final long[] CRC_TABLE = - { - 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, - 0xd4ca64eb, - 0x8ad958cf, - 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, - 0x105ec76f, - 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, - 0x9a879fa0, - 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, - 0x20bd8ede, - 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, - 0xaa64d611, - 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, - 0x30e349b1, - 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, - 0xba3a117e, - 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, - 0x417b1dbc, - 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, - 0xcba24573, - 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, - 0x5125dad3, - 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, - 0xdbfc821c, - 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, - 0x61c69362, - 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, - 0xeb1fcbad, - 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, - 0x7198540d, - 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, - 0xfb410cc2, - 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, - 0x82f63b78, - 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, - 0x082f63b7, - 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, - 0x92a8fc17, - 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, - 0x1871a4d8, - 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, - 0xa24bb5a6, - 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, - 0x2892ed69, - 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, - 0xb21572c9, - 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, - 0x38cc2a06, - 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, - 0xc38d26c4, - 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, - 0x49547e0b, - 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, - 0xd3d3e1ab, - 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, - 0x590ab964, - 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, - 0xe330a81a, - 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, - 0x69e9f0d5, - 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, - 0xf36e6f75, - 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, - 0x79b737ba, - 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 - }; + private static final long[] CRC_TABLE = + { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, 0x26a1e7e8, + 0xd4ca64eb, + 0x8ad958cf, + 0x78b2dbcc, 0x6be22838, 0x9989ab3b, 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, + 0xe235446c, 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, + 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, + 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, + 0x580f5512, 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, + 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, + 0x4851927d, 0x5b016189, 0xa96ae28a, 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, + 0xb3109ebf, 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, + 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, + 0xa34e59d0, 0xb01eaa24, 0x42752927, 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, + 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, + 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, + 0x197448ae, 0x0a24bb5a, 0xf84f3859, 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, + 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, + 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, + 0x709db87b, 0x63cd4b8f, 0x91a6c88c, 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, + 0xfa44e0b4, 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, + 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, + 0xea1a27db, 0xf94ad42f, 0x0b21572c, 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, + 0x502036a5, 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, + 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, + 0x407ef1ca, 0x532e023e, 0xa145813d, 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, + 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, + 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, + 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, + 0x21b862a8, 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, + 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, + 0x115b2b19, 0x020bd8ed, 0xf0605bee, 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, + 0x9b8273d6, 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, + 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, + 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 + }; - private static final long LONG_MASK = 0xffffffffL; - private static final long BYTE_MASK = 0xff; + private static final long LONG_MASK = 0xffffffffL; + private static final long BYTE_MASK = 0xff; - private long crc; + private long crc; - public CRC32C() { - crc = 0; - } + public CRC32C() { + crc = 0; + } - /** - * Updates the checksum with a new byte. - * - * @param b the new byte. - */ - @Override - public void update(int b) { - long newCrc = crc ^ LONG_MASK; - newCrc = updateByte((byte) b, newCrc); - crc = newCrc ^ LONG_MASK; - } + /** + * Updates the checksum with a new byte. + * + * @param b the new byte. + */ + @Override + public void update(int b) { + long newCrc = crc ^ LONG_MASK; + newCrc = updateByte((byte) b, newCrc); + crc = newCrc ^ LONG_MASK; + } - /** - * Updates the checksum with an array of bytes. - * - * @param bArray the array of bytes. - * @param off the offset into the array where the update should begin. - * @param len the length of data to examine. - */ - @Override - public void update(byte[] bArray, int off, int len) { - long newCrc = crc ^ LONG_MASK; - for (int i = off; i < off + len; i++) { - newCrc = updateByte(bArray[i], newCrc); + /** + * Updates the checksum with an array of bytes. + * + * @param bArray the array of bytes. + * @param off the offset into the array where the update should begin. + * @param len the length of data to examine. + */ + @Override + public void update(byte[] bArray, int off, int len) { + long newCrc = crc ^ LONG_MASK; + for (int i = off; i < off + len; i++) { + newCrc = updateByte(bArray[i], newCrc); + } + crc = newCrc ^ LONG_MASK; } - crc = newCrc ^ LONG_MASK; - } - public void update(ByteBuffer buf) { - long newCrc = crc ^ LONG_MASK; - while (buf.remaining() > 0) { - newCrc = updateByte(buf.get(), newCrc); + public void update(ByteBuffer buf) { + long newCrc = crc ^ LONG_MASK; + while (buf.remaining() > 0) { + newCrc = updateByte(buf.get(), newCrc); + } + crc = newCrc ^ LONG_MASK; } - crc = newCrc ^ LONG_MASK; - } - /** - * Returns the value of the checksum. - * - * @return the long representation of the checksum (high bits set to zero). - */ - @Override - public long getValue() { - return crc; - } + /** + * Returns the value of the checksum. + * + * @return the long representation of the checksum (high bits set to zero). + */ + @Override + public long getValue() { + return crc; + } - /** - * Returns the value of the checksum. - * - * @return the 4-byte array representation of the checksum in network byte order (big endian). - */ - public byte[] getValueAsBytes() { - long value = crc; - byte[] result = new byte[4]; - for (int i = 3; i >= 0; i--) { - result[i] = (byte) (value & 0xffL); - value >>= 8; + /** + * Returns the value of the checksum. + * + * @return the 4-byte array representation of the checksum in network byte order (big endian). + */ + public byte[] getValueAsBytes() { + long value = crc; + byte[] result = new byte[4]; + for (int i = 3; i >= 0; i--) { + result[i] = (byte) (value & 0xffL); + value >>= 8; + } + return result; } - return result; - } - /** - * Resets the crc. - */ - @Override - public void reset() { - crc = 0; - } + /** + * Resets the crc. + */ + @Override + public void reset() { + crc = 0; + } - private long updateByte(byte newByte, long crc) { - byte b = (byte) (newByte & BYTE_MASK); - int index = (int) ((crc ^ b) & BYTE_MASK); - return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK; - } + private long updateByte(byte newByte, long crc) { + byte b = (byte) (newByte & BYTE_MASK); + int index = (int) ((crc ^ b) & BYTE_MASK); + return (CRC_TABLE[index] ^ (crc >> 8)) & LONG_MASK; + } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java index 406300a..b108aca 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java @@ -4,188 +4,188 @@ public class IndexingUtils { - public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape) { - return computeChunkCoords(arrayShape, chunkShape, new long[arrayShape.length], - Utils.toIntArray(arrayShape)); - } - - public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) { - return computeChunkCoords(Utils.toLongArray(arrayShape), chunkShape); - } - - public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, - int[] selShape) { - final int ndim = arrayShape.length; - long[] start = new long[ndim]; - long[] end = new long[ndim]; - int numChunks = 1; - for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - final int staIdx = (int) (selOffset[dimIdx] / chunkShape[dimIdx]); - final int endIdx = (int) ((selOffset[dimIdx] + selShape[dimIdx] - 1) / chunkShape[dimIdx]); - numChunks *= (endIdx - staIdx + 1); - start[dimIdx] = staIdx; - end[dimIdx] = endIdx; + public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape) { + return computeChunkCoords(arrayShape, chunkShape, new long[arrayShape.length], + Utils.toIntArray(arrayShape)); } - final long[][] chunkCoords = new long[numChunks][]; - - final long[] currentIdx = Arrays.copyOf(start, ndim); - for (int i = 0; i < chunkCoords.length; i++) { - chunkCoords[i] = Arrays.copyOf(currentIdx, ndim); - int dimIdx = ndim - 1; - while (dimIdx >= 0) { - if (currentIdx[dimIdx] >= end[dimIdx]) { - currentIdx[dimIdx] = start[dimIdx]; - dimIdx--; - } else { - currentIdx[dimIdx]++; - dimIdx = -1; - } - } + public static long[][] computeChunkCoords(int[] arrayShape, int[] chunkShape) { + return computeChunkCoords(Utils.toLongArray(arrayShape), chunkShape); } - return chunkCoords; - } - - public static ChunkProjection computeProjection(long[] chunkCoords, int[] arrayShape, - int[] chunkShape) { - return computeProjection(chunkCoords, Utils.toLongArray(arrayShape), chunkShape); - } - - public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, - int[] chunkShape) { - return computeProjection(chunkCoords, arrayShape, chunkShape, new long[chunkCoords.length], - Utils.toIntArray(arrayShape) - ); - } - - public static ChunkProjection computeProjection( - final long[] chunkCoords, final long[] arrayShape, - final int[] chunkShape, final long[] selOffset, - final int[] selShape - ) { - final int ndim = chunkCoords.length; - final int[] chunkOffset = new int[ndim]; - final int[] outOffset = new int[ndim]; - final int[] shape = new int[ndim]; - - for (int dimIdx = 0; dimIdx < chunkCoords.length; dimIdx++) { - // compute offsets for chunk within overall array - final long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx]; - final long dimLimit = Math.min(arrayShape[dimIdx], - (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]); - - if (selOffset[dimIdx] < dimOffset) { - // selection starts before current chunk - chunkOffset[dimIdx] = 0; - // compute number of previous items, provides offset into output array - outOffset[dimIdx] = (int) (dimOffset - selOffset[dimIdx]); - } else { - // selection starts within current chunk - chunkOffset[dimIdx] = (int) (selOffset[dimIdx] - dimOffset); - outOffset[dimIdx] = 0; - } - - if (selOffset[dimIdx] + selShape[dimIdx] > dimLimit) { - // selection ends after current chunk - shape[dimIdx] = (int) (chunkShape[dimIdx] - (selOffset[dimIdx] % chunkShape[dimIdx])); - } else { - // selection ends within current chunk - shape[dimIdx] = (int) (selOffset[dimIdx] + selShape[dimIdx] - dimOffset - - chunkOffset[dimIdx]); - } + + public static long[][] computeChunkCoords(long[] arrayShape, int[] chunkShape, long[] selOffset, + int[] selShape) { + final int ndim = arrayShape.length; + long[] start = new long[ndim]; + long[] end = new long[ndim]; + int numChunks = 1; + for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { + final int staIdx = (int) (selOffset[dimIdx] / chunkShape[dimIdx]); + final int endIdx = (int) ((selOffset[dimIdx] + selShape[dimIdx] - 1) / chunkShape[dimIdx]); + numChunks *= (endIdx - staIdx + 1); + start[dimIdx] = staIdx; + end[dimIdx] = endIdx; + } + + final long[][] chunkCoords = new long[numChunks][]; + + final long[] currentIdx = Arrays.copyOf(start, ndim); + for (int i = 0; i < chunkCoords.length; i++) { + chunkCoords[i] = Arrays.copyOf(currentIdx, ndim); + int dimIdx = ndim - 1; + while (dimIdx >= 0) { + if (currentIdx[dimIdx] >= end[dimIdx]) { + currentIdx[dimIdx] = start[dimIdx]; + dimIdx--; + } else { + currentIdx[dimIdx]++; + dimIdx = -1; + } + } + } + return chunkCoords; } - return new ChunkProjection(chunkCoords, chunkOffset, outOffset, shape); - } + public static ChunkProjection computeProjection(long[] chunkCoords, int[] arrayShape, + int[] chunkShape) { + return computeProjection(chunkCoords, Utils.toLongArray(arrayShape), chunkShape); + } + public static ChunkProjection computeProjection(long[] chunkCoords, long[] arrayShape, + int[] chunkShape) { + return computeProjection(chunkCoords, arrayShape, chunkShape, new long[chunkCoords.length], + Utils.toIntArray(arrayShape) + ); + } - public static long cOrderIndex(final long[] chunkCoords, final long[] arrayShape) { - long index = 0; - long multiplier = 1; + public static ChunkProjection computeProjection( + final long[] chunkCoords, final long[] arrayShape, + final int[] chunkShape, final long[] selOffset, + final int[] selShape + ) { + final int ndim = chunkCoords.length; + final int[] chunkOffset = new int[ndim]; + final int[] outOffset = new int[ndim]; + final int[] shape = new int[ndim]; + + for (int dimIdx = 0; dimIdx < chunkCoords.length; dimIdx++) { + // compute offsets for chunk within overall array + final long dimOffset = (long) chunkShape[dimIdx] * chunkCoords[dimIdx]; + final long dimLimit = Math.min(arrayShape[dimIdx], + (chunkCoords[dimIdx] + 1) * (long) chunkShape[dimIdx]); + + if (selOffset[dimIdx] < dimOffset) { + // selection starts before current chunk + chunkOffset[dimIdx] = 0; + // compute number of previous items, provides offset into output array + outOffset[dimIdx] = (int) (dimOffset - selOffset[dimIdx]); + } else { + // selection starts within current chunk + chunkOffset[dimIdx] = (int) (selOffset[dimIdx] - dimOffset); + outOffset[dimIdx] = 0; + } + + if (selOffset[dimIdx] + selShape[dimIdx] > dimLimit) { + // selection ends after current chunk + shape[dimIdx] = (int) (chunkShape[dimIdx] - (selOffset[dimIdx] % chunkShape[dimIdx])); + } else { + // selection ends within current chunk + shape[dimIdx] = (int) (selOffset[dimIdx] + selShape[dimIdx] - dimOffset + - chunkOffset[dimIdx]); + } + } - for (int i = arrayShape.length - 1; i >= 0; i--) { - index += chunkCoords[i] * multiplier; - multiplier *= arrayShape[i]; + return new ChunkProjection(chunkCoords, chunkOffset, outOffset, shape); } - return index; - } - public static long fOrderIndex(final long[] chunkCoords, final long[] arrayShape) { - int index = 0; - int multiplier = 1; + public static long cOrderIndex(final long[] chunkCoords, final long[] arrayShape) { + long index = 0; + long multiplier = 1; + + for (int i = arrayShape.length - 1; i >= 0; i--) { + index += chunkCoords[i] * multiplier; + multiplier *= arrayShape[i]; + } - for (int i = 0; i < arrayShape.length; i++) { - index += chunkCoords[i] * multiplier; - multiplier *= arrayShape[i]; + return index; } - return index; - } + public static long fOrderIndex(final long[] chunkCoords, final long[] arrayShape) { + int index = 0; + int multiplier = 1; - public static boolean isFullChunk(final int[] selOffset, final int[] selShape, - final int[] chunkShape) { - if (selOffset.length != selShape.length) { - throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); - } - if (selOffset.length != chunkShape.length) { - throw new IllegalArgumentException( - "'selOffset' and 'chunkShape' need to have the same rank."); - } + for (int i = 0; i < arrayShape.length; i++) { + index += chunkCoords[i] * multiplier; + multiplier *= arrayShape[i]; + } - for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { - if (selOffset[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) { - return false; - } + return index; } - return true; - } - public static boolean isSingleFullChunk(final long[] selOffset, final int[] selShape, - final int[] chunkShape) { - if (selOffset.length != selShape.length) { - throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); - } - if (selOffset.length != chunkShape.length) { - throw new IllegalArgumentException( - "'selOffset' and 'chunkShape' need to have the same rank."); - } - for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { - if (selOffset[dimIdx] % chunkShape[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) { - return false; - } + public static boolean isFullChunk(final int[] selOffset, final int[] selShape, + final int[] chunkShape) { + if (selOffset.length != selShape.length) { + throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); + } + if (selOffset.length != chunkShape.length) { + throw new IllegalArgumentException( + "'selOffset' and 'chunkShape' need to have the same rank."); + } + + for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { + if (selOffset[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) { + return false; + } + } + return true; } - return true; - } - public static long[] computeSingleChunkCoords(final long[] selOffset, final int[] chunkShape) { - if (selOffset.length != chunkShape.length) { - throw new IllegalArgumentException( - "'selOffset' and 'chunkShape' need to have the same rank."); + public static boolean isSingleFullChunk(final long[] selOffset, final int[] selShape, + final int[] chunkShape) { + if (selOffset.length != selShape.length) { + throw new IllegalArgumentException("'selOffset' and 'selShape' need to have the same rank."); + } + if (selOffset.length != chunkShape.length) { + throw new IllegalArgumentException( + "'selOffset' and 'chunkShape' need to have the same rank."); + } + for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { + if (selOffset[dimIdx] % chunkShape[dimIdx] != 0 || selShape[dimIdx] != chunkShape[dimIdx]) { + return false; + } + } + return true; } - final long[] chunkCoords = new long[selOffset.length]; - for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { - chunkCoords[dimIdx] = (selOffset[dimIdx] / chunkShape[dimIdx]); + + public static long[] computeSingleChunkCoords(final long[] selOffset, final int[] chunkShape) { + if (selOffset.length != chunkShape.length) { + throw new IllegalArgumentException( + "'selOffset' and 'chunkShape' need to have the same rank."); + } + final long[] chunkCoords = new long[selOffset.length]; + for (int dimIdx = 0; dimIdx < selOffset.length; dimIdx++) { + chunkCoords[dimIdx] = (selOffset[dimIdx] / chunkShape[dimIdx]); + } + return chunkCoords; } - return chunkCoords; - } - public static final class ChunkProjection { + public static final class ChunkProjection { - final public long[] chunkCoords; - final public int[] chunkOffset; + final public long[] chunkCoords; + final public int[] chunkOffset; - final public int[] outOffset; - final public int[] shape; + final public int[] outOffset; + final public int[] shape; - public ChunkProjection( - final long[] chunkCoords, final int[] chunkOffset, final int[] outOffset, - final int[] shape - ) { - this.chunkCoords = chunkCoords; - this.chunkOffset = chunkOffset; - this.outOffset = outOffset; - this.shape = shape; + public ChunkProjection( + final long[] chunkCoords, final int[] chunkOffset, final int[] outOffset, + final int[] shape + ) { + this.chunkCoords = chunkCoords; + this.chunkOffset = chunkOffset; + this.outOffset = outOffset; + this.shape = shape; + } } - } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/utils/MultiArrayUtils.java b/src/main/java/dev/zarr/zarrjava/utils/MultiArrayUtils.java index 2d3aa6f..af8a302 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/MultiArrayUtils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/MultiArrayUtils.java @@ -1,295 +1,296 @@ package dev.zarr.zarrjava.utils; -import java.util.ArrayList; -import java.util.Arrays; -import javax.annotation.Nonnull; import ucar.ma2.Array; import ucar.ma2.IndexIterator; import ucar.ma2.InvalidRangeException; import ucar.ma2.Range; -public class MultiArrayUtils { - - public static void copyRegion(Array source, int[] sourceOffset, Array target, int[] targetOffset, - int[] shape) { - if (sourceOffset.length != targetOffset.length) { - throw new IllegalArgumentException( - "'sourceOffset' and 'targetOffset' do not have the same rank."); - } - if (source.getRank() != sourceOffset.length) { - throw new IllegalArgumentException("'sourceOffset' and 'source' do not have the same rank."); - } - if (target.getRank() != targetOffset.length) { - throw new IllegalArgumentException("'targetOffset' and 'target' do not have the same rank."); - } - if (shape.length != sourceOffset.length) { - throw new IllegalArgumentException("'shape' and 'sourceOffset' do not have the same rank."); - } - - try { - final ArrayList sourceRanges = new ArrayList<>(); - final ArrayList targetRanges = new ArrayList<>(); - for (int dimIdx = 0; dimIdx < shape.length; dimIdx++) { - if (sourceOffset[dimIdx] + shape[dimIdx] > source.getShape()[dimIdx]) { - throw new IllegalArgumentException( - "'sourceOffset + shape' needs to be less or equal than " + "'source.getShape()'."); - } - if (targetOffset[dimIdx] + shape[dimIdx] > target.getShape()[dimIdx]) { - throw new IllegalArgumentException( - "'targetOffset + shape' needs to be less or equal than " + "'target.getShape()'."); - } - - sourceRanges.add(new Range(sourceOffset[dimIdx], sourceOffset[dimIdx] + shape[dimIdx] - 1)); - targetRanges.add(new Range(targetOffset[dimIdx], targetOffset[dimIdx] + shape[dimIdx] - 1)); - } - final IndexIterator sourceRangeIterator = source.getRangeIterator(sourceRanges); - final IndexIterator targetRangeIterator = target.getRangeIterator(targetRanges); - final Class elementType = source.getElementType(); - final ValueAccessor accessor = createValueAccessor(elementType); - - while (sourceRangeIterator.hasNext()) { - accessor.copy(sourceRangeIterator, targetRangeIterator); - } - } catch (InvalidRangeException ex) { - throw new RuntimeException("Unreachable"); - } - } - - public static Array fill(@Nonnull Array array, @Nonnull Object fillValue) { - IndexIterator iterator = array.getIndexIterator(); - final Class elementType = array.getElementType(); - final ValueAccessor accessor = createValueAccessor(elementType); - while (iterator.hasNext()) { - accessor.set(iterator, fillValue); - } - return array; - } - - public static boolean allValuesEqual(Array array, Object value) { - IndexIterator iterator = array.getIndexIterator(); - final Class elementType = array.getElementType(); - final ValueAccessor accessor = createValueAccessor(elementType); - while (iterator.hasNext()) { - boolean isEqual = accessor.isEqual(iterator, value); - if (!isEqual) { - return false; - } - } - return true; - } - - public static boolean allValuesEqual(Array source, Array target) { - if (!Arrays.equals(source.getShape(), target.getShape())) { - return false; - } - if (!source.getElementType() - .equals(target.getElementType())) { - return false; - } - - IndexIterator sourceIterator = source.getIndexIterator(); - IndexIterator targetIterator = target.getIndexIterator(); - final Class elementType = source.getElementType(); - final ValueAccessor accessor = createValueAccessor(elementType); - while (sourceIterator.hasNext()) { - boolean isEqual = accessor.isEqual(sourceIterator, targetIterator); - if (!isEqual) { - return false; - } - } - return true; - } - - static ValueAccessor createValueAccessor(Class elementType) { - if (elementType == double.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setDoubleNext(sourceIterator.getDoubleNext()); - } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setDoubleNext((double) value); - } - - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getDoubleNext() == (double) value; - } - - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getDoubleNext() == sourceIterator.getDoubleNext(); - } - }; - } else if (elementType == float.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setFloatNext(sourceIterator.getFloatNext()); - } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setFloatNext((float) value); - } - - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getFloatNext() == (float) value; - } - - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getFloatNext() == sourceIterator.getFloatNext(); - } - }; - } else if (elementType == long.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setLongNext(sourceIterator.getLongNext()); - } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setLongNext((long) value); - } - - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getLongNext() == (long) value; - } - - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getLongNext() == sourceIterator.getLongNext(); - } - }; - } else if (elementType == int.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setIntNext(sourceIterator.getIntNext()); - } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setIntNext((int) value); - } +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getIntNext() == (int) value; - } +public class MultiArrayUtils { - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getIntNext() == sourceIterator.getIntNext(); + public static void copyRegion(Array source, int[] sourceOffset, Array target, int[] targetOffset, + int[] shape) { + if (sourceOffset.length != targetOffset.length) { + throw new IllegalArgumentException( + "'sourceOffset' and 'targetOffset' do not have the same rank."); } - }; - } else if (elementType == short.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setShortNext(sourceIterator.getShortNext()); + if (source.getRank() != sourceOffset.length) { + throw new IllegalArgumentException("'sourceOffset' and 'source' do not have the same rank."); } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setShortNext((short) value); + if (target.getRank() != targetOffset.length) { + throw new IllegalArgumentException("'targetOffset' and 'target' do not have the same rank."); } - - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getShortNext() == (short) value; + if (shape.length != sourceOffset.length) { + throw new IllegalArgumentException("'shape' and 'sourceOffset' do not have the same rank."); } - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getShortNext() == sourceIterator.getShortNext(); - } - }; - } else if (elementType == byte.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setByteNext(sourceIterator.getByteNext()); + try { + final ArrayList sourceRanges = new ArrayList<>(); + final ArrayList targetRanges = new ArrayList<>(); + for (int dimIdx = 0; dimIdx < shape.length; dimIdx++) { + if (sourceOffset[dimIdx] + shape[dimIdx] > source.getShape()[dimIdx]) { + throw new IllegalArgumentException( + "'sourceOffset + shape' needs to be less or equal than " + "'source.getShape()'."); + } + if (targetOffset[dimIdx] + shape[dimIdx] > target.getShape()[dimIdx]) { + throw new IllegalArgumentException( + "'targetOffset + shape' needs to be less or equal than " + "'target.getShape()'."); + } + + sourceRanges.add(new Range(sourceOffset[dimIdx], sourceOffset[dimIdx] + shape[dimIdx] - 1)); + targetRanges.add(new Range(targetOffset[dimIdx], targetOffset[dimIdx] + shape[dimIdx] - 1)); + } + final IndexIterator sourceRangeIterator = source.getRangeIterator(sourceRanges); + final IndexIterator targetRangeIterator = target.getRangeIterator(targetRanges); + final Class elementType = source.getElementType(); + final ValueAccessor accessor = createValueAccessor(elementType); + + while (sourceRangeIterator.hasNext()) { + accessor.copy(sourceRangeIterator, targetRangeIterator); + } + } catch (InvalidRangeException ex) { + throw new RuntimeException("Unreachable"); } + } - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setByteNext((byte) value); + public static Array fill(@Nonnull Array array, @Nonnull Object fillValue) { + IndexIterator iterator = array.getIndexIterator(); + final Class elementType = array.getElementType(); + final ValueAccessor accessor = createValueAccessor(elementType); + while (iterator.hasNext()) { + accessor.set(iterator, fillValue); } + return array; + } - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getByteNext() == (byte) value; + public static boolean allValuesEqual(Array array, Object value) { + IndexIterator iterator = array.getIndexIterator(); + final Class elementType = array.getElementType(); + final ValueAccessor accessor = createValueAccessor(elementType); + while (iterator.hasNext()) { + boolean isEqual = accessor.isEqual(iterator, value); + if (!isEqual) { + return false; + } } + return true; + } - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getByteNext() == sourceIterator.getByteNext(); + public static boolean allValuesEqual(Array source, Array target) { + if (!Arrays.equals(source.getShape(), target.getShape())) { + return false; } - }; - } else if (elementType == boolean.class) { - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setBooleanNext(sourceIterator.getBooleanNext()); + if (!source.getElementType() + .equals(target.getElementType())) { + return false; } - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setBooleanNext((boolean) value); - } - - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getBooleanNext() == (boolean) value; + IndexIterator sourceIterator = source.getIndexIterator(); + IndexIterator targetIterator = target.getIndexIterator(); + final Class elementType = source.getElementType(); + final ValueAccessor accessor = createValueAccessor(elementType); + while (sourceIterator.hasNext()) { + boolean isEqual = accessor.isEqual(sourceIterator, targetIterator); + if (!isEqual) { + return false; + } } + return true; + } - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getBooleanNext() == sourceIterator.getBooleanNext(); + static ValueAccessor createValueAccessor(Class elementType) { + if (elementType == double.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setDoubleNext(sourceIterator.getDoubleNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setDoubleNext((double) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getDoubleNext() == (double) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getDoubleNext() == sourceIterator.getDoubleNext(); + } + }; + } else if (elementType == float.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setFloatNext(sourceIterator.getFloatNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setFloatNext((float) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getFloatNext() == (float) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getFloatNext() == sourceIterator.getFloatNext(); + } + }; + } else if (elementType == long.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setLongNext(sourceIterator.getLongNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setLongNext((long) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getLongNext() == (long) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getLongNext() == sourceIterator.getLongNext(); + } + }; + } else if (elementType == int.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setIntNext(sourceIterator.getIntNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setIntNext((int) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getIntNext() == (int) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getIntNext() == sourceIterator.getIntNext(); + } + }; + } else if (elementType == short.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setShortNext(sourceIterator.getShortNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setShortNext((short) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getShortNext() == (short) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getShortNext() == sourceIterator.getShortNext(); + } + }; + } else if (elementType == byte.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setByteNext(sourceIterator.getByteNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setByteNext((byte) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getByteNext() == (byte) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getByteNext() == sourceIterator.getByteNext(); + } + }; + } else if (elementType == boolean.class) { + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setBooleanNext(sourceIterator.getBooleanNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setBooleanNext((boolean) value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getBooleanNext() == (boolean) value; + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getBooleanNext() == sourceIterator.getBooleanNext(); + } + }; } - }; + return new ValueAccessor() { + @Override + public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { + targetIterator.setObjectNext(sourceIterator.getObjectNext()); + } + + @Override + public void set(IndexIterator iterator, Object value) { + iterator.setObjectNext(value); + } + + @Override + public boolean isEqual(IndexIterator iterator, Object value) { + return iterator.getObjectNext() + .equals(value); + } + + @Override + public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { + return targetIterator.getObjectNext() + .equals(sourceIterator.getObjectNext()); + } + }; } - return new ValueAccessor() { - @Override - public void copy(IndexIterator sourceIterator, IndexIterator targetIterator) { - targetIterator.setObjectNext(sourceIterator.getObjectNext()); - } - - @Override - public void set(IndexIterator iterator, Object value) { - iterator.setObjectNext(value); - } - @Override - public boolean isEqual(IndexIterator iterator, Object value) { - return iterator.getObjectNext() - .equals(value); - } + private interface ValueAccessor { - @Override - public boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator) { - return targetIterator.getObjectNext() - .equals(sourceIterator.getObjectNext()); - } - }; - } + void copy(IndexIterator sourceIterator, IndexIterator targetIterator); - private interface ValueAccessor { + void set(IndexIterator iterator, Object value); - void copy(IndexIterator sourceIterator, IndexIterator targetIterator); + boolean isEqual(IndexIterator iterator, Object value); - void set(IndexIterator iterator, Object value); + boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator); - boolean isEqual(IndexIterator iterator, Object value); - - boolean isEqual(IndexIterator sourceIterator, IndexIterator targetIterator); - - } + } } diff --git a/src/main/java/dev/zarr/zarrjava/utils/Utils.java b/src/main/java/dev/zarr/zarrjava/utils/Utils.java index 7cc88d2..4a95d1e 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/Utils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/Utils.java @@ -14,97 +14,97 @@ public class Utils { - public static ByteBuffer allocateNative(int capacity) { - return ByteBuffer.allocate(capacity) - .order(ByteOrder.nativeOrder()); - } - - public static ByteBuffer makeByteBuffer(int capacity, Function func) { - ByteBuffer buf = ByteBuffer.allocate(capacity) - .order(ByteOrder.LITTLE_ENDIAN); - buf = func.apply(buf); - buf.rewind(); - return buf; - } - - public static ByteBuffer asByteBuffer(InputStream inputStream) throws IOException { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - - int nRead; - byte[] data = new byte[1024]; - - while ((nRead = inputStream.read(data, 0, data.length)) != -1) { - buffer.write(data, 0, nRead); + public static ByteBuffer allocateNative(int capacity) { + return ByteBuffer.allocate(capacity) + .order(ByteOrder.nativeOrder()); } - buffer.flush(); - return ByteBuffer.wrap(buffer.toByteArray()); - } - - public static long[] toLongArray(int[] array) { - return Arrays.stream(array) - .mapToLong(i -> (long) i) - .toArray(); - } - - public static int[] toIntArray(long[] array) { - return Arrays.stream(array) - .mapToInt(Math::toIntExact) - .toArray(); - } - - public static byte[] toArray(ByteBuffer buffer) { - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - return bytes; - } - - public static Stream asStream(Iterator sourceIterator) { - Iterable iterable = () -> sourceIterator; - return StreamSupport.stream(iterable.spliterator(), false); - } - - public static T[] concatArrays(T[] array1, T[]... arrays) { - if (arrays.length == 0) { - return array1; + public static ByteBuffer makeByteBuffer(int capacity, Function func) { + ByteBuffer buf = ByteBuffer.allocate(capacity) + .order(ByteOrder.LITTLE_ENDIAN); + buf = func.apply(buf); + buf.rewind(); + return buf; } - T[] result = Arrays.copyOf(array1, array1.length + Arrays.stream(arrays) - .mapToInt(a -> a.length) - .sum()); - int offset = array1.length; - for (T[] array2 : arrays) { - System.arraycopy(array2, 0, result, offset, array2.length); - offset += array2.length; + + public static ByteBuffer asByteBuffer(InputStream inputStream) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + int nRead; + byte[] data = new byte[1024]; + + while ((nRead = inputStream.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + + buffer.flush(); + return ByteBuffer.wrap(buffer.toByteArray()); + } + + public static long[] toLongArray(int[] array) { + return Arrays.stream(array) + .mapToLong(i -> (long) i) + .toArray(); + } + + public static int[] toIntArray(long[] array) { + return Arrays.stream(array) + .mapToInt(Math::toIntExact) + .toArray(); + } + + public static byte[] toArray(ByteBuffer buffer) { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + return bytes; } - return result; - } - public static void copyStream(InputStream inputStream, OutputStream outputStream) throws IOException { + public static Stream asStream(Iterator sourceIterator) { + Iterable iterable = () -> sourceIterator; + return StreamSupport.stream(iterable.spliterator(), false); + } + + public static T[] concatArrays(T[] array1, T[]... arrays) { + if (arrays.length == 0) { + return array1; + } + T[] result = Arrays.copyOf(array1, array1.length + Arrays.stream(arrays) + .mapToInt(a -> a.length) + .sum()); + int offset = array1.length; + for (T[] array2 : arrays) { + System.arraycopy(array2, 0, result, offset, array2.length); + offset += array2.length; + } + return result; + } + + public static void copyStream(InputStream inputStream, OutputStream outputStream) throws IOException { byte[] buffer = new byte[4096]; int len; while ((len = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, len); + outputStream.write(buffer, 0, len); } - } + } - public static boolean isPermutation(int[] array) { - if (array.length==0){ - return false; + public static boolean isPermutation(int[] array) { + if (array.length == 0) { + return false; + } + int[] arange = new int[array.length]; + Arrays.setAll(arange, i -> i); + int[] orderSorted = array.clone(); + Arrays.sort(orderSorted); + return Arrays.equals(orderSorted, arange); } - int[] arange = new int[array.length]; - Arrays.setAll(arange, i -> i); - int[] orderSorted = array.clone(); - Arrays.sort(orderSorted); - return Arrays.equals(orderSorted, arange); - } - - public static int[] inversePermutation(int[] origin){ - assert isPermutation(origin); - int[] inverse = new int[origin.length]; - for (int i = 0; i < origin.length; i++) { - inverse[origin[i]] = i; + + public static int[] inversePermutation(int[] origin) { + assert isPermutation(origin); + int[] inverse = new int[origin.length]; + for (int i = 0; i < origin.length; i++) { + inverse[origin[i]] = i; + } + return inverse; } - return inverse; - } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 421dbc6..87d767a 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -4,11 +4,11 @@ import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.core.BytesCodec; @@ -20,238 +20,241 @@ import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; + import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; import static dev.zarr.zarrjava.v2.Node.makeObjectWriter; public class Array extends dev.zarr.zarrjava.core.Array implements Node { - private final ArrayMetadata metadata; - - public ArrayMetadata metadata() { - return metadata; - } + private final ArrayMetadata metadata; - protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { - super(storeHandle); - this.metadata = arrayMetadata; - this.codecPipeline = new CodecPipeline(Utils.concatArrays( - new Codec[]{}, - metadata.filters == null ? new Codec[]{} : metadata.filters, - new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, - metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} - ), metadata.coreArrayMetadata); - } + protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { + super(storeHandle); + this.metadata = arrayMetadata; + this.codecPipeline = new CodecPipeline(Utils.concatArrays( + new Codec[]{}, + metadata.filters == null ? new Codec[]{} : metadata.filters, + new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, + metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} + ), metadata.coreArrayMetadata); + } - /** - * Opens an existing Zarr array at a specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @throws IOException throws IOException if the metadata cannot be read - * @throws ZarrException throws ZarrException if the Zarr array cannot be opened - */ - public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { - ObjectMapper mapper = makeObjectMapper(); - ArrayMetadata metadata = mapper.readValue( - Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), - ArrayMetadata.class - ); - if (storeHandle.resolve(ZATTRS).exists()) - metadata.attributes = mapper.readValue( - Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), - Attributes.class - ); - return new Array( - storeHandle, - metadata - ); - } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper mapper = makeObjectMapper(); + ArrayMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), + ArrayMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + Attributes.class + ); + return new Array( + storeHandle, + metadata + ); + } - /** - * Opens an existing Zarr array at a specified storage location. - * - * @param path the storage location of the Zarr array - * @throws IOException throws IOException if the metadata cannot be read - * @throws ZarrException throws ZarrException if the Zarr array cannot be opened - */ - public static Array open(Path path) throws IOException, ZarrException { - return open(new StoreHandle(new FilesystemStore(path))); - } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } /** * Opens an existing Zarr array at a specified storage location. * * @param path the storage location of the Zarr array - * @throws IOException throws IOException if the metadata cannot be read + * @throws IOException throws IOException if the metadata cannot be read * @throws ZarrException throws ZarrException if the Zarr array cannot be opened */ public static Array open(String path) throws IOException, ZarrException { - return open(Paths.get(path)); + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr array with the provided metadata in an in-memory store + * + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(ArrayMetadata arrayMetadata) throws ZarrException, IOException { + return create(new StoreHandle(new MemoryStore()), arrayMetadata); } - /** - * Creates a new Zarr array with the provided metadata in an in-memory store - * - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(ArrayMetadata arrayMetadata) throws ZarrException, IOException { - return create(new StoreHandle(new MemoryStore()), arrayMetadata); - } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param path the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(Path path, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return create(new StoreHandle(new FilesystemStore(path)), arrayMetadata); - } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), arrayMetadata); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param path the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(String path, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return create(Paths.get(path), arrayMetadata); - } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param storeHandle the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(storeHandle, arrayMetadata, false); - } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(Paths.get(path), arrayMetadata); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. If - * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the - * specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @param existsOk if true, no exception is raised if the Zarr array already exists - * @throws IOException throws IOException if the metadata cannot be serialized - * @throws ZarrException throws ZarrException if the Zarr array cannot be created - */ - public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) - throws IOException, ZarrException { - StoreHandle metadataHandle = storeHandle.resolve(ZARRAY); - if (!existsOk && metadataHandle.exists()) { - throw new RuntimeException( - "Trying to create a new array in " + storeHandle + ". But " + metadataHandle - + " already exists."); + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle, arrayMetadata, false); } - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); - if (arrayMetadata.attributes != null) { - StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); - ByteBuffer attrsBytes = ByteBuffer.wrap( - objectWriter.writeValueAsBytes(arrayMetadata.attributes)); - attrsHandle.set(attrsBytes); + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. If + * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the + * specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException throws IOException if the metadata cannot be serialized + * @throws ZarrException throws ZarrException if the Zarr array cannot be created + */ + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) + throws IOException, ZarrException { + StoreHandle metadataHandle = storeHandle.resolve(ZARRAY); + if (!existsOk && metadataHandle.exists()) { + throw new RuntimeException( + "Trying to create a new array in " + storeHandle + ". But " + metadataHandle + + " already exists."); + } + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); + if (arrayMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectWriter.writeValueAsBytes(arrayMetadata.attributes)); + attrsHandle.set(attrsBytes); + } + metadataHandle.set(metadataBytes); + return new Array(storeHandle, arrayMetadata); } - metadataHandle.set(metadataBytes); - return new Array(storeHandle, arrayMetadata); - } - public static Array create(StoreHandle storeHandle, - Function arrayMetadataBuilderMapper, - boolean existsOk) throws IOException, ZarrException { - return create(storeHandle, - arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); - } + public static Array create(StoreHandle storeHandle, + Function arrayMetadataBuilderMapper, + boolean existsOk) throws IOException, ZarrException { + return create(storeHandle, + arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); + } - @Nonnull - public static ArrayMetadataBuilder metadataBuilder() { - return new ArrayMetadataBuilder(); - } + @Nonnull + public static ArrayMetadataBuilder metadataBuilder() { + return new ArrayMetadataBuilder(); + } - @Nonnull - public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadata) { - return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); - } + @Nonnull + public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadata) { + return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); + } - private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - return Array.create(storeHandle, newArrayMetadata, true); - } + public ArrayMetadata metadata() { + return metadata; + } - /** - * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or - * deleted. This method returns a new instance of the Zarr array class and the old instance - * becomes invalid. - * - * @param newShape the new shape of the Zarr array - * @throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array resize(long[] newShape) throws ZarrException, IOException { - if (newShape.length != metadata.ndim()) { - throw new IllegalArgumentException( - "'newShape' needs to have rank '" + metadata.ndim() + "'."); + private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { + return Array.create(storeHandle, newArrayMetadata, true); } - ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) - .withShape(newShape) - .build(); - return writeMetadata(newArrayMetadata); - } + /** + * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or + * deleted. This method returns a new instance of the Zarr array class and the old instance + * becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape) throws ZarrException, IOException { + if (newShape.length != metadata.ndim()) { + throw new IllegalArgumentException( + "'newShape' needs to have rank '" + metadata.ndim() + "'."); + } - /** - * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This - * method returns a new instance of the Zarr array class and the old instance becomes invalid. - * - * @param newAttributes the new attributes of the Zarr array - * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { - ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata, false) - .withAttributes(newAttributes) - .build(); - return writeMetadata(newArrayMetadata); - } + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withShape(newShape) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { + ArrayMetadata newArrayMetadata = + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) + .withAttributes(newAttributes) + .build(); + return writeMetadata(newArrayMetadata); + } - /** - * Updates the attributes of the Zarr array. It provides a callback that gets the current - * attributes as input and needs to return the new set of attributes. The attributes in the - * callback may be mutated. This method overwrites and removes any existing attributes. This - * method returns a new instance of the Zarr array class and the old instance becomes invalid. - * - * @param attributeMapper the callback that is used to construct the new attributes - * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); - } + /** + * Updates the attributes of the Zarr array. It provides a callback that gets the current + * attributes as input and needs to return the new set of attributes. The attributes in the + * callback may be mutated. This method overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param attributeMapper the callback that is used to construct the new attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } - @Override - public String toString() { - return String.format("", storeHandle, - Arrays.stream(metadata.shape) - .mapToObj(Long::toString) - .collect(Collectors.joining(", ")), - metadata.dataType - ); - } + @Override + public String toString() { + return String.format("", storeHandle, + Arrays.stream(metadata.shape) + .mapToObj(Long::toString) + .collect(Collectors.joining(", ")), + metadata.dataType + ); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index b53eee3..a16d59e 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -6,8 +6,8 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; -import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v2.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; @@ -17,129 +17,128 @@ public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { - static final int ZARR_FORMAT = 2; - - @JsonProperty("zarr_format") - public final int zarrFormat = ZARR_FORMAT; - - public final int[] chunks; - - @JsonProperty("dtype") - public final DataType dataType; - - @JsonIgnore - public final Endianness endianness; - - @JsonProperty("order") - public final Order order; - - @JsonProperty("dimension_separator") - public final Separator dimensionSeparator; - - @Nullable - public final Codec[] filters; - @Nullable - public final Codec compressor; - - @Nullable - @JsonIgnore - public Attributes attributes; - - @JsonIgnore - public CoreArrayMetadata coreArrayMetadata; - - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ArrayMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat, - @JsonProperty(value = "shape", required = true) long[] shape, - @JsonProperty(value = "chunks", required = true) int[] chunks, - @JsonProperty(value = "dtype", required = true) DataType dataType, - @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, - @JsonProperty(value = "order", required = true) Order order, - @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, - @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, - @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator - ) throws ZarrException { - this(zarrFormat, shape, chunks, dataType, fillValue, order, filters, compressor, dimensionSeparator, null); - } - - - public ArrayMetadata( - int zarrFormat, - long[] shape, - int[] chunks, - DataType dataType, - @Nullable Object fillValue, - Order order, - @Nullable Codec[] filters, - @Nullable Codec compressor, - @Nullable Separator dimensionSeparator, - @Nullable Attributes attributes - ) throws ZarrException { - super(shape, fillValue, dataType); - if (zarrFormat != this.zarrFormat) { - throw new ZarrException( - "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + static final int ZARR_FORMAT = 2; + + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + + public final int[] chunks; + + @JsonProperty("dtype") + public final DataType dataType; + + @JsonIgnore + public final Endianness endianness; + + @JsonProperty("order") + public final Order order; + + @JsonProperty("dimension_separator") + public final Separator dimensionSeparator; + + @Nullable + public final Codec[] filters; + @Nullable + public final Codec compressor; + + @Nullable + @JsonIgnore + public Attributes attributes; + + @JsonIgnore + public CoreArrayMetadata coreArrayMetadata; + + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ArrayMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "shape", required = true) long[] shape, + @JsonProperty(value = "chunks", required = true) int[] chunks, + @JsonProperty(value = "dtype", required = true) DataType dataType, + @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, + @JsonProperty(value = "order", required = true) Order order, + @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, + @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, + @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator + ) throws ZarrException { + this(zarrFormat, shape, chunks, dataType, fillValue, order, filters, compressor, dimensionSeparator, null); } - this.chunks = chunks; - this.dataType = dataType; - this.endianness = dataType.getEndianness(); - this.order = order; - this.dimensionSeparator = dimensionSeparator; - this.coreArrayMetadata = - new CoreArrayMetadata(shape, chunks, - this.dataType, - this.parsedFillValue - ); - if (filters == null) this.filters = null; - else{ - this.filters = new Codec[filters.length]; - for(int i = 0; i < filters.length; i++) { - this.filters[i] = filters[i].evolveFromCoreArrayMetadata(this.coreArrayMetadata); - } + + + public ArrayMetadata( + int zarrFormat, + long[] shape, + int[] chunks, + DataType dataType, + @Nullable Object fillValue, + Order order, + @Nullable Codec[] filters, + @Nullable Codec compressor, + @Nullable Separator dimensionSeparator, + @Nullable Attributes attributes + ) throws ZarrException { + super(shape, fillValue, dataType); + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + this.chunks = chunks; + this.dataType = dataType; + this.endianness = dataType.getEndianness(); + this.order = order; + this.dimensionSeparator = dimensionSeparator; + this.coreArrayMetadata = + new CoreArrayMetadata(shape, chunks, + this.dataType, + this.parsedFillValue + ); + if (filters == null) this.filters = null; + else { + this.filters = new Codec[filters.length]; + for (int i = 0; i < filters.length; i++) { + this.filters[i] = filters[i].evolveFromCoreArrayMetadata(this.coreArrayMetadata); + } + } + this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); + this.attributes = attributes; + } + + + @Override + public int[] chunkShape() { + return chunks; } - this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); - this.attributes = attributes; - } - - - - @Override - public int[] chunkShape() { - return chunks; - } - - @Override - public DataType dataType() { - return dataType; - } - - @Override - public Array allocateFillValueChunk() { - Array outputArray = Array.factory(dataType.getMA2DataType(), chunks); - if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue); - return outputArray; - } - - @Override - public ChunkKeyEncoding chunkKeyEncoding() { - Separator separator = dimensionSeparator == null ? Separator.DOT : dimensionSeparator; - return new V2ChunkKeyEncoding(separator); - } - - @Override - public Object parsedFillValue() { - return parsedFillValue; - } - - @Override - public @Nonnull Attributes attributes() throws ZarrException { - if (attributes == null) { - throw new ZarrException("Array attributes have not been set."); + + @Override + public DataType dataType() { + return dataType; + } + + @Override + public Array allocateFillValueChunk() { + Array outputArray = Array.factory(dataType.getMA2DataType(), chunks); + if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue); + return outputArray; + } + + @Override + public ChunkKeyEncoding chunkKeyEncoding() { + Separator separator = dimensionSeparator == null ? Separator.DOT : dimensionSeparator; + return new V2ChunkKeyEncoding(separator); + } + + @Override + public Object parsedFillValue() { + return parsedFillValue; + } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; } - return attributes; - } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 7f27567..8ec6707 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -9,159 +9,160 @@ import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; public class ArrayMetadataBuilder { - long[] shape = null; - int[] chunks = null; - DataType dataType = null; - Order order = Order.C; - Separator dimensionSeparator = Separator.DOT; - Object fillValue = null; - Codec[] filters = null; - Codec compressor = null; - Attributes attributes = new Attributes(); - - - protected ArrayMetadataBuilder() { - } - - protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { - return fromArrayMetadata(arrayMetadata, true); - } - - protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { - ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); - builder.shape = arrayMetadata.shape; - builder.chunks = arrayMetadata.chunks; - builder.dataType = arrayMetadata.dataType; - builder.order = arrayMetadata.order; - builder.dimensionSeparator = arrayMetadata.dimensionSeparator; - builder.fillValue = arrayMetadata.parsedFillValue; - builder.filters = arrayMetadata.filters; - builder.compressor = arrayMetadata.compressor; - if (withAttributes) { - builder.attributes = arrayMetadata.attributes; - } - return builder; - } - - public ArrayMetadataBuilder withShape(long... shape) { - this.shape = shape; - return this; - } - - public ArrayMetadataBuilder withChunks(int... chunks) { - this.chunks = chunks; - return this; - } - - public ArrayMetadataBuilder withDataType(DataType dataTypeV2) { - this.dataType = dataTypeV2; - return this; - } - - public ArrayMetadataBuilder withOrder(Order order) { - this.order = order; - return this; - } - - public ArrayMetadataBuilder withDimensionSeparator(Separator dimensionSeparator) { - this.dimensionSeparator = dimensionSeparator; - return this; - } - - public ArrayMetadataBuilder withFillValue(Object fillValue) { - this.fillValue = fillValue; - return this; - } - - public ArrayMetadataBuilder withCompressor(Codec compressor) { - this.compressor = compressor; - return this; - } - - public ArrayMetadataBuilder withBloscCompressor( - Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, - int blockSize - ) { - try { - this.compressor = new BloscCodec(cname, shuffle, clevel, typeSize, blockSize); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel, int blockSize) { - if (shuffle.equals("shuffle")) { - shuffle = "byteshuffle"; - } - return withBloscCompressor(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, - dataType.getByteCount(), blockSize - ); - } - - public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { - return withBloscCompressor(cname, shuffle, clevel, 0); - } - - public ArrayMetadataBuilder withBloscCompressor(String cname, int clevel) { - return withBloscCompressor(cname, "noshuffle", clevel); - } - - public ArrayMetadataBuilder withBloscCompressor(String cname) { - return withBloscCompressor(cname, 5); - } - - public ArrayMetadataBuilder withBloscCompressor() { - return withBloscCompressor("zstd"); - } - - public ArrayMetadataBuilder withZlibCompressor(int level) { - try { - this.compressor = new ZlibCodec(level); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public ArrayMetadataBuilder withZlibCompressor() { - return withZlibCompressor(5); - } - - public ArrayMetadataBuilder putAttribute(String key, Object value) { - this.attributes.put(key, value); - return this; - } - - public ArrayMetadataBuilder withAttributes(Attributes attributes) { - if (this.attributes == null) { - this.attributes = attributes; - } else { - this.attributes.putAll(attributes); - } - return this; - } - public ArrayMetadata build() throws ZarrException { - if (shape == null) { - throw new IllegalStateException("Please call `withShape` first."); - } - if (chunks == null) { - throw new IllegalStateException("Please call `withChunks` first."); - } - if (dataType == null) { - throw new IllegalStateException("Please call `withDataType` first."); - } - return new ArrayMetadata( - 2, - shape, - chunks, - dataType, - fillValue, - order, - filters, - compressor, - dimensionSeparator, - attributes - ); - } + long[] shape = null; + int[] chunks = null; + DataType dataType = null; + Order order = Order.C; + Separator dimensionSeparator = Separator.DOT; + Object fillValue = null; + Codec[] filters = null; + Codec compressor = null; + Attributes attributes = new Attributes(); + + + protected ArrayMetadataBuilder() { + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { + ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); + builder.shape = arrayMetadata.shape; + builder.chunks = arrayMetadata.chunks; + builder.dataType = arrayMetadata.dataType; + builder.order = arrayMetadata.order; + builder.dimensionSeparator = arrayMetadata.dimensionSeparator; + builder.fillValue = arrayMetadata.parsedFillValue; + builder.filters = arrayMetadata.filters; + builder.compressor = arrayMetadata.compressor; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } + return builder; + } + + public ArrayMetadataBuilder withShape(long... shape) { + this.shape = shape; + return this; + } + + public ArrayMetadataBuilder withChunks(int... chunks) { + this.chunks = chunks; + return this; + } + + public ArrayMetadataBuilder withDataType(DataType dataTypeV2) { + this.dataType = dataTypeV2; + return this; + } + + public ArrayMetadataBuilder withOrder(Order order) { + this.order = order; + return this; + } + + public ArrayMetadataBuilder withDimensionSeparator(Separator dimensionSeparator) { + this.dimensionSeparator = dimensionSeparator; + return this; + } + + public ArrayMetadataBuilder withFillValue(Object fillValue) { + this.fillValue = fillValue; + return this; + } + + public ArrayMetadataBuilder withCompressor(Codec compressor) { + this.compressor = compressor; + return this; + } + + public ArrayMetadataBuilder withBloscCompressor( + Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, + int blockSize + ) { + try { + this.compressor = new BloscCodec(cname, shuffle, clevel, typeSize, blockSize); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel, int blockSize) { + if (shuffle.equals("shuffle")) { + shuffle = "byteshuffle"; + } + return withBloscCompressor(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, + dataType.getByteCount(), blockSize + ); + } + + public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { + return withBloscCompressor(cname, shuffle, clevel, 0); + } + + public ArrayMetadataBuilder withBloscCompressor(String cname, int clevel) { + return withBloscCompressor(cname, "noshuffle", clevel); + } + + public ArrayMetadataBuilder withBloscCompressor(String cname) { + return withBloscCompressor(cname, 5); + } + + public ArrayMetadataBuilder withBloscCompressor() { + return withBloscCompressor("zstd"); + } + + public ArrayMetadataBuilder withZlibCompressor(int level) { + try { + this.compressor = new ZlibCodec(level); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public ArrayMetadataBuilder withZlibCompressor() { + return withZlibCompressor(5); + } + + public ArrayMetadataBuilder putAttribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } + return this; + } + + public ArrayMetadata build() throws ZarrException { + if (shape == null) { + throw new IllegalStateException("Please call `withShape` first."); + } + if (chunks == null) { + throw new IllegalStateException("Please call `withChunks` first."); + } + if (dataType == null) { + throw new IllegalStateException("Please call `withDataType` first."); + } + return new ArrayMetadata( + 2, + shape, + chunks, + dataType, + fillValue, + order, + filters, + compressor, + dimensionSeparator, + attributes + ); + } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java index 4583487..df06a7b 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -3,69 +3,69 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum DataType implements dev.zarr.zarrjava.core.DataType { - BOOL("b1", Endianness.UNSPECIFIED), - INT8("i1", Endianness.UNSPECIFIED), - INT16("i2", Endianness.LITTLE), - INT32("i4", Endianness.LITTLE), - INT64("i8", Endianness.LITTLE), - UINT8("u1", Endianness.UNSPECIFIED), - UINT16("u2", Endianness.LITTLE), - UINT32("u4", Endianness.LITTLE), - UINT64("u8", Endianness.LITTLE), - FLOAT32("f4", Endianness.LITTLE), - FLOAT64("f8", Endianness.LITTLE); + BOOL("b1", Endianness.UNSPECIFIED), + INT8("i1", Endianness.UNSPECIFIED), + INT16("i2", Endianness.LITTLE), + INT32("i4", Endianness.LITTLE), + INT64("i8", Endianness.LITTLE), + UINT8("u1", Endianness.UNSPECIFIED), + UINT16("u2", Endianness.LITTLE), + UINT32("u4", Endianness.LITTLE), + UINT64("u8", Endianness.LITTLE), + FLOAT32("f4", Endianness.LITTLE), + FLOAT64("f8", Endianness.LITTLE); - private final String dtype; - private final Endianness endianness; + private final String dtype; + private final Endianness endianness; - DataType(String dtype, Endianness endianness) { - this.dtype = dtype; - this.endianness = endianness; - } + DataType(String dtype, Endianness endianness) { + this.dtype = dtype; + this.endianness = endianness; + } - public Endianness getEndianness() { - return endianness; - } + public Endianness getEndianness() { + return endianness; + } - @JsonValue - public String getValue() { - return String.format("%s%s", endianness.getValue(), dtype); - } + @JsonValue + public String getValue() { + return String.format("%s%s", endianness.getValue(), dtype); + } - @Override + @Override public ucar.ma2.DataType getMA2DataType() { - switch (this) { - case BOOL: - return ucar.ma2.DataType.BOOLEAN; - case INT8: - return ucar.ma2.DataType.BYTE; - case INT16: - return ucar.ma2.DataType.SHORT; - case INT32: - return ucar.ma2.DataType.INT; - case INT64: - return ucar.ma2.DataType.LONG; - case UINT8: - return ucar.ma2.DataType.UBYTE; - case UINT16: - return ucar.ma2.DataType.USHORT; - case UINT32: - return ucar.ma2.DataType.UINT; - case UINT64: - return ucar.ma2.DataType.ULONG; - case FLOAT32: - return ucar.ma2.DataType.FLOAT; - case FLOAT64: - return ucar.ma2.DataType.DOUBLE; - default: - throw new RuntimeException("Unreachable"); + switch (this) { + case BOOL: + return ucar.ma2.DataType.BOOLEAN; + case INT8: + return ucar.ma2.DataType.BYTE; + case INT16: + return ucar.ma2.DataType.SHORT; + case INT32: + return ucar.ma2.DataType.INT; + case INT64: + return ucar.ma2.DataType.LONG; + case UINT8: + return ucar.ma2.DataType.UBYTE; + case UINT16: + return ucar.ma2.DataType.USHORT; + case UINT32: + return ucar.ma2.DataType.UINT; + case UINT64: + return ucar.ma2.DataType.ULONG; + case FLOAT32: + return ucar.ma2.DataType.FLOAT; + case FLOAT64: + return ucar.ma2.DataType.DOUBLE; + default: + throw new RuntimeException("Unreachable"); + } } - } - @Override - public int getByteCount() { - return Integer.parseInt(dtype.substring(1)); - } + @Override + public int getByteCount() { + return Integer.parseInt(dtype.substring(1)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java index 1f064cb..f5bf206 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java @@ -4,30 +4,30 @@ import dev.zarr.zarrjava.v2.codec.core.BytesCodec; public enum Endianness { - LITTLE("<"), - BIG(">"), - UNSPECIFIED("|"); + LITTLE("<"), + BIG(">"), + UNSPECIFIED("|"); - private final String value; + private final String value; - Endianness(String value) { - this.value = value; - } + Endianness(String value) { + this.value = value; + } - @JsonValue - public String getValue() { - return value; - } + @JsonValue + public String getValue() { + return value; + } - public BytesCodec.Endian toEndian() { - switch (this) { - case LITTLE: - return BytesCodec.Endian.LITTLE; - case BIG: - return BytesCodec.Endian.BIG; - case UNSPECIFIED: - default: - return BytesCodec.Endian.LITTLE; + public BytesCodec.Endian toEndian() { + switch (this) { + case LITTLE: + return BytesCodec.Endian.LITTLE; + case BIG: + return BytesCodec.Endian.BIG; + case UNSPECIFIED: + default: + return BytesCodec.Endian.LITTLE; + } } - } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index c568294..29bd477 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -17,259 +17,260 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.function.Function; + import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; import static dev.zarr.zarrjava.v2.Node.makeObjectWriter; -public class Group extends dev.zarr.zarrjava.core.Group implements Node{ - public GroupMetadata metadata; +public class Group extends dev.zarr.zarrjava.core.Group implements Node { + public GroupMetadata metadata; - protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { - super(storeHandle); - this.metadata = groupMetadata; - } + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + super(storeHandle); + this.metadata = groupMetadata; + } - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { - ObjectMapper mapper = makeObjectMapper(); - GroupMetadata metadata = mapper.readValue( - Utils.toArray(storeHandle.resolve(ZGROUP).readNonNull()), - GroupMetadata.class - ); - if (storeHandle.resolve(ZATTRS).exists()) - metadata.attributes = mapper.readValue( - Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), - dev.zarr.zarrjava.core.Attributes.class - ); - return new Group(storeHandle, metadata); - } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { + ObjectMapper mapper = makeObjectMapper(); + GroupMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZGROUP).readNonNull()), + GroupMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + dev.zarr.zarrjava.core.Attributes.class + ); + return new Group(storeHandle, metadata); + } - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(Path path) throws IOException { - return open(new StoreHandle(new FilesystemStore(path))); + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); } - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(String path) throws IOException { - return open(Paths.get(path)); + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(String path) throws IOException { + return open(Paths.get(path)); } - /** - * Creates a new Zarr group with the provided metadata in an in-memory store. - * - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { - return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); - } + /** + * Creates a new Zarr group with the provided metadata in an in-memory store. + * + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { + return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); + } - /** - * Creates a new Zarr group with the provided metadata at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public static Group create( - @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata - ) throws IOException { - return new Group(storeHandle, groupMetadata).writeMetadata(); - } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create( + @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata + ) throws IOException { + return new Group(storeHandle, groupMetadata).writeMetadata(); + } - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { - return create(storeHandle, new GroupMetadata()); - } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata()); + } - /** - * Creates a new Zarr group with the provided attributes at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @param attributes the attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the attributes are invalid - */ - public static Group create(@Nonnull StoreHandle storeHandle, Attributes attributes) throws IOException, ZarrException { - return create(storeHandle, new GroupMetadata(attributes)); - } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create(@Nonnull StoreHandle storeHandle, Attributes attributes) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata(attributes)); + } - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(Path path) throws IOException, ZarrException { - return create(new StoreHandle(new FilesystemStore(path))); - } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(Path path) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path))); + } - /** - * Creates a new Zarr group with the provided attributes at a specified storage location. - * - * @param path the storage location of the Zarr group - * @param attributes the attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the attributes are invalid - */ - public static Group create(Path path, Attributes attributes) throws IOException, ZarrException { - return create(new StoreHandle(new FilesystemStore(path)), attributes); - } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create(Path path, Attributes attributes) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), attributes); + } - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(String path) throws IOException, ZarrException { - return create(Paths.get(path)); - } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); + } - /** - * Creates a new Zarr group with the provided attributes at a specified storage location. - * - * @param path the storage location of the Zarr group - * @param attributes the attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the attributes are invalid - */ - public static Group create(String path, Attributes attributes) throws IOException, ZarrException { - return create(Paths.get(path), attributes); - } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create(String path, Attributes attributes) throws IOException, ZarrException { + return create(Paths.get(path), attributes); + } - /** - * Retrieves a node (group or array) at the specified key within the current group. - * - * @param key the key of the node to retrieve - * @return the node at the specified key, or null if it does not exist - * @throws ZarrException if the node cannot be opened - * @throws IOException if there is an error accessing the storage - */ - @Nullable - public Node get(String key) throws ZarrException, IOException { - StoreHandle keyHandle = storeHandle.resolve(key); - try { - return Node.open(keyHandle); - } catch (NoSuchFileException e) { - return null; + /** + * Retrieves a node (group or array) at the specified key within the current group. + * + * @param key the key of the node to retrieve + * @return the node at the specified key, or null if it does not exist + * @throws ZarrException if the node cannot be opened + * @throws IOException if there is an error accessing the storage + */ + @Nullable + public Node get(String key) throws ZarrException, IOException { + StoreHandle keyHandle = storeHandle.resolve(key); + try { + return Node.open(keyHandle); + } catch (NoSuchFileException e) { + return null; + } } - } - /** - * Creates a new subgroup with default metadata at the specified key. - * - * @param key the key of the new Zarr group within the current group - * @return the created subgroup - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the group cannot be created - */ - public Group createGroup(String key) throws IOException, ZarrException { - return Group.create(storeHandle.resolve(key)); - } + /** + * Creates a new subgroup with default metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @return the created subgroup + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the group cannot be created + */ + public Group createGroup(String key) throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key)); + } - /** - * Creates a new array with the provided metadata at the specified key. - * - * @param key the key of the new Zarr array within the current group - * @param arrayMetadata the metadata of the Zarr array - * @return the created array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the array cannot be created - */ - public Array createArray(String key, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(storeHandle.resolve(key), arrayMetadata); - } + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadata the metadata of the Zarr array + * @return the created array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created + */ + public Array createArray(String key, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadata); + } - /** - * Creates a new array with the provided metadata at the specified key. - * - * @param key the key of the new Zarr array within the current group - * @param arrayMetadataBuilderMapper a function that modifies the array metadata - * @return the created array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the array cannot be created - */ - public Array createArray(String key, Function arrayMetadataBuilderMapper) - throws IOException, ZarrException { - return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); - } + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadataBuilderMapper a function that modifies the array metadata + * @return the created array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created + */ + public Array createArray(String key, Function arrayMetadataBuilderMapper) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); + } - private Group writeMetadata() throws IOException { - return writeMetadata(this.metadata); - } + private Group writeMetadata() throws IOException { + return writeMetadata(this.metadata); + } - private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); - storeHandle.resolve(ZGROUP).set(metadataBytes); - if (newGroupMetadata.attributes != null) { - StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); - ByteBuffer attrsBytes = ByteBuffer.wrap( - objectWriter.writeValueAsBytes(newGroupMetadata.attributes)); - attrsHandle.set(attrsBytes); + private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); + storeHandle.resolve(ZGROUP).set(metadataBytes); + if (newGroupMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectWriter.writeValueAsBytes(newGroupMetadata.attributes)); + attrsHandle.set(attrsBytes); + } + this.metadata = newGroupMetadata; + return this; } - this.metadata = newGroupMetadata; - return this; - } - /** - * Sets new attributes for the group, replacing any existing attributes. - * - * @param newAttributes the new attributes to set - * @return the updated group - * @throws ZarrException if the new attributes are invalid - * @throws IOException if the metadata cannot be serialized - */ - public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { - GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); - return writeMetadata(newGroupMetadata); - } + /** + * Sets new attributes for the group, replacing any existing attributes. + * + * @param newAttributes the new attributes to set + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { + GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); + return writeMetadata(newGroupMetadata); + } - /** - * Updates the attributes of the group using a mapper function. - * - * @param attributeMapper a function that takes the current attributes and returns the updated attributes - * @return the updated group - * @throws ZarrException if the new attributes are invalid - * @throws IOException if the metadata cannot be serialized - */ - public Group updateAttributes(Function attributeMapper) - throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); - } + /** + * Updates the attributes of the group using a mapper function. + * + * @param attributeMapper a function that takes the current attributes and returns the updated attributes + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ + public Group updateAttributes(Function attributeMapper) + throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } - @Override - public String toString() { - return String.format("", storeHandle); - } + @Override + public String toString() { + return String.format("", storeHandle); + } - @Override - public GroupMetadata metadata() { - return metadata; - } + @Override + public GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java index e0e6c00..98fd74d 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -1,49 +1,49 @@ package dev.zarr.zarrjava.v2; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { - static final int ZARR_FORMAT = 2; - @JsonProperty("zarr_format") - public final int zarrFormat = ZARR_FORMAT; - - @Nullable - @JsonIgnore - public Attributes attributes; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public GroupMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat, - @JsonProperty(value = "attributes", required = false) @Nullable Attributes attributes - ) throws ZarrException { - if (zarrFormat != this.zarrFormat) { - throw new ZarrException( - "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + static final int ZARR_FORMAT = 2; + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + + @Nullable + @JsonIgnore + public Attributes attributes; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public GroupMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "attributes", required = false) @Nullable Attributes attributes + ) throws ZarrException { + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + this.attributes = attributes; } - this.attributes = attributes; - } - public GroupMetadata() throws ZarrException { - this(ZARR_FORMAT, null); - } + public GroupMetadata() throws ZarrException { + this(ZARR_FORMAT, null); + } - public GroupMetadata(Attributes attributes) throws ZarrException { - this(ZARR_FORMAT, attributes); - } + public GroupMetadata(Attributes attributes) throws ZarrException { + this(ZARR_FORMAT, attributes); + } - @Override - public @Nonnull Attributes attributes() throws ZarrException { - if (attributes == null) { - throw new ZarrException("Group attributes have not been set."); + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } + return attributes; } - return attributes; - } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java index 09b972a..438e0d0 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -1,6 +1,5 @@ package dev.zarr.zarrjava.v2; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; @@ -16,47 +15,47 @@ public interface Node extends dev.zarr.zarrjava.core.Node { - static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - return objectMapper; - } + static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } - static ObjectWriter makeObjectWriter() { - return makeObjectMapper().writerWithDefaultPrettyPrinter(); - } + static ObjectWriter makeObjectWriter() { + return makeObjectMapper().writerWithDefaultPrettyPrinter(); + } /** - * Opens an existing Zarr array or group at a specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @throws IOException throws IOException if the metadata cannot be read - * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened - */ - static Node open(StoreHandle storeHandle) throws IOException, ZarrException { - boolean isGroup = storeHandle.resolve(ZGROUP).exists(); - boolean isArray = storeHandle.resolve(ZARRAY).exists(); - - if (isGroup && isArray) { - throw new ZarrException("Store handle '" + storeHandle + "' contains both a " + ZGROUP + " and a " + ZARRAY + " file."); - } else if (isGroup) { - return Group.open(storeHandle); - } else if (isArray) { - try { - return Array.open(storeHandle); - } catch (IOException e) { - throw new ZarrException("Failed to read array metadata for store handle '" + storeHandle + "'.", e); - } + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isGroup = storeHandle.resolve(ZGROUP).exists(); + boolean isArray = storeHandle.resolve(ZARRAY).exists(); + + if (isGroup && isArray) { + throw new ZarrException("Store handle '" + storeHandle + "' contains both a " + ZGROUP + " and a " + ZARRAY + " file."); + } else if (isGroup) { + return Group.open(storeHandle); + } else if (isArray) { + try { + return Array.open(storeHandle); + } catch (IOException e) { + throw new ZarrException("Failed to read array metadata for store handle '" + storeHandle + "'.", e); + } + } + throw new NoSuchFileException("Store handle '" + storeHandle + "' does not contain a " + ZGROUP + " or a " + ZARRAY + " file."); } - throw new NoSuchFileException("Store handle '" + storeHandle + "' does not contain a " + ZGROUP + " or a " + ZARRAY + " file."); - } - static Node open(Path path) throws IOException, ZarrException { - return open(new StoreHandle(new FilesystemStore(path))); - } + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } - static Node open(String path) throws IOException, ZarrException { - return open(Paths.get(path)); - } + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Order.java b/src/main/java/dev/zarr/zarrjava/v2/Order.java index 72f5f6e..1865a46 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Order.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Order.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.v2; public enum Order { - F, - C + F, + C } diff --git a/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java index 712306e..d8fab3e 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java @@ -10,24 +10,24 @@ public class V2ChunkKeyEncoding implements ChunkKeyEncoding { - public final String name = "v2"; - @Nonnull - public final Separator separator; + public final String name = "v2"; + @Nonnull + public final Separator separator; - public V2ChunkKeyEncoding( - @Nonnull Separator separator - ) { - this.separator = separator; - } + public V2ChunkKeyEncoding( + @Nonnull Separator separator + ) { + this.separator = separator; + } - @Override - public String[] encodeChunkKey(long[] chunkCoords) { - Stream keys = Arrays.stream(chunkCoords) - .mapToObj(Long::toString); - if (separator == Separator.SLASH) { - return keys.toArray(String[]::new); + @Override + public String[] encodeChunkKey(long[] chunkCoords) { + Stream keys = Arrays.stream(chunkCoords) + .mapToObj(Long::toString); + if (separator == Separator.SLASH) { + return keys.toArray(String[]::new); + } + return new String[]{keys.collect(Collectors.joining(this.separator.getValue()))}; } - return new String[]{keys.collect(Collectors.joining(this.separator.getValue()))}; - } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java index 2b0f7a6..2a1a9fa 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java @@ -1,29 +1,30 @@ package dev.zarr.zarrjava.v2.codec; import com.fasterxml.jackson.databind.jsontype.NamedType; -import dev.zarr.zarrjava.v2.codec.core.*; +import dev.zarr.zarrjava.v2.codec.core.BloscCodec; +import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; import java.util.HashMap; import java.util.Map; public class CodecRegistry { - static Map> map = new HashMap<>(); + static Map> map = new HashMap<>(); - static { - addType("blosc", BloscCodec.class); - addType("zlib", ZlibCodec.class); - } + static { + addType("blosc", BloscCodec.class); + addType("zlib", ZlibCodec.class); + } - public static void addType(String name, Class codecClass) { - map.put(name, codecClass); - } + public static void addType(String name, Class codecClass) { + map.put(name, codecClass); + } - public static NamedType[] getNamedTypes() { - return map.entrySet() - .stream() - .map(entry -> new NamedType(entry.getValue(), entry.getKey())) - .toArray( - NamedType[]::new); - } + public static NamedType[] getNamedTypes() { + return map.entrySet() + .stream() + .map(entry -> new NamedType(entry.getValue(), entry.getKey())) + .toArray( + NamedType[]::new); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 4e618d4..487bb95 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -23,102 +23,102 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { - @JsonIgnore - public final String id = "blosc"; - - @Nonnull - @JsonSerialize(using = CustomCompressorSerializer.class) - public final Blosc.Compressor cname; - @Nonnull - @JsonSerialize(using = CustomShuffleSerializer.class) - public final Blosc.Shuffle shuffle; - public final int clevel; - public final int typesize; - public final int blocksize; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public BloscCodec( - @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") - @JsonDeserialize(using = CustomCompressorDeserializer.class) - Blosc.Compressor cname, - @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") - @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, - @JsonProperty(value = "clevel", defaultValue = "5") int clevel, - @JsonProperty(value = "typesize", defaultValue = "0") int typesize, - @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize - ) throws ZarrException { - if (clevel < 0 || clevel > 9) { - throw new ZarrException("'clevel' needs to be between 0 and 9."); + @JsonIgnore + public final String id = "blosc"; + + @Nonnull + @JsonSerialize(using = CustomCompressorSerializer.class) + public final Blosc.Compressor cname; + @Nonnull + @JsonSerialize(using = CustomShuffleSerializer.class) + public final Blosc.Shuffle shuffle; + public final int clevel; + public final int typesize; + public final int blocksize; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public BloscCodec( + @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") + @JsonDeserialize(using = CustomCompressorDeserializer.class) + Blosc.Compressor cname, + @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") + @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, + @JsonProperty(value = "clevel", defaultValue = "5") int clevel, + @JsonProperty(value = "typesize", defaultValue = "0") int typesize, + @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize + ) throws ZarrException { + if (clevel < 0 || clevel > 9) { + throw new ZarrException("'clevel' needs to be between 0 and 9."); + } + this.cname = cname; + this.shuffle = shuffle; + this.clevel = clevel; + this.typesize = typesize; + this.blocksize = blocksize; } - this.cname = cname; - this.shuffle = shuffle; - this.clevel = clevel; - this.typesize = typesize; - this.blocksize = blocksize; - } - - @Override - public ByteBuffer encode(ByteBuffer chunkBytes) - throws ZarrException { - try { - return ByteBuffer.wrap( - Blosc.compress(Utils.toArray(chunkBytes), this.typesize, this.cname, - this.clevel, - this.shuffle, this.blocksize - )); - } catch (Exception ex) { - throw new ZarrException("Error in encoding blosc.", ex); + + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap( + Blosc.compress(Utils.toArray(chunkBytes), this.typesize, this.cname, + this.clevel, + this.shuffle, this.blocksize + )); + } catch (Exception ex) { + throw new ZarrException("Error in encoding blosc.", ex); + } } - } - - @Override - public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - if (typesize == 0) { - return new BloscCodec( - this.cname, - this.shuffle, - this.clevel, - arrayMetadata.dataType.getByteCount(), - this.blocksize - ); + + @Override + public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + if (typesize == 0) { + return new BloscCodec( + this.cname, + this.shuffle, + this.clevel, + arrayMetadata.dataType.getByteCount(), + this.blocksize + ); + } + return this; } - return this; - } - public static final class CustomShuffleSerializer extends StdSerializer { + public static final class CustomShuffleSerializer extends StdSerializer { - public CustomShuffleSerializer() { - super(Blosc.Shuffle.class); - } + public CustomShuffleSerializer() { + super(Blosc.Shuffle.class); + } - public CustomShuffleSerializer(Class t) { - super(t); - } + public CustomShuffleSerializer(Class t) { + super(t); + } - @Override - public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, - SerializerProvider provider) - throws IOException { - generator.writeNumber(shuffle.ordinal()); + @Override + public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, + SerializerProvider provider) + throws IOException { + generator.writeNumber(shuffle.ordinal()); + } } - } - public static final class CustomShuffleDeserializer extends StdDeserializer { + public static final class CustomShuffleDeserializer extends StdDeserializer { - public CustomShuffleDeserializer() { - this(null); - } + public CustomShuffleDeserializer() { + this(null); + } - public CustomShuffleDeserializer(Class vc) { - super(vc); - } + public CustomShuffleDeserializer(Class vc) { + super(vc); + } - @Override - public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - int shuffle = jsonParser.getCodec() - .readValue(jsonParser, int.class); - return Blosc.Shuffle.values()[shuffle]; + @Override + public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) + throws IOException { + int shuffle = jsonParser.getCodec() + .readValue(jsonParser, int.class); + return Blosc.Shuffle.values()[shuffle]; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 3e9b5cf..4e62817 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -5,15 +5,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.ArrayMetadata; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.zip.*; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; public class ZlibCodec extends BytesBytesCodec implements Codec { @@ -24,7 +26,7 @@ public class ZlibCodec extends BytesBytesCodec implements Codec { @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ZlibCodec( - @JsonProperty(value = "level", defaultValue = "1") int level) throws ZarrException { + @JsonProperty(value = "level", defaultValue = "1") int level) throws ZarrException { if (level < 0 || level > 9) { throw new ZarrException("'level' needs to be between 0 and 9."); } @@ -35,7 +37,7 @@ public ZlibCodec( @Override public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InflaterInputStream inputStream = new InflaterInputStream( - new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { + new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { Utils.copyStream(inputStream, outputStream); inputStream.close(); return ByteBuffer.wrap(outputStream.toByteArray()); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 8e263a1..72aad1e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -3,11 +3,13 @@ import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.core.codec.CodecPipeline; + +import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; @@ -15,249 +17,249 @@ import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; -import javax.annotation.Nonnull; + import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; import static dev.zarr.zarrjava.v3.Node.makeObjectWriter; public class Array extends dev.zarr.zarrjava.core.Array implements Node { - private final ArrayMetadata metadata; - - public ArrayMetadata metadata(){ - return metadata; - } + private final ArrayMetadata metadata; - protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws ZarrException { - super(storeHandle); - this.metadata = arrayMetadata; - this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); - } + protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws ZarrException { + super(storeHandle); + this.metadata = arrayMetadata; + this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); + } - /** - * Opens an existing Zarr array at a specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @throws IOException throws IOException if the metadata cannot be read - * @throws ZarrException throws ZarrException if the Zarr array cannot be opened - */ - public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { - return new Array( - storeHandle, - makeObjectMapper() - .readValue( - Utils.toArray(storeHandle.resolve(ZARR_JSON).readNonNull()), - ArrayMetadata.class - ) - ); - } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + return new Array( + storeHandle, + makeObjectMapper() + .readValue( + Utils.toArray(storeHandle.resolve(ZARR_JSON).readNonNull()), + ArrayMetadata.class + ) + ); + } - /** + /** * Opens an existing Zarr array at a specified storage location using a Path. * * @param path the storage location of the Zarr array - * @throws IOException if the metadata cannot be read + * @throws IOException if the metadata cannot be read * @throws ZarrException if the Zarr array cannot be opened */ public static Array open(Path path) throws IOException, ZarrException { - return open(new FilesystemStore(path).resolve()); + return open(new FilesystemStore(path).resolve()); } /** * Opens an existing Zarr array at a specified storage location using a String path. * * @param path the storage location of the Zarr array - * @throws IOException if the metadata cannot be read + * @throws IOException if the metadata cannot be read * @throws ZarrException if the Zarr array cannot be opened */ public static Array open(String path) throws IOException, ZarrException { - return open(Paths.get(path)); + return open(Paths.get(path)); } - /** - * Creates a new Zarr array with the provided metadata in an in-memory store - * - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(new MemoryStore().resolve(), arrayMetadata); - } + /** + * Creates a new Zarr array with the provided metadata in an in-memory store + * + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(new MemoryStore().resolve(), arrayMetadata); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param path the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(Path path, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(new FilesystemStore(path).resolve(), arrayMetadata); - } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadata); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param path the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(String path, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(Paths.get(path), arrayMetadata); - } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadata); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. This - * method will raise an exception if a Zarr array already exists at the specified storage - * location. - * - * @param storeHandle the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(storeHandle, arrayMetadata, false); - } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle, arrayMetadata, false); + } - /** - * Creates a new Zarr array with the provided metadata at a specified storage location. If - * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the - * specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @param arrayMetadata the metadata of the Zarr array - * @param existsOk if true, no exception is raised if the Zarr array already exists - * @throws IOException throws IOException if the metadata cannot be serialized - * @throws ZarrException throws ZarrException if the Zarr array cannot be created - */ - public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) - throws IOException, ZarrException { - StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); - if (!existsOk && metadataHandle.exists()) { - throw new RuntimeException( - "Trying to create a new array in " + storeHandle + ". But " + metadataHandle - + " already exists."); + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. If + * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the + * specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException throws IOException if the metadata cannot be serialized + * @throws ZarrException throws ZarrException if the Zarr array cannot be created + */ + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) + throws IOException, ZarrException { + StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); + if (!existsOk && metadataHandle.exists()) { + throw new RuntimeException( + "Trying to create a new array in " + storeHandle + ". But " + metadataHandle + + " already exists."); + } + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); + metadataHandle.set(metadataBytes); + return new Array(storeHandle, arrayMetadata); } - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); - metadataHandle.set(metadataBytes); - return new Array(storeHandle, arrayMetadata); - } - /** - * Creates a new Zarr array at a specified storage location. This method provides a callback that - * gets an ArrayMetadataBuilder and needs to return such an ArrayMetadataBuilder. The callback can - * be used to construct the metadata of the Zarr array. If `existsOk` is false, this method will - * raise an exception if a Zarr array already exists at the specified storage location. - * - * @param storeHandle the storage location of the Zarr array - * @param arrayMetadataBuilderMapper a callback that is used to construct the metadata of the Zarr array - * @param existsOk if true, no exception is raised if the Zarr array already exists - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the Zarr array cannot be created - */ - public static Array create(StoreHandle storeHandle, - Function arrayMetadataBuilderMapper, - boolean existsOk) throws IOException, ZarrException { - return create(storeHandle, - arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); - } + /** + * Creates a new Zarr array at a specified storage location. This method provides a callback that + * gets an ArrayMetadataBuilder and needs to return such an ArrayMetadataBuilder. The callback can + * be used to construct the metadata of the Zarr array. If `existsOk` is false, this method will + * raise an exception if a Zarr array already exists at the specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadataBuilderMapper a callback that is used to construct the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(StoreHandle storeHandle, + Function arrayMetadataBuilderMapper, + boolean existsOk) throws IOException, ZarrException { + return create(storeHandle, + arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); + } - public static Array create(Path path, Function arrayMetadataBuilderMapper, boolean existsOk) - throws IOException, ZarrException { - return Array.create(new FilesystemStore(path).resolve(), arrayMetadataBuilderMapper, existsOk); - } + public static Array create(Path path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadataBuilderMapper, existsOk); + } - public static Array create(String path, Function arrayMetadataBuilderMapper, boolean existsOk) - throws IOException, ZarrException { - return Array.create(Paths.get(path), arrayMetadataBuilderMapper, existsOk); - } + public static Array create(String path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadataBuilderMapper, existsOk); + } - @Nonnull - public static ArrayMetadataBuilder metadataBuilder() { - return new ArrayMetadataBuilder(); - } + @Nonnull + public static ArrayMetadataBuilder metadataBuilder() { + return new ArrayMetadataBuilder(); + } - @Nonnull - public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadata) { - return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); - } + @Nonnull + public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadata) { + return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); + } - private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - return Array.create(storeHandle, newArrayMetadata, true); - } + public ArrayMetadata metadata() { + return metadata; + } - /** - * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or - * deleted. This method returns a new instance of the Zarr array class and the old instance - * becomes invalid. - * - * @param newShape the new shape of the Zarr array - * @throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array resize(long[] newShape) throws ZarrException, IOException { - if (newShape.length != metadata.ndim()) { - throw new IllegalArgumentException( - "'newShape' needs to have rank '" + metadata.ndim() + "'."); + private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { + return Array.create(storeHandle, newArrayMetadata, true); } - ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) - .withShape(newShape) - .build(); - return writeMetadata(newArrayMetadata); - } + /** + * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or + * deleted. This method returns a new instance of the Zarr array class and the old instance + * becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape) throws ZarrException, IOException { + if (newShape.length != metadata.ndim()) { + throw new IllegalArgumentException( + "'newShape' needs to have rank '" + metadata.ndim() + "'."); + } - /** - * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This - * method returns a new instance of the Zarr array class and the old instance becomes invalid. - * - * @param newAttributes the new attributes of the Zarr array - * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { - ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata, false) - .withAttributes(newAttributes) - .build(); - return writeMetadata(newArrayMetadata); - } + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withShape(newShape) + .build(); + return writeMetadata(newArrayMetadata); + } - /** - * Updates the attributes of the Zarr array. It provides a callback that gets the current - * attributes as input and needs to return the new set of attributes. The attributes in the - * callback may be mutated. This method overwrites and removes any existing attributes. This - * method returns a new instance of the Zarr array class and the old instance becomes invalid. - * - * @param attributeMapper the callback that is used to construct the new attributes - * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized - */ - public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); - } + /** + * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { + ArrayMetadata newArrayMetadata = + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) + .withAttributes(newAttributes) + .build(); + return writeMetadata(newArrayMetadata); + } - @Override - public String toString() { - return String.format("", storeHandle, - Arrays.stream(metadata.shape) - .mapToObj(Long::toString) - .collect(Collectors.joining(", ")), - metadata.dataType - ); - } + /** + * Updates the attributes of the Zarr array. It provides a callback that gets the current + * attributes as input and needs to return the new set of attributes. The attributes in the + * callback may be mutated. This method overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param attributeMapper the callback that is used to construct the new attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + + @Override + public String toString() { + return String.format("", storeHandle, + Arrays.stream(metadata.shape) + .mapToObj(Long::toString) + .collect(Collectors.joining(", ")), + metadata.dataType + ); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index eb50032..58d5733 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -11,168 +11,167 @@ import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.Arrays; import java.util.Map; import java.util.Optional; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { - public static final String NODE_TYPE = "array"; - static final int ZARR_FORMAT = 3; - - @JsonProperty("zarr_format") - public final int zarrFormat = ZARR_FORMAT; - @JsonProperty("node_type") - public final String nodeType = NODE_TYPE; - - @JsonProperty("data_type") - public final DataType dataType; - - @JsonProperty("chunk_grid") - public final ChunkGrid chunkGrid; - - @JsonProperty("chunk_key_encoding") - public final ChunkKeyEncoding chunkKeyEncoding; - - @JsonProperty("codecs") - public final Codec[] codecs; - @Nullable - @JsonProperty("attributes") - public final Attributes attributes; - @Nullable - @JsonProperty("dimension_names") - public final String[] dimensionNames; - @Nullable - @JsonProperty("storage_transformers") - public final Map[] storageTransformers; - - @JsonIgnore - public CoreArrayMetadata coreArrayMetadata; - - public ArrayMetadata( - long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding, - Object fillValue, - @Nonnull Codec[] codecs, - @Nullable String[] dimensionNames, - @Nullable Attributes attributes, - @Nullable Map[] storageTransformers - ) throws ZarrException { - this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, - dimensionNames, - attributes, storageTransformers - ); - } - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ArrayMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat, - @JsonProperty(value = "node_type", required = true) String nodeType, - @JsonProperty(value = "shape", required = true) long[] shape, - @JsonProperty(value = "data_type", required = true) DataType dataType, - @JsonProperty(value = "chunk_grid", required = true) ChunkGrid chunkGrid, - @JsonProperty(value = "chunk_key_encoding", required = true) ChunkKeyEncoding chunkKeyEncoding, - @JsonProperty(value = "fill_value", required = true) Object fillValue, - @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, - @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, - @Nullable @JsonProperty(value = "attributes") Attributes attributes, - @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers - ) throws ZarrException { - super(shape, fillValue, dataType); - if (zarrFormat != this.zarrFormat) { - throw new ZarrException( - "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + public static final String NODE_TYPE = "array"; + static final int ZARR_FORMAT = 3; + + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + @JsonProperty("node_type") + public final String nodeType = NODE_TYPE; + + @JsonProperty("data_type") + public final DataType dataType; + + @JsonProperty("chunk_grid") + public final ChunkGrid chunkGrid; + + @JsonProperty("chunk_key_encoding") + public final ChunkKeyEncoding chunkKeyEncoding; + + @JsonProperty("codecs") + public final Codec[] codecs; + @Nullable + @JsonProperty("attributes") + public final Attributes attributes; + @Nullable + @JsonProperty("dimension_names") + public final String[] dimensionNames; + @Nullable + @JsonProperty("storage_transformers") + public final Map[] storageTransformers; + + @JsonIgnore + public CoreArrayMetadata coreArrayMetadata; + + public ArrayMetadata( + long[] shape, DataType dataType, ChunkGrid chunkGrid, ChunkKeyEncoding chunkKeyEncoding, + Object fillValue, + @Nonnull Codec[] codecs, + @Nullable String[] dimensionNames, + @Nullable Attributes attributes, + @Nullable Map[] storageTransformers + ) throws ZarrException { + this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, + dimensionNames, + attributes, storageTransformers + ); + } + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ArrayMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "node_type", required = true) String nodeType, + @JsonProperty(value = "shape", required = true) long[] shape, + @JsonProperty(value = "data_type", required = true) DataType dataType, + @JsonProperty(value = "chunk_grid", required = true) ChunkGrid chunkGrid, + @JsonProperty(value = "chunk_key_encoding", required = true) ChunkKeyEncoding chunkKeyEncoding, + @JsonProperty(value = "fill_value", required = true) Object fillValue, + @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, + @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, + @Nullable @JsonProperty(value = "attributes") Attributes attributes, + @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers + ) throws ZarrException { + super(shape, fillValue, dataType); + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + if (!nodeType.equals(this.nodeType)) { + throw new ZarrException( + "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); + } + if (storageTransformers != null && storageTransformers.length > 0) { + throw new ZarrException( + "Storage transformers are not supported in this version of Zarr Java."); + } + if (chunkGrid instanceof RegularChunkGrid) { + int[] chunkShape = ((RegularChunkGrid) chunkGrid).configuration.chunkShape; + if (shape.length != chunkShape.length) { + throw new ZarrException("Shape (ndim=" + shape.length + ") and chunk shape (ndim=" + + chunkShape.length + ") need to have the same number of dimensions."); + } + + Optional shardingCodec = getShardingIndexedCodec(codecs); + int[] outerChunkShape = chunkShape; + while (shardingCodec.isPresent()) { + ShardingIndexedCodec.Configuration shardingConfig = ((ShardingIndexedCodec) shardingCodec.get()).configuration; + int[] innerChunkShape = shardingConfig.chunkShape; + if (outerChunkShape.length != innerChunkShape.length) + throw new ZarrException("Sharding dimensions mismatch of outer chunk shape " + Arrays.toString(outerChunkShape) + " and inner chunk shape" + Arrays.toString(innerChunkShape)); + for (int i = 0; i < outerChunkShape.length; i++) { + if (outerChunkShape[i] % innerChunkShape[i] != 0) + throw new ZarrException("Sharding inner chunk shape " + Arrays.toString(innerChunkShape) + " does not evenly divide the outer chunk size " + Arrays.toString(outerChunkShape)); + } + outerChunkShape = innerChunkShape; + shardingCodec = getShardingIndexedCodec(shardingConfig.codecs); + } + } + this.chunkGrid = chunkGrid; + this.dataType = dataType; + this.coreArrayMetadata = + new CoreArrayMetadata(this.shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, + this.dataType, + this.parsedFillValue + ); + + this.chunkKeyEncoding = chunkKeyEncoding; + this.codecs = codecs; + this.dimensionNames = dimensionNames; + this.attributes = attributes; + this.storageTransformers = storageTransformers; } - if (!nodeType.equals(this.nodeType)) { - throw new ZarrException( - "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); + + public static Optional getShardingIndexedCodec(Codec[] codecs) { + return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } - if (storageTransformers != null && storageTransformers.length > 0) { - throw new ZarrException( - "Storage transformers are not supported in this version of Zarr Java."); + + public ucar.ma2.Array allocateFillValueChunk() { + return coreArrayMetadata.allocateFillValueChunk(); } - if (chunkGrid instanceof RegularChunkGrid) { - int[] chunkShape = ((RegularChunkGrid) chunkGrid).configuration.chunkShape; - if (shape.length != chunkShape.length) { - throw new ZarrException("Shape (ndim=" + shape.length + ") and chunk shape (ndim=" + - chunkShape.length + ") need to have the same number of dimensions."); - } - - Optional shardingCodec = getShardingIndexedCodec(codecs); - int[] outerChunkShape = chunkShape; - while (shardingCodec.isPresent()) { - ShardingIndexedCodec.Configuration shardingConfig = ((ShardingIndexedCodec) shardingCodec.get()).configuration; - int[] innerChunkShape = shardingConfig.chunkShape; - if (outerChunkShape.length != innerChunkShape.length) - throw new ZarrException("Sharding dimensions mismatch of outer chunk shape " + Arrays.toString(outerChunkShape) + " and inner chunk shape" + Arrays.toString(innerChunkShape)); - for (int i = 0; i < outerChunkShape.length; i++) { - if (outerChunkShape[i] % innerChunkShape[i] != 0) - throw new ZarrException("Sharding inner chunk shape " + Arrays.toString(innerChunkShape) + " does not evenly divide the outer chunk size " + Arrays.toString(outerChunkShape)); - } - outerChunkShape = innerChunkShape; - shardingCodec = getShardingIndexedCodec(shardingConfig.codecs); - } + + @Override + public ChunkKeyEncoding chunkKeyEncoding() { + return chunkKeyEncoding; } - this.chunkGrid = chunkGrid; - this.dataType = dataType; - this.coreArrayMetadata = - new CoreArrayMetadata(this.shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, - this.dataType, - this.parsedFillValue - ); - this.chunkKeyEncoding = chunkKeyEncoding; - this.codecs = codecs; - this.dimensionNames = dimensionNames; - this.attributes = attributes; - this.storageTransformers = storageTransformers; - } - - - public ucar.ma2.Array allocateFillValueChunk() { - return coreArrayMetadata.allocateFillValueChunk(); - } - - @Override - public ChunkKeyEncoding chunkKeyEncoding() { - return chunkKeyEncoding; - } - - @Override - public Object parsedFillValue() { - return parsedFillValue; - } - - @Nonnull - @Override - public Attributes attributes() throws ZarrException { - if (attributes == null) { - throw new ZarrException("Array attributes have not been set."); + @Override + public Object parsedFillValue() { + return parsedFillValue; } - return attributes; - } - public static Optional getShardingIndexedCodec(Codec[] codecs) { - return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); - } + @Nonnull + @Override + public Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } - public int[] chunkShape() { - return ((RegularChunkGrid) this.chunkGrid).configuration.chunkShape; - } + public int[] chunkShape() { + return ((RegularChunkGrid) this.chunkGrid).configuration.chunkShape; + } - @Override - public DataType dataType() { - return dataType; - } + @Override + public DataType dataType() { + return dataType; + } - public int chunkSize() { - return coreArrayMetadata.chunkSize(); - } + public int chunkSize() { + return coreArrayMetadata.chunkSize(); + } - public int chunkByteLength() { - return coreArrayMetadata.chunkByteLength(); - } + public int chunkByteLength() { + return coreArrayMetadata.chunkByteLength(); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 1da2015..83a6e1a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -2,16 +2,16 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.DefaultChunkKeyEncoding; -import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import java.util.HashMap; import java.util.Map; @@ -19,155 +19,155 @@ public class ArrayMetadataBuilder { - long[] shape = null; - DataType dataType = null; - ChunkGrid chunkGrid = null; - ChunkKeyEncoding chunkKeyEncoding = - new DefaultChunkKeyEncoding(new DefaultChunkKeyEncoding.Configuration(Separator.SLASH)); - - Object fillValue = 0; - Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; - Attributes attributes = new Attributes(); - Map[] storageTransformers = new HashMap[]{}; - String[] dimensionNames = null; - - protected ArrayMetadataBuilder() { - } - - protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { - return fromArrayMetadata(arrayMetadata, true); - } - - protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { - ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); - builder.shape = arrayMetadata.shape; - builder.dataType = arrayMetadata.dataType; - builder.chunkGrid = arrayMetadata.chunkGrid; - builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; - builder.fillValue = arrayMetadata.parsedFillValue; - builder.codecs = arrayMetadata.codecs; - builder.dimensionNames = arrayMetadata.dimensionNames; - builder.storageTransformers = arrayMetadata.storageTransformers; - if (withAttributes) { - builder.attributes = arrayMetadata.attributes; - } - return builder; - } - - public ArrayMetadataBuilder withShape(long... shape) { - this.shape = shape; - return this; - } - - public ArrayMetadataBuilder withDataType(DataType dataType) { - this.dataType = dataType; - return this; - } - - public ArrayMetadataBuilder withDataType(String dataType) { - this.dataType = DataType.valueOf(dataType); - return this; - } - - public ArrayMetadataBuilder withChunkShape(int... chunkShape) { - this.chunkGrid = new RegularChunkGrid(new RegularChunkGrid.Configuration(chunkShape)); - return this; - } - - public ArrayMetadataBuilder withDefaultChunkKeyEncoding(Separator separator) { - this.chunkKeyEncoding = new DefaultChunkKeyEncoding( - new DefaultChunkKeyEncoding.Configuration(separator)); - return this; - } + long[] shape = null; + DataType dataType = null; + ChunkGrid chunkGrid = null; + ChunkKeyEncoding chunkKeyEncoding = + new DefaultChunkKeyEncoding(new DefaultChunkKeyEncoding.Configuration(Separator.SLASH)); + + Object fillValue = 0; + Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; + Attributes attributes = new Attributes(); + Map[] storageTransformers = new HashMap[]{}; + String[] dimensionNames = null; + + protected ArrayMetadataBuilder() { + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { + ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); + builder.shape = arrayMetadata.shape; + builder.dataType = arrayMetadata.dataType; + builder.chunkGrid = arrayMetadata.chunkGrid; + builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; + builder.fillValue = arrayMetadata.parsedFillValue; + builder.codecs = arrayMetadata.codecs; + builder.dimensionNames = arrayMetadata.dimensionNames; + builder.storageTransformers = arrayMetadata.storageTransformers; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } + return builder; + } + + public ArrayMetadataBuilder withShape(long... shape) { + this.shape = shape; + return this; + } + + public ArrayMetadataBuilder withDataType(DataType dataType) { + this.dataType = dataType; + return this; + } + + public ArrayMetadataBuilder withDataType(String dataType) { + this.dataType = DataType.valueOf(dataType); + return this; + } + + public ArrayMetadataBuilder withChunkShape(int... chunkShape) { + this.chunkGrid = new RegularChunkGrid(new RegularChunkGrid.Configuration(chunkShape)); + return this; + } + + public ArrayMetadataBuilder withDefaultChunkKeyEncoding(Separator separator) { + this.chunkKeyEncoding = new DefaultChunkKeyEncoding( + new DefaultChunkKeyEncoding.Configuration(separator)); + return this; + } public ArrayMetadataBuilder withDefaultChunkKeyEncoding() { this.chunkKeyEncoding = new DefaultChunkKeyEncoding( - new DefaultChunkKeyEncoding.Configuration(Separator.SLASH)); - return this; - } - - public ArrayMetadataBuilder withDefaultChunkKeyEncoding(String separator) { - this.chunkKeyEncoding = - new DefaultChunkKeyEncoding( - new DefaultChunkKeyEncoding.Configuration(Separator.valueOf(separator))); - return this; - } - - public ArrayMetadataBuilder withV2ChunkKeyEncoding(Separator separator) { - this.chunkKeyEncoding = new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(separator)); - return this; - } - - public ArrayMetadataBuilder withV2ChunkKeyEncoding() { - this.chunkKeyEncoding = new V2ChunkKeyEncoding( - new V2ChunkKeyEncoding.Configuration(Separator.DOT)); - return this; - } - - public ArrayMetadataBuilder withV2ChunkKeyEncoding(String separator) { - this.chunkKeyEncoding = - new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(Separator.valueOf(separator))); - return this; - } - - public ArrayMetadataBuilder withFillValue(Object fillValue) { - this.fillValue = fillValue; - return this; - } - - public ArrayMetadataBuilder withCodecs(Codec... codecs) { - this.codecs = codecs; - return this; - } - - public ArrayMetadataBuilder withCodecs(Function codecBuilder) { - if (dataType == null) { - throw new IllegalStateException("Please call `withDataType` first."); - } - CodecBuilder nestedCodecBuilder = new CodecBuilder(dataType); - this.codecs = codecBuilder.apply(nestedCodecBuilder) - .build(); - return this; - } - - public ArrayMetadataBuilder withDimensionNames(String... dimensionNames) { - this.dimensionNames = dimensionNames; - return this; - } - - public ArrayMetadataBuilder putAttribute(String key, Object value) { - this.attributes.put(key, value); - return this; - } - - public ArrayMetadataBuilder withAttributes(Attributes attributes) { - if (this.attributes == null) { - this.attributes = attributes; - } else { - this.attributes.putAll(attributes); - } - return this; - } - - public ArrayMetadataBuilder withStorageTransformers(Map[] storageTransformers) { - this.storageTransformers = storageTransformers; - return this; - } - - public ArrayMetadata build() throws ZarrException { - if (shape == null) { - throw new ZarrException("Shape needs to be provided. Please call `.withShape`."); - } - if (dataType == null) { - throw new ZarrException("Data type needs to be provided. Please call `.withDataType`."); - } - if (chunkGrid == null) { - throw new ZarrException("Chunk grid needs to be provided. Please call `.withChunkShape`."); - } - return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, - dimensionNames, - attributes, - storageTransformers - ); - } + new DefaultChunkKeyEncoding.Configuration(Separator.SLASH)); + return this; + } + + public ArrayMetadataBuilder withDefaultChunkKeyEncoding(String separator) { + this.chunkKeyEncoding = + new DefaultChunkKeyEncoding( + new DefaultChunkKeyEncoding.Configuration(Separator.valueOf(separator))); + return this; + } + + public ArrayMetadataBuilder withV2ChunkKeyEncoding(Separator separator) { + this.chunkKeyEncoding = new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(separator)); + return this; + } + + public ArrayMetadataBuilder withV2ChunkKeyEncoding() { + this.chunkKeyEncoding = new V2ChunkKeyEncoding( + new V2ChunkKeyEncoding.Configuration(Separator.DOT)); + return this; + } + + public ArrayMetadataBuilder withV2ChunkKeyEncoding(String separator) { + this.chunkKeyEncoding = + new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(Separator.valueOf(separator))); + return this; + } + + public ArrayMetadataBuilder withFillValue(Object fillValue) { + this.fillValue = fillValue; + return this; + } + + public ArrayMetadataBuilder withCodecs(Codec... codecs) { + this.codecs = codecs; + return this; + } + + public ArrayMetadataBuilder withCodecs(Function codecBuilder) { + if (dataType == null) { + throw new IllegalStateException("Please call `withDataType` first."); + } + CodecBuilder nestedCodecBuilder = new CodecBuilder(dataType); + this.codecs = codecBuilder.apply(nestedCodecBuilder) + .build(); + return this; + } + + public ArrayMetadataBuilder withDimensionNames(String... dimensionNames) { + this.dimensionNames = dimensionNames; + return this; + } + + public ArrayMetadataBuilder putAttribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } + return this; + } + + public ArrayMetadataBuilder withStorageTransformers(Map[] storageTransformers) { + this.storageTransformers = storageTransformers; + return this; + } + + public ArrayMetadata build() throws ZarrException { + if (shape == null) { + throw new ZarrException("Shape needs to be provided. Please call `.withShape`."); + } + if (dataType == null) { + throw new ZarrException("Data type needs to be provided. Please call `.withDataType`."); + } + if (chunkGrid == null) { + throw new ZarrException("Chunk grid needs to be provided. Please call `.withChunkShape`."); + } + return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, + dimensionNames, + attributes, + storageTransformers + ); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index 9d9842c..65ab28a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -3,67 +3,67 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum DataType implements dev.zarr.zarrjava.core.DataType { - BOOL("bool", 1), - INT8("int8", 1), - INT16("int16", 2), - INT32("int32", 4), - INT64("int64", 8), - UINT8( - "uint8", - 1 - ), - UINT16("uint16", 2), - UINT32("uint32", 4), - UINT64("uint64", 8), - FLOAT32("float32", 4), - FLOAT64( - "float64", - 8 - ); + BOOL("bool", 1), + INT8("int8", 1), + INT16("int16", 2), + INT32("int32", 4), + INT64("int64", 8), + UINT8( + "uint8", + 1 + ), + UINT16("uint16", 2), + UINT32("uint32", 4), + UINT64("uint64", 8), + FLOAT32("float32", 4), + FLOAT64( + "float64", + 8 + ); - private final String dtype; - private final int byteCount; + private final String dtype; + private final int byteCount; - DataType(String dtype, int byteCount) { - this.dtype = dtype; - this.byteCount = byteCount; - } + DataType(String dtype, int byteCount) { + this.dtype = dtype; + this.byteCount = byteCount; + } - @JsonValue - public String getValue() { - return dtype; - } + @JsonValue + public String getValue() { + return dtype; + } - public int getByteCount() { - return byteCount; - } + public int getByteCount() { + return byteCount; + } - public ucar.ma2.DataType getMA2DataType() { - switch (this) { - case BOOL: - return ucar.ma2.DataType.BOOLEAN; - case INT8: - return ucar.ma2.DataType.BYTE; - case INT16: - return ucar.ma2.DataType.SHORT; - case INT32: - return ucar.ma2.DataType.INT; - case INT64: - return ucar.ma2.DataType.LONG; - case UINT8: - return ucar.ma2.DataType.UBYTE; - case UINT16: - return ucar.ma2.DataType.USHORT; - case UINT32: - return ucar.ma2.DataType.UINT; - case UINT64: - return ucar.ma2.DataType.ULONG; - case FLOAT32: - return ucar.ma2.DataType.FLOAT; - case FLOAT64: - return ucar.ma2.DataType.DOUBLE; - default: - throw new RuntimeException("Unreachable"); + public ucar.ma2.DataType getMA2DataType() { + switch (this) { + case BOOL: + return ucar.ma2.DataType.BOOLEAN; + case INT8: + return ucar.ma2.DataType.BYTE; + case INT16: + return ucar.ma2.DataType.SHORT; + case INT32: + return ucar.ma2.DataType.INT; + case INT64: + return ucar.ma2.DataType.LONG; + case UINT8: + return ucar.ma2.DataType.UBYTE; + case UINT16: + return ucar.ma2.DataType.USHORT; + case UINT32: + return ucar.ma2.DataType.UINT; + case UINT64: + return ucar.ma2.DataType.ULONG; + case FLOAT32: + return ucar.ma2.DataType.FLOAT; + case FLOAT64: + return ucar.ma2.DataType.DOUBLE; + default: + throw new RuntimeException("Unreachable"); + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index d17eb77..305dcd1 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -7,298 +7,300 @@ import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.function.Function; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; + import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; import static dev.zarr.zarrjava.v3.Node.makeObjectWriter; public class Group extends dev.zarr.zarrjava.core.Group implements Node { - public GroupMetadata metadata; - - protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { - super(storeHandle); - this.metadata = groupMetadata; - } - - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { - StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); - ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, makeObjectMapper().readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); - } - - - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(Path path) throws IOException { - return open(new StoreHandle(new FilesystemStore(path))); - } - - /** - * Opens an existing Zarr group at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be read - */ - public static Group open(String path) throws IOException { - return open(Paths.get(path)); - } - - /** - * Creates a new Zarr group with default metadata in an in-memory store. - * - * @throws IOException if the metadata cannot be serialized - */ - public static Group create() throws IOException { - return new Group(new MemoryStore().resolve(), GroupMetadata.defaultValue()).writeMetadata(); - } - - /** - * Creates a new Zarr group with the provided attributes in an in-memory store. - * - * @param attributes the attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the attributes are invalid - */ - public static Group create(@Nonnull Attributes attributes) throws IOException, ZarrException { - return new Group(new MemoryStore().resolve(), new GroupMetadata(attributes)).writeMetadata(); - } - - /** - * Creates a new Zarr group with the provided metadata in an in-memory store. - * - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { - return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); - } - - /** - * Creates a new Zarr group with the provided metadata at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { - return new Group(storeHandle, groupMetadata).writeMetadata(); - } - - /** - * Creates a new Zarr group with the provided attributes at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @param attributes the attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the attributes are invalid - */ - public static Group create( - @Nonnull StoreHandle storeHandle, - @Nonnull Attributes attributes - ) throws IOException, ZarrException { - return create(storeHandle, new GroupMetadata(attributes)); - } - - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param storeHandle the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public static Group create(@Nonnull StoreHandle storeHandle) throws IOException { - return create(storeHandle, GroupMetadata.defaultValue()); - } - - /** - * Creates a new Zarr group with the provided metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { - return create(new FilesystemStore(path).resolve(), groupMetadata); - } - - /** - * Creates a new Zarr group with the provided metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { - return create(Paths.get(path), groupMetadata); - } - - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(Path path) throws IOException, ZarrException { - return create(new FilesystemStore(path).resolve()); - } - - /** - * Creates a new Zarr group with default metadata at a specified storage location. - * - * @param path the storage location of the Zarr group - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the metadata is invalid - */ - public static Group create(String path) throws IOException, ZarrException { - return create(Paths.get(path)); - } - - /** - * Retrieves a node (group or array) at the specified key within the current group. - * - * @param key the key of the node to retrieve - * @return the node at the specified key, or null if it does not exist - * @throws ZarrException if the node cannot be opened - * @throws IOException if there is an error accessing the storage - */ - @Nullable - public Node get(String key) throws ZarrException, IOException{ - StoreHandle keyHandle = storeHandle.resolve(key); - try { - return Node.open(keyHandle); - } catch (NoSuchFileException e) { - return null; + public GroupMetadata metadata; + + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + super(storeHandle); + this.metadata = groupMetadata; + } + + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { + StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); + ByteBuffer metadataBytes = metadataHandle.readNonNull(); + return new Group(storeHandle, makeObjectMapper().readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + } + + + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr group with default metadata in an in-memory store. + * + * @throws IOException if the metadata cannot be serialized + */ + public static Group create() throws IOException { + return new Group(new MemoryStore().resolve(), GroupMetadata.defaultValue()).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided attributes in an in-memory store. + * + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create(@Nonnull Attributes attributes) throws IOException, ZarrException { + return new Group(new MemoryStore().resolve(), new GroupMetadata(attributes)).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided metadata in an in-memory store. + * + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { + return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + return new Group(storeHandle, groupMetadata).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create( + @Nonnull StoreHandle storeHandle, + @Nonnull Attributes attributes + ) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata(attributes)); + } + + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull StoreHandle storeHandle) throws IOException { + return create(storeHandle, GroupMetadata.defaultValue()); + } + + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve(), groupMetadata); + } + + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(Paths.get(path), groupMetadata); + } + + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(Path path) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve()); + } + + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); + } + + /** + * Retrieves a node (group or array) at the specified key within the current group. + * + * @param key the key of the node to retrieve + * @return the node at the specified key, or null if it does not exist + * @throws ZarrException if the node cannot be opened + * @throws IOException if there is an error accessing the storage + */ + @Nullable + public Node get(String key) throws ZarrException, IOException { + StoreHandle keyHandle = storeHandle.resolve(key); + try { + return Node.open(keyHandle); + } catch (NoSuchFileException e) { + return null; + } + } + + /** + * Creates a new subgroup with the provided metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public Group createGroup(String key, GroupMetadata groupMetadata) + throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key), groupMetadata); + } + + /** + * Creates a new subgroup with the provided attributes at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @param attributes attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public Group createGroup(String key, Attributes attributes) + throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); + } + + /** + * Creates a new subgroup with default metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @return the created subgroup + * @throws IOException if the metadata cannot be serialized + */ + public Group createGroup(String key) throws IOException { + return Group.create(storeHandle.resolve(key), GroupMetadata.defaultValue()); + } + + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadata the metadata of the Zarr array + * @return the created array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created + */ + public Array createArray(String key, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadata); } - } - - /** - * Creates a new subgroup with the provided metadata at the specified key. - * - * @param key the key of the new Zarr group within the current group - * @param groupMetadata the metadata of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public Group createGroup(String key, GroupMetadata groupMetadata) - throws IOException, ZarrException { - return Group.create(storeHandle.resolve(key), groupMetadata); - } - - /** - * Creates a new subgroup with the provided attributes at the specified key. - * - * @param key the key of the new Zarr group within the current group - * @param attributes attributes of the Zarr group - * @throws IOException if the metadata cannot be serialized - */ - public Group createGroup(String key, Attributes attributes) - throws IOException, ZarrException { - return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); - } - - /** - * Creates a new subgroup with default metadata at the specified key. - * - * @param key the key of the new Zarr group within the current group - * @return the created subgroup - * @throws IOException if the metadata cannot be serialized - */ - public Group createGroup(String key) throws IOException{ - return Group.create(storeHandle.resolve(key), GroupMetadata.defaultValue()); - } - - /** - * Creates a new array with the provided metadata at the specified key. - * - * @param key the key of the new Zarr array within the current group - * @param arrayMetadata the metadata of the Zarr array - * @return the created array - * @throws IOException if the metadata cannot be serialized - * @throws ZarrException if the array cannot be created - */ - public Array createArray(String key, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { - return Array.create(storeHandle.resolve(key), arrayMetadata); - } /** * Creates a new array with the provided metadata builder mapper at the specified key. * - * @param key the key of the new Zarr array within the current group - * @param arrayMetadataBuilderMapper a function building the metadata of the Zarr array + * @param key the key of the new Zarr array within the current group + * @param arrayMetadataBuilderMapper a function building the metadata of the Zarr array * @throws IOException if the metadata cannot be serialized */ - public Array createArray(String key, - Function arrayMetadataBuilderMapper) - throws IOException, ZarrException { - return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); - } - - private Group writeMetadata() throws IOException { - return writeMetadata(this.metadata); - } - - private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); - this.metadata = newGroupMetadata; - return this; - } - - /** - * Updates the attributes of the group using a mapper function. - * - * @param attributeMapper a function that takes the current attributes and returns the updated attributes - * @return the updated group - * @throws ZarrException if the new attributes are invalid - * @throws IOException if the metadata cannot be serialized - */ - public Group updateAttributes(Function attributeMapper) - throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); - } - - /** - * Sets new attributes for the group, replacing any existing attributes. - * - * @param newAttributes the new attributes to set - * @return the updated group - * @throws ZarrException if the new attributes are invalid - * @throws IOException if the metadata cannot be serialized - */ - public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { - GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); - return writeMetadata(newGroupMetadata); - } - - @Override - public String toString() { - return String.format("", storeHandle); - } - - @Override - public GroupMetadata metadata() { - return metadata; - } + public Array createArray(String key, + Function arrayMetadataBuilderMapper) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); + } + + private Group writeMetadata() throws IOException { + return writeMetadata(this.metadata); + } + + private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); + storeHandle.resolve(ZARR_JSON) + .set(metadataBytes); + this.metadata = newGroupMetadata; + return this; + } + + /** + * Updates the attributes of the group using a mapper function. + * + * @param attributeMapper a function that takes the current attributes and returns the updated attributes + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ + public Group updateAttributes(Function attributeMapper) + throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + + /** + * Sets new attributes for the group, replacing any existing attributes. + * + * @param newAttributes the new attributes to set + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { + GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); + return writeMetadata(newGroupMetadata); + } + + @Override + public String toString() { + return String.format("", storeHandle); + } + + @Override + public GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index e8acd93..281087d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -1,7 +1,6 @@ package dev.zarr.zarrjava.v3; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; @@ -11,53 +10,53 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { - static final String NODE_TYPE = "group"; - static final int ZARR_FORMAT = 3; - @JsonProperty("zarr_format") - public final int zarrFormat = ZARR_FORMAT; - @JsonProperty("node_type") - public final String nodeType = "group"; - @JsonProperty("consolidated_metadata") - public final String consolidatedMetadata = null; - - @Nullable - public final Attributes attributes; - - public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { - this(ZARR_FORMAT, NODE_TYPE, attributes); - } - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public GroupMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat, - @JsonProperty(value = "node_type", required = true) String nodeType, - @Nullable @JsonProperty(value = "attributes") Attributes attributes - ) throws ZarrException { - if (zarrFormat != this.zarrFormat) { - throw new ZarrException( - "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + static final String NODE_TYPE = "group"; + static final int ZARR_FORMAT = 3; + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + @JsonProperty("node_type") + public final String nodeType = "group"; + @JsonProperty("consolidated_metadata") + public final String consolidatedMetadata = null; + + @Nullable + public final Attributes attributes; + + public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { + this(ZARR_FORMAT, NODE_TYPE, attributes); } - if (!nodeType.equals(this.nodeType)) { - throw new ZarrException( - "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public GroupMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "node_type", required = true) String nodeType, + @Nullable @JsonProperty(value = "attributes") Attributes attributes + ) throws ZarrException { + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + if (!nodeType.equals(this.nodeType)) { + throw new ZarrException( + "Expected node type '" + this.nodeType + "', got '" + nodeType + "'."); + } + this.attributes = attributes; } - this.attributes = attributes; - } - public static GroupMetadata defaultValue() { - try { - return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); + public static GroupMetadata defaultValue() { + try { + return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); } catch (ZarrException e) { - // This should never happen - throw new RuntimeException(e); + // This should never happen + throw new RuntimeException(e); + } } - } - @Override - public @Nonnull Attributes attributes() throws ZarrException { - if (attributes == null) { - throw new ZarrException("Group attributes have not been set."); + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } + return attributes; } - return attributes; - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 4c125fe..6d7bcb1 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -12,56 +12,55 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; -public interface Node extends dev.zarr.zarrjava.core.Node{ +public interface Node extends dev.zarr.zarrjava.core.Node { - static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); - return objectMapper; - } + static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); + return objectMapper; + } - static ObjectWriter makeObjectWriter() { - return makeObjectMapper().writerWithDefaultPrettyPrinter(); - } + static ObjectWriter makeObjectWriter() { + return makeObjectMapper().writerWithDefaultPrettyPrinter(); + } - /** - * Opens an existing Zarr array or group at a specified storage location. - * - * @param storeHandle the storage location of the Zarr array or group - * @throws IOException throws IOException if the metadata cannot be read - * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened - */ - static Node open(StoreHandle storeHandle) throws IOException, ZarrException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = storeHandle.resolve(ZARR_JSON).readNonNull(); - byte[] metadataBytearray = Utils.toArray(metadataBytes); - String nodeType = objectMapper.readTree(metadataBytearray) - .get("node_type") - .asText(); - switch (nodeType) { - case ArrayMetadata.NODE_TYPE: - return new Array(storeHandle, - objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); - case GroupMetadata.NODE_TYPE: - return new Group(storeHandle, - objectMapper.readValue(metadataBytearray, GroupMetadata.class)); - default: - throw new ZarrException("Unsupported node_type '" + nodeType + "' at " + storeHandle); + /** + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array or group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = storeHandle.resolve(ZARR_JSON).readNonNull(); + byte[] metadataBytearray = Utils.toArray(metadataBytes); + String nodeType = objectMapper.readTree(metadataBytearray) + .get("node_type") + .asText(); + switch (nodeType) { + case ArrayMetadata.NODE_TYPE: + return new Array(storeHandle, + objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); + case GroupMetadata.NODE_TYPE: + return new Group(storeHandle, + objectMapper.readValue(metadataBytearray, GroupMetadata.class)); + default: + throw new ZarrException("Unsupported node_type '" + nodeType + "' at " + storeHandle); + } } - } - static Node open(Path path) throws IOException, ZarrException { - return open(new StoreHandle(new FilesystemStore(path))); - } + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } - static Node open(String path) throws IOException, ZarrException { - return open(Paths.get(path)); - } + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/ChunkGrid.java b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/ChunkGrid.java index a8ab5f6..3e185b8 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/ChunkGrid.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/ChunkGrid.java @@ -4,11 +4,11 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "name") + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "name") @JsonSubTypes({ - @JsonSubTypes.Type(value = RegularChunkGrid.class, name = "regular") + @JsonSubTypes.Type(value = RegularChunkGrid.class, name = "regular") }) public abstract class ChunkGrid { diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java index 0e89769..1867239 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java @@ -3,33 +3,34 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; + import javax.annotation.Nonnull; public class RegularChunkGrid extends ChunkGrid { - @JsonIgnore - public final String name = "regular"; - @Nonnull - public final Configuration configuration; + @JsonIgnore + public final String name = "regular"; + @Nonnull + public final Configuration configuration; - @JsonCreator - public RegularChunkGrid( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration - ) { - this.configuration = configuration; - } + @JsonCreator + public RegularChunkGrid( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; + } - public static final class Configuration { + public static final class Configuration { - @Nonnull - @JsonProperty("chunk_shape") - public final int[] chunkShape; + @Nonnull + @JsonProperty("chunk_shape") + public final int[] chunkShape; - @JsonCreator - public Configuration( - @Nonnull @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape) { - this.chunkShape = chunkShape; + @JsonCreator + public Configuration( + @Nonnull @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape) { + this.chunkShape = chunkShape; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java index cf82ad8..f599e72 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java @@ -6,12 +6,12 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "name") @JsonSubTypes({ - @JsonSubTypes.Type(value = DefaultChunkKeyEncoding.class, name = "default"), - @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2") + @JsonSubTypes.Type(value = DefaultChunkKeyEncoding.class, name = "default"), + @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2") }) public abstract class ChunkKeyEncoding implements dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding { - public abstract String[] encodeChunkKey(long[] chunkCoords); + public abstract String[] encodeChunkKey(long[] chunkCoords); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java index bf85706..83d94c0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java @@ -5,49 +5,49 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import javax.annotation.Nonnull; import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nonnull; public class DefaultChunkKeyEncoding extends ChunkKeyEncoding { - @JsonIgnore - public final String name = "default"; - @Nonnull - public final Configuration configuration; - - @JsonCreator - public DefaultChunkKeyEncoding( - @JsonProperty(value = "configuration") Configuration configuration - ) { - if (configuration == null) { - this.configuration = new Configuration(Separator.SLASH); - } else { - this.configuration = configuration; - } - } - - - @Override - public String[] encodeChunkKey(long[] chunkCoords) { - Stream keys = Stream.concat(Stream.of("c"), Arrays.stream(chunkCoords) - .mapToObj(Long::toString)); - if (configuration.separator == Separator.SLASH) { - return keys.toArray(String[]::new); + @JsonIgnore + public final String name = "default"; + @Nonnull + public final Configuration configuration; + + @JsonCreator + public DefaultChunkKeyEncoding( + @JsonProperty(value = "configuration") Configuration configuration + ) { + if (configuration == null) { + this.configuration = new Configuration(Separator.SLASH); + } else { + this.configuration = configuration; + } + } + + + @Override + public String[] encodeChunkKey(long[] chunkCoords) { + Stream keys = Stream.concat(Stream.of("c"), Arrays.stream(chunkCoords) + .mapToObj(Long::toString)); + if (configuration.separator == Separator.SLASH) { + return keys.toArray(String[]::new); + } + return new String[]{keys.collect(Collectors.joining(this.configuration.separator.getValue()))}; } - return new String[]{keys.collect(Collectors.joining(this.configuration.separator.getValue()))}; - } - public static final class Configuration { + public static final class Configuration { - @Nonnull - public final Separator separator; + @Nonnull + public final Separator separator; - @JsonCreator - public Configuration( - @Nonnull @JsonProperty(value = "separator", defaultValue = "/") Separator separator) { - this.separator = separator; + @JsonCreator + public Configuration( + @Nonnull @JsonProperty(value = "separator", defaultValue = "/") Separator separator) { + this.separator = separator; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java index 85bf19a..2b1e71e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java @@ -5,48 +5,48 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import javax.annotation.Nonnull; import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.annotation.Nonnull; public class V2ChunkKeyEncoding extends ChunkKeyEncoding { - @JsonIgnore - public final String name = "v2"; - @Nonnull - public final Configuration configuration; - - @JsonCreator - public V2ChunkKeyEncoding( - @JsonProperty(value = "configuration") Configuration configuration - ) { - if (configuration == null) { - this.configuration = new Configuration(Separator.DOT); - } else { - this.configuration = configuration; - } - } - - @Override - public String[] encodeChunkKey(long[] chunkCoords) { - Stream keys = Arrays.stream(chunkCoords) - .mapToObj(Long::toString); - if (configuration.separator == Separator.SLASH) { - return keys.toArray(String[]::new); + @JsonIgnore + public final String name = "v2"; + @Nonnull + public final Configuration configuration; + + @JsonCreator + public V2ChunkKeyEncoding( + @JsonProperty(value = "configuration") Configuration configuration + ) { + if (configuration == null) { + this.configuration = new Configuration(Separator.DOT); + } else { + this.configuration = configuration; + } } - return new String[]{keys.collect(Collectors.joining(this.configuration.separator.getValue()))}; - } - public static final class Configuration { + @Override + public String[] encodeChunkKey(long[] chunkCoords) { + Stream keys = Arrays.stream(chunkCoords) + .mapToObj(Long::toString); + if (configuration.separator == Separator.SLASH) { + return keys.toArray(String[]::new); + } + return new String[]{keys.collect(Collectors.joining(this.configuration.separator.getValue()))}; + } - public final Separator separator; + public static final class Configuration { - @JsonCreator - public Configuration( - @Nonnull @JsonProperty(value = "separator", defaultValue = ".") Separator separator) { - this.separator = separator; + public final Separator separator; + + @JsonCreator + public Configuration( + @Nonnull @JsonProperty(value = "separator", defaultValue = ".") Separator separator) { + this.separator = separator; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index e281cde..8bddb8a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -5,10 +5,10 @@ import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.core.*; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration; -import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import java.util.ArrayList; import java.util.Collections; @@ -17,153 +17,155 @@ public class CodecBuilder extends dev.zarr.zarrjava.core.codec.CodecBuilder { - protected List codecs; - public CodecBuilder(DataType dataType) { - super(dataType); - this.codecs = new ArrayList<>(); - } - - public CodecBuilder withBlosc( - Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, - int blockSize - ) { - try { - codecs.add(new BloscCodec( - new BloscCodec.Configuration(cname, shuffle, clevel, typeSize, blockSize))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withBlosc(String cname, String shuffle, int clevel, int blockSize) { - if (shuffle.equals("shuffle")){ - shuffle = "byteshuffle"; - } - return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, - dataType.getByteCount(), blockSize - ); - } - - public CodecBuilder withBlosc(String cname, String shuffle, int clevel) { - return withBlosc(cname, shuffle, clevel, 0); - } - - public CodecBuilder withBlosc(String cname, int clevel) { - return withBlosc(cname, "noshuffle", clevel); - } - - public CodecBuilder withBlosc(String cname) { - return withBlosc(cname, 5); - } - - public CodecBuilder withBlosc() { - return withBlosc("zstd"); - } - - public CodecBuilder withTranspose(int[] order) { - codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); - return this; - } - - public CodecBuilder withBytes(Endian endian) { - if (dataType.getByteCount() <= 1) - codecs.add(new BytesCodec()); - else - codecs.add(new BytesCodec(endian)); - return this; - } - - public CodecBuilder withBytes(String endian) { - return withBytes(BytesCodec.Endian.valueOf(endian)); - } - - public CodecBuilder withBytes() { - return withBytes(Endian.nativeOrder()); - } - - public CodecBuilder withGzip(int clevel) { - try { - codecs.add(new GzipCodec(new GzipCodec.Configuration(clevel))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withGzip() { - return withGzip(5); - } - - public CodecBuilder withZstd(int clevel, boolean checksum) { - try { - codecs.add(new ZstdCodec(new ZstdCodec.Configuration(clevel, checksum))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withZstd() { - return withZstd(5, true); - } - - public CodecBuilder withZstd(int clevel) { - return withZstd(clevel, true); - } - - public CodecBuilder withSharding(int[] chunkShape) { - try { - codecs.add( - new ShardingIndexedCodec(new ShardingIndexedCodec.Configuration(chunkShape, - new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE))}, - new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE)), new Crc32cCodec()}, - "end"))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withSharding(int[] chunkShape, - Function codecBuilder) { - return withSharding(chunkShape, codecBuilder, "end"); - } - - public CodecBuilder withSharding(int[] chunkShape, - Function codecBuilder, String indexLocation) { - CodecBuilder nestedBuilder = new CodecBuilder((DataType) dataType); - try { - codecs.add(new ShardingIndexedCodec( - new ShardingIndexedCodec.Configuration(chunkShape, - codecBuilder.apply(nestedBuilder).build(), - new Codec[]{new BytesCodec(Endian.LITTLE), new Crc32cCodec()}, - indexLocation))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withCrc32c() { - codecs.add(new Crc32cCodec()); - return this; - } - private void autoInsertBytesCodec() { - if (codecs.stream().noneMatch(c -> c instanceof ArrayBytesCodec)) { - Codec[] arrayArrayCodecs = codecs.stream().filter(c -> c instanceof ArrayArrayCodec) - .toArray(Codec[]::new); - Codec[] bytesBytesCodecs = codecs.stream().filter(c -> c instanceof BytesBytesCodec) - .toArray(Codec[]::new); - this.codecs = new ArrayList<>(); - Collections.addAll(this.codecs, arrayArrayCodecs); - this.codecs.add(new BytesCodec(new BytesCodec.Configuration(Endian.LITTLE))); - Collections.addAll(this.codecs, bytesBytesCodecs); - } - } - - public Codec[] build() { - autoInsertBytesCodec(); - return codecs.toArray(new Codec[0]); - } + protected List codecs; + + public CodecBuilder(DataType dataType) { + super(dataType); + this.codecs = new ArrayList<>(); + } + + public CodecBuilder withBlosc( + Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, + int blockSize + ) { + try { + codecs.add(new BloscCodec( + new BloscCodec.Configuration(cname, shuffle, clevel, typeSize, blockSize))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withBlosc(String cname, String shuffle, int clevel, int blockSize) { + if (shuffle.equals("shuffle")) { + shuffle = "byteshuffle"; + } + return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, + dataType.getByteCount(), blockSize + ); + } + + public CodecBuilder withBlosc(String cname, String shuffle, int clevel) { + return withBlosc(cname, shuffle, clevel, 0); + } + + public CodecBuilder withBlosc(String cname, int clevel) { + return withBlosc(cname, "noshuffle", clevel); + } + + public CodecBuilder withBlosc(String cname) { + return withBlosc(cname, 5); + } + + public CodecBuilder withBlosc() { + return withBlosc("zstd"); + } + + public CodecBuilder withTranspose(int[] order) { + codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); + return this; + } + + public CodecBuilder withBytes(Endian endian) { + if (dataType.getByteCount() <= 1) + codecs.add(new BytesCodec()); + else + codecs.add(new BytesCodec(endian)); + return this; + } + + public CodecBuilder withBytes(String endian) { + return withBytes(BytesCodec.Endian.valueOf(endian)); + } + + public CodecBuilder withBytes() { + return withBytes(Endian.nativeOrder()); + } + + public CodecBuilder withGzip(int clevel) { + try { + codecs.add(new GzipCodec(new GzipCodec.Configuration(clevel))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withGzip() { + return withGzip(5); + } + + public CodecBuilder withZstd(int clevel, boolean checksum) { + try { + codecs.add(new ZstdCodec(new ZstdCodec.Configuration(clevel, checksum))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withZstd() { + return withZstd(5, true); + } + + public CodecBuilder withZstd(int clevel) { + return withZstd(clevel, true); + } + + public CodecBuilder withSharding(int[] chunkShape) { + try { + codecs.add( + new ShardingIndexedCodec(new ShardingIndexedCodec.Configuration(chunkShape, + new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE))}, + new Codec[]{new BytesCodec(new Configuration(Endian.LITTLE)), new Crc32cCodec()}, + "end"))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withSharding(int[] chunkShape, + Function codecBuilder) { + return withSharding(chunkShape, codecBuilder, "end"); + } + + public CodecBuilder withSharding(int[] chunkShape, + Function codecBuilder, String indexLocation) { + CodecBuilder nestedBuilder = new CodecBuilder((DataType) dataType); + try { + codecs.add(new ShardingIndexedCodec( + new ShardingIndexedCodec.Configuration(chunkShape, + codecBuilder.apply(nestedBuilder).build(), + new Codec[]{new BytesCodec(Endian.LITTLE), new Crc32cCodec()}, + indexLocation))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withCrc32c() { + codecs.add(new Crc32cCodec()); + return this; + } + + private void autoInsertBytesCodec() { + if (codecs.stream().noneMatch(c -> c instanceof ArrayBytesCodec)) { + Codec[] arrayArrayCodecs = codecs.stream().filter(c -> c instanceof ArrayArrayCodec) + .toArray(Codec[]::new); + Codec[] bytesBytesCodecs = codecs.stream().filter(c -> c instanceof BytesBytesCodec) + .toArray(Codec[]::new); + this.codecs = new ArrayList<>(); + Collections.addAll(this.codecs, arrayArrayCodecs); + this.codecs.add(new BytesCodec(new BytesCodec.Configuration(Endian.LITTLE))); + Collections.addAll(this.codecs, bytesBytesCodecs); + } + } + + public Codec[] build() { + autoInsertBytesCodec(); + return codecs.toArray(new Codec[0]); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index 16b52a1..ad249e0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -8,27 +8,27 @@ public class CodecRegistry { - static Map> map = new HashMap<>(); + static Map> map = new HashMap<>(); - static { - addType("transpose", TransposeCodec.class); - addType("bytes", BytesCodec.class); - addType("blosc", BloscCodec.class); - addType("gzip", GzipCodec.class); - addType("zstd", ZstdCodec.class); - addType("crc32c", Crc32cCodec.class); - addType("sharding_indexed", ShardingIndexedCodec.class); - } + static { + addType("transpose", TransposeCodec.class); + addType("bytes", BytesCodec.class); + addType("blosc", BloscCodec.class); + addType("gzip", GzipCodec.class); + addType("zstd", ZstdCodec.class); + addType("crc32c", Crc32cCodec.class); + addType("sharding_indexed", ShardingIndexedCodec.class); + } - public static void addType(String name, Class codecClass) { - map.put(name, codecClass); - } + public static void addType(String name, Class codecClass) { + map.put(name, codecClass); + } - public static NamedType[] getNamedTypes() { - return map.entrySet() - .stream() - .map(entry -> new NamedType(entry.getValue(), entry.getKey())) - .toArray( - NamedType[]::new); - } + public static NamedType[] getNamedTypes() { + return map.entrySet() + .stream() + .map(entry -> new NamedType(entry.getValue(), entry.getKey())) + .toArray( + NamedType[]::new); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index 2daaec3..9e866ce 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -14,144 +14,144 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; +import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; -import javax.annotation.Nonnull; -public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec{ - - @JsonIgnore - public final String name = "blosc"; - @Nonnull - public final Configuration configuration; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public BloscCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { - this.configuration = configuration; - } - - @Override - public ByteBuffer encode(ByteBuffer chunkBytes) - throws ZarrException { - try { - return ByteBuffer.wrap( - Blosc.compress(Utils.toArray(chunkBytes), configuration.typesize, configuration.cname, - configuration.clevel, - configuration.shuffle, configuration.blocksize - )); - } catch (Exception ex) { - throw new ZarrException("Error in encoding blosc.", ex); - } - } +public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Blosc codec."); - } - - public static final class CustomShuffleSerializer extends StdSerializer { - - public CustomShuffleSerializer() { - super(Blosc.Shuffle.class); - } + @JsonIgnore + public final String name = "blosc"; + @Nonnull + public final Configuration configuration; - public CustomShuffleSerializer(Class t) { - super(t); + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public BloscCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { + this.configuration = configuration; } @Override - public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, - SerializerProvider provider) - throws IOException { - switch (shuffle) { - case NO_SHUFFLE: - generator.writeString("noshuffle"); - break; - case BIT_SHUFFLE: - generator.writeString("bitshuffle"); - break; - case BYTE_SHUFFLE: - generator.writeString("shuffle"); - break; - } + public ByteBuffer encode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap( + Blosc.compress(Utils.toArray(chunkBytes), configuration.typesize, configuration.cname, + configuration.clevel, + configuration.shuffle, configuration.blocksize + )); + } catch (Exception ex) { + throw new ZarrException("Error in encoding blosc.", ex); + } } - } - - public static final class CustomShuffleDeserializer extends StdDeserializer { - public CustomShuffleDeserializer() { - this(null); + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Blosc codec."); } - public CustomShuffleDeserializer(Class vc) { - super(vc); + public static final class CustomShuffleSerializer extends StdSerializer { + + public CustomShuffleSerializer() { + super(Blosc.Shuffle.class); + } + + public CustomShuffleSerializer(Class t) { + super(t); + } + + @Override + public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, + SerializerProvider provider) + throws IOException { + switch (shuffle) { + case NO_SHUFFLE: + generator.writeString("noshuffle"); + break; + case BIT_SHUFFLE: + generator.writeString("bitshuffle"); + break; + case BYTE_SHUFFLE: + generator.writeString("shuffle"); + break; + } + } } - @Override - public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - String shuffle = jsonParser.getCodec() - .readValue(jsonParser, String.class); - switch (shuffle) { - case "noshuffle": - return Blosc.Shuffle.NO_SHUFFLE; - case "bitshuffle": - return Blosc.Shuffle.BIT_SHUFFLE; - case "shuffle": - return Blosc.Shuffle.BYTE_SHUFFLE; - default: - throw new JsonParseException( - jsonParser, - String.format( - "Could not parse the value for Blosc.Shuffle." + " Got '%s'", - shuffle - ) - ); - } + public static final class CustomShuffleDeserializer extends StdDeserializer { + + public CustomShuffleDeserializer() { + this(null); + } + + public CustomShuffleDeserializer(Class vc) { + super(vc); + } + + @Override + public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) + throws IOException { + String shuffle = jsonParser.getCodec() + .readValue(jsonParser, String.class); + switch (shuffle) { + case "noshuffle": + return Blosc.Shuffle.NO_SHUFFLE; + case "bitshuffle": + return Blosc.Shuffle.BIT_SHUFFLE; + case "shuffle": + return Blosc.Shuffle.BYTE_SHUFFLE; + default: + throw new JsonParseException( + jsonParser, + String.format( + "Could not parse the value for Blosc.Shuffle." + " Got '%s'", + shuffle + ) + ); + } + } } - } - - public static final class Configuration { - @Nonnull - @JsonSerialize(using = BloscCodec.CustomCompressorSerializer.class) - public final Blosc.Compressor cname; - @Nonnull - @JsonSerialize(using = BloscCodec.CustomShuffleSerializer.class) - public final Blosc.Shuffle shuffle; - public final int clevel; - public final int typesize; - public final int blocksize; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration( - @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") - @JsonDeserialize(using = BloscCodec.CustomCompressorDeserializer.class) - Blosc.Compressor cname, - @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") - @JsonDeserialize(using = BloscCodec.CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, - @JsonProperty(value = "clevel", defaultValue = "5") int clevel, - @JsonProperty(value = "typesize", defaultValue = "0") int typesize, - @JsonProperty(value = "blocksize", defaultValue = "0") - int blocksize - ) throws ZarrException { - if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { - throw new ZarrException("'typesize' needs to be larger than 0."); - } - if (clevel < 0 || clevel > 9) { - throw new ZarrException("'clevel' needs to be between 0 and 9."); - } - this.cname = cname; - this.shuffle = shuffle; - this.clevel = clevel; - this.typesize = typesize; - this.blocksize = blocksize; + public static final class Configuration { + + @Nonnull + @JsonSerialize(using = BloscCodec.CustomCompressorSerializer.class) + public final Blosc.Compressor cname; + @Nonnull + @JsonSerialize(using = BloscCodec.CustomShuffleSerializer.class) + public final Blosc.Shuffle shuffle; + public final int clevel; + public final int typesize; + public final int blocksize; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration( + @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") + @JsonDeserialize(using = BloscCodec.CustomCompressorDeserializer.class) + Blosc.Compressor cname, + @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") + @JsonDeserialize(using = BloscCodec.CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, + @JsonProperty(value = "clevel", defaultValue = "5") int clevel, + @JsonProperty(value = "typesize", defaultValue = "0") int typesize, + @JsonProperty(value = "blocksize", defaultValue = "0") + int blocksize + ) throws ZarrException { + if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { + throw new ZarrException("'typesize' needs to be larger than 0."); + } + if (clevel < 0 || clevel > 9) { + throw new ZarrException("'clevel' needs to be between 0 and 9."); + } + this.cname = cname; + this.shuffle = shuffle; + this.clevel = clevel; + this.typesize = typesize; + this.blocksize = blocksize; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index ed9380b..5e38075 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -4,60 +4,60 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; -import java.nio.ByteOrder; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.nio.ByteOrder; public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { - @JsonIgnore - public final String name = "bytes"; - @Nullable - public final Configuration configuration; - - @JsonCreator - public BytesCodec( - @JsonProperty(value = "configuration") Configuration configuration - ) { - this.configuration = configuration; - } - - public BytesCodec() { - this((Configuration) null); - } - - public BytesCodec(Endian endian) { - this(new BytesCodec.Configuration(endian)); - } - - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength; - } - - @Override - protected ByteOrder getByteOrder() throws ZarrException { - if (configuration == null) { - throw new ZarrException("BytesCodec configuration is required to determine endianess."); + @JsonIgnore + public final String name = "bytes"; + @Nullable + public final Configuration configuration; + + @JsonCreator + public BytesCodec( + @JsonProperty(value = "configuration") Configuration configuration + ) { + this.configuration = configuration; } - return configuration.endian.getByteOrder(); - } + public BytesCodec() { + this((Configuration) null); + } - public static final class Configuration{ + public BytesCodec(Endian endian) { + this(new BytesCodec.Configuration(endian)); + } - @Nonnull - public final BytesCodec.Endian endian; + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength; + } - @JsonCreator - public Configuration( - @JsonProperty(value = "endian", defaultValue = "little") BytesCodec.Endian endian) { - this.endian = endian; + @Override + protected ByteOrder getByteOrder() throws ZarrException { + if (configuration == null) { + throw new ZarrException("BytesCodec configuration is required to determine endianess."); + } + return configuration.endian.getByteOrder(); + } + + + public static final class Configuration { + + @Nonnull + public final BytesCodec.Endian endian; + + @JsonCreator + public Configuration( + @JsonProperty(value = "endian", defaultValue = "little") BytesCodec.Endian endian) { + this.endian = endian; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 6132921..8a62965 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -3,65 +3,67 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import dev.zarr.zarrjava.utils.CRC32C; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.core.codec.BytesBytesCodec; -import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; + import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class Crc32cCodec extends BytesBytesCodec implements Codec { +public class Crc32cCodec extends BytesBytesCodec implements Codec { - @JsonIgnore - public final String name = "crc32c"; + @JsonIgnore + public final String name = "crc32c"; - @JsonCreator - public Crc32cCodec(){} + @JsonCreator + public Crc32cCodec() { + } - @Override - public ByteBuffer decode(ByteBuffer chunkBytes) - throws ZarrException { - ByteBuffer buffer = chunkBytes.slice(); - buffer.order(ByteOrder.LITTLE_ENDIAN); + @Override + public ByteBuffer decode(ByteBuffer chunkBytes) + throws ZarrException { + ByteBuffer buffer = chunkBytes.slice(); + buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.limit(buffer.capacity() - 4); + buffer.limit(buffer.capacity() - 4); - final CRC32C crc32c = new CRC32C(); - crc32c.update(buffer); - int computedCrc32c = (int) crc32c.getValue(); + final CRC32C crc32c = new CRC32C(); + crc32c.update(buffer); + int computedCrc32c = (int) crc32c.getValue(); - buffer.limit(buffer.capacity()); - int storedCrc32c = buffer.getInt(); + buffer.limit(buffer.capacity()); + int storedCrc32c = buffer.getInt(); - if (computedCrc32c != storedCrc32c) { - throw new ZarrException( - "The checksum of the sharding index is invalid. Stored: " + storedCrc32c + " " - + "Computed: " + - computedCrc32c); + if (computedCrc32c != storedCrc32c) { + throw new ZarrException( + "The checksum of the sharding index is invalid. Stored: " + storedCrc32c + " " + + "Computed: " + + computedCrc32c); + } + buffer.rewind(); + buffer.limit(buffer.capacity() - 4); + return buffer.slice(); } - buffer.rewind(); - buffer.limit(buffer.capacity() - 4); - return buffer.slice(); - } - @Override - public ByteBuffer encode(ByteBuffer chunkBytes) { - return Utils.makeByteBuffer(chunkBytes.capacity() + 4, b -> { - final CRC32C crc32c = new CRC32C(); - crc32c.update(chunkBytes); - int computedCrc32c = (int) crc32c.getValue(); - chunkBytes.rewind(); - b.put(chunkBytes); - b.putInt(computedCrc32c); - return b; - }); - } + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) { + return Utils.makeByteBuffer(chunkBytes.capacity() + 4, b -> { + final CRC32C crc32c = new CRC32C(); + crc32c.update(chunkBytes); + int computedCrc32c = (int) crc32c.getValue(); + chunkBytes.rewind(); + b.put(chunkBytes); + b.putInt(computedCrc32c); + return b; + }); + } - @Override - public long computeEncodedSize(long inputByteLength, - CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength + 4; - } + @Override + public long computeEncodedSize(long inputByteLength, + CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength + 4; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index cb03b26..80ef9d2 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -4,78 +4,78 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.v3.codec.Codec; + +import javax.annotation.Nonnull; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import javax.annotation.Nonnull; public class GzipCodec extends BytesBytesCodec implements Codec { - @JsonIgnore - public final String name = "gzip"; - @Nonnull - public final Configuration configuration; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public GzipCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { - this.configuration = configuration; - } + @JsonIgnore + public final String name = "gzip"; + @Nonnull + public final Configuration configuration; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public GzipCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { + this.configuration = configuration; + } - @Override - public ByteBuffer decode(ByteBuffer chunkBytes) - throws ZarrException { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream( - new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { - Utils.copyStream(inputStream, outputStream); - inputStream.close(); - return ByteBuffer.wrap(outputStream.toByteArray()); - } catch (IOException ex) { - throw new ZarrException("Error in decoding gzip.", ex); + @Override + public ByteBuffer decode(ByteBuffer chunkBytes) + throws ZarrException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream( + new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { + Utils.copyStream(inputStream, outputStream); + inputStream.close(); + return ByteBuffer.wrap(outputStream.toByteArray()); + } catch (IOException ex) { + throw new ZarrException("Error in decoding gzip.", ex); + } } - } - @Override - public ByteBuffer encode(ByteBuffer chunkBytes) - throws ZarrException { - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream( - outputStream)) { - gzipStream.write(Utils.toArray(chunkBytes)); - gzipStream.close(); - return ByteBuffer.wrap(outputStream.toByteArray()); - } catch (IOException ex) { - throw new ZarrException("Error in encoding gzip.", ex); + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) + throws ZarrException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPOutputStream gzipStream = new GZIPOutputStream( + outputStream)) { + gzipStream.write(Utils.toArray(chunkBytes)); + gzipStream.close(); + return ByteBuffer.wrap(outputStream.toByteArray()); + } catch (IOException ex) { + throw new ZarrException("Error in encoding gzip.", ex); + } } - } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Gzip codec."); - } + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Gzip codec."); + } - public static final class Configuration { + public static final class Configuration { - public final int level; + public final int level; - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "level", defaultValue = "5") int level) - throws ZarrException { - if (level < 0 || level > 9) { - throw new ZarrException("'level' needs to be between 0 and 9."); - } - this.level = level; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "level", defaultValue = "5") int level) + throws ZarrException { + if (level < 0 || level > 9) { + throw new ZarrException("'level' needs to be between 0 and 9."); + } + this.level = level; + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 6e598ab..733dd0e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -4,354 +4,355 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; import dev.zarr.zarrjava.v3.DataType; -import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.core.codec.CodecPipeline; +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; + +import javax.annotation.Nonnull; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import javax.annotation.Nonnull; -import ucar.ma2.Array; -import ucar.ma2.InvalidRangeException; public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode implements Codec { - @JsonIgnore - public final String name = "sharding_indexed"; - @Nonnull - public final Configuration configuration; - CodecPipeline codecPipeline; - CodecPipeline indexCodecPipeline; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ShardingIndexedCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) - Configuration configuration - ) throws ZarrException { - this.configuration = configuration; - } - - @Override - public void setCoreArrayMetadata(CoreArrayMetadata arrayMetadata) throws ZarrException { - super.setCoreArrayMetadata(arrayMetadata); - final ArrayMetadata.CoreArrayMetadata shardMetadata = - new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), - configuration.chunkShape, arrayMetadata.dataType, - arrayMetadata.parsedFillValue - ); - this.codecPipeline = new CodecPipeline(configuration.codecs, shardMetadata); - this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs, getShardIndexArrayMetadata(getChunksPerShard(arrayMetadata))); - } - - ArrayMetadata.CoreArrayMetadata getShardIndexArrayMetadata(int[] chunksPerShard) { - int[] indexShape = extendArrayBy1(chunksPerShard, 2); - return new ArrayMetadata.CoreArrayMetadata( - Utils.toLongArray(indexShape), indexShape, DataType.UINT64, -1); - } - - public int[] getChunksPerShard(ArrayMetadata.CoreArrayMetadata arrayMetadata) { - final int ndim = arrayMetadata.ndim(); - final int[] chunksPerShard = new int[ndim]; - for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { - chunksPerShard[dimIdx] = - arrayMetadata.chunkShape[dimIdx] / configuration.chunkShape[dimIdx]; - } - return chunksPerShard; - } - - int[] extendArrayBy1(int[] array, int value) { - int[] out = new int[array.length + 1]; - System.arraycopy(array, 0, out, 0, array.length); - out[out.length - 1] = value; - return out; - } - - long[] extendArrayBy1(long[] array, long value) { - long[] out = new long[array.length + 1]; - System.arraycopy(array, 0, out, 0, array.length); - out[out.length - 1] = value; - return out; - } - - long getValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int idx) { - return shardIndexArray.getLong( - shardIndexArray.getIndex() - .set(Utils.toIntArray(extendArrayBy1(chunkCoords, idx)))); - } - - void setValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int idx, long value) { - shardIndexArray.setLong( - shardIndexArray.getIndex() - .set(Utils.toIntArray(extendArrayBy1(chunkCoords, idx))), value); - } - - @Override - public Array decode(ByteBuffer shardBytes) - throws ZarrException { - return decodeInternal(new ByteBufferDataProvider(shardBytes), new long[arrayMetadata.ndim()], - arrayMetadata.chunkShape, arrayMetadata); - } - - @Override - public ByteBuffer encode(final Array shardArray) throws ZarrException { - final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; - final int[] chunksPerShard = getChunksPerShard(arrayMetadata); - final int chunkCount = Arrays.stream(chunksPerShard) - .reduce(1, (r, a) -> r * a); - - final Array shardIndexArray = Array.factory(ucar.ma2.DataType.ULONG, - extendArrayBy1(chunksPerShard, 2)); - final List chunkBytesList = new ArrayList<>(chunkCount); - - Arrays.stream( - IndexingUtils.computeChunkCoords(shardMetadata.shape, shardMetadata.chunkShape)) - .parallel() - .forEach( - chunkCoords -> { - try { - final int i = - (int) IndexingUtils.cOrderIndex(chunkCoords, Utils.toLongArray(chunksPerShard)); - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, - shardMetadata.chunkShape - ); - final Array chunkArray = - shardArray.sectionNoReduce(chunkProjection.outOffset, chunkProjection.shape, - null - ); - if (MultiArrayUtils.allValuesEqual(chunkArray, shardMetadata.parsedFillValue)) { - setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, -1); - setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, -1); - } else { - final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); - synchronized (chunkBytesList) { - int chunkByteOffset = chunkBytesList.stream() - .mapToInt(ByteBuffer::capacity) - .sum(); - if (configuration.indexLocation.equals("start")) { - chunkByteOffset += (int) getShardIndexSize(arrayMetadata); - } - setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, chunkByteOffset); - setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, - chunkBytes.capacity()); - chunkBytesList.add(chunkBytes); - } - } - } catch (ZarrException | InvalidRangeException e) { - throw new RuntimeException(e); - } - }); - final int shardBytesLength = chunkBytesList.stream() - .mapToInt(ByteBuffer::capacity) - .sum() + (int) getShardIndexSize(arrayMetadata); - final ByteBuffer shardBytes = ByteBuffer.allocate(shardBytesLength); - if(configuration.indexLocation.equals("start")){ - shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); - } - for (final ByteBuffer chunkBytes : chunkBytesList) { - shardBytes.put(chunkBytes); + @JsonIgnore + public final String name = "sharding_indexed"; + @Nonnull + public final Configuration configuration; + CodecPipeline codecPipeline; + CodecPipeline indexCodecPipeline; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ShardingIndexedCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) + Configuration configuration + ) throws ZarrException { + this.configuration = configuration; } - if(configuration.indexLocation.equals("end")){ - shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); + + @Override + public void setCoreArrayMetadata(CoreArrayMetadata arrayMetadata) throws ZarrException { + super.setCoreArrayMetadata(arrayMetadata); + final ArrayMetadata.CoreArrayMetadata shardMetadata = + new ArrayMetadata.CoreArrayMetadata(Utils.toLongArray(arrayMetadata.chunkShape), + configuration.chunkShape, arrayMetadata.dataType, + arrayMetadata.parsedFillValue + ); + this.codecPipeline = new CodecPipeline(configuration.codecs, shardMetadata); + this.indexCodecPipeline = new CodecPipeline(configuration.indexCodecs, getShardIndexArrayMetadata(getChunksPerShard(arrayMetadata))); } - shardBytes.rewind(); - return shardBytes; - } - - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength + getShardIndexSize(arrayMetadata); - } - - private long getShardIndexSize(CoreArrayMetadata arrayMetadata) throws ZarrException { - return indexCodecPipeline.computeEncodedSize( - 16 * (long) Arrays.stream(getChunksPerShard(arrayMetadata)).reduce(1, (r, a) -> r * a), - arrayMetadata - ); - } - - private Array decodeInternal( - DataProvider dataProvider, long[] offset, int[] shape, - ArrayMetadata.CoreArrayMetadata arrayMetadata - ) throws ZarrException { - final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; - - final Array outputArray = Array.factory(arrayMetadata.dataType.getMA2DataType(), shape); - final int shardIndexByteLength = (int) getShardIndexSize(arrayMetadata); - ByteBuffer shardIndexBytes; - if (this.configuration.indexLocation.equals("start")) { - shardIndexBytes = dataProvider.readPrefix(shardIndexByteLength); - }else if(this.configuration.indexLocation.equals("end")){ - shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength); - }else{ - throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); + + ArrayMetadata.CoreArrayMetadata getShardIndexArrayMetadata(int[] chunksPerShard) { + int[] indexShape = extendArrayBy1(chunksPerShard, 2); + return new ArrayMetadata.CoreArrayMetadata( + Utils.toLongArray(indexShape), indexShape, DataType.UINT64, -1); } - if (shardIndexBytes == null) { - throw new ZarrException("Could not read shard index."); + + public int[] getChunksPerShard(ArrayMetadata.CoreArrayMetadata arrayMetadata) { + final int ndim = arrayMetadata.ndim(); + final int[] chunksPerShard = new int[ndim]; + for (int dimIdx = 0; dimIdx < ndim; dimIdx++) { + chunksPerShard[dimIdx] = + arrayMetadata.chunkShape[dimIdx] / configuration.chunkShape[dimIdx]; + } + return chunksPerShard; } - final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); - long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, - shardMetadata.chunkShape, offset, - shape); - - Arrays.stream(allChunkCoords) - // .parallel() - .forEach( - chunkCoords -> { - try { - final long chunkByteOffset = getValueFromShardIndexArray(shardIndexArray, - chunkCoords, 0); - final long chunkByteLength = getValueFromShardIndexArray(shardIndexArray, - chunkCoords, 1); - Array chunkArray = null; - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, - shardMetadata.chunkShape, offset, shape - ); - if (chunkByteOffset != -1 && chunkByteLength != -1) { - final ByteBuffer chunkBytes = dataProvider.read(chunkByteOffset, chunkByteLength); - if (chunkBytes == null) { - throw new ZarrException(String.format("Could not load byte data for chunk %s", - Arrays.toString(chunkCoords))); - } - chunkArray = codecPipeline.decode(chunkBytes); - } - if (chunkArray == null) { - chunkArray = shardMetadata.allocateFillValueChunk(); - } - MultiArrayUtils.copyRegion(chunkArray, chunkProjection.chunkOffset, outputArray, - chunkProjection.outOffset, chunkProjection.shape - ); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }); - - return outputArray; - } - - @Override - public Array decodePartial(StoreHandle chunkHandle, long[] offset, int[] shape) throws ZarrException { - if (Arrays.equals(shape, arrayMetadata.chunkShape)) { - ByteBuffer chunkBytes = chunkHandle.read(); - if (chunkBytes == null) { - return arrayMetadata.allocateFillValueChunk(); - } - return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, arrayMetadata); + + int[] extendArrayBy1(int[] array, int value) { + int[] out = new int[array.length + 1]; + System.arraycopy(array, 0, out, 0, array.length); + out[out.length - 1] = value; + return out; } - return decodeInternal(new StoreHandleDataProvider(chunkHandle), offset, shape, arrayMetadata); - } + long[] extendArrayBy1(long[] array, long value) { + long[] out = new long[array.length + 1]; + System.arraycopy(array, 0, out, 0, array.length); + out[out.length - 1] = value; + return out; + } - interface DataProvider { + long getValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int idx) { + return shardIndexArray.getLong( + shardIndexArray.getIndex() + .set(Utils.toIntArray(extendArrayBy1(chunkCoords, idx)))); + } - ByteBuffer read(long start, long length); + void setValueFromShardIndexArray(Array shardIndexArray, long[] chunkCoords, int idx, long value) { + shardIndexArray.setLong( + shardIndexArray.getIndex() + .set(Utils.toIntArray(extendArrayBy1(chunkCoords, idx))), value); + } - ByteBuffer readSuffix(long suffixLength); + @Override + public Array decode(ByteBuffer shardBytes) + throws ZarrException { + return decodeInternal(new ByteBufferDataProvider(shardBytes), new long[arrayMetadata.ndim()], + arrayMetadata.chunkShape, arrayMetadata); + } - ByteBuffer readPrefix(long prefixLength); - } + @Override + public ByteBuffer encode(final Array shardArray) throws ZarrException { + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; + final int[] chunksPerShard = getChunksPerShard(arrayMetadata); + final int chunkCount = Arrays.stream(chunksPerShard) + .reduce(1, (r, a) -> r * a); + + final Array shardIndexArray = Array.factory(ucar.ma2.DataType.ULONG, + extendArrayBy1(chunksPerShard, 2)); + final List chunkBytesList = new ArrayList<>(chunkCount); + + Arrays.stream( + IndexingUtils.computeChunkCoords(shardMetadata.shape, shardMetadata.chunkShape)) + .parallel() + .forEach( + chunkCoords -> { + try { + final int i = + (int) IndexingUtils.cOrderIndex(chunkCoords, Utils.toLongArray(chunksPerShard)); + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, + shardMetadata.chunkShape + ); + final Array chunkArray = + shardArray.sectionNoReduce(chunkProjection.outOffset, chunkProjection.shape, + null + ); + if (MultiArrayUtils.allValuesEqual(chunkArray, shardMetadata.parsedFillValue)) { + setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, -1); + setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, -1); + } else { + final ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); + synchronized (chunkBytesList) { + int chunkByteOffset = chunkBytesList.stream() + .mapToInt(ByteBuffer::capacity) + .sum(); + if (configuration.indexLocation.equals("start")) { + chunkByteOffset += (int) getShardIndexSize(arrayMetadata); + } + setValueFromShardIndexArray(shardIndexArray, chunkCoords, 0, chunkByteOffset); + setValueFromShardIndexArray(shardIndexArray, chunkCoords, 1, + chunkBytes.capacity()); + chunkBytesList.add(chunkBytes); + } + } + } catch (ZarrException | InvalidRangeException e) { + throw new RuntimeException(e); + } + }); + final int shardBytesLength = chunkBytesList.stream() + .mapToInt(ByteBuffer::capacity) + .sum() + (int) getShardIndexSize(arrayMetadata); + final ByteBuffer shardBytes = ByteBuffer.allocate(shardBytesLength); + if (configuration.indexLocation.equals("start")) { + shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); + } + for (final ByteBuffer chunkBytes : chunkBytesList) { + shardBytes.put(chunkBytes); + } + if (configuration.indexLocation.equals("end")) { + shardBytes.put(indexCodecPipeline.encode(shardIndexArray)); + } + shardBytes.rewind(); + return shardBytes; + } - public static final class Configuration { + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength + getShardIndexSize(arrayMetadata); + } - @JsonProperty("chunk_shape") - public final int[] chunkShape; - @Nonnull - @JsonProperty("codecs") - public final Codec[] codecs; - @Nonnull - @JsonProperty("index_codecs") - public final Codec[] indexCodecs; - @Nonnull - @JsonProperty("index_location") - public String indexLocation; + private long getShardIndexSize(CoreArrayMetadata arrayMetadata) throws ZarrException { + return indexCodecPipeline.computeEncodedSize( + 16 * (long) Arrays.stream(getChunksPerShard(arrayMetadata)).reduce(1, (r, a) -> r * a), + arrayMetadata + ); + } - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration( - @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape, - @Nonnull @JsonProperty("codecs") Codec[] codecs, - @Nonnull @JsonProperty("index_codecs") Codec[] indexCodecs, - @JsonProperty(value = "index_location", defaultValue = "end") String indexLocation + private Array decodeInternal( + DataProvider dataProvider, long[] offset, int[] shape, + ArrayMetadata.CoreArrayMetadata arrayMetadata ) throws ZarrException { - if (indexLocation == null) { - indexLocation = "end"; - } - if (!indexLocation.equals("start") && !indexLocation.equals("end")) { - throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); - } - this.chunkShape = chunkShape; - this.codecs = codecs; - this.indexCodecs = indexCodecs; - this.indexLocation = indexLocation; + final ArrayMetadata.CoreArrayMetadata shardMetadata = codecPipeline.arrayMetadata; + + final Array outputArray = Array.factory(arrayMetadata.dataType.getMA2DataType(), shape); + final int shardIndexByteLength = (int) getShardIndexSize(arrayMetadata); + ByteBuffer shardIndexBytes; + if (this.configuration.indexLocation.equals("start")) { + shardIndexBytes = dataProvider.readPrefix(shardIndexByteLength); + } else if (this.configuration.indexLocation.equals("end")) { + shardIndexBytes = dataProvider.readSuffix(shardIndexByteLength); + } else { + throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); + } + if (shardIndexBytes == null) { + throw new ZarrException("Could not read shard index."); + } + final Array shardIndexArray = indexCodecPipeline.decode(shardIndexBytes); + long[][] allChunkCoords = IndexingUtils.computeChunkCoords(shardMetadata.shape, + shardMetadata.chunkShape, offset, + shape); + + Arrays.stream(allChunkCoords) + // .parallel() + .forEach( + chunkCoords -> { + try { + final long chunkByteOffset = getValueFromShardIndexArray(shardIndexArray, + chunkCoords, 0); + final long chunkByteLength = getValueFromShardIndexArray(shardIndexArray, + chunkCoords, 1); + Array chunkArray = null; + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, shardMetadata.shape, + shardMetadata.chunkShape, offset, shape + ); + if (chunkByteOffset != -1 && chunkByteLength != -1) { + final ByteBuffer chunkBytes = dataProvider.read(chunkByteOffset, chunkByteLength); + if (chunkBytes == null) { + throw new ZarrException(String.format("Could not load byte data for chunk %s", + Arrays.toString(chunkCoords))); + } + chunkArray = codecPipeline.decode(chunkBytes); + } + if (chunkArray == null) { + chunkArray = shardMetadata.allocateFillValueChunk(); + } + MultiArrayUtils.copyRegion(chunkArray, chunkProjection.chunkOffset, outputArray, + chunkProjection.outOffset, chunkProjection.shape + ); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + }); + + return outputArray; } - } - static class ByteBufferDataProvider implements DataProvider { + @Override + public Array decodePartial(StoreHandle chunkHandle, long[] offset, int[] shape) throws ZarrException { + if (Arrays.equals(shape, arrayMetadata.chunkShape)) { + ByteBuffer chunkBytes = chunkHandle.read(); + if (chunkBytes == null) { + return arrayMetadata.allocateFillValueChunk(); + } + return decodeInternal(new ByteBufferDataProvider(chunkHandle.read()), offset, shape, arrayMetadata); + } + return decodeInternal(new StoreHandleDataProvider(chunkHandle), offset, shape, arrayMetadata); + } - @Nonnull - final ByteBuffer buffer; + interface DataProvider { - ByteBufferDataProvider(@Nonnull ByteBuffer buffer) { - this.buffer = buffer; - } + ByteBuffer read(long start, long length); - @Override - public ByteBuffer readSuffix(long suffixLength) { - ByteBuffer bufferSlice = buffer.slice(); - bufferSlice.position((int) (bufferSlice.capacity() - suffixLength)); - return bufferSlice.slice(); - } + ByteBuffer readSuffix(long suffixLength); - public ByteBuffer readPrefix(long prefixLength) { - ByteBuffer bufferSlice = buffer.slice(); - bufferSlice.limit((int) (prefixLength)); - return bufferSlice.slice(); + ByteBuffer readPrefix(long prefixLength); } - @Override - public ByteBuffer read(long start, long length) { - ByteBuffer bufferSlice = buffer.slice(); - bufferSlice.position((int) start); - bufferSlice.limit((int) (start + length)); - return bufferSlice.slice(); + public static final class Configuration { + + @JsonProperty("chunk_shape") + public final int[] chunkShape; + @Nonnull + @JsonProperty("codecs") + public final Codec[] codecs; + @Nonnull + @JsonProperty("index_codecs") + public final Codec[] indexCodecs; + @Nonnull + @JsonProperty("index_location") + public String indexLocation; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration( + @JsonProperty(value = "chunk_shape", required = true) int[] chunkShape, + @Nonnull @JsonProperty("codecs") Codec[] codecs, + @Nonnull @JsonProperty("index_codecs") Codec[] indexCodecs, + @JsonProperty(value = "index_location", defaultValue = "end") String indexLocation + ) throws ZarrException { + if (indexLocation == null) { + indexLocation = "end"; + } + if (!indexLocation.equals("start") && !indexLocation.equals("end")) { + throw new ZarrException("Only index_location \"start\" or \"end\" are supported."); + } + this.chunkShape = chunkShape; + this.codecs = codecs; + this.indexCodecs = indexCodecs; + this.indexLocation = indexLocation; + } } - } - static class StoreHandleDataProvider implements DataProvider { + static class ByteBufferDataProvider implements DataProvider { - @Nonnull - final StoreHandle storeHandle; + @Nonnull + final ByteBuffer buffer; - StoreHandleDataProvider(@Nonnull StoreHandle storeHandle) { - this.storeHandle = storeHandle; - } + ByteBufferDataProvider(@Nonnull ByteBuffer buffer) { + this.buffer = buffer; + } - @Override - public ByteBuffer readSuffix(long suffixLength) { - return storeHandle.read(-suffixLength); - } + @Override + public ByteBuffer readSuffix(long suffixLength) { + ByteBuffer bufferSlice = buffer.slice(); + bufferSlice.position((int) (bufferSlice.capacity() - suffixLength)); + return bufferSlice.slice(); + } - @Override - public ByteBuffer readPrefix(long prefixLength) { - return storeHandle.read(0, prefixLength); + public ByteBuffer readPrefix(long prefixLength) { + ByteBuffer bufferSlice = buffer.slice(); + bufferSlice.limit((int) (prefixLength)); + return bufferSlice.slice(); + } + + @Override + public ByteBuffer read(long start, long length) { + ByteBuffer bufferSlice = buffer.slice(); + bufferSlice.position((int) start); + bufferSlice.limit((int) (start + length)); + return bufferSlice.slice(); + } } - @Override - public ByteBuffer read(long start, long length) { - return storeHandle.read(start, start + length); + static class StoreHandleDataProvider implements DataProvider { + + @Nonnull + final StoreHandle storeHandle; + + StoreHandleDataProvider(@Nonnull StoreHandle storeHandle) { + this.storeHandle = storeHandle; + } + + + @Override + public ByteBuffer readSuffix(long suffixLength) { + return storeHandle.read(-suffixLength); + } + + @Override + public ByteBuffer readPrefix(long prefixLength) { + return storeHandle.read(0, prefixLength); + } + + @Override + public ByteBuffer read(long start, long length) { + return storeHandle.read(start, start + length); + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index c085532..c731546 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -4,9 +4,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; import ucar.ma2.Array; import javax.annotation.Nonnull; @@ -15,7 +15,7 @@ import static dev.zarr.zarrjava.utils.Utils.inversePermutation; import static dev.zarr.zarrjava.utils.Utils.isPermutation; -public class TransposeCodec extends ArrayArrayCodec implements Codec{ +public class TransposeCodec extends ArrayArrayCodec implements Codec { @JsonIgnore @Nonnull @@ -33,7 +33,7 @@ public TransposeCodec( @Override public Array decode(Array chunkArray) throws ZarrException { - if (!isPermutation(configuration.order)){ + if (!isPermutation(configuration.order)) { throw new ZarrException("Order is no permutation array"); } if (arrayMetadata.ndim() != configuration.order.length) { @@ -44,10 +44,9 @@ public Array decode(Array chunkArray) throws ZarrException { } - @Override public Array encode(Array chunkArray) throws ZarrException { - if (!isPermutation(configuration.order)){ + if (!isPermutation(configuration.order)) { throw new ZarrException("Order is no permutation array"); } if (arrayMetadata.ndim() != configuration.order.length) { @@ -63,15 +62,6 @@ public long computeEncodedSize(long inputByteLength, return inputByteLength; } - public static final class Configuration { - public final int[] order; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "order") int[] order) { - this.order = order; - } - } - @Override public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { super.resolveArrayMetadata(); @@ -82,7 +72,7 @@ public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrExcepti //only chunk shape gets transformed, the outer shape stays the same long[] transposedArrayShape = new long[arrayMetadata.ndim()]; - Arrays.setAll(transposedArrayShape, i -> arrayMetadata.shape[i]/arrayMetadata.chunkShape[i]*transposedArrayShape[i]); + Arrays.setAll(transposedArrayShape, i -> arrayMetadata.shape[i] / arrayMetadata.chunkShape[i] * transposedArrayShape[i]); return new ArrayMetadata.CoreArrayMetadata( transposedArrayShape, @@ -91,4 +81,13 @@ public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrExcepti arrayMetadata.parsedFillValue ); } + + public static final class Configuration { + public final int[] order; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "order") int[] order) { + this.order = order; + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index bc5304b..d7599bc 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -6,9 +6,9 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; import javax.annotation.Nonnull; import java.nio.ByteBuffer; diff --git a/src/test/java/dev/zarr/zarrjava/TestUtils.java b/src/test/java/dev/zarr/zarrjava/TestUtils.java index 8165b4a..2a49f6b 100644 --- a/src/test/java/dev/zarr/zarrjava/TestUtils.java +++ b/src/test/java/dev/zarr/zarrjava/TestUtils.java @@ -8,11 +8,10 @@ import static dev.zarr.zarrjava.utils.Utils.inversePermutation; import static dev.zarr.zarrjava.utils.Utils.isPermutation; -import static org.junit.Assert.assertFalse; public class TestUtils { @Test - public void testIsPermutation(){ + public void testIsPermutation() { assert isPermutation(new int[]{2, 1, 0}); assert isPermutation(new int[]{4, 2, 1, 3, 0}); assert !isPermutation(new int[]{0, 1, 2, 0}); @@ -21,7 +20,7 @@ public void testIsPermutation(){ } @Test - public void testInversePermutation(){ + public void testInversePermutation() { Assertions.assertArrayEquals(new int[]{1, 0, 2}, inversePermutation(new int[]{1, 0, 2})); Assertions.assertArrayEquals(new int[]{2, 0, 1}, inversePermutation(new int[]{1, 2, 0})); Assertions.assertArrayEquals(new int[]{0, 3, 2, 4, 1}, inversePermutation(new int[]{0, 4, 2, 1, 3})); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 1a6faad..6186fe8 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -17,13 +17,14 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.stream.Stream; @@ -66,19 +67,13 @@ public static void setupUV() { } } - public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { - int exitCode = runCommand(Stream.concat(Stream.of("uv", "run", PYTHON_TEST_PATH.resolve(scriptName) - .toString()), Arrays.stream(args)).toArray(String[]::new)); - assert exitCode == 0; - } - - static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt){ + static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { ucar.ma2.DataType ma2Type = dt.getMA2DataType(); - ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); + ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); for (int i = 0; i < array.getSize(); i++) { switch (ma2Type) { case BOOLEAN: - array.setBoolean(i, i%2 == 0); + array.setBoolean(i, i % 2 == 0); break; case BYTE: case UBYTE: @@ -151,45 +146,79 @@ static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataT static Stream compressorAndDataTypeProviderV3() { Stream datatypeTests = Stream.of( - DataType.BOOL, - DataType.INT8, - DataType.UINT8, - DataType.INT16, - DataType.UINT16, - DataType.INT32, - DataType.UINT32, - DataType.INT64, - DataType.UINT64, - DataType.FLOAT32, - DataType.FLOAT64 + DataType.BOOL, + DataType.INT8, + DataType.UINT8, + DataType.INT16, + DataType.UINT16, + DataType.INT32, + DataType.UINT32, + DataType.INT64, + DataType.UINT64, + DataType.FLOAT32, + DataType.FLOAT64 ).flatMap(dt -> Stream.of( - new Object[]{"sharding", "end", dt}, - new Object[]{"blosc", "blosclz_shuffle_3", dt} + new Object[]{"sharding", "end", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} )); Stream codecsTests = Stream.of( - new Object[]{"blosc", "blosclz_noshuffle_0", DataType.INT32}, - new Object[]{"blosc", "lz4_shuffle_6", DataType.INT32}, - new Object[]{"blosc", "lz4hc_bitshuffle_3", DataType.INT32}, - new Object[]{"blosc", "zlib_shuffle_5", DataType.INT32}, - new Object[]{"blosc", "zstd_bitshuffle_9", DataType.INT32}, - new Object[]{"gzip", "0", DataType.INT32}, - new Object[]{"gzip", "5", DataType.INT32}, - new Object[]{"zstd", "0_true", DataType.INT32}, - new Object[]{"zstd", "5_true", DataType.INT32}, - new Object[]{"zstd", "0_false", DataType.INT32}, - new Object[]{"zstd", "5_false", DataType.INT32}, - new Object[]{"bytes", "BIG", DataType.INT32}, - new Object[]{"bytes", "LITTLE", DataType.INT32}, - new Object[]{"transpose", "_", DataType.INT32}, - new Object[]{"sharding", "start", DataType.INT32}, - new Object[]{"sharding_nested", "_", DataType.INT32}, - new Object[]{"crc32c", "_", DataType.INT32} + new Object[]{"blosc", "blosclz_noshuffle_0", DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", DataType.INT32}, + new Object[]{"gzip", "0", DataType.INT32}, + new Object[]{"gzip", "5", DataType.INT32}, + new Object[]{"zstd", "0_true", DataType.INT32}, + new Object[]{"zstd", "5_true", DataType.INT32}, + new Object[]{"zstd", "0_false", DataType.INT32}, + new Object[]{"zstd", "5_false", DataType.INT32}, + new Object[]{"bytes", "BIG", DataType.INT32}, + new Object[]{"bytes", "LITTLE", DataType.INT32}, + new Object[]{"transpose", "_", DataType.INT32}, + new Object[]{"sharding", "start", DataType.INT32}, + new Object[]{"sharding_nested", "_", DataType.INT32}, + new Object[]{"crc32c", "_", DataType.INT32} ); return Stream.concat(datatypeTests, codecsTests); } + static Stream compressorAndDataTypeProviderV2() { + Stream datatypeTests = Stream.of( + dev.zarr.zarrjava.v2.DataType.BOOL, + dev.zarr.zarrjava.v2.DataType.INT8, + dev.zarr.zarrjava.v2.DataType.UINT8, + dev.zarr.zarrjava.v2.DataType.INT16, + dev.zarr.zarrjava.v2.DataType.UINT16, + dev.zarr.zarrjava.v2.DataType.INT32, + dev.zarr.zarrjava.v2.DataType.UINT32, + dev.zarr.zarrjava.v2.DataType.INT64, + dev.zarr.zarrjava.v2.DataType.UINT64, + dev.zarr.zarrjava.v2.DataType.FLOAT32, + dev.zarr.zarrjava.v2.DataType.FLOAT64 + ).flatMap(dt -> Stream.of( + new Object[]{"zlib", "0", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} + )); + + Stream bloscTests = Stream.of( + new Object[]{"blosc", "blosclz_noshuffle_0", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", dev.zarr.zarrjava.v2.DataType.INT32} + ); + + return Stream.concat(datatypeTests, bloscTests); + } + + public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + int exitCode = runCommand(Stream.concat(Stream.of("uv", "run", PYTHON_TEST_PATH.resolve(scriptName) + .toString()), Arrays.stream(args)).toArray(String[]::new)); + assert exitCode == 0; + } @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV3") @@ -215,11 +244,11 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(dataType) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withAttributes(attributes); + .withShape(16, 16, 16) + .withDataType(dataType) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withAttributes(attributes); switch (codec) { case "blosc": @@ -273,36 +302,6 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro run_python_script("zarr_python_read.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); } - - static Stream compressorAndDataTypeProviderV2() { - Stream datatypeTests = Stream.of( - dev.zarr.zarrjava.v2.DataType.BOOL, - dev.zarr.zarrjava.v2.DataType.INT8, - dev.zarr.zarrjava.v2.DataType.UINT8, - dev.zarr.zarrjava.v2.DataType.INT16, - dev.zarr.zarrjava.v2.DataType.UINT16, - dev.zarr.zarrjava.v2.DataType.INT32, - dev.zarr.zarrjava.v2.DataType.UINT32, - dev.zarr.zarrjava.v2.DataType.INT64, - dev.zarr.zarrjava.v2.DataType.UINT64, - dev.zarr.zarrjava.v2.DataType.FLOAT32, - dev.zarr.zarrjava.v2.DataType.FLOAT64 - ).flatMap(dt -> Stream.of( - new Object[]{"zlib", "0", dt}, - new Object[]{"blosc", "blosclz_shuffle_3", dt} - )); - - Stream bloscTests = Stream.of( - new Object[]{"blosc", "blosclz_noshuffle_0", dev.zarr.zarrjava.v2.DataType.INT32}, - new Object[]{"blosc", "lz4_shuffle_6", dev.zarr.zarrjava.v2.DataType.INT32}, - new Object[]{"blosc", "lz4hc_bitshuffle_3", dev.zarr.zarrjava.v2.DataType.INT32}, - new Object[]{"blosc", "zlib_shuffle_5", dev.zarr.zarrjava.v2.DataType.INT32}, - new Object[]{"blosc", "zstd_bitshuffle_9", dev.zarr.zarrjava.v2.DataType.INT32} - ); - - return Stream.concat(datatypeTests, bloscTests); - } - @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV2") public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException { @@ -329,11 +328,11 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(dt) - .withChunks(2, 4, 8) - .withAttributes(attributes) - .withFillValue(0); + .withShape(16, 16, 16) + .withDataType(dt) + .withChunks(2, 4, 8) + .withAttributes(attributes) + .withFillValue(0); switch (compressor) { case "blosc": @@ -391,11 +390,11 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException //decompress in python int exitCode = ZarrPythonTests.runCommand( - "uv", - "run", - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) + "uv", + "run", + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) ); assert exitCode == 0; } @@ -410,7 +409,7 @@ public void testGroupReadWriteV2() throws Exception { .withShape(16, 16, 16) .withDataType(dataType) .withChunks(2, 4, 8) - ); + ); array.write(testdata(dataType)); @@ -437,7 +436,7 @@ public void testGroupReadWriteV3() throws Exception { .withShape(16, 16, 16) .withDataType(dataType) .withChunkShape(2, 4, 8) - ); + ); array.write(testdata(dataType)); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index e771b86..cbb21f8 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -2,9 +2,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.Attributes; -import dev.zarr.zarrjava.store.*; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.MemoryStore; +import dev.zarr.zarrjava.store.S3Store; import dev.zarr.zarrjava.v3.*; -import java.net.URI; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -15,6 +17,7 @@ import software.amazon.awssdk.services.s3.S3Configuration; import java.io.IOException; +import java.net.URI; import java.nio.file.Files; import java.util.Arrays; import java.util.stream.Stream; @@ -28,8 +31,8 @@ public void testFileSystemStores() throws IOException, ZarrException { ObjectMapper objectMapper = makeObjectMapper(); GroupMetadata groupMetadata = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class ); String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); @@ -37,8 +40,8 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); @@ -63,23 +66,23 @@ public void testFileSystemStores() throws IOException, ZarrException { @Test public void testS3Store() throws IOException, ZarrException { S3Store s3Store = new S3Store(S3Client.builder() - .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) - .region(Region.US_EAST_1) // required, but ignored - .serviceConfiguration( - S3Configuration.builder() - .pathStyleAccessEnabled(true) // required - .build() - ) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "idr", "zarr/v0.5/idr0033A"); + .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) + .region(Region.US_EAST_1) // required, but ignored + .serviceConfiguration( + S3Configuration.builder() + .pathStyleAccessEnabled(true) // required + .build() + ) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "idr", "zarr/v0.5/idr0033A"); Array arrayV3 = Array.open(s3Store.resolve("BR00109990_C2.zarr", "0", "0")); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayV3.metadata().shape); - Assertions.assertEquals(574, arrayV3.read(new long[]{0,0,0}, new int[]{1,1,1}).getInt(0)); + Assertions.assertEquals(574, arrayV3.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(s3Store.resolve("BR00109990_C2.zarr", "0", "0")); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayCore.metadata().shape); - Assertions.assertEquals(574, arrayCore.read(new long[]{0,0,0}, new int[]{1,1,1}).getInt(0)); + Assertions.assertEquals(574, arrayCore.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); } @Test diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 8a6309d..7657b83 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -23,12 +23,12 @@ public static void clearTestoutputFolder() throws IOException { if (Files.exists(TESTOUTPUT)) { try (Stream walk = Files.walk(TESTOUTPUT)) { walk.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(file -> { - if (!file.delete()) { - throw new RuntimeException("Failed to delete file: " + file.getAbsolutePath()); - } - }); + .map(Path::toFile) + .forEach(file -> { + if (!file.delete()) { + throw new RuntimeException("Failed to delete file: " + file.getAbsolutePath()); + } + }); } } Files.createDirectory(TESTOUTPUT); @@ -70,12 +70,12 @@ protected Attributes defaultTestAttributes() { put("element", "value"); }}); put("array_of_attributes", new Attributes[]{ - new Attributes() {{ - put("a", 1); - }}, - new Attributes() {{ - put("b", 2); - }} + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} }); }}; } @@ -100,15 +100,15 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx Assertions.assertArrayEquals(new boolean[]{true, false, true}, attributes.getBooleanArray("boolean_array")); Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); Assertions.assertArrayEquals( - new Attributes[]{ - new Attributes() {{ - put("a", 1); - }}, - new Attributes() {{ - put("b", 2); - }} - }, - attributes.getArray("array_of_attributes", Attributes.class) + new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }, + attributes.getArray("array_of_attributes", Attributes.class) ); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 346fd2a..084d831 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,8 +1,8 @@ package dev.zarr.zarrjava; -import dev.zarr.zarrjava.core.Attributes; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -28,18 +28,26 @@ import static dev.zarr.zarrjava.core.Node.ZARRAY; public class ZarrV2Test extends ZarrTest { + static Stream> compressorBuilder() { + return Stream.of( + ArrayMetadataBuilder::withBloscCompressor, + ArrayMetadataBuilder::withZlibCompressor, + b -> b + ); + } + @ParameterizedTest @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), - Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) - .withFillValue(1) - .withBloscCompressor(cname, shuffle, clevel) - .build() + new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), + Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withFillValue(1) + .withBloscCompressor(cname, shuffle, clevel) + .build() ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); @@ -50,7 +58,7 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE @ParameterizedTest @CsvSource({ - "BOOL", "FLOAT64" + "BOOL", "FLOAT64" }) public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrException { String arrayname = dt == DataType.BOOL ? "bool" : "double"; @@ -65,13 +73,13 @@ public void testCreate() throws IOException, ZarrException { DataType dataType = DataType.UINT32; Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create"), - Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dataType) - .withChunks(5, 5) - .withFillValue(2) - .build() + new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dataType) + .withChunks(5, 5) + .withFillValue(2) + .build() ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); @@ -84,14 +92,14 @@ public void testCreate() throws IOException, ZarrException { @ValueSource(ints = {0, 1, 5, 9}) public void testCreateZlib(int level) throws IOException, ZarrException { Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), - Array.metadataBuilder() - .withShape(15, 10) - .withDataType(DataType.UINT8) - .withChunks(4, 5) - .withFillValue(5) - .withZlibCompressor(level) - .build() + new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), + Array.metadataBuilder() + .withShape(15, 10) + .withDataType(DataType.UINT8) + .withChunks(4, 5) + .withFillValue(5) + .withZlibCompressor(level) + .build() ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); @@ -106,12 +114,12 @@ public void testNoFillValue(DataType dataType) throws IOException, ZarrException StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); Array array = Array.create( - storeHandle, - Array.metadataBuilder() - .withShape(15, 10) - .withDataType(dataType) - .withChunks(4, 5) - .build() + storeHandle, + Array.metadataBuilder() + .withShape(15, 10) + .withDataType(dataType) + .withChunks(4, 5) + .build() ); Assertions.assertNull(array.metadata().fillValue); @@ -123,12 +131,11 @@ public void testNoFillValue(DataType dataType) throws IOException, ZarrException } Array array2 = Array.open( - storeHandle + storeHandle ); Assertions.assertNull(array2.metadata().fillValue); } - @Test public void testOpen() throws ZarrException, IOException { StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); @@ -236,16 +243,15 @@ public void testGroup() throws IOException, ZarrException { Group group = Group.create(fsStore.resolve("v2_testgroup")); Group group2 = group.createGroup("test2"); Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunks); } - @Test public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV2"); @@ -271,15 +277,15 @@ public void testAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) - .putAttribute("specific", "attribute") - .withAttributes(defaultTestAttributes()) - .withAttributes(new Attributes() {{ - put("another", "attribute"); - }}) - .build(); + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); Array array = Array.create(storeHandle, arrayMetadata); assertContainsTestAttributes(array.metadata().attributes()); @@ -297,11 +303,11 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV2"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) - .withAttributes(new Attributes(b -> b.set("some", "value"))) - .build(); + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("some", "value"))) + .build(); Array array = Array.create(storeHandle, arrayMetadata); Assertions.assertEquals("value", array.metadata().attributes().getString("some")); @@ -332,11 +338,11 @@ public void testResizeArray() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV2"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT32) - .withChunks(5, 5) - .withFillValue(1) - .build(); + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(1) + .build(); ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); Array array = Array.create(storeHandle, arrayMetadata); array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); @@ -367,15 +373,6 @@ public void testGroupAttributes() throws IOException, ZarrException { Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); } - - static Stream> compressorBuilder() { - return Stream.of( - ArrayMetadataBuilder::withBloscCompressor, - ArrayMetadataBuilder::withZlibCompressor, - b -> b - ); - } - @ParameterizedTest @MethodSource("compressorBuilder") public void testZarrJsonFormat(Function compressorBuilder) throws ZarrException, IOException { @@ -406,7 +403,7 @@ public void testMemoryStore() throws ZarrException, IOException { ); group.createGroup("subgroup"); Assertions.assertEquals(2, group.list().count()); - for(String s: storeHandle.list().toArray(String[]::new)) + for (String s : storeHandle.list().toArray(String[]::new)) System.out.println(s); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 54fded2..9a669be 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,17 +1,20 @@ package dev.zarr.zarrjava; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.core.Attributes; - -import com.fasterxml.jackson.databind.JsonMappingException; -import dev.zarr.zarrjava.store.*; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.Node; import dev.zarr.zarrjava.v3.*; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.*; +import dev.zarr.zarrjava.v3.codec.core.BloscCodec; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -21,7 +24,8 @@ import org.junit.jupiter.params.provider.ValueSource; import ucar.ma2.MAMath; -import java.io.*; +import java.io.BufferedReader; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -41,22 +45,76 @@ public class ZarrV3Test extends ZarrTest { static Stream> invalidCodecBuilder() { return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + static Stream invalidChunkSizes() { + return Stream.of( + new int[]{1}, + new int[]{1, 1, 1} + ); + } + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension ); } + static Stream invalidShardSizesWithNested() { + return invalidShardSizes().flatMap(shardSize -> + Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) + ); + } + + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } + + static Stream> codecBuilders() { + return Stream.of( + CodecBuilder::withBlosc, + c -> c.withTranspose(new int[]{1, 0}), + CodecBuilder::withBytes, + CodecBuilder::withGzip, + CodecBuilder::withZstd, + c -> c.withSharding(new int[]{2, 2}), + CodecBuilder::withCrc32c + ); + } + + static Stream> chunkKeyEncodingsAndCodecs() { + Stream> builders = Stream.of( + ArrayMetadataBuilder::withDefaultChunkKeyEncoding, + ArrayMetadataBuilder::withV2ChunkKeyEncoding + ); + + return Stream.concat(builders, codecBuilders().map(codecFunc -> b -> b.withCodecs(codecFunc))); + } + @ParameterizedTest @MethodSource("invalidCodecBuilder") public void testCheckInvalidCodecConfiguration(Function codecBuilder) { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(4, 4) - .withDataType(DataType.UINT32) - .withChunkShape(2, 2) - .withCodecs(codecBuilder); + .withShape(4, 4) + .withDataType(DataType.UINT32) + .withChunkShape(2, 2) + .withCodecs(codecBuilder); assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); } @@ -68,11 +126,11 @@ public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); ArrayMetadata metadata = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(32, 32, 32) - .withFillValue(0) - .build(); + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(32, 32, 32) + .withFillValue(0) + .build(); Array writeArray = Array.create(storeHandle, metadata); writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); @@ -83,41 +141,19 @@ public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); } - static Stream invalidChunkSizes() { - return Stream.of( - new int[]{1}, - new int[]{1, 1, 1} - ); - } - @ParameterizedTest @MethodSource("invalidChunkSizes") public void testCheckInvalidChunkDimensions(int[] chunkSize) { long[] shape = new long[]{4, 4}; ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32) - .withChunkShape(chunkSize); + .withShape(shape) + .withDataType(DataType.UINT32) + .withChunkShape(chunkSize); assertThrows(ZarrException.class, builder::build); } - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - static Stream invalidShardSizesWithNested() { - return invalidShardSizes().flatMap(shardSize -> - Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) - ); - } @ParameterizedTest @MethodSource("invalidShardSizesWithNested") public void testCheckShardingBounds(int[] shardSize, boolean nested) { @@ -125,8 +161,8 @@ public void testCheckShardingBounds(int[] shardSize, boolean nested) { int[] innerChunkSize = new int[]{2, 2}; ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); if (nested) { int[] nestedChunkSize = new int[]{4, 4}; @@ -144,11 +180,11 @@ public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrExce StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(clevel, checksum)); + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); Array writeArray = Array.create(storeHandle, builder.build()); writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); @@ -161,30 +197,21 @@ public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrExce @Test public void testTransposeCodec() throws ZarrException { ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); } - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - @ParameterizedTest @MethodSource("invalidTransposeOrder") public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { @@ -193,10 +220,10 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); + shapeLong, + shapeInt, + DataType.UINT32, + null)); ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); @@ -216,14 +243,14 @@ public void testAccess() throws IOException, ZarrException { Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); + .withShape(1, 64, 64, 64) + .read(); Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); Assertions.assertEquals(-98, outArray.getByte(0)); Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata() + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata() ); writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); } @@ -232,11 +259,11 @@ public void testAccess() throws IOException, ZarrException { @ValueSource(strings = {"start", "end"}) public void testShardingReadWrite(String indexLocation) throws IOException, ZarrException { Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); ucar.ma2.Array readArrayContent = readArray.read(); Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata() + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata() ); writeArray.write(readArrayContent); ucar.ma2.Array outArray = writeArray.read(); @@ -248,12 +275,12 @@ public void testShardingReadWrite(String indexLocation) throws IOException, Zarr public void testCodecs() throws IOException, ZarrException { int[] readShape = new int[]{1, 1, 1024, 1024}; Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); { Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() ); gzipArray.write(readArrayContent); ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); @@ -261,8 +288,8 @@ public void testCodecs() throws IOException, ZarrException { } { Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() ); bloscArray.write(readArrayContent); ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); @@ -270,8 +297,8 @@ public void testCodecs() throws IOException, ZarrException { } { Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() ); zstdArray.write(readArrayContent); ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); @@ -287,13 +314,13 @@ public void testArrayMetadataBuilder() throws ZarrException { int fillValue = 0; ArrayMetadata metadata = Array.metadataBuilder() - .withShape(shape) - .withDataType(dataType) - .withChunkShape(chunkShape) - .withFillValue(fillValue) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) - .build(); + .withShape(shape) + .withDataType(dataType) + .withChunkShape(chunkShape) + .withFillValue(fillValue) + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); Assertions.assertArrayEquals(shape, metadata.shape); Assertions.assertEquals(dataType, metadata.dataType); Assertions.assertArrayEquals(chunkShape, metadata.chunkShape()); @@ -317,14 +344,14 @@ public void testFillValue() throws ZarrException { @Test public void testReadme1() throws IOException, ZarrException { Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") ); Group color = (Group) hierarchy.get("color"); Array array = (Array) color.get("1"); ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape ); Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); } @@ -332,19 +359,19 @@ public void testReadme1() throws IOException, ZarrException { @Test public void testReadme2() throws IOException, ZarrException { Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() + new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() ); ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); array.write( - new long[]{0, 0, 0, 0}, // offset - data + new long[]{0, 0, 0, 0}, // offset + data ); ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); assert MultiArrayUtils.allValuesEqual(data, output); @@ -390,11 +417,11 @@ public void testParallel(boolean useParallel) throws IOException, ZarrException StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead", useParallel ? "parallel" : "serial"); ArrayMetadata metadata = Array.metadataBuilder() - .withShape(512, 512, 512) - .withDataType(DataType.UINT32) - .withChunkShape(100, 100, 100) - .withFillValue(0) - .build(); + .withShape(512, 512, 512) + .withDataType(DataType.UINT32) + .withChunkShape(100, 100, 100) + .withFillValue(0) + .build(); Array writeArray = Array.create(storeHandle, metadata); writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); @@ -409,25 +436,25 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I // non-empty storage transformers are currently not supported Map[] storageTransformersEmpty = Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") ).metadata().storageTransformers; assert storageTransformersEmpty.length == 0; assertThrows(JsonMappingException.class, () -> Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) ); ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap() {{ - put("some", "value"); - }}}); + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap() {{ + put("some", "value"); + }}}); assertThrows(ZarrException.class, () -> Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - builderWithStorageTransformer.build() + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() )); } @@ -520,9 +547,9 @@ public void testGroup() throws IOException, ZarrException { Group group = Group.create(fsStore.resolve("testgroup")); Group group2 = group.createGroup("test2", attributes); Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); @@ -550,7 +577,6 @@ public void testCreateArray() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); } - @Test public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); @@ -577,37 +603,37 @@ public void testAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - .putAttribute("specific", "attribute") - .withAttributes(defaultTestAttributes()) - .withAttributes(new Attributes() {{ - put("another", "attribute"); - }}) + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) - .build(); + .build(); - Array array = Array.create(storeHandle, arrayMetadata); - assertContainsTestAttributes(array.metadata().attributes()); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); - Array arrayOpened = Array.open(storeHandle); - assertContainsTestAttributes(arrayOpened.metadata().attributes()); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(arrayOpened.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); } @Test public void testCodecWithoutConfiguration() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecWithoutConfigurationV3"); Array array = Array.create(storeHandle, Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - .withCodecs(CodecBuilder::withBytes) - .build() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withCodecs(CodecBuilder::withBytes) + .build() ); Assertions.assertTrue(storeHandle.resolve(ZARR_JSON).exists()); Codec bytesCodec = array.metadata().codecs[0]; @@ -615,27 +641,6 @@ public void testCodecWithoutConfiguration() throws ZarrException, IOException { Assertions.assertNull(((BytesCodec) bytesCodec).configuration); } - static Stream> codecBuilders() { - return Stream.of( - CodecBuilder::withBlosc, - c -> c.withTranspose(new int[]{1, 0}), - CodecBuilder::withBytes, - CodecBuilder::withGzip, - CodecBuilder::withZstd, - c -> c.withSharding(new int[]{2, 2}), - CodecBuilder::withCrc32c - ); - } - - static Stream> chunkKeyEncodingsAndCodecs() { - Stream> builders = Stream.of( - ArrayMetadataBuilder::withDefaultChunkKeyEncoding, - ArrayMetadataBuilder::withV2ChunkKeyEncoding - ); - - return Stream.concat(builders, codecBuilders().map(codecFunc -> b -> b.withCodecs(codecFunc))); - } - @ParameterizedTest @MethodSource("chunkKeyEncodingsAndCodecs") public void testZarrJsonFormat(Function chunkKeyEncodingsAndCodecs) throws ZarrException, IOException { @@ -661,11 +666,11 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - .withAttributes(new Attributes(b -> b.set("some", "value"))) - .build(); + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.set("some", "value"))) + .build(); Array array = Array.create(storeHandle, arrayMetadata); Assertions.assertEquals("value", array.metadata().attributes().getString("some")); @@ -696,11 +701,11 @@ public void testResizeArray() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV3"); ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT32) - .withChunkShape(5, 5) - .withFillValue(1) - .build(); + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(1) + .build(); ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); Array array = Array.create(storeHandle, arrayMetadata); array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index 772b34d..edc4fd1 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -1,3 +1,4 @@ +import numcodecs import zarr from zarr.codecs.blosc import BloscCodec from zarr.codecs.bytes import BytesCodec @@ -6,7 +7,7 @@ from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -import numcodecs + def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: int = 3): compressor = None @@ -40,14 +41,14 @@ def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: filters = [TransposeCodec(order=(1, 0, 2))] elif codec_string == "sharding" and zarr_version == 3: serializer = ShardingCodec(chunk_shape=(2, 2, 4), codecs=(BytesCodec(endian="little"),), - index_location=ShardingCodecIndexLocation.start if param_string == "start" - else ShardingCodecIndexLocation.end) + index_location=ShardingCodecIndexLocation.start if param_string == "start" + else ShardingCodecIndexLocation.end) elif codec_string == "sharding_nested" and zarr_version == 3: serializer = ShardingCodec(chunk_shape=(2, 2, 4), codecs=(ShardingCodec(chunk_shape=(2, 1, 2), - codecs=[BytesCodec(endian="little")]),)) + codecs=[BytesCodec(endian="little")]),)) elif codec_string == "crc32c" and zarr_version == 3: compressor = Crc32cCodec() else: raise ValueError(f"Invalid codec: {codec_string}, zarr_version: {zarr_version}") - return compressor, serializer, filters \ No newline at end of file + return compressor, serializer, filters diff --git a/src/test/python-scripts/zarr_python_group.py b/src/test/python-scripts/zarr_python_group.py index 3aa76c7..e5181a2 100644 --- a/src/test/python-scripts/zarr_python_group.py +++ b/src/test/python-scripts/zarr_python_group.py @@ -1,15 +1,13 @@ -import sys -from pathlib import Path - import numpy as np - +import sys import zarr +from pathlib import Path from zarr.storage import LocalStore store_path_read = Path(sys.argv[1]) store_path_write = Path(sys.argv[2]) zarr_format = int(sys.argv[3]) -assert zarr_format in (2,3), f"unexpected zarr format: {zarr_format}" +assert zarr_format in (2, 3), f"unexpected zarr format: {zarr_format}" expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) @@ -28,4 +26,4 @@ dtype="int32", fill_value=0, ) -a2[:] = expected_data \ No newline at end of file +a2[:] = expected_data diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py index d6cee0c..411f70d 100644 --- a/src/test/python-scripts/zarr_python_read.py +++ b/src/test/python-scripts/zarr_python_read.py @@ -1,10 +1,9 @@ -import sys -from pathlib import Path - import numpy as np - +import sys import zarr +from pathlib import Path from zarr.storage import LocalStore + from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5d0b1c6..2655231 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -1,11 +1,9 @@ -import sys -from pathlib import Path - import numpy as np - +import sys import zarr - +from pathlib import Path from zarr.storage import LocalStore + from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py index 328400f..9b3b36e 100644 --- a/src/test/python-scripts/zarr_python_write.py +++ b/src/test/python-scripts/zarr_python_write.py @@ -1,10 +1,9 @@ -import sys -from pathlib import Path - import numpy as np - +import sys import zarr +from pathlib import Path from zarr.storage import LocalStore + from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -18,7 +17,6 @@ else: testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) - a = zarr.create_array( LocalStore(store_path), shape=(16, 16, 16), diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py index 0068e04..f362b0c 100644 --- a/src/test/python-scripts/zarr_python_write_v2.py +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -1,11 +1,9 @@ -import sys -from pathlib import Path - import numpy as np - +import sys import zarr - +from pathlib import Path from zarr.storage import LocalStore + from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1]