Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ target/
!**/src/test/**/target/

### IntelliJ IDEA ###
.idea/
.idea/*
!.idea/codeStyles/
!.idea/codeStyles/**
*.iws
*.iml
*.ipr
Expand Down Expand Up @@ -43,4 +45,4 @@ build/
/main.py
/pyproject.toml
/uv.lock
**/__pycache__
**/__pycache__
5 changes: 5 additions & 0 deletions .idea/codeStyles/codeStyleConfig.xml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

47 changes: 31 additions & 16 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
This repository contains a Java implementation of Zarr version 2 and 3.

## Usage

```java
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.HttpStore;
Expand All @@ -11,40 +12,54 @@ import dev.zarr.zarrjava.v3.DataType;
import dev.zarr.zarrjava.v3.Group;

Group hierarchy = Group.open(
new HttpStore("https://static.webknossos.org/data/zarr_v3")
.resolve("l4_sample")
new HttpStore("https://static.webknossos.org/data/zarr_v3")
.resolve("l4_sample")
);
Group color = (Group) hierarchy.get("color");
Array array = (Array) color.get("1");
ucar.ma2.Array outArray = array.read(
new long[]{0, 3073, 3073, 513}, // offset
new int[]{1, 64, 64, 64} // shape
new long[]{0, 3073, 3073, 513}, // offset
new int[]{1, 64, 64, 64} // shape
);

Array array = Array.create(
new FilesystemStore("/path/to/zarr").resolve("array"),
Array.metadataBuilder()
.withShape(1, 4096, 4096, 1536)
.withDataType(DataType.UINT32)
.withChunkShape(1, 1024, 1024, 1024)
.withFillValue(0)
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
.build()
new FilesystemStore("/path/to/zarr").resolve("array"),
Array.metadataBuilder()
.withShape(1, 4096, 4096, 1536)
.withDataType(DataType.UINT32)
.withChunkShape(1, 1024, 1024, 1024)
.withFillValue(0)
.withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc()))
.build()
);
ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1024, 1024, 1024});
array.write(
new long[]{0, 0, 0, 0}, // offset
data
array.

write(
new long[] {
0, 0, 0, 0
}, // offset
data
);
```

## Development Start-Guide

### Run Tests Locally
To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.

To be able to run the tests locally, make sure to have `python3.11` and `uv` installed.

Furthermore, you will need the `l4_sample` test data:

`curl https://static.webknossos.org/data/zarr_v3/l4_sample.zip -o testdata/l4_sample.zip
&& cd testdata
&& unzip l4_sample.zip
`

### Code Style & Formatting

This project uses IntelliJ IDEA default Java formatting

Before submitting changes, please run:

- IntelliJ: `Reformat Code` and `Optimize Imports`
24 changes: 12 additions & 12 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,10 @@
<version>3.5.0</version>
<executions>
<execution>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
<id>attach-javadoc</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
<configuration>
Expand All @@ -210,14 +210,14 @@
<version>1.6</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
<configuration>
<keyname>9F88D86AD9A0D91E</keyname>
</configuration>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
<configuration>
<keyname>9F88D86AD9A0D91E</keyname>
</configuration>
</execution>
</executions>
</plugin>
Expand Down
12 changes: 6 additions & 6 deletions src/main/java/dev/zarr/zarrjava/ZarrException.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

public class ZarrException extends Exception {

public ZarrException(String message, Throwable cause) {
super(message, cause);
}
public ZarrException(String message, Throwable cause) {
super(message, cause);
}

public ZarrException(String message) {
super(message);
}
public ZarrException(String message) {
super(message);
}
}
130 changes: 66 additions & 64 deletions src/main/java/dev/zarr/zarrjava/core/Array.java
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package dev.zarr.zarrjava.core;

import dev.zarr.zarrjava.ZarrException;
import dev.zarr.zarrjava.core.codec.CodecPipeline;
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.StoreHandle;
import dev.zarr.zarrjava.utils.IndexingUtils;
import dev.zarr.zarrjava.utils.MultiArrayUtils;
import dev.zarr.zarrjava.utils.Utils;
import dev.zarr.zarrjava.core.codec.CodecPipeline;
import ucar.ma2.InvalidRangeException;

import javax.annotation.Nonnull;
Expand All @@ -21,7 +21,6 @@
public abstract class Array extends AbstractNode {

protected CodecPipeline codecPipeline;
public abstract ArrayMetadata metadata();

protected Array(StoreHandle storeHandle) throws ZarrException {
super(storeHandle);
Expand All @@ -47,7 +46,8 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
throw new ZarrException("No Zarr array found at the specified location.");
}
}
/**

/**
* Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version.
*
* @param path the storage location of the Zarr array
Expand All @@ -69,6 +69,8 @@ public static Array open(String path) throws IOException, ZarrException {
return open(Paths.get(path));
}

public abstract ArrayMetadata metadata();

/**
* Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array
* needs be large enough for the write.
Expand All @@ -94,32 +96,32 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
chunkStream = chunkStream.parallel();
}
chunkStream.forEach(
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

ucar.ma2.Array chunkArray;
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
chunkShape
)) {
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
chunkProjection.shape,
null
);
} else {
chunkArray = readChunk(chunkCoords);
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
chunkProjection.chunkOffset, chunkProjection.shape
);
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

ucar.ma2.Array chunkArray;
if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape,
chunkShape
)) {
chunkArray = array.sectionNoReduce(chunkProjection.outOffset,
chunkProjection.shape,
null
);
} else {
chunkArray = readChunk(chunkCoords);
MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray,
chunkProjection.chunkOffset, chunkProjection.shape
);
}
writeChunk(chunkCoords, chunkArray);
} catch (ZarrException | InvalidRangeException e) {
throw new RuntimeException(e);
}
writeChunk(chunkCoords, chunkArray);
} catch (ZarrException | InvalidRangeException e) {
throw new RuntimeException(e);
}
});
});

}

Expand Down Expand Up @@ -246,7 +248,7 @@ boolean chunkIsInArray(long[] chunkCoords) {
final int[] chunkShape = metadata().chunkShape();
for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) {
if (chunkCoords[dimIdx] < 0
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
|| chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) {
return false;
}
}
Expand Down Expand Up @@ -282,47 +284,47 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
}

final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
shape);
shape);
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
if (parallel) {
chunkStream = chunkStream.parallel();
}
chunkStream.forEach(
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

if (chunkIsInArray(chunkCoords)) {
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
chunkProjection.shape
);
}

final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
if (!chunkHandle.exists()) {
return;
chunkCoords -> {
try {
final IndexingUtils.ChunkProjection chunkProjection =
IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset,
shape
);

if (chunkIsInArray(chunkCoords)) {
MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(),
chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset,
chunkProjection.shape
);
}

final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
if (!chunkHandle.exists()) {
return;
}
if (codecPipeline.supportsPartialDecode()) {
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
chunkProjection.outOffset, chunkProjection.shape
);
} else {
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
outputArray, chunkProjection.outOffset, chunkProjection.shape
);
}

} catch (ZarrException e) {
throw new RuntimeException(e);
}
if (codecPipeline.supportsPartialDecode()) {
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
chunkProjection.outOffset, chunkProjection.shape
);
} else {
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
outputArray, chunkProjection.outOffset, chunkProjection.shape
);
}

} catch (ZarrException e) {
throw new RuntimeException(e);
}
});
});
return outputArray;
}

Expand Down
Loading
Loading