From 8e79825d32a8c9e3e8ded0e83fe8daa334e378f9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 28 Nov 2025 13:53:16 +0100 Subject: [PATCH 01/61] add ZipStore tests diff --git c/src/main/java/dev/zarr/zarrjava/store/ZipStore.java i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java new file mode 100644 index 0000000..054917f --- /dev/null +++ i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -0,0 +1,72 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; + +public class ZipStore implements Store, Store.ListableStore { + @Nonnull + private final Path path; + + public ZipStore(@Nonnull Path path) { + this.path = path; + } + + public ZipStore(@Nonnull String path) { + this.path = Paths.get(path); + } + + + @Override + public Stream list(String[] keys) { + return Stream.empty(); + } + + @Override + public boolean exists(String[] keys) { + return false; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return null; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return null; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + return null; + } + + @Override + public void set(String[] keys, ByteBuffer bytes) { + + } + + @Override + public void delete(String[] keys) { + + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return this.path.toUri().toString().replaceAll("\\/$", ""); + } + +} diff --git c/src/test/java/dev/zarr/zarrjava/Utils.java i/src/test/java/dev/zarr/zarrjava/Utils.java new file mode 100644 index 0000000..0026200 --- /dev/null +++ i/src/test/java/dev/zarr/zarrjava/Utils.java @@ -0,0 +1,40 @@ +package dev.zarr.zarrjava; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class Utils { + + static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { + if (fileToZip.isHidden()) { + return; + } + if (fileToZip.isDirectory()) { + if (fileName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(fileName)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(fileName + "/")); + zipOut.closeEntry(); + } + File[] children = fileToZip.listFiles(); + for (File childFile : children) { + zipFile(childFile, fileName + "/" + childFile.getName(), zipOut); + } + return; + } + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileName); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + } + +} diff --git c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4a369c9..c7d2ab4 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -7,16 +7,22 @@ import dev.zarr.zarrjava.v3.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.CsvSource; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; import java.util.stream.Stream; +import java.nio.file.Path; +import java.util.zip.ZipOutputStream; +import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { @@ -132,4 +138,72 @@ public class ZarrStoreTest extends ZarrTest { Assertions.assertEquals("test group", attrs.getString("description")); } + + @Test + public void testZipStore() throws ZarrException, IOException { + Path sourceDir = TESTOUTPUT.resolve("testZipStore"); + Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); + FilesystemStore fsStore = new FilesystemStore(sourceDir); + writeTestGroupV3(fsStore, true); + + FileOutputStream fos = new FileOutputStream(targetDir.toFile()); + ZipOutputStream zipOut = new ZipOutputStream(fos); + + File fileToZip = new File(sourceDir.toUri()); + zipFile(fileToZip, fileToZip.getName(), zipOut); + zipOut.close(); + fos.close(); + + ZipStore zipStore = new ZipStore(targetDir); + assertIsTestGroupV3(Group.open(zipStore.resolve()), true); + } + + static Stream localStores() { + return Stream.of( +// new ConcurrentMemoryStore(), + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), + new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + ); + } + + @ParameterizedTest + @MethodSource("localStores") + public void testLocalStores(Store store) throws IOException, ZarrException { + boolean useParallel = true; + Group group = writeTestGroupV3(store, useParallel); + assertIsTestGroupV3(group, useParallel); + } + + int[] testData(){ + int[] testData = new int[1024 * 1024]; + Arrays.setAll(testData, p -> p); + return testData; + } + + Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException { + StoreHandle storeHandle = store.resolve(); + + Group group = Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + return group; + } + + void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException { + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + Array array = (Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } } --- .../dev/zarr/zarrjava/store/ZipStore.java | 72 ++++++++++++++++++ src/test/java/dev/zarr/zarrjava/Utils.java | 40 ++++++++++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 74 +++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 src/main/java/dev/zarr/zarrjava/store/ZipStore.java create mode 100644 src/test/java/dev/zarr/zarrjava/Utils.java diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java new file mode 100644 index 0000000..054917f --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -0,0 +1,72 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; + +public class ZipStore implements Store, Store.ListableStore { + @Nonnull + private final Path path; + + public ZipStore(@Nonnull Path path) { + this.path = path; + } + + public ZipStore(@Nonnull String path) { + this.path = Paths.get(path); + } + + + @Override + public Stream list(String[] keys) { + return Stream.empty(); + } + + @Override + public boolean exists(String[] keys) { + return false; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return null; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return null; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + return null; + } + + @Override + public void set(String[] keys, ByteBuffer bytes) { + + } + + @Override + public void delete(String[] keys) { + + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return this.path.toUri().toString().replaceAll("\\/$", ""); + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/Utils.java b/src/test/java/dev/zarr/zarrjava/Utils.java new file mode 100644 index 0000000..0026200 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/Utils.java @@ -0,0 +1,40 @@ +package dev.zarr.zarrjava; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +public class Utils { + + static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { + if (fileToZip.isHidden()) { + return; + } + if (fileToZip.isDirectory()) { + if (fileName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(fileName)); + zipOut.closeEntry(); + } else { + zipOut.putNextEntry(new ZipEntry(fileName + "/")); + zipOut.closeEntry(); + } + File[] children = fileToZip.listFiles(); + for (File childFile : children) { + zipFile(childFile, fileName + "/" + childFile.getName(), zipOut); + } + return; + } + FileInputStream fis = new FileInputStream(fileToZip); + ZipEntry zipEntry = new ZipEntry(fileName); + zipOut.putNextEntry(zipEntry); + byte[] bytes = new byte[1024]; + int length; + while ((length = fis.read(bytes)) >= 0) { + zipOut.write(bytes, 0, length); + } + fis.close(); + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4a369c9..c7d2ab4 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -7,16 +7,22 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.CsvSource; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; import java.util.stream.Stream; +import java.nio.file.Path; +import java.util.zip.ZipOutputStream; +import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { @@ -132,4 +138,72 @@ public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOExcep Assertions.assertEquals("test group", attrs.getString("description")); } + + @Test + public void testZipStore() throws ZarrException, IOException { + Path sourceDir = TESTOUTPUT.resolve("testZipStore"); + Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); + FilesystemStore fsStore = new FilesystemStore(sourceDir); + writeTestGroupV3(fsStore, true); + + FileOutputStream fos = new FileOutputStream(targetDir.toFile()); + ZipOutputStream zipOut = new ZipOutputStream(fos); + + File fileToZip = new File(sourceDir.toUri()); + zipFile(fileToZip, fileToZip.getName(), zipOut); + zipOut.close(); + fos.close(); + + ZipStore zipStore = new ZipStore(targetDir); + assertIsTestGroupV3(Group.open(zipStore.resolve()), true); + } + + static Stream localStores() { + return Stream.of( +// new ConcurrentMemoryStore(), + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), + new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + ); + } + + @ParameterizedTest + @MethodSource("localStores") + public void testLocalStores(Store store) throws IOException, ZarrException { + boolean useParallel = true; + Group group = writeTestGroupV3(store, useParallel); + assertIsTestGroupV3(group, useParallel); + } + + int[] testData(){ + int[] testData = new int[1024 * 1024]; + Arrays.setAll(testData, p -> p); + return testData; + } + + Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException { + StoreHandle storeHandle = store.resolve(); + + Group group = Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + return group; + } + + void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException { + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + Array array = (Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } } From 2611738824f132a28aeff456e0ee41f3733e162f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 1 Dec 2025 20:32:32 +0100 Subject: [PATCH 02/61] refactor and unify outputs of Store.list --- .../java/dev/zarr/zarrjava/core/Group.java | 7 ++++++- .../zarr/zarrjava/store/FilesystemStore.java | 12 ++++++++--- .../dev/zarr/zarrjava/store/MemoryStore.java | 21 +++++++++---------- .../java/dev/zarr/zarrjava/store/S3Store.java | 4 ++-- .../java/dev/zarr/zarrjava/store/Store.java | 15 ++++++++++++- .../dev/zarr/zarrjava/store/StoreHandle.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/Group.java | 2 +- src/main/java/dev/zarr/zarrjava/v3/Group.java | 2 +- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 4 ++-- 9 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index d8b9a6b..6f8a4d0 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -64,7 +64,12 @@ public static Group open(String path) throws IOException, ZarrException { } @Nullable - public abstract Node get(String key) throws ZarrException, IOException; + public abstract Node get(String[] key) throws ZarrException, IOException; + + @Nullable + public Node get(String key) throws ZarrException, IOException { + return get(new String[]{key}); + } public Stream list() { return storeHandle.list() diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index f0b01cc..5640a1a 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -120,10 +120,16 @@ public void delete(String[] keys) { throw new RuntimeException(e); } } - - public Stream list(String[] keys) { + public Stream list(String[] keys) { try { - return Files.list(resolveKeys(keys)).map(p -> p.toFile().getName()); + return Files.list(resolveKeys(keys)).map(path -> { + Path relativePath = resolveKeys(keys).relativize(path); + String[] parts = new String[relativePath.getNameCount()]; + for (int i = 0; i < relativePath.getNameCount(); i++) { + parts[i] = relativePath.getName(i).toString(); + } + return parts; + }); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index c1bbb9d..d97cffe 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -59,19 +59,18 @@ public void delete(String[] keys) { map.remove(resolveKeys(keys)); } - public Stream list(String[] keys) { - List prefix = resolveKeys(keys); - Set allKeys = new HashSet<>(); + public Stream list(String[] keys) { + List prefix = resolveKeys(keys); + Set> allKeys = new HashSet<>(); - for (List k : map.keySet()) { - if (k.size() <= prefix.size() || ! k.subList(0, prefix.size()).equals(prefix)) - continue; - for (int i = 0; i < k.size(); i++) { - List subKey = k.subList(0, i+1); - allKeys.add(String.join("/", subKey)); + for (List k : map.keySet()) { + if (k.size() <= prefix.size() || ! k.subList(0, prefix.size()).equals(prefix)) + continue; + for (int i = prefix.size(); i < k.size(); i++) { + allKeys.add(k.subList(0, i+1)); + } } - } - return allKeys.stream(); + return allKeys.stream().map(k -> k.toArray(new String[0])); } @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 27aef77..58c8d08 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -104,7 +104,7 @@ public void delete(String[] keys) { } @Override - public Stream list(String[] keys) { + public Stream list(String[] keys) { final String fullKey = resolveKeys(keys); ListObjectsRequest req = ListObjectsRequest.builder() .bucket(bucketName).prefix(fullKey) @@ -112,7 +112,7 @@ public Stream list(String[] keys) { ListObjectsResponse res = s3client.listObjects(req); return res.contents() .stream() - .map(p -> p.key().substring(fullKey.length() + 1)); + .map(p -> p.key().substring(fullKey.length() + 1).split("/")); } @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index c92906d..451bf79 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -27,6 +27,19 @@ public interface Store { interface ListableStore extends Store { - Stream list(String[] keys); + /** + * Lists all keys in the store that match the given prefix keys. Keys are represented as arrays of strings, + * where each string is a segment of the key path. + * Keys that are exactly equal to the prefix are not included in the results. + * Keys that do not contain data (i.e. "directories") are included in the results. + * + * @param keys The prefix keys to match. + * @return A stream of key arrays that match the given prefix. Prefixed keys are not included in the results. + */ + Stream list(String[] keys); + + default Stream list() { + return list(new String[]{}); + } } } diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index b82424f..e731a39 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -56,7 +56,7 @@ public boolean exists() { return store.exists(keys); } - public Stream list() { + public Stream list() { if (!(store instanceof Store.ListableStore)) { throw new UnsupportedOperationException("The underlying store does not support listing."); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index c568294..8551a76 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -169,7 +169,7 @@ public static Group create(String path, Attributes attributes) throws IOExceptio * @throws IOException if there is an error accessing the storage */ @Nullable - public Node get(String key) throws ZarrException, IOException { + public Node get(String[] key) throws ZarrException, IOException { StoreHandle keyHandle = storeHandle.resolve(key); try { return Node.open(keyHandle); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index d17eb77..2436051 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -182,7 +182,7 @@ public static Group create(String path) throws IOException, ZarrException { * @throws IOException if there is an error accessing the storage */ @Nullable - public Node get(String key) throws ZarrException, IOException{ + public Node get(String[] key) throws ZarrException, IOException{ StoreHandle keyHandle = storeHandle.resolve(key); try { return Node.open(keyHandle); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index c7d2ab4..2cb0888 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -123,7 +123,7 @@ public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOExcep dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b .withShape(1024, 1024) .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) - .withChunks(5, 5) + .withChunks(512, 512) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); group.createGroup("subgroup"); @@ -195,7 +195,7 @@ Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, I return group; } - void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException { + void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, IOException { Stream nodes = group.list(); Assertions.assertEquals(2, nodes.count()); Array array = (Array) group.get("array"); From 5e2e017c6657e5707ee1d212bd3f6fb866b183d3 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 1 Dec 2025 21:18:54 +0100 Subject: [PATCH 03/61] read zip store --- .../zarr/zarrjava/store/BufferedZipStore.java | 185 ++++++++++++++++++ .../dev/zarr/zarrjava/store/ZipStore.java | 72 ------- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 23 ++- 3 files changed, 202 insertions(+), 78 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java delete mode 100644 src/main/java/dev/zarr/zarrjava/store/ZipStore.java diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java new file mode 100644 index 0000000..93ab126 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -0,0 +1,185 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + + +/** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. + */ +public class BufferedZipStore implements Store, Store.ListableStore { + + private final StoreHandle underlyingStore; + private final Store.ListableStore bufferStore; + + private void writeBuffer() throws IOException{ + // create zip file bytes from buffer store and write to underlying store + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + +// try (ZipOutputStream zos = new ZipOutputStream(baos)) { +// // iterate over bufferStore.list() +// for (String entry: bufferStore.list(new String[]{}).toArray(String[]::new)) { +// List pathComponents = entry.getKey(); +// byte[] data = entry.getValue(); +// +// // Build the ZIP path (e.g. ["dir", "sub", "file.txt"] → "dir/sub/file.txt") +// String path = String.join("/", pathComponents); +// +// ZipEntry zipEntry = new ZipEntry(path); +// zos.putNextEntry(zipEntry); +// +// zos.write(data); +// zos.closeEntry(); +// } +// } + +// byte[] zipBytes = baos.toByteArray(); +// return ByteBuffer.wrap(zipBytes); +// +// underlyingStore.set(); + } + + private void loadBuffer() throws IOException{ + // read zip file bytes from underlying store and populate buffer store + ByteBuffer buffer = underlyingStore.read(); + if (buffer == null) { + return; + } + try (ZipInputStream zis = new ZipInputStream(new ByteBufferBackedInputStream(buffer))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + zis.closeEntry(); + continue; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] tmp = new byte[8192]; + int read; + while ((read = zis.read(tmp)) != -1) { + baos.write(tmp, 0, read); + } + + byte[] bytes = baos.toByteArray(); + System.out.println("Loading entry: " + entry.getName() + " (" + bytes.length + " bytes)"); + + bufferStore.set(new String[]{entry.getName()}, ByteBuffer.wrap(bytes)); + + zis.closeEntry(); + } + } + + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { + this.underlyingStore = underlyingStore; + this.bufferStore = bufferStore; + try { + loadBuffer(); + } catch (IOException e) { + throw new RuntimeException("Failed to load buffer from underlying store", e); + } + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore) { + this(underlyingStore, new MemoryStore()); + } + + public BufferedZipStore(@Nonnull Path underlyingStore) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString())); + System.out.println("Created BufferedZipStore with underlying path: " + this.underlyingStore.toString()); + + } + + public BufferedZipStore(@Nonnull String underlyingStorePath) { + this(Paths.get(underlyingStorePath)); + } + + /** + * Flushes the buffer to the underlying store as a zip file. + */ + public void flush() throws IOException { + writeBuffer(); + } + + @Override + public Stream list(String[] keys) { + return bufferStore.list(keys); + } + + @Override + public boolean exists(String[] keys) { + return bufferStore.exists(keys); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return bufferStore.get(keys); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return bufferStore.get(keys, start); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + return bufferStore.get(keys, start, end); + } + + @Override + public void set(String[] keys, ByteBuffer bytes) { + bufferStore.set(keys, bytes); + } + + @Override + public void delete(String[] keys) { + bufferStore.delete(keys); + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return "BufferedZipStore(" + underlyingStore.toString() + ")"; + } + + static class ByteBufferBackedInputStream extends InputStream { + private final ByteBuffer buf; + + public ByteBufferBackedInputStream(ByteBuffer buf) { + this.buf = buf; + } + + @Override + public int read() { + return buf.hasRemaining() ? (buf.get() & 0xFF) : -1; + } + + @Override + public int read(byte[] bytes, int off, int len) { + if (!buf.hasRemaining()) { + return -1; + } + + int toRead = Math.min(len, buf.remaining()); + buf.get(bytes, off, toRead); + return toRead; + } + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java deleted file mode 100644 index 054917f..0000000 --- a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java +++ /dev/null @@ -1,72 +0,0 @@ -package dev.zarr.zarrjava.store; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.stream.Stream; - -public class ZipStore implements Store, Store.ListableStore { - @Nonnull - private final Path path; - - public ZipStore(@Nonnull Path path) { - this.path = path; - } - - public ZipStore(@Nonnull String path) { - this.path = Paths.get(path); - } - - - @Override - public Stream list(String[] keys) { - return Stream.empty(); - } - - @Override - public boolean exists(String[] keys) { - return false; - } - - @Nullable - @Override - public ByteBuffer get(String[] keys) { - return null; - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start) { - return null; - } - - @Nullable - @Override - public ByteBuffer get(String[] keys, long start, long end) { - return null; - } - - @Override - public void set(String[] keys, ByteBuffer bytes) { - - } - - @Override - public void delete(String[] keys) { - - } - - @Nonnull - @Override - public StoreHandle resolve(String... keys) { - return new StoreHandle(this, keys); - } - - @Override - public String toString() { - return this.path.toUri().toString().replaceAll("\\/$", ""); - } - -} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 2cb0888..93813fa 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -140,7 +140,7 @@ public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOExcep } @Test - public void testZipStore() throws ZarrException, IOException { + public void testOpenZipStore() throws ZarrException, IOException { Path sourceDir = TESTOUTPUT.resolve("testZipStore"); Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); FilesystemStore fsStore = new FilesystemStore(sourceDir); @@ -150,19 +150,30 @@ public void testZipStore() throws ZarrException, IOException { ZipOutputStream zipOut = new ZipOutputStream(fos); File fileToZip = new File(sourceDir.toUri()); - zipFile(fileToZip, fileToZip.getName(), zipOut); + zipFile(fileToZip, "", zipOut); zipOut.close(); fos.close(); - ZipStore zipStore = new ZipStore(targetDir); + BufferedZipStore zipStore = new BufferedZipStore(targetDir); assertIsTestGroupV3(Group.open(zipStore.resolve()), true); } + @Test + public void testWriteZipStore() throws ZarrException, IOException { + Path targetDir = TESTOUTPUT.resolve("testWriteZipStore.zip"); + BufferedZipStore zipStore = new BufferedZipStore(targetDir); + writeTestGroupV3(zipStore, true); + zipStore.flush(); + + BufferedZipStore zipStoreRead = new BufferedZipStore(targetDir); + assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); + } + static Stream localStores() { return Stream.of( -// new ConcurrentMemoryStore(), + new MemoryStore(), new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) ); } @@ -187,7 +198,7 @@ Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, I Array array = group.createArray("array", b -> b .withShape(1024, 1024) .withDataType(DataType.UINT32) - .withChunkShape(5, 5) + .withChunkShape(512, 512) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); group.createGroup("subgroup"); From 4a9f7f0e09fbf30949a1a0eb76bbeb46dcd213e5 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 4 Dec 2025 10:45:22 +0100 Subject: [PATCH 04/61] Bump to 0.0.6 to trigger release There are apparently cases where release: [created] leads to the deploy not being triggered. Attempting an expansion to [created, published]. diff --git c/pom.xml i/pom.xml index f4c1091..9c9d45d 100644 --- c/pom.xml +++ i/pom.xml @@ -6,7 +6,7 @@ dev.zarr zarr-java - 0.0.9 + 0.0.6 zarr-java @@ -123,6 +123,17 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + unidata-all @@ -221,16 +232,6 @@ - - org.sonatype.central - central-publishing-maven-plugin - 0.8.0 - true - - central - true - - --- pom.xml | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index f4c1091..9c9d45d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ dev.zarr zarr-java - 0.0.9 + 0.0.6 zarr-java @@ -123,6 +123,17 @@ + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + unidata-all @@ -221,16 +232,6 @@ - - org.sonatype.central - central-publishing-maven-plugin - 0.8.0 - true - - central - true - - From 99081e542773fb9e28140f5477685bb900093916 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 4 Dec 2025 15:33:11 +0100 Subject: [PATCH 05/61] Bump to 0.0.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c9d45d..b88c855 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ dev.zarr zarr-java - 0.0.6 + 0.0.7 zarr-java From 268890e4ae9a03bdd34da65f1f762716f7322f58 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 4 Dec 2025 16:15:07 +0100 Subject: [PATCH 06/61] Bump to 0.0.8 Use new sonatype plugin for upload --- pom.xml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index b88c855..685393b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ dev.zarr zarr-java - 0.0.7 + 0.0.8 zarr-java @@ -123,17 +123,6 @@ - - - ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - unidata-all @@ -232,6 +221,16 @@ + + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 + true + + central + true + + From b9e6db43a97660beda94969968004b0b04a1f26c Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 4 Dec 2025 17:06:42 +0100 Subject: [PATCH 07/61] Bump to 0.0.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 685393b..f4c1091 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ dev.zarr zarr-java - 0.0.8 + 0.0.9 zarr-java From 08afc3682c7bf576a2f9c7f605cd18c19616338c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 5 Dec 2025 13:13:59 +0100 Subject: [PATCH 08/61] write buffer of zip store --- .../zarr/zarrjava/store/BufferedZipStore.java | 65 +++++++++++++------ src/test/java/dev/zarr/zarrjava/Utils.java | 48 ++++++++++++++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 28 ++++---- 3 files changed, 105 insertions(+), 36 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 93ab126..bf55e1b 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -11,6 +11,7 @@ import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; /** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. @@ -23,28 +24,50 @@ public class BufferedZipStore implements Store, Store.ListableStore { private void writeBuffer() throws IOException{ // create zip file bytes from buffer store and write to underlying store ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + // iterate all entries provided by bufferStore.list() + bufferStore.list().forEach(keys -> { + try { + if (keys == null || keys.length == 0) { + // skip root entry + return; + } + String entryName = String.join("/", keys); + ByteBuffer bb = bufferStore.get(keys); + if (bb == null) { + // directory entry: ensure trailing slash + if (!entryName.endsWith("/")) { + entryName = entryName + "/"; + } + zos.putNextEntry(new ZipEntry(entryName)); + zos.closeEntry(); + } else { + // read bytes from ByteBuffer without modifying original + ByteBuffer dup = bb.duplicate(); + int len = dup.remaining(); + byte[] bytes = new byte[len]; + dup.get(bytes); + zos.putNextEntry(new ZipEntry(entryName)); + zos.write(bytes); + zos.closeEntry(); + } + } catch (IOException e) { + // wrap checked exception so it can be rethrown from stream for handling below + throw new RuntimeException(e); + } + }); + zos.finish(); + } catch (RuntimeException e) { + // unwrap and rethrow IOExceptions thrown inside the lambda + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + throw e; + } -// try (ZipOutputStream zos = new ZipOutputStream(baos)) { -// // iterate over bufferStore.list() -// for (String entry: bufferStore.list(new String[]{}).toArray(String[]::new)) { -// List pathComponents = entry.getKey(); -// byte[] data = entry.getValue(); -// -// // Build the ZIP path (e.g. ["dir", "sub", "file.txt"] → "dir/sub/file.txt") -// String path = String.join("/", pathComponents); -// -// ZipEntry zipEntry = new ZipEntry(path); -// zos.putNextEntry(zipEntry); -// -// zos.write(data); -// zos.closeEntry(); -// } -// } - -// byte[] zipBytes = baos.toByteArray(); -// return ByteBuffer.wrap(zipBytes); -// -// underlyingStore.set(); + byte[] zipBytes = baos.toByteArray(); + // write zip bytes back to underlying store + underlyingStore.set(ByteBuffer.wrap(zipBytes)); } private void loadBuffer() throws IOException{ diff --git a/src/test/java/dev/zarr/zarrjava/Utils.java b/src/test/java/dev/zarr/zarrjava/Utils.java index 0026200..da57f0d 100644 --- a/src/test/java/dev/zarr/zarrjava/Utils.java +++ b/src/test/java/dev/zarr/zarrjava/Utils.java @@ -2,12 +2,28 @@ import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.BufferedOutputStream; +import java.nio.file.Path; +import java.nio.file.Files; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import java.util.zip.ZipInputStream; public class Utils { + static void zipFile(Path sourceDir, Path targetDir) throws IOException { + FileOutputStream fos = new FileOutputStream(targetDir.toFile()); + ZipOutputStream zipOut = new ZipOutputStream(fos); + + File fileToZip = new File(sourceDir.toUri()); + + zipFile(fileToZip, "", zipOut); + zipOut.close(); + fos.close(); + } + static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { if (fileToZip.isHidden()) { return; @@ -37,4 +53,36 @@ static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) thr fis.close(); } + /** + * Unzip sourceZip into targetDir. + * Protects against Zip Slip by ensuring extracted paths remain inside targetDir. + */ + static void unzipFile(Path sourceZip, Path targetDir) throws IOException { + Files.createDirectories(targetDir); + try (FileInputStream fis = new FileInputStream(sourceZip.toFile()); + ZipInputStream zis = new ZipInputStream(fis)) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + Path outPath = targetDir.resolve(entry.getName()).normalize(); + Path targetDirNorm = targetDir.normalize(); + if (!outPath.startsWith(targetDirNorm)) { + throw new IOException("Zip entry is outside of the target dir: " + entry.getName()); + } + if (entry.isDirectory() || entry.getName().endsWith("/")) { + Files.createDirectories(outPath); + } else { + Files.createDirectories(outPath.getParent()); + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(outPath.toFile()))) { + byte[] buffer = new byte[1024]; + int len; + while ((len = zis.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } + } + zis.closeEntry(); + } + } + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 93813fa..4a26215 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -13,15 +13,13 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; import java.util.stream.Stream; import java.nio.file.Path; -import java.util.zip.ZipOutputStream; +import static dev.zarr.zarrjava.Utils.unzipFile; import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; @@ -146,13 +144,7 @@ public void testOpenZipStore() throws ZarrException, IOException { FilesystemStore fsStore = new FilesystemStore(sourceDir); writeTestGroupV3(fsStore, true); - FileOutputStream fos = new FileOutputStream(targetDir.toFile()); - ZipOutputStream zipOut = new ZipOutputStream(fos); - - File fileToZip = new File(sourceDir.toUri()); - zipFile(fileToZip, "", zipOut); - zipOut.close(); - fos.close(); + zipFile(sourceDir, targetDir); BufferedZipStore zipStore = new BufferedZipStore(targetDir); assertIsTestGroupV3(Group.open(zipStore.resolve()), true); @@ -160,20 +152,26 @@ public void testOpenZipStore() throws ZarrException, IOException { @Test public void testWriteZipStore() throws ZarrException, IOException { - Path targetDir = TESTOUTPUT.resolve("testWriteZipStore.zip"); - BufferedZipStore zipStore = new BufferedZipStore(targetDir); + Path path = TESTOUTPUT.resolve("testWriteZipStore.zip"); + BufferedZipStore zipStore = new BufferedZipStore(path); writeTestGroupV3(zipStore, true); zipStore.flush(); - BufferedZipStore zipStoreRead = new BufferedZipStore(targetDir); + BufferedZipStore zipStoreRead = new BufferedZipStore(path); assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); + + Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped"); + + unzipFile(path, unzippedPath); + FilesystemStore fsStore = new FilesystemStore(unzippedPath); + assertIsTestGroupV3(Group.open(fsStore.resolve()), true); } static Stream localStores() { return Stream.of( new MemoryStore(), - new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")) +// new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) ); } From 0cacc5bc2e86f90b3f1f0fc52ee218987dfdb08f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 5 Dec 2025 19:42:27 +0100 Subject: [PATCH 09/61] use apache commons compress for zip file read and write --- pom.xml | 5 ++ .../zarr/zarrjava/store/BufferedZipStore.java | 88 ++++++++++++++----- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 71 +++++++++++++++ 3 files changed, 141 insertions(+), 23 deletions(-) diff --git a/pom.xml b/pom.xml index f4c1091..ea7e597 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,11 @@ 4.13.1 test + + org.apache.commons + commons-compress + 1.28.0 + diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index bf55e1b..ac67286 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -9,9 +9,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.Zip64Mode; + +import java.util.zip.CRC32; +import java.util.zip.ZipEntry; // for STORED constant /** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. @@ -20,11 +26,19 @@ public class BufferedZipStore implements Store, Store.ListableStore { private final StoreHandle underlyingStore; private final Store.ListableStore bufferStore; + private final String archiveComment; private void writeBuffer() throws IOException{ // create zip file bytes from buffer store and write to underlying store ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zos = new ZipOutputStream(baos)) { + try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) { + // always use zip64 + zos.setUseZip64(Zip64Mode.Always); + // set archive comment if provided + if (archiveComment != null) { + zos.setComment(archiveComment); + } + // iterate all entries provided by bufferStore.list() bufferStore.list().forEach(keys -> { try { @@ -39,17 +53,30 @@ private void writeBuffer() throws IOException{ if (!entryName.endsWith("/")) { entryName = entryName + "/"; } - zos.putNextEntry(new ZipEntry(entryName)); - zos.closeEntry(); + ZipArchiveEntry dirEntry = new ZipArchiveEntry(entryName); + dirEntry.setMethod(ZipEntry.STORED); + dirEntry.setSize(0); + dirEntry.setCrc(0); + zos.putArchiveEntry(dirEntry); + zos.closeArchiveEntry(); } else { // read bytes from ByteBuffer without modifying original ByteBuffer dup = bb.duplicate(); int len = dup.remaining(); byte[] bytes = new byte[len]; dup.get(bytes); - zos.putNextEntry(new ZipEntry(entryName)); + + // compute CRC and set size for STORED (no compression) + CRC32 crc = new CRC32(); + crc.update(bytes, 0, bytes.length); + ZipArchiveEntry fileEntry = new ZipArchiveEntry(entryName); + fileEntry.setMethod(ZipEntry.STORED); + fileEntry.setSize(bytes.length); + fileEntry.setCrc(crc.getValue()); + + zos.putArchiveEntry(fileEntry); zos.write(bytes); - zos.closeEntry(); + zos.closeArchiveEntry(); } } catch (IOException e) { // wrap checked exception so it can be rethrown from stream for handling below @@ -76,11 +103,12 @@ private void loadBuffer() throws IOException{ if (buffer == null) { return; } - try (ZipInputStream zis = new ZipInputStream(new ByteBufferBackedInputStream(buffer))) { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { +// this.archiveComment = zis.getComment(); + ArchiveEntry aentry; + while ((aentry = zis.getNextEntry()) != null) { + ZipArchiveEntry entry = (ZipArchiveEntry) aentry; if (entry.isDirectory()) { - zis.closeEntry(); continue; } ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -89,21 +117,17 @@ private void loadBuffer() throws IOException{ while ((read = zis.read(tmp)) != -1) { baos.write(tmp, 0, read); } - byte[] bytes = baos.toByteArray(); - System.out.println("Loading entry: " + entry.getName() + " (" + bytes.length + " bytes)"); - bufferStore.set(new String[]{entry.getName()}, ByteBuffer.wrap(bytes)); - - zis.closeEntry(); } } } - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { this.underlyingStore = underlyingStore; this.bufferStore = bufferStore; + this.archiveComment = archiveComment; try { loadBuffer(); } catch (IOException e) { @@ -111,27 +135,45 @@ public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.Lis } } + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { + this(underlyingStore, bufferStore, null); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment) { + this(underlyingStore, new MemoryStore(), archiveComment); + } + public BufferedZipStore(@Nonnull StoreHandle underlyingStore) { - this(underlyingStore, new MemoryStore()); + this(underlyingStore, (String) null); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment); } public BufferedZipStore(@Nonnull Path underlyingStore) { - this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString())); - System.out.println("Created BufferedZipStore with underlying path: " + this.underlyingStore.toString()); + this(underlyingStore, null); + } + public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment) { + this(Paths.get(underlyingStorePath), archiveComment); } public BufferedZipStore(@Nonnull String underlyingStorePath) { - this(Paths.get(underlyingStorePath)); + this(underlyingStorePath, null); } /** - * Flushes the buffer to the underlying store as a zip file. + * Flushes the buffer and archiveComment to the underlying store as a zip file. */ public void flush() throws IOException { writeBuffer(); } + public String getArchiveComment() { + return archiveComment; + } + @Override public Stream list(String[] keys) { return bufferStore.list(keys); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4a26215..ddaad8f 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -4,6 +4,10 @@ import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.v3.*; +import org.apache.commons.compress.archivers.zip.*; + +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -16,11 +20,14 @@ import java.io.IOException; import java.nio.file.Files; import java.util.Arrays; +import java.util.Collections; import java.util.stream.Stream; import java.nio.file.Path; +import java.util.zip.ZipEntry; import static dev.zarr.zarrjava.Utils.unzipFile; import static dev.zarr.zarrjava.Utils.zipFile; + import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { @@ -167,6 +174,70 @@ public void testWriteZipStore() throws ZarrException, IOException { assertIsTestGroupV3(Group.open(fsStore.resolve()), true); } + @Test + public void testZipStoreWithComment() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreWithComment.zip"); + String comment = "{\"ome\": { \"version\": \"XX.YY\" }}"; + BufferedZipStore zipStore = new BufferedZipStore(path, comment); + writeTestGroupV3(zipStore, true); + zipStore.flush(); + + try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) { + String retrievedComment = zipFile.getComment(); + Assertions.assertEquals(comment, retrievedComment, "ZIP archive comment does not match expected value."); + } + + Assertions.assertEquals(comment, new BufferedZipStore(path).getArchiveComment(), "ZIP archive comment from store does not match expected value."); + } + + /** + * Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr + * @see RFC-9: Zipped OME-Zarr + * + * Features to test: + * - ZIP64 format + * - No ZIP-level compression + * - Option to add archive comments in the ZIP file header + * - Prohibit nested or multi-part ZIP archives + * - "The root-level zarr.json file SHOULD be the first ZIP file entry and the first entry in the central directory header; other zarr.json files SHOULD follow immediately afterwards, in breadth-first order." + */ + @Test + public void testZipStoreRequirements() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip"); + BufferedZipStore zipStore = new BufferedZipStore(path); + writeTestGroupV3(zipStore, true); + zipStore.flush(); + + // test for ZIP64 +// List fileHeaders = new ZipFile(path.toFile()).getFileHeaders(); +// +// HeaderReader headerReader = new HeaderReader(); +// ZipModel zipModel = headerReader.readAllHeaders(new RandomAccessFile(generatedZipFile, +// RandomAccessFileMode.READ.getValue()), buildDefaultConfig()); +// assertThat(zipModel.getZip64EndOfCentralDirectoryLocator()).isNotNull(); +// assertThat(zipModel.getZip64EndOfCentralDirectoryRecord()).isNotNull(); +// assertThat(zipModel.isZip64Format()).isTrue(); + + try (ZipFile zip = new ZipFile(path.toFile())) { + for (ZipArchiveEntry e : Collections.list(zip.getEntries())) { + System.out.println(e.getName()); + ZipExtraField[] extraFields = e.getExtraFields(); + System.out.println(extraFields.length); + Assertions.assertNotNull(extraFields, "Entry " + e.getName() + " has no extra fields"); + Assertions.assertTrue(Arrays.stream(extraFields).anyMatch(xf -> xf instanceof Zip64ExtendedInformationExtraField), + "Entry " + e.getName() + " is missing ZIP64 extra field"); + } + } + // no compression + try (ZipFile zip = new ZipFile(path.toFile())) { + for (ZipArchiveEntry e : Collections.list(zip.getEntries())) { + Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed"); + } + } + + + } + static Stream localStores() { return Stream.of( new MemoryStore(), From 5b74372b7b13bcfec632d02c5e374a6462a68be9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 11 Dec 2025 11:25:07 +0100 Subject: [PATCH 10/61] set Zip64Mode.AsNeeded --- .../java/dev/zarr/zarrjava/store/BufferedZipStore.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index ac67286..1f1f33b 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -11,10 +11,7 @@ import java.util.stream.Stream; import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.compress.archivers.zip.Zip64Mode; +import org.apache.commons.compress.archivers.zip.*; import java.util.zip.CRC32; import java.util.zip.ZipEntry; // for STORED constant @@ -32,14 +29,11 @@ private void writeBuffer() throws IOException{ // create zip file bytes from buffer store and write to underlying store ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) { - // always use zip64 - zos.setUseZip64(Zip64Mode.Always); - // set archive comment if provided + zos.setUseZip64(Zip64Mode.AsNeeded); if (archiveComment != null) { zos.setComment(archiveComment); } - // iterate all entries provided by bufferStore.list() bufferStore.list().forEach(keys -> { try { if (keys == null || keys.length == 0) { From ee92e278366d0785b2eb5713edf3722b31d5b804 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 11 Dec 2025 11:25:31 +0100 Subject: [PATCH 11/61] test Zipped OME-Zarr requirements --- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index ddaad8f..6f764aa 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.stream.Stream; @@ -193,49 +194,54 @@ public void testZipStoreWithComment() throws ZarrException, IOException { /** * Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr * @see RFC-9: Zipped OME-Zarr - * - * Features to test: - * - ZIP64 format - * - No ZIP-level compression - * - Option to add archive comments in the ZIP file header - * - Prohibit nested or multi-part ZIP archives - * - "The root-level zarr.json file SHOULD be the first ZIP file entry and the first entry in the central directory header; other zarr.json files SHOULD follow immediately afterwards, in breadth-first order." */ @Test public void testZipStoreRequirements() throws ZarrException, IOException { Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip"); BufferedZipStore zipStore = new BufferedZipStore(path); - writeTestGroupV3(zipStore, true); - zipStore.flush(); - // test for ZIP64 -// List fileHeaders = new ZipFile(path.toFile()).getFileHeaders(); -// -// HeaderReader headerReader = new HeaderReader(); -// ZipModel zipModel = headerReader.readAllHeaders(new RandomAccessFile(generatedZipFile, -// RandomAccessFileMode.READ.getValue()), buildDefaultConfig()); -// assertThat(zipModel.getZip64EndOfCentralDirectoryLocator()).isNotNull(); -// assertThat(zipModel.getZip64EndOfCentralDirectoryRecord()).isNotNull(); -// assertThat(zipModel.isZip64Format()).isTrue(); + Group group = Group.create(zipStore.resolve()); + Array array = group.createArray("a1", b -> b + .withShape(1024, 1024) + .withDataType(DataType.UINT32) + .withChunkShape(512, 512) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), true); + + Group g1 = group.createGroup("g1"); + g1.createGroup("g1_1").createGroup("g1_1_1"); + g1.createGroup("g1_2"); + group.createGroup("g2").createGroup("g2_1"); + group.createGroup("g3"); + + zipStore.flush(); try (ZipFile zip = new ZipFile(path.toFile())) { - for (ZipArchiveEntry e : Collections.list(zip.getEntries())) { - System.out.println(e.getName()); - ZipExtraField[] extraFields = e.getExtraFields(); - System.out.println(extraFields.length); - Assertions.assertNotNull(extraFields, "Entry " + e.getName() + " has no extra fields"); - Assertions.assertTrue(Arrays.stream(extraFields).anyMatch(xf -> xf instanceof Zip64ExtendedInformationExtraField), - "Entry " + e.getName() + " is missing ZIP64 extra field"); - } - } - // no compression - try (ZipFile zip = new ZipFile(path.toFile())) { - for (ZipArchiveEntry e : Collections.list(zip.getEntries())) { + ArrayList entries = Collections.list(zip.getEntries()); + + // no compression + for (ZipArchiveEntry e : entries) { Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed"); } - } - + // correct order of zarr.json files + String[] expectedZarrJsonOrder = new String[]{ + "zarr.json", + "a1/zarr.json", + "g1/zarr.json", + "g2/zarr.json", + "g3/zarr.json", + "g1/g1_1/zarr.json", + "g1/g1_2/zarr.json", + "g2/g2_1/zarr.json", + "g1/g1_1/g1_1_1/zarr.json" + }; + String[] actualZarrJsonOrder = entries.stream() + .map(ZipArchiveEntry::getName) + .limit(expectedZarrJsonOrder.length) + .toArray(String[]::new); + Assertions.assertArrayEquals(expectedZarrJsonOrder, actualZarrJsonOrder, "zarr.json files are not in the expected breadth-first order"); + } } static Stream localStores() { From a34465587cb2a55c55de8b8225ad8c3c973652be Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 11 Dec 2025 11:42:07 +0100 Subject: [PATCH 12/61] Sort zarr.json files in breadth-first order within BufferedZipStore --- .../zarr/zarrjava/store/BufferedZipStore.java | 24 ++++++++++++++++++- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 9 +++---- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 1f1f33b..fd4bc60 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -33,8 +33,30 @@ private void writeBuffer() throws IOException{ if (archiveComment != null) { zos.setComment(archiveComment); } + Stream entries = bufferStore.list().sorted( + (a, b) -> { + boolean aIsZarr = a.length > 0 && a[a.length - 1].equals("zarr.json"); + boolean bIsZarr = b.length > 0 && b[b.length - 1].equals("zarr.json"); + // first all zarr.json files + if (aIsZarr && !bIsZarr) { + return -1; + } else if (!aIsZarr && bIsZarr) { + return 1; + } else if (aIsZarr && bIsZarr) { + // sort zarr.json in BFS order within same depth by lexicographical order + if (a.length != b.length) { + return Integer.compare(a.length, b.length); + } else { + return String.join("/", a).compareTo(String.join("/", b)); + } + } else { + // then all other files in lexicographical order + return String.join("/", a).compareTo(String.join("/", b)); + } + } + ); - bufferStore.list().forEach(keys -> { + entries.forEach(keys -> { try { if (keys == null || keys.length == 0) { // skip root entry diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 6f764aa..22afb68 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -225,7 +225,7 @@ public void testZipStoreRequirements() throws ZarrException, IOException { } // correct order of zarr.json files - String[] expectedZarrJsonOrder = new String[]{ + String[] expectedFirstEntries = new String[]{ "zarr.json", "a1/zarr.json", "g1/zarr.json", @@ -236,11 +236,12 @@ public void testZipStoreRequirements() throws ZarrException, IOException { "g2/g2_1/zarr.json", "g1/g1_1/g1_1_1/zarr.json" }; - String[] actualZarrJsonOrder = entries.stream() + String[] actualFirstEntries = entries.stream() .map(ZipArchiveEntry::getName) - .limit(expectedZarrJsonOrder.length) + .limit(expectedFirstEntries.length) .toArray(String[]::new); - Assertions.assertArrayEquals(expectedZarrJsonOrder, actualZarrJsonOrder, "zarr.json files are not in the expected breadth-first order"); + + Assertions.assertArrayEquals(expectedFirstEntries, actualFirstEntries, "zarr.json files are not in the expected breadth-first order"); } } From ea16692df732e791d00c48740916495257f8d431 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 11:44:20 +0100 Subject: [PATCH 13/61] manually read zip comment --- .../zarr/zarrjava/store/BufferedZipStore.java | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index fd4bc60..6304e99 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -23,7 +23,7 @@ public class BufferedZipStore implements Store, Store.ListableStore { private final StoreHandle underlyingStore; private final Store.ListableStore bufferStore; - private final String archiveComment; + private String archiveComment; private void writeBuffer() throws IOException{ // create zip file bytes from buffer store and write to underlying store @@ -113,14 +113,62 @@ private void writeBuffer() throws IOException{ underlyingStore.set(ByteBuffer.wrap(zipBytes)); } + // Source - https://stackoverflow.com/a/9918966 + // Retrieved 2025-12-12, License - CC BY-SA 3.0 + private static String getZipCommentFromBuffer (byte[] buffer, int len) { + byte[] magicDirEnd = {0x50, 0x4b, 0x05, 0x06}; + int buffLen = Math.min(buffer.length, len); + + // Check the buffer from the end + for (int i = buffLen - magicDirEnd.length - 22; i >= 0; i--) { + boolean isMagicStart = true; + + for (int k = 0; k < magicDirEnd.length; k++) { + if (buffer[i + k] != magicDirEnd[k]) { + isMagicStart = false; + break; + } + } + + if (isMagicStart) { + // Magic Start found! + int commentLen = buffer[i + 20] + buffer[i + 21] * 256; + int realLen = buffLen - i - 22; + System.out.println ("ZIP comment found at buffer position " + + (i + 22) + " with len = " + commentLen + ", good!"); + + if (commentLen != realLen) { + System.out.println ("WARNING! ZIP comment size mismatch: " + + "directory says len is " + commentLen + + ", but file ends after " + realLen + " bytes!"); + } + + String comment = new String (buffer, i + 22, Math.min(commentLen, realLen)); + return comment; + } + } + + System.out.println ("ERROR! ZIP comment NOT found!"); + return null; + } + private void loadBuffer() throws IOException{ // read zip file bytes from underlying store and populate buffer store ByteBuffer buffer = underlyingStore.read(); if (buffer == null) { return; } + + // read archive comment + byte[] bufArray; + if (buffer.hasArray()) { + bufArray = buffer.array(); + } else { + bufArray = new byte[buffer.remaining()]; + buffer.duplicate().get(bufArray); + } + this.archiveComment = getZipCommentFromBuffer(bufArray, bufArray.length); try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { -// this.archiveComment = zis.getComment(); ArchiveEntry aentry; while ((aentry = zis.getNextEntry()) != null) { ZipArchiveEntry entry = (ZipArchiveEntry) aentry; From 7e0164f6f2de7b338d5224b16364a6754fc27e56 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 13:38:46 +0100 Subject: [PATCH 14/61] refactor read zip comment --- .../zarr/zarrjava/store/BufferedZipStore.java | 60 +++++++------------ 1 file changed, 22 insertions(+), 38 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 6304e99..5ccdf5c 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -113,42 +113,30 @@ private void writeBuffer() throws IOException{ underlyingStore.set(ByteBuffer.wrap(zipBytes)); } - // Source - https://stackoverflow.com/a/9918966 - // Retrieved 2025-12-12, License - CC BY-SA 3.0 - private static String getZipCommentFromBuffer (byte[] buffer, int len) { - byte[] magicDirEnd = {0x50, 0x4b, 0x05, 0x06}; - int buffLen = Math.min(buffer.length, len); - + // adopted from https://stackoverflow.com/a/9918966 + @Nullable + private String getZipCommentFromBuffer(byte[] bufArray) throws IOException { + // End of Central Directory (EOCD) record magic number + byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; + int buffLen = bufArray.length; // Check the buffer from the end - for (int i = buffLen - magicDirEnd.length - 22; i >= 0; i--) { - boolean isMagicStart = true; - - for (int k = 0; k < magicDirEnd.length; k++) { - if (buffer[i + k] != magicDirEnd[k]) { - isMagicStart = false; - break; + search: + for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { + for (int k = 0; k < EOCD.length; k++) { + if (bufArray[i + k] != EOCD[k]) { + continue search; } } - - if (isMagicStart) { - // Magic Start found! - int commentLen = buffer[i + 20] + buffer[i + 21] * 256; - int realLen = buffLen - i - 22; - System.out.println ("ZIP comment found at buffer position " - + (i + 22) + " with len = " + commentLen + ", good!"); - - if (commentLen != realLen) { - System.out.println ("WARNING! ZIP comment size mismatch: " - + "directory says len is " + commentLen - + ", but file ends after " + realLen + " bytes!"); - } - - String comment = new String (buffer, i + 22, Math.min(commentLen, realLen)); - return comment; + // End of Central Directory found! + int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; + int realLen = buffLen - i - 22; + if (commentLen != realLen) { + throw new IOException("ZIP comment size mismatch: " + + "directory says len is " + commentLen + + ", but file ends after " + realLen + " bytes!"); } + return new String(bufArray, i + 22, commentLen); } - - System.out.println ("ERROR! ZIP comment NOT found!"); return null; } @@ -158,8 +146,6 @@ private void loadBuffer() throws IOException{ if (buffer == null) { return; } - - // read archive comment byte[] bufArray; if (buffer.hasArray()) { bufArray = buffer.array(); @@ -167,11 +153,10 @@ private void loadBuffer() throws IOException{ bufArray = new byte[buffer.remaining()]; buffer.duplicate().get(bufArray); } - this.archiveComment = getZipCommentFromBuffer(bufArray, bufArray.length); + this.archiveComment = getZipCommentFromBuffer(bufArray); try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { - ArchiveEntry aentry; - while ((aentry = zis.getNextEntry()) != null) { - ZipArchiveEntry entry = (ZipArchiveEntry) aentry; + ZipArchiveEntry entry; + while ((entry = zis.getNextEntry()) != null) { if (entry.isDirectory()) { continue; } @@ -185,7 +170,6 @@ private void loadBuffer() throws IOException{ bufferStore.set(new String[]{entry.getName()}, ByteBuffer.wrap(bytes)); } } - } public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { From 02445e009b0fdd2726b06aa42ef724053df67267 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 13:58:39 +0100 Subject: [PATCH 15/61] test zip store with v2 --- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 72 +++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 22afb68..89a8b70 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.*; -import dev.zarr.zarrjava.v3.*; +import dev.zarr.zarrjava.core.*; import org.apache.commons.compress.archivers.zip.*; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; @@ -39,7 +39,7 @@ public void testFileSystemStores() throws IOException, ZarrException { GroupMetadata groupMetadata = objectMapper.readValue( Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class + dev.zarr.zarrjava.v3.GroupMetadata.class ); String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); @@ -48,7 +48,7 @@ public void testFileSystemStores() throws IOException, ZarrException { ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); + dev.zarr.zarrjava.v3.ArrayMetadata.class); String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); @@ -100,10 +100,10 @@ public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOExcep int[] testData = new int[1024 * 1024]; Arrays.setAll(testData, p -> p); - Group group = Group.create(new MemoryStore().resolve()); + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve()); Array array = group.createArray("array", b -> b .withShape(1024, 1024) - .withDataType(DataType.UINT32) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) .withChunkShape(5, 5) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); @@ -200,15 +200,15 @@ public void testZipStoreRequirements() throws ZarrException, IOException { Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip"); BufferedZipStore zipStore = new BufferedZipStore(path); - Group group = Group.create(zipStore.resolve()); + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(zipStore.resolve()); Array array = group.createArray("a1", b -> b .withShape(1024, 1024) - .withDataType(DataType.UINT32) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) .withChunkShape(512, 512) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), true); - Group g1 = group.createGroup("g1"); + dev.zarr.zarrjava.v3.Group g1 = group.createGroup("g1"); g1.createGroup("g1_1").createGroup("g1_1_1"); g1.createGroup("g1_2"); group.createGroup("g2").createGroup("g2_1"); @@ -245,6 +245,25 @@ public void testZipStoreRequirements() throws ZarrException, IOException { } } + + @Test + public void testZipStoreV2() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreV2.zip"); + BufferedZipStore zipStore = new BufferedZipStore(path); + writeTestGroupV2(zipStore, true); + zipStore.flush(); + + BufferedZipStore zipStoreRead = new BufferedZipStore(path); + assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(zipStoreRead.resolve()), true); + + Path unzippedPath = TESTOUTPUT.resolve("testZipStoreV2Unzipped"); + + unzipFile(path, unzippedPath); + FilesystemStore fsStore = new FilesystemStore(unzippedPath); + assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(fsStore.resolve()), true); + } + + static Stream localStores() { return Stream.of( new MemoryStore(), @@ -261,6 +280,7 @@ public void testLocalStores(Store store) throws IOException, ZarrException { assertIsTestGroupV3(group, useParallel); } + int[] testData(){ int[] testData = new int[1024 * 1024]; Arrays.setAll(testData, p -> p); @@ -270,10 +290,10 @@ int[] testData(){ Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException { StoreHandle storeHandle = store.resolve(); - Group group = Group.create(storeHandle); - Array array = group.createArray("array", b -> b + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle); + dev.zarr.zarrjava.v3.Array array = group.createArray("array", b -> b .withShape(1024, 1024) - .withDataType(DataType.UINT32) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) .withChunkShape(512, 512) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); @@ -289,7 +309,35 @@ void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, Assertions.assertNotNull(array); ucar.ma2.Array result = array.read(useParallel); Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes; + Attributes attrs = group.metadata().attributes(); + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } + + + dev.zarr.zarrjava.v2.Group writeTestGroupV2(Store store, boolean useParallel) throws ZarrException, IOException { + StoreHandle storeHandle = store.resolve(); + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); + dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withChunks(512, 512) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes().set("some", "value")); + return group; + } + + void assertIsTestGroupV2(dev.zarr.zarrjava.core.Group group, boolean useParallel) throws ZarrException, IOException { + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes(); Assertions.assertNotNull(attrs); Assertions.assertEquals("value", attrs.getString("some")); } From caafad0997a7a285a1a669a69b23746d8a63d67d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 15:59:39 +0100 Subject: [PATCH 16/61] use com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream instead of own implementation --- .../zarr/zarrjava/store/BufferedZipStore.java | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 5ccdf5c..047fd14 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -4,13 +4,12 @@ import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.Stream; -import org.apache.commons.compress.archivers.ArchiveEntry; +import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; import org.apache.commons.compress.archivers.zip.*; import java.util.zip.CRC32; @@ -270,29 +269,4 @@ public StoreHandle resolve(String... keys) { public String toString() { return "BufferedZipStore(" + underlyingStore.toString() + ")"; } - - static class ByteBufferBackedInputStream extends InputStream { - private final ByteBuffer buf; - - public ByteBufferBackedInputStream(ByteBuffer buf) { - this.buf = buf; - } - - @Override - public int read() { - return buf.hasRemaining() ? (buf.get() & 0xFF) : -1; - } - - @Override - public int read(byte[] bytes, int off, int len) { - if (!buf.hasRemaining()) { - return -1; - } - - int toRead = Math.min(len, buf.remaining()); - buf.get(bytes, off, toRead); - return toRead; - } - } - } From dbc559c7eb80ddcd57bce39ec2cd3a7cfdea62da Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 16:01:38 +0100 Subject: [PATCH 17/61] add ReadOnlyZipStore --- .../zarr/zarrjava/store/BufferedZipStore.java | 28 +-- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 159 ++++++++++++++++++ .../dev/zarr/zarrjava/utils/ZipUtils.java | 35 ++++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 13 ++ 4 files changed, 209 insertions(+), 26 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java create mode 100644 src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 047fd14..af12ef0 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -15,6 +15,8 @@ import java.util.zip.CRC32; import java.util.zip.ZipEntry; // for STORED constant +import static dev.zarr.zarrjava.utils.ZipUtils.getZipCommentFromBuffer; + /** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. */ @@ -112,32 +114,6 @@ private void writeBuffer() throws IOException{ underlyingStore.set(ByteBuffer.wrap(zipBytes)); } - // adopted from https://stackoverflow.com/a/9918966 - @Nullable - private String getZipCommentFromBuffer(byte[] bufArray) throws IOException { - // End of Central Directory (EOCD) record magic number - byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; - int buffLen = bufArray.length; - // Check the buffer from the end - search: - for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { - for (int k = 0; k < EOCD.length; k++) { - if (bufArray[i + k] != EOCD[k]) { - continue search; - } - } - // End of Central Directory found! - int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; - int realLen = buffLen - i - 22; - if (commentLen != realLen) { - throw new IOException("ZIP comment size mismatch: " - + "directory says len is " + commentLen - + ", but file ends after " + realLen + " bytes!"); - } - return new String(bufArray, i + 22, commentLen); - } - return null; - } private void loadBuffer() throws IOException{ // read zip file bytes from underlying store and populate buffer store diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java new file mode 100644 index 0000000..516a647 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -0,0 +1,159 @@ +package dev.zarr.zarrjava.store; + +import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; + +import static dev.zarr.zarrjava.utils.ZipUtils.getZipCommentFromBuffer; + +public class ReadOnlyZipStore implements Store, Store.ListableStore { + + private final StoreHandle underlyingStore; + + String resolveKeys(String[] keys) { + return String.join("/", keys); + } + + String[] resolveEntryKeys(String entryKey) { + return entryKey.split("/"); + } + + @Override + public boolean exists(String[] keys) { + return get(keys, 0, 0) != null; + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return get(keys, 0); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return get(keys, start, -1); + } + + public String getArchiveComment() throws IOException { + ByteBuffer buffer = underlyingStore.read(); + if (buffer == null) { + return null; + } + byte[] bufArray; + if (buffer.hasArray()) { + bufArray = buffer.array(); + } else { + bufArray = new byte[buffer.remaining()]; + buffer.duplicate().get(bufArray); + } + return getZipCommentFromBuffer(bufArray); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + ByteBuffer buffer = underlyingStore.read(); + if (buffer == null) { + return null; + } + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { + ZipArchiveEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory() || !entry.getName().equals(resolveKeys(keys))) { + continue; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + if (end == -1) { + end = entry.getSize(); + } + if (start > end) { + throw new IllegalArgumentException("Start position can not be larger than end position. Got start=" + start + ", end=" + end); + } + if (start < 0 || end > entry.getSize()) { + throw new IllegalArgumentException("Start and end positions must be within the bounds of the zip entry size. Entry size=" + entry.getSize() + ", got start=" + start + ", end=" + end); + } + zis.skip(start); + long bytesToRead = end - start; + byte[] bufferArray = new byte[8192]; + int len; + while (bytesToRead > 0 && (len = zis.read(bufferArray, 0, (int) Math.min(bufferArray.length, bytesToRead))) != -1) { + baos.write(bufferArray, 0, len); + bytesToRead -= len; + } + byte[] bytes = baos.toByteArray(); + return ByteBuffer.wrap(bytes); + } + } catch (IOException e) { + return null; + } + return null; + } + + @Override + public void set(String[] keys, ByteBuffer bytes) { + throw new UnsupportedOperationException("ReadOnlyZipStore does not support set operation."); + } + + @Override + public void delete(String[] keys) { + throw new UnsupportedOperationException("ReadOnlyZipStore does not support delete operation."); + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return "ReadOnlyZipStore(" + underlyingStore.toString() + ")"; + } + + public ReadOnlyZipStore(@Nonnull StoreHandle underlyingStore) { + this.underlyingStore = underlyingStore; + } + + public ReadOnlyZipStore(@Nonnull Path underlyingStore) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString())); + } + + public ReadOnlyZipStore(@Nonnull String underlyingStorePath) { + this(Paths.get(underlyingStorePath)); + } + + @Override + public Stream list(String[] keys) { + Stream.Builder builder = Stream.builder(); + + ByteBuffer buffer = underlyingStore.read(); + if (buffer == null) { + return builder.build(); + } + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { + ZipArchiveEntry entry; + String prefix = resolveKeys(keys); + while ((entry = zis.getNextEntry()) != null) { + String entryKey = entry.getName(); + if (!entryKey.startsWith(prefix) || entryKey.equals(prefix)) { + continue; + } + String[] entryKeys = resolveEntryKeys(entryKey.substring(prefix.length())); + builder.add(entryKeys); + } + } catch (IOException e) { + return null; + } + return builder.build(); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java b/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java new file mode 100644 index 0000000..e08d930 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java @@ -0,0 +1,35 @@ +package dev.zarr.zarrjava.utils; + +import javax.annotation.Nullable; +import java.io.IOException; + +public class ZipUtils { + + // adopted from https://stackoverflow.com/a/9918966 + @Nullable + public static String getZipCommentFromBuffer(byte[] bufArray) throws IOException { + // End of Central Directory (EOCD) record magic number + byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; + int buffLen = bufArray.length; + // Check the buffer from the end + search: + for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { + for (int k = 0; k < EOCD.length; k++) { + if (bufArray[i + k] != EOCD[k]) { + continue search; + } + } + // End of Central Directory found! + int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; + int realLen = buffLen - i - 22; + if (commentLen != realLen) { + throw new IOException("ZIP comment size mismatch: " + + "directory says len is " + commentLen + + ", but file ends after " + realLen + " bytes!"); + } + return new String(bufArray, i + 22, commentLen); + } + return null; + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 89a8b70..bb83142 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -263,6 +263,19 @@ public void testZipStoreV2() throws ZarrException, IOException { assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(fsStore.resolve()), true); } + @Test + public void testReadOnlyZipStore() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testReadOnlyZipStore.zip"); + String archiveComment = "This is a test ZIP archive comment."; + BufferedZipStore zipStore = new BufferedZipStore(path, archiveComment); + writeTestGroupV3(zipStore, true); + zipStore.flush(); + + ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(path); + Assertions.assertEquals(archiveComment, readOnlyZipStore.getArchiveComment(), "ZIP archive comment from ReadOnlyZipStore does not match expected value."); + assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); + } + static Stream localStores() { return Stream.of( From db57be7eb93a9c30e1d05cf900664242cca96397 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 16:54:56 +0100 Subject: [PATCH 18/61] fix ReadOnlyZipStore for zips with 1. leading slashes in paths 2. no sizes in entry headers --- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 516a647..49388d7 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -15,6 +15,11 @@ import static dev.zarr.zarrjava.utils.ZipUtils.getZipCommentFromBuffer; + +/** A Store implementation that provides read-only access to a zip archive stored in an underlying Store. + * Compared to BufferedZipStore, this implementation reads directly from the zip archive without parsing + * its contents into a buffer store first making it more efficient for read-only access to large zip archives. + */ public class ReadOnlyZipStore implements Store, Store.ListableStore { private final StoreHandle underlyingStore; @@ -69,21 +74,24 @@ public ByteBuffer get(String[] keys, long start, long end) { try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { ZipArchiveEntry entry; while ((entry = zis.getNextEntry()) != null) { - if (entry.isDirectory() || !entry.getName().equals(resolveKeys(keys))) { - continue; - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - if (end == -1) { - end = entry.getSize(); + String entryName = entry.getName(); + + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); } - if (start > end) { - throw new IllegalArgumentException("Start position can not be larger than end position. Got start=" + start + ", end=" + end); + if (entry.isDirectory() || !entryName.equals(resolveKeys(keys))) { + continue; } - if (start < 0 || end > entry.getSize()) { - throw new IllegalArgumentException("Start and end positions must be within the bounds of the zip entry size. Entry size=" + entry.getSize() + ", got start=" + start + ", end=" + end); + + if (zis.skip(start) != start) { + throw new IOException("Failed to skip to start position " + start + " in zip entry " + entryName); } - zis.skip(start); - long bytesToRead = end - start; + + long bytesToRead; + if (end != -1) bytesToRead = end - start; + else bytesToRead = Long.MAX_VALUE; + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); byte[] bufferArray = new byte[8192]; int len; while (bytesToRead > 0 && (len = zis.read(bufferArray, 0, (int) Math.min(bufferArray.length, bytesToRead))) != -1) { @@ -144,11 +152,15 @@ public Stream list(String[] keys) { ZipArchiveEntry entry; String prefix = resolveKeys(keys); while ((entry = zis.getNextEntry()) != null) { - String entryKey = entry.getName(); - if (!entryKey.startsWith(prefix) || entryKey.equals(prefix)) { + String entryName = entry.getName(); + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } + + if (!entryName.startsWith(prefix) || entryName.equals(prefix)) { continue; } - String[] entryKeys = resolveEntryKeys(entryKey.substring(prefix.length())); + String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length())); builder.add(entryKeys); } } catch (IOException e) { From 768bd62e19f7ad739f0393df4d77fbe2728f1583 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 17:01:01 +0100 Subject: [PATCH 19/61] add BufferedZipStore parameter flushOnWrite --- .../zarr/zarrjava/store/BufferedZipStore.java | 51 ++++++++++++++++++- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 42 ++++++++------- 2 files changed, 74 insertions(+), 19 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index af12ef0..dc82672 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -25,6 +25,7 @@ public class BufferedZipStore implements Store, Store.ListableStore { private final StoreHandle underlyingStore; private final Store.ListableStore bufferStore; private String archiveComment; + private boolean flushOnWrite; private void writeBuffer() throws IOException{ // create zip file bytes from buffer store and write to underlying store @@ -147,10 +148,11 @@ private void loadBuffer() throws IOException{ } } - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment, boolean flushOnWrite) { this.underlyingStore = underlyingStore; this.bufferStore = bufferStore; this.archiveComment = archiveComment; + this.flushOnWrite = flushOnWrite; try { loadBuffer(); } catch (IOException e) { @@ -158,6 +160,10 @@ public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.Lis } } + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { + this(underlyingStore, bufferStore, archiveComment, true); + } + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { this(underlyingStore, bufferStore, null); } @@ -186,6 +192,35 @@ public BufferedZipStore(@Nonnull String underlyingStorePath) { this(underlyingStorePath, null); } + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, boolean flushOnWrite) { + this(underlyingStore, bufferStore, null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment, boolean flushOnWrite) { + this(underlyingStore, new MemoryStore(), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, boolean flushOnWrite) { + this(underlyingStore, (String) null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment, boolean flushOnWrite) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, boolean flushOnWrite) { + this(underlyingStore, null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment, boolean flushOnWrite) { + this(Paths.get(underlyingStorePath), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath, boolean flushOnWrite) { + this(underlyingStorePath, null, flushOnWrite); + } + + /** * Flushes the buffer and archiveComment to the underlying store as a zip file. */ @@ -228,11 +263,25 @@ public ByteBuffer get(String[] keys, long start, long end) { @Override public void set(String[] keys, ByteBuffer bytes) { bufferStore.set(keys, bytes); + if (flushOnWrite) { + try { + writeBuffer(); + } catch (IOException e) { + throw new RuntimeException("Failed to flush buffer to underlying store after set operation", e); + } + } } @Override public void delete(String[] keys) { bufferStore.delete(keys); + if (flushOnWrite) { + try { + writeBuffer(); + } catch (IOException e) { + throw new RuntimeException("Failed to flush buffer to underlying store after delete operation", e); + } + } } @Nonnull diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index bb83142..86c9606 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -156,32 +156,37 @@ public void testOpenZipStore() throws ZarrException, IOException { BufferedZipStore zipStore = new BufferedZipStore(targetDir); assertIsTestGroupV3(Group.open(zipStore.resolve()), true); + + ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(targetDir); + assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); } - @Test - public void testWriteZipStore() throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testWriteZipStore.zip"); - BufferedZipStore zipStore = new BufferedZipStore(path); + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testWriteZipStore" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); writeTestGroupV3(zipStore, true); - zipStore.flush(); + if(!flushOnWrite) zipStore.flush(); BufferedZipStore zipStoreRead = new BufferedZipStore(path); assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); - Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped"); + Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped" + (flushOnWrite ? "Flush" : "NoFlush")); unzipFile(path, unzippedPath); FilesystemStore fsStore = new FilesystemStore(unzippedPath); assertIsTestGroupV3(Group.open(fsStore.resolve()), true); } - @Test - public void testZipStoreWithComment() throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreWithComment.zip"); + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreWithComment"+ (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); String comment = "{\"ome\": { \"version\": \"XX.YY\" }}"; - BufferedZipStore zipStore = new BufferedZipStore(path, comment); + BufferedZipStore zipStore = new BufferedZipStore(path, comment, flushOnWrite); writeTestGroupV3(zipStore, true); - zipStore.flush(); + if(!flushOnWrite) zipStore.flush(); try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) { String retrievedComment = zipFile.getComment(); @@ -246,12 +251,13 @@ public void testZipStoreRequirements() throws ZarrException, IOException { } - @Test - public void testZipStoreV2() throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreV2.zip"); - BufferedZipStore zipStore = new BufferedZipStore(path); + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreV2" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); writeTestGroupV2(zipStore, true); - zipStore.flush(); + if(!flushOnWrite) zipStore.flush(); BufferedZipStore zipStoreRead = new BufferedZipStore(path); assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(zipStoreRead.resolve()), true); @@ -280,8 +286,8 @@ public void testReadOnlyZipStore() throws ZarrException, IOException { static Stream localStores() { return Stream.of( new MemoryStore(), - new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")) -// new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), + new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) ); } From 38bec2760ad3224789278a6aa3471f1172b41e2a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 17:01:16 +0100 Subject: [PATCH 20/61] fix testMemoryStore --- src/test/java/dev/zarr/zarrjava/ZarrV2Test.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 346fd2a..6522fae 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -406,8 +406,6 @@ public void testMemoryStore() throws ZarrException, IOException { ); group.createGroup("subgroup"); Assertions.assertEquals(2, group.list().count()); - for(String s: storeHandle.list().toArray(String[]::new)) - System.out.println(s); } } \ No newline at end of file From bdcbc463b720935f4c20858ca0da081161b0acfc Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 12 Dec 2025 17:17:52 +0100 Subject: [PATCH 21/61] default flushOnWrite to false --- src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java | 2 +- src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index dc82672..4e0a257 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -161,7 +161,7 @@ public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.Lis } public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { - this(underlyingStore, bufferStore, archiveComment, true); + this(underlyingStore, bufferStore, archiveComment, false); } public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 86c9606..b4cbc63 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -287,7 +287,7 @@ static Stream localStores() { return Stream.of( new MemoryStore(), new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip")) + new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true) ); } From 9014feffa7b7b8fa31ea7a635446e3062a383989 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Dec 2025 11:19:38 +0100 Subject: [PATCH 22/61] fix s3 store get range --- .../java/dev/zarr/zarrjava/store/S3Store.java | 4 ++-- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 58c8d08..6ab1452 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -71,7 +71,7 @@ public ByteBuffer get(String[] keys, long start) { GetObjectRequest req = GetObjectRequest.builder() .bucket(bucketName) .key(resolveKeys(keys)) - .range(String.valueOf(start)) + .range(String.format("bytes=%d-", start)) .build(); return get(req); } @@ -82,7 +82,7 @@ public ByteBuffer get(String[] keys, long start, long end) { GetObjectRequest req = GetObjectRequest.builder() .bucket(bucketName) .key(resolveKeys(keys)) - .range(start +"-"+ end) + .range(String.format("bytes=%d-%d", start, end-1)) // S3 range is inclusive .build(); return get(req); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index b4cbc63..620b304 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -86,6 +86,24 @@ public void testS3Store() throws IOException, ZarrException { Assertions.assertEquals(0, arrayCore.read(new long[]{0,0,0,0}, new int[]{1,1,1,1}).getInt(0)); } + @Test + public void testS3StoreGet() throws IOException, ZarrException { + S3Store s3Store = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "static.webknossos.org", "data"); + String[] keys = new String[]{"zarr_v3", "l4_sample", "color", "1", "zarr.json"}; + + ByteBuffer buffer = s3Store.get(keys); + ByteBuffer bufferWithStart = s3Store.get(keys, 10); + Assertions.assertEquals(10, buffer.remaining()-bufferWithStart.remaining()); + + ByteBuffer bufferWithStartAndEnd = s3Store.get(keys, 0, 10); + Assertions.assertEquals(10, bufferWithStartAndEnd.remaining()); + + } + + @Test public void testHttpStore() throws IOException, ZarrException { HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample"); From 5c74445f353b0f1ce0c7b8333fc4bf2f80dda45b Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 18 Dec 2025 13:56:37 +0100 Subject: [PATCH 23/61] add store.getInputStream --- .../zarr/zarrjava/store/BufferedZipStore.java | 6 ++ .../zarr/zarrjava/store/FilesystemStore.java | 24 ++++++++ .../dev/zarr/zarrjava/store/HttpStore.java | 33 ++++++++++- .../dev/zarr/zarrjava/store/MemoryStore.java | 12 +++- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 39 ++++++++++++- .../java/dev/zarr/zarrjava/store/S3Store.java | 11 ++++ .../java/dev/zarr/zarrjava/store/Store.java | 7 +++ .../dev/zarr/zarrjava/store/StoreHandle.java | 10 ++++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 57 +++++++++++++++++-- 9 files changed, 192 insertions(+), 7 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 4e0a257..c3466af 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -4,6 +4,7 @@ import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; @@ -290,6 +291,11 @@ public StoreHandle resolve(String... keys) { return new StoreHandle(this, keys); } + @Override + public InputStream getInputStream(String[] keys, long start, long end) { + return bufferStore.getInputStream(keys, start, end); + } + @Override public String toString() { return "BufferedZipStore(" + underlyingStore.toString() + ")"; diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 5640a1a..9aeba12 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -1,7 +1,10 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.utils.Utils; +import org.apache.commons.io.input.BoundedInputStream; + import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; @@ -146,4 +149,25 @@ public String toString() { return this.path.toUri().toString().replaceAll("\\/$", ""); } + @Override + public InputStream getInputStream(String[] keys, long start, long end) { + Path keyPath = resolveKeys(keys); + try { + InputStream inputStream = Files.newInputStream(keyPath); + if (start > 0) { + long skipped = inputStream.skip(start); + if (skipped < start) { + throw new IOException("Unable to skip to the desired start position."); + } + } + if (end != -1) { + long bytesToRead = end - start; + return new BoundedInputStream(inputStream, bytesToRead); + } else { + return inputStream; + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 343d251..7fb044f 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -5,7 +5,10 @@ import com.squareup.okhttp.Request; import com.squareup.okhttp.Response; import com.squareup.okhttp.ResponseBody; + +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -101,6 +104,34 @@ public StoreHandle resolve(String... keys) { @Override public String toString() { - return uri; + return uri; } + + @Override + @Nullable + public InputStream getInputStream(String[] keys, long start, long end) { + if (start < 0) { + throw new IllegalArgumentException("Argument 'start' needs to be non-negative."); + } + Request request = new Request.Builder().url(resolveKeys(keys)).header( + "Range", String.format("Bytes=%d-%d", start, end - 1)).build(); + Call call = httpClient.newCall(request); + try { + Response response = call.execute(); + ResponseBody body = response.body(); + if (body == null) return null; + InputStream stream = body.byteStream(); + + // Ensure closing the stream also closes the response + return new FilterInputStream(stream) { + @Override + public void close() throws IOException { + super.close(); + body.close(); + } + }; + } catch (IOException e) { + return null; + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index d97cffe..371e413 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -2,6 +2,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -45,7 +46,7 @@ public ByteBuffer get(String[] keys, long start, long end) { if (bytes == null) return null; if (end < 0) end = bytes.length; if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); - return ByteBuffer.wrap(bytes, (int) start, (int) end); + return ByteBuffer.wrap(bytes, (int) start, (int) (end - start)); } @@ -83,5 +84,14 @@ public StoreHandle resolve(String... keys) { public String toString() { return String.format("", hashCode()); } + + @Override + public InputStream getInputStream(String[] keys, long start, long end) { + byte[] bytes = map.get(resolveKeys(keys)); + if (bytes == null) return null; + if (end < 0) end = bytes.length; + if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); + return new java.io.ByteArrayInputStream(bytes, (int) start, (int)(end - start)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 49388d7..d7a8404 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -3,11 +3,13 @@ import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.io.input.BoundedInputStream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; @@ -83,7 +85,8 @@ public ByteBuffer get(String[] keys, long start, long end) { continue; } - if (zis.skip(start) != start) { + long skipResult = zis.skip(start); + if (skipResult != start) { throw new IOException("Failed to skip to start position " + start + " in zip entry " + entryName); } @@ -168,4 +171,38 @@ public Stream list(String[] keys) { } return builder.build(); } + + @Override + public InputStream getInputStream(String[] keys, long start, long end) { + InputStream baseStream = underlyingStore.getInputStream(); + + try { + ZipArchiveInputStream zis = new ZipArchiveInputStream(baseStream); + ZipArchiveEntry entry; + while ((entry = zis.getNextEntry()) != null) { + String entryName = entry.getName(); + + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } + if (entry.isDirectory() || !entryName.equals(resolveKeys(keys))) { + continue; + } + + long skipResult = zis.skip(start); + if (skipResult != start) { + throw new IOException("Failed to skip to start position " + start + " in zip entry " + entryName); + } + + long bytesToRead; + if (end != -1) bytesToRead = end - start; + else bytesToRead = Long.MAX_VALUE; + + return new BoundedInputStream(zis, bytesToRead); + } + return null; + } catch (IOException e) { + } + return null; + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 6ab1452..37eca90 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -121,6 +121,17 @@ public StoreHandle resolve(String... keys) { return new StoreHandle(this, keys); } + @Override + public InputStream getInputStream(String[] keys, long start, long end) { + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(String.format("bytes=%d-%d", start, end-1)) // S3 range is inclusive + .build(); + ResponseInputStream responseInputStream = s3client.getObject(req); + return responseInputStream; + } + @Override public String toString() { return "s3://" + bucketName + "/" + prefix; diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index 451bf79..ecd2242 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -1,5 +1,6 @@ package dev.zarr.zarrjava.store; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -42,4 +43,10 @@ default Stream list() { return list(new String[]{}); } } + + InputStream getInputStream(String[] keys, long start, long end); + + default InputStream getInputStream(String[] keys) { + return getInputStream(keys, 0, -1); + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index e731a39..84665e8 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -1,6 +1,8 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.utils.Utils; + +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -44,6 +46,14 @@ public ByteBuffer read(long start, long end) { return store.get(keys, start, end); } + public InputStream getInputStream(int start, int end) { + return store.getInputStream(keys, start, end); + } + + public InputStream getInputStream() { + return store.getInputStream(keys); + } + public void set(ByteBuffer bytes) { store.set(keys, bytes); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 620b304..99b373d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -18,6 +18,8 @@ import software.amazon.awssdk.services.s3.S3Client; import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; @@ -103,6 +105,55 @@ public void testS3StoreGet() throws IOException, ZarrException { } + static Stream inputStreamStores() throws IOException { + String[] s3StoreKeys = new String[]{"zarr_v3", "l4_sample", "color", "1", "zarr.json"}; + StoreHandle s3StoreHandle = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "static.webknossos.org", "data") + .resolve(s3StoreKeys); + + byte[] testData = new byte[100]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) i; + } + + StoreHandle memoryStoreHandle = new MemoryStore().resolve(); + memoryStoreHandle.set(ByteBuffer.wrap(testData)); + + StoreHandle fsStoreHandle = new FilesystemStore(TESTOUTPUT.resolve("testInputStreamFS")).resolve("testfile"); + fsStoreHandle.set(ByteBuffer.wrap(testData)); + + zipFile(TESTOUTPUT.resolve("testInputStreamFS"), TESTOUTPUT.resolve("testInputStreamZIP.zip")); + StoreHandle bufferedZipStoreHandle = new BufferedZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip"), true) + .resolve("testfile"); + + StoreHandle readOnlyZipStoreHandle = new ReadOnlyZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip")) + .resolve("testfile"); + + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample") + .resolve("color", "1", "zarr.json"); + return Stream.of( + memoryStoreHandle, + s3StoreHandle, + fsStoreHandle, + bufferedZipStoreHandle, + readOnlyZipStoreHandle, + httpStoreHandle + ); + } + + @ParameterizedTest + @MethodSource("inputStreamStores") + public void testStoreInputStream(StoreHandle storeHandle) throws IOException, ZarrException { + InputStream is = storeHandle.getInputStream(10, 20); + byte[] buffer = new byte[10]; + int bytesRead = is.read(buffer); + Assertions.assertEquals(10, bytesRead); + byte[] expectedBuffer = new byte[10]; + storeHandle.read(10, 20).get(expectedBuffer); + Assertions.assertArrayEquals(expectedBuffer, buffer); + } @Test public void testHttpStore() throws IOException, ZarrException { @@ -115,8 +166,7 @@ public void testHttpStore() throws IOException, ZarrException { @ParameterizedTest @CsvSource({"false", "true",}) public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException { - int[] testData = new int[1024 * 1024]; - Arrays.setAll(testData, p -> p); + int[] testData = testData(); dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve()); Array array = group.createArray("array", b -> b @@ -140,8 +190,7 @@ public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOExcep @ParameterizedTest @CsvSource({"false", "true",}) public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { - int[] testData = new int[1024 * 1024]; - Arrays.setAll(testData, p -> p); + int[] testData = testData(); dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve()); dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b From 7e0e90e8cd81eb269afffd14048db9bde88f7b9d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 18 Dec 2025 15:18:19 +0100 Subject: [PATCH 24/61] add store.getSize --- .../zarr/zarrjava/store/BufferedZipStore.java | 4 +++ .../zarr/zarrjava/store/FilesystemStore.java | 7 +++++ .../dev/zarr/zarrjava/store/HttpStore.java | 26 +++++++++++++++++++ .../dev/zarr/zarrjava/store/MemoryStore.java | 10 ++++++- .../java/dev/zarr/zarrjava/store/S3Store.java | 13 ++++++++++ .../java/dev/zarr/zarrjava/store/Store.java | 2 ++ .../dev/zarr/zarrjava/store/StoreHandle.java | 4 +++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 8 ++++++ 8 files changed, 73 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index c3466af..c0ac7b3 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -296,6 +296,10 @@ public InputStream getInputStream(String[] keys, long start, long end) { return bufferStore.getInputStream(keys, start, end); } + public long getSize(String[] keys) { + return bufferStore.getSize(keys); + } + @Override public String toString() { return "BufferedZipStore(" + underlyingStore.toString() + ")"; diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 9aeba12..dbc8a83 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -170,4 +170,11 @@ public InputStream getInputStream(String[] keys, long start, long end) { throw new RuntimeException(e); } } + public long getSize(String[] keys) { + try { + return Files.size(resolveKeys(keys)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 7fb044f..8dcd75b 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -134,4 +134,30 @@ public void close() throws IOException { return null; } } + @Override + public long getSize(String[] keys) { + // Explicitly request "identity" encoding to prevent OkHttp from adding "gzip" + // and subsequently stripping the Content-Length header. + Request request = new Request.Builder() + .head() + .url(resolveKeys(keys)) + .header("Accept-Encoding", "identity") + .build(); + + Call call = httpClient.newCall(request); + try { + Response response = call.execute(); + if (!response.isSuccessful()) { + throw new IOException("Failed to get size: " + response.code()); + } + + String contentLength = response.header("Content-Length"); + if (contentLength != null) { + return Long.parseLong(contentLength); + } + return -1; + } catch (IOException e) { + throw new RuntimeException(e); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 371e413..09ee39b 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -93,5 +93,13 @@ public InputStream getInputStream(String[] keys, long start, long end) { if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); return new java.io.ByteArrayInputStream(bytes, (int) start, (int)(end - start)); } -} + @Override + public long getSize(String[] keys) { + byte[] bytes = map.get(resolveKeys(keys)); + if (bytes == null) { + throw new RuntimeException(new java.io.FileNotFoundException("Key not found: " + String.join("/", keys))); + } + return bytes.length; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 37eca90..d112db0 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -132,6 +132,19 @@ public InputStream getInputStream(String[] keys, long start, long end) { return responseInputStream; } + @Override + public long getSize(String[] keys) { + HeadObjectRequest req = HeadObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .build(); + try { + return s3client.headObject(req).contentLength(); + } catch (NoSuchKeyException e) { + throw new RuntimeException(e); + } + } + @Override public String toString() { return "s3://" + bucketName + "/" + prefix; diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index ecd2242..3923bde 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -49,4 +49,6 @@ default Stream list() { default InputStream getInputStream(String[] keys) { return getInputStream(keys, 0, -1); } + + long getSize(String[] keys); } diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index 84665e8..e2c9273 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -73,6 +73,10 @@ public Stream list() { return ((Store.ListableStore) store).list(keys); } + public long getSize() { + return store.getSize(keys); + } + @Override public String toString() { return store + "/" + String.join("/", keys); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 99b373d..7b12c75 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -155,6 +155,14 @@ public void testStoreInputStream(StoreHandle storeHandle) throws IOException, Za Assertions.assertArrayEquals(expectedBuffer, buffer); } + @ParameterizedTest + @MethodSource("inputStreamStores") + public void testStoreGetSize(StoreHandle storeHandle) throws IOException, ZarrException { + long size = storeHandle.getSize(); + long actual_size = storeHandle.read().remaining(); + Assertions.assertEquals(actual_size, size); + } + @Test public void testHttpStore() throws IOException, ZarrException { HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample"); From 086d3f8df66eba6b3a956d0673c1ef0aa33bbcfc Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 18 Dec 2025 15:20:48 +0100 Subject: [PATCH 25/61] improve performance of ReadOnlyZipStore.getArchiveComment --- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index d7a8404..454b08e 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -52,28 +52,49 @@ public ByteBuffer get(String[] keys, long start) { } public String getArchiveComment() throws IOException { - ByteBuffer buffer = underlyingStore.read(); - if (buffer == null) { - return null; - } - byte[] bufArray; - if (buffer.hasArray()) { - bufArray = buffer.array(); - } else { - bufArray = new byte[buffer.remaining()]; - buffer.duplicate().get(bufArray); + // Attempt to read from the end of the file to find the EOCD record. + // We try a small chunk first (1KB) which covers most short comments (or no comment), + // then the maximum possible EOCD size (approx 65KB). + int[] readSizes = {1024, 65535 + 22}; + + for (int size : readSizes) { + ByteBuffer buffer; + long fileSize = underlyingStore.getSize(); + + if (fileSize < size){ + buffer = underlyingStore.read(); + } + else { + buffer = underlyingStore.read(fileSize - size); + } + + if (buffer == null) { + return null; + } + + byte[] bufArray; + if (buffer.hasArray()) { + bufArray = buffer.array(); + } else { + bufArray = new byte[buffer.remaining()]; + buffer.duplicate().get(bufArray); + } + + String comment = getZipCommentFromBuffer(bufArray); + if (comment != null) { + return comment; + } } - return getZipCommentFromBuffer(bufArray); + return null; } - @Nullable @Override public ByteBuffer get(String[] keys, long start, long end) { - ByteBuffer buffer = underlyingStore.read(); - if (buffer == null) { + InputStream inputStream = underlyingStore.getInputStream(); + if (inputStream == null) { return null; } - try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { ZipArchiveEntry entry; while ((entry = zis.getNextEntry()) != null) { String entryName = entry.getName(); @@ -147,11 +168,11 @@ public ReadOnlyZipStore(@Nonnull String underlyingStorePath) { public Stream list(String[] keys) { Stream.Builder builder = Stream.builder(); - ByteBuffer buffer = underlyingStore.read(); - if (buffer == null) { + InputStream inputStream = underlyingStore.getInputStream(); + if (inputStream == null) { return builder.build(); } - try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { ZipArchiveEntry entry; String prefix = resolveKeys(keys); while ((entry = zis.getNextEntry()) != null) { @@ -166,9 +187,7 @@ public Stream list(String[] keys) { String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length())); builder.add(entryKeys); } - } catch (IOException e) { - return null; - } + } catch (IOException ignored) {} return builder.build(); } @@ -201,8 +220,42 @@ public InputStream getInputStream(String[] keys, long start, long end) { return new BoundedInputStream(zis, bytesToRead); } return null; + } catch (IOException ignored) {} + return null; + } + + @Override + public long getSize(String[] keys) { + InputStream inputStream = underlyingStore.getInputStream(); + if (inputStream == null) { + throw new RuntimeException(new IOException("Underlying store input stream is null")); + } + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { + ZipArchiveEntry entry; + while ((entry = zis.getNextEntry()) != null) { + String entryName = entry.getName(); + + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } + if (entry.isDirectory() || !entryName.equals(resolveKeys(keys))) { + continue; + } + long size = entry.getSize(); + if (size < 0) { + // read the entire entry to determine size + size = 0; + byte[] bufferArray = new byte[8192]; + int len; + while ((len = zis.read(bufferArray)) != -1) { + size += len; + } + } + return size; + } + throw new RuntimeException(new java.io.FileNotFoundException("Key not found: " + resolveKeys(keys))); } catch (IOException e) { + throw new RuntimeException(e); } - return null; } } From 7d4f4873e31fe7e1e23a961ddb4454f62328898a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 18 Dec 2025 18:00:40 +0100 Subject: [PATCH 26/61] inherit zipstores from common parent and reduce buffers in memory in loadBuffer --- .../zarr/zarrjava/store/BufferedZipStore.java | 96 ++++++++++--------- .../zarr/zarrjava/store/FilesystemStore.java | 4 + .../zarr/zarrjava/store/ReadOnlyZipStore.java | 44 +-------- .../dev/zarr/zarrjava/store/ZipStore.java | 81 ++++++++++++++++ .../dev/zarr/zarrjava/utils/ZipUtils.java | 35 ------- 5 files changed, 138 insertions(+), 122 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/store/ZipStore.java delete mode 100644 src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index c0ac7b3..c0b72a1 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -8,27 +8,45 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Comparator; import java.util.stream.Stream; -import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; import org.apache.commons.compress.archivers.zip.*; import java.util.zip.CRC32; import java.util.zip.ZipEntry; // for STORED constant -import static dev.zarr.zarrjava.utils.ZipUtils.getZipCommentFromBuffer; - /** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. */ -public class BufferedZipStore implements Store, Store.ListableStore { +public class BufferedZipStore extends ZipStore { - private final StoreHandle underlyingStore; private final Store.ListableStore bufferStore; private String archiveComment; - private boolean flushOnWrite; + private final boolean flushOnWrite; + + private final Comparator zipEntryComparator = (a, b) -> { + boolean aIsZarr = a.length > 0 && a[a.length - 1].equals("zarr.json"); + boolean bIsZarr = b.length > 0 && b[b.length - 1].equals("zarr.json"); + // first all zarr.json files + if (aIsZarr && !bIsZarr) { + return -1; + } else if (!aIsZarr && bIsZarr) { + return 1; + } else if (aIsZarr && bIsZarr) { + // sort zarr.json in BFS order within same depth by lexicographical order + if (a.length != b.length) { + return Integer.compare(a.length, b.length); + } else { + return String.join("/", a).compareTo(String.join("/", b)); + } + } else { + // then all other files in lexicographical order + return String.join("/", a).compareTo(String.join("/", b)); + } + }; - private void writeBuffer() throws IOException{ + private void writeBuffer() throws IOException { // create zip file bytes from buffer store and write to underlying store ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) { @@ -36,30 +54,7 @@ private void writeBuffer() throws IOException{ if (archiveComment != null) { zos.setComment(archiveComment); } - Stream entries = bufferStore.list().sorted( - (a, b) -> { - boolean aIsZarr = a.length > 0 && a[a.length - 1].equals("zarr.json"); - boolean bIsZarr = b.length > 0 && b[b.length - 1].equals("zarr.json"); - // first all zarr.json files - if (aIsZarr && !bIsZarr) { - return -1; - } else if (!aIsZarr && bIsZarr) { - return 1; - } else if (aIsZarr && bIsZarr) { - // sort zarr.json in BFS order within same depth by lexicographical order - if (a.length != b.length) { - return Integer.compare(a.length, b.length); - } else { - return String.join("/", a).compareTo(String.join("/", b)); - } - } else { - // then all other files in lexicographical order - return String.join("/", a).compareTo(String.join("/", b)); - } - } - ); - - entries.forEach(keys -> { + bufferStore.list().sorted(zipEntryComparator).forEach(keys -> { try { if (keys == null || keys.length == 0) { // skip root entry @@ -116,22 +111,32 @@ private void writeBuffer() throws IOException{ underlyingStore.set(ByteBuffer.wrap(zipBytes)); } + public void setArchiveComment(@Nullable String archiveComment) throws IOException { + this.archiveComment = archiveComment; + if (flushOnWrite) { + writeBuffer(); + } + } + + public void deleteArchiveComment() throws IOException { + this.setArchiveComment(null); + } - private void loadBuffer() throws IOException{ - // read zip file bytes from underlying store and populate buffer store - ByteBuffer buffer = underlyingStore.read(); - if (buffer == null) { - return; + /** + * Loads the buffer from the underlying store zip file. + */ + private void loadBuffer() throws IOException { + String loadedArchiveComment = super.getArchiveComment(); + if (loadedArchiveComment != null && this.archiveComment == null) { + // don't overwrite existing archiveComment + this.archiveComment = loadedArchiveComment; } - byte[] bufArray; - if (buffer.hasArray()) { - bufArray = buffer.array(); - } else { - bufArray = new byte[buffer.remaining()]; - buffer.duplicate().get(bufArray); + + InputStream inputStream = underlyingStore.getInputStream(); + if (inputStream == null) { + return; } - this.archiveComment = getZipCommentFromBuffer(bufArray); - try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) { + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { ZipArchiveEntry entry; while ((entry = zis.getNextEntry()) != null) { if (entry.isDirectory()) { @@ -150,7 +155,7 @@ private void loadBuffer() throws IOException{ } public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment, boolean flushOnWrite) { - this.underlyingStore = underlyingStore; + super(underlyingStore); this.bufferStore = bufferStore; this.archiveComment = archiveComment; this.flushOnWrite = flushOnWrite; @@ -229,6 +234,7 @@ public void flush() throws IOException { writeBuffer(); } + @Override public String getArchiveComment() { return archiveComment; } diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index dbc8a83..d8d992d 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -153,6 +153,9 @@ public String toString() { public InputStream getInputStream(String[] keys, long start, long end) { Path keyPath = resolveKeys(keys); try { + if (!Files.exists(keyPath)) { + return null; + } InputStream inputStream = Files.newInputStream(keyPath); if (start > 0) { long skipped = inputStream.skip(start); @@ -170,6 +173,7 @@ public InputStream getInputStream(String[] keys, long start, long end) { throw new RuntimeException(e); } } + public long getSize(String[] keys) { try { return Files.size(resolveKeys(keys)); diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 454b08e..7fa2bed 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -15,16 +15,12 @@ import java.nio.file.Paths; import java.util.stream.Stream; -import static dev.zarr.zarrjava.utils.ZipUtils.getZipCommentFromBuffer; - /** A Store implementation that provides read-only access to a zip archive stored in an underlying Store. * Compared to BufferedZipStore, this implementation reads directly from the zip archive without parsing * its contents into a buffer store first making it more efficient for read-only access to large zip archives. */ -public class ReadOnlyZipStore implements Store, Store.ListableStore { - - private final StoreHandle underlyingStore; +public class ReadOnlyZipStore extends ZipStore { String resolveKeys(String[] keys) { return String.join("/", keys); @@ -51,42 +47,6 @@ public ByteBuffer get(String[] keys, long start) { return get(keys, start, -1); } - public String getArchiveComment() throws IOException { - // Attempt to read from the end of the file to find the EOCD record. - // We try a small chunk first (1KB) which covers most short comments (or no comment), - // then the maximum possible EOCD size (approx 65KB). - int[] readSizes = {1024, 65535 + 22}; - - for (int size : readSizes) { - ByteBuffer buffer; - long fileSize = underlyingStore.getSize(); - - if (fileSize < size){ - buffer = underlyingStore.read(); - } - else { - buffer = underlyingStore.read(fileSize - size); - } - - if (buffer == null) { - return null; - } - - byte[] bufArray; - if (buffer.hasArray()) { - bufArray = buffer.array(); - } else { - bufArray = new byte[buffer.remaining()]; - buffer.duplicate().get(bufArray); - } - - String comment = getZipCommentFromBuffer(bufArray); - if (comment != null) { - return comment; - } - } - return null; - } @Nullable @Override public ByteBuffer get(String[] keys, long start, long end) { @@ -153,7 +113,7 @@ public String toString() { } public ReadOnlyZipStore(@Nonnull StoreHandle underlyingStore) { - this.underlyingStore = underlyingStore; + super(underlyingStore); } public ReadOnlyZipStore(@Nonnull Path underlyingStore) { diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java new file mode 100644 index 0000000..5865456 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -0,0 +1,81 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class ZipStore implements Store, Store.ListableStore { + protected final StoreHandle underlyingStore; + + public ZipStore(@Nonnull StoreHandle underlyingStore) { + this.underlyingStore = underlyingStore; + } + + public String getArchiveComment() throws IOException { + // Attempt to read from the end of the file to find the EOCD record. + // We try a small chunk first (1KB) which covers most short comments (or no comment), + // then the maximum possible EOCD size (approx 65KB). + if (!underlyingStore.exists()) { + return null; + } + int[] readSizes = {1024, 65535 + 22}; + + for (int size : readSizes) { + ByteBuffer buffer; + long fileSize = underlyingStore.getSize(); + + if (fileSize < size){ + buffer = underlyingStore.read(); + } + else { + buffer = underlyingStore.read(fileSize - size); + } + + if (buffer == null) { + return null; + } + + byte[] bufArray; + if (buffer.hasArray()) { + bufArray = buffer.array(); + } else { + bufArray = new byte[buffer.remaining()]; + buffer.duplicate().get(bufArray); + } + + String comment = getZipCommentFromBuffer(bufArray); + if (comment != null) { + return comment; + } + } + return null; + } + + // adopted from https://stackoverflow.com/a/9918966 + @Nullable + public static String getZipCommentFromBuffer(byte[] bufArray) throws IOException { + // End of Central Directory (EOCD) record magic number + byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; + int buffLen = bufArray.length; + // Check the buffer from the end + search: + for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { + for (int k = 0; k < EOCD.length; k++) { + if (bufArray[i + k] != EOCD[k]) { + continue search; + } + } + // End of Central Directory found! + int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; + int realLen = buffLen - i - 22; + if (commentLen != realLen) { + throw new IOException("ZIP comment size mismatch: " + + "directory says len is " + commentLen + + ", but file ends after " + realLen + " bytes!"); + } + return new String(bufArray, i + 22, commentLen); + } + return null; + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java b/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java deleted file mode 100644 index e08d930..0000000 --- a/src/main/java/dev/zarr/zarrjava/utils/ZipUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -package dev.zarr.zarrjava.utils; - -import javax.annotation.Nullable; -import java.io.IOException; - -public class ZipUtils { - - // adopted from https://stackoverflow.com/a/9918966 - @Nullable - public static String getZipCommentFromBuffer(byte[] bufArray) throws IOException { - // End of Central Directory (EOCD) record magic number - byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; - int buffLen = bufArray.length; - // Check the buffer from the end - search: - for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { - for (int k = 0; k < EOCD.length; k++) { - if (bufArray[i + k] != EOCD[k]) { - continue search; - } - } - // End of Central Directory found! - int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; - int realLen = buffLen - i - 22; - if (commentLen != realLen) { - throw new IOException("ZIP comment size mismatch: " - + "directory says len is " + commentLen - + ", but file ends after " + realLen + " bytes!"); - } - return new String(bufArray, i + 22, commentLen); - } - return null; - } - -} From 72ef229905339751a182566a3b45af53996c366c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 11:12:52 +0100 Subject: [PATCH 27/61] fix s3 test buckets --- .../dev/zarr/zarrjava/store/StoreHandle.java | 4 +- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 39 ++++++++----------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index e2c9273..fe5c87c 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -13,9 +13,9 @@ public class StoreHandle { @Nonnull - final Store store; + public final Store store; @Nonnull - final String[] keys; + public final String[] keys; public StoreHandle(@Nonnull Store store, @Nonnull String... keys) { this.store = store; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 07bbd21..8f93631 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -74,8 +74,7 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } - @Test - public void testS3Store() throws IOException, ZarrException { + static StoreHandle createS3StoreHandle() { S3Store s3Store = new S3Store(S3Client.builder() .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) .region(Region.US_EAST_1) // required, but ignored @@ -86,40 +85,36 @@ public void testS3Store() throws IOException, ZarrException { ) .credentialsProvider(AnonymousCredentialsProvider.create()) .build(), "idr", "zarr/v0.5/idr0033A"); + return s3Store.resolve("BR00109990_C2.zarr", "0", "0"); + } - Array arrayV3 = Array.open(s3Store.resolve("BR00109990_C2.zarr", "0", "0")); + @Test + public void testS3Store() throws IOException, ZarrException { + StoreHandle s3StoreHandle = createS3StoreHandle(); + Array arrayV3 = Array.open(s3StoreHandle); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayV3.metadata().shape); Assertions.assertEquals(574, arrayV3.read(new long[]{0,0,0}, new int[]{1,1,1}).getInt(0)); - dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(s3Store.resolve("BR00109990_C2.zarr", "0", "0")); + dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(s3StoreHandle); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayCore.metadata().shape); Assertions.assertEquals(574, arrayCore.read(new long[]{0,0,0}, new int[]{1,1,1}).getInt(0)); } @Test - public void testS3StoreGet() throws IOException, ZarrException { - S3Store s3Store = new S3Store(S3Client.builder() - .region(Region.of("eu-west-1")) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "static.webknossos.org", "data"); - String[] keys = new String[]{"zarr_v3", "l4_sample", "color", "1", "zarr.json"}; - - ByteBuffer buffer = s3Store.get(keys); - ByteBuffer bufferWithStart = s3Store.get(keys, 10); + public void testS3StoreGet() throws ZarrException { + StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); + S3Store s3Store = (S3Store) s3StoreHandle.store; + ByteBuffer buffer = s3Store.get(s3StoreHandle.keys); + ByteBuffer bufferWithStart = s3Store.get(s3StoreHandle.keys, 10); Assertions.assertEquals(10, buffer.remaining()-bufferWithStart.remaining()); - ByteBuffer bufferWithStartAndEnd = s3Store.get(keys, 0, 10); + ByteBuffer bufferWithStartAndEnd = s3Store.get(s3StoreHandle.keys, 0, 10); Assertions.assertEquals(10, bufferWithStartAndEnd.remaining()); } static Stream inputStreamStores() throws IOException { - String[] s3StoreKeys = new String[]{"zarr_v3", "l4_sample", "color", "1", "zarr.json"}; - StoreHandle s3StoreHandle = new S3Store(S3Client.builder() - .region(Region.of("eu-west-1")) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "static.webknossos.org", "data") - .resolve(s3StoreKeys); + StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); byte[] testData = new byte[100]; for (int i = 0; i < testData.length; i++) { @@ -153,7 +148,7 @@ static Stream inputStreamStores() throws IOException { @ParameterizedTest @MethodSource("inputStreamStores") - public void testStoreInputStream(StoreHandle storeHandle) throws IOException, ZarrException { + public void testStoreInputStream(StoreHandle storeHandle) throws IOException { InputStream is = storeHandle.getInputStream(10, 20); byte[] buffer = new byte[10]; int bytesRead = is.read(buffer); @@ -165,7 +160,7 @@ public void testStoreInputStream(StoreHandle storeHandle) throws IOException, Za @ParameterizedTest @MethodSource("inputStreamStores") - public void testStoreGetSize(StoreHandle storeHandle) throws IOException, ZarrException { + public void testStoreGetSize(StoreHandle storeHandle) { long size = storeHandle.getSize(); long actual_size = storeHandle.read().remaining(); Assertions.assertEquals(actual_size, size); From e8fb440112bb5be9c48ae80ba8f7e75d1c2ccb1c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 12:58:32 +0100 Subject: [PATCH 28/61] reduce getArchiveComment store requests --- src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java | 2 ++ src/main/java/dev/zarr/zarrjava/store/HttpStore.java | 2 +- src/main/java/dev/zarr/zarrjava/store/MemoryStore.java | 2 +- .../java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java | 2 +- src/main/java/dev/zarr/zarrjava/store/S3Store.java | 2 +- src/main/java/dev/zarr/zarrjava/store/Store.java | 6 ++++++ src/main/java/dev/zarr/zarrjava/store/ZipStore.java | 7 +++++-- 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index d8d992d..f78c130 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -177,6 +177,8 @@ public InputStream getInputStream(String[] keys, long start, long end) { public long getSize(String[] keys) { try { return Files.size(resolveKeys(keys)); + } catch (NoSuchFileException e) { + return -1; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index 8dcd75b..805e446 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -148,7 +148,7 @@ public long getSize(String[] keys) { try { Response response = call.execute(); if (!response.isSuccessful()) { - throw new IOException("Failed to get size: " + response.code()); + return -1; } String contentLength = response.header("Content-Length"); diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 09ee39b..7b0de58 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -98,7 +98,7 @@ public InputStream getInputStream(String[] keys, long start, long end) { public long getSize(String[] keys) { byte[] bytes = map.get(resolveKeys(keys)); if (bytes == null) { - throw new RuntimeException(new java.io.FileNotFoundException("Key not found: " + String.join("/", keys))); + return -1; } return bytes.length; } diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 7fa2bed..e47f3e8 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -213,7 +213,7 @@ public long getSize(String[] keys) { } return size; } - throw new RuntimeException(new java.io.FileNotFoundException("Key not found: " + resolveKeys(keys))); + return -1; // file not found } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index d112db0..1c8b1a0 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -141,7 +141,7 @@ public long getSize(String[] keys) { try { return s3client.headObject(req).contentLength(); } catch (NoSuchKeyException e) { - throw new RuntimeException(e); + return -1; } } diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index 3923bde..1625b41 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -50,5 +50,11 @@ default InputStream getInputStream(String[] keys) { return getInputStream(keys, 0, -1); } +/** + * Gets the size in bytes of the data stored at the given keys. + * + * @param keys The keys identifying the data. + * @return The size in bytes of the data stored at the given keys. -1 if the keys do not exist. + */ long getSize(String[] keys); } diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java index 5865456..d239e92 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -16,14 +16,14 @@ public String getArchiveComment() throws IOException { // Attempt to read from the end of the file to find the EOCD record. // We try a small chunk first (1KB) which covers most short comments (or no comment), // then the maximum possible EOCD size (approx 65KB). - if (!underlyingStore.exists()) { + long fileSize = underlyingStore.getSize(); + if (fileSize < 22) { return null; } int[] readSizes = {1024, 65535 + 22}; for (int size : readSizes) { ByteBuffer buffer; - long fileSize = underlyingStore.getSize(); if (fileSize < size){ buffer = underlyingStore.read(); @@ -48,6 +48,9 @@ public String getArchiveComment() throws IOException { if (comment != null) { return comment; } + if (fileSize < size){ + break; + } } return null; } From 8b390bc9719dc60b28c130da0b14ac9fb39c7d91 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 14:23:18 +0100 Subject: [PATCH 29/61] format --- .../zarr/zarrjava/store/BufferedZipStore.java | 177 +++++++++--------- .../zarr/zarrjava/store/FilesystemStore.java | 38 ++-- .../dev/zarr/zarrjava/store/HttpStore.java | 21 +-- .../dev/zarr/zarrjava/store/MemoryStore.java | 24 +-- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 34 ++-- .../java/dev/zarr/zarrjava/store/S3Store.java | 46 ++--- .../java/dev/zarr/zarrjava/store/Store.java | 56 +++--- .../dev/zarr/zarrjava/store/StoreHandle.java | 33 ++-- .../dev/zarr/zarrjava/store/ZipStore.java | 61 +++--- src/test/java/dev/zarr/zarrjava/Utils.java | 10 +- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 171 ++++++++--------- 11 files changed, 325 insertions(+), 346 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index c0b72a1..934da19 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -1,5 +1,10 @@ package dev.zarr.zarrjava.store; +import org.apache.commons.compress.archivers.zip.Zip64Mode; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; + import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.ByteArrayOutputStream; @@ -10,21 +15,17 @@ import java.nio.file.Paths; import java.util.Comparator; import java.util.stream.Stream; - -import org.apache.commons.compress.archivers.zip.*; - import java.util.zip.CRC32; -import java.util.zip.ZipEntry; // for STORED constant +import java.util.zip.ZipEntry; -/** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. +/** + * A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file. */ public class BufferedZipStore extends ZipStore { private final Store.ListableStore bufferStore; - private String archiveComment; private final boolean flushOnWrite; - private final Comparator zipEntryComparator = (a, b) -> { boolean aIsZarr = a.length > 0 && a[a.length - 1].equals("zarr.json"); boolean bIsZarr = b.length > 0 && b[b.length - 1].equals("zarr.json"); @@ -45,6 +46,79 @@ public class BufferedZipStore extends ZipStore { return String.join("/", a).compareTo(String.join("/", b)); } }; + private String archiveComment; + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment, boolean flushOnWrite) { + super(underlyingStore); + this.bufferStore = bufferStore; + this.archiveComment = archiveComment; + this.flushOnWrite = flushOnWrite; + try { + loadBuffer(); + } catch (IOException e) { + throw new RuntimeException("Failed to load buffer from underlying store", e); + } + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { + this(underlyingStore, bufferStore, archiveComment, false); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { + this(underlyingStore, bufferStore, null); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment) { + this(underlyingStore, new MemoryStore(), archiveComment); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore) { + this(underlyingStore, (String) null); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment); + } + + public BufferedZipStore(@Nonnull Path underlyingStore) { + this(underlyingStore, null); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment) { + this(Paths.get(underlyingStorePath), archiveComment); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath) { + this(underlyingStorePath, null); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, boolean flushOnWrite) { + this(underlyingStore, bufferStore, null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment, boolean flushOnWrite) { + this(underlyingStore, new MemoryStore(), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull StoreHandle underlyingStore, boolean flushOnWrite) { + this(underlyingStore, (String) null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment, boolean flushOnWrite) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull Path underlyingStore, boolean flushOnWrite) { + this(underlyingStore, null, flushOnWrite); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment, boolean flushOnWrite) { + this(Paths.get(underlyingStorePath), archiveComment, flushOnWrite); + } + + public BufferedZipStore(@Nonnull String underlyingStorePath, boolean flushOnWrite) { + this(underlyingStorePath, null, flushOnWrite); + } private void writeBuffer() throws IOException { // create zip file bytes from buffer store and write to underlying store @@ -111,13 +185,6 @@ private void writeBuffer() throws IOException { underlyingStore.set(ByteBuffer.wrap(zipBytes)); } - public void setArchiveComment(@Nullable String archiveComment) throws IOException { - this.archiveComment = archiveComment; - if (flushOnWrite) { - writeBuffer(); - } - } - public void deleteArchiveComment() throws IOException { this.setArchiveComment(null); } @@ -154,81 +221,8 @@ private void loadBuffer() throws IOException { } } - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment, boolean flushOnWrite) { - super(underlyingStore); - this.bufferStore = bufferStore; - this.archiveComment = archiveComment; - this.flushOnWrite = flushOnWrite; - try { - loadBuffer(); - } catch (IOException e) { - throw new RuntimeException("Failed to load buffer from underlying store", e); - } - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) { - this(underlyingStore, bufferStore, archiveComment, false); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) { - this(underlyingStore, bufferStore, null); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment) { - this(underlyingStore, new MemoryStore(), archiveComment); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore) { - this(underlyingStore, (String) null); - } - - public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment) { - this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment); - } - - public BufferedZipStore(@Nonnull Path underlyingStore) { - this(underlyingStore, null); - } - - public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment) { - this(Paths.get(underlyingStorePath), archiveComment); - } - - public BufferedZipStore(@Nonnull String underlyingStorePath) { - this(underlyingStorePath, null); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, boolean flushOnWrite) { - this(underlyingStore, bufferStore, null, flushOnWrite); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment, boolean flushOnWrite) { - this(underlyingStore, new MemoryStore(), archiveComment, flushOnWrite); - } - - public BufferedZipStore(@Nonnull StoreHandle underlyingStore, boolean flushOnWrite) { - this(underlyingStore, (String) null, flushOnWrite); - } - - public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment, boolean flushOnWrite) { - this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment, flushOnWrite); - } - - public BufferedZipStore(@Nonnull Path underlyingStore, boolean flushOnWrite) { - this(underlyingStore, null, flushOnWrite); - } - - public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment, boolean flushOnWrite) { - this(Paths.get(underlyingStorePath), archiveComment, flushOnWrite); - } - - public BufferedZipStore(@Nonnull String underlyingStorePath, boolean flushOnWrite) { - this(underlyingStorePath, null, flushOnWrite); - } - - /** - * Flushes the buffer and archiveComment to the underlying store as a zip file. + * Flushes the buffer and archiveComment to the underlying store as a zip file. */ public void flush() throws IOException { writeBuffer(); @@ -239,6 +233,13 @@ public String getArchiveComment() { return archiveComment; } + public void setArchiveComment(@Nullable String archiveComment) throws IOException { + this.archiveComment = archiveComment; + if (flushOnWrite) { + writeBuffer(); + } + } + @Override public Stream list(String[] keys) { return bufferStore.list(keys); diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index e4e886e..c89b3e6 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -1,11 +1,10 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.utils.Utils; +import org.apache.commons.io.input.BoundedInputStream; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.apache.commons.io.input.BoundedInputStream; - import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; @@ -110,26 +109,27 @@ public void set(String[] keys, ByteBuffer bytes) { } } - @Override - public void delete(String[] keys) { - try { - Files.delete(resolveKeys(keys)); - } catch (NoSuchFileException e) { - // ignore - } catch (IOException e) { - throw new RuntimeException(e); + @Override + public void delete(String[] keys) { + try { + Files.delete(resolveKeys(keys)); + } catch (NoSuchFileException e) { + // ignore + } catch (IOException e) { + throw new RuntimeException(e); + } } - } - public Stream list(String[] keys) { + + public Stream list(String[] keys) { try { return Files.list(resolveKeys(keys)).map(path -> { - Path relativePath = resolveKeys(keys).relativize(path); - String[] parts = new String[relativePath.getNameCount()]; - for (int i = 0; i < relativePath.getNameCount(); i++) { - parts[i] = relativePath.getName(i).toString(); - } - return parts; - }); + Path relativePath = resolveKeys(keys).relativize(path); + String[] parts = new String[relativePath.getNameCount()]; + for (int i = 0; i < relativePath.getNameCount(); i++) { + parts[i] = relativePath.getName(i).toString(); + } + return parts; + }); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java index beac6dd..31d5a0a 100644 --- a/src/main/java/dev/zarr/zarrjava/store/HttpStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/HttpStore.java @@ -1,20 +1,12 @@ package dev.zarr.zarrjava.store; -import com.squareup.okhttp.Call; -import com.squareup.okhttp.OkHttpClient; -import com.squareup.okhttp.Request; -import com.squareup.okhttp.Response; -import com.squareup.okhttp.ResponseBody; - -import java.io.FilterInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; import com.squareup.okhttp.*; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.FilterInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; public class HttpStore implements Store { @@ -106,10 +98,10 @@ public StoreHandle resolve(String... keys) { return new StoreHandle(this, keys); } - @Override - public String toString() { - return uri; - } + @Override + public String toString() { + return uri; + } @Override @Nullable @@ -138,6 +130,7 @@ public void close() throws IOException { return null; } } + @Override public long getSize(String[] keys) { // Explicitly request "identity" encoding to prevent OkHttp from adding "gzip" diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 51506d3..5e4a7c7 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -61,17 +61,17 @@ public void delete(String[] keys) { } public Stream list(String[] keys) { - List prefix = resolveKeys(keys); - Set> allKeys = new HashSet<>(); - - for (List k : map.keySet()) { - if (k.size() <= prefix.size() || ! k.subList(0, prefix.size()).equals(prefix)) - continue; - for (int i = prefix.size(); i < k.size(); i++) { - allKeys.add(k.subList(0, i+1)); - } - } - return allKeys.stream().map(k -> k.toArray(new String[0])); + List prefix = resolveKeys(keys); + Set> allKeys = new HashSet<>(); + + for (List k : map.keySet()) { + if (k.size() <= prefix.size() || !k.subList(0, prefix.size()).equals(prefix)) + continue; + for (int i = prefix.size(); i < k.size(); i++) { + allKeys.add(k.subList(0, i + 1)); + } + } + return allKeys.stream().map(k -> k.toArray(new String[0])); } @Nonnull @@ -91,7 +91,7 @@ public InputStream getInputStream(String[] keys, long start, long end) { if (bytes == null) return null; if (end < 0) end = bytes.length; if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); - return new java.io.ByteArrayInputStream(bytes, (int) start, (int)(end - start)); + return new java.io.ByteArrayInputStream(bytes, (int) start, (int) (end - start)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index e47f3e8..d518a1e 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -1,6 +1,5 @@ package dev.zarr.zarrjava.store; -import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.input.BoundedInputStream; @@ -16,12 +15,25 @@ import java.util.stream.Stream; -/** A Store implementation that provides read-only access to a zip archive stored in an underlying Store. +/** + * A Store implementation that provides read-only access to a zip archive stored in an underlying Store. * Compared to BufferedZipStore, this implementation reads directly from the zip archive without parsing * its contents into a buffer store first making it more efficient for read-only access to large zip archives. */ public class ReadOnlyZipStore extends ZipStore { + public ReadOnlyZipStore(@Nonnull StoreHandle underlyingStore) { + super(underlyingStore); + } + + public ReadOnlyZipStore(@Nonnull Path underlyingStore) { + this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString())); + } + + public ReadOnlyZipStore(@Nonnull String underlyingStorePath) { + this(Paths.get(underlyingStorePath)); + } + String resolveKeys(String[] keys) { return String.join("/", keys); } @@ -112,18 +124,6 @@ public String toString() { return "ReadOnlyZipStore(" + underlyingStore.toString() + ")"; } - public ReadOnlyZipStore(@Nonnull StoreHandle underlyingStore) { - super(underlyingStore); - } - - public ReadOnlyZipStore(@Nonnull Path underlyingStore) { - this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString())); - } - - public ReadOnlyZipStore(@Nonnull String underlyingStorePath) { - this(Paths.get(underlyingStorePath)); - } - @Override public Stream list(String[] keys) { Stream.Builder builder = Stream.builder(); @@ -147,7 +147,8 @@ public Stream list(String[] keys) { String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length())); builder.add(entryKeys); } - } catch (IOException ignored) {} + } catch (IOException ignored) { + } return builder.build(); } @@ -180,7 +181,8 @@ public InputStream getInputStream(String[] keys, long start, long end) { return new BoundedInputStream(zis, bytesToRead); } return null; - } catch (IOException ignored) {} + } catch (IOException ignored) { + } return null; } diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index db5027a..2b9c73d 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -82,7 +82,7 @@ public ByteBuffer get(String[] keys, long start, long end) { GetObjectRequest req = GetObjectRequest.builder() .bucket(bucketName) .key(resolveKeys(keys)) - .range(String.format("bytes=%d-%d", start, end-1)) // S3 range is inclusive + .range(String.format("bytes=%d-%d", start, end - 1)) // S3 range is inclusive .build(); return get(req); } @@ -122,30 +122,30 @@ public StoreHandle resolve(String... keys) { } @Override - public InputStream getInputStream(String[] keys, long start, long end) { - GetObjectRequest req = GetObjectRequest.builder() - .bucket(bucketName) - .key(resolveKeys(keys)) - .range(String.format("bytes=%d-%d", start, end-1)) // S3 range is inclusive - .build(); - ResponseInputStream responseInputStream = s3client.getObject(req); - return responseInputStream; - } - - @Override - public long getSize(String[] keys) { - HeadObjectRequest req = HeadObjectRequest.builder() - .bucket(bucketName) - .key(resolveKeys(keys)) - .build(); - try { - return s3client.headObject(req).contentLength(); - } catch (NoSuchKeyException e) { - return -1; + public InputStream getInputStream(String[] keys, long start, long end) { + GetObjectRequest req = GetObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .range(String.format("bytes=%d-%d", start, end - 1)) // S3 range is inclusive + .build(); + ResponseInputStream responseInputStream = s3client.getObject(req); + return responseInputStream; + } + + @Override + public long getSize(String[] keys) { + HeadObjectRequest req = HeadObjectRequest.builder() + .bucket(bucketName) + .key(resolveKeys(keys)) + .build(); + try { + return s3client.headObject(req).contentLength(); + } catch (NoSuchKeyException e) { + return -1; + } } - } - @Override + @Override public String toString() { return "s3://" + bucketName + "/" + prefix; } diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index 8aa1563..3747aed 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -1,10 +1,10 @@ package dev.zarr.zarrjava.store; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.io.InputStream; import java.nio.ByteBuffer; import java.util.stream.Stream; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; public interface Store { @@ -26,35 +26,35 @@ public interface Store { @Nonnull StoreHandle resolve(String... keys); - interface ListableStore extends Store { + InputStream getInputStream(String[] keys, long start, long end); - /** - * Lists all keys in the store that match the given prefix keys. Keys are represented as arrays of strings, - * where each string is a segment of the key path. - * Keys that are exactly equal to the prefix are not included in the results. - * Keys that do not contain data (i.e. "directories") are included in the results. - * - * @param keys The prefix keys to match. - * @return A stream of key arrays that match the given prefix. Prefixed keys are not included in the results. - */ - Stream list(String[] keys); - - default Stream list() { - return list(new String[]{}); + default InputStream getInputStream(String[] keys) { + return getInputStream(keys, 0, -1); } - } - InputStream getInputStream(String[] keys, long start, long end); + /** + * Gets the size in bytes of the data stored at the given keys. + * + * @param keys The keys identifying the data. + * @return The size in bytes of the data stored at the given keys. -1 if the keys do not exist. + */ + long getSize(String[] keys); - default InputStream getInputStream(String[] keys) { - return getInputStream(keys, 0, -1); - } + interface ListableStore extends Store { -/** - * Gets the size in bytes of the data stored at the given keys. - * - * @param keys The keys identifying the data. - * @return The size in bytes of the data stored at the given keys. -1 if the keys do not exist. - */ - long getSize(String[] keys); + /** + * Lists all keys in the store that match the given prefix keys. Keys are represented as arrays of strings, + * where each string is a segment of the key path. + * Keys that are exactly equal to the prefix are not included in the results. + * Keys that do not contain data (i.e. "directories") are included in the results. + * + * @param keys The prefix keys to match. + * @return A stream of key arrays that match the given prefix. Prefixed keys are not included in the results. + */ + Stream list(String[] keys); + + default Stream list() { + return list(new String[]{}); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index dd3a7b6..e7bd8eb 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -4,7 +4,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; - import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; @@ -15,8 +14,8 @@ public class StoreHandle { @Nonnull public final Store store; - @Nonnull - public final String[] keys; + @Nonnull + public final String[] keys; public StoreHandle(@Nonnull Store store, @Nonnull String... keys) { this.store = store; @@ -48,16 +47,16 @@ public ByteBuffer read(long start, long end) { } public InputStream getInputStream(int start, int end) { - return store.getInputStream(keys, start, end); - } + return store.getInputStream(keys, start, end); + } - public InputStream getInputStream() { - return store.getInputStream(keys); - } + public InputStream getInputStream() { + return store.getInputStream(keys); + } - public void set(ByteBuffer bytes) { - store.set(keys, bytes); - } + public void set(ByteBuffer bytes) { + store.set(keys, bytes); + } public void delete() { store.delete(keys); @@ -75,13 +74,13 @@ public Stream list() { } public long getSize() { - return store.getSize(keys); - } + return store.getSize(keys); + } - @Override - public String toString() { - return store + "/" + String.join("/", keys); - } + @Override + public String toString() { + return store + "/" + String.join("/", keys); + } public StoreHandle resolve(String... subKeys) { return new StoreHandle(store, Utils.concatArrays(keys, subKeys)); diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java index d239e92..11cb7f2 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -12,6 +12,33 @@ public ZipStore(@Nonnull StoreHandle underlyingStore) { this.underlyingStore = underlyingStore; } + // adopted from https://stackoverflow.com/a/9918966 + @Nullable + public static String getZipCommentFromBuffer(byte[] bufArray) throws IOException { + // End of Central Directory (EOCD) record magic number + byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; + int buffLen = bufArray.length; + // Check the buffer from the end + search: + for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { + for (int k = 0; k < EOCD.length; k++) { + if (bufArray[i + k] != EOCD[k]) { + continue search; + } + } + // End of Central Directory found! + int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; + int realLen = buffLen - i - 22; + if (commentLen != realLen) { + throw new IOException("ZIP comment size mismatch: " + + "directory says len is " + commentLen + + ", but file ends after " + realLen + " bytes!"); + } + return new String(bufArray, i + 22, commentLen); + } + return null; + } + public String getArchiveComment() throws IOException { // Attempt to read from the end of the file to find the EOCD record. // We try a small chunk first (1KB) which covers most short comments (or no comment), @@ -25,10 +52,9 @@ public String getArchiveComment() throws IOException { for (int size : readSizes) { ByteBuffer buffer; - if (fileSize < size){ + if (fileSize < size) { buffer = underlyingStore.read(); - } - else { + } else { buffer = underlyingStore.read(fileSize - size); } @@ -48,37 +74,10 @@ public String getArchiveComment() throws IOException { if (comment != null) { return comment; } - if (fileSize < size){ + if (fileSize < size) { break; } } return null; } - - // adopted from https://stackoverflow.com/a/9918966 - @Nullable - public static String getZipCommentFromBuffer(byte[] bufArray) throws IOException { - // End of Central Directory (EOCD) record magic number - byte[] EOCD = {0x50, 0x4b, 0x05, 0x06}; - int buffLen = bufArray.length; - // Check the buffer from the end - search: - for (int i = buffLen - EOCD.length - 22; i >= 0; i--) { - for (int k = 0; k < EOCD.length; k++) { - if (bufArray[i + k] != EOCD[k]) { - continue search; - } - } - // End of Central Directory found! - int commentLen = bufArray[i + 20] + bufArray[i + 21] * 256; - int realLen = buffLen - i - 22; - if (commentLen != realLen) { - throw new IOException("ZIP comment size mismatch: " - + "directory says len is " + commentLen - + ", but file ends after " + realLen + " bytes!"); - } - return new String(bufArray, i + 22, commentLen); - } - return null; - } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/Utils.java b/src/test/java/dev/zarr/zarrjava/Utils.java index da57f0d..5356af6 100644 --- a/src/test/java/dev/zarr/zarrjava/Utils.java +++ b/src/test/java/dev/zarr/zarrjava/Utils.java @@ -1,15 +1,11 @@ package dev.zarr.zarrjava; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.BufferedOutputStream; -import java.nio.file.Path; +import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; public class Utils { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 7af2597..b9fae63 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -3,22 +3,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.*; -import dev.zarr.zarrjava.core.*; -import org.apache.commons.compress.archivers.zip.*; - import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; -import java.net.URI; -import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.HttpStore; -import dev.zarr.zarrjava.store.MemoryStore; -import dev.zarr.zarrjava.store.S3Store; -import dev.zarr.zarrjava.v3.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -26,29 +17,83 @@ import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; import java.net.URI; +import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.stream.Stream; -import java.nio.file.Path; import java.util.zip.ZipEntry; import static dev.zarr.zarrjava.Utils.unzipFile; import static dev.zarr.zarrjava.Utils.zipFile; - import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { + static StoreHandle createS3StoreHandle() { + S3Store s3Store = new S3Store(S3Client.builder() + .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) + .region(Region.US_EAST_1) // required, but ignored + .serviceConfiguration( + S3Configuration.builder() + .pathStyleAccessEnabled(true) // required + .build() + ) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "idr", "zarr/v0.5/idr0033A"); + return s3Store.resolve("BR00109990_C2.zarr", "0", "0"); + } + + static Stream inputStreamStores() throws IOException { + StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); + + byte[] testData = new byte[100]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) i; + } + + StoreHandle memoryStoreHandle = new MemoryStore().resolve(); + memoryStoreHandle.set(ByteBuffer.wrap(testData)); + + StoreHandle fsStoreHandle = new FilesystemStore(TESTOUTPUT.resolve("testInputStreamFS")).resolve("testfile"); + fsStoreHandle.set(ByteBuffer.wrap(testData)); + + zipFile(TESTOUTPUT.resolve("testInputStreamFS"), TESTOUTPUT.resolve("testInputStreamZIP.zip")); + StoreHandle bufferedZipStoreHandle = new BufferedZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip"), true) + .resolve("testfile"); + + StoreHandle readOnlyZipStoreHandle = new ReadOnlyZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip")) + .resolve("testfile"); + + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample") + .resolve("color", "1", "zarr.json"); + return Stream.of( + memoryStoreHandle, + s3StoreHandle, + fsStoreHandle, + bufferedZipStoreHandle, + readOnlyZipStoreHandle, + httpStoreHandle + ); + } + + static Stream localStores() { + return Stream.of( + new MemoryStore(), + new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), + new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true) + ); + } + @Test public void testFileSystemStores() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTDATA); ObjectMapper objectMapper = makeObjectMapper(); GroupMetadata groupMetadata = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), dev.zarr.zarrjava.v3.GroupMetadata.class ); @@ -57,7 +102,7 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), dev.zarr.zarrjava.v3.ArrayMetadata.class); String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); @@ -80,20 +125,6 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } - static StoreHandle createS3StoreHandle() { - S3Store s3Store = new S3Store(S3Client.builder() - .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) - .region(Region.US_EAST_1) // required, but ignored - .serviceConfiguration( - S3Configuration.builder() - .pathStyleAccessEnabled(true) // required - .build() - ) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "idr", "zarr/v0.5/idr0033A"); - return s3Store.resolve("BR00109990_C2.zarr", "0", "0"); - } - @Test public void testS3Store() throws IOException, ZarrException { StoreHandle s3StoreHandle = createS3StoreHandle(); @@ -112,46 +143,13 @@ public void testS3StoreGet() throws ZarrException { S3Store s3Store = (S3Store) s3StoreHandle.store; ByteBuffer buffer = s3Store.get(s3StoreHandle.keys); ByteBuffer bufferWithStart = s3Store.get(s3StoreHandle.keys, 10); - Assertions.assertEquals(10, buffer.remaining()-bufferWithStart.remaining()); + Assertions.assertEquals(10, buffer.remaining() - bufferWithStart.remaining()); ByteBuffer bufferWithStartAndEnd = s3Store.get(s3StoreHandle.keys, 0, 10); Assertions.assertEquals(10, bufferWithStartAndEnd.remaining()); } - static Stream inputStreamStores() throws IOException { - StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); - - byte[] testData = new byte[100]; - for (int i = 0; i < testData.length; i++) { - testData[i] = (byte) i; - } - - StoreHandle memoryStoreHandle = new MemoryStore().resolve(); - memoryStoreHandle.set(ByteBuffer.wrap(testData)); - - StoreHandle fsStoreHandle = new FilesystemStore(TESTOUTPUT.resolve("testInputStreamFS")).resolve("testfile"); - fsStoreHandle.set(ByteBuffer.wrap(testData)); - - zipFile(TESTOUTPUT.resolve("testInputStreamFS"), TESTOUTPUT.resolve("testInputStreamZIP.zip")); - StoreHandle bufferedZipStoreHandle = new BufferedZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip"), true) - .resolve("testfile"); - - StoreHandle readOnlyZipStoreHandle = new ReadOnlyZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip")) - .resolve("testfile"); - - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample") - .resolve("color", "1", "zarr.json"); - return Stream.of( - memoryStoreHandle, - s3StoreHandle, - fsStoreHandle, - bufferedZipStoreHandle, - readOnlyZipStoreHandle, - httpStoreHandle - ); - } - @ParameterizedTest @MethodSource("inputStreamStores") public void testStoreInputStream(StoreHandle storeHandle) throws IOException { @@ -251,7 +249,7 @@ public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOExce Path path = TESTOUTPUT.resolve("testWriteZipStore" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); writeTestGroupV3(zipStore, true); - if(!flushOnWrite) zipStore.flush(); + if (!flushOnWrite) zipStore.flush(); BufferedZipStore zipStoreRead = new BufferedZipStore(path); assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); @@ -266,11 +264,11 @@ public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOExce @ParameterizedTest @CsvSource({"false", "true",}) public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreWithComment"+ (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + Path path = TESTOUTPUT.resolve("testZipStoreWithComment" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); String comment = "{\"ome\": { \"version\": \"XX.YY\" }}"; BufferedZipStore zipStore = new BufferedZipStore(path, comment, flushOnWrite); writeTestGroupV3(zipStore, true); - if(!flushOnWrite) zipStore.flush(); + if (!flushOnWrite) zipStore.flush(); try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) { String retrievedComment = zipFile.getComment(); @@ -282,6 +280,7 @@ public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, /** * Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr + * * @see RFC-9: Zipped OME-Zarr */ @Test @@ -306,7 +305,7 @@ public void testZipStoreRequirements() throws ZarrException, IOException { zipStore.flush(); try (ZipFile zip = new ZipFile(path.toFile())) { - ArrayList entries = Collections.list(zip.getEntries()); + ArrayList entries = Collections.list(zip.getEntries()); // no compression for (ZipArchiveEntry e : entries) { @@ -315,33 +314,32 @@ public void testZipStoreRequirements() throws ZarrException, IOException { // correct order of zarr.json files String[] expectedFirstEntries = new String[]{ - "zarr.json", - "a1/zarr.json", - "g1/zarr.json", - "g2/zarr.json", - "g3/zarr.json", - "g1/g1_1/zarr.json", - "g1/g1_2/zarr.json", - "g2/g2_1/zarr.json", - "g1/g1_1/g1_1_1/zarr.json" + "zarr.json", + "a1/zarr.json", + "g1/zarr.json", + "g2/zarr.json", + "g3/zarr.json", + "g1/g1_1/zarr.json", + "g1/g1_2/zarr.json", + "g2/g2_1/zarr.json", + "g1/g1_1/g1_1_1/zarr.json" }; String[] actualFirstEntries = entries.stream() - .map(ZipArchiveEntry::getName) - .limit(expectedFirstEntries.length) - .toArray(String[]::new); + .map(ZipArchiveEntry::getName) + .limit(expectedFirstEntries.length) + .toArray(String[]::new); Assertions.assertArrayEquals(expectedFirstEntries, actualFirstEntries, "zarr.json files are not in the expected breadth-first order"); } } - @ParameterizedTest @CsvSource({"false", "true",}) public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOException { Path path = TESTOUTPUT.resolve("testZipStoreV2" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); writeTestGroupV2(zipStore, true); - if(!flushOnWrite) zipStore.flush(); + if (!flushOnWrite) zipStore.flush(); BufferedZipStore zipStoreRead = new BufferedZipStore(path); assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(zipStoreRead.resolve()), true); @@ -366,15 +364,6 @@ public void testReadOnlyZipStore() throws ZarrException, IOException { assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); } - - static Stream localStores() { - return Stream.of( - new MemoryStore(), - new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true) - ); - } - @ParameterizedTest @MethodSource("localStores") public void testLocalStores(Store store) throws IOException, ZarrException { @@ -384,7 +373,7 @@ public void testLocalStores(Store store) throws IOException, ZarrException { } - int[] testData(){ + int[] testData() { int[] testData = new int[1024 * 1024]; Arrays.setAll(testData, p -> p); return testData; From 7b064c8cb48c0ced58c38eba0b7451dab2099028 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 5 Jan 2026 14:31:49 +0100 Subject: [PATCH 30/61] fix merge --- src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index b9fae63..4e6d6ce 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -1,7 +1,7 @@ package dev.zarr.zarrjava; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.*; import dev.zarr.zarrjava.store.*; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; @@ -342,13 +342,13 @@ public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOExcepti if (!flushOnWrite) zipStore.flush(); BufferedZipStore zipStoreRead = new BufferedZipStore(path); - assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(zipStoreRead.resolve()), true); + assertIsTestGroupV2(Group.open(zipStoreRead.resolve()), true); Path unzippedPath = TESTOUTPUT.resolve("testZipStoreV2Unzipped"); unzipFile(path, unzippedPath); FilesystemStore fsStore = new FilesystemStore(unzippedPath); - assertIsTestGroupV2(dev.zarr.zarrjava.core.Group.open(fsStore.resolve()), true); + assertIsTestGroupV2(Group.open(fsStore.resolve()), true); } @Test @@ -395,7 +395,7 @@ Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, I } void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, IOException { - Stream nodes = group.list(); + Stream nodes = group.list(); Assertions.assertEquals(2, nodes.count()); Array array = (Array) group.get("array"); Assertions.assertNotNull(array); @@ -422,10 +422,10 @@ dev.zarr.zarrjava.v2.Group writeTestGroupV2(Store store, boolean useParallel) th return group; } - void assertIsTestGroupV2(dev.zarr.zarrjava.core.Group group, boolean useParallel) throws ZarrException, IOException { - Stream nodes = group.list(); + void assertIsTestGroupV2(Group group, boolean useParallel) throws ZarrException, IOException { + Stream nodes = group.list(); Assertions.assertEquals(2, nodes.count()); - dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) group.get("array"); + Array array = (Array) group.get("array"); Assertions.assertNotNull(array); ucar.ma2.Array result = array.read(useParallel); Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); From d98767c8c1be252de00e5070aac5c383edf0481d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 11 Jan 2026 23:42:00 +0100 Subject: [PATCH 31/61] improve Group::list performance --- .../java/dev/zarr/zarrjava/core/Group.java | 13 +----- src/main/java/dev/zarr/zarrjava/v2/Group.java | 20 +++++++++ src/main/java/dev/zarr/zarrjava/v3/Group.java | 41 +++++++++++-------- 3 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index fb8428c..7b425a3 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Objects; import java.util.stream.Stream; public abstract class Group extends AbstractNode { @@ -71,17 +70,7 @@ public Node get(String key) throws ZarrException, IOException { return get(new String[]{key}); } - public Stream list() { - return storeHandle.list() - .map(key -> { - try { - return get(key); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) - .filter(Objects::nonNull); - } + public abstract Stream list(); public Node[] listAsArray() { try (Stream nodeStream = list()) { diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 99eb663..33c917c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -16,7 +16,9 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.function.Function; +import java.util.stream.Stream; import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; import static dev.zarr.zarrjava.v2.Node.makeObjectWriter; @@ -179,6 +181,24 @@ public Node get(String[] key) throws ZarrException, IOException { } } + @Override + public Stream list() { + Stream metadataKeys = storeHandle.list() + .filter(key -> { + if (key.length <= 1) return false; // exclude root from list + String fileName = key[key.length - 1]; + return fileName.equals(ZARRAY) || fileName.equals(ZGROUP); + }); + + return metadataKeys.map(key -> { + try { + return get(Arrays.copyOf(key, key.length - 1)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + /** * Creates a new subgroup with default metadata at the specified key. * diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 5c017e0..5df6d5b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -15,7 +15,9 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.function.Function; +import java.util.stream.Stream; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; import static dev.zarr.zarrjava.v3.Node.makeObjectWriter; @@ -112,10 +114,7 @@ public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetad * @throws IOException if the metadata cannot be serialized * @throws ZarrException if the attributes are invalid */ - public static Group create( - @Nonnull StoreHandle storeHandle, - @Nonnull Attributes attributes - ) throws IOException, ZarrException { + public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull Attributes attributes) throws IOException, ZarrException { return create(storeHandle, new GroupMetadata(attributes)); } @@ -193,6 +192,21 @@ public Node get(String[] key) throws ZarrException, IOException { } } + @Override + public Stream list() { + Stream metadataKeys = storeHandle.list() + .filter(key -> key[key.length - 1].equals(ZARR_JSON)) + .filter(key -> key.length > 1); // exclude root from list + return metadataKeys.map(key -> { + try { + return get(Arrays.copyOf(key, key.length - 1)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + /** * Creates a new subgroup with the provided metadata at the specified key. * @@ -200,8 +214,7 @@ public Node get(String[] key) throws ZarrException, IOException { * @param groupMetadata the metadata of the Zarr group * @throws IOException if the metadata cannot be serialized */ - public Group createGroup(String key, GroupMetadata groupMetadata) - throws IOException, ZarrException { + public Group createGroup(String key, GroupMetadata groupMetadata) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), groupMetadata); } @@ -212,8 +225,7 @@ public Group createGroup(String key, GroupMetadata groupMetadata) * @param attributes attributes of the Zarr group * @throws IOException if the metadata cannot be serialized */ - public Group createGroup(String key, Attributes attributes) - throws IOException, ZarrException { + public Group createGroup(String key, Attributes attributes) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); } @@ -237,8 +249,7 @@ public Group createGroup(String key) throws IOException { * @throws IOException if the metadata cannot be serialized * @throws ZarrException if the array cannot be created */ - public Array createArray(String key, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { + public Array createArray(String key, ArrayMetadata arrayMetadata) throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadata); } @@ -249,9 +260,7 @@ public Array createArray(String key, ArrayMetadata arrayMetadata) * @param arrayMetadataBuilderMapper a function building the metadata of the Zarr array * @throws IOException if the metadata cannot be serialized */ - public Array createArray(String key, - Function arrayMetadataBuilderMapper) - throws IOException, ZarrException { + public Array createArray(String key, Function arrayMetadataBuilderMapper) throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); } @@ -262,8 +271,7 @@ private Group writeMetadata() throws IOException { private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { ObjectWriter objectWriter = makeObjectWriter(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); + storeHandle.resolve(ZARR_JSON).set(metadataBytes); this.metadata = newGroupMetadata; return this; } @@ -276,8 +284,7 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { * @throws ZarrException if the new attributes are invalid * @throws IOException if the metadata cannot be serialized */ - public Group updateAttributes(Function attributeMapper) - throws ZarrException, IOException { + public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); } From e4514b206ec4d9a1701de34f4e3b92c40e0d3831 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 11 Jan 2026 23:43:49 +0100 Subject: [PATCH 32/61] fix ReadOnlyZipStore::list --- src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index d518a1e..c4facc2 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -144,7 +144,7 @@ public Stream list(String[] keys) { if (!entryName.startsWith(prefix) || entryName.equals(prefix)) { continue; } - String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length())); + String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length() + 1)); builder.add(entryKeys); } } catch (IOException ignored) { From 76aacb8e617b4ddc835941134b5244efede3c35a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 00:42:05 +0100 Subject: [PATCH 33/61] less requests to store on read --- src/main/java/dev/zarr/zarrjava/core/Array.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index a8efaeb..482458e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -16,6 +16,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; public abstract class Array extends AbstractNode { @@ -289,6 +292,8 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean if (parallel) { chunkStream = chunkStream.parallel(); } + final Set> existingKeys = storeHandle.list().map(Arrays::asList).collect(Collectors.toSet()); + chunkStream.forEach( chunkCoords -> { try { @@ -306,9 +311,8 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - if (!chunkHandle.exists()) { + if (existingKeys.stream().noneMatch(Arrays.asList(chunkKeys)::equals)) return; - } if (codecPipeline.supportsPartialDecode()) { final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); From 60e11d719aa2a4951769efb72a8a272aea6f588c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 00:42:40 +0100 Subject: [PATCH 34/61] fix FilesystemStore::list --- .../zarr/zarrjava/store/FilesystemStore.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index c89b3e6..90da970 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -121,15 +121,18 @@ public void delete(String[] keys) { } public Stream list(String[] keys) { + Path keyPath = resolveKeys(keys); try { - return Files.list(resolveKeys(keys)).map(path -> { - Path relativePath = resolveKeys(keys).relativize(path); - String[] parts = new String[relativePath.getNameCount()]; - for (int i = 0; i < relativePath.getNameCount(); i++) { - parts[i] = relativePath.getName(i).toString(); - } - return parts; - }); + return Files.walk(keyPath) + .filter(path -> !path.equals(keyPath)) + .map(path -> { + Path relativePath = keyPath.relativize(path); + String[] parts = new String[relativePath.getNameCount()]; + for (int i = 0; i < relativePath.getNameCount(); i++) { + parts[i] = relativePath.getName(i).toString(); + } + return parts; + }); } catch (IOException e) { throw new RuntimeException(e); } From cd349f9cd23c49b6d2838cb044596c34cfa63f86 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 00:42:54 +0100 Subject: [PATCH 35/61] fix MemoryStore::list --- src/main/java/dev/zarr/zarrjava/store/MemoryStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 5e4a7c7..ea855c0 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -68,7 +68,7 @@ public Stream list(String[] keys) { if (k.size() <= prefix.size() || !k.subList(0, prefix.size()).equals(prefix)) continue; for (int i = prefix.size(); i < k.size(); i++) { - allKeys.add(k.subList(0, i + 1)); + allKeys.add(k.subList(prefix.size(), i + 1)); } } return allKeys.stream().map(k -> k.toArray(new String[0])); From a1c54eaf353eaa0fe8241a858586882bcbbaee66 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 00:43:05 +0100 Subject: [PATCH 36/61] fix ReadOnlyZipStore::list --- src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index c4facc2..8b906fd 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -140,7 +140,9 @@ public Stream list(String[] keys) { if (entryName.startsWith("/")) { entryName = entryName.substring(1); } - + if (entryName.endsWith("/")) { + entryName = entryName.substring(0, entryName.length() - 1); + } if (!entryName.startsWith(prefix) || entryName.equals(prefix)) { continue; } From a2aef00e70c8bd686331b9ac80a31146d29b34fd Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 00:46:09 +0100 Subject: [PATCH 37/61] test Store::list for local stores --- .../dev/zarr/zarrjava/store/ZipStore.java | 2 +- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 55 ++++++++++++++++--- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java index 11cb7f2..603d8e3 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ZipStore.java @@ -6,7 +6,7 @@ import java.nio.ByteBuffer; public abstract class ZipStore implements Store, Store.ListableStore { - protected final StoreHandle underlyingStore; + public final StoreHandle underlyingStore; public ZipStore(@Nonnull StoreHandle underlyingStore) { this.underlyingStore = underlyingStore; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4e6d6ce..3cb051b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -24,6 +24,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; @@ -79,11 +81,12 @@ static Stream inputStreamStores() throws IOException { ); } - static Stream localStores() { + static Stream localStores() { return Stream.of( new MemoryStore(), new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true) + new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true), + new ReadOnlyZipStore(TESTOUTPUT.resolve("testLocalStoresReadOnlyZIP.zip")) ); } @@ -113,8 +116,7 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1"))); Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new); - Assertions.assertEquals(2, subNodes.length); - Assertions.assertInstanceOf(Group.class, subNodes[0]); + Assertions.assertEquals(12, subNodes.length); Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new); @@ -366,9 +368,34 @@ public void testReadOnlyZipStore() throws ZarrException, IOException { @ParameterizedTest @MethodSource("localStores") - public void testLocalStores(Store store) throws IOException, ZarrException { + public void testLocalStores(Store.ListableStore store) throws IOException, ZarrException { boolean useParallel = true; - Group group = writeTestGroupV3(store, useParallel); + Store writeStore = store; + if (store instanceof ReadOnlyZipStore) { + StoreHandle underlyingStore = ((ReadOnlyZipStore)store).underlyingStore; + writeStore = new BufferedZipStore(underlyingStore, true); + } + Group group = writeTestGroupV3(writeStore, useParallel); + + java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( + "array/c/1/1", + "array/c/0/0", + "array/c/0/1", + "zarr.json", + "array", + "array/c/1/0", + "array/c/1", + "array/c/0", + "array/zarr.json", + "array/c" + )); + + java.util.Set actualKeys = store.resolve("subgroup").list() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + + Assertions.assertEquals(expectedSubgroupKeys, actualKeys); + assertIsTestGroupV3(group, useParallel); } @@ -389,18 +416,30 @@ Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, I .withChunkShape(512, 512) ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); - group.createGroup("subgroup"); + dev.zarr.zarrjava.v3.Group subgroup = group.createGroup("subgroup"); + dev.zarr.zarrjava.v3.Array subgrouparray = subgroup.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) + .withChunkShape(512, 512) + ); + subgrouparray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); return group; } void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, IOException { Stream nodes = group.list(); - Assertions.assertEquals(2, nodes.count()); + List nodeList = nodes.collect(Collectors.toList()); + Assertions.assertEquals(3, nodeList.size()); Array array = (Array) group.get("array"); Assertions.assertNotNull(array); ucar.ma2.Array result = array.read(useParallel); Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Group subgroup = (Group) group.get("subgroup"); + Array subgrouparray = (Array) subgroup.get("array"); + result = subgrouparray.read(useParallel); + Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); Attributes attrs = group.metadata().attributes(); Assertions.assertNotNull(attrs); Assertions.assertEquals("value", attrs.getString("some")); From 4f7bee87c5838dde46f0ad90358b1a922d92804a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 15:10:31 +0100 Subject: [PATCH 38/61] less store.exists() for v2.Group.list() --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 33c917c..d3229e4 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -17,6 +17,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Stream; @@ -183,20 +184,22 @@ public Node get(String[] key) throws ZarrException, IOException { @Override public Stream list() { - Stream metadataKeys = storeHandle.list() - .filter(key -> { - if (key.length <= 1) return false; // exclude root from list - String fileName = key[key.length - 1]; - return fileName.equals(ZARRAY) || fileName.equals(ZGROUP); - }); - - return metadataKeys.map(key -> { + return storeHandle.list().map(key -> { + if (key.length <= 1) return null; // exclude root from list + String fileName = key[key.length - 1]; + StoreHandle parent = storeHandle.resolve(Arrays.copyOf(key, key.length - 1)); try { - return get(Arrays.copyOf(key, key.length - 1)); + if (fileName.equals(ZARRAY)) { + return Array.open(parent); + } + if (fileName.equals(ZGROUP)) { + return (dev.zarr.zarrjava.core.Node) Group.open(parent); + } } catch (Exception e) { throw new RuntimeException(e); } - }); + return null; + }).filter(Objects::nonNull); } /** From 60e28cd2a8c4e9b096feb5cd57831bb7ecf1512c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 17:35:34 +0100 Subject: [PATCH 39/61] add S3Mock for tests --- .gitignore | 3 +- pom.xml | 24 ++++++++ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 60 ++++++++++++++++++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index fcafa91..4c52107 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ build/ /main.py /pyproject.toml /uv.lock -**/__pycache__ \ No newline at end of file +**/__pycache__ +/dependency-reduced-pom.xml diff --git a/pom.xml b/pom.xml index 0190403..22d8d99 100644 --- a/pom.xml +++ b/pom.xml @@ -126,6 +126,30 @@ commons-compress 1.28.0 + + com.adobe.testing + s3mock + 4.11.0 + test + + + com.adobe.testing + s3mock-testcontainers + 4.11.0 + test + + + org.testcontainers + junit-jupiter + 1.20.4 + test + + + org.testcontainers + testcontainers + 2.0.2 + test + diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 3cb051b..6e491be 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -1,19 +1,26 @@ package dev.zarr.zarrjava; +import com.adobe.testing.s3mock.testcontainers.S3MockContainer; import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.*; import dev.zarr.zarrjava.store.*; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.utils.AttributeMap; import java.io.IOException; import java.io.InputStream; @@ -32,6 +39,55 @@ import static dev.zarr.zarrjava.Utils.unzipFile; import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; +import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; + + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Testcontainers +class S3StoreTest { + private final String TEST_BUCKET = "test-bucket"; + @Container + private S3MockContainer s3Mock; + private S3Client s3Client; + + @BeforeAll + void setUp() { + s3Mock = new S3MockContainer("latest").withInitialBuckets(TEST_BUCKET); + s3Mock.start(); + SdkHttpClient httpClient = ApacheHttpClient.builder().buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, Boolean.TRUE).build()); + s3Client = S3Client.builder().endpointOverride(URI.create(s3Mock.getHttpsEndpoint())).serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("foo", "bar"))).region(Region.US_EAST_1) // required, but ignored + .httpClient(httpClient).build(); + } + + @AfterAll + void tearDown() { + if (s3Mock.isRunning()) { + s3Mock.stop(); + } + } + + @Test + void testS3Mock() { + Assertions.assertTrue(s3Mock.isRunning()); + } + + @Test + void testReadWriteS3Store() { + S3Store s3Store = new S3Store(s3Client, TEST_BUCKET, ""); + + StoreHandle storeHandle = s3Store.resolve("testfile"); + byte[] testData = new byte[100]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) i; + } + storeHandle.set(ByteBuffer.wrap(testData)); + ByteBuffer retrievedData = storeHandle.read(); + byte[] retrievedBytes = new byte[retrievedData.remaining()]; + retrievedData.get(retrievedBytes); + Assertions.assertArrayEquals(testData, retrievedBytes); + } + +} public class ZarrStoreTest extends ZarrTest { static StoreHandle createS3StoreHandle() { From 51d0f365da59454f6c52bdac743f422922d61f59 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 17:53:44 +0100 Subject: [PATCH 40/61] fallback to chunkHandle.exists() for non listable stores --- .../java/dev/zarr/zarrjava/core/Array.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 482458e..a08f62b 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -3,6 +3,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.Store; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; @@ -292,7 +293,14 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean if (parallel) { chunkStream = chunkStream.parallel(); } - final Set> existingKeys = storeHandle.list().map(Arrays::asList).collect(Collectors.toSet()); + + boolean isListableStore = storeHandle.store instanceof Store.ListableStore; + Set> existingKeys; + if (isListableStore) { + existingKeys = storeHandle.list().map(Arrays::asList).collect(Collectors.toSet()); + } else { + existingKeys = null; + } chunkStream.forEach( chunkCoords -> { @@ -311,8 +319,13 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - if (existingKeys.stream().noneMatch(Arrays.asList(chunkKeys)::equals)) - return; + + // chunkHandle.exists() can be expensive on some store types, so we optimize for ListableStore + if (isListableStore) { + if (existingKeys.stream().noneMatch(Arrays.asList(chunkKeys)::equals)) + return; + } else if (!chunkHandle.exists()) return; + if (codecPipeline.supportsPartialDecode()) { final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); From b30992208959141995e1850698fb48379572e02d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 18:00:56 +0100 Subject: [PATCH 41/61] fix ZarrV2Test::testGroup --- src/test/java/dev/zarr/zarrjava/ZarrV2Test.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 07bb622..534a351 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -248,8 +248,9 @@ public void testGroup() throws IOException, ZarrException { .withChunks(5, 5) ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunks); + Array[] arrays = group.list().filter(n -> n instanceof Array).toArray(Array[]::new); + Assertions.assertEquals(1, arrays.length); + Assertions.assertArrayEquals(new int[]{5, 5}, arrays[0].metadata().chunks); } @Test From cac3e220cc0a1083d38faf1cefadd331287eb871 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 18:20:19 +0100 Subject: [PATCH 42/61] downgrade s3mock version to match java-version 11 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 22d8d99..83c490b 100644 --- a/pom.xml +++ b/pom.xml @@ -129,13 +129,13 @@ com.adobe.testing s3mock - 4.11.0 + 3.12.0 test com.adobe.testing s3mock-testcontainers - 4.11.0 + 3.12.0 test @@ -147,7 +147,7 @@ org.testcontainers testcontainers - 2.0.2 + 1.20.4 test From 8d5d5833dca0ba083183b47d385a3aa71435f4ce Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 12 Jan 2026 18:50:47 +0100 Subject: [PATCH 43/61] downgrade s3mock version to match java-version 11 --- pom.xml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 83c490b..ec7c2ba 100644 --- a/pom.xml +++ b/pom.xml @@ -129,13 +129,13 @@ com.adobe.testing s3mock - 3.12.0 + 2.17.0 test com.adobe.testing s3mock-testcontainers - 3.12.0 + 2.17.0 test @@ -150,6 +150,13 @@ 1.20.4 test + + + ch.qos.logback + logback-classic + 1.4.14 + test + @@ -168,6 +175,10 @@ 3.2.5 false + + + 1.44 + From c64ea1c27af5f8c476ed2894b4c427587e66d7a5 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 11:25:24 +0100 Subject: [PATCH 44/61] run S3 Mock with docker --- .github/workflows/ci.yml | 31 ++++++++- pom.xml | 31 --------- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 63 +++++++++---------- 3 files changed, 59 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0108c47..f62ffae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,28 @@ jobs: steps: - uses: actions/checkout@v5 + - name: Start S3Mock + if: runner.os != 'macOS' + run: | + docker run -d -p 9090:9090 -p 9191:9191 \ + --name s3mock \ + -e "initialBuckets=zarr-test-bucket" \ + adobe/s3mock:3.11.0 + + # TODO: move this waiting logic right in front of the tests that need it + - name: Wait for S3Mock + if: runner.os != 'macOS' + run: | + echo "Waiting for S3Mock to start..." + for i in {1..30}; do + if curl -s http://localhost:9090/favicon.ico > /dev/null; then + echo "S3Mock is up!" + exit 0 + fi + sleep 2 + done + echo "S3Mock failed to start" + exit 1 - name: Set up JDK uses: actions/setup-java@v4 with: @@ -49,8 +71,13 @@ jobs: - name: Test env: MAVEN_OPTS: "-Xmx6g" - run: mvn --no-transfer-progress test -DargLine="-Xmx6g" - + SKIP_S3_TESTS: ${{ matrix.os == 'macos-latest' }} + run: | + if [ "$SKIP_S3_TESTS" = "true" ]; then + mvn --no-transfer-progress test -DargLine="-Xmx6g" -Dgroups="!s3" + else + mvn --no-transfer-progress test -DargLine="-Xmx6g" + fi - name: Assemble JAR run: mvn package -DskipTests diff --git a/pom.xml b/pom.xml index ec7c2ba..5d1e750 100644 --- a/pom.xml +++ b/pom.xml @@ -126,37 +126,6 @@ commons-compress 1.28.0 - - com.adobe.testing - s3mock - 2.17.0 - test - - - com.adobe.testing - s3mock-testcontainers - 2.17.0 - test - - - org.testcontainers - junit-jupiter - 1.20.4 - test - - - org.testcontainers - testcontainers - 1.20.4 - test - - - - ch.qos.logback - logback-classic - 1.4.14 - test - diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 6e491be..65c8364 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -1,6 +1,5 @@ package dev.zarr.zarrjava; -import com.adobe.testing.s3mock.testcontainers.S3MockContainer; import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.*; import dev.zarr.zarrjava.store.*; @@ -10,17 +9,12 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.http.SdkHttpClient; -import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.utils.AttributeMap; import java.io.IOException; import java.io.InputStream; @@ -39,41 +33,44 @@ import static dev.zarr.zarrjava.Utils.unzipFile; import static dev.zarr.zarrjava.Utils.zipFile; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -import static software.amazon.awssdk.http.SdkHttpConfigurationOption.TRUST_ALL_CERTIFICATES; - -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@Testcontainers +/** + * Tests for S3Store + *

+ * Requires a local S3 mock server running at http://localhost:9090 + * with a bucket named "zarr-test-bucket" + *

+ * Execute the following command to start a local S3 mock server: + * 

+ * docker run -p 9090:9090 -p 9191:9191 -e "initialBuckets=zarr-test-bucket" adobe/s3mock:3.11.0
+ *
+ */ +@Tag("s3") class S3StoreTest { - private final String TEST_BUCKET = "test-bucket"; - @Container - private S3MockContainer s3Mock; - private S3Client s3Client; - @BeforeAll - void setUp() { - s3Mock = new S3MockContainer("latest").withInitialBuckets(TEST_BUCKET); - s3Mock.start(); - SdkHttpClient httpClient = ApacheHttpClient.builder().buildWithDefaults(AttributeMap.builder().put(TRUST_ALL_CERTIFICATES, Boolean.TRUE).build()); - s3Client = S3Client.builder().endpointOverride(URI.create(s3Mock.getHttpsEndpoint())).serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("foo", "bar"))).region(Region.US_EAST_1) // required, but ignored - .httpClient(httpClient).build(); - } + static String s3Endpoint = "http://localhost:9090"; + static String bucket = "zarr-test-bucket"; + static S3Client s3Client; - @AfterAll - void tearDown() { - if (s3Mock.isRunning()) { - s3Mock.stop(); - } - } - - @Test - void testS3Mock() { - Assertions.assertTrue(s3Mock.isRunning()); + @BeforeAll + static void setUp() { + s3Client = S3Client.builder() + .endpointOverride(URI.create(s3Endpoint)) + .region(Region.US_EAST_1) // required, but ignored + .serviceConfiguration( + S3Configuration.builder() + .pathStyleAccessEnabled(true) // required + .build() + ) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("accessKey", "secretKey") + )) + .build(); } @Test void testReadWriteS3Store() { - S3Store s3Store = new S3Store(s3Client, TEST_BUCKET, ""); + S3Store s3Store = new S3Store(s3Client, bucket, ""); StoreHandle storeHandle = s3Store.resolve("testfile"); byte[] testData = new byte[100]; From 55d5475ee157055d407908c155140c788de29e6c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 11:35:13 +0100 Subject: [PATCH 45/61] S3Mock tests only on linux --- .github/workflows/ci.yml | 38 ++++++++++++-------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f62ffae..1fc7edb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,14 @@ jobs: os: [ ubuntu-latest, windows-latest, macos-latest ] fail-fast: false runs-on: ${{ matrix.os }} + services: + s3mock: + # This service will only start if the runner is Linux + image: ${{ matrix.os == 'ubuntu-latest' && 'adobe/s3mock:3.11.0' || '' }} + ports: + - 9090:9090 + env: + initialBuckets: zarr-test-bucket defaults: run: shell: bash @@ -21,28 +29,6 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Start S3Mock - if: runner.os != 'macOS' - run: | - docker run -d -p 9090:9090 -p 9191:9191 \ - --name s3mock \ - -e "initialBuckets=zarr-test-bucket" \ - adobe/s3mock:3.11.0 - - # TODO: move this waiting logic right in front of the tests that need it - - name: Wait for S3Mock - if: runner.os != 'macOS' - run: | - echo "Waiting for S3Mock to start..." - for i in {1..30}; do - if curl -s http://localhost:9090/favicon.ico > /dev/null; then - echo "S3Mock is up!" - exit 0 - fi - sleep 2 - done - echo "S3Mock failed to start" - exit 1 - name: Set up JDK uses: actions/setup-java@v4 with: @@ -71,12 +57,12 @@ jobs: - name: Test env: MAVEN_OPTS: "-Xmx6g" - SKIP_S3_TESTS: ${{ matrix.os == 'macos-latest' }} run: | - if [ "$SKIP_S3_TESTS" = "true" ]; then - mvn --no-transfer-progress test -DargLine="-Xmx6g" -Dgroups="!s3" - else + if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then mvn --no-transfer-progress test -DargLine="-Xmx6g" + else + # Skip S3 tests on Windows/macOS where the service isn't running (labeled with @Tag("s3")) + mvn --no-transfer-progress test -DargLine="-Xmx6g" -DexcludedGroups="s3" fi - name: Assemble JAR run: mvn package -DskipTests From 5ae5ac2189550fe472ff64bc66ca1a65ede95707 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 11:52:22 +0100 Subject: [PATCH 46/61] fix ZarrV3Test::testGroup for windows, mac --- src/test/java/dev/zarr/zarrjava/ZarrV3Test.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 9a669be..44ea9d8 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -553,7 +553,9 @@ public void testGroup() throws IOException, ZarrException { ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunkShape()); + Array[] arrays = group.list().filter(n -> n instanceof Array).toArray(Array[]::new); + Assertions.assertEquals(1, arrays.length); + Assertions.assertArrayEquals(new int[]{5, 5}, arrays[0].metadata().chunkShape()); } @Test From 8c4e988974bc7a45ada30a3336bedb70d0477a44 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 15:18:40 +0100 Subject: [PATCH 47/61] refactor and add missing store tests --- src/test/java/dev/zarr/zarrjava/Utils.java | 4 +- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 528 ------------------ .../zarrjava/store/BufferedZipStoreTest.java | 168 ++++++ .../zarrjava/store/FileSystemStoreTest.java | 62 ++ .../zarr/zarrjava/store/HttpStoreTest.java | 25 + .../zarr/zarrjava/store/MemoryStoreTest.java | 77 +++ .../zarrjava/store/OnlineS3StoreTest.java | 67 +++ .../zarrjava/store/ReadOnlyZipStoreTest.java | 78 +++ .../dev/zarr/zarrjava/store/S3StoreTest.java | 97 ++++ .../dev/zarr/zarrjava/store/StoreTest.java | 126 +++++ .../zarrjava/store/WritableStoreTest.java | 49 ++ 11 files changed, 751 insertions(+), 530 deletions(-) delete mode 100644 src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/StoreTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java diff --git a/src/test/java/dev/zarr/zarrjava/Utils.java b/src/test/java/dev/zarr/zarrjava/Utils.java index 5356af6..b41a714 100644 --- a/src/test/java/dev/zarr/zarrjava/Utils.java +++ b/src/test/java/dev/zarr/zarrjava/Utils.java @@ -9,7 +9,7 @@ public class Utils { - static void zipFile(Path sourceDir, Path targetDir) throws IOException { + public static void zipFile(Path sourceDir, Path targetDir) throws IOException { FileOutputStream fos = new FileOutputStream(targetDir.toFile()); ZipOutputStream zipOut = new ZipOutputStream(fos); @@ -53,7 +53,7 @@ static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) thr * Unzip sourceZip into targetDir. * Protects against Zip Slip by ensuring extracted paths remain inside targetDir. */ - static void unzipFile(Path sourceZip, Path targetDir) throws IOException { + public static void unzipFile(Path sourceZip, Path targetDir) throws IOException { Files.createDirectories(targetDir); try (FileInputStream fis = new FileInputStream(sourceZip.toFile()); ZipInputStream zis = new ZipInputStream(fis)) { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java deleted file mode 100644 index 65c8364..0000000 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ /dev/null @@ -1,528 +0,0 @@ -package dev.zarr.zarrjava; - -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.core.*; -import dev.zarr.zarrjava.store.*; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipFile; -import org.junit.jupiter.api.*; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3Configuration; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.ZipEntry; - -import static dev.zarr.zarrjava.Utils.unzipFile; -import static dev.zarr.zarrjava.Utils.zipFile; -import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; - -/** - * Tests for S3Store - *

- * Requires a local S3 mock server running at http://localhost:9090 - * with a bucket named "zarr-test-bucket" - *

- * Execute the following command to start a local S3 mock server: - * 

- * docker run -p 9090:9090 -p 9191:9191 -e "initialBuckets=zarr-test-bucket" adobe/s3mock:3.11.0
- *
- */ -@Tag("s3") -class S3StoreTest { - - static String s3Endpoint = "http://localhost:9090"; - static String bucket = "zarr-test-bucket"; - static S3Client s3Client; - - @BeforeAll - static void setUp() { - s3Client = S3Client.builder() - .endpointOverride(URI.create(s3Endpoint)) - .region(Region.US_EAST_1) // required, but ignored - .serviceConfiguration( - S3Configuration.builder() - .pathStyleAccessEnabled(true) // required - .build() - ) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create("accessKey", "secretKey") - )) - .build(); - } - - @Test - void testReadWriteS3Store() { - S3Store s3Store = new S3Store(s3Client, bucket, ""); - - StoreHandle storeHandle = s3Store.resolve("testfile"); - byte[] testData = new byte[100]; - for (int i = 0; i < testData.length; i++) { - testData[i] = (byte) i; - } - storeHandle.set(ByteBuffer.wrap(testData)); - ByteBuffer retrievedData = storeHandle.read(); - byte[] retrievedBytes = new byte[retrievedData.remaining()]; - retrievedData.get(retrievedBytes); - Assertions.assertArrayEquals(testData, retrievedBytes); - } - -} - -public class ZarrStoreTest extends ZarrTest { - static StoreHandle createS3StoreHandle() { - S3Store s3Store = new S3Store(S3Client.builder() - .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) - .region(Region.US_EAST_1) // required, but ignored - .serviceConfiguration( - S3Configuration.builder() - .pathStyleAccessEnabled(true) // required - .build() - ) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "idr", "zarr/v0.5/idr0033A"); - return s3Store.resolve("BR00109990_C2.zarr", "0", "0"); - } - - static Stream inputStreamStores() throws IOException { - StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); - - byte[] testData = new byte[100]; - for (int i = 0; i < testData.length; i++) { - testData[i] = (byte) i; - } - - StoreHandle memoryStoreHandle = new MemoryStore().resolve(); - memoryStoreHandle.set(ByteBuffer.wrap(testData)); - - StoreHandle fsStoreHandle = new FilesystemStore(TESTOUTPUT.resolve("testInputStreamFS")).resolve("testfile"); - fsStoreHandle.set(ByteBuffer.wrap(testData)); - - zipFile(TESTOUTPUT.resolve("testInputStreamFS"), TESTOUTPUT.resolve("testInputStreamZIP.zip")); - StoreHandle bufferedZipStoreHandle = new BufferedZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip"), true) - .resolve("testfile"); - - StoreHandle readOnlyZipStoreHandle = new ReadOnlyZipStore(TESTOUTPUT.resolve("testInputStreamZIP.zip")) - .resolve("testfile"); - - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample") - .resolve("color", "1", "zarr.json"); - return Stream.of( - memoryStoreHandle, - s3StoreHandle, - fsStoreHandle, - bufferedZipStoreHandle, - readOnlyZipStoreHandle, - httpStoreHandle - ); - } - - static Stream localStores() { - return Stream.of( - new MemoryStore(), - new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")), - new BufferedZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"), true), - new ReadOnlyZipStore(TESTOUTPUT.resolve("testLocalStoresReadOnlyZIP.zip")) - ); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = makeObjectMapper(); - - GroupMetadata groupMetadata = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - dev.zarr.zarrjava.v3.GroupMetadata.class - ); - - String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); - Assertions.assertTrue(groupMetadataString.contains("\"zarr_format\":3")); - Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - dev.zarr.zarrjava.v3.ArrayMetadata.class); - - String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); - Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); - Assertions.assertTrue(arrayMetadataString.contains("\"node_type\":\"array\"")); - Assertions.assertTrue(arrayMetadataString.contains("\"shape\":[1,4096,4096,2048]")); - - Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1"))); - - Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new); - Assertions.assertEquals(12, subNodes.length); - - Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new); - - Assertions.assertEquals(5, colorSubNodes.length); - Assertions.assertInstanceOf(Array.class, colorSubNodes[0]); - - Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1"); - Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - StoreHandle s3StoreHandle = createS3StoreHandle(); - Array arrayV3 = Array.open(s3StoreHandle); - Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayV3.metadata().shape); - Assertions.assertEquals(574, arrayV3.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); - - dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(s3StoreHandle); - Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayCore.metadata().shape); - Assertions.assertEquals(574, arrayCore.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); - } - - @Test - public void testS3StoreGet() throws ZarrException { - StoreHandle s3StoreHandle = createS3StoreHandle().resolve("zarr.json"); - S3Store s3Store = (S3Store) s3StoreHandle.store; - ByteBuffer buffer = s3Store.get(s3StoreHandle.keys); - ByteBuffer bufferWithStart = s3Store.get(s3StoreHandle.keys, 10); - Assertions.assertEquals(10, buffer.remaining() - bufferWithStart.remaining()); - - ByteBuffer bufferWithStartAndEnd = s3Store.get(s3StoreHandle.keys, 0, 10); - Assertions.assertEquals(10, bufferWithStartAndEnd.remaining()); - - } - - @ParameterizedTest - @MethodSource("inputStreamStores") - public void testStoreInputStream(StoreHandle storeHandle) throws IOException { - InputStream is = storeHandle.getInputStream(10, 20); - byte[] buffer = new byte[10]; - int bytesRead = is.read(buffer); - Assertions.assertEquals(10, bytesRead); - byte[] expectedBuffer = new byte[10]; - storeHandle.read(10, 20).get(expectedBuffer); - Assertions.assertArrayEquals(expectedBuffer, buffer); - } - - @ParameterizedTest - @MethodSource("inputStreamStores") - public void testStoreGetSize(StoreHandle storeHandle) { - long size = storeHandle.getSize(); - long actual_size = storeHandle.read().remaining(); - Assertions.assertEquals(actual_size, size); - } - - @Test - public void testHttpStore() throws IOException, ZarrException { - HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.5/idr0033A"); - Array array = Array.open(httpStore.resolve("BR00109990_C2.zarr", "0", "0")); - - Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, array.metadata().shape); - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException { - int[] testData = testData(); - - dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve()); - Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) - .withChunkShape(5, 5) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); - group.createGroup("subgroup"); - group.setAttributes(new Attributes(b -> b.set("some", "value"))); - Stream nodes = group.list(); - Assertions.assertEquals(2, nodes.count()); - - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes; - Assertions.assertNotNull(attrs); - Assertions.assertEquals("value", attrs.getString("some")); - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { - int[] testData = testData(); - - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve()); - dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) - .withChunks(512, 512) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); - group.createGroup("subgroup"); - Stream nodes = group.list(); - group.setAttributes(new Attributes().set("description", "test group")); - Assertions.assertEquals(2, nodes.count()); - - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes; - Assertions.assertNotNull(attrs); - Assertions.assertEquals("test group", attrs.getString("description")); - - } - - @Test - public void testOpenZipStore() throws ZarrException, IOException { - Path sourceDir = TESTOUTPUT.resolve("testZipStore"); - Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); - FilesystemStore fsStore = new FilesystemStore(sourceDir); - writeTestGroupV3(fsStore, true); - - zipFile(sourceDir, targetDir); - - BufferedZipStore zipStore = new BufferedZipStore(targetDir); - assertIsTestGroupV3(Group.open(zipStore.resolve()), true); - - ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(targetDir); - assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testWriteZipStore" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); - BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); - writeTestGroupV3(zipStore, true); - if (!flushOnWrite) zipStore.flush(); - - BufferedZipStore zipStoreRead = new BufferedZipStore(path); - assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); - - Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped" + (flushOnWrite ? "Flush" : "NoFlush")); - - unzipFile(path, unzippedPath); - FilesystemStore fsStore = new FilesystemStore(unzippedPath); - assertIsTestGroupV3(Group.open(fsStore.resolve()), true); - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreWithComment" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); - String comment = "{\"ome\": { \"version\": \"XX.YY\" }}"; - BufferedZipStore zipStore = new BufferedZipStore(path, comment, flushOnWrite); - writeTestGroupV3(zipStore, true); - if (!flushOnWrite) zipStore.flush(); - - try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) { - String retrievedComment = zipFile.getComment(); - Assertions.assertEquals(comment, retrievedComment, "ZIP archive comment does not match expected value."); - } - - Assertions.assertEquals(comment, new BufferedZipStore(path).getArchiveComment(), "ZIP archive comment from store does not match expected value."); - } - - /** - * Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr - * - * @see RFC-9: Zipped OME-Zarr - */ - @Test - public void testZipStoreRequirements() throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip"); - BufferedZipStore zipStore = new BufferedZipStore(path); - - dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(zipStore.resolve()); - Array array = group.createArray("a1", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) - .withChunkShape(512, 512) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), true); - - dev.zarr.zarrjava.v3.Group g1 = group.createGroup("g1"); - g1.createGroup("g1_1").createGroup("g1_1_1"); - g1.createGroup("g1_2"); - group.createGroup("g2").createGroup("g2_1"); - group.createGroup("g3"); - - zipStore.flush(); - - try (ZipFile zip = new ZipFile(path.toFile())) { - ArrayList entries = Collections.list(zip.getEntries()); - - // no compression - for (ZipArchiveEntry e : entries) { - Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed"); - } - - // correct order of zarr.json files - String[] expectedFirstEntries = new String[]{ - "zarr.json", - "a1/zarr.json", - "g1/zarr.json", - "g2/zarr.json", - "g3/zarr.json", - "g1/g1_1/zarr.json", - "g1/g1_2/zarr.json", - "g2/g2_1/zarr.json", - "g1/g1_1/g1_1_1/zarr.json" - }; - String[] actualFirstEntries = entries.stream() - .map(ZipArchiveEntry::getName) - .limit(expectedFirstEntries.length) - .toArray(String[]::new); - - Assertions.assertArrayEquals(expectedFirstEntries, actualFirstEntries, "zarr.json files are not in the expected breadth-first order"); - } - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testZipStoreV2" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); - BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); - writeTestGroupV2(zipStore, true); - if (!flushOnWrite) zipStore.flush(); - - BufferedZipStore zipStoreRead = new BufferedZipStore(path); - assertIsTestGroupV2(Group.open(zipStoreRead.resolve()), true); - - Path unzippedPath = TESTOUTPUT.resolve("testZipStoreV2Unzipped"); - - unzipFile(path, unzippedPath); - FilesystemStore fsStore = new FilesystemStore(unzippedPath); - assertIsTestGroupV2(Group.open(fsStore.resolve()), true); - } - - @Test - public void testReadOnlyZipStore() throws ZarrException, IOException { - Path path = TESTOUTPUT.resolve("testReadOnlyZipStore.zip"); - String archiveComment = "This is a test ZIP archive comment."; - BufferedZipStore zipStore = new BufferedZipStore(path, archiveComment); - writeTestGroupV3(zipStore, true); - zipStore.flush(); - - ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(path); - Assertions.assertEquals(archiveComment, readOnlyZipStore.getArchiveComment(), "ZIP archive comment from ReadOnlyZipStore does not match expected value."); - assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); - } - - @ParameterizedTest - @MethodSource("localStores") - public void testLocalStores(Store.ListableStore store) throws IOException, ZarrException { - boolean useParallel = true; - Store writeStore = store; - if (store instanceof ReadOnlyZipStore) { - StoreHandle underlyingStore = ((ReadOnlyZipStore)store).underlyingStore; - writeStore = new BufferedZipStore(underlyingStore, true); - } - Group group = writeTestGroupV3(writeStore, useParallel); - - java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( - "array/c/1/1", - "array/c/0/0", - "array/c/0/1", - "zarr.json", - "array", - "array/c/1/0", - "array/c/1", - "array/c/0", - "array/zarr.json", - "array/c" - )); - - java.util.Set actualKeys = store.resolve("subgroup").list() - .map(node -> String.join("/", node)) - .collect(Collectors.toSet()); - - Assertions.assertEquals(expectedSubgroupKeys, actualKeys); - - assertIsTestGroupV3(group, useParallel); - } - - - int[] testData() { - int[] testData = new int[1024 * 1024]; - Arrays.setAll(testData, p -> p); - return testData; - } - - Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException { - StoreHandle storeHandle = store.resolve(); - - dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle); - dev.zarr.zarrjava.v3.Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) - .withChunkShape(512, 512) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); - dev.zarr.zarrjava.v3.Group subgroup = group.createGroup("subgroup"); - dev.zarr.zarrjava.v3.Array subgrouparray = subgroup.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) - .withChunkShape(512, 512) - ); - subgrouparray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); - - group.setAttributes(new Attributes(b -> b.set("some", "value"))); - return group; - } - - void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, IOException { - Stream nodes = group.list(); - List nodeList = nodes.collect(Collectors.toList()); - Assertions.assertEquals(3, nodeList.size()); - Array array = (Array) group.get("array"); - Assertions.assertNotNull(array); - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Group subgroup = (Group) group.get("subgroup"); - Array subgrouparray = (Array) subgroup.get("array"); - result = subgrouparray.read(useParallel); - Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes(); - Assertions.assertNotNull(attrs); - Assertions.assertEquals("value", attrs.getString("some")); - } - - - dev.zarr.zarrjava.v2.Group writeTestGroupV2(Store store, boolean useParallel) throws ZarrException, IOException { - StoreHandle storeHandle = store.resolve(); - - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); - dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) - .withChunks(512, 512) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel); - group.createGroup("subgroup"); - group.setAttributes(new Attributes().set("some", "value")); - return group; - } - - void assertIsTestGroupV2(Group group, boolean useParallel) throws ZarrException, IOException { - Stream nodes = group.list(); - Assertions.assertEquals(2, nodes.count()); - Array array = (Array) group.get("array"); - Assertions.assertNotNull(array); - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes(); - Assertions.assertNotNull(attrs); - Assertions.assertEquals("value", attrs.getString("some")); - } -} diff --git a/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java new file mode 100644 index 0000000..4b55797 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java @@ -0,0 +1,168 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.Utils; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Array; +import dev.zarr.zarrjava.core.Group; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.zip.ZipEntry; + +import static dev.zarr.zarrjava.Utils.unzipFile; + +public class BufferedZipStoreTest extends WritableStoreTest { + + Path testGroupDir = TESTOUTPUT.resolve("testZipStore.zip"); + + @BeforeAll + void writeTestGroup() throws ZarrException, IOException { + Path sourceDir = TESTOUTPUT.resolve("testZipStore"); + FilesystemStore fsStore = new FilesystemStore(sourceDir); + writeTestGroupV3(fsStore.resolve(), true); + Utils.zipFile(sourceDir, testGroupDir); + } + + @Override + StoreHandle storeHandleWithData() { + return new BufferedZipStore(testGroupDir).resolve("zarr.json"); + } + + @Test + public void testOpenZipStore() throws ZarrException, IOException { + BufferedZipStore zipStore = new BufferedZipStore(testGroupDir); + assertIsTestGroupV3(Group.open(zipStore.resolve()), true); + } + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testWriteZipStore" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); + writeTestGroupV3(zipStore.resolve(), true); + if (!flushOnWrite) zipStore.flush(); + + BufferedZipStore zipStoreRead = new BufferedZipStore(path); + assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true); + + Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped" + (flushOnWrite ? "Flush" : "NoFlush")); + + unzipFile(path, unzippedPath); + FilesystemStore fsStore = new FilesystemStore(unzippedPath); + assertIsTestGroupV3(Group.open(fsStore.resolve()), true); + } + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreWithComment" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + String comment = "{\"ome\": { \"version\": \"XX.YY\" }}"; + BufferedZipStore zipStore = new BufferedZipStore(path, comment, flushOnWrite); + writeTestGroupV3(zipStore.resolve(), true); + if (!flushOnWrite) zipStore.flush(); + + try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) { + String retrievedComment = zipFile.getComment(); + Assertions.assertEquals(comment, retrievedComment, "ZIP archive comment does not match expected value."); + } + + Assertions.assertEquals(comment, new BufferedZipStore(path).getArchiveComment(), "ZIP archive comment from store does not match expected value."); + } + + /** + * Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr + * + * @see RFC-9: Zipped OME-Zarr + */ + @Test + public void testZipStoreRequirements() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip"); + BufferedZipStore zipStore = new BufferedZipStore(path); + + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(zipStore.resolve()); + Array array = group.createArray("a1", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) + .withChunkShape(512, 512) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testDataInt()), true); + + dev.zarr.zarrjava.v3.Group g1 = group.createGroup("g1"); + g1.createGroup("g1_1").createGroup("g1_1_1"); + g1.createGroup("g1_2"); + group.createGroup("g2").createGroup("g2_1"); + group.createGroup("g3"); + + zipStore.flush(); + + try (ZipFile zip = new ZipFile(path.toFile())) { + ArrayList entries = Collections.list(zip.getEntries()); + + // no compression + for (ZipArchiveEntry e : entries) { + Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed"); + } + + // correct order of zarr.json files + String[] expectedFirstEntries = new String[]{ + "zarr.json", + "a1/zarr.json", + "g1/zarr.json", + "g2/zarr.json", + "g3/zarr.json", + "g1/g1_1/zarr.json", + "g1/g1_2/zarr.json", + "g2/g2_1/zarr.json", + "g1/g1_1/g1_1_1/zarr.json" + }; + String[] actualFirstEntries = entries.stream() + .map(ZipArchiveEntry::getName) + .limit(expectedFirstEntries.length) + .toArray(String[]::new); + + Assertions.assertArrayEquals(expectedFirstEntries, actualFirstEntries, "zarr.json files are not in the expected breadth-first order"); + } + } + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testZipStoreV2" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip"); + BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite); + writeTestGroupV2(zipStore.resolve(), true); + if (!flushOnWrite) zipStore.flush(); + + BufferedZipStore zipStoreRead = new BufferedZipStore(path); + assertIsTestGroupV2(Group.open(zipStoreRead.resolve()), true); + + Path unzippedPath = TESTOUTPUT.resolve("testZipStoreV2Unzipped"); + + unzipFile(path, unzippedPath); + FilesystemStore fsStore = new FilesystemStore(unzippedPath); + assertIsTestGroupV2(Group.open(fsStore.resolve()), true); + } + + + @Override + Store writableStore() { + Path path = TESTOUTPUT.resolve("writableStore.ZIP"); + if (Files.exists(path)) { + try{ + Files.delete(path); + }catch (IOException e) { + throw new RuntimeException("Failed to delete existing test ZIP store at: " + path.toAbsolutePath(), e); + } + } + return new BufferedZipStore(TESTOUTPUT.resolve("writableStore.ZIP"), true); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java new file mode 100644 index 0000000..ae6a088 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java @@ -0,0 +1,62 @@ +package dev.zarr.zarrjava.store; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; + +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; + +public class FileSystemStoreTest extends WritableStoreTest{ + + @Override + StoreHandle storeHandleWithData() { + return new FilesystemStore(TESTDATA).resolve("l4_sample", "zarr.json"); + } + + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = makeObjectMapper(); + + GroupMetadata groupMetadata = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + dev.zarr.zarrjava.v3.GroupMetadata.class + ); + + String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); + Assertions.assertTrue(groupMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + dev.zarr.zarrjava.v3.ArrayMetadata.class); + + String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); + Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(arrayMetadataString.contains("\"node_type\":\"array\"")); + Assertions.assertTrue(arrayMetadataString.contains("\"shape\":[1,4096,4096,2048]")); + + Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1"))); + + Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new); + Assertions.assertEquals(12, subNodes.length); + + Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new); + + Assertions.assertEquals(5, colorSubNodes.length); + Assertions.assertInstanceOf(Array.class, colorSubNodes[0]); + + Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1"); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); + } + + @Override + Store writableStore() { + return new FilesystemStore(TESTOUTPUT.resolve("writableFSStore")); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java new file mode 100644 index 0000000..3d45cf7 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java @@ -0,0 +1,25 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Array; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +public class HttpStoreTest extends StoreTest { + + @Override + StoreHandle storeHandleWithData() { + HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.5/idr0033A"); + return httpStore.resolve("BR00109990_C2.zarr", "0", "0"); + + } + + @Test + public void testHttpStore() throws IOException, ZarrException { + Array array = Array.open(storeHandleWithData()); + Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, array.metadata().shape); + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java new file mode 100644 index 0000000..5367705 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java @@ -0,0 +1,77 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Array; +import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.Node; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.stream.Stream; + +public class MemoryStoreTest extends WritableStoreTest { + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException { + int[] testData = testDataInt(); + + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve()); + Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) + .withChunkShape(64, 64) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { + int[] testData = testDataInt(); + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve()); + dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withChunks(512, 512) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); + group.createGroup("subgroup"); + Stream nodes = group.list(); + group.setAttributes(new Attributes().set("description", "test group")); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("test group", attrs.getString("description")); + + } + + @Override + Store writableStore() { + return new MemoryStore(); + } + + @Override + StoreHandle storeHandleWithData() { + StoreHandle memoryStoreHandle = new MemoryStore().resolve(); + memoryStoreHandle.set(ByteBuffer.wrap(testData())); + return memoryStoreHandle; + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java new file mode 100644 index 0000000..1330df4 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java @@ -0,0 +1,67 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Array; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class OnlineS3StoreTest extends StoreTest { + StoreHandle storeHandle; + + @BeforeAll + void createStore() { + S3Store s3Store = new S3Store(S3Client.builder() + .endpointOverride(URI.create("https://uk1s3.embassy.ebi.ac.uk")) + .region(Region.US_EAST_1) // required, but ignored + .serviceConfiguration( + S3Configuration.builder() + .pathStyleAccessEnabled(true) // required + .build() + ) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "idr", "zarr/v0.5/idr0033A"); + storeHandle = s3Store.resolve("BR00109990_C2.zarr", "0", "0"); + } + + @Test + public void testOpen() throws IOException, ZarrException { + Array arrayV3 = Array.open(storeHandle); + Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayV3.metadata().shape); + Assertions.assertEquals(574, arrayV3.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); + + dev.zarr.zarrjava.core.Array arrayCore = dev.zarr.zarrjava.core.Array.open(storeHandle); + Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, arrayCore.metadata().shape); + Assertions.assertEquals(574, arrayCore.read(new long[]{0, 0, 0}, new int[]{1, 1, 1}).getInt(0)); + } + + + @Test + public void testGet() { + StoreHandle s3StoreHandle = storeHandle.resolve("zarr.json"); + S3Store s3Store = (S3Store) s3StoreHandle.store; + ByteBuffer buffer = s3Store.get(s3StoreHandle.keys); + ByteBuffer bufferWithStart = s3Store.get(s3StoreHandle.keys, 10); + Assertions.assertEquals(10, buffer.remaining() - bufferWithStart.remaining()); + + ByteBuffer bufferWithStartAndEnd = s3Store.get(s3StoreHandle.keys, 0, 10); + Assertions.assertEquals(10, bufferWithStartAndEnd.remaining()); + } + + @Override + StoreHandle storeHandleWithData() { + return storeHandle.resolve("zarr.json"); + } +} + + diff --git a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java new file mode 100644 index 0000000..7af3558 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java @@ -0,0 +1,78 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.Utils; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Group; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Collectors; + +public class ReadOnlyZipStoreTest extends StoreTest { + + StoreHandle storeHandleWithData; + + @BeforeAll + void writeStoreHandleWithData() throws ZarrException, IOException { + Path source = TESTDATA.resolve("v2_sample").resolve("bool"); + Path target = TESTOUTPUT.resolve("readOnlyZipStoreTest.zip"); + Utils.zipFile(source, target); + storeHandleWithData = new ReadOnlyZipStore(target).resolve("0.0.0"); + } + + @Override + StoreHandle storeHandleWithData() { + return storeHandleWithData; + } + + @Test + public void testOpen() throws ZarrException, IOException { + Path sourceDir = TESTOUTPUT.resolve("testZipStore"); + Path targetDir = TESTOUTPUT.resolve("testZipStore.zip"); + FilesystemStore fsStore = new FilesystemStore(sourceDir); + writeTestGroupV3(fsStore.resolve(), true); + + Utils.zipFile(sourceDir, targetDir); + + ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(targetDir); + assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); + } + + + @Test + public void testReadFromBufferedZipStore() throws ZarrException, IOException { + Path path = TESTOUTPUT.resolve("testReadOnlyZipStore.zip"); + String archiveComment = "This is a test ZIP archive comment."; + BufferedZipStore zipStore = new BufferedZipStore(path, archiveComment); + writeTestGroupV3(zipStore.resolve(), true); + zipStore.flush(); + + ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(path); + Assertions.assertEquals(archiveComment, readOnlyZipStore.getArchiveComment(), "ZIP archive comment from ReadOnlyZipStore does not match expected value."); + + java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( + "array/c/1/1", + "array/c/0/0", + "array/c/0/1", + "zarr.json", + "array", + "array/c/1/0", + "array/c/1", + "array/c/0", + "array/zarr.json", + "array/c" + )); + + java.util.Set actualKeys = readOnlyZipStore.resolve("subgroup").list() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + + Assertions.assertEquals(expectedSubgroupKeys, actualKeys); + + assertIsTestGroupV3(Group.open(readOnlyZipStore.resolve()), true); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java new file mode 100644 index 0000000..520d3a0 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java @@ -0,0 +1,97 @@ +package dev.zarr.zarrjava.store; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; + +/** + * Tests for S3Store + *

+ * Requires a local S3 mock server running at http://localhost:9090 + * with a bucket named "zarr-test-bucket" + *

+ * Execute the following command to start a local S3 mock server: + * 

+ * docker run -p 9090:9090 -p 9191:9191 -e "initialBuckets=zarr-test-bucket" adobe/s3mock:3.11.0
+ *
+ */ +@Tag("s3") +public class S3StoreTest extends WritableStoreTest { + + String s3Endpoint = "http://localhost:9090"; + String bucketName = "zarr-test-bucket"; + S3Client s3Client; + String testDataKey = "testData"; + + @BeforeAll + void setUpS3Client() { + s3Client = S3Client.builder() + .endpointOverride(URI.create(s3Endpoint)) + .region(Region.US_EAST_1) // required, but ignored + .serviceConfiguration( + S3Configuration.builder() + .pathStyleAccessEnabled(true) // required + .build() + ) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("accessKey", "secretKey") + )) + .build(); + // Clean up the bucket + try { + s3Client.listObjectsV2Paginator(builder -> builder.bucket(bucketName).build()) + .contents() + .forEach(s3Object -> { + s3Client.deleteObject(builder -> builder.bucket(bucketName).key(s3Object.key()).build()); + }); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + void testReadWriteS3Store() { + S3Store s3Store = new S3Store(s3Client, bucketName, ""); + + StoreHandle storeHandle = s3Store.resolve("testfile"); + byte[] testData = new byte[100]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) i; + } + storeHandle.set(ByteBuffer.wrap(testData)); + ByteBuffer retrievedData = storeHandle.read(); + byte[] retrievedBytes = new byte[retrievedData.remaining()]; + retrievedData.get(retrievedBytes); + Assertions.assertArrayEquals(testData, retrievedBytes); + } + + + @Override + Store writableStore() { + return new S3Store(s3Client, bucketName, ""); + } + + @Override + StoreHandle storeHandleWithData() { + try (InputStream byteStream = new ByteArrayInputStream(testData())) { + s3Client.putObject(PutObjectRequest.builder().bucket(bucketName).key("/" + testDataKey).build(), RequestBody.fromContentProvider(() -> byteStream, "application/octet-stream")); + } catch (IOException e) { + throw new RuntimeException(e); + } + return new S3Store(s3Client, bucketName, "").resolve(testDataKey); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java new file mode 100644 index 0000000..9862879 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -0,0 +1,126 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.ZarrTest; +import dev.zarr.zarrjava.core.Array; +import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.core.Group; +import dev.zarr.zarrjava.core.Node; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import ucar.ma2.DataType; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class StoreTest extends ZarrTest { + + abstract StoreHandle storeHandleWithData(); + + @Test + public void testInputStream() throws IOException { + StoreHandle storeHandle = storeHandleWithData(); + InputStream is = storeHandle.getInputStream(10, 20); + byte[] buffer = new byte[10]; + int bytesRead = is.read(buffer); + Assertions.assertEquals(10, bytesRead); + byte[] expectedBuffer = new byte[10]; + storeHandle.read(10, 20).get(expectedBuffer); + Assertions.assertArrayEquals(expectedBuffer, buffer); + } + + + @Test + public void testStoreGetSize() { + StoreHandle storeHandle = storeHandleWithData(); + long size = storeHandle.getSize(); + long actual_size = storeHandle.read().remaining(); + Assertions.assertEquals(actual_size, size); + } + + + byte[] testData() { + byte[] testData = new byte[1024 * 1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = (byte) (i % 256); + } + return testData; + } + int[] testDataInt() { + int[] testData = new int[1024 * 1024]; + for (int i = 0; i < testData.length; i++) { + testData[i] = i; + } + return testData; + } + + + Group writeTestGroupV3(StoreHandle storeHandle, boolean useParallel) throws ZarrException, IOException { + + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle); + dev.zarr.zarrjava.v3.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) + .withChunkShape(512, 512) + ); + array.write(ucar.ma2.Array.factory(DataType.BYTE, new int[]{1024, 1024}, testData()), useParallel); + dev.zarr.zarrjava.v3.Group subgroup = group.createGroup("subgroup"); + dev.zarr.zarrjava.v3.Array subgrouparray = subgroup.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) + .withChunkShape(512, 512) + ); + subgrouparray.write(ucar.ma2.Array.factory(DataType.BYTE, new int[]{1024, 1024}, testData()), useParallel); + + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + return group; + } + + void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException, IOException { + Stream nodes = group.list(); + List nodeList = nodes.collect(Collectors.toList()); + Assertions.assertEquals(3, nodeList.size()); + Array array = (Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (byte[]) result.get1DJavaArray(DataType.BYTE)); + Group subgroup = (Group) group.get("subgroup"); + Array subgrouparray = (Array) subgroup.get("array"); + result = subgrouparray.read(useParallel); + Assertions.assertArrayEquals(testData(), (byte[]) result.get1DJavaArray(ucar.ma2.DataType.BYTE)); + Attributes attrs = group.metadata().attributes(); + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } + + + dev.zarr.zarrjava.v2.Group writeTestGroupV2(StoreHandle storeHandle, boolean useParallel) throws ZarrException, IOException { + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); + dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(512, 512) + ); + array.write(ucar.ma2.Array.factory(DataType.BYTE, new int[]{1024, 1024}, testData()), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes().set("some", "value")); + return group; + } + + void assertIsTestGroupV2(Group group, boolean useParallel) throws ZarrException, IOException { + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + Array array = (Array) group.get("array"); + Assertions.assertNotNull(array); + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData(), (byte[]) result.get1DJavaArray(DataType.BYTE)); + Attributes attrs = group.metadata().attributes(); + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java new file mode 100644 index 0000000..8378040 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -0,0 +1,49 @@ +package dev.zarr.zarrjava.store; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Group; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Collectors; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class WritableStoreTest extends StoreTest { + abstract Store writableStore(); + + @Test + public void testList() throws ZarrException, IOException { + StoreHandle storeHandle = writableStore().resolve("testList"); + boolean useParallel = true; + writeTestGroupV3(storeHandle, useParallel); + java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( + "array/c/1/1", + "array/c/0/0", + "array/c/0/1", + "zarr.json", + "array", + "array/c/1/0", + "array/c/1", + "array/c/0", + "array/zarr.json", + "array/c" + )); + + java.util.Set actualKeys = storeHandle.resolve("subgroup").list() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + Assertions.assertEquals(expectedSubgroupKeys, actualKeys); + } + + @Test + public void testWriteRead() throws IOException, ZarrException { + StoreHandle storeHandle = writableStore().resolve("testWriteRead"); + boolean useParallel = true; + Group group = writeTestGroupV3(storeHandle, useParallel); + assertIsTestGroupV3(group, useParallel); + } + +} From 4e4f1e816a6b5e152b34bf73a8d1a50452e03703 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 18:06:05 +0100 Subject: [PATCH 48/61] fix s3Store::list --- .../java/dev/zarr/zarrjava/store/S3Store.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 2b9c73d..3dcdbfd 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -12,6 +12,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.stream.Stream; public class S3Store implements Store, Store.ListableStore { @@ -110,9 +112,20 @@ public Stream list(String[] keys) { .bucket(bucketName).prefix(fullKey) .build(); ListObjectsResponse res = s3client.listObjects(req); - return res.contents() - .stream() - .map(p -> p.key().substring(fullKey.length() + 1).split("/")); + return res.contents().stream() + .map(S3Object::key) + .flatMap(key -> { + List pathSegments = new ArrayList<>(); + int index = fullKey.length(); + while ((index = key.indexOf('/', index + 1)) != -1) { + pathSegments.add(key.substring(fullKey.length() + 1, index)); + } + pathSegments.add(key.substring(fullKey.length() + 1)); + return pathSegments.stream(); + }) + .distinct() + .map(s -> s.split("/")) + .filter(arr -> arr.length > 0); } @Nonnull From 8d643859cc53019f73160962a4f3d79f5a0cdc13 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 18:07:28 +0100 Subject: [PATCH 49/61] fix ReadOnlyZipStore::list --- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 6 ++++- src/test/java/dev/zarr/zarrjava/Utils.java | 3 --- .../zarrjava/store/ReadOnlyZipStoreTest.java | 25 +++++++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 8b906fd..1e50da5 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -146,7 +146,11 @@ public Stream list(String[] keys) { if (!entryName.startsWith(prefix) || entryName.equals(prefix)) { continue; } - String[] entryKeys = resolveEntryKeys(entryName.substring(prefix.length() + 1)); + entryName = entryName.substring(prefix.length()); + if (entryName.startsWith("/")) { + entryName = entryName.substring(1); + } + String[] entryKeys = resolveEntryKeys(entryName); builder.add(entryKeys); } } catch (IOException ignored) { diff --git a/src/test/java/dev/zarr/zarrjava/Utils.java b/src/test/java/dev/zarr/zarrjava/Utils.java index b41a714..e07bbb4 100644 --- a/src/test/java/dev/zarr/zarrjava/Utils.java +++ b/src/test/java/dev/zarr/zarrjava/Utils.java @@ -21,9 +21,6 @@ public static void zipFile(Path sourceDir, Path targetDir) throws IOException { } static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException { - if (fileToZip.isHidden()) { - return; - } if (fileToZip.isDirectory()) { if (fileName.endsWith("/")) { zipOut.putNextEntry(new ZipEntry(fileName)); diff --git a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java index 7af3558..7b73af4 100644 --- a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java @@ -12,16 +12,16 @@ import java.util.Arrays; import java.util.stream.Collectors; -public class ReadOnlyZipStoreTest extends StoreTest { +public class ReadOnlyZipStoreTest extends StoreTest { + Path storePath = TESTOUTPUT.resolve("readOnlyZipStoreTest.zip"); StoreHandle storeHandleWithData; @BeforeAll void writeStoreHandleWithData() throws ZarrException, IOException { - Path source = TESTDATA.resolve("v2_sample").resolve("bool"); - Path target = TESTOUTPUT.resolve("readOnlyZipStoreTest.zip"); - Utils.zipFile(source, target); - storeHandleWithData = new ReadOnlyZipStore(target).resolve("0.0.0"); + Path source = TESTDATA.resolve("v2_sample").resolve("subgroup"); + Utils.zipFile(source, storePath); + storeHandleWithData = new ReadOnlyZipStore(storePath).resolve("array", "0.0.0"); } @Override @@ -29,6 +29,21 @@ StoreHandle storeHandleWithData() { return storeHandleWithData; } + @Override + @Test + void testList() { + ReadOnlyZipStore zipStore = new ReadOnlyZipStore(storePath); + BufferedZipStore bufferedZipStore = new BufferedZipStore(storePath); + + java.util.Set expectedKeys = bufferedZipStore.resolve().list() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + java.util.Set actualKeys = zipStore.resolve().list() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + Assertions.assertEquals(expectedKeys, actualKeys); + } + @Test public void testOpen() throws ZarrException, IOException { Path sourceDir = TESTOUTPUT.resolve("testZipStore"); From 78c0e3a98b8dee36a3cc4a3cb2ae85adbbb30f94 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 18:08:13 +0100 Subject: [PATCH 50/61] add tests --- .../zarr/zarrjava/store/HttpStoreTest.java | 12 ++++ .../zarr/zarrjava/store/MemoryStoreTest.java | 47 ------------- .../zarrjava/store/OnlineS3StoreTest.java | 6 ++ .../dev/zarr/zarrjava/store/StoreTest.java | 3 + .../zarrjava/store/WritableStoreTest.java | 66 ++++++++++++++++++- 5 files changed, 86 insertions(+), 48 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java index 3d45cf7..751ab6a 100644 --- a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java @@ -22,4 +22,16 @@ public void testHttpStore() throws IOException, ZarrException { Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, array.metadata().shape); } + @Override + @Test + public void testStoreGetSize() { + // size is not defined in BR00109990_C2.zarr + long size = storeHandleWithData().getSize(); + Assertions.assertEquals(-1, size); + } + + @Override + void testList() throws ZarrException, IOException { + // listing is not supported in HttpStore + } } diff --git a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java index 5367705..c57905a 100644 --- a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java @@ -14,54 +14,7 @@ public class MemoryStoreTest extends WritableStoreTest { - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException { - int[] testData = testDataInt(); - dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve()); - Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) - .withChunkShape(64, 64) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); - group.createGroup("subgroup"); - group.setAttributes(new Attributes(b -> b.set("some", "value"))); - Stream nodes = group.list(); - Assertions.assertEquals(2, nodes.count()); - - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes; - Assertions.assertNotNull(attrs); - Assertions.assertEquals("value", attrs.getString("some")); - } - - @ParameterizedTest - @CsvSource({"false", "true",}) - public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { - int[] testData = testDataInt(); - - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve()); - dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b - .withShape(1024, 1024) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) - .withChunks(512, 512) - ); - array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); - group.createGroup("subgroup"); - Stream nodes = group.list(); - group.setAttributes(new Attributes().set("description", "test group")); - Assertions.assertEquals(2, nodes.count()); - - ucar.ma2.Array result = array.read(useParallel); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Attributes attrs = group.metadata().attributes; - Assertions.assertNotNull(attrs); - Assertions.assertEquals("test group", attrs.getString("description")); - - } @Override Store writableStore() { diff --git a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java index 1330df4..885cbf6 100644 --- a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java @@ -62,6 +62,12 @@ public void testGet() { StoreHandle storeHandleWithData() { return storeHandle.resolve("zarr.json"); } + + @Override + @Test + void testList() { + Assertions.assertTrue(storeHandle.list().count() > 1); + } } diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index 9862879..ece5318 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -123,4 +123,7 @@ void assertIsTestGroupV2(Group group, boolean useParallel) throws ZarrException, Assertions.assertNotNull(attrs); Assertions.assertEquals("value", attrs.getString("some")); } + + @Test + abstract void testList() throws ZarrException, IOException; } diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java index 8378040..77be681 100644 --- a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -1,21 +1,28 @@ package dev.zarr.zarrjava.store; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Array; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.Group; +import dev.zarr.zarrjava.core.Node; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.stream.Collectors; +import java.util.stream.Stream; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class WritableStoreTest extends StoreTest { abstract Store writableStore(); @Test - public void testList() throws ZarrException, IOException { + public void testList() throws IOException, ZarrException { StoreHandle storeHandle = writableStore().resolve("testList"); boolean useParallel = true; writeTestGroupV3(storeHandle, useParallel); @@ -36,6 +43,11 @@ public void testList() throws ZarrException, IOException { .map(node -> String.join("/", node)) .collect(Collectors.toSet()); Assertions.assertEquals(expectedSubgroupKeys, actualKeys); + + List allKeys = storeHandle.list() + .map(node -> String.join("/", node)) + .collect(Collectors.toList()); + Assertions.assertEquals(21, allKeys.size(), "Total number of keys in store should be 21 but was: " + allKeys); } @Test @@ -46,4 +58,56 @@ public void testWriteRead() throws IOException, ZarrException { assertIsTestGroupV3(group, useParallel); } + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testWriteReadV3(boolean useParallel) throws ZarrException, IOException { + int[] testData = testDataInt(); + Store store = writableStore(); + StoreHandle storeHandle = store.resolve("testWriteReadV3").resolve(store.getClass().getSimpleName()); + + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT32) + .withChunkShape(64, 64) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); + group.createGroup("subgroup"); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); + } + + @ParameterizedTest + @CsvSource({"false", "true",}) + public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { + int[] testData = testDataInt(); + Store store = writableStore(); + StoreHandle storeHandle = store.resolve("testMemoryStoreV2").resolve(store.getClass().getSimpleName()); + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); + dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withChunks(512, 512) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); + group.createGroup("subgroup"); + Stream nodes = group.list(); + group.setAttributes(new Attributes().set("description", "test group")); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("test group", attrs.getString("description")); + } + } From 9b738298e81cb3076aec04e3c50b8498f1771f93 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 16 Jan 2026 18:12:25 +0100 Subject: [PATCH 51/61] fix tests with path collision --- .../java/dev/zarr/zarrjava/store/WritableStoreTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java index 77be681..e1c9b45 100644 --- a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -63,7 +63,7 @@ public void testWriteRead() throws IOException, ZarrException { public void testWriteReadV3(boolean useParallel) throws ZarrException, IOException { int[] testData = testDataInt(); Store store = writableStore(); - StoreHandle storeHandle = store.resolve("testWriteReadV3").resolve(store.getClass().getSimpleName()); + StoreHandle storeHandle = store.resolve("testWriteReadV3").resolve(store.getClass().getSimpleName()).resolve("" + useParallel); dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle); Array array = group.createArray("array", b -> b @@ -86,11 +86,10 @@ public void testWriteReadV3(boolean useParallel) throws ZarrException, IOExcepti @ParameterizedTest @CsvSource({"false", "true",}) - public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { + public void testWriteReadV2(boolean useParallel) throws ZarrException, IOException { int[] testData = testDataInt(); Store store = writableStore(); - StoreHandle storeHandle = store.resolve("testMemoryStoreV2").resolve(store.getClass().getSimpleName()); - + StoreHandle storeHandle = store.resolve("testMemoryStoreV2").resolve(store.getClass().getSimpleName()).resolve("" + useParallel); dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b .withShape(1024, 1024) From 77060dea47fa9b7692305a4f6d4d52ebfe00f28e Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 11:05:30 +0100 Subject: [PATCH 52/61] reformat code --- src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java | 9 ++++----- .../dev/zarr/zarrjava/store/BufferedZipStoreTest.java | 4 ++-- .../dev/zarr/zarrjava/store/FileSystemStoreTest.java | 2 +- .../java/dev/zarr/zarrjava/store/MemoryStoreTest.java | 11 ----------- src/test/java/dev/zarr/zarrjava/store/StoreTest.java | 1 + 5 files changed, 8 insertions(+), 19 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 6186fe8..859314d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -32,7 +32,6 @@ public class ZarrPythonTests extends ZarrTest { final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); - public static int runCommand(String... command) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(); pb.command().addAll(Arrays.asList(command)); @@ -91,13 +90,13 @@ static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) { break; case LONG: case ULONG: - array.setLong(i, (long) i); + array.setLong(i, i); break; case FLOAT: array.setFloat(i, (float) i); break; case DOUBLE: - array.setDouble(i, (double) i); + array.setDouble(i, i); break; default: throw new IllegalArgumentException("Invalid DataType: " + dt); @@ -130,13 +129,13 @@ static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataT break; case LONG: case ULONG: - Assertions.assertEquals((long) i, result.getLong(i)); + Assertions.assertEquals(i, result.getLong(i)); break; case FLOAT: Assertions.assertEquals((float) i, result.getFloat(i), 1e-6); break; case DOUBLE: - Assertions.assertEquals((double) i, result.getDouble(i), 1e-12); + Assertions.assertEquals(i, result.getDouble(i), 1e-12); break; default: throw new IllegalArgumentException("Invalid DataType: " + dt); diff --git a/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java index 4b55797..ba7d404 100644 --- a/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java @@ -157,9 +157,9 @@ public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOExcepti Store writableStore() { Path path = TESTOUTPUT.resolve("writableStore.ZIP"); if (Files.exists(path)) { - try{ + try { Files.delete(path); - }catch (IOException e) { + } catch (IOException e) { throw new RuntimeException("Failed to delete existing test ZIP store at: " + path.toAbsolutePath(), e); } } diff --git a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java index ae6a088..e8e58fa 100644 --- a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java @@ -11,7 +11,7 @@ import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -public class FileSystemStoreTest extends WritableStoreTest{ +public class FileSystemStoreTest extends WritableStoreTest { @Override StoreHandle storeHandleWithData() { diff --git a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java index c57905a..5ca7d5f 100644 --- a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java @@ -1,21 +1,10 @@ package dev.zarr.zarrjava.store; -import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.core.Array; -import dev.zarr.zarrjava.core.Attributes; -import dev.zarr.zarrjava.core.Node; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; - -import java.io.IOException; import java.nio.ByteBuffer; -import java.util.stream.Stream; public class MemoryStoreTest extends WritableStoreTest { - @Override Store writableStore() { return new MemoryStore(); diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index ece5318..51a0e26 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -51,6 +51,7 @@ byte[] testData() { } return testData; } + int[] testDataInt() { int[] testData = new int[1024 * 1024]; for (int i = 0; i < testData.length; i++) { From 2e048f772c72d0732e4d46f11cc8316bc11e6d70 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 13:40:06 +0100 Subject: [PATCH 53/61] improve httpStore testing --- .../dev/zarr/zarrjava/store/HttpStoreTest.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java index 751ab6a..0d1e087 100644 --- a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java @@ -11,25 +11,20 @@ public class HttpStoreTest extends StoreTest { @Override StoreHandle storeHandleWithData() { + return br00109990StoreHandle().resolve("c", "0", "0", "0"); + } + + StoreHandle br00109990StoreHandle() { HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.5/idr0033A"); return httpStore.resolve("BR00109990_C2.zarr", "0", "0"); - } @Test - public void testHttpStore() throws IOException, ZarrException { - Array array = Array.open(storeHandleWithData()); + public void testOpen() throws IOException, ZarrException { + Array array = Array.open(br00109990StoreHandle()); Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, array.metadata().shape); } - @Override - @Test - public void testStoreGetSize() { - // size is not defined in BR00109990_C2.zarr - long size = storeHandleWithData().getSize(); - Assertions.assertEquals(-1, size); - } - @Override void testList() throws ZarrException, IOException { // listing is not supported in HttpStore From a1e36011df4137438946ebf1f20b8968ca0c2fe7 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 16:53:55 +0100 Subject: [PATCH 54/61] change store::list to list no directories add store::listChildren containing direct children including directories --- .../zarr/zarrjava/store/BufferedZipStore.java | 5 + .../zarr/zarrjava/store/FilesystemStore.java | 49 ++++++--- .../dev/zarr/zarrjava/store/MemoryStore.java | 38 ++++--- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 74 +++++++++----- .../java/dev/zarr/zarrjava/store/S3Store.java | 99 +++++++++++++++---- .../java/dev/zarr/zarrjava/store/Store.java | 28 ++++-- 6 files changed, 222 insertions(+), 71 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 934da19..4fffb1a 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -245,6 +245,11 @@ public Stream list(String[] keys) { return bufferStore.list(keys); } + @Override + public Stream listChildren(String[] prefix) { + return bufferStore.listChildren(prefix); + } + @Override public boolean exists(String[] keys) { return bufferStore.exists(keys); diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index 90da970..a255ab8 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -120,21 +120,44 @@ public void delete(String[] keys) { } } - public Stream list(String[] keys) { - Path keyPath = resolveKeys(keys); + /** + * Helper to convert a filesystem Path back into the full String[] key array + * relative to the prefix + */ + private String[] pathToKeyArray(Path rootPath, Path currentPath, String[] prefix) { + Path relativePath = rootPath.relativize(currentPath); + int relativeCount = relativePath.getNameCount(); + + String[] result = new String[relativeCount]; + for (int i = 0; i < relativeCount; i++) { + result[i] = relativePath.getName(i).toString(); + } + return result; + } + + @Override + public Stream list(String[] prefix) { + Path rootPath = resolveKeys(prefix); try { - return Files.walk(keyPath) - .filter(path -> !path.equals(keyPath)) - .map(path -> { - Path relativePath = keyPath.relativize(path); - String[] parts = new String[relativePath.getNameCount()]; - for (int i = 0; i < relativePath.getNameCount(); i++) { - parts[i] = relativePath.getName(i).toString(); - } - return parts; - }); + return Files.walk(rootPath) + .filter(Files::isRegularFile) + .map(path -> pathToKeyArray(rootPath, path, prefix)); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException("Failed to list store content", e); + } + } + + @Override + public Stream listChildren(String[] prefix) { + Path rootPath = resolveKeys(prefix); + if (!Files.exists(rootPath) || !Files.isDirectory(rootPath)) { + return Stream.empty(); + } + try { + return Files.list(rootPath) // note: Files.list is non-recursive + .map(path -> pathToKeyArray(rootPath, path, prefix)); + } catch (IOException e) { + throw new RuntimeException("Failed to list store children", e); } } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index ea855c0..be7e6b5 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -60,18 +60,32 @@ public void delete(String[] keys) { map.remove(resolveKeys(keys)); } - public Stream list(String[] keys) { - List prefix = resolveKeys(keys); - Set> allKeys = new HashSet<>(); - - for (List k : map.keySet()) { - if (k.size() <= prefix.size() || !k.subList(0, prefix.size()).equals(prefix)) - continue; - for (int i = prefix.size(); i < k.size(); i++) { - allKeys.add(k.subList(prefix.size(), i + 1)); - } - } - return allKeys.stream().map(k -> k.toArray(new String[0])); + @Override + public Stream list(String[] prefix) { + List prefixList = resolveKeys(prefix); + int prefixSize = prefixList.size(); + + return map.keySet().stream() + .filter(key -> key.size() >= prefixSize && key.subList(0, prefixSize).equals(prefixList)) + .map(key -> key.subList(prefixSize, key.size()).toArray(new String[0])); + } + + @Override + public Stream listChildren(String[] prefix) { + List prefixList = resolveKeys(prefix); + int prefixSize = prefixList.size(); + + return map.keySet().stream() + .filter(key -> key.size() > prefixSize && key.subList(0, prefixSize).equals(prefixList)) + // Identify the immediate child segment + // e.g. if prefix is [a], and key is [a, b, c], the child is [a, b] + .map(key -> { + List childPath = new ArrayList<>(prefixList); + childPath.add(key.get(prefixSize)); + return childPath; + }) + .distinct() + .map(list -> list.toArray(new String[0])); } @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 1e50da5..6c6e035 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -125,39 +125,69 @@ public String toString() { } @Override - public Stream list(String[] keys) { + public Stream list(String[] prefixKeys) { Stream.Builder builder = Stream.builder(); - InputStream inputStream = underlyingStore.getInputStream(); - if (inputStream == null) { - return builder.build(); + if (inputStream == null) return builder.build(); + + String prefix = resolveKeys(prefixKeys); + if (!prefix.isEmpty() && !prefix.endsWith("/")) { + prefix += "/"; } + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { ZipArchiveEntry entry; - String prefix = resolveKeys(keys); while ((entry = zis.getNextEntry()) != null) { - String entryName = entry.getName(); - if (entryName.startsWith("/")) { - entryName = entryName.substring(1); - } - if (entryName.endsWith("/")) { - entryName = entryName.substring(0, entryName.length() - 1); + String name = normalizeEntryName(entry.getName()); + if (name.startsWith(prefix) && !entry.isDirectory()) { + builder.add(resolveEntryKeys(name.substring(prefix.length()))); } - if (!entryName.startsWith(prefix) || entryName.equals(prefix)) { - continue; - } - entryName = entryName.substring(prefix.length()); - if (entryName.startsWith("/")) { - entryName = entryName.substring(1); - } - String[] entryKeys = resolveEntryKeys(entryName); - builder.add(entryKeys); } - } catch (IOException ignored) { - } + } catch (IOException ignored) {} return builder.build(); } + @Override + public Stream listChildren(String[] prefixKeys) { + java.util.Set children = new java.util.LinkedHashSet<>(); + InputStream inputStream = underlyingStore.getInputStream(); + if (inputStream == null) return Stream.empty(); + + String prefix = resolveKeys(prefixKeys); + if (!prefix.isEmpty() && !prefix.endsWith("/")) { + prefix += "/"; + } + + try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) { + ZipArchiveEntry entry; + while ((entry = zis.getNextEntry()) != null) { + String name = normalizeEntryName(entry.getName()); + + if (name.startsWith(prefix) && !name.equals(prefix)) { + String relative = name.substring(prefix.length()); + String[] parts = relative.split("/"); + // The child is the prefix + the very next segment + String childSegment = parts[0]; + children.add(childSegment); + } + } + } catch (IOException ignored) {} + + return children.stream().map(segment -> { + String[] result = new String[prefixKeys.length + 1]; + System.arraycopy(prefixKeys, 0, result, 0, prefixKeys.length); + result[prefixKeys.length] = segment; + return result; + }); + } + + private String normalizeEntryName(String name) { + if (name.startsWith("/")) name = name.substring(1); + if (name.endsWith("/")) name = name.substring(0, name.length() - 1); + return name; + } + + @Override public InputStream getInputStream(String[] keys, long start, long end) { InputStream baseStream = underlyingStore.getInputStream(); diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 3dcdbfd..8706946 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -52,6 +52,9 @@ ByteBuffer get(GetObjectRequest getObjectRequest) { @Override public boolean exists(String[] keys) { + if (keys == null || keys.length == 0) { + return true; + } HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(); try { return s3client.headObject(req).sdkHttpResponse().statusCode() == 200; @@ -105,27 +108,89 @@ public void delete(String[] keys) { .build()); } +// @Override +// public Stream list(String[] keys) { +// final String fullKey = resolveKeys(keys); +// ListObjectsRequest req = ListObjectsRequest.builder() +// .bucket(bucketName).prefix(fullKey) +// .build(); +// ListObjectsResponse res = s3client.listObjects(req); +// return res.contents().stream() +// .map(S3Object::key) +// .flatMap(key -> { +// List pathSegments = new ArrayList<>(); +// int index = fullKey.length(); +// while ((index = key.indexOf('/', index + 1)) != -1) { +// pathSegments.add(key.substring(fullKey.length() + 1, index)); +// } +// pathSegments.add(key.substring(fullKey.length() + 1)); +// return pathSegments.stream(); +// }) +// .distinct() +// .map(s -> s.split("/")) +// .filter(arr -> arr.length > 0); +// } + @Override public Stream list(String[] keys) { - final String fullKey = resolveKeys(keys); - ListObjectsRequest req = ListObjectsRequest.builder() - .bucket(bucketName).prefix(fullKey) + String fullPrefix = resolveKeys(keys); + // Ensure prefix ends with / for precise matching if not empty + if (!fullPrefix.isEmpty() && !fullPrefix.endsWith("/")) { + fullPrefix += "/"; + } + + ListObjectsV2Request req = ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(fullPrefix) .build(); - ListObjectsResponse res = s3client.listObjects(req); - return res.contents().stream() + + final String finalFullPrefix = fullPrefix; + return s3client.listObjectsV2Paginator(req).contents().stream() .map(S3Object::key) - .flatMap(key -> { - List pathSegments = new ArrayList<>(); - int index = fullKey.length(); - while ((index = key.indexOf('/', index + 1)) != -1) { - pathSegments.add(key.substring(fullKey.length() + 1, index)); - } - pathSegments.add(key.substring(fullKey.length() + 1)); - return pathSegments.stream(); - }) - .distinct() - .map(s -> s.split("/")) - .filter(arr -> arr.length > 0); + .filter(key -> !key.equals(finalFullPrefix) && !key.endsWith("/")) + .map(k -> keyToRelativeArray(k, finalFullPrefix)); + } + + @Override + public Stream listChildren(String[] keys) { + String fullPrefix = resolveKeys(keys); + if (!fullPrefix.isEmpty() && !fullPrefix.endsWith("/")) { + fullPrefix += "/"; + } + + ListObjectsV2Request req = ListObjectsV2Request.builder() + .bucket(bucketName) + .prefix(fullPrefix) + .delimiter("/") + .build(); + + ListObjectsV2Response res = s3client.listObjectsV2(req); + + // Combine CommonPrefixes (folders) and Contents (files) + Stream folders = res.commonPrefixes().stream().map(CommonPrefix::prefix); + final String finalFullPrefix = fullPrefix; + Stream files = res.contents().stream().map(S3Object::key) + .filter(key -> !key.equals(finalFullPrefix)); + + return Stream.concat(folders, files) + .map(k -> keyToRelativeArray(k, finalFullPrefix)); + } + + /** + * Helper to convert a full S3 key back into a String[] relative to the prefix. + */ + private String[] keyToRelativeArray(String fullS3Key, String prefix) { + String relativePath = fullS3Key; + if (prefix != null && fullS3Key.startsWith(prefix)) { + relativePath = fullS3Key.substring(prefix.length()); + } + if (relativePath.startsWith("/")) { + relativePath = relativePath.substring(1); + } + if (relativePath.endsWith("/")) { + relativePath = relativePath.substring(0, relativePath.length() - 1); + } + return relativePath.isEmpty() ? new String[0] : relativePath.split("/"); } @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index 3747aed..37b7aa3 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -40,19 +40,33 @@ default InputStream getInputStream(String[] keys) { */ long getSize(String[] keys); + /** + * A store that supports discovery of keys. + */ interface ListableStore extends Store { /** - * Lists all keys in the store that match the given prefix keys. Keys are represented as arrays of strings, - * where each string is a segment of the key path. - * Keys that are exactly equal to the prefix are not included in the results. - * Keys that do not contain data (i.e. "directories") are included in the results. + * Recursively lists all keys that contain data (leaf nodes) under the given prefix + * relative to the prefix. + * Directory-only entries are excluded. + * + * @param prefix The prefix keys to match. + * @return A stream of full key arrays containing data. + */ + Stream list(String[] prefix); + + /** + * Lists the immediate children (files and virtual directories) under the given prefix. + * This is useful for UI navigation or browsing the store hierarchy. * - * @param keys The prefix keys to match. - * @return A stream of key arrays that match the given prefix. Prefixed keys are not included in the results. + * @param prefix The prefix keys to explore. + * @return A stream of key arrays representing one level deeper than the prefix. */ - Stream list(String[] keys); + Stream listChildren(String[] prefix); + /** + * Lists all data-bearing keys in the entire store. + */ default Stream list() { return list(new String[]{}); } From ffd5debb6711a16bf41187bcf45f6a74234a7468 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 17:01:59 +0100 Subject: [PATCH 55/61] adjust test for store::list --- .../dev/zarr/zarrjava/store/WritableStoreTest.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java index e1c9b45..a13dd4e 100644 --- a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -31,12 +31,8 @@ public void testList() throws IOException, ZarrException { "array/c/0/0", "array/c/0/1", "zarr.json", - "array", "array/c/1/0", - "array/c/1", - "array/c/0", - "array/zarr.json", - "array/c" + "array/zarr.json" )); java.util.Set actualKeys = storeHandle.resolve("subgroup").list() @@ -44,10 +40,10 @@ public void testList() throws IOException, ZarrException { .collect(Collectors.toSet()); Assertions.assertEquals(expectedSubgroupKeys, actualKeys); - List allKeys = storeHandle.list() + List allKeys = ((Store.ListableStore) storeHandle.store).list() .map(node -> String.join("/", node)) .collect(Collectors.toList()); - Assertions.assertEquals(21, allKeys.size(), "Total number of keys in store should be 21 but was: " + allKeys); + Assertions.assertEquals(12, allKeys.size(), "Total number of keys in store should be 12 but was: " + allKeys); } @Test From 4f80830479ee9b148d86b4d1bbeb6095790011d0 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 19:20:29 +0100 Subject: [PATCH 56/61] test store::exist, list, listChildren --- .../zarr/zarrjava/store/BufferedZipStore.java | 2 +- .../zarr/zarrjava/store/FilesystemStore.java | 7 ++-- .../dev/zarr/zarrjava/store/MemoryStore.java | 18 ++++---- .../zarr/zarrjava/store/ReadOnlyZipStore.java | 25 +++++------ .../java/dev/zarr/zarrjava/store/S3Store.java | 9 +--- .../java/dev/zarr/zarrjava/store/Store.java | 17 ++++++-- .../dev/zarr/zarrjava/store/StoreHandle.java | 7 ++++ .../zarrjava/store/BufferedZipStoreTest.java | 10 +++++ .../zarrjava/store/FileSystemStoreTest.java | 10 +++++ .../zarr/zarrjava/store/HttpStoreTest.java | 32 ++++++++++++-- .../zarr/zarrjava/store/MemoryStoreTest.java | 15 +++++++ .../zarrjava/store/OnlineS3StoreTest.java | 26 +++++++++++- .../zarrjava/store/ReadOnlyZipStoreTest.java | 40 +++++++++++++++--- .../dev/zarr/zarrjava/store/S3StoreTest.java | 14 ++++++- .../dev/zarr/zarrjava/store/StoreTest.java | 42 +++++++++++++++++-- .../zarrjava/store/WritableStoreTest.java | 33 ++++++++++++++- 16 files changed, 252 insertions(+), 55 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java index 4fffb1a..fb4f1c4 100644 --- a/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java @@ -246,7 +246,7 @@ public Stream list(String[] keys) { } @Override - public Stream listChildren(String[] prefix) { + public Stream listChildren(String[] prefix) { return bufferStore.listChildren(prefix); } diff --git a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java index a255ab8..02c552b 100644 --- a/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java @@ -35,7 +35,7 @@ Path resolveKeys(String[] keys) { @Override public boolean exists(String[] keys) { - return Files.exists(resolveKeys(keys)); + return Files.isRegularFile(resolveKeys(keys)); } @Nullable @@ -148,14 +148,13 @@ public Stream list(String[] prefix) { } @Override - public Stream listChildren(String[] prefix) { + public Stream listChildren(String[] prefix) { Path rootPath = resolveKeys(prefix); if (!Files.exists(rootPath) || !Files.isDirectory(rootPath)) { return Stream.empty(); } try { - return Files.list(rootPath) // note: Files.list is non-recursive - .map(path -> pathToKeyArray(rootPath, path, prefix)); + return Files.list(rootPath).map(path -> path.getFileName().toString()); } catch (IOException e) { throw new RuntimeException("Failed to list store children", e); } diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index be7e6b5..c1664e1 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -4,7 +4,10 @@ import javax.annotation.Nullable; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; @@ -71,21 +74,14 @@ public Stream list(String[] prefix) { } @Override - public Stream listChildren(String[] prefix) { + public Stream listChildren(String[] prefix) { List prefixList = resolveKeys(prefix); int prefixSize = prefixList.size(); return map.keySet().stream() .filter(key -> key.size() > prefixSize && key.subList(0, prefixSize).equals(prefixList)) - // Identify the immediate child segment - // e.g. if prefix is [a], and key is [a, b, c], the child is [a, b] - .map(key -> { - List childPath = new ArrayList<>(prefixList); - childPath.add(key.get(prefixSize)); - return childPath; - }) - .distinct() - .map(list -> list.toArray(new String[0])); + .map(key -> key.get(prefixSize)) + .distinct(); } @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java index 6c6e035..0912df1 100644 --- a/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java @@ -12,6 +12,8 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.stream.Stream; @@ -143,13 +145,14 @@ public Stream list(String[] prefixKeys) { builder.add(resolveEntryKeys(name.substring(prefix.length()))); } } - } catch (IOException ignored) {} + } catch (IOException ignored) { + } return builder.build(); } @Override - public Stream listChildren(String[] prefixKeys) { - java.util.Set children = new java.util.LinkedHashSet<>(); + public Stream listChildren(String[] prefixKeys) { + Set children = new LinkedHashSet<>(); InputStream inputStream = underlyingStore.getInputStream(); if (inputStream == null) return Stream.empty(); @@ -166,19 +169,13 @@ public Stream listChildren(String[] prefixKeys) { if (name.startsWith(prefix) && !name.equals(prefix)) { String relative = name.substring(prefix.length()); String[] parts = relative.split("/"); - // The child is the prefix + the very next segment - String childSegment = parts[0]; - children.add(childSegment); + children.add(parts[0]); } } - } catch (IOException ignored) {} - - return children.stream().map(segment -> { - String[] result = new String[prefixKeys.length + 1]; - System.arraycopy(prefixKeys, 0, result, 0, prefixKeys.length); - result[prefixKeys.length] = segment; - return result; - }); + } catch (IOException ignored) { + } + + return children.stream(); } private String normalizeEntryName(String name) { diff --git a/src/main/java/dev/zarr/zarrjava/store/S3Store.java b/src/main/java/dev/zarr/zarrjava/store/S3Store.java index 8706946..3a4be98 100644 --- a/src/main/java/dev/zarr/zarrjava/store/S3Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/S3Store.java @@ -12,8 +12,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; import java.util.stream.Stream; public class S3Store implements Store, Store.ListableStore { @@ -52,9 +50,6 @@ ByteBuffer get(GetObjectRequest getObjectRequest) { @Override public boolean exists(String[] keys) { - if (keys == null || keys.length == 0) { - return true; - } HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build(); try { return s3client.headObject(req).sdkHttpResponse().statusCode() == 200; @@ -152,7 +147,7 @@ public Stream list(String[] keys) { } @Override - public Stream listChildren(String[] keys) { + public Stream listChildren(String[] keys) { String fullPrefix = resolveKeys(keys); if (!fullPrefix.isEmpty() && !fullPrefix.endsWith("/")) { fullPrefix += "/"; @@ -173,7 +168,7 @@ public Stream listChildren(String[] keys) { .filter(key -> !key.equals(finalFullPrefix)); return Stream.concat(folders, files) - .map(k -> keyToRelativeArray(k, finalFullPrefix)); + .map(k -> keyToRelativeArray(k, finalFullPrefix)[0]); } /** diff --git a/src/main/java/dev/zarr/zarrjava/store/Store.java b/src/main/java/dev/zarr/zarrjava/store/Store.java index 37b7aa3..7d478e5 100644 --- a/src/main/java/dev/zarr/zarrjava/store/Store.java +++ b/src/main/java/dev/zarr/zarrjava/store/Store.java @@ -51,7 +51,7 @@ interface ListableStore extends Store { * Directory-only entries are excluded. * * @param prefix The prefix keys to match. - * @return A stream of full key arrays containing data. + * @return A stream of key arrays containing data. */ Stream list(String[] prefix); @@ -60,12 +60,23 @@ interface ListableStore extends Store { * This is useful for UI navigation or browsing the store hierarchy. * * @param prefix The prefix keys to explore. - * @return A stream of key arrays representing one level deeper than the prefix. + * @return A stream of keys representing one level deeper than the prefix. */ - Stream listChildren(String[] prefix); + Stream listChildren(String[] prefix); + + /** + * Lists the immediate children (files and virtual directories) under the store root. + * + * @return A stream of keys. + */ + default Stream listChildren() { + return listChildren(new String[]{}); + } /** * Lists all data-bearing keys in the entire store. + * + * @return A stream of key arrays containing data. */ default Stream list() { return list(new String[]{}); diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index e7bd8eb..1729adf 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -73,6 +73,13 @@ public Stream list() { return ((Store.ListableStore) store).list(keys); } + public Stream listChildren() { + if (!(store instanceof Store.ListableStore)) { + throw new UnsupportedOperationException("The underlying store does not support listing."); + } + return ((Store.ListableStore) store).listChildren(keys); + } + public long getSize() { return store.getSize(keys); } diff --git a/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java index ba7d404..91a2235 100644 --- a/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java @@ -38,6 +38,16 @@ StoreHandle storeHandleWithData() { return new BufferedZipStore(testGroupDir).resolve("zarr.json"); } + @Override + StoreHandle storeHandleWithoutData() { + return new BufferedZipStore(testGroupDir).resolve("nonexistent", "path", "zarr.json"); + } + + @Override + Store storeWithArrays() { + return new BufferedZipStore(testGroupDir); + } + @Test public void testOpenZipStore() throws ZarrException, IOException { BufferedZipStore zipStore = new BufferedZipStore(testGroupDir); diff --git a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java index e8e58fa..dec204e 100644 --- a/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java @@ -18,6 +18,16 @@ StoreHandle storeHandleWithData() { return new FilesystemStore(TESTDATA).resolve("l4_sample", "zarr.json"); } + @Override + StoreHandle storeHandleWithoutData() { + return new FilesystemStore(TESTDATA).resolve("nonexistent_key"); + } + + @Override + Store storeWithArrays() { + return new FilesystemStore(TESTDATA.resolve("l4_sample")); + } + @Test public void testFileSystemStores() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTDATA); diff --git a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java index 0d1e087..0b33331 100644 --- a/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java @@ -3,17 +3,28 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Array; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.io.IOException; -public class HttpStoreTest extends StoreTest { +class HttpStoreTest extends StoreTest { @Override StoreHandle storeHandleWithData() { return br00109990StoreHandle().resolve("c", "0", "0", "0"); } + @Override + StoreHandle storeHandleWithoutData() { + return br00109990StoreHandle().resolve("nonexistent", "path", "to", "data"); + } + + @Override + Store storeWithArrays() { + return br00109990StoreHandle().store; + } + StoreHandle br00109990StoreHandle() { HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.5/idr0033A"); return httpStore.resolve("BR00109990_C2.zarr", "0", "0"); @@ -26,7 +37,22 @@ public void testOpen() throws IOException, ZarrException { } @Override - void testList() throws ZarrException, IOException { - // listing is not supported in HttpStore + @Test + @Disabled("List is not supported in HttpStore") + public void testList() { } + + @Override + @Test + @Disabled("List is not supported in HttpStore") + public void testListedItemsExist() { + } + + @Override + @Test + @Disabled("List is not supported in HttpStore") + public void testListChildren() { + } + + } diff --git a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java index 5ca7d5f..e8a9185 100644 --- a/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java @@ -1,5 +1,8 @@ package dev.zarr.zarrjava.store; +import dev.zarr.zarrjava.ZarrException; + +import java.io.IOException; import java.nio.ByteBuffer; public class MemoryStoreTest extends WritableStoreTest { @@ -16,4 +19,16 @@ StoreHandle storeHandleWithData() { memoryStoreHandle.set(ByteBuffer.wrap(testData())); return memoryStoreHandle; } + + @Override + StoreHandle storeHandleWithoutData() { + return new MemoryStore().resolve(); + } + + @Override + Store storeWithArrays() throws ZarrException, IOException { + MemoryStore memoryStore = new MemoryStore(); + writeTestGroupV3(memoryStore.resolve("array"), false); + return memoryStore; + } } diff --git a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java index 885cbf6..c1600f2 100644 --- a/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/OnlineS3StoreTest.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; +import java.util.*; +import java.util.stream.Collectors; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class OnlineS3StoreTest extends StoreTest { @@ -64,8 +66,30 @@ StoreHandle storeHandleWithData() { } @Override + StoreHandle storeHandleWithoutData() { + return storeHandle.resolve("nonexistent_key"); + } + + @Override + Store storeWithArrays() { + return storeHandle.store; + } + @Test - void testList() { + @Override + public void testListChildren() { + List children = ((Store.ListableStore) storeHandle.store).listChildren().collect(Collectors.toList()); + List expectedChildren = Collections.singletonList("BR00109990_C2.zarr"); + Assertions.assertEquals(expectedChildren, children); + + Set storeHandleChildren = storeHandle.listChildren().collect(Collectors.toSet()); + Set expectedStoreHandleChildren = new HashSet<>(Arrays.asList("c", "zarr.json")); + Assertions.assertEquals(expectedStoreHandleChildren, storeHandleChildren); + } + + @Test + @Override + public void testList() { Assertions.assertTrue(storeHandle.list().count() > 1); } } diff --git a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java index 7b73af4..df00b5c 100644 --- a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java @@ -10,6 +10,8 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Collectors; public class ReadOnlyZipStoreTest extends StoreTest { @@ -29,16 +31,44 @@ StoreHandle storeHandleWithData() { return storeHandleWithData; } + @Override + StoreHandle storeHandleWithoutData() { + return new ReadOnlyZipStore(storePath).resolve("nonexistent_key"); + } + + @Override + Store storeWithArrays() { + return new ReadOnlyZipStore(storePath); + } + + + @Override + @Test + public void testListChildren() { + ReadOnlyZipStore zipStore = new ReadOnlyZipStore(storePath); + BufferedZipStore bufferedZipStore = new BufferedZipStore(storePath); + + Set expectedKeys = bufferedZipStore.resolve().listChildren() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + Set actualKeys = zipStore.resolve().listChildren() + .map(node -> String.join("/", node)) + .collect(Collectors.toSet()); + + Assertions.assertFalse(actualKeys.isEmpty()); + Assertions.assertEquals(expectedKeys, actualKeys); + } + @Override @Test - void testList() { + public void testList() { ReadOnlyZipStore zipStore = new ReadOnlyZipStore(storePath); BufferedZipStore bufferedZipStore = new BufferedZipStore(storePath); - java.util.Set expectedKeys = bufferedZipStore.resolve().list() + Set expectedKeys = bufferedZipStore.resolve().list() .map(node -> String.join("/", node)) .collect(Collectors.toSet()); - java.util.Set actualKeys = zipStore.resolve().list() + Set actualKeys = zipStore.resolve().list() .map(node -> String.join("/", node)) .collect(Collectors.toSet()); Assertions.assertEquals(expectedKeys, actualKeys); @@ -69,7 +99,7 @@ public void testReadFromBufferedZipStore() throws ZarrException, IOException { ReadOnlyZipStore readOnlyZipStore = new ReadOnlyZipStore(path); Assertions.assertEquals(archiveComment, readOnlyZipStore.getArchiveComment(), "ZIP archive comment from ReadOnlyZipStore does not match expected value."); - java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( + Set expectedSubgroupKeys = new HashSet<>(Arrays.asList( "array/c/1/1", "array/c/0/0", "array/c/0/1", @@ -82,7 +112,7 @@ public void testReadFromBufferedZipStore() throws ZarrException, IOException { "array/c" )); - java.util.Set actualKeys = readOnlyZipStore.resolve("subgroup").list() + Set actualKeys = readOnlyZipStore.resolve("subgroup").list() .map(node -> String.join("/", node)) .collect(Collectors.toSet()); diff --git a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java index 520d3a0..3e63480 100644 --- a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java @@ -1,5 +1,6 @@ package dev.zarr.zarrjava.store; +import dev.zarr.zarrjava.ZarrException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -79,7 +80,6 @@ void testReadWriteS3Store() { Assertions.assertArrayEquals(testData, retrievedBytes); } - @Override Store writableStore() { return new S3Store(s3Client, bucketName, ""); @@ -94,4 +94,16 @@ StoreHandle storeHandleWithData() { } return new S3Store(s3Client, bucketName, "").resolve(testDataKey); } + + @Override + StoreHandle storeHandleWithoutData() { + return new S3Store(s3Client, bucketName, "").resolve("nonexistent_key"); + } + + @Override + Store storeWithArrays() throws ZarrException, IOException { + S3Store s3Store = new S3Store(s3Client, bucketName, ""); + writeTestGroupV3(s3Store.resolve("array"), false); + return s3Store; + } } diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index 51a0e26..2f79611 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -14,14 +14,25 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public abstract class StoreTest extends ZarrTest { + /** + * Returns a StoreHandle with some test data written to it (optimally not written by the same Store implementation). + */ abstract StoreHandle storeHandleWithData(); + abstract StoreHandle storeHandleWithoutData(); + + /** + * Returns a Store with some test arrays written to it used to test list() and exist() (can be written by the same Store implementation). + */ + abstract Store storeWithArrays() throws ZarrException, IOException; + @Test public void testInputStream() throws IOException { StoreHandle storeHandle = storeHandleWithData(); @@ -34,15 +45,42 @@ public void testInputStream() throws IOException { Assertions.assertArrayEquals(expectedBuffer, buffer); } + @Test + public void testExists() throws ZarrException, IOException { + Assertions.assertTrue(storeHandleWithData().exists()); + Assertions.assertFalse(storeHandleWithoutData().exists()); + Assertions.assertFalse(storeWithArrays().resolve("").exists()); + } + + @Test + public void testListedItemsExist() throws IOException, ZarrException { + Store store = storeWithArrays(); + if (!(store instanceof Store.ListableStore)) { + Assertions.fail("Store is not listable"); + } + Set nodes = ((Store.ListableStore) store).list().limit(10).collect(Collectors.toSet()); + Assertions.assertFalse(nodes.isEmpty()); // to ensure the sensitivity of this test + + nodes.forEach(keys -> { + System.out.println("Checking existence of key: " + String.join("/", keys)); + StoreHandle handle = store.resolve(keys); + Assertions.assertTrue(handle.exists(), "Listed key does not exist: " + String.join("/", keys)); + }); + } + + @Test + public abstract void testListChildren() throws ZarrException, IOException; @Test - public void testStoreGetSize() { + public void testGetSize() { StoreHandle storeHandle = storeHandleWithData(); long size = storeHandle.getSize(); long actual_size = storeHandle.read().remaining(); Assertions.assertEquals(actual_size, size); } + @Test + public abstract void testList() throws ZarrException, IOException; byte[] testData() { byte[] testData = new byte[1024 * 1024]; @@ -125,6 +163,4 @@ void assertIsTestGroupV2(Group group, boolean useParallel) throws ZarrException, Assertions.assertEquals("value", attrs.getString("some")); } - @Test - abstract void testList() throws ZarrException, IOException; } diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java index a13dd4e..61bcdbd 100644 --- a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -13,7 +13,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -26,7 +28,7 @@ public void testList() throws IOException, ZarrException { StoreHandle storeHandle = writableStore().resolve("testList"); boolean useParallel = true; writeTestGroupV3(storeHandle, useParallel); - java.util.Set expectedSubgroupKeys = new java.util.HashSet<>(Arrays.asList( + Set expectedSubgroupKeys = new HashSet<>(Arrays.asList( "array/c/1/1", "array/c/0/0", "array/c/0/1", @@ -35,7 +37,7 @@ public void testList() throws IOException, ZarrException { "array/zarr.json" )); - java.util.Set actualKeys = storeHandle.resolve("subgroup").list() + Set actualKeys = storeHandle.resolve("subgroup").list() .map(node -> String.join("/", node)) .collect(Collectors.toSet()); Assertions.assertEquals(expectedSubgroupKeys, actualKeys); @@ -46,6 +48,33 @@ public void testList() throws IOException, ZarrException { Assertions.assertEquals(12, allKeys.size(), "Total number of keys in store should be 12 but was: " + allKeys); } + @Test + @Override + public void testListChildren() throws ZarrException, IOException { + StoreHandle storeHandle = writableStore().resolve("testListChildren"); + boolean useParallel = true; + writeTestGroupV3(storeHandle, useParallel); + + Set expectedChildren = new HashSet<>(Arrays.asList( + "array", + "zarr.json", + "subgroup" + )); + + Set actualChildren = storeHandle.resolve().listChildren() + .collect(Collectors.toSet()); + Assertions.assertEquals(expectedChildren, actualChildren); + + Set expectedSubgroupKeys = new HashSet<>(Arrays.asList( + "array", + "zarr.json" + )); + Set subgroupChildren = storeHandle.resolve("subgroup").listChildren() + .collect(Collectors.toSet()); + + Assertions.assertEquals(expectedSubgroupKeys, subgroupChildren); + } + @Test public void testWriteRead() throws IOException, ZarrException { StoreHandle storeHandle = writableStore().resolve("testWriteRead"); From 82e0ef3ea88978aba5cfb61b4ca6a4a208b9271e Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 19:30:01 +0100 Subject: [PATCH 57/61] test store::get with start and end --- .../dev/zarr/zarrjava/store/StoreTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index 2f79611..44f7d57 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -79,6 +80,26 @@ public void testGetSize() { Assertions.assertEquals(actual_size, size); } + @Test + public void testGetWithStartEnd() { + StoreHandle storeHandle = storeHandleWithData(); + long size = storeHandle.getSize(); + System.out.println("Store size: " + size); + if (size < 20) { + Assertions.fail("Store size is too small to test get with start and end"); + } + ByteBuffer buffer = storeHandle.read(5, 15); + Assertions.assertEquals(10, buffer.remaining()); + + ByteBuffer fullBuffer = storeHandle.read(); + byte[] expectedBytes = new byte[10]; + fullBuffer.position(5); + fullBuffer.get(expectedBytes, 0, 10); + byte[] actualBytes = new byte[10]; + buffer.get(actualBytes, 0, 10); + Assertions.assertArrayEquals(expectedBytes, actualBytes); + } + @Test public abstract void testList() throws ZarrException, IOException; From 2897f0a4e9a1d4032d8d85d18a9d0667c5f1d766 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 19 Jan 2026 19:30:19 +0100 Subject: [PATCH 58/61] test store::delete --- .../dev/zarr/zarrjava/store/WritableStoreTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java index 61bcdbd..29f181d 100644 --- a/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/WritableStoreTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -83,6 +84,18 @@ public void testWriteRead() throws IOException, ZarrException { assertIsTestGroupV3(group, useParallel); } + @Test + public void testDelete() { + Store store = writableStore(); + StoreHandle storeHandle = store.resolve("testDelete"); + + storeHandle.resolve("toBeDeleted").set(ByteBuffer.allocate(0)); + Assertions.assertTrue(storeHandle.resolve("toBeDeleted").exists(), "Key toBeDeleted should exist before deletion."); + storeHandle.resolve("toBeDeleted").delete(); + Assertions.assertFalse(storeHandle.resolve("toBeDeleted").exists(), "Key toBeDeleted should not exist after deletion."); + + } + @ParameterizedTest @CsvSource({"false", "true",}) public void testWriteReadV3(boolean useParallel) throws ZarrException, IOException { From 91e5e3db9c1c5ec711225ba6556a1a8694ea0ccf Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 23 Jan 2026 09:59:14 +0100 Subject: [PATCH 59/61] fix ReadOnlyZipStoreTest.testReadFromBufferedZipStore and S3StoreTest>StoreTest.testExists --- .../zarr/zarrjava/store/ReadOnlyZipStoreTest.java | 6 +----- .../java/dev/zarr/zarrjava/store/S3StoreTest.java | 14 ++++++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java index df00b5c..54b5373 100644 --- a/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/ReadOnlyZipStoreTest.java @@ -104,12 +104,8 @@ public void testReadFromBufferedZipStore() throws ZarrException, IOException { "array/c/0/0", "array/c/0/1", "zarr.json", - "array", "array/c/1/0", - "array/c/1", - "array/c/0", - "array/zarr.json", - "array/c" + "array/zarr.json" )); Set actualKeys = readOnlyZipStore.resolve("subgroup").list() diff --git a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java index 3e63480..1ff0bc8 100644 --- a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java @@ -37,9 +37,9 @@ public class S3StoreTest extends WritableStoreTest { String bucketName = "zarr-test-bucket"; S3Client s3Client; String testDataKey = "testData"; - + S3Store s3Store; @BeforeAll - void setUpS3Client() { + void setUpS3Client() throws ZarrException, IOException { s3Client = S3Client.builder() .endpointOverride(URI.create(s3Endpoint)) .region(Region.US_EAST_1) // required, but ignored @@ -62,6 +62,9 @@ void setUpS3Client() { } catch (Exception e) { throw new RuntimeException(e); } + // Write a test group to the S3 store + s3Store = new S3Store(s3Client, bucketName, "storeWithArrays"); + writeTestGroupV3(s3Store.resolve(), false); } @Test @@ -100,10 +103,9 @@ StoreHandle storeHandleWithoutData() { return new S3Store(s3Client, bucketName, "").resolve("nonexistent_key"); } + @Override - Store storeWithArrays() throws ZarrException, IOException { - S3Store s3Store = new S3Store(s3Client, bucketName, ""); - writeTestGroupV3(s3Store.resolve("array"), false); - return s3Store; + Store storeWithArrays() { + return new S3Store(s3Client, bucketName, "storeWithArrays"); } } From 933f58bc3f84a0f2aa5e277a5b3e585620ff8167 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 23 Jan 2026 10:03:43 +0100 Subject: [PATCH 60/61] fix S3StoreTest.testList --- src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java | 2 +- src/test/java/dev/zarr/zarrjava/store/StoreTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java index 1ff0bc8..50d1973 100644 --- a/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/S3StoreTest.java @@ -85,7 +85,7 @@ void testReadWriteS3Store() { @Override Store writableStore() { - return new S3Store(s3Client, bucketName, ""); + return new S3Store(s3Client, bucketName, "writableStore"); } @Override diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index 44f7d57..6da5e12 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -63,7 +63,6 @@ public void testListedItemsExist() throws IOException, ZarrException { Assertions.assertFalse(nodes.isEmpty()); // to ensure the sensitivity of this test nodes.forEach(keys -> { - System.out.println("Checking existence of key: " + String.join("/", keys)); StoreHandle handle = store.resolve(keys); Assertions.assertTrue(handle.exists(), "Listed key does not exist: " + String.join("/", keys)); }); From 52de5f43cd7406abddfc85b3b895520150286239 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 23 Jan 2026 10:12:20 +0100 Subject: [PATCH 61/61] test store.get with start argument --- src/test/java/dev/zarr/zarrjava/store/StoreTest.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java index 6da5e12..6e21bfc 100644 --- a/src/test/java/dev/zarr/zarrjava/store/StoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/store/StoreTest.java @@ -83,7 +83,6 @@ public void testGetSize() { public void testGetWithStartEnd() { StoreHandle storeHandle = storeHandleWithData(); long size = storeHandle.getSize(); - System.out.println("Store size: " + size); if (size < 20) { Assertions.fail("Store size is too small to test get with start and end"); } @@ -97,6 +96,13 @@ public void testGetWithStartEnd() { byte[] actualBytes = new byte[10]; buffer.get(actualBytes, 0, 10); Assertions.assertArrayEquals(expectedBytes, actualBytes); + + buffer = storeHandle.read(size-10); + Assertions.assertEquals(10, buffer.remaining()); + fullBuffer.position((int)(size-10)); + fullBuffer.get(expectedBytes, 0, 10); + buffer.get(actualBytes, 0, 10); + Assertions.assertArrayEquals(expectedBytes, actualBytes); } @Test