From d1f6075135ab63a809c42c4836c5ff700a42c6c1 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 18 Mar 2025 10:19:11 -0400 Subject: [PATCH 1/3] Use N5Factory in N5 client to support cloud storage Co-authored-by: Eric Trautman --- .../janelia/render/client/spark/n5/N5Client.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 491505447..4b1dcb3f0 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -38,10 +38,11 @@ import org.janelia.render.client.zspacing.ThicknessCorrectionData; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -475,7 +476,7 @@ public static void setupFullScaleExportN5(final Parameters parameters, final int[] blockSize, final DataType dataType) { - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter(StorageFormat.N5, parameters.n5Path)) { n5.createDataset(fullScaleDatasetName, dimensions, blockSize, @@ -494,7 +495,7 @@ public static void updateFullScaleExportAttributes(final Parameters parameters, String exportAttributesDatasetName = fullScaleDatasetName; - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter(StorageFormat.N5, parameters.n5Path)) { final Map export_attributes = new HashMap<>(); export_attributes.put("runTimestamp", new Date()); export_attributes.put("runParameters", parameters); @@ -661,8 +662,9 @@ private static void saveRenderStack(final JavaSparkContext sc, } } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter(StorageFormat.N5, n5Path); // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); } @@ -722,8 +724,9 @@ private static void save2DRenderStack(final JavaSparkContext sc, out.next().set(in.next()); } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter(StorageFormat.N5, n5Path); // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); LOG.info("save2DRenderStack: exit"); From dd6333ecdf41ffb7f8668ccc9d92acfaf466faf0 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 25 Mar 2025 19:21:33 -0400 Subject: [PATCH 2/3] Use N5Factory in Util.java to support sources other than files --- .../main/java/org/janelia/render/client/spark/n5/Util.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java index 437526422..e85361a52 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java @@ -2,9 +2,9 @@ import java.io.IOException; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; /** * Utilities for N5 operations. @@ -22,7 +22,7 @@ public N5PathSupplier(final String path) { @Override public N5Writer get() throws IOException { - return new N5FSWriter(path); + return new N5Factory().openWriter(N5Factory.StorageFormat.N5, path); } } From 55de48c0219463b5d37f21d15252d065c1ad25be Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Sep 2025 11:03:55 -0400 Subject: [PATCH 3/3] Make NeurglancerAttributes::write accept N5Writer instead of path --- .../util/NeuroglancerAttributes.java | 29 +++++++++---------- ...rrelationWithNextRegionalDataN5Writer.java | 6 ++-- .../ExportMichalSegmentationsClient.java | 4 ++- .../spark/n5/H5TileToN5PreviewClient.java | 3 +- .../render/client/spark/n5/N5Client.java | 9 ++---- .../render/client/spark/n5/N5ClientTest.java | 8 ++--- 6 files changed, 28 insertions(+), 31 deletions(-) diff --git a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java index c389eb275..35aa0e537 100644 --- a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java +++ b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java @@ -9,7 +9,6 @@ import java.util.List; import java.util.Map; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,13 +134,13 @@ public NeuroglancerAttributes(final List stackResolutionValues, /** * Writes all attribute.json files required by neuroglancer to display the specified dataset. * - * @param n5BasePath base path for n5. + * @param n5Writer N5 writer to use for writing the attributes. * @param fullScaleDatasetPath path of the full scale data set. * * @throws IOException * if the writes fail for any reason. */ - public void write(final Path n5BasePath, + public void write(final N5Writer n5Writer, final Path fullScaleDatasetPath) throws IOException { @@ -150,10 +149,8 @@ public void write(final Path n5BasePath, final Path ngAttributesPath = isMultiScaleDataset ? fullScaleDatasetPath.getParent() : fullScaleDatasetPath; - LOG.info("write: entry, n5BasePath={}, fullScaleDatasetPath={}, ngAttributesPath={}", - n5BasePath, fullScaleDatasetPath, ngAttributesPath); - - final N5Writer n5Writer = new N5FSWriter(n5BasePath.toAbsolutePath().toString()); + LOG.info("write: entry, n5Base={}, fullScaleDatasetPath={}, ngAttributesPath={}", + n5Writer.getURI(), fullScaleDatasetPath, ngAttributesPath); // Neuroglancer recursively looks for attribute.json files from root path and stops at // the first subdirectory without an attributes.json file. @@ -164,7 +161,7 @@ public void write(final Path n5BasePath, for (Path path = ngAttributesPath.getParent(); (path != null) && (! path.endsWith("/")); path = path.getParent()) { - LOG.info("write: saving supported attribute to {}{}/attributes.json", n5BasePath, path); + LOG.info("write: saving supported attribute to {}{}/attributes.json", n5Writer.getURI(), path); n5Writer.setAttribute(path.toString(), SUPPORTED_KEY, true); } @@ -180,7 +177,7 @@ public void write(final Path n5BasePath, attributes.put("pixelResolution", pixelResolution); attributes.put("translate", translate); - LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5BasePath, ngAttributesPath); + LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5Writer.getURI(), ngAttributesPath); n5Writer.setAttributes(ngAttributesPath.toString(), attributes); if (isMultiScaleDataset) { @@ -188,7 +185,7 @@ public void write(final Path n5BasePath, writeScaleLevelTransformAttributes(scaleLevel, scales.get(scaleLevel), n5Writer, - n5BasePath, + n5Writer.getURI().toString(), ngAttributesPath); } } @@ -197,16 +194,18 @@ public void write(final Path n5BasePath, private void writeScaleLevelTransformAttributes(final int scaleLevel, final List scaleLevelFactors, final N5Writer n5Writer, - final Path n5BasePath, + final String n5Base, final Path ngAttributesPath) throws IOException { final String scaleName = "s" + scaleLevel; final Path scaleAttributesPath = Paths.get(ngAttributesPath.toString(), scaleName); - final Path scaleLevelDirectoryPath = Paths.get(n5BasePath.toString(), ngAttributesPath.toString(), scaleName); - if (! scaleLevelDirectoryPath.toFile().exists()) { - throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + if (n5Base.startsWith("/") || n5Base.startsWith("\\")) { + final Path scaleLevelDirectoryPath = Paths.get(n5Base, ngAttributesPath.toString(), scaleName); + if (! scaleLevelDirectoryPath.toFile().exists()) { + throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + } } final Map transformAttributes = new HashMap<>(); @@ -232,7 +231,7 @@ private void writeScaleLevelTransformAttributes(final int scaleLevel, final Map attributes = new HashMap<>(); attributes.put("transform", transformAttributes); - LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5BasePath, scaleAttributesPath); + LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5Base, scaleAttributesPath); n5Writer.setAttributes(scaleAttributesPath.toString(), attributes); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java index 998464f23..1c838e4fe 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java @@ -132,7 +132,7 @@ private static void createDataSet(final List downsampledDatasetPaths = downsampleScalePyramid(sparkContext, n5Supplier, @@ -362,8 +361,7 @@ public void run() Arrays.asList(min[0], min[1], min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - ngAttributes.write(Paths.get(parameters.n5Path), - Paths.get(fullScaleDatasetName)); + ngAttributes.write(n5Supplier.get(), Paths.get(fullScaleDatasetName)); if (downsampleStackForReview) { @@ -407,8 +405,7 @@ public void run() Arrays.asList(min[0], min[1], min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - reviewNgAttributes.write(Paths.get(parameters.n5Path), - Paths.get(fullScaleReviewDatasetName)); + reviewNgAttributes.write(n5ReviewSupplier.get(), Paths.get(fullScaleReviewDatasetName)); } sparkContext.close(); diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java index eabac8d66..c10d6e87e 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java @@ -150,13 +150,13 @@ public void testSetupFullScaleExportN5() throws Exception { @Test public void testNeuroglancerAttributes() throws Exception { - final Path n5Path = n5PathDirectory.toPath().toAbsolutePath(); + final String n5Path = n5PathDirectory.getAbsolutePath(); final Path fullScaleDatasetPath = Paths.get("/render/test_stack/one_more_nested_dir/s0"); final String datasetName = fullScaleDatasetPath.toString(); final long[] dimensions = { 100L, 200L, 300L }; final int[] blockSize = { 10, 20, 30 }; - try (final N5Writer n5Writer = new N5FSWriter(n5Path.toString())) { + try (final N5Writer n5Writer = new N5FSWriter(n5Path)) { final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, @@ -164,7 +164,7 @@ public void testNeuroglancerAttributes() throws Exception { new GzipCompression()); n5Writer.createDataset(datasetName, datasetAttributes); - final N5Reader n5Reader = new N5FSReader(n5Path.toString()); + final N5Reader n5Reader = new N5FSReader(n5Path); Assert.assertTrue("dataset " + datasetName + " is missing", n5Reader.datasetExists(datasetName)); final Map originalDatasetAttributes = datasetAttributes.asMap(); @@ -198,7 +198,7 @@ public void testNeuroglancerAttributes() throws Exception { Arrays.asList(5L, 25L, 125L), NeuroglancerAttributes.NumpyContiguousOrdering.C); - ngAttributes.write(n5Path, fullScaleDatasetPath); + ngAttributes.write(n5Writer, fullScaleDatasetPath); final String testStackDatasetName = fullScaleDatasetPath.getParent().toString();