From 55192d47e4b8e8b957fc8e879669f16087995f75 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 18 Mar 2025 10:14:53 -0400 Subject: [PATCH 1/7] fix non-compiling generics statement (I agree with the compiler :) ) --- .../client/newsolver/solvers/intensity/IntensityTile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java index 9dbab3782..64b282bac 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java @@ -52,7 +52,7 @@ public IntensityTile( for (int i = 0; i < N; i++) { final Affine1D model = modelSupplier.get(); - this.subTiles.add(new Tile<>((Model) model)); + this.subTiles.add(new Tile((Model) model)); } } From 954c0eb1e781876cdf006ec47c1b4b8af552cc9d Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Tue, 18 Mar 2025 10:19:11 -0400 Subject: [PATCH 2/7] change N5 client to support cloud storage, add n5.close() statements --- .../janelia/render/client/spark/n5/N5Client.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 491505447..54af9722a 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -38,10 +38,11 @@ import org.janelia.render.client.zspacing.ThicknessCorrectionData; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -475,12 +476,13 @@ public static void setupFullScaleExportN5(final Parameters parameters, final int[] blockSize, final DataType dataType) { - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter( StorageFormat.N5, parameters.n5Path )) /*new N5FSWriter(parameters.n5Path))*/ { n5.createDataset(fullScaleDatasetName, dimensions, blockSize, dataType, new GzipCompression()); + n5.close(); } updateFullScaleExportAttributes(parameters, @@ -494,7 +496,7 @@ public static void updateFullScaleExportAttributes(final Parameters parameters, String exportAttributesDatasetName = fullScaleDatasetName; - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter( StorageFormat.N5, parameters.n5Path )/*new N5FSWriter(parameters.n5Path)*/) { final Map export_attributes = new HashMap<>(); export_attributes.put("runTimestamp", new Date()); export_attributes.put("runParameters", parameters); @@ -508,6 +510,7 @@ public static void updateFullScaleExportAttributes(final Parameters parameters, exportAttributesDatasetName = fullScaleDatasetPath.getParent().toString(); } n5.setAttributes(exportAttributesDatasetName, attributes); + n5.close(); } LOG.info("updateFullScaleExportAttributes: saved {}", @@ -661,8 +664,9 @@ private static void saveRenderStack(final JavaSparkContext sc, } } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter( StorageFormat.N5, n5Path )/*new N5FSWriter(n5Path)*/; // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); } @@ -722,8 +726,9 @@ private static void save2DRenderStack(final JavaSparkContext sc, out.next().set(in.next()); } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter( StorageFormat.N5, n5Path ) /*new N5FSWriter(n5Path)*/; // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); LOG.info("save2DRenderStack: exit"); From 119d4d0219824bd63cd72a9adc79a5e2bd4514f6 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 25 Mar 2025 14:47:08 -0400 Subject: [PATCH 3/7] remove unnecessary close calls that are automatically handled by the try block --- .../main/java/org/janelia/render/client/spark/n5/N5Client.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 54af9722a..438aa0880 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -482,7 +482,6 @@ public static void setupFullScaleExportN5(final Parameters parameters, blockSize, dataType, new GzipCompression()); - n5.close(); } updateFullScaleExportAttributes(parameters, @@ -510,7 +509,6 @@ public static void updateFullScaleExportAttributes(final Parameters parameters, exportAttributesDatasetName = fullScaleDatasetPath.getParent().toString(); } n5.setAttributes(exportAttributesDatasetName, attributes); - n5.close(); } LOG.info("updateFullScaleExportAttributes: saved {}", From c2514774743f96b49f6ba426e7329d67adfb9687 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 25 Mar 2025 16:33:20 -0400 Subject: [PATCH 4/7] revert n5-google-cloud.version to 4.1.0 to get past N5Exception$N5NoSuchKeyException class not found error --- pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pom.xml b/pom.xml index 5f9d5a485..5dc76fab9 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,15 @@ --> 3.2.0 + + 4.1.0 + 7.1.4 0.17.2 From cf4ba4d740635053226baf7128f98272dbdf8646 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 25 Mar 2025 19:21:33 -0400 Subject: [PATCH 5/7] fix n5 writer in Util.java to support sources other than files --- .../main/java/org/janelia/render/client/spark/n5/Util.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java index 437526422..e85361a52 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java @@ -2,9 +2,9 @@ import java.io.IOException; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; /** * Utilities for N5 operations. @@ -22,7 +22,7 @@ public N5PathSupplier(final String path) { @Override public N5Writer get() throws IOException { - return new N5FSWriter(path); + return new N5Factory().openWriter(N5Factory.StorageFormat.N5, path); } } From 2f90f7446b243fb93914ba8443d58b036547d4fc Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Wed, 26 Mar 2025 10:37:24 -0400 Subject: [PATCH 6/7] Change NeuroglancerAttributes utility to work with an N5Writer that can write to both typical file storage and to Google storage buckets. This forced me to include the n5-universe library in the root render-app pom.xml which then caused a bunch of duplicate resource issues which needed to be corrected with exclusions. --- render-app/pom.xml | 4 +-- .../util/NeuroglancerAttributes.java | 30 ++++++++++--------- render-ws-java-client/pom.xml | 7 +++++ ...rrelationWithNextRegionalDataN5Writer.java | 2 +- .../ExportMichalSegmentationsClient.java | 2 +- .../spark/n5/H5TileToN5PreviewClient.java | 2 +- .../render/client/spark/n5/N5Client.java | 4 +-- .../render/client/spark/n5/N5ClientTest.java | 6 ++-- render-ws/pom.xml | 8 +++++ 9 files changed, 41 insertions(+), 24 deletions(-) diff --git a/render-app/pom.xml b/render-app/pom.xml index c82f740a8..f132fc84c 100644 --- a/render-app/pom.xml +++ b/render-app/pom.xml @@ -156,12 +156,12 @@ org.janelia.saalfeldlab - n5-imglib2 + n5-hdf5 org.janelia.saalfeldlab - n5-hdf5 + n5-universe diff --git a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java index c389eb275..739a3e1e8 100644 --- a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java +++ b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java @@ -9,8 +9,8 @@ import java.util.List; import java.util.Map; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,13 +135,13 @@ public NeuroglancerAttributes(final List stackResolutionValues, /** * Writes all attribute.json files required by neuroglancer to display the specified dataset. * - * @param n5BasePath base path for n5. + * @param n5Base base path or URL string for n5. * @param fullScaleDatasetPath path of the full scale data set. * * @throws IOException * if the writes fail for any reason. */ - public void write(final Path n5BasePath, + public void write(final String n5Base, final Path fullScaleDatasetPath) throws IOException { @@ -150,10 +150,10 @@ public void write(final Path n5BasePath, final Path ngAttributesPath = isMultiScaleDataset ? fullScaleDatasetPath.getParent() : fullScaleDatasetPath; - LOG.info("write: entry, n5BasePath={}, fullScaleDatasetPath={}, ngAttributesPath={}", - n5BasePath, fullScaleDatasetPath, ngAttributesPath); + LOG.info("write: entry, n5Base={}, fullScaleDatasetPath={}, ngAttributesPath={}", + n5Base, fullScaleDatasetPath, ngAttributesPath); - final N5Writer n5Writer = new N5FSWriter(n5BasePath.toAbsolutePath().toString()); + final N5Writer n5Writer = new N5Factory().openWriter(N5Factory.StorageFormat.N5, n5Base); //new N5FSWriter(n5BasePath.toAbsolutePath().toString()); // Neuroglancer recursively looks for attribute.json files from root path and stops at // the first subdirectory without an attributes.json file. @@ -164,7 +164,7 @@ public void write(final Path n5BasePath, for (Path path = ngAttributesPath.getParent(); (path != null) && (! path.endsWith("/")); path = path.getParent()) { - LOG.info("write: saving supported attribute to {}{}/attributes.json", n5BasePath, path); + LOG.info("write: saving supported attribute to {}{}/attributes.json", n5Base, path); n5Writer.setAttribute(path.toString(), SUPPORTED_KEY, true); } @@ -180,7 +180,7 @@ public void write(final Path n5BasePath, attributes.put("pixelResolution", pixelResolution); attributes.put("translate", translate); - LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5BasePath, ngAttributesPath); + LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5Base, ngAttributesPath); n5Writer.setAttributes(ngAttributesPath.toString(), attributes); if (isMultiScaleDataset) { @@ -188,7 +188,7 @@ public void write(final Path n5BasePath, writeScaleLevelTransformAttributes(scaleLevel, scales.get(scaleLevel), n5Writer, - n5BasePath, + n5Base, ngAttributesPath); } } @@ -197,16 +197,18 @@ public void write(final Path n5BasePath, private void writeScaleLevelTransformAttributes(final int scaleLevel, final List scaleLevelFactors, final N5Writer n5Writer, - final Path n5BasePath, + final String n5Base, final Path ngAttributesPath) throws IOException { final String scaleName = "s" + scaleLevel; final Path scaleAttributesPath = Paths.get(ngAttributesPath.toString(), scaleName); - final Path scaleLevelDirectoryPath = Paths.get(n5BasePath.toString(), ngAttributesPath.toString(), scaleName); - if (! scaleLevelDirectoryPath.toFile().exists()) { - throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + if (n5Base.startsWith("/") || n5Base.startsWith("\\")) { + final Path scaleLevelDirectoryPath = Paths.get(n5Base, ngAttributesPath.toString(), scaleName); + if (! scaleLevelDirectoryPath.toFile().exists()) { + throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + } } final Map transformAttributes = new HashMap<>(); @@ -232,7 +234,7 @@ private void writeScaleLevelTransformAttributes(final int scaleLevel, final Map attributes = new HashMap<>(); attributes.put("transform", transformAttributes); - LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5BasePath, scaleAttributesPath); + LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5Base, scaleAttributesPath); n5Writer.setAttributes(scaleAttributesPath.toString(), attributes); } diff --git a/render-ws-java-client/pom.xml b/render-ws-java-client/pom.xml index 7aee937b0..d20fff8ad 100644 --- a/render-ws-java-client/pom.xml +++ b/render-ws-java-client/pom.xml @@ -74,6 +74,13 @@ LICENSE META-INF/* META-INF/versions/** + canonical.json + cosem.json + n5-compression.jq + n5.jq + zyx.zattrs.json + META-INF/services/io.grpc.LoadBalancerProvider + META-INF/services/io.grpc.NameResolverProvider diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java index 998464f23..c46b18b65 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java @@ -132,7 +132,7 @@ private static void createDataSet(final List originalDatasetAttributes = datasetAttributes.asMap(); diff --git a/render-ws/pom.xml b/render-ws/pom.xml index 7cb96b0de..7a4dd9ab6 100644 --- a/render-ws/pom.xml +++ b/render-ws/pom.xml @@ -67,11 +67,19 @@ render-app ${project.version} + org.slf4j slf4j-api + + + + javax.annotation + javax.annotation-api + + From 518ab576805dfb96dc92734fc9fc130c0c3c9a56 Mon Sep 17 00:00:00 2001 From: Stephan Preibisch Date: Sat, 3 May 2025 09:28:37 -0400 Subject: [PATCH 7/7] Added TODO for solve with per-tile dynamic regularization fix typo --- .../newsolver/solvers/affine/AffineAlignBlockWorker.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/affine/AffineAlignBlockWorker.java b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/affine/AffineAlignBlockWorker.java index 35ea0a49f..6ad18ca91 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/affine/AffineAlignBlockWorker.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/affine/AffineAlignBlockWorker.java @@ -132,7 +132,7 @@ public AffineAlignBlockWorker( // NOTE: if you choose to stitch first, you need to pre-align, otherwise, it's OK to use the initial alignment for each tile if (stitchFirst && parameters.preAlign() == PreAlign.NONE) { - throw new IllegalArgumentException("AffineBlockSolverSetup with --stitchFirst requires --preAlign to be TRANSLATION or RIGID"); + throw new IllegalArgumentException("AffineBlockSolverSetup with --stitchFirst requires --preAlign to be TRANSLATION or RIGID or MULTI_SEM"); } this.coreTileSpecIds = new HashSet<>(); // will be populated by call to assembleMatchData @@ -922,6 +922,7 @@ protected void solve( for (final Tile tile : tileConfig.getTiles()) { final AlignmentModel model = (AlignmentModel) tile.getModel(); model.setWeights(weights); + // TODO: per Tile sigmoidal weight regularization depending on numMatches * weights } final int numIterations = blockOptimizerIterations.get(k);