diff --git a/pom.xml b/pom.xml index 5f9d5a485..5dc76fab9 100644 --- a/pom.xml +++ b/pom.xml @@ -141,6 +141,15 @@ --> 3.2.0 + + 4.1.0 + 7.1.4 0.17.2 diff --git a/render-app/pom.xml b/render-app/pom.xml index c82f740a8..f132fc84c 100644 --- a/render-app/pom.xml +++ b/render-app/pom.xml @@ -156,12 +156,12 @@ org.janelia.saalfeldlab - n5-imglib2 + n5-hdf5 org.janelia.saalfeldlab - n5-hdf5 + n5-universe diff --git a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java index c389eb275..739a3e1e8 100644 --- a/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java +++ b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java @@ -9,8 +9,8 @@ import java.util.List; import java.util.Map; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.universe.N5Factory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,13 +135,13 @@ public NeuroglancerAttributes(final List stackResolutionValues, /** * Writes all attribute.json files required by neuroglancer to display the specified dataset. * - * @param n5BasePath base path for n5. + * @param n5Base base path or URL string for n5. * @param fullScaleDatasetPath path of the full scale data set. * * @throws IOException * if the writes fail for any reason. */ - public void write(final Path n5BasePath, + public void write(final String n5Base, final Path fullScaleDatasetPath) throws IOException { @@ -150,10 +150,10 @@ public void write(final Path n5BasePath, final Path ngAttributesPath = isMultiScaleDataset ? fullScaleDatasetPath.getParent() : fullScaleDatasetPath; - LOG.info("write: entry, n5BasePath={}, fullScaleDatasetPath={}, ngAttributesPath={}", - n5BasePath, fullScaleDatasetPath, ngAttributesPath); + LOG.info("write: entry, n5Base={}, fullScaleDatasetPath={}, ngAttributesPath={}", + n5Base, fullScaleDatasetPath, ngAttributesPath); - final N5Writer n5Writer = new N5FSWriter(n5BasePath.toAbsolutePath().toString()); + final N5Writer n5Writer = new N5Factory().openWriter(N5Factory.StorageFormat.N5, n5Base); //new N5FSWriter(n5BasePath.toAbsolutePath().toString()); // Neuroglancer recursively looks for attribute.json files from root path and stops at // the first subdirectory without an attributes.json file. @@ -164,7 +164,7 @@ public void write(final Path n5BasePath, for (Path path = ngAttributesPath.getParent(); (path != null) && (! path.endsWith("/")); path = path.getParent()) { - LOG.info("write: saving supported attribute to {}{}/attributes.json", n5BasePath, path); + LOG.info("write: saving supported attribute to {}{}/attributes.json", n5Base, path); n5Writer.setAttribute(path.toString(), SUPPORTED_KEY, true); } @@ -180,7 +180,7 @@ public void write(final Path n5BasePath, attributes.put("pixelResolution", pixelResolution); attributes.put("translate", translate); - LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5BasePath, ngAttributesPath); + LOG.info("write: saving neuroglancer attributes to {}{}/attributes.json", n5Base, ngAttributesPath); n5Writer.setAttributes(ngAttributesPath.toString(), attributes); if (isMultiScaleDataset) { @@ -188,7 +188,7 @@ public void write(final Path n5BasePath, writeScaleLevelTransformAttributes(scaleLevel, scales.get(scaleLevel), n5Writer, - n5BasePath, + n5Base, ngAttributesPath); } } @@ -197,16 +197,18 @@ public void write(final Path n5BasePath, private void writeScaleLevelTransformAttributes(final int scaleLevel, final List scaleLevelFactors, final N5Writer n5Writer, - final Path n5BasePath, + final String n5Base, final Path ngAttributesPath) throws IOException { final String scaleName = "s" + scaleLevel; final Path scaleAttributesPath = Paths.get(ngAttributesPath.toString(), scaleName); - final Path scaleLevelDirectoryPath = Paths.get(n5BasePath.toString(), ngAttributesPath.toString(), scaleName); - if (! scaleLevelDirectoryPath.toFile().exists()) { - throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + if (n5Base.startsWith("/") || n5Base.startsWith("\\")) { + final Path scaleLevelDirectoryPath = Paths.get(n5Base, ngAttributesPath.toString(), scaleName); + if (! scaleLevelDirectoryPath.toFile().exists()) { + throw new IOException(scaleLevelDirectoryPath.toAbsolutePath() + " does not exist"); + } } final Map transformAttributes = new HashMap<>(); @@ -232,7 +234,7 @@ private void writeScaleLevelTransformAttributes(final int scaleLevel, final Map attributes = new HashMap<>(); attributes.put("transform", transformAttributes); - LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5BasePath, scaleAttributesPath); + LOG.info("writeScaleLevelTransformAttributes: saving {}{}/attributes.json", n5Base, scaleAttributesPath); n5Writer.setAttributes(scaleAttributesPath.toString(), attributes); } diff --git a/render-ws-java-client/pom.xml b/render-ws-java-client/pom.xml index 7aee937b0..d20fff8ad 100644 --- a/render-ws-java-client/pom.xml +++ b/render-ws-java-client/pom.xml @@ -74,6 +74,13 @@ LICENSE META-INF/* META-INF/versions/** + canonical.json + cosem.json + n5-compression.jq + n5.jq + zyx.zattrs.json + META-INF/services/io.grpc.LoadBalancerProvider + META-INF/services/io.grpc.NameResolverProvider diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java index 998464f23..c46b18b65 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/n5/CrossCorrelationWithNextRegionalDataN5Writer.java @@ -132,7 +132,7 @@ private static void createDataSet(final List(); // will be populated by call to assembleMatchData @@ -922,6 +922,7 @@ protected void solve( for (final Tile tile : tileConfig.getTiles()) { final AlignmentModel model = (AlignmentModel) tile.getModel(); model.setWeights(weights); + // TODO: per Tile sigmoidal weight regularization depending on numMatches * weights } final int numIterations = blockOptimizerIterations.get(k); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java index 9dbab3782..64b282bac 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/newsolver/solvers/intensity/IntensityTile.java @@ -52,7 +52,7 @@ public IntensityTile( for (int i = 0; i < N; i++) { final Affine1D model = modelSupplier.get(); - this.subTiles.add(new Tile<>((Model) model)); + this.subTiles.add(new Tile((Model) model)); } } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/ExportMichalSegmentationsClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/ExportMichalSegmentationsClient.java index 526a2ad40..89de060bb 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/ExportMichalSegmentationsClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/ExportMichalSegmentationsClient.java @@ -242,6 +242,6 @@ private void runForStack( Arrays.asList(min[0], min[1], min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - ngAttributes.write(Paths.get(parameters.targetN5Path), Paths.get(targetDataset, "s0")); + ngAttributes.write(parameters.targetN5Path, Paths.get(targetDataset, "s0")); } } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/H5TileToN5PreviewClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/H5TileToN5PreviewClient.java index 4f6b96c1a..eb72052e8 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/H5TileToN5PreviewClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/H5TileToN5PreviewClient.java @@ -559,7 +559,7 @@ private void exportPreview(final JavaSparkContext sparkContext, Arrays.asList(exportInfo.min[0], exportInfo.min[1], exportInfo.min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - ngAttributes.write(Paths.get(exportInfo.n5PathString), + ngAttributes.write(exportInfo.n5PathString, Paths.get(exportInfo.fullScaleDatasetName)); } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 491505447..94e931825 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -38,10 +38,11 @@ import org.janelia.render.client.zspacing.ThicknessCorrectionData; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.GzipCompression; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -361,7 +362,7 @@ public void run() Arrays.asList(min[0], min[1], min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - ngAttributes.write(Paths.get(parameters.n5Path), + ngAttributes.write(parameters.n5Path, Paths.get(fullScaleDatasetName)); if (downsampleStackForReview) { @@ -406,7 +407,7 @@ public void run() Arrays.asList(min[0], min[1], min[2]), NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); - reviewNgAttributes.write(Paths.get(parameters.n5Path), + reviewNgAttributes.write(parameters.n5Path, Paths.get(fullScaleReviewDatasetName)); } @@ -475,7 +476,7 @@ public static void setupFullScaleExportN5(final Parameters parameters, final int[] blockSize, final DataType dataType) { - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter( StorageFormat.N5, parameters.n5Path )) /*new N5FSWriter(parameters.n5Path))*/ { n5.createDataset(fullScaleDatasetName, dimensions, blockSize, @@ -494,7 +495,7 @@ public static void updateFullScaleExportAttributes(final Parameters parameters, String exportAttributesDatasetName = fullScaleDatasetName; - try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + try (final N5Writer n5 = new N5Factory().openWriter( StorageFormat.N5, parameters.n5Path )/*new N5FSWriter(parameters.n5Path)*/) { final Map export_attributes = new HashMap<>(); export_attributes.put("runTimestamp", new Date()); export_attributes.put("runParameters", parameters); @@ -661,8 +662,9 @@ private static void saveRenderStack(final JavaSparkContext sc, } } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter( StorageFormat.N5, n5Path )/*new N5FSWriter(n5Path)*/; // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); } @@ -722,8 +724,9 @@ private static void save2DRenderStack(final JavaSparkContext sc, out.next().set(in.next()); } - final N5Writer anotherN5Writer = new N5FSWriter(n5Path); // needed to prevent Spark serialization error + final N5Writer anotherN5Writer = new N5Factory().openWriter( StorageFormat.N5, n5Path ) /*new N5FSWriter(n5Path)*/; // needed to prevent Spark serialization error N5Utils.saveNonEmptyBlock(block, anotherN5Writer, datasetName, gridBlock.gridPosition, new UnsignedByteType(0)); + anotherN5Writer.close(); }); LOG.info("save2DRenderStack: exit"); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java index 437526422..e85361a52 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java @@ -2,9 +2,9 @@ import java.io.IOException; -import org.janelia.saalfeldlab.n5.N5FSWriter; import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; +import org.janelia.saalfeldlab.n5.universe.N5Factory; /** * Utilities for N5 operations. @@ -22,7 +22,7 @@ public N5PathSupplier(final String path) { @Override public N5Writer get() throws IOException { - return new N5FSWriter(path); + return new N5Factory().openWriter(N5Factory.StorageFormat.N5, path); } } diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java index eabac8d66..f534e5162 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java @@ -150,13 +150,13 @@ public void testSetupFullScaleExportN5() throws Exception { @Test public void testNeuroglancerAttributes() throws Exception { - final Path n5Path = n5PathDirectory.toPath().toAbsolutePath(); + final String n5Path = n5PathDirectory.getAbsolutePath(); final Path fullScaleDatasetPath = Paths.get("/render/test_stack/one_more_nested_dir/s0"); final String datasetName = fullScaleDatasetPath.toString(); final long[] dimensions = { 100L, 200L, 300L }; final int[] blockSize = { 10, 20, 30 }; - try (final N5Writer n5Writer = new N5FSWriter(n5Path.toString())) { + try (final N5Writer n5Writer = new N5FSWriter(n5Path)) { final DatasetAttributes datasetAttributes = new DatasetAttributes(dimensions, blockSize, @@ -164,7 +164,7 @@ public void testNeuroglancerAttributes() throws Exception { new GzipCompression()); n5Writer.createDataset(datasetName, datasetAttributes); - final N5Reader n5Reader = new N5FSReader(n5Path.toString()); + final N5Reader n5Reader = new N5FSReader(n5Path); Assert.assertTrue("dataset " + datasetName + " is missing", n5Reader.datasetExists(datasetName)); final Map originalDatasetAttributes = datasetAttributes.asMap(); diff --git a/render-ws/pom.xml b/render-ws/pom.xml index 7cb96b0de..7a4dd9ab6 100644 --- a/render-ws/pom.xml +++ b/render-ws/pom.xml @@ -67,11 +67,19 @@ render-app ${project.version} + org.slf4j slf4j-api + + + + javax.annotation + javax.annotation-api + +