From a90a30509c0aaec896a18d03eff2b0ea042818f4 Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 7 Sep 2023 16:36:32 +0100 Subject: [PATCH 01/37] ZarrReader performance improvements --- src/loci/formats/in/ZarrReader.java | 118 ++++++++++++++++------------ 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 123e88d..17f981d 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -82,6 +82,8 @@ public class ZarrReader extends FormatReader { + public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; + public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; protected transient ZarrService zarrService; private ArrayList arrayPaths= new ArrayList(); private HashMap> resSeries = new HashMap>(); @@ -168,8 +170,8 @@ protected void initFile(String id) throws FormatException, IOException { Map attr = zarrService.getGroupAttr(canonicalPath); int attrIndex = 0; if (attr != null && !attr.isEmpty()) { - parseResolutionCount(zarrRootPath, ""); - parseOmeroMetadata(zarrRootPath, ""); + parseResolutionCount(zarrRootPath, "", attr); + parseOmeroMetadata(zarrRootPath, attr); String jsonAttr; try { jsonAttr = ZarrUtils.toJson(attr, true); @@ -186,45 +188,49 @@ protected void initFile(String id) throws FormatException, IOException { for (String key: zarrService.getGroupKeys(canonicalPath)) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); if (attributes != null && !attributes.isEmpty()) { - parseResolutionCount(zarrRootPath, key); - parseLabels(zarrRootPath, key); - parseImageLabels(zarrRootPath, key); + parseResolutionCount(zarrRootPath, key, attributes); + parseLabels(zarrRootPath, attributes); + parseImageLabels(zarrRootPath, attributes); attrIndex++; - String jsonAttr; - try { - jsonAttr = ZarrUtils.toJson(attributes, true); - store.setXMLAnnotationValue(jsonAttr, attrIndex); - String xml_id = MetadataTools.createLSID("Annotation", attrIndex); - store.setXMLAnnotationID(xml_id, attrIndex); - } catch (JZarrException e) { - LOGGER.warn("Failed to convert attributes to JSON"); - e.printStackTrace(); + if (saveAnnotations()) { + String jsonAttr; + try { + jsonAttr = ZarrUtils.toJson(attributes, true); + store.setXMLAnnotationValue(jsonAttr, attrIndex); + String xml_id = MetadataTools.createLSID("Annotation", attrIndex); + store.setXMLAnnotationID(xml_id, attrIndex); + } catch (JZarrException e) { + LOGGER.warn("Failed to convert attributes to JSON"); + e.printStackTrace(); + } } } } // Parse array attributes - for (String key: zarrService.getArrayKeys(canonicalPath)) { - Map attributes = zarrService.getArrayAttr(canonicalPath+File.separator+key); - if (attributes != null && !attributes.isEmpty()) { - attrIndex++; - String jsonAttr; - try { - jsonAttr = ZarrUtils.toJson(attributes, true); - store.setXMLAnnotationValue(jsonAttr, attrIndex); - String xml_id = MetadataTools.createLSID("Annotation", attrIndex); - store.setXMLAnnotationID(xml_id, attrIndex); - } catch (JZarrException e) { - LOGGER.warn("Failed to convert attributes to JSON"); - e.printStackTrace(); - } - } - } - arrayPaths = new ArrayList(); arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); orderArrayPaths(zarrRootPath); + if (saveAnnotations()) { + for (String key: arrayPaths) { + Map attributes = zarrService.getArrayAttr(zarrRootPath+File.separator+key); + if (attributes != null && !attributes.isEmpty()) { + attrIndex++; + String jsonAttr; + try { + jsonAttr = ZarrUtils.toJson(attributes, true); + store.setXMLAnnotationValue(jsonAttr, attrIndex); + String xml_id = MetadataTools.createLSID("Annotation", attrIndex); + store.setXMLAnnotationID(xml_id, attrIndex); + } catch (JZarrException e) { + LOGGER.warn("Failed to convert attributes to JSON"); + e.printStackTrace(); + } + } + } + } + core.clear(); int resolutionTotal = 0; for (int i=0; i attr = zarrService.getGroupAttr(canonicalPath); + private void parseResolutionCount(String root, String key, Map attr) throws IOException, FormatException { ArrayList multiscales = (ArrayList) attr.get("multiscales"); if (multiscales != null) { for (int x = 0; x < multiscales.size(); x++) { @@ -646,10 +656,7 @@ private void parseWells(String root, String key, MetadataStore store, int plateI } } - private void parseLabels(String root, String key) throws IOException, FormatException { - String path = key.isEmpty() ? root : root + File.separator + key; - String canonicalPath = new Location(path).getCanonicalPath(); - Map attr = zarrService.getGroupAttr(canonicalPath); + private void parseLabels(String root, Map attr) throws IOException, FormatException { ArrayList labels = (ArrayList) attr.get("labels"); if (labels != null) { for (int l = 0; l < labels.size(); l++) { @@ -658,10 +665,7 @@ private void parseLabels(String root, String key) throws IOException, FormatExce } } - private void parseImageLabels(String root, String key) throws IOException, FormatException { - String path = key.isEmpty() ? root : root + File.separator + key; - String canonicalPath = new Location(path).getCanonicalPath(); - Map attr = zarrService.getGroupAttr(canonicalPath); + private void parseImageLabels(String root, Map attr) throws IOException, FormatException { Map imageLabel = (Map) attr.get("image-label"); if (imageLabel != null) { String version = (String) imageLabel.get("version"); @@ -698,10 +702,7 @@ private void parseImageLabels(String root, String key) throws IOException, Forma } } - private void parseOmeroMetadata(String root, String key) throws IOException, FormatException { - String path = key.isEmpty() ? root : root + File.separator + key; - String canonicalPath = new Location(path).getCanonicalPath(); - Map attr = zarrService.getGroupAttr(canonicalPath); + private void parseOmeroMetadata(String root, Map attr) throws IOException, FormatException { Map omeroMetadata = (Map) attr.get("omero"); if (omeroMetadata != null) { Integer id = (Integer) omeroMetadata.get("id"); @@ -883,4 +884,21 @@ public String[] getDomains() { FormatTools.NON_SPECIAL_DOMAINS; } + /* @see loci.formats.FormatReader#initFile(String) */ + @Override + protected ArrayList getAvailableOptions() { + ArrayList optionsList = super.getAvailableOptions(); + optionsList.add(SAVE_ANNOTATIONS_KEY); + return optionsList; + } + + public boolean saveAnnotations() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).getBoolean( + SAVE_ANNOTATIONS_KEY, SAVE_ANNOTATIONS_DEFAULT); + } + return SAVE_ANNOTATIONS_DEFAULT; + } + } From d30c0f013dddf01e21130bf00524355f3705a27b Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 7 Sep 2023 19:46:35 +0100 Subject: [PATCH 02/37] Add new setResolution if option to open-air --- src/loci/formats/in/ZarrReader.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 17f981d..19157ed 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -434,10 +434,18 @@ public void setSeries(int no, boolean openZarr) { @Override public void setResolution(int no) { + setResolution(no, false); + } + + @Override + public void setResolution(int no, boolean openZarr) { super.setResolution(no); - openZarr(); + if (openZarr) { + openZarr(); + } } + private void openZarr() { try { if (currentId != null && zarrService != null) { From 434cf20ec1a9b8362986698351a2cbcfe4d0221b Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 7 Sep 2023 21:58:47 +0100 Subject: [PATCH 03/37] ZarrReader: Remove incorrect override --- src/loci/formats/in/ZarrReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 19157ed..7f274eb 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -437,7 +437,6 @@ public void setResolution(int no) { setResolution(no, false); } - @Override public void setResolution(int no, boolean openZarr) { super.setResolution(no); if (openZarr) { From a1d4bea5640918a2fe5d4e303f0ab149f59ad40b Mon Sep 17 00:00:00 2001 From: David Gault Date: Sun, 10 Sep 2023 19:04:10 +0100 Subject: [PATCH 04/37] Merge list pixels option for getUsedFiles --- src/loci/formats/in/ZarrReader.java | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 7f274eb..61b884f 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -84,6 +84,8 @@ public class ZarrReader extends FormatReader { public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; + public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; + public static final boolean LIST_PIXELS_DEFAULT = false; protected transient ZarrService zarrService; private ArrayList arrayPaths= new ArrayList(); private HashMap> resSeries = new HashMap>(); @@ -871,9 +873,13 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); + boolean skipPixels = noPixels || !listPixels(); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { - paths.filter(Files::isRegularFile) - .forEach(path -> usedFiles.add(path.toFile().getAbsolutePath())); + paths.filter(Files::isRegularFile) + .forEach(path -> {if (!skipPixels || + (skipPixels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) + usedFiles.add(path.toFile().getAbsolutePath()); + }); } catch (IOException e) { e.printStackTrace(); } @@ -896,6 +902,7 @@ public String[] getDomains() { protected ArrayList getAvailableOptions() { ArrayList optionsList = super.getAvailableOptions(); optionsList.add(SAVE_ANNOTATIONS_KEY); + optionsList.add(LIST_PIXELS_KEY); return optionsList; } @@ -907,5 +914,14 @@ public boolean saveAnnotations() { } return SAVE_ANNOTATIONS_DEFAULT; } + + public boolean listPixels() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).getBoolean( + LIST_PIXELS_KEY, LIST_PIXELS_DEFAULT); + } + return LIST_PIXELS_DEFAULT; + } } From 3f09254740d03a7809ce14e3ae49a7bae2c2dde8 Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 11 Sep 2023 15:06:14 +0100 Subject: [PATCH 05/37] Add quick read option if shape and pixel type remains consistent --- src/loci/formats/in/ZarrReader.java | 46 ++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 61b884f..de83282 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -64,6 +64,8 @@ import loci.formats.FormatReader; import loci.formats.FormatTools; import loci.formats.MetadataTools; +import loci.formats.in.DynamicMetadataOptions; +import loci.formats.in.MetadataOptions; import loci.formats.meta.MetadataStore; import loci.formats.ome.OMEXMLMetadata; import loci.formats.services.JZarrServiceImpl; @@ -82,6 +84,8 @@ public class ZarrReader extends FormatReader { + public static final String QUICK_READ_KEY = "zarrreader.quick_read"; + public static final boolean QUICK_READ_DEFAULT = true; public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; @@ -235,6 +239,10 @@ protected void initFile(String id) throws FormatException, IOException { core.clear(); int resolutionTotal = 0; + + HashMap resShapes = new HashMap(); + int pixelType = -1; + for (int i=0; i Date: Tue, 12 Sep 2023 18:30:00 +0100 Subject: [PATCH 06/37] Generate groupKeys and arrayPaths from top level attributes --- src/loci/formats/in/ZarrReader.java | 66 +++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index de83282..a328ff9 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -91,7 +91,8 @@ public class ZarrReader extends FormatReader { public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; public static final boolean LIST_PIXELS_DEFAULT = false; protected transient ZarrService zarrService; - private ArrayList arrayPaths= new ArrayList(); + private ArrayList arrayPaths = new ArrayList(); + private ArrayList groupKeys = new ArrayList(); private HashMap> resSeries = new HashMap>(); private HashMap resCounts = new HashMap(); private HashMap resIndexes = new HashMap(); @@ -130,6 +131,7 @@ public boolean isThisType(String name, boolean open) { @Override public void close() throws IOException { arrayPaths.clear(); + groupKeys.clear(); resSeries.clear(); resCounts.clear(); resIndexes.clear(); @@ -190,8 +192,13 @@ protected void initFile(String id) throws FormatException, IOException { } } + quickParsePlate(attr, zarrRootPath, "", store); + // Parse group attributes - for (String key: zarrService.getGroupKeys(canonicalPath)) { + if (groupKeys.isEmpty()) { + groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); + } + for (String key: groupKeys) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); if (attributes != null && !attributes.isEmpty()) { parseResolutionCount(zarrRootPath, key, attributes); @@ -214,8 +221,27 @@ protected void initFile(String id) throws FormatException, IOException { } // Parse array attributes - arrayPaths = new ArrayList(); - arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); + Map plates = (Map) attr.get("plate"); + if (plates != null) { + ArrayList columns = (ArrayList)plates.get("columns"); + ArrayList rows = (ArrayList)plates.get("rows"); + Integer fieldCount = (Integer) plates.get("field_count"); + for (Object row: rows) { + String rowName = ((Map) row).get("name"); + for (Object column: columns) { + String columnName = ((Map) column).get("name"); + for (int i = 0; i < fieldCount; i++) { + for (int j = 0; j < 4; j++) { + arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); + } + } + } + } + } + + if (arrayPaths.isEmpty()) { + arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); + } orderArrayPaths(zarrRootPath); if (saveAnnotations()) { @@ -287,7 +313,6 @@ protected void initFile(String id) throws FormatException, IOException { shape = resShapes.get(resolutionIndex); } - ms.sizeX = shape[4]; ms.sizeY = shape[3]; ms.sizeT = shape[0]; @@ -319,8 +344,7 @@ protected void initFile(String id) throws FormatException, IOException { store.setImageName(arrayPaths.get(seriesToCoreIndex(i)), i); store.setImageID(MetadataTools.createLSID("Image", i), i); } - parsePlate(zarrRootPath, "", store); - + parsePlate(attr, zarrRootPath, "", store); setSeries(0); } @@ -566,10 +590,28 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { } } - private void parsePlate(String root, String key, MetadataStore store) throws IOException, FormatException { - String path = key.isEmpty() ? root : root + File.separator + key; - String canonicalPath = new Location(path).getCanonicalPath(); - Map attr = zarrService.getGroupAttr(canonicalPath); + private void quickParsePlate(Map attr, String root, String key, MetadataStore store) throws IOException, FormatException { + Map plates = (Map) attr.get("plate"); + if (plates != null) { + ArrayList columns = (ArrayList)plates.get("columns"); + ArrayList rows = (ArrayList)plates.get("rows"); + Integer fieldCount = (Integer) plates.get("field_count"); + + for (Object row: rows) { + String rowName = ((Map) row).get("name"); + groupKeys.add(rowName); + for (Object column: columns) { + String columnName = ((Map) column).get("name"); + groupKeys.add(rowName + File.separator + columnName); + for (int i = 0; i < fieldCount; i++) { + groupKeys.add(rowName + File.separator + columnName + File.separator + i); + } + } + } + } + } + + private void parsePlate(Map attr, String root, String key, MetadataStore store) throws IOException, FormatException { Map plates = (Map) attr.get("plate"); if (plates != null) { ArrayList columns = (ArrayList)plates.get("columns"); @@ -577,7 +619,7 @@ private void parsePlate(String root, String key, MetadataStore store) throws IOE ArrayList wells = (ArrayList)plates.get("wells"); ArrayList acquisitions = (ArrayList )plates.get("acquisitions"); String plateName = (String) plates.get("name"); - String fieldCount = (String) plates.get("filed_count"); + Integer fieldCount = (Integer) plates.get("field_count"); String plate_id = MetadataTools.createLSID("Plate", 0); store.setPlateID(plate_id, 0); From f14c44bc8ea4ff25b39441ed01c88ee4bb14e6fc Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 13 Sep 2023 09:21:34 +0100 Subject: [PATCH 07/37] Update arrayKey generation to revert if resolution counts differ --- src/loci/formats/in/ZarrReader.java | 50 +++++++++++++++++------------ 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index a328ff9..6565f52 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -38,6 +38,7 @@ import java.nio.file.FileVisitOption; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -95,6 +96,7 @@ public class ZarrReader extends FormatReader { private ArrayList groupKeys = new ArrayList(); private HashMap> resSeries = new HashMap>(); private HashMap resCounts = new HashMap(); + private HashSet uniqueResCounts = new HashSet(); private HashMap resIndexes = new HashMap(); private String dimensionOrder = "XYZCT"; private int wellCount = 0; @@ -192,7 +194,7 @@ protected void initFile(String id) throws FormatException, IOException { } } - quickParsePlate(attr, zarrRootPath, "", store); + generateGroupKeys(attr); // Parse group attributes if (groupKeys.isEmpty()) { @@ -221,24 +223,7 @@ protected void initFile(String id) throws FormatException, IOException { } // Parse array attributes - Map plates = (Map) attr.get("plate"); - if (plates != null) { - ArrayList columns = (ArrayList)plates.get("columns"); - ArrayList rows = (ArrayList)plates.get("rows"); - Integer fieldCount = (Integer) plates.get("field_count"); - for (Object row: rows) { - String rowName = ((Map) row).get("name"); - for (Object column: columns) { - String columnName = ((Map) column).get("name"); - for (int i = 0; i < fieldCount; i++) { - for (int j = 0; j < 4; j++) { - arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); - } - } - } - } - } - + generateArrayKeys(attr); if (arrayPaths.isEmpty()) { arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); } @@ -567,6 +552,7 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { int numRes = multiscalePaths.size(); if (i == 0) { resCounts.put(key.isEmpty() ? scalePath : key + File.separator + scalePath, numRes); + uniqueResCounts.add(numRes); } resIndexes.put(key.isEmpty() ? scalePath : key + File.separator + scalePath, i); ArrayList list = resSeries.get(resCounts.size() - 1); @@ -590,7 +576,31 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { } } - private void quickParsePlate(Map attr, String root, String key, MetadataStore store) throws IOException, FormatException { + private void generateArrayKeys(Map attr) { + if (uniqueResCounts.size() != 1) { + LOGGER.info("Cannout automatically generate ArrayKeys as resolution counts differ"); + } + Map plates = (Map) attr.get("plate"); + if (plates != null) { + ArrayList columns = (ArrayList)plates.get("columns"); + ArrayList rows = (ArrayList)plates.get("rows"); + Integer fieldCount = (Integer) plates.get("field_count"); + for (Object row: rows) { + String rowName = ((Map) row).get("name"); + for (Object column: columns) { + String columnName = ((Map) column).get("name"); + for (int i = 0; i < fieldCount; i++) { + int resolutionCount = ((Integer[])uniqueResCounts.toArray())[0]; + for (int j = 0; j < resolutionCount; j++) { + arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); + } + } + } + } + } + } + + private void generateGroupKeys(Map attr) { Map plates = (Map) attr.get("plate"); if (plates != null) { ArrayList columns = (ArrayList)plates.get("columns"); From 3246a712842ac89c7f1e75d55515c939c77d6101 Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 14 Sep 2023 19:32:05 +0100 Subject: [PATCH 08/37] update setResolution to call openZarr --- src/loci/formats/in/ZarrReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 6565f52..0f0f646 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -277,7 +277,7 @@ protected void initFile(String id) throws FormatException, IOException { } else { setSeries(coreIndexToSeries(i), openZarr); - setResolution(resolutionIndex); + setResolution(resolutionIndex, openZarr); if (i == resolutionTotal + resolutionCount - 1) { resolutionTotal += resolutionCount; } @@ -590,7 +590,7 @@ private void generateArrayKeys(Map attr) { for (Object column: columns) { String columnName = ((Map) column).get("name"); for (int i = 0; i < fieldCount; i++) { - int resolutionCount = ((Integer[])uniqueResCounts.toArray())[0]; + int resolutionCount = (Integer)(uniqueResCounts.toArray())[0]; for (int j = 0; j < resolutionCount; j++) { arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); } From f32b8bcb2bd35ab1c413ee013bb4d03432e6dd63 Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 14 Sep 2023 19:57:45 +0100 Subject: [PATCH 09/37] Avoid duplicate calls in openZarr --- src/loci/formats/in/ZarrReader.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 0f0f646..f8506fa 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -104,6 +104,7 @@ public class ZarrReader extends FormatReader { private HashMap> pathArrayDimensions = new HashMap>(); private boolean planesPrePopulated = false; private boolean hasSPW = false; + private transient int currentOpenZarr = -1; public ZarrReader() { super("Zarr", "zarr"); @@ -489,9 +490,13 @@ private void openZarr() { if (!hasFlattenedResolutions()) { seriesIndex += resolution; } - newZarrPath += File.separator + arrayPaths.get(seriesIndex); - String canonicalPath = new Location(newZarrPath).getCanonicalPath(); - zarrService.open(canonicalPath); + if (seriesIndex != currentOpenZarr) { + newZarrPath += File.separator + arrayPaths.get(seriesIndex); + String canonicalPath = new Location(newZarrPath).getCanonicalPath(); + LOGGER.info("Opening zarr for series {} at path: {}", seriesIndex, canonicalPath); + zarrService.open(canonicalPath); + currentOpenZarr = seriesIndex; + } } } } catch (IOException | FormatException e) { From 3e7dfaae7ccdd6c34a40d20d20361d49383aca82 Mon Sep 17 00:00:00 2001 From: David Gault Date: Fri, 15 Sep 2023 19:28:38 +0100 Subject: [PATCH 10/37] Store resolution shapes after converting to 5D --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index f8506fa..a78271e 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -291,8 +291,8 @@ protected void initFile(String id) throws FormatException, IOException { shape = zarrService.getShape(); if (shape.length < 5) { shape = get5DShape(shape); - resShapes.put(resolutionIndex, shape); } + resShapes.put(resolutionIndex, shape); } else { ms.pixelType = pixelType; From 66d15590a66596548e1bce9c5fd1a6e0204735f3 Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 18 Sep 2023 14:51:39 +0100 Subject: [PATCH 11/37] Update close to clear existing variables --- src/loci/formats/in/ZarrReader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index a78271e..14d74a2 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -137,7 +137,9 @@ public void close() throws IOException { groupKeys.clear(); resSeries.clear(); resCounts.clear(); + uniqueResCounts.clear(); resIndexes.clear(); + pathArrayDimensions.clear(); if (zarrService != null) { zarrService.close(); } From a96f0adf82608560a50dba289c7ff4812e08df09 Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 18 Sep 2023 17:00:10 +0100 Subject: [PATCH 12/37] Add additional initFile logging --- src/loci/formats/in/ZarrReader.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 14d74a2..e675bba 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -166,6 +166,7 @@ public int getOptimalTileWidth() { @Override protected void initFile(String id) throws FormatException, IOException { super.initFile(id); + LOGGER.info("ZarrReader attempting to initialize file: {}", id); final MetadataStore store = makeFilterMetadata(); Location zarrFolder = new Location(id); String zarrPath = zarrFolder.getAbsolutePath(); @@ -174,14 +175,17 @@ protected void initFile(String id) throws FormatException, IOException { Location omeMetaFile = new Location( zarrRootPath + File.separator + "OME", "METADATA.ome.xml" ); String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); + LOGGER.info("ZarrReader initializing ZarrService: {}", canonicalPath); initializeZarrService(canonicalPath); if(omeMetaFile.exists()) { + LOGGER.info("ZarrReader parsing existing OME-XML"); parseOMEXML(omeMetaFile, store); } // Parse base level attributes Map attr = zarrService.getGroupAttr(canonicalPath); int attrIndex = 0; + LOGGER.info("ZarrReader parsing top level group attributes: {}", canonicalPath); if (attr != null && !attr.isEmpty()) { parseResolutionCount(zarrRootPath, "", attr); parseOmeroMetadata(zarrRootPath, attr); @@ -196,13 +200,15 @@ protected void initFile(String id) throws FormatException, IOException { e.printStackTrace(); } } - + LOGGER.info("ZarrReader generating group keys"); generateGroupKeys(attr); // Parse group attributes if (groupKeys.isEmpty()) { + LOGGER.info("ZarrReader adding hroup keys from ZarrService"); groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); } + LOGGER.info("ZarrReader parsing group Keys"); for (String key: groupKeys) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); if (attributes != null && !attributes.isEmpty()) { @@ -226,10 +232,13 @@ protected void initFile(String id) throws FormatException, IOException { } // Parse array attributes + LOGGER.info("ZarrReader attempting to generate Array Keys"); generateArrayKeys(attr); if (arrayPaths.isEmpty()) { + LOGGER.info("ZarrReader adding Array Keys from ZarrService"); arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); } + LOGGER.info("ZarrReader ordering Array Paths"); orderArrayPaths(zarrRootPath); if (saveAnnotations()) { @@ -258,6 +267,7 @@ protected void initFile(String id) throws FormatException, IOException { int pixelType = -1; for (int i=0; i Date: Mon, 18 Sep 2023 17:01:08 +0100 Subject: [PATCH 13/37] Update getRequiredDirectories --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index e675bba..a800435 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -117,7 +117,7 @@ public ZarrReader() { public int getRequiredDirectories(String[] files) throws FormatException, IOException { - return 1; + return FormatTools.getRequiredDirectories(files); } /* @see loci.formats.IFormatReader#isThisType(String, boolean) */ From 2a7bd44ecf66c3bf776a9efd76ce8ff3ef436599 Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 18 Sep 2023 17:03:20 +0100 Subject: [PATCH 14/37] Update close to reset variables --- src/loci/formats/in/ZarrReader.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index a800435..524674a 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -143,6 +143,11 @@ public void close() throws IOException { if (zarrService != null) { zarrService.close(); } + planesPrePopulated = false; + hasSPW = false; + currentOpenZarr = -1; + wellCount = 0; + wellSamplesCount = 0; super.close(); } From 6371c790a70306bbfcf1365d64bad04c19fc538d Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 18 Sep 2023 20:00:46 +0100 Subject: [PATCH 15/37] Merge reorder group keys changes from ZarrReader/pull/53 --- src/loci/formats/in/ZarrReader.java | 109 +++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 4 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 524674a..73839e1 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -37,11 +37,14 @@ import java.nio.file.Paths; import java.nio.file.FileVisitOption; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; @@ -183,9 +186,10 @@ protected void initFile(String id) throws FormatException, IOException { LOGGER.info("ZarrReader initializing ZarrService: {}", canonicalPath); initializeZarrService(canonicalPath); + ArrayList omeSeriesOrder = new ArrayList(); if(omeMetaFile.exists()) { LOGGER.info("ZarrReader parsing existing OME-XML"); - parseOMEXML(omeMetaFile, store); + parseOMEXML(omeMetaFile, store, omeSeriesOrder); } // Parse base level attributes Map attr = zarrService.getGroupAttr(canonicalPath); @@ -214,7 +218,8 @@ protected void initFile(String id) throws FormatException, IOException { groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); } LOGGER.info("ZarrReader parsing group Keys"); - for (String key: groupKeys) { + List orderedGroupKeys = reorderGroupKeys(groupKeys, omeSeriesOrder); + for (String key: orderedGroupKeys) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); if (attributes != null && !attributes.isEmpty()) { parseResolutionCount(zarrRootPath, key, attributes); @@ -353,6 +358,68 @@ protected void initFile(String id) throws FormatException, IOException { setSeries(0); LOGGER.info("ZarrReader initialization complete"); } + + private List reorderGroupKeys(ArrayList groupKeys, List originalKeys) { + // Reorder group keys to maintain the original order from the OME-XML provided by bioformats2raw + if (originalKeys.isEmpty() || !groupKeys.containsAll(originalKeys)) { + LOGGER.warn("Mismatch with group key paths and original OME-XML metadata, original ordering wont be maintained"); + return reorderGroupKeys(groupKeys); + } + List groupKeysList = new ArrayList(); + groupKeys.removeAll(originalKeys); + groupKeysList.addAll(originalKeys); + groupKeysList.addAll(groupKeys); + return groupKeysList; + } + + private List reorderGroupKeys(ArrayList groupKeys) { + // Reorder group keys to avoid order such A/1, A/10, A/11, A/12, A/2, A/20, A/3, A/4 + List groupKeysList = new ArrayList(); + groupKeysList.addAll(groupKeys); + Collections.sort(groupKeysList, keyComparator); + return groupKeysList; + } + + private static Comparator keyComparator = (a,b)->{ + String[] aParts = a.split("/"); + String[] bParts = b.split("/"); + + int numParts = aParts.length - bParts.length; + if (numParts != 0) return numParts; + + for (int i = 0; i < aParts.length; i++) { + String aPart = aParts[i]; + String bPart = bParts[i]; + + boolean isAInt = isInteger(aPart); + boolean isBInt = isInteger(bPart); + if (isAInt && !isBInt) return -1; + if (!isAInt && isBInt) return 1; + + if (isAInt) { + int numResult = Integer.compare(Integer.valueOf(aPart), Integer.valueOf(bPart)); + if (numResult != 0) return numResult; + } + else { + int stringResult = aPart.compareTo(bPart); + if (stringResult != 0) return stringResult; + } + } + + return 0; + }; + + private static boolean isInteger(String s) { + if(s.isEmpty()) return false; + for(int i = 0; i < s.length(); i++) { + if(i == 0 && s.charAt(i) == '-') { + if(s.length() == 1) return false; + else continue; + } + if(Character.digit(s.charAt(i), 10) < 0) return false; + } + return true; + } /** * In the event that .zarray does not contain a 5d shape @@ -843,7 +910,7 @@ private void parseOmeroMetadata(String root, Map attr) throws IO } } - private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOException, FormatException { + private void parseOMEXML(Location omeMetaFile, MetadataStore store, ArrayList origSeries) throws IOException, FormatException { Document omeDocument = null; try (RandomAccessInputStream measurement = new RandomAccessInputStream(omeMetaFile.getAbsolutePath())) { @@ -885,6 +952,24 @@ private void parseOMEXML(Location omeMetaFile, MetadataStore store) throws IOExc int numDatasets = omexmlMeta.getImageCount(); + // Map of the well location for each imageReference + // Later we will map the series index to the imageReference + // This allows us to maintain the series order when parsing the Zarr groups + Map imageRefPaths = new HashMap(); + for (int plateIndex = 0; plateIndex < omexmlMeta.getPlateCount(); plateIndex++) { + for (int wellIndex = 0; wellIndex < omexmlMeta.getWellCount(plateIndex); wellIndex++) { + NonNegativeInteger col = omexmlMeta.getWellColumn(plateIndex, wellIndex); + NonNegativeInteger row = omexmlMeta.getWellRow(plateIndex, wellIndex); + + String rowLetter = getRowString(row.getValue()); + String expectedPath = rowLetter + File.separator + (col.getValue() + 1) + File.separator + "0"; + for (int wellSampleIndex = 0; wellSampleIndex < omexmlMeta.getWellSampleCount(plateIndex, wellIndex); wellSampleIndex++) { + String imageRef = omexmlMeta.getWellSampleImageRef(plateIndex, wellIndex, wellSampleIndex); + imageRefPaths.put(imageRef, expectedPath); + } + } + } + int oldSeries = getSeries(); core.clear(); for (int i=0; i 0) { + sb.append((char)('A' + (rowIndex % 26))); + rowIndex /= 26; + } + return sb.reverse().toString(); + } private Double getDouble(Map src, String key) { Number val = (Number) src.get(key); From 38d5fa843c27f1ab1dba56b054f872bd10780120 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 20 Sep 2023 14:58:43 +0100 Subject: [PATCH 16/37] Temporarily use error level logging --- src/loci/formats/in/ZarrReader.java | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 73839e1..724776f 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -174,7 +174,7 @@ public int getOptimalTileWidth() { @Override protected void initFile(String id) throws FormatException, IOException { super.initFile(id); - LOGGER.info("ZarrReader attempting to initialize file: {}", id); + LOGGER.error("ZarrReader attempting to initialize file: {}", id); final MetadataStore store = makeFilterMetadata(); Location zarrFolder = new Location(id); String zarrPath = zarrFolder.getAbsolutePath(); @@ -183,18 +183,18 @@ protected void initFile(String id) throws FormatException, IOException { Location omeMetaFile = new Location( zarrRootPath + File.separator + "OME", "METADATA.ome.xml" ); String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); - LOGGER.info("ZarrReader initializing ZarrService: {}", canonicalPath); + LOGGER.error("ZarrReader initializing ZarrService: {}", canonicalPath); initializeZarrService(canonicalPath); ArrayList omeSeriesOrder = new ArrayList(); if(omeMetaFile.exists()) { - LOGGER.info("ZarrReader parsing existing OME-XML"); + LOGGER.error("ZarrReader parsing existing OME-XML"); parseOMEXML(omeMetaFile, store, omeSeriesOrder); } // Parse base level attributes Map attr = zarrService.getGroupAttr(canonicalPath); int attrIndex = 0; - LOGGER.info("ZarrReader parsing top level group attributes: {}", canonicalPath); + LOGGER.error("ZarrReader parsing top level group attributes: {}", canonicalPath); if (attr != null && !attr.isEmpty()) { parseResolutionCount(zarrRootPath, "", attr); parseOmeroMetadata(zarrRootPath, attr); @@ -209,15 +209,15 @@ protected void initFile(String id) throws FormatException, IOException { e.printStackTrace(); } } - LOGGER.info("ZarrReader generating group keys"); + LOGGER.error("ZarrReader generating group keys"); generateGroupKeys(attr); // Parse group attributes if (groupKeys.isEmpty()) { - LOGGER.info("ZarrReader adding hroup keys from ZarrService"); + LOGGER.error("ZarrReader adding hroup keys from ZarrService"); groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); } - LOGGER.info("ZarrReader parsing group Keys"); + LOGGER.error("ZarrReader parsing group Keys"); List orderedGroupKeys = reorderGroupKeys(groupKeys, omeSeriesOrder); for (String key: orderedGroupKeys) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); @@ -242,13 +242,13 @@ protected void initFile(String id) throws FormatException, IOException { } // Parse array attributes - LOGGER.info("ZarrReader attempting to generate Array Keys"); + LOGGER.error("ZarrReader attempting to generate Array Keys"); generateArrayKeys(attr); if (arrayPaths.isEmpty()) { - LOGGER.info("ZarrReader adding Array Keys from ZarrService"); + LOGGER.error("ZarrReader adding Array Keys from ZarrService"); arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); } - LOGGER.info("ZarrReader ordering Array Paths"); + LOGGER.error("ZarrReader ordering Array Paths"); orderArrayPaths(zarrRootPath); if (saveAnnotations()) { @@ -277,7 +277,7 @@ protected void initFile(String id) throws FormatException, IOException { int pixelType = -1; for (int i=0; i reorderGroupKeys(ArrayList groupKeys, List originalKeys) { @@ -580,7 +580,7 @@ private void openZarr() { if (seriesIndex != currentOpenZarr) { newZarrPath += File.separator + arrayPaths.get(seriesIndex); String canonicalPath = new Location(newZarrPath).getCanonicalPath(); - LOGGER.info("Opening zarr for series {} at path: {}", seriesIndex, canonicalPath); + LOGGER.error("Opening zarr for series {} at path: {}", seriesIndex, canonicalPath); zarrService.open(canonicalPath); currentOpenZarr = seriesIndex; } @@ -670,7 +670,7 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { private void generateArrayKeys(Map attr) { if (uniqueResCounts.size() != 1) { - LOGGER.info("Cannout automatically generate ArrayKeys as resolution counts differ"); + LOGGER.error("Cannout automatically generate ArrayKeys as resolution counts differ"); } Map plates = (Map) attr.get("plate"); if (plates != null) { From bff43c9d8cade861dd5141b90871ec489b219567 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 20 Sep 2023 15:01:35 +0100 Subject: [PATCH 17/37] Add logging for getUsedFiles --- src/loci/formats/in/ZarrReader.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 724776f..7e60480 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1075,6 +1075,8 @@ public String[] getUsedFiles(boolean noPixels) { String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); boolean skipPixels = noPixels || !listPixels(); + LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); + LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { paths.filter(Files::isRegularFile) .forEach(path -> {if (!skipPixels || @@ -1084,7 +1086,7 @@ public String[] getUsedFiles(boolean noPixels) { } catch (IOException e) { e.printStackTrace(); } - + LOGGER.error("ZarrReader returning list of used files of size: {}", usedFiles.size()); String[] fileArr = new String[usedFiles.size()]; fileArr = usedFiles.toArray(fileArr); return fileArr; From ff7b03150669df40174e31682885d4b2d89aa095 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 20 Sep 2023 15:51:24 +0100 Subject: [PATCH 18/37] Check directory when generating group keys and array paths --- src/loci/formats/in/ZarrReader.java | 41 ++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 7e60480..3875896 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -210,11 +210,11 @@ protected void initFile(String id) throws FormatException, IOException { } } LOGGER.error("ZarrReader generating group keys"); - generateGroupKeys(attr); + generateGroupKeys(attr, canonicalPath); // Parse group attributes if (groupKeys.isEmpty()) { - LOGGER.error("ZarrReader adding hroup keys from ZarrService"); + LOGGER.error("ZarrReader adding group keys from ZarrService"); groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); } LOGGER.error("ZarrReader parsing group Keys"); @@ -243,7 +243,7 @@ protected void initFile(String id) throws FormatException, IOException { // Parse array attributes LOGGER.error("ZarrReader attempting to generate Array Keys"); - generateArrayKeys(attr); + generateArrayKeys(attr, canonicalPath); if (arrayPaths.isEmpty()) { LOGGER.error("ZarrReader adding Array Keys from ZarrService"); arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); @@ -668,7 +668,7 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { } } - private void generateArrayKeys(Map attr) { + private void generateArrayKeys(Map attr, String canonicalPath) { if (uniqueResCounts.size() != 1) { LOGGER.error("Cannout automatically generate ArrayKeys as resolution counts differ"); } @@ -684,7 +684,13 @@ private void generateArrayKeys(Map attr) { for (int i = 0; i < fieldCount; i++) { int resolutionCount = (Integer)(uniqueResCounts.toArray())[0]; for (int j = 0; j < resolutionCount; j++) { - arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); + String key = rowName + File.separator + columnName + File.separator + i + File.separator + j; + if (Files.isDirectory(Paths.get(canonicalPath+File.separator+key))) { + arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); + } + else { + LOGGER.error("Skipping array path as sparse data: {}", key); + } } } } @@ -692,7 +698,7 @@ private void generateArrayKeys(Map attr) { } } - private void generateGroupKeys(Map attr) { + private void generateGroupKeys(Map attr, String canonicalPath) { Map plates = (Map) attr.get("plate"); if (plates != null) { ArrayList columns = (ArrayList)plates.get("columns"); @@ -701,12 +707,29 @@ private void generateGroupKeys(Map attr) { for (Object row: rows) { String rowName = ((Map) row).get("name"); - groupKeys.add(rowName); + if (Files.isDirectory(Paths.get(canonicalPath+File.separator+rowName))) { + groupKeys.add(rowName); + } + else { + LOGGER.error("Skipping group key as sparse data: {}", rowName); + } for (Object column: columns) { String columnName = ((Map) column).get("name"); - groupKeys.add(rowName + File.separator + columnName); + String columnKey = rowName + File.separator + columnName; + if (Files.isDirectory(Paths.get(canonicalPath+File.separator+columnKey))) { + groupKeys.add(columnKey); + } + else { + LOGGER.error("Skipping group key as sparse data: {}", columnKey); + } for (int i = 0; i < fieldCount; i++) { - groupKeys.add(rowName + File.separator + columnName + File.separator + i); + String key = rowName + File.separator + columnName + File.separator + i; + if (Files.isDirectory(Paths.get(canonicalPath+File.separator+key))) { + groupKeys.add(key); + } + else { + LOGGER.error("Skipping group key as sparse data: {}", key); + } } } } From aeadb2d1345b8e0500d5f93fc700e04ad3ca68b7 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 20 Sep 2023 17:34:03 +0100 Subject: [PATCH 19/37] Add additional logging to getUsedFiles --- src/loci/formats/in/ZarrReader.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 3875896..e276ec2 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1105,6 +1105,7 @@ public String[] getUsedFiles(boolean noPixels) { .forEach(path -> {if (!skipPixels || (skipPixels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) usedFiles.add(path.toFile().getAbsolutePath()); + LOGGER.error("Adding to the used files list: {}", path.toFile().getAbsolutePath()); }); } catch (IOException e) { e.printStackTrace(); From 8aee5d81bc09f20fd9856498b157d4360e1bb9cb Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 21 Sep 2023 19:38:52 +0100 Subject: [PATCH 20/37] revert getUsedFiles behaviour to remove skipPixels --- src/loci/formats/in/ZarrReader.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index e276ec2..9850dd9 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1097,16 +1097,11 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - boolean skipPixels = noPixels || !listPixels(); - LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { - paths.filter(Files::isRegularFile) - .forEach(path -> {if (!skipPixels || - (skipPixels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) - usedFiles.add(path.toFile().getAbsolutePath()); - LOGGER.error("Adding to the used files list: {}", path.toFile().getAbsolutePath()); - }); + paths.filter(Files::isRegularFile) + .forEach(path -> usedFiles.add(path.toFile().getAbsolutePath())); + } catch (IOException e) { e.printStackTrace(); } From 0f3fcdda1cd046f5bfa5311baa8e899e5e7a87b3 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 26 Sep 2023 13:53:58 +0100 Subject: [PATCH 21/37] Remove unused listPixels option --- src/loci/formats/in/ZarrReader.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 9850dd9..b9778c2 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -44,7 +44,6 @@ import java.util.Hashtable; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; @@ -92,8 +91,6 @@ public class ZarrReader extends FormatReader { public static final boolean QUICK_READ_DEFAULT = true; public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; - public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; - public static final boolean LIST_PIXELS_DEFAULT = false; protected transient ZarrService zarrService; private ArrayList arrayPaths = new ArrayList(); private ArrayList groupKeys = new ArrayList(); @@ -1124,7 +1121,7 @@ public String[] getDomains() { protected ArrayList getAvailableOptions() { ArrayList optionsList = super.getAvailableOptions(); optionsList.add(SAVE_ANNOTATIONS_KEY); - optionsList.add(LIST_PIXELS_KEY); + optionsList.add(QUICK_READ_KEY); return optionsList; } @@ -1137,15 +1134,6 @@ public boolean saveAnnotations() { return SAVE_ANNOTATIONS_DEFAULT; } - public boolean listPixels() { - MetadataOptions options = getMetadataOptions(); - if (options instanceof DynamicMetadataOptions) { - return ((DynamicMetadataOptions) options).getBoolean( - LIST_PIXELS_KEY, LIST_PIXELS_DEFAULT); - } - return LIST_PIXELS_DEFAULT; - } - public boolean quickRead() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { From f2a2fb3dc7e1548377c77cc2ead906a9e1b136b2 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 26 Sep 2023 13:59:08 +0100 Subject: [PATCH 22/37] Update logging statements --- src/loci/formats/in/ZarrReader.java | 36 +++++++++++------------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index b9778c2..9d3c733 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -171,7 +171,7 @@ public int getOptimalTileWidth() { @Override protected void initFile(String id) throws FormatException, IOException { super.initFile(id); - LOGGER.error("ZarrReader attempting to initialize file: {}", id); + LOGGER.debug("ZarrReader attempting to initialize file: {}", id); final MetadataStore store = makeFilterMetadata(); Location zarrFolder = new Location(id); String zarrPath = zarrFolder.getAbsolutePath(); @@ -180,18 +180,16 @@ protected void initFile(String id) throws FormatException, IOException { Location omeMetaFile = new Location( zarrRootPath + File.separator + "OME", "METADATA.ome.xml" ); String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); - LOGGER.error("ZarrReader initializing ZarrService: {}", canonicalPath); initializeZarrService(canonicalPath); ArrayList omeSeriesOrder = new ArrayList(); if(omeMetaFile.exists()) { - LOGGER.error("ZarrReader parsing existing OME-XML"); + LOGGER.debug("ZarrReader parsing existing OME-XML"); parseOMEXML(omeMetaFile, store, omeSeriesOrder); } // Parse base level attributes Map attr = zarrService.getGroupAttr(canonicalPath); int attrIndex = 0; - LOGGER.error("ZarrReader parsing top level group attributes: {}", canonicalPath); if (attr != null && !attr.isEmpty()) { parseResolutionCount(zarrRootPath, "", attr); parseOmeroMetadata(zarrRootPath, attr); @@ -206,15 +204,14 @@ protected void initFile(String id) throws FormatException, IOException { e.printStackTrace(); } } - LOGGER.error("ZarrReader generating group keys"); generateGroupKeys(attr, canonicalPath); // Parse group attributes if (groupKeys.isEmpty()) { - LOGGER.error("ZarrReader adding group keys from ZarrService"); + LOGGER.debug("ZarrReader adding group keys from ZarrService"); groupKeys.addAll(zarrService.getGroupKeys(canonicalPath)); } - LOGGER.error("ZarrReader parsing group Keys"); + List orderedGroupKeys = reorderGroupKeys(groupKeys, omeSeriesOrder); for (String key: orderedGroupKeys) { Map attributes = zarrService.getGroupAttr(canonicalPath+File.separator+key); @@ -239,13 +236,11 @@ protected void initFile(String id) throws FormatException, IOException { } // Parse array attributes - LOGGER.error("ZarrReader attempting to generate Array Keys"); generateArrayKeys(attr, canonicalPath); if (arrayPaths.isEmpty()) { - LOGGER.error("ZarrReader adding Array Keys from ZarrService"); + LOGGER.debug("ZarrReader adding Array Keys from ZarrService"); arrayPaths.addAll(zarrService.getArrayKeys(canonicalPath)); } - LOGGER.error("ZarrReader ordering Array Paths"); orderArrayPaths(zarrRootPath); if (saveAnnotations()) { @@ -274,7 +269,6 @@ protected void initFile(String id) throws FormatException, IOException { int pixelType = -1; for (int i=0; i reorderGroupKeys(ArrayList groupKeys, List originalKeys) { @@ -577,7 +569,7 @@ private void openZarr() { if (seriesIndex != currentOpenZarr) { newZarrPath += File.separator + arrayPaths.get(seriesIndex); String canonicalPath = new Location(newZarrPath).getCanonicalPath(); - LOGGER.error("Opening zarr for series {} at path: {}", seriesIndex, canonicalPath); + LOGGER.debug("Opening zarr for series {} at path: {}", seriesIndex, canonicalPath); zarrService.open(canonicalPath); currentOpenZarr = seriesIndex; } @@ -667,7 +659,7 @@ else if (multiscaleAxes.get(i) instanceof HashMap) { private void generateArrayKeys(Map attr, String canonicalPath) { if (uniqueResCounts.size() != 1) { - LOGGER.error("Cannout automatically generate ArrayKeys as resolution counts differ"); + LOGGER.debug("Cannout automatically generate ArrayKeys as resolution counts differ"); } Map plates = (Map) attr.get("plate"); if (plates != null) { @@ -686,7 +678,7 @@ private void generateArrayKeys(Map attr, String canonicalPath) { arrayPaths.add(rowName + File.separator + columnName + File.separator + i + File.separator + j); } else { - LOGGER.error("Skipping array path as sparse data: {}", key); + LOGGER.debug("Skipping array path as sparse data: {}", key); } } } @@ -708,7 +700,7 @@ private void generateGroupKeys(Map attr, String canonicalPath) { groupKeys.add(rowName); } else { - LOGGER.error("Skipping group key as sparse data: {}", rowName); + LOGGER.debug("Skipping group key as sparse data: {}", rowName); } for (Object column: columns) { String columnName = ((Map) column).get("name"); @@ -717,7 +709,7 @@ private void generateGroupKeys(Map attr, String canonicalPath) { groupKeys.add(columnKey); } else { - LOGGER.error("Skipping group key as sparse data: {}", columnKey); + LOGGER.debug("Skipping group key as sparse data: {}", columnKey); } for (int i = 0; i < fieldCount; i++) { String key = rowName + File.separator + columnName + File.separator + i; @@ -725,7 +717,7 @@ private void generateGroupKeys(Map attr, String canonicalPath) { groupKeys.add(key); } else { - LOGGER.error("Skipping group key as sparse data: {}", key); + LOGGER.debug("Skipping group key as sparse data: {}", key); } } } @@ -1094,7 +1086,6 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { paths.filter(Files::isRegularFile) .forEach(path -> usedFiles.add(path.toFile().getAbsolutePath())); @@ -1102,7 +1093,6 @@ public String[] getUsedFiles(boolean noPixels) { } catch (IOException e) { e.printStackTrace(); } - LOGGER.error("ZarrReader returning list of used files of size: {}", usedFiles.size()); String[] fileArr = new String[usedFiles.size()]; fileArr = usedFiles.toArray(fileArr); return fileArr; From 82ac198fd3c5cc80f6ada1bdb03509fa0294ebb7 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 4 Oct 2023 14:50:47 +0100 Subject: [PATCH 23/37] Revert "Remove unused listPixels option" This reverts commit 0f3fcdda1cd046f5bfa5311baa8e899e5e7a87b3. --- src/loci/formats/in/ZarrReader.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 9d3c733..c77d582 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -44,6 +44,7 @@ import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Stream; import javax.xml.parsers.ParserConfigurationException; @@ -91,6 +92,8 @@ public class ZarrReader extends FormatReader { public static final boolean QUICK_READ_DEFAULT = true; public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; + public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; + public static final boolean LIST_PIXELS_DEFAULT = false; protected transient ZarrService zarrService; private ArrayList arrayPaths = new ArrayList(); private ArrayList groupKeys = new ArrayList(); @@ -1111,7 +1114,7 @@ public String[] getDomains() { protected ArrayList getAvailableOptions() { ArrayList optionsList = super.getAvailableOptions(); optionsList.add(SAVE_ANNOTATIONS_KEY); - optionsList.add(QUICK_READ_KEY); + optionsList.add(LIST_PIXELS_KEY); return optionsList; } @@ -1124,6 +1127,15 @@ public boolean saveAnnotations() { return SAVE_ANNOTATIONS_DEFAULT; } + public boolean listPixels() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).getBoolean( + LIST_PIXELS_KEY, LIST_PIXELS_DEFAULT); + } + return LIST_PIXELS_DEFAULT; + } + public boolean quickRead() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { From b20ef9c57cc236db83a5935713723f87050fbf21 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 4 Oct 2023 14:53:34 +0100 Subject: [PATCH 24/37] Revert "revert getUsedFiles behaviour to remove skipPixels" This reverts commit 8aee5d81bc09f20fd9856498b157d4360e1bb9cb. # Conflicts: # src/loci/formats/in/ZarrReader.java --- src/loci/formats/in/ZarrReader.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index c77d582..5a3b696 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1089,10 +1089,17 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { - paths.filter(Files::isRegularFile) - .forEach(path -> usedFiles.add(path.toFile().getAbsolutePath())); + boolean skipPixels = noPixels || !listPixels(); + LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); + LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); + try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { + paths.filter(Files::isRegularFile) + .forEach(path -> {if (!skipPixels || + (skipPixels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) + usedFiles.add(path.toFile().getAbsolutePath()); + LOGGER.error("Adding to the used files list: {}", path.toFile().getAbsolutePath()); + }); } catch (IOException e) { e.printStackTrace(); } From e3b1d23bf663ca5c0c2cd569b81c32a9bd9c750c Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 23 Oct 2023 16:12:57 +0100 Subject: [PATCH 25/37] Do not assume label images have same resolution levels --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 5a3b696..f80cb6d 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -285,7 +285,7 @@ protected void initFile(String id) throws FormatException, IOException { core.add(ms); boolean openZarr = true; - if (quickRead() && resShapes.containsKey(resolutionIndex)) { + if (quickRead() && resShapes.containsKey(resolutionIndex) && !arrayPaths.get(i).toLowerCase().contains("label")) { openZarr = false; } From 7bba003d7317141612b6a61f0115d70e9aefff95 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 24 Oct 2023 15:50:30 +0100 Subject: [PATCH 26/37] Add new option for handling images in labels folder --- src/loci/formats/in/ZarrReader.java | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index f80cb6d..4ae5b87 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -94,6 +94,8 @@ public class ZarrReader extends FormatReader { public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; public static final boolean LIST_PIXELS_DEFAULT = false; + public static final String INCLUDE_LABELS_KEY = "omezarr.include_labels"; + public static final boolean INCLUDE_LABELS_DEFAULT = false; protected transient ZarrService zarrService; private ArrayList arrayPaths = new ArrayList(); private ArrayList groupKeys = new ArrayList(); @@ -589,7 +591,9 @@ private void orderArrayPaths(String root) { arrayPaths.remove(arrayPath); } for (String arrayPath: resSeries.get(i)) { - arrayPaths.add(arrayPath); + if (includeLabels() || !arrayPath.toLowerCase().contains("labels")) { + arrayPaths.add(arrayPath); + } } } } @@ -1090,13 +1094,17 @@ public String[] getUsedFiles(boolean noPixels) { String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); + boolean skipPixels = noPixels || !listPixels(); + boolean includeLabels = includeLabels(); LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { paths.filter(Files::isRegularFile) - .forEach(path -> {if (!skipPixels || - (skipPixels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) + .forEach(path -> {if ((!skipPixels && includeLabels) || + (!skipPixels && !includeLabels && !path.toString().toLowerCase().contains("labels")) || + (skipPixels && includeLabels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml"))) || + (skipPixels && !includeLabels && !path.toString().toLowerCase().contains("labels") &&(path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) usedFiles.add(path.toFile().getAbsolutePath()); LOGGER.error("Adding to the used files list: {}", path.toFile().getAbsolutePath()); }); @@ -1122,6 +1130,8 @@ protected ArrayList getAvailableOptions() { ArrayList optionsList = super.getAvailableOptions(); optionsList.add(SAVE_ANNOTATIONS_KEY); optionsList.add(LIST_PIXELS_KEY); + optionsList.add(QUICK_READ_KEY); + optionsList.add(INCLUDE_LABELS_KEY); return optionsList; } @@ -1151,5 +1161,14 @@ public boolean quickRead() { } return QUICK_READ_DEFAULT; } + + public boolean includeLabels() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).getBoolean( + INCLUDE_LABELS_KEY, INCLUDE_LABELS_DEFAULT); + } + return INCLUDE_LABELS_DEFAULT; + } } From 290d7678c05d7687f493f8a2e94620539958461f Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 26 Oct 2023 15:35:22 +0100 Subject: [PATCH 27/37] Add support for environment variable OME_ZARR_LIST_PIXELS --- src/loci/formats/in/ZarrReader.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 4ae5b87..675bfe3 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -94,6 +94,7 @@ public class ZarrReader extends FormatReader { public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; public static final boolean LIST_PIXELS_DEFAULT = false; + public static final String LIST_PIXELS_ENV_KEY = "OME_ZARR_LIST_PIXELS"; public static final String INCLUDE_LABELS_KEY = "omezarr.include_labels"; public static final boolean INCLUDE_LABELS_DEFAULT = false; protected transient ZarrService zarrService; @@ -1094,8 +1095,7 @@ public String[] getUsedFiles(boolean noPixels) { String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - - boolean skipPixels = noPixels || !listPixels(); + boolean skipPixels = noPixels || !listPixels() || !systemEnvListPixels(); boolean includeLabels = includeLabels(); LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); @@ -1171,4 +1171,10 @@ public boolean includeLabels() { return INCLUDE_LABELS_DEFAULT; } + private boolean systemEnvListPixels() { + String value = System.getenv(LIST_PIXELS_ENV_KEY); + if (value != null && value.equalsIgnoreCase("true")) return true; + if (value != null && value.toLowerCase().equals("false")) return false; + return LIST_PIXELS_DEFAULT; + } } From 12617b837eed190c86d96ed78739ade8581263d1 Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 26 Oct 2023 15:35:44 +0100 Subject: [PATCH 28/37] Change default value of list pixels to true --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 675bfe3..1d7010b 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -93,7 +93,7 @@ public class ZarrReader extends FormatReader { public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; - public static final boolean LIST_PIXELS_DEFAULT = false; + public static final boolean LIST_PIXELS_DEFAULT = true; public static final String LIST_PIXELS_ENV_KEY = "OME_ZARR_LIST_PIXELS"; public static final String INCLUDE_LABELS_KEY = "omezarr.include_labels"; public static final boolean INCLUDE_LABELS_DEFAULT = false; From f51ae44c006cb0dbdb723c3c28594bb89ec3f21f Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 26 Oct 2023 16:44:55 +0100 Subject: [PATCH 29/37] Add new method to reload options file --- src/loci/formats/in/ZarrReader.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 1d7010b..82052df 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1177,4 +1177,18 @@ private boolean systemEnvListPixels() { if (value != null && value.toLowerCase().equals("false")) return false; return LIST_PIXELS_DEFAULT; } + + private void reloadOptionsFile(String id) { + String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); + if (optionsFile != null) { + MetadataOptions options = getMetadataOptions(); + if (options != null && options instanceof DynamicMetadataOptions) { + try { + ((DynamicMetadataOptions) options).loadOptions(optionsFile, getAvailableOptions()); + } catch (Exception e) { + LOGGER.warn("Exception while attempting to read metadata options file", e); + } + } + } + } } From a7ec0c4e4f5543cd9742611c71c427f131e01e03 Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 30 Oct 2023 15:19:58 +0000 Subject: [PATCH 30/37] Force ZarrReader to read bfoptions file --- src/loci/formats/in/ZarrReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 82052df..7e756a0 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -187,6 +187,7 @@ protected void initFile(String id) throws FormatException, IOException { String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); initializeZarrService(canonicalPath); + reloadOptionsFile(zarrRootPath+".bfoptions"); ArrayList omeSeriesOrder = new ArrayList(); if(omeMetaFile.exists()) { @@ -1094,6 +1095,7 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); + reloadOptionsFile(zarrRootPath+".bfoptions"); boolean skipPixels = noPixels || !listPixels() || !systemEnvListPixels(); boolean includeLabels = includeLabels(); @@ -1181,6 +1183,7 @@ private boolean systemEnvListPixels() { private void reloadOptionsFile(String id) { String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); if (optionsFile != null) { + LOGGER.error("ZarrReader loaded options file from: {}", id); MetadataOptions options = getMetadataOptions(); if (options != null && options instanceof DynamicMetadataOptions) { try { From 4357f54557083e907ebc8c4507b726b352aede42 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 31 Oct 2023 11:54:47 +0000 Subject: [PATCH 31/37] Remove unnecessary bfoptions extension --- src/loci/formats/in/ZarrReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 7e756a0..99899a3 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -187,7 +187,7 @@ protected void initFile(String id) throws FormatException, IOException { String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); initializeZarrService(canonicalPath); - reloadOptionsFile(zarrRootPath+".bfoptions"); + reloadOptionsFile(zarrRootPath); ArrayList omeSeriesOrder = new ArrayList(); if(omeMetaFile.exists()) { @@ -1095,7 +1095,7 @@ public String[] getUsedFiles(boolean noPixels) { FormatTools.assertId(currentId, true, 1); String zarrRootPath = currentId.substring(0, currentId.indexOf(".zarr") + 5); ArrayList usedFiles = new ArrayList(); - reloadOptionsFile(zarrRootPath+".bfoptions"); + reloadOptionsFile(zarrRootPath); boolean skipPixels = noPixels || !listPixels() || !systemEnvListPixels(); boolean includeLabels = includeLabels(); From 6b1da5172b3f85b109dc27ea6edcfe0b8cfe84c3 Mon Sep 17 00:00:00 2001 From: David Gault Date: Fri, 3 Nov 2023 15:44:22 +0000 Subject: [PATCH 32/37] Ensure WellSampleIndex is used when comparing to OME-XML --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 99899a3..2b239fd 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -983,8 +983,8 @@ private void parseOMEXML(Location omeMetaFile, MetadataStore store, ArrayList Date: Mon, 13 Nov 2023 15:25:34 +0000 Subject: [PATCH 33/37] Remove unnecessary logging statements --- src/loci/formats/in/ZarrReader.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 2b239fd..9289737 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1099,8 +1099,6 @@ public String[] getUsedFiles(boolean noPixels) { boolean skipPixels = noPixels || !listPixels() || !systemEnvListPixels(); boolean includeLabels = includeLabels(); - LOGGER.error("ZarrReader getUsed files, skipPixels: {}", skipPixels); - LOGGER.error("ZarrReader fetching list of used files: {}", zarrRootPath); try (Stream paths = Files.walk(Paths.get(zarrRootPath), FileVisitOption.FOLLOW_LINKS)) { paths.filter(Files::isRegularFile) .forEach(path -> {if ((!skipPixels && includeLabels) || @@ -1108,7 +1106,6 @@ public String[] getUsedFiles(boolean noPixels) { (skipPixels && includeLabels && (path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml"))) || (skipPixels && !includeLabels && !path.toString().toLowerCase().contains("labels") &&(path.endsWith(".zgroup") || path.endsWith(".zattrs") || path.endsWith(".xml")))) usedFiles.add(path.toFile().getAbsolutePath()); - LOGGER.error("Adding to the used files list: {}", path.toFile().getAbsolutePath()); }); } catch (IOException e) { e.printStackTrace(); @@ -1183,7 +1180,6 @@ private boolean systemEnvListPixels() { private void reloadOptionsFile(String id) { String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); if (optionsFile != null) { - LOGGER.error("ZarrReader loaded options file from: {}", id); MetadataOptions options = getMetadataOptions(); if (options != null && options instanceof DynamicMetadataOptions) { try { From 30f87f7d3ce51d7482e19543027a60ca921fd6c2 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 15 Nov 2023 17:58:37 +0000 Subject: [PATCH 34/37] Update getRequiredDirectories --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 9289737..0acfdf6 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -123,7 +123,7 @@ public ZarrReader() { public int getRequiredDirectories(String[] files) throws FormatException, IOException { - return FormatTools.getRequiredDirectories(files); + return 1; } /* @see loci.formats.IFormatReader#isThisType(String, boolean) */ From e50eb50c6b0dcbfb663ff8e7873b069bcca75a57 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 5 Dec 2023 11:53:41 +0000 Subject: [PATCH 35/37] Unify options prefixes to omezarr --- src/loci/formats/in/ZarrReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 0acfdf6..a60384d 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -88,9 +88,9 @@ public class ZarrReader extends FormatReader { - public static final String QUICK_READ_KEY = "zarrreader.quick_read"; + public static final String QUICK_READ_KEY = "omezarr.quick_read"; public static final boolean QUICK_READ_DEFAULT = true; - public static final String SAVE_ANNOTATIONS_KEY = "zarrreader.save_annotations"; + public static final String SAVE_ANNOTATIONS_KEY = "omezarr.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; public static final boolean LIST_PIXELS_DEFAULT = true; From 94f707f11318f9c492c12e2423397519d65c94d1 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 5 Dec 2023 11:54:25 +0000 Subject: [PATCH 36/37] Change default behaviour to not include performance changes --- src/loci/formats/in/ZarrReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index a60384d..a4b19bf 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -89,7 +89,7 @@ public class ZarrReader extends FormatReader { public static final String QUICK_READ_KEY = "omezarr.quick_read"; - public static final boolean QUICK_READ_DEFAULT = true; + public static final boolean QUICK_READ_DEFAULT = false; public static final String SAVE_ANNOTATIONS_KEY = "omezarr.save_annotations"; public static final boolean SAVE_ANNOTATIONS_DEFAULT = false; public static final String LIST_PIXELS_KEY = "omezarr.list_pixels"; From d4b1130060e47a7aeee2ad81f7a91a07cbf11701 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 5 Dec 2023 12:12:45 +0000 Subject: [PATCH 37/37] Add some Javadoc for options --- src/loci/formats/in/ZarrReader.java | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index a4b19bf..450706c 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -562,7 +562,6 @@ public void setResolution(int no, boolean openZarr) { } } - private void openZarr() { try { if (currentId != null && zarrService != null) { @@ -1134,6 +1133,10 @@ protected ArrayList getAvailableOptions() { return optionsList; } + /** + * Used to decide if all the zarr metadata is additionally stored as XML annotations + * @return boolean true if all metadata should be saved as an annotation, default is false + */ public boolean saveAnnotations() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { @@ -1143,6 +1146,10 @@ public boolean saveAnnotations() { return SAVE_ANNOTATIONS_DEFAULT; } + /** + * Used to decide if getUsedFiles should list all of the pixel chunks + * @return boolean true if the full list of files including pixels should be returned, default is true + */ public boolean listPixels() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { @@ -1152,6 +1159,11 @@ public boolean listPixels() { return LIST_PIXELS_DEFAULT; } + /** + * Used to decide if performance improvements are applied during initialization + * This makes assumptions about the data, assuming that the shape of images remains consistent + * @return boolean true if performance improvements should be applied, default is false + */ public boolean quickRead() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { @@ -1160,7 +1172,11 @@ public boolean quickRead() { } return QUICK_READ_DEFAULT; } - + + /** + * Used to decide if images stored in the label sub folder should be included in the list of images + * @return boolean true if images in the label folder should be included, default is false + */ public boolean includeLabels() { MetadataOptions options = getMetadataOptions(); if (options instanceof DynamicMetadataOptions) { @@ -1177,6 +1193,11 @@ private boolean systemEnvListPixels() { return LIST_PIXELS_DEFAULT; } + /** + * Reloads the bfoptions file so that the options are able to be read for each getUsedFiles + * Otherwise the options are read when initialised and saved as part of the memo file + * @param id of the options file to reload + */ private void reloadOptionsFile(String id) { String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); if (optionsFile != null) {