diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..7cbb3e98 --- /dev/null +++ b/.gitignore @@ -0,0 +1,97 @@ +*/.DS_Store +**/__pycache__/* +*/index.html +_build +_bikeshed +.tox +.vscode +.*plist + +# do not sync auto-generated md files +docs/examples/ +docs/schemas/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +*_version.py + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +.napari_cache + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Sphinx documentation +docs/_build/ + +# MkDocs documentation +/site/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# OS +.DS_Store + +# written by setuptools_scm +*/_version.py + +# PyBuilder +.idea +venv/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..cc67767f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,19 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.13" + jobs: + pre_build: + # Generate the Sphinx configuration for this Jupyter Book so it builds. + - "jupyter-book config sphinx ." + - "python ./docs/pre_build.py" + +python: + install: + - requirements: docs/requirements.txt + + +sphinx: + configuration: conf.py \ No newline at end of file diff --git a/_config.yml b/_config.yml new file mode 100644 index 00000000..8feb92b1 --- /dev/null +++ b/_config.yml @@ -0,0 +1,83 @@ +# Book settings +# Learn more at https://jupyterbook.org/customize/config.html +# Comprehensive example: https://github.com/executablebooks/jupyter-book/blob/master/docs/_config.yml + +title: Next-generation file format specification +author: NGFF-community +logo: https://www.openmicroscopy.org/img/logos/ome-logomark.svg + +# Force re-execution of notebooks on each build. +# See https://jupyterbook.org/content/execute.html +# execute: +# execute_notebooks: off + +# Define the name of the latex output file for PDF builds +latex: + latex_documents: + targetname: book.tex + +# Add a bibtex file so that we can create citations +bibtex_bibfiles: + - references.bib + +# Information about where the book exists on the web +repository: + url: https://github.com/ome/ngff-spec # Online location of your book + path_to_book: docs # Optional path to your book, relative to the repository root + branch: main # Which branch of the repository should be used when creating links (optional) + +# Add GitHub buttons to your book +# See https://jupyterbook.org/customize/config.html#add-a-link-to-your-repository +html: + use_issues_button: true + use_repository_button: true + extra_footer: | +
+ Copyright © 2020- + OME® + (U. Dundee). + OME trademark rules apply. +
+ + +parse: + myst_heading_anchors: 3 + +myst: + enable_extensions: + - colon_fence + - deflist + - dollarmath + - linkify + - substitution + + +sphinx: + extra_extensions: + - sphinx.ext.autosummary + - sphinx.ext.autodoc + - sphinx.ext.napoleon # Enable support for NumPy and Google style docstrings + - sphinx.ext.intersphinx + - sphinx_inline_tabs + - sphinx_proof + - sphinx_examples + config: + html_show_copyright: false + add_module_names: True + autosummary_generate: True + + myst_substitutions: + title: "OME-Zarr specification" + shortname: "ome-zarr" + level: "1" + status: "CG-FINAL" + tr_url: "https://ngff.openmicroscopy.org/0.5/" + url: "https://ngff.openmicroscopy.org/0.6-dev/" + issue_tracking: "[Forum](https://forum.image.sc/tag/ome-ngff), [Github](https://github.com/ome/ngff/issues)" + logo: "http://www.openmicroscopy.org/img/logos/ome-logomark.svg" + editor_name: "[Josh Moore](https://gerbi-gmb.de/)" + editor_affiliation: "German BioImaging e.V." + editor_orcid: "[https://orcid.org/0000-0003-4028-811X](https://orcid.org/0000-0003-4028-811X)" + version: "0.6.dev1" \ No newline at end of file diff --git a/_toc.yml b/_toc.yml new file mode 100644 index 00000000..9df6edc7 --- /dev/null +++ b/_toc.yml @@ -0,0 +1,17 @@ +# Table of contents +# Learn more at https://jupyterbook.org/customize/toc.html + +format: jb-book +root: specification + +chapters: +- file: docs/examples + sections: + - glob: docs/examples/* +- file: docs/schemas/index + sections: + - glob: docs/schemas/markdown/* + +- file: contribute +- file: citing +- file: version_history \ No newline at end of file diff --git a/citing.md b/citing.md new file mode 100644 index 00000000..3e4777d0 --- /dev/null +++ b/citing.md @@ -0,0 +1,13 @@ +Citing +====== +(citing)= + +[Next-generation file format (NGFF) specifications for storing bioimaging data in the cloud.](https://ngff.openmicroscopy.org/0.4) +J. Moore, *et al*. Open Microscopy Environment Consortium, 8 February 2022. + +This edition of the specification is [https://ngff.openmicroscopy.org/{{version}}](https://ngff.openmicroscopy.org/{{version}}]). +The latest edition is publically available {cite}`ngff_spec_latest`. + +```{bibliography} +:style: unsrt +``` \ No newline at end of file diff --git a/contribute.md b/contribute.md new file mode 100644 index 00000000..ce0777a4 --- /dev/null +++ b/contribute.md @@ -0,0 +1,83 @@ +# Contribution guide + +Contributions to the spec text, examples and schemas are highly welcome and +appreciated by the ngff community. If you propose an RFC (i.e., major change) +or a aminor change (pull request), please make sure to follow [these guidelines](https://ngff.openmicroscopy.org/contributing/index.html). + +Major changes should follow the RFC process as it was laid out in [RFC1 ](https://ngff.openmicroscopy.org/rfc/1/index.html). + +## Building the documentation + +Build and inspect changes to the documentation before submitting a PR. To do so, you first need +to install the necessary dependecies: + +```bash +pip install -r docs/requirements.txt +``` + +This document uses [jupyter-book](https://jupyterbook.org) to generate the pages and [MyST](https://mystmd.org) +markdown for formatting. After installing these via the dependencies, navigate into the repository on your machine +and build the book using the following command: + +```bash +python docs/pre_build.py +jupyter-book build . --path-output docs +``` + +You'll find the built webpages under `docs/_build/html`. + +### Formating hints + +The specification uses MyST extensively for a number of formatting options to make the text +readible and improve structure. + +#### Referencing + +MyST allows a number of ways to reference and cross-reference inside this text and across +several of the pages in this repo. For an overview of supported referencing syntax, see the +[MyST doc pages](https://mystmd.org/guide/cross-references). It is recommended to use the +following syntax in this document for consitency: +``` +anchor: (your-reference-name)= +reference: [This is a reference](your-reference-name) +``` + +#### Admonitions + +We suggest using [admonitions](https://mystmd.org/guide/admonitions) for example code and +other highlighting. For examples, please use the following syntax to highlight your examples: + +`````markdown +````{admonition} Example + +Some informative text about your example +```json +"key": "value" +``` +```` +````` + +which results in + +````{admonition} Example + +Some informative text about your example +```json +"key": "value" +``` +```` + +If you want to link in example metadata from somewhere in this repo (i.e, a json file), +use this syntax: + +`````markdown +````{admonition} Example + +Some informative text about your example +```{literalinclude} path/to/example.json +:language: json +``` +```` +````` + +Other useful admonitions (e.g., `hint`, `note`) can be found [here](https://mystmd.org/guide/directives) diff --git a/copyright.include b/copyright.include deleted file mode 100644 index f0def708..00000000 --- a/copyright.include +++ /dev/null @@ -1,4 +0,0 @@ -Copyright © 2020-[YEAR] -OME® -(U. Dundee). -OME trademark rules apply. diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 00000000..d7c1c369 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,3 @@ +# JSON Examples + +This document contains JSON examples for ngff-compliant metadata layouts. \ No newline at end of file diff --git a/docs/pre_build.py b/docs/pre_build.py new file mode 100644 index 00000000..873ea0dc --- /dev/null +++ b/docs/pre_build.py @@ -0,0 +1,132 @@ +# %% +import os +import json +import glob +from pathlib import Path +import jsonc as json +import yaml + +# change working directory to the location of this script +os.chdir(os.path.dirname(os.path.abspath(__file__))) + + +def build_json_examples(): + """Build markdown files from json examples.""" + # glob recursively to find all json files + input_directory = '../examples' + output_directory = 'examples' + os.makedirs(output_directory, exist_ok=True) + example_types = os.listdir(input_directory) + + for example in example_types: + json_files = glob.glob(os.path.join(input_directory, example, '*.json'), recursive=True) + markdown_file_name = os.path.join(output_directory, f'{example}.md') + + # add header + markdown_content = f"""# {example}\n\n + +This document contains JSON examples for {example} metadata layouts. + +""" + + # append each json file content + for json_file in json_files: + print(f'Processing {json_file}...') + with open(json_file, 'r') as file: + json_data = json.load(file) + json_str = json.dumps(json_data, indent=4) + json_file_name = Path(json_file).stem + + # Create the Markdown content + markdown_content += f""" +## {os.path.splitext(json_file_name)[0]} +(examples:{example}:{Path(json_file).stem})= + +```{{code-block}} json +:caption: {json_file_name} +:linenos: + +{json_str} +``` +""" + # create + with open(markdown_file_name, 'w') as md_file: + md_file.write(markdown_content) + +def build_json_schemas(): + from json_schema_for_humans.generate import generate_from_filename + from json_schema_for_humans.generation_configuration import GenerationConfiguration + + schema_source_dir = '../schemas' + output_directory = 'schemas' + os.makedirs(output_directory, exist_ok=True) + schema_files = glob.glob(os.path.join(schema_source_dir, '*.schema'), recursive=True) + + index_markdown = """# JSON Schemas + +This section contains JSON schemas for various metadata layouts. +Find below links to autogenerated markdown pages or interactive HTML pages for each schema. + +| Schema | Markdown | HTML | +|--------|----------|------| +""" + + for schema_file in schema_files: + if 'strict' in schema_file: + continue # skip strict schemas + + print(f'Processing {schema_file}...') + output_path_md = os.path.join(output_directory, "markdown", f"{Path(schema_file).stem}" + ".md") + output_path_html = os.path.join(output_directory, "html", f"{Path(schema_file).stem}" + ".html") + os.makedirs(os.path.dirname(output_path_md), exist_ok=True) + os.makedirs(os.path.dirname(output_path_html), exist_ok=True) + + # Generate the documentation + try: + config_md = GenerationConfiguration( + template_name='md', + with_footer=True, + show_toc=False, + link_to_reused_ref=False) + generate_from_filename( + os.path.abspath(schema_file), + result_file_name=os.path.abspath(output_path_md), + config=config_md + ) + + # insert mySt crossreference at top of markdown files + with open(output_path_md, 'r') as md_file: + md_content = md_file.read() + crossref = f"(schemas:{Path(schema_file).stem})" + md_content = f"{crossref}\n\n{md_content}" + with open(output_path_md, 'w') as md_file: + md_file.write(md_content) + + link_markdown = f"[{Path(schema_file).stem}]{crossref}" + except Exception as e: + link_markdown = "" + + try: + config_html = GenerationConfiguration( + template_name='js', + with_footer=True, + show_toc=False, + link_to_reused_ref=False) + + generate_from_filename( + os.path.abspath(schema_file), + result_file_name=os.path.abspath(output_path_html), + config=config_html + ) + link_html = f"[{Path(schema_file).stem}]({output_path_html})" + + except Exception as e: + link_html = "" + + index_markdown += f"| {Path(schema_file).stem} | {link_markdown} | {link_html} |\n" + + with open(os.path.join(output_directory, "index.md"), 'w') as index_file: + index_file.write(index_markdown) + +build_json_examples() +build_json_schemas() diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..cfc70258 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,10 @@ +sphinx +sphinx-book-theme +sphinx-proof +sphinx-inline-tabs +sphinx-examples +json-schema-for-humans +myst-parser +json_with_comments +jupyter-book +jsonschema_markdown \ No newline at end of file diff --git a/examples/multiscales_strict/multiscales_example.json b/examples/multiscales_strict/multiscales_example.json index 4b2c718a..55799eb2 100644 --- a/examples/multiscales_strict/multiscales_example.json +++ b/examples/multiscales_strict/multiscales_example.json @@ -3,16 +3,21 @@ "node_type": "group", "attributes": { "ome": { - "version": "0.5", + "version": "0.6dev2", "multiscales": [ { - "name": "example", - "axes": [ - { "name": "t", "type": "time", "unit": "millisecond" }, - { "name": "c", "type": "channel" }, - { "name": "z", "type": "space", "unit": "micrometer" }, - { "name": "y", "type": "space", "unit": "micrometer" }, - { "name": "x", "type": "space", "unit": "micrometer" } + "name": "physical", + "coordinateSystems": [ + { + "name": "physical", + "axes": [ + { "name": "t", "type": "time", "unit": "millisecond" }, + { "name": "c", "type": "channel" }, + { "name": "z", "type": "space", "unit": "micrometer" }, + { "name": "y", "type": "space", "unit": "micrometer" }, + { "name": "x", "type": "space", "unit": "micrometer" } + ] + } ], "datasets": [ { @@ -20,8 +25,11 @@ "coordinateTransformations": [ { // the voxel size for the first scale level (0.5 micrometer) + // the time unit (0.1 milliseconds), which is the same for each scale level "type": "scale", - "scale": [1.0, 1.0, 0.5, 0.5, 0.5] + "scale": [0.1, 1.0, 0.5, 0.5, 0.5], + "input": "0", + "output": "physical" } ] }, @@ -29,9 +37,12 @@ "path": "1", "coordinateTransformations": [ { - // the voxel size for the second scale level (downscaled by a factor of 2 -> 1 micrometer) + // the voxel size for the second scale level (1 micrometer) + // the time unit (0.1 milliseconds), which is the same for each scale level "type": "scale", - "scale": [1.0, 1.0, 1.0, 1.0, 1.0] + "scale": [0.1, 1.0, 1.0, 1.0, 1.0], + "input": "1", + "output": "physical" } ] }, @@ -39,18 +50,14 @@ "path": "2", "coordinateTransformations": [ { - // the voxel size for the third scale level (downscaled by a factor of 4 -> 2 micrometer) + // the voxel size for the third scale level (2 micrometer) + // the time unit (0.1 milliseconds), which is the same for each scale level "type": "scale", - "scale": [1.0, 1.0, 2.0, 2.0, 2.0] + "scale": [0.1, 1.0, 2.0, 2.0, 2.0], + "input": "2", + "output": "physical" } - ] - } - ], - "coordinateTransformations": [ - { - // the time unit (0.1 milliseconds), which is the same for each scale level - "type": "scale", - "scale": [0.1, 1.0, 1.0, 1.0, 1.0] + ] } ], "type": "gaussian", diff --git a/header.include b/header.include deleted file mode 100644 index 7bd9e500..00000000 --- a/header.include +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - [TITLE] - - - -
- - OME logo (6 circles in a hexagon) - -

[TITLE]

-

[LONGSTATUS], -

-
-
- -
-
- -
- -

Status of this document

-
-
- - -
diff --git a/references.bib b/references.bib new file mode 100644 index 00000000..90d674c8 --- /dev/null +++ b/references.bib @@ -0,0 +1,24 @@ +@article{moore2021ome, + title={OME-NGFF: a next-generation file format for expanding bioimaging data-access strategies}, + author={Moore, Josh and Allan, Chris and Besson, S{\'e}bastien and Burel, Jean-Marie and Diel, Erin and Gault, David and Kozlowski, Kevin and Lindner, Dominik and Linkert, Melissa and Manz, Trevor and others}, + journal={Nature methods}, + volume={18}, + number={12}, + pages={1496--1498}, + year={2021}, + publisher={Nature Publishing Group US New York} +} + +@software{ngff_spec_latest, + author = {Josh Moore}, + title = {ome/ngff: Next-generation file format (NGFF) + specifications for storing bioimaging data in the + cloud. + }, + month = nov, + year = 2020, + publisher = {Zenodo}, + version = {0.0.1}, + doi = {10.5281/zenodo.4282107}, + url = {https://ngff.openmicroscopy.org/latest/}, +} \ No newline at end of file diff --git a/schemas/coordinateTransformations.schema b/schemas/coordinateTransformations.schema deleted file mode 100644 index 8bd35f4f..00000000 --- a/schemas/coordinateTransformations.schema +++ /dev/null @@ -1,296 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://ngff.openmicroscopy.org/latest/schemas/coordinateTransformations.schema", - "title": "NGFF Coordinate Transformations", - "description": "JSON from OME-NGFF .zattrs", - "type": "array", - "minItems": 1, - "contains": { - "type": "object" - }, - "maxContains": 1, - "items": { - "$ref": "#/$defs/transformation" - }, - "$defs": { - "transformation" : { - "oneOf": [ - { - "$ref": "#/$defs/scaleTransformation" - }, - { - "$ref": "#/$defs/translationTransformation" - }, - { - "$ref": "#/$defs/affineTransformation" - }, - { - "$ref": "#/$defs/rotationTransformation" - }, - { - "$ref": "#/$defs/inverseOfTransformation" - }, - { - "$ref": "#/$defs/sequenceTransformation" - }, - { - "$ref": "#/$defs/coordinatesTransformation" - }, - { - "$ref": "#/$defs/displacementsTransformation" - }, - { - "$ref": "#/$defs/byDimensionTransformation" - }, - { - "$ref": "#/$defs/bijectionTransformation" - } - ] - }, - "scaleTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "scale" - ] - }, - "scale": { - "type": "array", - "minItems": 2, - "items": { - "type": "number" - } - }, - "path": { - "type": "string" - } - }, - "required": [ - "type" - ], - "oneOf": [ - { - "required": ["scale"] - }, - { - "required": ["path"] - } - ] - }, - "translationTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "translation" - ] - }, - "translation": { - "type": "array", - "minItems": 2, - "items": { - "type": "number" - } - }, - "path": { - "type": "string" - } - }, - "required": [ - "type" - ], - "oneOf": [ - { - "required": ["translation"] - }, - { - "required": ["path"] - } - ] - }, - "affineTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "affine" - ] - }, - "affine": { - "type": "array", - "minItems": 2, - "items": { - "type": "number" - } - }, - "path": { - "type": "string" - } - }, - "required": [ - "type" - ], - "oneOf": [ - { - "required": ["affine"] - }, - { - "required": ["path"] - } - ] - }, - "rotationTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "rotation" - ] - }, - "rotation": { - "type": "object", - "minItems": 2, - "items": { - "type": "number" - } - }, - "path": { - "type": "string" - } - }, - "required": [ - "type" - ], - "oneOf": [ - { - "required": ["rotation"] - }, - { - "required": ["path"] - } - ] - }, - "inverseOfTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "inverseOf" - ] - }, - "transformation": { - "type": "object" - } - }, - "required": [ - "type", "transformation" - ] - }, - "sequenceTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "sequence" - ] - }, - "transformations": { - "type": "array" - } - }, - "required": [ - "type", "transformations" - ] - }, - "coordinatesTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "coordinates" - ] - }, - "path": { - "type": "string" - }, - "interpolation": { - "type": "string", - "enum": [ - "nearest", "linear", "cubic" - ] - } - }, - "required": [ - "type", "path", "interpolation" - ] - }, - "displacementsTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "displacements" - ] - }, - "path": { - "type": "string" - }, - "interpolation": { - "type": "string", - "enum": [ - "nearest", "linear", "cubic" - ] - } - }, - "required": [ - "type", "path", "interpolation" - ] - }, - "byDimensionTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "byDimension" - ] - }, - "transformations": { - "type": "array" - } - }, - "required": [ - "type", "transformations" - ] - }, - "bijectionTransformation": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "bijection" - ] - }, - "forward": { - "type": "object" - }, - "inverse": { - "type": "object" - } - }, - "required": [ - "type", "forward", "inverse" - ] - } - } -} diff --git a/index.bs b/specification.md similarity index 81% rename from index.bs rename to specification.md index 6211d398..4e88b3dc 100644 --- a/index.bs +++ b/specification.md @@ -1,36 +1,34 @@ -
-Title: OME-Zarr specification
-Shortname: ome-zarr
-Level: 1
-Status: w3c/CG-FINAL
-TR: https://ngff.openmicroscopy.org/0.5/
-URL: https://ngff.openmicroscopy.org/0.6-dev/
-Repository: https://github.com/ome/ngff
-Issue Tracking: Forums https://forum.image.sc/tag/ome-ngff
-Logo: http://www.openmicroscopy.org/img/logos/ome-logomark.svg
-Local Boilerplate: header yes, copyright yes
-Boilerplate: style-darkmode off
-Markup Shorthands: markdown yes
-Editor: Josh Moore, German BioImaging e.V., https://gerbi-gmb.de/, https://orcid.org/0000-0003-4028-811X
-Text Macro: NGFFVERSION 0.5
-Abstract: This document contains next-generation file format (NGFF)
-Abstract: specifications for storing bioimaging data in the cloud.
-Abstract: All specifications are submitted to the https://image.sc community for review.
-Status Text: The current released version of this specification is 0.5. Migration scripts
-Status Text: will be provided between numbered versions. Data written with these latest changes
-Status Text: (an "editor's draft") will not necessarily be supported.
-
+# {{ title }} + +**Version**: {{version}}
+**Shortname:** {{ shortname }}
+**Level:** {{ level }}
+**Status:** {{ status }}
+ +**Feedback:** {{ issue_tracking }}
+ +**Editor:** {{ editor_name }}, ({{ editor_affiliation }}), {{ editor_orcid }} + +## Abstract + +```{warning} +This is **not** the released version of the ngff-specification. It is an explorative, work-in-progress document. + +``` + +This document contains next-generation file format (NGFF) specifications for storing bioimaging data in the cloud. All specifications are submitted to the https://image.sc community for review. + +## Status of This Document + +The current released version of this specification is 0.5. Migration scripts will be provided between numbered versions. Data written with these latest changes (an "editor's draft") will not necessarily be supported. -OME-Zarr specification {#ome-zarr} ----------------------------------- The conventions and specifications defined in this document are designed to enable next-generation file formats to represent the same bioimaging data that can be represented in \[OME-TIFF](http://www.openmicroscopy.org/ome-files/) and beyond. -Document conventions --------------------- +## Document conventions The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL NOT”, “SHOULD”, “SHOULD NOT”, “RECOMMENDED”, “MAY”, and “OPTIONAL” are to be interpreted as described in @@ -41,14 +39,13 @@ The key words “MUST”, “MUST NOT”, “REQUIRED”, “SHALL”, “SHALL intention of removing it in the future. Implementations may be expected (MUST) or encouraged (SHOULD) to support the reading of the data, but writing will usually be optional (MAY). Examples of transitional metadata include custom additions by -implementations that are later submitted as a formal specification. (See [[#bf2raw]]) +implementations that are later submitted as a formal specification. (See [bioformats2raw](bf2raw))

Some of the JSON examples in this document include comments. However, these are only for clarity purposes and comments MUST NOT be included in JSON objects. -Storage format {#storage-format} -================================ +# Storage format OME-Zarr is implemented using the Zarr format as defined by the [version 3 of the Zarr specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html). @@ -62,14 +59,13 @@ is represented here as it would appear locally but could equally be stored on a web server to be accessed via HTTP or in object storage like S3 or GCS. -Images {#image-layout} ----------------------- +## Images The following layout describes the expected Zarr hierarchy for images with multiple levels of resolutions and optionally associated labels. -Note that the number of dimensions is variable between 2 and 5 and that axis names are arbitrary, see [[#multiscale-md]] for details. +Note that the number of dimensions is variable between 2 and 5 and that axis names are arbitrary, see [multiscales metadata](multiscales-md) for details. -
+```
 ├── 123.zarr                  # One OME-Zarr image (id=123).
 │   ...
 │
@@ -107,23 +103,22 @@ Note that the number of dimensions is variable between 2 and 5 and that axis nam
                 ├── 0         # Each multiscale level is stored as a separate Zarr array, as above, but only integer values
                 └── ...       # are supported.
                 
-
+``` -High-content screening {#hcs-layout} ------------------------------------- +## High-content screening The following specification defines the hierarchy for a high-content screening dataset. Three groups MUST be defined above the images: - the group above the images defines the well and MUST implement the - [well specification](#well-md). All images contained in a well are fields + [well specification](well-md). All images contained in a well are fields of view of the same well - the group above the well defines a row of wells - the group above the well row defines an entire plate i.e. a two-dimensional collection of wells organized in rows and columns. It MUST implement the - [plate specification](#plate-md) + [plate specification](plate-md) A well row group SHOULD NOT be present if there are no images in the well row. A well group SHOULD NOT be present if there are no images in the well. @@ -151,8 +146,8 @@ A well group SHOULD NOT be present if there are no images in the well. └── ... # Other rows -OME-Zarr Metadata {#metadata} -============================= +# OME-Zarr Metadata +(metadata)= The "OME-Zarr Metadata" contains metadata keys as specified below for discovering certain types of data, especially images. @@ -164,7 +159,7 @@ The version of the OME-Zarr Metadata is denoted as a string in the `version` att The OME-Zarr Metadata version MUST be consistent within a hierarchy. -```json +
 {
   ...
   "attributes": {
@@ -174,13 +169,13 @@ The OME-Zarr Metadata version MUST be consistent within a hierarchy.
     }
   }
 }
-```
+
-"axes" metadata {#axes-md} --------------------------- +## "axes" metadata +(axes-md)= "axes" describes the dimensions of a coordinate systems and adds an interpretation to the data along that dimension. A named collection -of axes forms a coordinate system ([[#coord-sys-md]]) (see below). +of axes forms a [coordinate system](coord-sys-md). It is a list of dictionaries, where each dictionary describes a dimension (axis) and: - MUST contain the field "name" that gives the name for this dimension. The values MUST be unique across all "name" fields. - SHOULD contain the field "type". It SHOULD be one of the strings "array", "space", "time", "channel", "coordinate", or @@ -190,11 +185,11 @@ It is a list of dictionaries, where each dictionary describes a dimension (axis) - Units for "time" axes: 'attosecond', 'centisecond', 'day', 'decisecond', 'exasecond', 'femtosecond', 'gigasecond', 'hectosecond', 'hour', 'kilosecond', 'megasecond', 'microsecond', 'millisecond', 'minute', 'nanosecond', 'petasecond', 'picosecond', 'second', 'terasecond', 'yoctosecond', 'yottasecond', 'zeptosecond', 'zettasecond' - MAY contain the field "longName". The value MUST be a string, and can provide a longer name or description of an axis and its properties. -The "axes" are used as part of [[#multiscale-md]]. The length of "axes" MUST be equal to the number of dimensions of the arrays that contain the image data. +The "axes" are used as part of [multiscales metadata](multiscales-md). The length of "axes" MUST be equal to the number of dimensions of the arrays that contain the image data. The "dimension_names" attribute MUST be included in the `zarr.json` of the Zarr array of a multiscale level and MUST match the names in the "axes" metadata. -
+````{admonition} Example Examples of valid axes: @@ -209,7 +204,7 @@ Examples of valid axes: {"name": "freq", "type": "frequency", "unit": "megahertz"} ] ``` -
+```` Arrays are inherently discrete (see Array coordinate systems, below) but are often used to store discrete samples of a continuous variable. The continuous values "in between" discrete samples can be retrieved using an *interpolation* method. If an @@ -222,11 +217,11 @@ Note: The most common methods for interpolation are "nearest neighbor", "linear" to any method that obtains values at real valued coordinates using discrete samples as an "interpolator". As such, label images may be interpolated using "nearest neighbor" to obtain labels at points along the continuum. -
+````{admonition} Example For the coordinate system: -```json +``` { "name" : "index and interpolation", "axes" : [ @@ -240,17 +235,20 @@ For the coordinate system: Indexing an image at the point `(0.1, 0.2, 0.3, 0.4)` is not valid, because the value of the first coordinate (`0.1`) refers to the discrete axis `"c"`. Indexing an image at the point `(1, 0.2, 0.3, 0.4)` is valid. -
+```` -"coordinateSystems" metadata {#coord-sys-md} --------------------------- + +## "coordinateSystems" metadata +(coord-sys-md)= A "coordinate system" is a collection of "axes" / dimensions with a name. Every coordinate system: - MUST contain the field "name". The value MUST be a non-empty string that is unique among `coordinateSystem`s. - MUST contain the field "axes", whose value is an array of valid "axes". -
+````{admonition} Example + +Example of valid `coordinateSystems` metadata: ```json { @@ -262,7 +260,7 @@ A "coordinate system" is a collection of "axes" / dimensions with a name. Every ] } ``` -
+```` The order of the `"axes"` list matters and defines the index of each array dimension and coordinates for points in that coordinate system. For the above example, the `"x"` dimension is the last dimension. The "dimensionality" of a coordinate system @@ -275,7 +273,7 @@ point refer to different physical entities and therefore should not be analyzed regions of interest, etc., SHOULD ensure that they are in the same coordinate system (same name, with identical axes) or can be transformed to the same coordinate system before doing analysis. See the example below. -
+````{admonition} Example Two instruments simultaneously image the same sample from two different angles, and the 3D data from both instruments are calibrated to "micrometer" units. Two samples are collected ("sampleA" and "sampleB"). An analysis of sample A requires @@ -285,7 +283,7 @@ collected at different angles, a measurement by instrument 1 at the point with c measurement at the same point in instrument 2 (i.e., it may not be the same physical location in the sample). To analyze both images together, they must be in the same coordinate system. -The set of coordinate transformations ([[#trafo-md]]) encodes relationships between coordinate systems, specifically, how to +The set of [coordinate transformations](trafo-md) encodes relationships between coordinate systems, specifically, how to convert points and images to different coordinate systems. Implementations can apply the coordinate transform to images or points in coordinate system "sampleA_instrument2" to bring them into the "sampleA_instrument1" coordinate system. In this case, the ROI should be transformed to the "sampleA_image1" coordinate system, then used for quantification with the instrument 1 @@ -320,7 +318,7 @@ image. ] ``` -
+```` ### Array coordinate systems @@ -333,7 +331,8 @@ choice explicitly will be important for interoperability. This is possible by us Every array has a default coordinate system whose parameters need not be explicitly defined. Its name is the path to the array in the container, its axes have `"type":"array"`, are unitless, and have default "name"s. The ith axis has `"name":"dim_i"` (these are the same default names used by [xarray](https://docs.xarray.dev/en/stable/user-guide/terminology.html)). -
+ +````{admonition} Example For example, a 3D array at path `0` defines the coordinate system: ```json @@ -348,7 +347,8 @@ For example, a 3D array at path `0` defines the coordinate system: ``` though this object should not and need not explicitly appear in metadata. -
+ +```` The dimensionality of each array coordinate system equals the dimensionality of its corresponding zarr array. The axis with @@ -357,11 +357,11 @@ attribute in the zarr array attributes, and whose data depends on the byte order chunks. As described in the [zarr array metadata](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#array-metadata), the last dimension of an array in "C" order are stored contiguously on disk or in-memory when directly loaded. -
+````{admonition} Example For example, if `0/zarr.json` contains: -```json +``` { "zarr_format": 3, "node_type": "array", @@ -371,23 +371,22 @@ For example, if `0/zarr.json` contains: ``` Then `dim_0` has length 4, `dim_1` has length 3, and `dim_2` has length 5. -
+```` The name and axes names MAY be customized by including a `arrayCoordinateSystem` field in the user-defined attributes of the array whose value is a coordinate system object. The length of `axes` MUST be equal to the dimensionality. The value of `"type"` for each object in the axes array MUST equal `"array"`. -
+````{admonition} Example -
-path: examples/coordSystems/arrayCoordSys.json
-highlight: json
-
+ +```{literalinclude} examples/coordSystems/arrayCoordSys.json +``` Note that dimension `i` is contiguous in memory. -
+```` ### Coordinate convention @@ -402,9 +401,8 @@ system `(0.0, 0.0)` (when the transformation is the identity). The continuous re half-open interval `[-0.5, 0.5) x [-0.5, 0.5)` (i.e., -0.5 is included, +0.5 is excluded). See chapter 4 and figure 4.1 of the ITK Software Guide [[itk]]. - -"bioformats2raw.layout" (transitional) {#bf2raw} ------------------------------------------------- +## bioformats2raw.layout +(bf2raw)= [=Transitional=] "bioformats2raw.layout" metadata identifies a group which implicitly describes a series of images. The need for the collection stems from the common "multi-image file" scenario in microscopy. Parsers like Bio-Formats @@ -414,11 +412,12 @@ In order to capture that information within an OME-Zarr dataset, `bioformats2raw The bioformats2raw layout has been added to v0.4 as a transitional specification to specify filesets that already exist in the wild. An upcoming NGFF specification will replace this layout with explicit metadata. -

Layout

+### Layout +(bf2raw-layout)= Typical Zarr layout produced by running `bioformats2raw` on a fileset that contains more than one image (series > 1): -
+```
 series.ome.zarr               # One converted fileset from bioformats2raw
     ├── zarr.json             # Contains "bioformats2raw.layout" metadata
     ├── OME                   # Special group for containing OME metadata
@@ -427,33 +426,32 @@ series.ome.zarr               # One converted fileset from bioformats2raw
     ├── 0                     # First image in the collection
     ├── 1                     # Second image in the collection
     └── ...
-
+``` -

Attributes

+### bf2raw-attributes +(bf2raw-attributes)= The OME-Zarr Metadata in the top-level `zarr.json` file must contain the `bioformats2raw.layout` key: -
-path: examples/bf2raw/image.json
-highlight: json
-
+ +```{literalinclude} examples/bf2raw/image.json +:language: json +``` If the top-level group represents a plate, the `bioformats2raw.layout` metadata will be present but -the "plate" key MUST also be present, takes precedence and parsing of such datasets should follow [[#plate-md]]. It is not +the "plate" key MUST also be present, takes precedence and parsing of such datasets should follow (see [plate metadata](plate-md)). It is not possible to mix collections of images with plates at present. -
-path: examples/bf2raw/plate.json
-highlight: json
-
+```{literalinclude} examples/bf2raw/plate.json +``` The OME-Zarr Metadata in the `zarr.json` file within the OME group may contain the "series" key: -
-path: examples/ome/series-2.json
-highlight: json
-
+```{literalinclude} examples/ome/series-2.json +:language: json +``` -

Details

+### Details +(bf2raw-details)= Conforming groups: @@ -482,8 +480,8 @@ Conforming readers: - MAY ignore other groups or arrays under the root of the hierarchy. -"coordinateTransformations" metadata {#trafo-md} ------------------------------------------------- +## "coordinateTransformations" metadata +(trafo-md)= "coordinateTransformations" describe the mapping between two coordinate systems (defined by "axes"). For example, to map an array's discrete coordinate system to its corresponding physical coordinates. @@ -552,11 +550,11 @@ Conforming readers: - SHOULD be able to apply transformations to points; - SHOULD be able to apply transformations to images; -Coordinate transformations from array to physical coordinates MUST be stored in multiscales ([[#multiscale-md]]). +Coordinate transformations from array to physical coordinates MUST be stored in [multiscales](multiscales-md). Transformations between different images MUST be stored in the attributes of a parent zarr group. For transformations that store data or parameters in a zarr array, those zarr arrays SHOULD be stored in a zarr group `"coordinateTransformations"`. -
+```
 store.zarr                      # Root folder of the zarr store
 │
 ├── zarr.json                   # coordinate transformations describing the relationship between two image coordinate systems
@@ -579,7 +577,7 @@ store.zarr                      # Root folder of the zarr store
         └── image               # a zarr array
             └── zarr.json       # physical coordinate system and transformations here
                                 # the array attributes
-
+``` ### Additional details @@ -595,7 +593,7 @@ Transformations in the `transformations` list of a `byDimensions` transformation of strings corresponding to axis names of the parent transformation's input and output coordinate systems (see below for details). -
+````{admonition} Example The sequence transformation's input corresponds to an array coordinate system at path "my/array". @@ -644,7 +642,7 @@ The sequence transformation's input corresponds to an array coordinate system at ] ``` -
+```` Coordinate transformations are functions of *points* in the input space to *points* in the output space. We call this the "forward" direction. Points are ordered lists of coordinates, where a coordinate is the location/value of that point along its corresponding axis. @@ -674,26 +672,34 @@ transformation type, for example: ``` Implementations SHOULD be able to compute and apply the inverse of some coordinate transformations when they -are computable in closed-form (as the [Transformation types](#transformation-types) section below indicates). If an +are computable in closed-form (as the [Transformation types](transformation-types) section below indicates). If an operation is requested that requires the inverse of a transformation that can not be inverted in closed-form, implementations MAY estimate an inverse, or MAY output a warning that the requested operation is unsupported. #### Matrix transformations +(matrix-transformations)= -Two transformation types ([affine](#affine) and [rotation](#rotation)) are parametrized by matrices. Matrices are applied to +Two transformation types ([affine](affine) and [rotation](rotation)) are parametrized by matrices. Matrices are applied to column vectors that represent points in the input coordinate system. The first (last) axis in a coordinate system is the top (bottom) entry in the column vector. Matrices are stored as two-dimensional arrays, either as json or in a zarr array. When stored as a 2D zarr array, the first dimension indexes rows and the second dimension indexes columns (e.g., an array of `"shape":[3,4]` has 3 rows and 4 columns). When stored as a 2D json array, the inner array contains rows (e.g. `[[1,2,3], [4,5,6]]` has 2 rows and 3 columns). -
+````{admonition} Example For matrix transformations, points in the coordinate system: -``` -{ "name" : "in", "axes" : [{"name" : "z"}, {"name" : "y"}, {"name":"x"}] }, +```json +{ + "name" : "in", + "axes" : [ + {"name" : "z"}, + {"name" : "y"}, + {"name":"x"} + ] +}, ``` are represented as column vectors: @@ -713,30 +719,29 @@ results in the point [2,-1,3] because it is computed with the matrix-vector mult [ 0 0 -1] [3] [-3] ``` -
- +```` ### Transformation types +(transformation-types)= -Input and output dimensionality may be determined by the value of the "input" and "output" fields, respectively. If the value -of "input" is an array, it's length gives the input dimension, otherwise the length of "axes" for the coordinate -system with the name of the "input" value gives the input dimension. If the value of "input" is an array, it's -length gives the input dimension, otherwise it is given by the length of "axes" for the coordinate system with -the name of the "input". If the value of "output" is an array, its length gives the output dimension, +Input and output dimensionality may be determined by the value of the "input" and "output" fields, respectively. +If the value of "input" is an array, its shape gives the input dimension, +otherwise it is given by the length of "axes" for the coordinate system with the name of the "input". +If the value of "output" is an array, its shape gives the output dimension, otherwise it is given by the length of "axes" for the coordinate system with the name of the "output". -#### identity +#### identity +(identity)= `identity` transformations map input coordinates to output coordinates without modification. The position of the ith axis of the output coordinate system is set to the position of the ith axis of the input coordinate system. `identity` transformations are invertible. -
+````{admonition} Example -
-path: examples/transformations/identity.json
-highlight: json
-
+```{literalinclude} examples/transformations/identity.json +:language: json +``` defines the function: @@ -745,10 +750,11 @@ x = i y = j ``` -
+```` -#### mapAxis +#### mapAxis +(mapAxis)= `mapAxis` transformations describe axis permutations as a mapping of axis names. Transformations MUST include a `mapAxis` field whose value is an object, all of whose values are strings. If the object contains `"x":"i"`, then the transform sets the value @@ -757,12 +763,11 @@ system, the `mapAxis` MUST have a corresponding field. For every value of the ob coordinate system with that name. Note that the order of the keys could be reversed. -
+````{admonition} Example -
-path: examples/transformations/mapAxis1.json
-highlight: json
-
+```{literalinclude} examples/transformations/mapAxis1.json +:language: json +``` The "equivalent to identity" transformation defines the function: @@ -778,14 +783,13 @@ x = j y = i ``` -
+```` -
+````{admonition} Example -
-path: examples/transformations/mapAxis2.json
-highlight: json
-
+```{literalinclude} examples/transformations/mapAxis2.json +:language: json +``` The "projection_down" transformation defines the function: @@ -800,9 +804,10 @@ x = a y = b z = b ``` -
+```` -#### translation +#### translation +(translation)= `translation` transformations are special cases of affine transformations. When possible, a translation transformation should be preferred to its equivalent affine. Input and output dimensionality MUST be @@ -817,12 +822,11 @@ invertible.
The scale parameters stored as a JSON list of numbers. The list MUST have length `N`.
-
+````{admonition} Example -
-path: examples/transformations/translation.json
-highlight: json
-
+```{literalinclude} examples/transformations/translation.json +:language: json +``` defines the function: @@ -830,9 +834,10 @@ defines the function: x = i + 9 y = j - 1.42 ``` -
+```` #### scale +(scale)= `scale` transformations are special cases of affine transformations. When possible, a scale transformation SHOULD be preferred to its equivalent affine. Input and output dimensionality MUST be identical and MUST equal @@ -847,12 +852,11 @@ transformations are invertible.
The scale parameters stored as a JSON list of numbers. The list MUST have length `N`.
-
+````{admonition} Example -
-path: examples/transformations/scale.json
-highlight: json
-
+```{literalinclude} examples/transformations/scale.json +:language: json +``` defines the function: @@ -860,11 +864,11 @@ defines the function: x = 3.12 * i y = 2 * j ``` -
- -#### affine +```` +#### affine +(affine)= -`affine`s are [matrix transformations](#matrix-transformations) from N-dimensional inputs to M-dimensional outputs are +`affine`s are [matrix transformations](matrix-transformations) from N-dimensional inputs to M-dimensional outputs are represented as the upper `(M)x(N+1)` sub-matrix of a `(M+1)x(N+1)` matrix in [homogeneous coordinates](https://en.wikipedia.org/wiki/Homogeneous_coordinates) (see examples). This transformation type may be (but is not necessarily) invertible when `N` equals `M`. The matrix MUST be stored as a 2D array either as json or as a zarr array. @@ -877,69 +881,68 @@ necessarily) invertible when `N` equals `M`. The matrix MUST be stored as a 2D a
The affine parameters stored in JSON. The matrix MUST be stored as 2D nested array where the outer array MUST be length `M` and the inner arrays MUST be length `N+1`.
-
- A 2D-2D example: +````{admonition} Example +A 2D-2D example: -
-    path: examples/transformations/affine2d2d.json
-    highlight: json
-    
+```{literalinclude} examples/transformations/affine2d2d.json +:language: json +``` - defines the function: +defines the function: - ``` - x = 1*i + 2*j + 3 - y = 4*i + 5*j + 6 - ``` +``` +x = 1*i + 2*j + 3 +y = 4*i + 5*j + 6 +``` - it is equivalent to this matrix-vector multiplication in homogeneous coordinates: +it is equivalent to this matrix-vector multiplication in homogeneous coordinates: - ``` - [ 1 2 3 ][ i ] [ x ] - [ 4 5 6 ][ j ] = [ y ] - [ 0 0 1 ][ 1 ] [ 1 ] - ``` +``` +[ 1 2 3 ][ i ] [ x ] +[ 4 5 6 ][ j ] = [ y ] +[ 0 0 1 ][ 1 ] [ 1 ] +``` - where the last row `[0 0 1]` is omitted in the JSON representation. +where the last row `[0 0 1]` is omitted in the JSON representation. -
+```` -
- An example with two dimensional inputs and three dimensional outputs. +````{admonition} Example +An example with two dimensional inputs and three dimensional outputs. - Note that the order of the axes can in general be determined by the application or user. - These axes relate to the memory or on-disk order insofar as the last dimension is contiguous - when the zarr array is c-order (the default for zarr version 2, and the only option for zarr version 3). +Note that the order of the axes can in general be determined by the application or user. +These axes relate to the memory or on-disk order insofar as the last dimension is contiguous +when the zarr array is c-order (the default for zarr version 2, and the only option for zarr version 3). -
-    path: examples/transformations/affine2d3d.json
-    highlight: json
-    
+```{literalinclude} examples/transformations/affine2d3d.json +:language: json +``` - defines the function: +defines the function: - ``` - x = 1*i + 2*j + 3 - y = 4*i + 5*j + 6 - z = 7*i + 8*j + 9 - ``` +``` +x = 1*i + 2*j + 3 +y = 4*i + 5*j + 6 +z = 7*i + 8*j + 9 +``` - it is equivalent to this matrix-vector multiplication in homogeneous coordinates: +it is equivalent to this matrix-vector multiplication in homogeneous coordinates: - ``` - [ 1 2 3 ][ i ] [ x ] - [ 4 5 6 ][ j ] = [ y ] - [ 7 8 9 ][ 1 ] [ z ] - [ 0 0 1 ] [ 1 ] - ``` +``` +[ 1 2 3 ][ i ] [ x ] +[ 4 5 6 ][ j ] = [ y ] +[ 7 8 9 ][ 1 ] [ z ] +[ 0 0 1 ] [ 1 ] +``` - where the last row `[0 0 1]` is omitted in the JSON representation. -
+where the last row `[0 0 1]` is omitted in the JSON representation. +```` -#### rotation +#### rotation +(rotation)= -`rotation`s are [matrix transformations](#matrix-transformations) that are special cases of affine transformations. When possible, a rotation +`rotation`s are [matrix transformations](matrix-transformations) that are special cases of affine transformations. When possible, a rotation transformation SHOULD be preferred to its equivalent affine. Input and output dimensionality (N) MUST be identical. Rotations are stored as `NxN` matrices, see below, and MUST have determinant equal to one, with orthonormal rows and columns. The matrix MUST be stored as a 2D array either as json or in a zarr array. `rotation` transformations are invertible. @@ -952,49 +955,48 @@ MUST be stored as a 2D array either as json or in a zarr array. `rotation` trans
The parameters stored in JSON. The matrix MUST be stored as a 2D nested array where the outer array MUST be length `N` and the inner arrays MUST be length `N`.
-
- A 2D example +````{admonition} Example +A 2D example -
-    path: examples/transformations/rotation.json
-    highlight: json
-    
+```{literalinclude} examples/transformations/rotation.json +:language: json +``` - defines the function: +defines the function: - ``` - x = 0*i - 1*j - y = 1*i + 0*j - ``` -
+``` +x = 0*i - 1*j +y = 1*i + 0*j + ``` +```` -#### inverseOf +#### inverseOf +(inverseOf)= An `inverseOf` transformation contains another transformation (often non-linear), and indicates that transforming points from output to input coordinate systems is possible using the contained transformation. Transforming points from the input to the output coordinate systems requires the inverse of the contained transformation (if it exists). -
- Software libraries that perform image registration often return the transformation from fixed image - coordinates to moving image coordinates, because this "inverse" transformation is most often required - when rendering the transformed moving image. Results such as this may be enclosed in an `inverseOf` - transformation. This enables the "outer" coordinate transformation to specify the moving image coordinates - as `input` and fixed image coordinates as `output`, a choice that many users and developers find intuitive. -
- +```{note} +Software libraries that perform image registration often return the transformation from fixed image +coordinates to moving image coordinates, because this "inverse" transformation is most often required +when rendering the transformed moving image. Results such as this may be enclosed in an `inverseOf` +transformation. This enables the "outer" coordinate transformation to specify the moving image coordinates +as `input` and fixed image coordinates as `output`, a choice that many users and developers find intuitive. +``` -
-
-    path: examples/transformations/inverseOf.json
-    highlight: json
-    
+````{admonition} Example -
+```{literalinclude} examples/transformations/inverseOf.json +:language: json +``` +```` -#### sequence +#### sequence +(sequence)= A `sequence` transformation consists of an ordered array of coordinate transformations, and is invertible if every coordinate transform in the array is invertible (though could be invertible in other cases as well). To apply a sequence transformation @@ -1002,7 +1004,7 @@ to a point in the input coordinate system, apply the first transformation in the transformation to the result. Repeat until every transformation has been applied. The output of the last transformation is the result of the sequence. -
+````{note} Considering transformations as functions of points, if the list contains transformations `[f0, f1, f2]` in that order, applying this sequence to point `x` is equivalent to: @@ -1013,7 +1015,7 @@ f2(f1(f0(x))) `f0` is applied first, `f1` is applied second, and `f2` is applied last. -
+```` The transformations included in the `transformations` array may omit their `input` and `output` fields under the conditions outlined below: @@ -1035,14 +1037,13 @@ outlined below:
A non-empty array of transformations.
-
+````{admonition} Example This sequence: -
-path: examples/transformations/sequence.json
-highlight: json
-
+```{literalinclude} examples/transformations/sequence.json +:language: json +``` describes the function @@ -1052,10 +1053,11 @@ y = (j + 0.9) * 3 ``` and is invertible. -
+```` -#### coordinates and displacements +#### coordinates and displacements +(coordinates-displacements)= `coordinates` and `displacements` transformations store coordinates or displacements in an array and interpret them as a vector field that defines a transformation. The arrays must have a dimension corresponding to every axis of the input coordinate @@ -1076,7 +1078,7 @@ metadata for the array ("field coordinate system"). The `i`th value of the array along the `coordinate` or `displacement` axis refers to the coordinate or displacement of the `i`th output axis. See the example below. -
+````{admonition} Example In this example, the array located at `"displacementField"` MUST have three dimensions. One dimension MUST correspond to an axis with `type : displacement` (in this example, the last dimension), the other two dimensions MUST be axes @@ -1117,8 +1119,7 @@ x_displacement = displacementField[y][x][1] I.e. the y-displacement is first, because the y-axis is the first element of the input and output coordinate systems. -
- +```` `coordinates` and `displacements` transformations are not invertible in general, but implementations MAY approximate their inverses. Metadata for these coordinate transforms have the following field: @@ -1244,7 +1245,8 @@ The transformation specifies linear interpolation, which in this case yields input point, hence the output is `1.0 + (-0.5) = 0.5`. -#### byDimension +#### byDimension +(byDimension)= `byDimension` transformations build a high dimensional transformation using lower dimensional transformations on subsets of dimensions. @@ -1259,56 +1261,51 @@ on subsets of dimensions. -
+````{admonition} Example A valid `byDimension` transformation: -
-path: examples/transformations/byDimension1.json
-highlight: json
-
- -
+```{literalinclude} examples/transformations/byDimension1.json +:language: json +``` +```` -
+````{admonition} Example Another valid `byDimension` transformation: -
-path: examples/transformations/byDimension2.json
-highlight: json
-
- -
+```{literalinclude} examples/transformations/byDimension2.json +:language: json +``` +```` -
+````{admonition} Example This is an **invalid** `byDimension` transform: -
-path: examples/transformations/byDimensionInvalid1.json
-highlight: json
-
+```{literalinclude} examples/transformations/byDimensionInvalid1.json +:language: json +``` It is invalid for two reasons. First because input `0` used by the scale transformation is not an axis of the `byDimension` transformation's `input`. Second, the `x` axis of the `output` does not appear in the `output` of any child transformation. -
+```` -
+````{admonition} Example Another **invalid** `byDimension` transform: -
-path: examples/transformations/byDimensionInvalid2.json
-highlight: json
-
+```{literalinclude} examples/transformations/byDimensionInvalid2.json +:language: json +``` This transformation is invalid because the output axis `x` appears in more than one transformation in the `transformations` list. -
+```` -#### bijection +#### bijection +(bijection)= A bijection transformation is an invertible transformation in which both the `forward` and `inverse` transformations are explicitly defined. Each direction SHOULD be a transformation type that is not closed-form invertible. @@ -1323,25 +1320,23 @@ Practically, non-invertible transformations have finite extents, so bijection tr to be correct / consistent for points that fall within those extents. It may not be correct for any point of appropriate dimensionality. -
+````{admonition} Example -
-path: examples/transformations/bijection.json
-highlight: json
-
+```{literalinclude} examples/transformations/bijection.json +:language: json +``` the input and output of the `forward` and `inverse` transformations are understood to be: -
-path: examples/transformations/bijection_verbose.json
-highlight: json
-
+```{literalinclude} examples/transformations/bijection_verbose.json +:language: json +``` -
+```` -"multiscales" metadata {#multiscale-md} ---------------------------------------- +## "multiscales" metadata +(multiscales-md)= Metadata about an image can be found under the "multiscales" key in the group-level OME-Zarr Metadata. Here, image refers to 2 to 5 dimensional data representing image or volumetric data with optional time or channel axes. @@ -1349,7 +1344,7 @@ It is stored in a multiple resolution representation. "multiscales" contains a list of dictionaries where each entry describes a multiscale image. -Each "multiscales" dictionary MUST contain the field "coordinateSystems", see [[#coord-sys-md]], with the following constraints. +Each "multiscales" dictionary MUST contain the field "coordinateSystems", see [coordinateSystems metadata](coord-sys-md), with the following constraints. The length of "axes" must be between 2 and 5 and MUST be equal to the dimensionality of the zarr arrays storing the image data (see "datasets:path"). The "axes" MUST contain 2 or 3 entries of "type:space" and MAY contain one additional entry of "type:time" and MAY contain one additional entry of "type:channel" or a null / custom type. The order of the entries MUST correspond to the order of dimensions of the zarr arrays. In addition, the entries MUST be ordered by "type" where the "time" axis must come first (if present), followed by the "channel" or custom axis (if present) and the axes of type "space". @@ -1361,7 +1356,7 @@ to the current zarr group. The "path"s MUST be ordered from largest (i.e. highes Each "datasets" dictionary MUST have the same number of dimensions and MUST NOT have more than 5 dimensions. The number of dimensions and order MUST correspond to number and order of "axes". Each dictionary in "datasets" MUST contain the field "coordinateTransformations", which contains a list of transformations that map the data coordinates to the physical coordinates (as specified by "axes") for this resolution level. -The transformations are defined according to [[#trafo-md]]. +The transformations are defined according to [coordinateTransformations metadata](trafo-md). They MUST contain exactly one `scale` transformation that specifies the pixel size in physical units or time duration. If scaling information is not available or applicable for one of the axes, the value MUST express the scaling factor between the current resolution and the first resolution for the given axis, defaulting to 1.0 if there is no downsampling along the axis. It MAY contain exactly one `translation` that specifies the offset from the origin in physical units. If `translation` is given it MUST be listed after `scale` to ensure that it is given in physical coordinates. @@ -1376,11 +1371,11 @@ Each "multiscales" dictionary SHOULD contain the field "name". Each "multiscales" dictionary SHOULD contain the field "type", which gives the type of downscaling method used to generate the multiscale image pyramid. It SHOULD contain the field "metadata", which contains a dictionary with additional information about the downscaling method. -
-path: examples/multiscales_strict/multiscales_example.json
-highlight: json
-
- +````{admonition} Example +```{literalinclude} examples/multiscales_strict/multiscales_example.json +:language: json +``` +```` If only one multiscale is provided, use it. Otherwise, the user can choose by name, using the first multiscale as a fallback: @@ -1396,8 +1391,8 @@ if not datasets: datasets = [x["path"] for x in multiscales[0]["datasets"]] ``` -"omero" metadata (transitional) {#omero-md} -------------------------------------------- +## "omero" metadata (transitional) +(omero-md)= [=Transitional=] information specific to the channels of an image and how to render it can be found under the "omero" key in the group-level metadata: @@ -1437,8 +1432,8 @@ Each dictionary in "channels" MUST contain the field "window", which is a dictio The field "window" MUST contain the fields "min" and "max", which are the minimum and maximum values of the window, respectively. It MUST also contain the fields "start" and "end", which are the start and end values of the window, respectively. -"labels" metadata {#labels-md} ------------------------------- +## "labels" metadata +(labels-md)= In OME-Zarr, Zarr arrays representing pixel-annotation data are stored in a group called "labels". Some applications--notably image segmentation--produce a new image that is in the same coordinate system as a corresponding multiscale image (usually having the same dimensions and coordinate transformations). @@ -1452,11 +1447,12 @@ The "labels" group is not itself an image; it contains images. The pixels of the but these MUST NOT contain metadata. Names of the images in the "labels" group are arbitrary. The OME-Zarr Metadata in the `zarr.json` file associated with the "labels" group MUST contain a JSON object with the key `labels`, whose value is a JSON array of paths to the -labeled multiscale image(s). All label images SHOULD be listed within this metadata file. For example: +labeled multiscale image(s). All label images SHOULD be listed within this metadata file. +````{admonition} Example +For example: ```json { - ... "attributes": { "ome": { "version": "0.5", @@ -1467,6 +1463,7 @@ labeled multiscale image(s). All label images SHOULD be listed within this metad } } ``` +```` The `zarr.json` file for the label image MUST implement the multiscales specification. Within the `multiscales` object, the JSON array associated with the `datasets` key MUST have the same number of entries (scale levels) as the original unlabeled image. @@ -1495,18 +1492,18 @@ The value of the `source` key MUST be a JSON object containing information about This object MAY include a key `image`, whose value MUST be a string specifying the relative path to a Zarr image group. The default value is `../../` since most labeled images are stored in a "labels" group that is nested within the original image group. -Here is an example of a simple `image-label` object for a label image in which 0s and 1s represent intercellular and cellular space, respectively: -
-path: examples/label_strict/colors_properties.json
-highlight: json
-
+````{admonition} Example +Here is an example of a simple `image-label` object for a label image in which 0s and 1s represent intercellular and cellular space, respectively: +```{literalinclude} examples/label_strict/colors_properties.json +:language: json +``` +In this case, the pixels consisting of a 0 in the Zarr array will be displayed as 50% blue and 50% opacity. Pixels with a 1 in the Zarr array, which correspond to cellular space, will be displayed as 50% green and 50% opacity. +```` -In this case, the pixels consisting of a 0 in the Zarr array will be displayed as 50% blue and 50% opacity. Pixels with a 1 in the Zarr array, -which correspond to cellular space, will be displayed as 50% green and 50% opacity. -"plate" metadata {#plate-md} ----------------------------- +## "plate" metadata +(plate-md)= For high-content screening datasets, the plate layout can be found under the custom attributes of the plate group under the `plate` key in the group-level metadata. @@ -1515,7 +1512,7 @@ The `plate` dictionary MAY contain an `acquisitions` key whose value MUST be a l JSON objects defining the acquisitions for a given plate to which wells can refer to. Each acquisition object MUST contain an `id` key whose value MUST be an unique integer identifier greater than or equal to 0 within the context of the plate to which fields of view can refer -to (see #well-md). +to (see [well metadata](well-md)). Each acquisition object SHOULD contain a `name` key whose value MUST be a string identifying the name of the acquisition. Each acquisition object SHOULD contain a `maximumfieldcount` key whose value MUST be a positive integer indicating the maximum number of fields of view for the @@ -1561,24 +1558,24 @@ the index into the `rows` list and a `columnIndex` key whose value MUST be an in the index into the `columns` list. `rowIndex` and `columnIndex` MUST be 0-based. The `rowIndex`, `columnIndex`, and `path` MUST all refer to the same row/column pair. +````{admonition} Example For example the following JSON object defines a plate with two acquisitions and 6 wells (2 rows and 3 columns), containing up to 2 fields of view per acquisition. -
-path: examples/plate_strict/plate_6wells.json
-highlight: json
-
+```{literalinclude} examples/plate_strict/plate_6wells.json +:language: json +``` The following JSON object defines a sparse plate with one acquisition and 2 wells in a 96 well plate, containing one field of view per acquisition. -
-path: examples/plate_strict/plate_2wells.json
-highlight: json
-
+```{literalinclude} examples/plate_strict/plate_2wells.json +:language: json +``` +```` -"well" metadata {#well-md} --------------------------- +## "well" metadata +(well-md)= For high-content screening datasets, the metadata about all fields of views under a given well can be found under the "well" key in the attributes of the @@ -1590,209 +1587,37 @@ specifying all fields of views for a given well. Each image object MUST contain MUST contain only alphanumeric characters, MUST be case-sensitive, and MUST NOT be a duplicate of any other `path` in the `images` list. If multiple acquisitions were performed in the plate, it MUST contain an `acquisition` key whose value MUST be an integer identifying the acquisition -which MUST match one of the acquisition JSON objects defined in the plate metadata (see #plate-md). +which MUST match one of the acquisition JSON objects defined in the [plate metadata](plate-md). The `well` dictionary SHOULD contain a `version` key whose value MUST be a string specifying the version of the well specification. +````{admonition} Example For example the following JSON object defines a well with four fields of view. The first two fields of view were part of the first acquisition while the last two fields of view were part of the second acquisition. -
-path: examples/well_strict/well_4fields.json
-highlight: json
-
+```{literalinclude} examples/well_strict/well_4fields.json +:language: json +``` The following JSON object defines a well with two fields of view in a plate with four acquisitions. The first field is part of the first acquisition, and the second field is part of the last acquisition. -
-path: examples/well_strict/well_2fields.json
-highlight: json
-
+```{literalinclude} examples/well_strict/well_2fields.json +:language: json +``` +```` -Specification naming style {#naming-style} -========================================== +# Specification naming style +(naming-style)= Multi-word keys in this specification should use the `camelCase` style. NB: some parts of the specification don't obey this convention as they were added before this was adopted, but they should be updated in due course. -Implementations {#implementations} -================================== - -See [Tools](https://ngff.openmicroscopy.org/tools/index.html). - -Citing {#citing} -================ +# Implementations +(implementations)= -[Next-generation file format (NGFF) specifications for storing bioimaging data in the cloud.](https://ngff.openmicroscopy.org/0.4) -J. Moore, *et al*. Open Microscopy Environment Consortium, 8 February 2022. -This edition of the specification is [https://ngff.openmicroscopy.org/0.5/](https://ngff.openmicroscopy.org/0.5/]). -The latest edition is available at [https://ngff.openmicroscopy.org/latest/](https://ngff.openmicroscopy.org/latest/). -[(doi:10.5281/zenodo.4282107)](https://doi.org/10.5281/zenodo.4282107) - -Version History {#history} -========================== - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
RevisionDateDescription
0.5.22025-01-10Clarify that the dimension_names field in axes MUST be included.
0.5.12025-01-10Re-add the improved omero description in PR-191.
0.5.02024-11-21use Zarr v3 in OME-Zarr, see RFC-2.
0.4.12023-02-09expand on "labels" description
0.4.12022-09-26transitional metadata for image collections ("bioformats2raw.layout")
0.4.02022-02-08multiscales: add axes type, units and coordinateTransformations
0.4.02022-02-08plate: add rowIndex/columnIndex
0.3.02021-08-24Add axes field to multiscale metadata
0.2.02021-03-29Change chunk dimension separator to "/"
0.1.42020-11-26Add HCS specification
0.1.32020-09-14Add labels specification
0.1.2 2020-05-07Add description of "omero" metadata
0.1.1 2020-05-06Add info on the ordering of resolutions
0.1.0 2020-04-20First version for internal demo
- - -
-{
-  "blogNov2020": {
-    "href": "https://blog.openmicroscopy.org/file-formats/community/2020/11/04/zarr-data/",
-    "title": "Public OME-Zarr data (Nov. 2020)",
-    "authors": [
-      "OME Team"
-    ],
-    "status": "Informational",
-    "publisher": "OME",
-    "id": "blogNov2020",
-    "date": "04 November 2020"
-  },
-  "imagesc26952": {
-    "href": "https://forum.image.sc/t/ome-s-position-regarding-file-formats/26952",
-    "title": "OME’s position regarding file formats",
-    "authors": [
-      "OME Team"
-    ],
-    "status": "Informational",
-    "publisher": "OME",
-    "id": "imagesc26952",
-    "date": "19 June 2020"
-  },
-  "n5": {
-    "id": "n5",
-    "href": "https://github.com/saalfeldlab/n5/issues/62",
-    "title": "N5---a scalable Java API for hierarchies of chunked n-dimensional tensors and structured meta-data",
-    "status": "Informational",
-    "authors": [
-      "John A. Bogovic",
-      "Igor Pisarev",
-      "Philipp Hanslovsky",
-      "Neil Thistlethwaite",
-      "Stephan Saalfeld"
-    ],
-    "date": "2020"
-  },
-  "ome-zarr-py": {
-    "id": "ome-zarr-py",
-    "href": "https://doi.org/10.5281/zenodo.4113931",
-    "title": "ome-zarr-py: Experimental implementation of next-generation file format (NGFF) specifications for storing bioimaging data in the cloud.",
-    "status": "Informational",
-    "publisher": "Zenodo",
-    "authors": [
-      "OME",
-      "et al"
-    ],
-    "date": "06 October 2020"
-  },
-  "zarr": {
-    "id": "zarr",
-    "href": "https://doi.org/10.5281/zenodo.4069231",
-    "title": "Zarr: An implementation of chunked, compressed, N-dimensional arrays for Python.",
-    "status": "Informational",
-    "publisher": "Zenodo",
-    "authors": [
-      "Alistair Miles",
-      "et al"
-    ],
-    "date": "06 October 2020"
-  },
-  "itk":{
-    "id": "itk-book",
-    "href": "https://itk.org/ItkSoftwareGuide.pdf",
-    "title": "The ITK Software Guide",
-    "status": "Informational",
-    "publisher": "ITK",
-    "authors": [
-      "Hans J. Johnson",
-      "Matthew M. McCormick",
-      "Luis Ibanez",
-      "Insight Software Consortium"
-    ],
-    "date": "16 April 2021"
-  }
-}
-
+See [Tools](https://ngff.openmicroscopy.org/tools/index.html). \ No newline at end of file diff --git a/transform-details.bs b/transform-details.bs deleted file mode 100644 index b2b744c8..00000000 --- a/transform-details.bs +++ /dev/null @@ -1,106 +0,0 @@ -
-Title: Coordinates and Transformations
-Shortname: ome-ngff-transformations
-Level: 1
-Status: LS-COMMIT
-Status: w3c/ED
-Group: ome
-URL: https://ngff.openmicroscopy.org/latest/
-Repository: https://github.com/ome/ngff
-Issue Tracking: Forums https://forum.image.sc/tag/ome-ngff
-Logo: http://www.openmicroscopy.org/img/logos/ome-logomark.svg
-Local Boilerplate: header no
-Local Boilerplate: copyright no
-Boilerplate: style-darkmode off
-Markup Shorthands: markdown yes
-Editor: Josh Moore, Open Microscopy Environment (OME) https://www.openmicroscopy.org
-Editor: Sébastien Besson, Open Microscopy Environment (OME) https://www.openmicroscopy.org
-Editor: Constantin Pape, European Molecular Biology Laboratory (EMBL) https://www.embl.org/sites/heidelberg/
-Editor: John Bogovic, Hughes Medical Institute Janelia (HHMI) https://www.janelia.org/ 
-Abstract: This document contains next-generation file format (NGFF)
-Abstract: specifications for storing bioimaging data in the cloud.
-Abstract: All specifications are submitted to the https://image.sc community for review.
-Status Text: The current released version of this specification is
-Status Text: will be provided between numbered versions. Data written with these latest changes
-Status Text: (an "editor's draft") will not necessarily be supported.
-
- -Coordinates and Axes {#coords-axes} -===================== - -OME-NGFF datasets are arrays that hold values. The arrays may be indexed by discrete (integer) -coordinates in order to obtain a corresponding value. If values are desired at continuous (real-valued) -coordinates, then interpolation is required. - -Interpolation {#interp} ---------------------- - -Interpolation is the process of producing values at continuous coordinates from data sampled at discrete -coordinates. "Nearest-neighbor" and "N-Linear" are the two most commonly used interpolation methods. - - -Pixel coordinates {#pix-coords} ---------------------- - -**The pixel center is the origin of the continuous coordinate system.** - -### Top-left convention - -A common alternative convention is for the origin in the continuous space is at the "top-left" of the pixel. -This is not recommended, but can be acheived by explicitly adding a half-pixel translation, for example: - -```json -{ - "name": "center_to_top-left", - "type": "translation", - "translation" : [0.5, 0.5], - "output_space" : "top-left-space" -} -``` - -Coordinate Transformations {#coord-tforms} -===================== - -This document describes background and motivation that is outside the NGFF specification. - - -Direction {#direction} ---------------------- - -Specified coordinate transforms are in the "forward" direction. They represent functions -from *points* in the input space to *points* in the output space. For example, the transformation `"ij2xy"` - - -```json -{ - "name": "ij2xy", - "type": "scale", - "scale": [2, 0.5] - "input_axes" : ["i", "j"] - "output_axes" : ["x", "y"] -} -``` - -representes the function - -``` -x = 2 * i -y = 0.5 * j -``` - - -Recommendations {#recommendations} -===================== - - -"Native" physical space ---------------------- - -Datasets SHOULD define a transformation from array space to their "native physical space." -This transformation SHOULD describe physical pixel spacing and origin only, and therefore SHOULD consist of -`scale` and/or `translation` types only. - -Subsequent reorientation / registration transformations SHOULD use this native space as their `input_space`, -i.e., transformations should be defined in physical coordinates. - - diff --git a/version_history.md b/version_history.md new file mode 100644 index 00000000..a3214f19 --- /dev/null +++ b/version_history.md @@ -0,0 +1,83 @@ +Version History +=============== +(history)= + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
RevisionDateDescription
0.5.22025-01-10Clarify that the dimension_names field in axes MUST be included.
0.5.12025-01-10Re-add the improved omero description in PR-191.
0.5.02024-11-21use Zarr v3 in OME-Zarr, see RFC-2.
0.4.12023-02-09expand on "labels" description
0.4.12022-09-26transitional metadata for image collections ("bioformats2raw.layout")
0.4.02022-02-08multiscales: add axes type, units and coordinateTransformations
0.4.02022-02-08plate: add rowIndex/columnIndex
0.3.02021-08-24Add axes field to multiscale metadata
0.2.02021-03-29Change chunk dimension separator to "/"
0.1.42020-11-26Add HCS specification
0.1.32020-09-14Add labels specification
0.1.2 2020-05-07Add description of "omero" metadata
0.1.1 2020-05-06Add info on the ordering of resolutions
0.1.0 2020-04-20First version for internal demo
\ No newline at end of file