Merge remote-tracking branch 'upstream/main' into include-all-authors

Author: jo-mueller

.editorconfig

@@ -0,0 +1,15 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+trim_trailing_whitespace = true
+
+[*.{py,md,ini}]
+indent_style = space
+indent_size = 4
+
+[*.{json,yaml,yml,cff}]
+indent_style = space
+indent_size = 2

.github/workflows/codespell.yml

@@ -0,0 +1,23 @@
+# Codespell configuration is within .codespellrc
+---
+name: Codespell
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2

.github/workflows/review.yml

@@ -0,0 +1,25 @@
+---
+name: Add review url
+
+on:
+  pull_request_target:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: Automated Review URLs
+
+      - uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            #### Automated Review URLs
+            * [Readthedocs](https://ngff-spec--${{ github.event.pull_request.number }}.org.readthedocs.build/)
+          edit-mode: replace

.github/workflows/validation.yml

@@ -0,0 +1,16 @@
+name: Validation
+on:
+  push:
+  pull_request:
+
+jobs:
+  validate:
+    name: Validation
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -mpip install --upgrade wheel tox
+      - run: tox

.gitignore

@@ -8,7 +8,14 @@ _bikeshed
 .*plist
 
 # do not sync auto-generated md files
-ngff_spec/_generated
+footer.md
+examples.md
+schemas.md
+examples/**/*.md
+schemas/**/*.md
+schemas/**/*.html
+schemas/**/*.css
+schemas/**/*.js
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -35,7 +42,6 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
-*_version.py
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -88,9 +94,6 @@ target/
 # OS
 .DS_Store
 
-# written by setuptools_scm
-*/_version.py
-
 # PyBuilder
 .idea
 venv/

.readthedocs.yaml

@@ -11,10 +11,10 @@ build:
 
     pre_build:
       # Run the script that autogenerates content before building the docs
-      - "uv run python ngff_spec/pre_build.py"
+      - "uv run python pre_build.py"
 
     build:
       html: 
-        - cd ngff_spec/ && jupyter book build --html --ci
+        - jupyter book build --html --ci
         - mkdir -p $READTHEDOCS_OUTPUT
-        - mv ./ngff_spec/_build/html $READTHEDOCS_OUTPUT
+        - mv ./_build/html $READTHEDOCS_OUTPUT

CITATION.cff

@@ -0,0 +1,88 @@
+cff-version: 1.2.0
+message: Please cite the following works when using this project.
+abstract: >-
+  A growing community is constructing a next-generation file format (NGFF) for
+  bioimaging to overcome problems of scalability and heterogeneity. Organized by
+  the Open Microscopy Environment (OME), individuals and institutes across
+  diverse modalities facing these problems have designed a format specification
+  process (OME-NGFF) to address these needs. Those community members have built
+  the format itself – OME-Zarr – along with tools and data resources available
+  today to increase FAIR access and remove barriers in the scientific process.
+  The current momentum offers an opportunity to unify a key component of the
+  bioimaging domain — the file format that underlies so many personal,
+  institutional, and global data management and analysis tasks.
+title: Next-generation file format specification
+authors:
+  - family-names: NGFF-community
+    given-names: ''
+identifiers:
+  - type: doi
+    value: 10.1038/s41592-021-01326-w
+    description: Original Nature Methods article (2021)
+  - type: doi
+    value: 10.1101/2023.02.17.528834v2
+    description: OME-Zarr preprint (2023)
+keywords:
+  - FAIR
+  - community
+  - bioimaging
+  - data
+  - cloud
+  - format
+license: CC-BY-4.0
+url: https://ngff.openmicroscopy.org
+repository: https://github.com/ome/ngff-spec
+preferred-citation:
+  type: article
+  doi: 10.1038/s41592-021-01326-w
+  journal: Nature Methods
+  month: 11
+  start: 1496
+  end: 1498
+  title: >-
+    OME-NGFF: a next-generation file format for expanding bioimaging data-access
+    strategies
+  issue: 12
+  volume: 18
+  year: 2021
+  authors:
+    - given-names: Josh
+      family-names: Moore
+      orcid: https://orcid.org/0000-0003-4028-811X
+    - given-names: Chris
+      family-names: Allan
+    - given-names: Sébastien
+      family-names: Besson
+      orcid: https://orcid.org/0000-0001-8783-1429
+    - given-names: Jean-Marie
+      family-names: Burel
+      orcid: https://orcid.org/0000-0002-1789-1861
+    - given-names: Erin
+      family-names: Diel
+      orcid: https://orcid.org/0000-0003-2526-3512
+    - given-names: David
+      family-names: Gault
+    - given-names: Kevin
+      family-names: Kozlowski
+    - given-names: Dominik
+      family-names: Lindner
+      orcid: https://orcid.org/0000-0001-8038-1250
+    - given-names: Melissa
+      family-names: Linkert
+    - given-names: Trevor
+      family-names: Manz
+      orcid: https://orcid.org/0000-0001-7694-5164
+    - given-names: Will
+      family-names: Moore
+      orcid: https://orcid.org/0000-0002-7264-8338
+    - given-names: Constantin
+      family-names: Pape
+      orcid: https://orcid.org/0000-0001-6562-7187
+    - given-names: Christian
+      family-names: Tischer
+      orcid: https://orcid.org/0000-0003-4105-1990
+    - given-names: Jason R.
+      family-names: Swedlow
+      orcid: https://orcid.org/0000-0002-2198-1958
+type: software
+repository-code: https://github.com/ome/ngff

README.md

@@ -3,8 +3,80 @@
 NGFF is an initiative by the bioimaging community to develop imaging format specifications to
 address issues of scalability and interoperability.
 
-This repository contains the central [specification text](./ngff_spec/specification.md),
-a comprehensive list of [metadata examples](./ngff_spec/examples)
-as well as [json schemas](./ngff_spec/schemas) to validate written ome-zarr image data. 
+This repository contains the central [specification text](./specification.md),
+a comprehensive list of [metadata examples](./examples)
+as well as [json schemas](./schemas) to validate written ome-zarr image data.
 
-The built including contribution hints can be found **[here]([url](https://ngff-spec.readthedocs.io/en/latest/specification.html))**.
+The built documentation including contribution hints can be found **[here](https://ngff-spec.readthedocs.io/en/latest/)**.
+
+## Conformance tests
+
+Conformance can be tested at several levels.
+
+1. Validating that individual fields of a zarr attributes object are valid.
+2. Validating a single zarr attributes object (i.e. containing OME-Zarr metadata)
+
+    - Validates that correct data can be represented, and that internally inconsistent data can be caught
+    - Cannot validate references to other objects in the zarr hierarchy
+    - Cannot validate conformance to other zarr metadata e.g. array data type, dimensionality
+
+3. Validating a metadata-only zarr hierarchy
+
+    - Can validate references to other objects and other zarr metadata
+    - Cannot validate values e.g. the invertibility of an affine matrix defined as a zarr array
+
+4. Validating a zarr hierarchy with data
+
+This repository contains
+
+- JSON schemas which handle level 1
+- a set of test zarr attributes JSON for level 2 ([`./tests/attributes`](./tests/attributes/))
+- a set of metadata-only zarr hierarchies for level 3 ([`./tests/zarr`](./tests/zarr/))
+
+as well as a tool for feeding these test cases into an external validator.
+
+### Testing tool
+
+See [`ome_zarr_conformance.py`](./conformance/ome_zarr_conformance.py).
+
+Run it in either `attributes` or `zarr` mode, optionally with filters for test names or types.
+
+Wrap your own OME-Zarr implementation in a
+["dingus"](https://talk.commonmark.org/t/origin-of-the-usage-for-dingus/1226) CLI,
+which takes as its last argument the path to either
+
+- attributes mode: a JSON file representing zarr attributes (i.e. containing OME-Zarr metadata)
+- zarr mode: a zarr hierarchy root on the file system (i.e. a directory containing `zarr.json`)
+
+The dingus should print to STDOUT a JSON object with the keys:
+
+- `"valid"`: boolean, whether this is valid
+- optionally `"message"`: string, free text describing the success/ failure
+
+Call the tool like
+
+```sh
+python3 ./conformance/ome_zarr_conformance.py attributes -- path/to/my/dingus -dingusArg +argValue 10
+```
+
+Each call to the dingus will then look like
+
+```sh
+>>> path/to/my/dingus -dingusArg +argValue 10 /home/you/ngff-spec/tests/attributes/spec/valid/custom_type_axes.json
+
+{"valid": true}
+```
+
+`ome_zarr_conformance.py` will parse the JSON output and format the results of all requested tests in a tab-separated table.
+
+Full usage information is available with `./conformance/ome_zarr_conformance.py --help`.
+
+### JSON Schema tests
+
+You can use the conformance testing tool to test JSON Schema-based validation with
+
+```sh
+>>> ./conformance/ome_zarr_conformance.py attributes -- uv run ./conformance/jsonschema_dingus.py attributes
+```
+
+Some failures are expected as JSON Schema can only handle level 1 validation.

_version.py

@@ -0,0 +1 @@
+__version__ = '0.6.dev3'

ngff_spec/citing.md citing.mdngff_spec/citing.md renamed to citing.md

@@ -1,12 +1,14 @@
-Citing
-======
+---
+title: Citing
+author: ""
+---
 (citing)=
 
 [Next-generation file format (NGFF) specifications for storing bioimaging data in the cloud.](https://ngff.openmicroscopy.org/0.4)
 J. Moore, *et al*. Open Microscopy Environment Consortium, 8 February 2022.
 
 This edition of the specification is [https://ngff.openmicroscopy.org/{{version}}](https://ngff.openmicroscopy.org/{{version}}]).
-The latest edition is publically available {cite}`ngff_spec_latest`.
+The latest edition is publicly available {cite}`ngff_spec_latest`.
 
 ```{bibliography}
 :style: unsrt