Split tests rfc5

README.md

@@ -5,6 +5,78 @@ address issues of scalability and interoperability.
 
 This repository contains the central [specification text](./ngff_spec/specification.md),
 a comprehensive list of [metadata examples](./ngff_spec/examples)
-as well as [json schemas](./ngff_spec/schemas) to validate written ome-zarr image data. 
+as well as [json schemas](./ngff_spec/schemas) to validate written ome-zarr image data.
 
 The built documentation including contribution hints can be found **[here](https://ngff-spec.readthedocs.io/en/latest/specification.html)**.
+
+## Conformance tests
+
+Conformance can be tested at several levels.
+
+1. Validating that individual fields of a zarr attributes object are valid.
+2. Validating a single zarr attributes object (i.e. containing OME-Zarr metadata)
+
+    - Validates that correct data can be represented, and that internally inconsistent data can be caught
+    - Cannot validate references to other objects in the zarr hierarchy
+    - Cannot validate conformance to other zarr metadata e.g. array data type, dimensionality
+
+3. Validating a metadata-only zarr hierarchy
+
+    - Can validate references to other objects and other zarr metadata
+    - Cannot validate values e.g. the invertibility of an affine matrix defined as a zarr array
+
+4. Validating a zarr hierarchy with data
+
+This repository contains
+
+- JSON schemas which handle level 1
+- a set of test zarr attributes JSON for level 2 ([`./tests/attributes`](./tests/attributes/))
+- a set of metadata-only zarr hierarchies for level 3 ([`./tests/zarr`](./tests/zarr/))
+
+as well as a tool for feeding these test cases into an external validator.
+
+### Testing tool
+
+See [`ome_zarr_conformance.py`](./ome_zarr_conformance.py).
+
+Run it in either `attributes` or `zarr` mode, optionally with filters for test names or types.
+
+Wrap your own OME-Zarr implementation in a
+["dingus"](https://talk.commonmark.org/t/origin-of-the-usage-for-dingus/1226) CLI,
+which takes as its last argument the path to either
+
+- attributes mode: a JSON file representing zarr attributes (i.e. containing OME-Zarr metadata)
+- zarr mode: a zarr hierarchy root on the file system (i.e. a directory containing `zarr.json`)
+
+The dingus should print to STDOUT a JSON object with the keys:
+
+- `"valid"`: boolean, whether this is valid
+- optionally `"message"`: string, free text describing the success/ failure
+
+Call the tool like
+
+```sh
+python3 ./ome_zarr_conformance.py attributes -- path/to/my/dingus -dingusArg +argValue 10
+```
+
+Each call to the dingus will then look like
+
+```sh
+>>> path/to/my/dingus -dingusArg +argValue 10 /home/you/ngff-spec/tests/attributes/spec/valid/custom_type_axes.json
+
+{"valid": true}
+```
+
+`ome_zarr_conformance.py` will parse the JSON output and format the results of all requested tests in a tab-separated table.
+
+Full usage information is available with `./ome_zarr_conformance.py --help`.
+
+### JSON Schema tests
+
+You can use the conformance testing tool to test JSON Schema-based validation with
+
+```sh
+>>> ./ome_zarr_conformance.py attributes -- uv run jsonschema_dingus.py attributes
+```
+
+Some failures are expected as JSON Schema can only handle level 1 validation.

jsonschema_dingus.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# /// script
+# dependencies = [
+#   "jsonschema",
+#   "referencing",
+# ]
+# ///
+from __future__ import annotations
+from dataclasses import dataclass
+from pathlib import Path
+import json
+from argparse import ArgumentParser
+from typing import Any, Self
+
+from referencing import Registry, Resource
+from jsonschema import Draft202012Validator as Validator
+from jsonschema.exceptions import ValidationError
+
+HERE = Path(__file__).resolve().parent
+SCHEMA_DIR = HERE / "ngff_spec" / "schemas"
+
+
+@dataclass
+class SchemasInfo:
+    generic_id: str
+    base_url: str
+    registry: Registry
+
+    @classmethod
+    def load(cls) -> Self:
+        """Get the ID of the top-level schema, and the mapping of schema IDs to objects"""
+        registry = Registry()
+        generic = None
+        base_url = None
+        for p in SCHEMA_DIR.glob("*.schema*"):
+            schema = json.loads(p.read_text())
+            resource = Resource.from_contents(schema)
+            registry = resource @ registry
+
+            schema_id = resource.id()
+            if schema_id is None:
+                raise RuntimeError("schema has no ID")
+            if p.stem == "ome_zarr":
+                generic = schema_id
+                base_url = generic.split("/schemas/")[0]
+
+        if generic is None or base_url is None:
+            raise RuntimeError("Could not find generic ome_zarr schema")
+
+        return cls(generic, base_url, registry)
+
+
+def main(raw_args=None):
+    parser = ArgumentParser()
+    parser.add_argument("mode", choices=["attributes", "zarr"])
+    parser.add_argument("path", type=Path)
+
+    args = parser.parse_args(raw_args)
+
+    p: Path = args.path
+    attrs: None | dict[str, Any] = None
+    match args.mode:
+        case "attributes":
+            attrs = json.loads(p.read_text())
+        case "zarr":
+            attrs = json.loads(p.joinpath("zarr.json").read_text())["attributes"]
+        case _:
+            raise RuntimeError("unreachable")
+
+    if attrs is None:
+        raise RuntimeError("unreachable")
+
+    schemas = SchemasInfo.load()
+
+    generic_schema = schemas.registry.get(schemas.generic_id)
+    if generic_schema is None:
+        raise RuntimeError("could not find generic schema")
+
+    validator = Validator(
+        generic_schema.contents,
+        registry=schemas.registry,
+    )
+
+    result = dict()
+    try:
+        validator.validate(attrs)
+        result["valid"] = True
+    except ValidationError as e:
+        result["valid"] = False
+        result["message"] = str(e)
+
+    print(json.dumps(result))
+
+
+if __name__ == "__main__":
+    main()

ngff_spec/pre_build.py

@@ -74,6 +74,7 @@ def build_json_examples():
 def build_json_schemas():
     from json_schema_for_humans.generate import generate_from_filename
     from json_schema_for_humans.generation_configuration import GenerationConfiguration
+    import json
 
     schema_source_dir = 'schemas'
     output_directory = '_generated/schemas'

ngff_spec/schemas/strict_coordinate_systems.schema

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_coordinate_systems.schema",
   "allOf" : [
     {

ngff_spec/schemas/strict_image.schema

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_image.schema",
   "allOf": [
     {

ngff_spec/schemas/strict_label.schema

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_label.schema",
   "allOf": [
     {

ngff_spec/schemas/strict_ome_zarr.schema

@@ -0,0 +1,24 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_ome_zarr.schema",
+  "anyOf": [
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/bf2raw.schema"
+    },
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_image.schema"
+    },
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_label.schema"
+    },
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/ome.schema"
+    },
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_plate.schema"
+    },
+    {
+      "$ref": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_well.schema"
+    }
+  ]
+}

ngff_spec/schemas/strict_plate.schema

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_plate.schema",
   "allOf": [
     {

ngff_spec/schemas/strict_well.schema

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://ngff.openmicroscopy.org/0.6.dev2/schemas/strict_well.schema",
   "allOf": [
     {