Conformance testing docs

Also minor refactor test_validation

Author: clbarnes

README.md

@@ -5,6 +5,64 @@ address issues of scalability and interoperability.
 
 This repository contains the central [specification text](./ngff_spec/specification.md),
 a comprehensive list of [metadata examples](./ngff_spec/examples)
-as well as [json schemas](./ngff_spec/schemas) to validate written ome-zarr image data. 
+as well as [json schemas](./ngff_spec/schemas) to validate written ome-zarr image data.
 
 The built documentation including contribution hints can be found **[here](https://ngff-spec.readthedocs.io/en/latest/specification.html)**.
+
+## Conformance tests
+
+Conformance can be tested at several levels.
+
+1. Validating a single zarr attributes object (i.e. containing OME-Zarr metadata)
+
+    - Validates that correct data can be represented, and that internally inconsistent data can be caught
+    - Cannot validate references to other objects in the zarr hierarchy
+    - Cannot validate conformance to other zarr metadata e.g. array data type, dimensionality
+
+2. Validating a metadata-only zarr hierarchy
+
+    - Can validate references to other objects and other zarr metadata
+    - Cannot validate values e.g. the invertibility of an affine matrix defined as a zarr array
+
+3. Validating a zarr hierarchy with data
+
+This repository contains a set of test cases for levels
+1 ([`./tests/attributes`](./tests/attributes/)) and
+2 ([`./tests/zarr`](./tests/zarr/)),
+as well as a tool for feeding these test cases into an external validator.
+
+### Testing tool
+
+See [`ome_zarr_conformance.py`](./ome_zarr_conformance.py).
+
+Run it in either `attributes` or `zarr` mode, optionally with filters for test names or types.
+
+Wrap your own OME-Zarr implementation in a
+["dingus"](https://talk.commonmark.org/t/origin-of-the-usage-for-dingus/1226) CLI,
+which takes as its last argument the path to either
+
+- attributes mode: a JSON file representing zarr attributes (i.e. containing OME-Zarr metadata)
+- zarr mode: a zarr hierarchy root on the file system (i.e. a directory containing `zarr.json`)
+
+The dingus should print to STDOUT a JSON object with the keys:
+
+- `"valid"`: boolean, whether this is valid
+- optionally `"message"`: str, free text describing the success/ failure
+
+Call the tool like
+
+```sh
+python3 ./ome_zarr_conformance.py attributes -- path/to/my/dingus -dingusArg +argValue 10
+```
+
+Each call to the dingus will then look like
+
+```sh
+>>> path/to/my/dingus -dingusArg +argValue 10 /home/you/ngff-spec/tests/attributes/spec/valid/custom_type_axes.json
+
+{"valid": true}
+```
+
+`ome_zarr_conformance.py` will parse the JSON output and format the results of all requested tests in a tab-separated table.
+
+Full usage information is available with `./ome_zarr_conformance.py --help`.

ngff_spec/schemas/strict_ome_zarr.schema

@@ -1,6 +1,6 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://ngff.openmicroscopy.org/0.6dev2/schemas/ome_zarr.schema",
+  "$id": "https://ngff.openmicroscopy.org/0.6dev2/schemas/strict_ome_zarr.schema",
   "anyOf": [
     {
       "$ref": "https://ngff.openmicroscopy.org/0.6dev2/schemas/bf2raw.schema"

tests/test_validation.py

@@ -19,8 +19,9 @@
 GENERIC_SCHEMA = schema_store[
     "https://ngff.openmicroscopy.org/0.6dev2/schemas/ome_zarr.schema"
 ]
-
-print(schema_store)
+GENERIC_STRICT_SCHEMA = schema_store[
+    "https://ngff.openmicroscopy.org/0.6dev2/schemas/strict_ome_zarr.schema"
+]
 
 
 @dataclass
@@ -57,60 +58,60 @@ def pytest_generate_tests(metafunc):
         styled JSON tests. Metadata in each test defines which schema is used
         and whether or not the block is considered valid.
     """
-    if "suite" in metafunc.fixturenames:
-        suites: List[Schema] = []
-        ids: List[str] = []
-        schema_store = {}
-        for filename in glob.glob("ngff_spec/schemas/*.schema"):
-            with open(filename) as o:
-                schema = json.load(o)
-            schema_store[schema["$id"]] = schema
-
-        # Validation
-        for filename in glob.glob("tests/*.json"):
-            with open(filename) as o:
-                suite = json.load(o)
-            schema = suite["schema"]
-            with open(schema["id"]) as f:
-                schema = json.load(f)
-            for test in suite["tests"]:
-                ids.append("validate_" + str(test["formerly"]).split("/")[-1][0:-5])
-                suites.append(Suite(schema, test["data"], test["valid"]))
-
-        # Examples
-        for config_filename in glob.glob("examples/*/.config.json"):
-            with open(config_filename) as o:
-                data = json.load(o)
-            schema = data["schema"]
-            with open(schema) as f:
-                schema = json.load(f)
-            example_folder = os.path.dirname(config_filename)
-            for filename in glob.glob(f"{example_folder}/*.json"):
-                with open(filename) as f:
-                    # Strip comments
-                    data = "".join(
-                        line for line in f if not line.lstrip().startswith("//")
-                    )
-                    data = json.loads(data)
-                    data = data["attributes"]  # Only validate the attributes object
-                ids.append("example_" + str(filename).split("/")[-1][0:-5])
-                suites.append(Suite(schema, data, True))  # Assume true
-
-        metafunc.parametrize("suite", suites, ids=ids, indirect=True)
+    if "suite" not in metafunc.fixturenames:
+        return
+
+    suites: List[Suite] = []
+    ids: List[str] = []
+    schema_store = {}
+    for filename in glob.glob("ngff_spec/schemas/*.schema"):
+        with open(filename) as o:
+            schema = json.load(o)
+        schema_store[schema["$id"]] = schema
+
+    # Validation
+    for filename in glob.glob("tests/*.json"):
+        with open(filename) as o:
+            suite = json.load(o)
+        schema_path = suite["schema"]
+        with open(schema_path["id"]) as f:
+            schema = json.load(f)
+        for test in suite["tests"]:
+            ids.append("validate_" + str(test["formerly"]).split("/")[-1][0:-5])
+            suites.append(Suite(schema, test["data"], test["valid"]))
+
+    # Examples
+    for config_filename in glob.glob("examples/*/.config.json"):
+        with open(config_filename) as o:
+            data = json.load(o)
+        schema_path = data["schema"]
+        with open(schema_path) as f:
+            schema = json.load(f)
+        example_folder = os.path.dirname(config_filename)
+        for filename in glob.glob(f"{example_folder}/*.json"):
+            with open(filename) as f:
+                # Strip comments
+                data = "".join(line for line in f if not line.lstrip().startswith("//"))
+                data = json.loads(data)
+                data = data["attributes"]  # Only validate the attributes object
+            ids.append("example_" + str(filename).split("/")[-1][0:-5])
+            suites.append(Suite(schema, data, True))  # Assume true
+
+    metafunc.parametrize("suite", suites, ids=ids, indirect=True)
 
 
 @pytest.fixture
-def suite(request):
+def suite(request) -> Suite:
     return request.param
 
 
-def test_run(suite):
+def test_run(suite: Suite):
     resolver = RefResolver.from_schema(suite.schema, store=schema_store)
     validator = Validator(suite.schema, resolver=resolver)
     suite.validate(validator)
 
 
-def test_generic_run(suite):
+def test_generic_run(suite: Suite):
     resolver = RefResolver.from_schema(GENERIC_SCHEMA, store=schema_store)
     validator = Validator(GENERIC_SCHEMA, resolver=resolver)
     suite.maybe_validate(validator)