zarr-developers
diff --git a/‎.github/workflows/zarr-metadata.yml‎
Lines changed: 4 additions & 1 deletion b/‎.github/workflows/zarr-metadata.yml‎
Lines changed: 4 additions & 1 deletion
diff --git a/‎packages/zarr-metadata/README.md‎
Lines changed: 58 additions & 23 deletions b/‎packages/zarr-metadata/README.md‎
Lines changed: 58 additions & 23 deletions
diff --git a/‎packages/zarr-metadata/changes/4119.feature.md‎
Lines changed: 92 additions & 0 deletions b/‎packages/zarr-metadata/changes/4119.feature.md‎
Lines changed: 92 additions & 0 deletions
diff --git a/‎packages/zarr-metadata/changes/4119.removal.md‎
Lines changed: 41 additions & 0 deletions b/‎packages/zarr-metadata/changes/4119.removal.md‎
Lines changed: 41 additions & 0 deletions
diff --git a/‎packages/zarr-metadata/pyproject.toml‎
Lines changed: 13 additions & 6 deletions b/‎packages/zarr-metadata/pyproject.toml‎
Lines changed: 13 additions & 6 deletions
@@ -82,7 +82,10 @@ jobs:
       - name: Sync test dependency group
         run: uv sync --group test --python 3.11
       - name: Run pyright
-        run: uv run --group test --with pyright pyright src
+        # Pinned to the last version that types PEP 661 sentinels in class
+        # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115).
+        # Unpin when the fix lands.
+        run: uv run --group test --with 'pyright==1.1.404' pyright src
 
   zarr-metadata-complete:
     name: zarr-metadata complete
 
@@ -1,51 +1,86 @@
 # zarr-metadata
 
-Python type definitions for Zarr v2 and v3 metadata.
+Python types, models, and validators for Zarr v2 and v3 metadata.
 
 ## What this is
 
-A typed-data package: `TypedDict` definitions and `Literal` aliases for the
-JSON shapes specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html)
-and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html)
-specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/)
-and a few widely-used-but-unspecified entities (e.g. consolidated metadata).
+Two layers and an optional integration:
+
+- **Typed JSON shapes**: `TypedDict` definitions and `Literal` aliases for the
+  JSON documents specified by the [Zarr v2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html)
+  and [Zarr v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html)
+  specifications, plus types for [`zarr-extensions`](https://github.com/zarr-developers/zarr-extensions/)
+  and a few widely-used-but-unspecified entities (e.g. consolidated metadata).
+- **Document models** (`zarr_metadata.model`): canonical frozen-dataclass
+  models of whole metadata documents, with structural validators, loc-aware
+  parsers, and store-key (de)serialization. A document produced by `to_json`
+  shares no mutable state with the model that produced it.
+- **Optional Pydantic integration** (`zarr_metadata.pydantic`, requires
+  Pydantic 2.13 or newer): each model as a Pydantic field type that validates
+  raw documents through the same strict parser.
 
 ## What this is for
 
-These types describe the JSON shape of Zarr metadata. They are
-intended for libraries that **read, write, validate, or transform**
-Zarr metadata. Pair them with a runtime validator like
-[pydantic](https://docs.pydantic.dev/) to check JSON loaded from disk:
+The public `TypedDict` definitions describe the static JSON shape of Zarr
+metadata. For strict, loc-aware validation of JSON loaded from disk, use the
+model parser:
 
 ```python
 import json
-from pydantic import TypeAdapter
-from zarr_metadata.v3.array import ArrayMetadataV3
+from zarr_metadata.model import ZarrV3ArrayMetadata
 
 with open("zarr.json", "rb") as f:
     raw = json.load(f)
 
-metadata = TypeAdapter(ArrayMetadataV3).validate_python(raw)
+metadata = ZarrV3ArrayMetadata.from_json(raw)
+```
+
+The optional Pydantic integration delegates raw input to the same strict
+parser and returns the same normalized model class:
+
+```python
+from pydantic import TypeAdapter
+import zarr_metadata.pydantic as zmp
+
+metadata = TypeAdapter(zmp.ZarrV3ArrayMetadata).validate_python(raw)
+encoded = metadata.to_key_value()["zarr.json"]
 ```
 
-## What this is *not*
+A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape
+adapter, not a Zarr conformance validator; it may coerce values or discard
+members that the strict model parser rejects.
+
+## Validation boundary
 
-- Not a parser or builder. There are no `make_array_metadata(...)` factories —
-  that surface belongs to consumer libraries.
-- Not a runtime validator on its own. Pair with `pydantic`, `msgspec`, or
-  similar to enforce shapes at decode time.
+The model validators enforce the declared document structure and a small set
+of context-free consistency rules, including fixed format literals, finite
+JSON numbers, non-negative dimensions, non-empty v3 codec pipelines, and one
+`dimension_names` entry per array dimension. They do not interpret extension
+names or configurations, resolve codec pipelines, or decide whether a data
+type, chunk grid, codec, or storage transformer is supported. Those decisions
+belong to consumer implementations.
 
-Even with a runtime validator, these types only describe **structural**
-shape — they will not flag *semantically* invalid metadata, like a 3D v3
-array whose `dimension_names` has 4 entries instead of 3. That's a job
-for downstream validator routines.
+The Pydantic integration's generated JSON Schemas express independently
+checkable document structure and field constraints, but they are not a
+replacement for runtime model validation. Standard JSON Schema treats a
+mathematically integral number such as `1.0` as an integer, while the runtime
+boundary requires Python `int` values, and it cannot express arbitrary
+same-length relations such as `dimension_names` versus `shape` or v2 `chunks`
+versus `shape`. Consumers should run the model parser after schema validation.
 
 ## Scope
 
 At minimum, this library supports what Zarr-Python needs: the complete
 Zarr v2 and v3 specs, consolidated metadata, and a subset of the metadata
 defined in `zarr-extensions`. We are generally open to contributions that
-add types for Zarr metadata with a published spec.
+add types, models, or structural validation for Zarr metadata with a
+published spec.
+
+Runtime array behavior is out of scope: nothing here encodes or decodes
+chunks, resolves codec or data type names to implementations, or performs
+store I/O. The models begin and end at the metadata documents themselves —
+`from_key_value` / `to_key_value` map documents to store keys and bytes,
+and everything past that belongs to consumer libraries.
 
 ## Releasing
 
 
@@ -0,0 +1,92 @@
+Added `zarr_metadata.model`: frozen-dataclass models (`ZarrV2ArrayMetadata`,
+`ZarrV3ArrayMetadata`, `ZarrV2GroupMetadata`, `ZarrV3GroupMetadata`,
+`ZarrV2ConsolidatedMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV3NamedConfig`)
+that are canonical, semantically lossless representations of Zarr metadata
+documents, plus structural validators (`validate_*` / `is_*` / `parse_*`).
+Every v3 extension point (data type, chunk grid, chunk key encoding, codecs,
+storage transformers) is held as `ZarrV3NamedConfig`: a name, configuration,
+and `must_understand` obligation; nothing is interpreted. On the wire, an
+empty configuration with the default obligation uses the spec's plain-string
+shorthand. Model fields are annotated with the role alias
+`ZarrV3MetadataField` (today exactly `ZarrV3NamedConfig`), so annotations
+convey the logical meaning and stay put if the spec adds another field form.
+
+Validation is strict about what the types declare: v2 `dtype` / `order` /
+`compressor` / `filters` / `dimension_separator` shapes and the fixed
+`zarr_format` / `node_type` literals are all enforced. Every
+`ValidationProblem` carries a machine-readable `kind`
+(`missing_key` / `invalid_type` / `invalid_value` / `invalid_json`) so
+consumers can dispatch on the failure mode without matching message strings,
+and every ingestion failure — including missing store keys and undecodable
+bytes in `from_key_value` — surfaces as `MetadataValidationError`. An
+adversarial review added further structural checks: JSON booleans are not
+accepted as dimension lengths, dimensions are non-negative,
+`dimension_names` must have one entry per dimension of `shape`, `attributes`
+and `configuration` values are JSON-checked recursively (like `fill_value`),
+non-finite floats and non-standard JSON constants are rejected, abstract
+mappings and sequences normalize to encoder-safe canonical containers,
+v2 `shape` and `chunks` must have the same rank, non-null v2 filter pipelines
+contain at least one filter, document `TypeIs` guards only narrow values that
+already use the declared canonical containers,
+and the inline consolidated-metadata envelope and entries are deep-validated
+so the group validator's verdict always agrees with the model constructor.
+
+The v3 models expose `must_understand_fields`: the subset of `extra_fields`
+not explicitly waived with `must_understand: false` (fields are implicitly
+must-understand per the spec). Readers discharge the spec's fail-to-open
+duty by subtracting the extension names they recognize; the model only
+partitions by obligation, since recognition is reader-specific.
+
+Optional pydantic integration ships as `zarr_metadata.pydantic` (importing it
+requires pydantic 2.13 or newer; the core package does not depend on it): one
+`Annotated`
+field type per model, validating raw documents through `from_json`, passing
+core-model instances through unchanged, serializing via `to_json`, and
+publishing JSON Schemas derived from private constrained document types that
+mirror the independently expressible runtime rules. Cross-field cardinality
+relations still require runtime validation. The instances are the core model
+classes, so values interoperate freely with non-pydantic code.
+
+`create_default` keeps its output self-consistent: overriding `shape` without
+a chunk grid derives one regular chunk covering the array (v3
+`chunk_shape == shape`; v2 `chunks == shape`) instead of silently keeping the
+scalar default's 0-d grid.
+
+A v2 `.zarray` that omits `dimension_separator` is interpreted with the v2
+convention's default `"."` (the model previously normalized absence to `"/"`,
+which would misaddress the chunks of real-world default-separator arrays).
+The value is never null: absent, `"."`, or `"/"` are the only spellings.
+
+Optional document keys use `UNSET` — a PEP 661 sentinel
+(`typing_extensions.Sentinel`), usable directly in type expressions — never
+`None`: in a model, `None` always corresponds to a JSON `null` in the
+document (a v2 `compressor`, an unnamed dimension inside `dimension_names`),
+and `UNSET` always means the key is absent. Checker note: ty types the
+sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115
+is fixed (this package's CI pins it); mypy users need a `cast` or
+`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings
+distinct — an absent `dimension_names` ("there are no dimension names") and
+an explicit `[null, null]` ("every dimension has a name, which is null") are
+different documents and round-trip as such. The `consolidated_metadata: null`
+written by a historical zarr-python bug is the one deliberate exception to
+faithful round-tripping: those stores remain readable, but the bug spelling
+is repaired to absence on read and never written back.
+
+The v2 models treat the `.zattrs` file's presence as part of the store:
+`attributes` is `UNSET` when no `.zattrs` file exists (and `to_key_value`
+emits none), while an explicit empty `.zattrs` is `{}` and round-trips as a
+file. Previously `to_key_value` always emitted `.zattrs`, silently adding a
+file to stores that never had one.
+
+The store-key `Literal` aliases (`ZarrV2ArrayMetadataStoreKey`,
+`ZarrV2AttributesStoreKey`, ...) are exported from `zarr_metadata.model`
+alongside their constants, and each `to_key_value` return type is keyed by
+them, so the set of store keys a model can emit is visible in its signature.
+`from_key_value` deliberately keeps `Mapping[str, bytes]` input: it accepts
+any string-keyed store mapping and ignores unrelated keys.
+
+`to_json` returns a document that shares no mutable state with the model:
+every value that can hold a mutable container (attributes, configurations,
+extra fields, v2 codec configurations, fill values, consolidated entries) is
+deep-copied on the way out, so editing a serialized document can never
+silently mutate the frozen model that produced it.
@@ -0,0 +1,41 @@
+The document (TypedDict) types are renamed to put the format version at the
+front of the name and to mark the JSON-document form with a `JSON` suffix,
+so a format version can never be misread as a class revision and the bare
+entity names are reserved for the `zarr_metadata.model` dataclasses:
+
+- `ArrayMetadataV2` → `ZarrV2ArrayMetadataJSON` (and `...Partial` accordingly)
+- `ArrayMetadataV3` → `ZarrV3ArrayMetadataJSON` (and `...Partial` accordingly)
+- `GroupMetadataV2` → `ZarrV2GroupMetadataJSON` (and `...Partial` accordingly)
+- `GroupMetadataV3` → `ZarrV3GroupMetadataJSON` (and `...Partial` accordingly)
+- `ConsolidatedMetadataV2` → `ZarrV2ConsolidatedMetadataJSON`
+- `ConsolidatedMetadataV3` → `ZarrV3ConsolidatedMetadataJSON`
+- `NamedConfigV3` → `ZarrV3NamedConfigJSON`
+- `MetadataV3` → `ZarrV3MetadataFieldJSON` (the union of the bare-name and
+  named-configuration spellings of one metadata field)
+- `ExtensionFieldV3` → `ZarrV3ExtensionField`
+- `CodecMetadataV2` → `ZarrV2CodecMetadata`
+- `DataTypeMetadataV2` → `ZarrV2DataTypeMetadata`
+- `ArrayOrderV2` → `ZarrV2ArrayOrder`
+- `ArrayDimensionSeparatorV2` → `ZarrV2ArrayDimensionSeparator`
+- `ZArrayMetadata` → `ZarrV2ZArrayJSON` (the strict on-disk `.zarray` document)
+- `ZGroupMetadata` → `ZarrV2ZGroupJSON` (the strict on-disk `.zgroup` document)
+- `ZAttrsMetadata` → `ZarrV2ZAttrsJSON` (the `.zattrs` document)
+
+The old names are removed, not aliased. The `zarr_metadata.pydantic` field
+types take the bare entity names (`ZarrV3ArrayMetadata`, ...), matching the
+model classes they validate into.
+
+The conventions, stated once for future additions: CamelCase type names put
+the format version first (`ZarrV2ArrayMetadataJSON`,
+`ZarrV3ArrayMetadataStoreKey`), while SCREAMING_SNAKE constants and
+snake_case functions put it last (`ARRAY_METADATA_STORE_KEY_V2`,
+`validate_array_metadata_v3`). The `JSON` suffix marks a raw-document type
+whose bare name is taken by (or reserved for) a `zarr_metadata.model`
+dataclass; raw field-level types the models hold verbatim
+(`ZarrV2CodecMetadata`, `ZarrV3ExtensionField`) keep their bare names.
+Extension-entity types put the registered entity name first and end in
+exactly one role suffix (`BloscCodecMetadata`, `Uint8DataTypeName`) — the
+`V2` in `V2ChunkKeyEncodingMetadata` is that encoding's entity name, not a
+format version, which is always spelled `ZarrV2`/`ZarrV3`. Every public
+type name is checked against this grammar by
+`tests/test_public_api.py::test_public_type_names_comply_with_naming_grammar`.
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
 [project]
 name = "zarr-metadata"
 dynamic = ["version"]
-description = "Spec-defined metadata types for Zarr v2 and v3."
+description = "Spec-defined metadata types, models, and validators for Zarr v2 and v3."
 readme = "README.md"
 requires-python = ">=3.11"
 license = "MIT"
@@ -32,7 +32,11 @@ classifiers = [
 ]
 keywords = ["zarr"]
 dependencies = [
-    "typing_extensions>=4.13",
+    # >=4.16: first release where `Sentinel` pickles by reference
+    # (`__reduce__` returns the sentinel's name), so `UNSET` — and any model
+    # holding it — can cross process boundaries and be deep-copied with its
+    # singleton identity intact. 4.15 and earlier refuse to pickle sentinels.
+    "typing_extensions>=4.16",
 ]
 
 [project.urls]
@@ -43,7 +47,7 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z
 Documentation = "https://github.com/zarr-developers/zarr-python/blob/main/packages/zarr-metadata/README.md"
 
 [dependency-groups]
-test = ["pytest", "pydantic>=2"]
+test = ["pytest", "pydantic>=2.13", "jsonschema"]
 
 [tool.hatch.version]
 source = "vcs"
@@ -67,9 +71,9 @@ xfail_strict = true
 addopts = ["-ra", "--strict-config", "--strict-markers"]
 filterwarnings = [
     "error",
-    # pydantic warns about ReadOnly TypedDict items not being enforced at runtime.
-    # That's expected here — we rely on type-checker enforcement, not pydantic mutation guards.
-    "ignore::UserWarning:pydantic._internal._generate_schema",
+    # Pydantic validates these public immutable-shape TypedDicts correctly but
+    # cannot enforce the type checker's ReadOnly mutation restriction.
+    "ignore:Items? .* using the `ReadOnly` qualifier.*:UserWarning:pydantic._internal._generate_schema",
 ]
 
 [tool.numpydoc_validation]
@@ -82,6 +86,9 @@ checks = [
     "PR06",
 ]
 
+# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in
+# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel
+# relies on. Use the same pin locally; unpin when the fix lands.
 [tool.pyright]
 include = ["src"]
 enableExperimentalFeatures = true