feat(zarr-metadata): adopt the PEP 661 sentinel for UNSET

d-v-b's call: PEP 661 is Final, ty already types the sentinel spelling
exactly, mypy support is in review (python/mypy#21647) and treated as
imminent, and pyright has a known-good version — so use the standard
sentinel today rather than carrying the enum stopgap.

- UNSET is now typing_extensions.Sentinel("UNSET"), used directly in
  type expressions (tuple[str | None, ...] | UNSET); the UnsetType
  companion enum is gone from the API.
- typing_extensions floor bumped to 4.14 (where Sentinel arrived).
- CI pins pyright==1.1.404, the last version before the class-attribute
  sentinel regression (microsoft/pyright#11115); pyproject documents the
  same pin for local runs. 0 errors on the pin; ty checks the sentinel
  fields clean (its 2 remaining diagnostics are its incomplete PEP 728
  extra_items write support, unrelated).
- Known short-term cost, accepted deliberately: mypy-checked consumers
  need cast/type-ignore at narrowing sites until mypy#21647 merges, and
  contributors' Pylance may show phantom Unknowns until the pyright fix
  ships. Recorded in _sentinel.py and the changelog.
- The pydantic native-introspection test reverts to documenting that
  introspection is unsupported (pydantic 2.13 cannot schema a Sentinel);
  the delegation patterns are unaffected.

Assisted-by: ClaudeCode:claude-fable-5

Author: d-v-b

.github/workflows/zarr-metadata.yml

@@ -82,7 +82,10 @@ jobs:
       - name: Sync test dependency group
         run: uv sync --group test --python 3.11
       - name: Run pyright
-        run: uv run --group test --with pyright pyright src
+        # Pinned to the last version that types PEP 661 sentinels in class
+        # attributes correctly; 1.1.405+ regressed (microsoft/pyright#11115).
+        # Unpin when the fix lands.
+        run: uv run --group test --with 'pyright==1.1.404' pyright src
 
   zarr-metadata-complete:
     name: zarr-metadata complete

packages/zarr-metadata/changes/210.feature.md

@@ -47,10 +47,14 @@ convention's default `"."` (the model previously normalized absence to `"/"`,
 which would misaddress the chunks of real-world default-separator arrays).
 The value is never null: absent, `"."`, or `"/"` are the only spellings.
 
-Optional document keys use the `UNSET` sentinel, never `None`: in a model,
-`None` always corresponds to a JSON `null` in the document (a v2
-`compressor`, an unnamed dimension inside `dimension_names`), and `UNSET`
-always means the key is absent. This keeps semantically distinct spellings
+Optional document keys use `UNSET` — a PEP 661 sentinel
+(`typing_extensions.Sentinel`), usable directly in type expressions — never
+`None`: in a model, `None` always corresponds to a JSON `null` in the
+document (a v2 `compressor`, an unnamed dimension inside `dimension_names`),
+and `UNSET` always means the key is absent. Checker note: ty types the
+sentinel exactly; pyright needs `<= 1.1.404` until microsoft/pyright#11115
+is fixed (this package's CI pins it); mypy users need a `cast` or
+`type: ignore` at narrowing sites until python/mypy#21647 merges. This keeps semantically distinct spellings
 distinct — an absent `dimension_names` ("there are no dimension names") and
 an explicit `[null, null]` ("every dimension has a name, which is null") are
 different documents and round-trip as such. The `consolidated_metadata: null`

packages/zarr-metadata/pyproject.toml

@@ -32,7 +32,7 @@ classifiers = [
 ]
 keywords = ["zarr"]
 dependencies = [
-    "typing_extensions>=4.13",
+    "typing_extensions>=4.14",
 ]
 
 [project.urls]
@@ -82,6 +82,9 @@ checks = [
     "PR06",
 ]
 
+# CI pins pyright==1.1.404: later versions regress PEP 661 sentinel typing in
+# class attributes (microsoft/pyright#11115), which zarr_metadata.model._sentinel
+# relies on. Use the same pin locally; unpin when the fix lands.
 [tool.pyright]
 include = ["src"]
 enableExperimentalFeatures = true

packages/zarr-metadata/src/zarr_metadata/__init__.py

@@ -17,7 +17,6 @@
     MetadataValidationError,
     NamedConfigModelV3,
     ProblemKind,
-    UnsetType,
     ValidationProblem,
 )
 from zarr_metadata.v2.array import (
@@ -356,7 +355,6 @@
     "Uint32FillValue",
     "Uint64DataTypeName",
     "Uint64FillValue",
-    "UnsetType",
     "V2ChunkKeyEncodingMetadata",
     "V2ChunkKeyEncodingName",
     "V2ChunkKeyEncodingSeparator",

packages/zarr-metadata/src/zarr_metadata/model/__init__.py

@@ -34,7 +34,7 @@
     GroupMetadataModelV3,
     GroupMetadataModelV3Partial,
 )
-from zarr_metadata.model._sentinel import UNSET, UnsetType
+from zarr_metadata.model._sentinel import UNSET
 from zarr_metadata.model._validation import (
     ARRAY_METADATA_OPTIONAL_KEYS_V3,
     ARRAY_METADATA_REQUIRED_KEYS_V2,
@@ -98,7 +98,6 @@
     "MetadataValidationError",
     "NamedConfigModelV3",
     "ProblemKind",
-    "UnsetType",
     "ValidationProblem",
     "is_array_metadata_v2",
     "is_array_metadata_v3",

packages/zarr-metadata/src/zarr_metadata/model/_array.py

@@ -10,7 +10,7 @@
 
 from typing_extensions import TypedDict, Unpack
 
-from zarr_metadata.model._sentinel import UNSET, UnsetType
+from zarr_metadata.model._sentinel import UNSET
 from zarr_metadata.model._validation import (
     ARRAY_METADATA_STANDARD_KEYS_V3,
     MetadataValidationError,
@@ -134,7 +134,7 @@ class ArrayMetadataModelV3Partial(TypedDict, total=False):
     chunk_grid: MetadataFieldModelV3
     codecs: tuple[MetadataFieldModelV3, ...]
     chunk_key_encoding: MetadataFieldModelV3
-    dimension_names: tuple[str | None, ...] | UnsetType
+    dimension_names: tuple[str | None, ...] | UNSET
     attributes: dict[str, JSONValue]
     storage_transformers: tuple[MetadataFieldModelV3, ...]
     extra_fields: dict[str, ExtensionFieldV3]
@@ -160,7 +160,7 @@ class ArrayMetadataModelV3:
     chunk_grid: MetadataFieldModelV3
     codecs: tuple[MetadataFieldModelV3, ...]
     chunk_key_encoding: MetadataFieldModelV3
-    dimension_names: tuple[str | None, ...] | UnsetType
+    dimension_names: tuple[str | None, ...] | UNSET
     attributes: dict[str, JSONValue]
     storage_transformers: tuple[MetadataFieldModelV3, ...]
     extra_fields: dict[str, ExtensionFieldV3]

packages/zarr-metadata/src/zarr_metadata/model/_group.py

@@ -15,7 +15,7 @@
     ArrayMetadataModelV3,
     must_understand_subset,
 )
-from zarr_metadata.model._sentinel import UNSET, UnsetType
+from zarr_metadata.model._sentinel import UNSET
 from zarr_metadata.model._validation import (
     GROUP_METADATA_STANDARD_KEYS_V3,
     MetadataValidationError,
@@ -64,7 +64,7 @@ class GroupMetadataModelV3Partial(TypedDict, total=False):
     """
 
     attributes: dict[str, JSONValue]
-    consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType
+    consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET
     extra_fields: dict[str, ExtensionFieldV3]
 
 
@@ -81,7 +81,7 @@ class GroupMetadataModelV3:
     zarr_format: Literal[3] = field(default=3, init=False)
     node_type: Literal["group"] = field(default="group", init=False)
     attributes: dict[str, JSONValue]
-    consolidated_metadata: ConsolidatedMetadataModelV3 | UnsetType
+    consolidated_metadata: ConsolidatedMetadataModelV3 | UNSET
     extra_fields: dict[str, ExtensionFieldV3]
 
     def __post_init__(self) -> None:
@@ -147,7 +147,7 @@ def from_json(cls, data: object) -> GroupMetadataModelV3:
         # Cast to object: the TypedDict's extra_items type does not admit null,
         # but wild documents (historical zarr-python) contain it.
         consolidated_raw = cast("object", parsed.get(CONSOLIDATED_METADATA_KEY_V3, UNSET))
-        consolidated: ConsolidatedMetadataModelV3 | UnsetType
+        consolidated: ConsolidatedMetadataModelV3 | UNSET
         if consolidated_raw is UNSET or consolidated_raw is None:
             # consolidated_metadata: null was written by a historical
             # zarr-python bug; it gets no model representation. It is read as

packages/zarr-metadata/src/zarr_metadata/model/_sentinel.py

@@ -3,43 +3,27 @@
 The models observe one invariant: `None` in a model always corresponds to a
 JSON `null` in the document (a v2 `compressor`/`filters` value, an unnamed
 dimension inside `dimension_names`), and `UNSET` always means the document
-key is absent. The two are never interchangeable, so a model value can never leak
-into a document as a spelling the writer did not intend.
+key is absent. The two are never interchangeable, so a model value can never
+leak into a document as a spelling the writer did not intend.
 
 Check with identity: `if model.dimension_names is UNSET: ...`.
 
-Implementation note: `typing_extensions.Sentinel` (PEP 661, Final as of
-2026-04-23, stdlib in Python 3.15) is the intended spelling, but two
-independent checker gaps block it for now. Pyright: a confirmed regression
-(1.1.405 through at least 1.1.411; worked in <= 1.1.404,
-https://github.com/microsoft/pyright/issues/11115) degrades a Sentinel to
-`Unknown` when read from any class-body attribute annotation. Mypy (2.1.0):
-has not yet implemented PEP 661 — a sentinel in type position is a hard
-`[valid-type]` error, so downstream mypy users (zarr-python itself) would
-see these fields as `Any`. Pinning a working pyright in this package's CI
-would fix neither contributors' IDEs nor downstream checkers reading the
-py.typed annotations. For calibration: ty (0.0.56)
-already types the Sentinel spelling perfectly — exact unions and
-`is`/`is not` narrowing in dataclass fields — so the standard is landing;
-pyright and mypy are the laggards. The single-member enum gives the same
-identity semantics with exact `Literal` narrowing on every checker; switch
-to `Sentinel` once the pyright regression is fixed and mypy support lands.
+Checker support (PEP 661 is Final; stdlib `sentinel` arrives in Python
+3.15): ty types this spelling exactly, including `is`/`is not` narrowing.
+Pyright supports it but a regression (1.1.405+, tracked as
+https://github.com/microsoft/pyright/issues/11115) degrades class-attribute
+reads to `Unknown`, so this package pins pyright to the last good version
+until the fix lands. Mypy support is in review
+(https://github.com/python/mypy/pull/21647); until it merges, mypy-checked
+consumers of these fields need a `cast` or `type: ignore` at narrowing
+sites. This is a deliberate short-term cost: the sentinel is the standard,
+and the checkers are converging on it.
 """
 
 from __future__ import annotations
 
-from enum import Enum
-from typing import Final, Literal
+from typing_extensions import Sentinel
 
-
-class UnsetType(Enum):
-    """The type of `UNSET`; use in annotations as `T | UnsetType`."""
-
-    UNSET = "UNSET"
-
-    def __repr__(self) -> str:
-        return "UNSET"
-
-
-UNSET: Final[Literal[UnsetType.UNSET]] = UnsetType.UNSET
-"""Marks a metadata-document key as absent. Test with `is UNSET`."""
+UNSET = Sentinel("UNSET")
+"""Marks a metadata-document key as absent (PEP 661 sentinel; usable directly
+in type expressions, e.g. `tuple[str, ...] | UNSET`). Test with `is UNSET`."""

packages/zarr-metadata/tests/model/test_pydantic.py

@@ -34,6 +34,8 @@
     ConfigDict,
     InstanceOf,
     PlainSerializer,
+    PydanticSchemaGenerationError,
+    PydanticUserError,
     TypeAdapter,
     ValidationError,
     model_validator,
@@ -142,59 +144,40 @@ def test_type_adapter_standalone() -> None:
 # --- the road not taken: native dataclass introspection ----------------------
 
 
-def test_native_dataclass_introspection_is_possible_but_diverges() -> None:
-    """Pydantic CAN introspect the model dataclass after a namespace rebuild,
-    but that path validates the model shape, not the document: it rejects the
-    document form, coerces booleans into dimensions, and skips the library's
-    cross-field checks. This test documents why the delegation pattern above
-    is the recommended integration."""
+def test_native_dataclass_introspection_is_not_supported() -> None:
+    """Pydantic cannot field-introspect the model dataclasses: the UNSET
+    sentinel (PEP 661, typing_extensions.Sentinel) in the optional-field
+    annotations has no pydantic schema (as of pydantic 2.13), so even the
+    rebuild-with-namespace recipe fails. Introspection was already the wrong
+    tool before the sentinel existed — it validated the model shape rather
+    than the document, and its lax coercion re-opened validator holes (e.g.
+    shape=[True, -5] coerced to (1, -5)) — so the delegation patterns above
+    are the only supported integrations. If this test ever fails because
+    pydantic learned to handle sentinels, revisit whether the introspection
+    path needs its divergences documented again."""
     from zarr_metadata._common import JSONValue
-    from zarr_metadata.model import UNSET, UnsetType
+    from zarr_metadata.model import UNSET
     from zarr_metadata.v3._common import MetadataV3
     from zarr_metadata.v3.array import ArrayMetadataV3, ExtensionFieldV3
 
-    adapter = TypeAdapter(ArrayMetadataModelV3)
-    adapter.rebuild(
-        force=True,
-        _types_namespace={
-            "JSONValue": JSONValue,
-            "ExtensionFieldV3": ExtensionFieldV3,
-            "MetadataV3": MetadataV3,
-            "ArrayMetadataV3": ArrayMetadataV3,
-            "NamedConfigModelV3": NamedConfigModelV3,
-            "MetadataFieldModelV3": NamedConfigModelV3,
-            "UnsetType": UnsetType,
-        },
-    )
-    model_shaped = {
-        "shape": [10],
-        "fill_value": 0,
-        "data_type": {"name": "uint8", "configuration": {}},
-        "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [5]}},
-        "codecs": [{"name": "bytes", "configuration": {}}],
-        "chunk_key_encoding": {"name": "default", "configuration": {}},
-        "dimension_names": UNSET,
-        "attributes": {},
-        "storage_transformers": [],
-        "extra_fields": {},
-    }
-    # model-shaped data validates, nested named configs and all
-    model = adapter.validate_python(model_shaped)
-    assert isinstance(model.data_type, NamedConfigModelV3)
-
-    # divergence 1: the DOCUMENT form is rejected — no from_json normalization
-    with pytest.raises(ValidationError):
-        adapter.validate_python(model_shaped | {"data_type": "uint8"})
-
-    # divergence 2: lax coercion re-opens holes the library validators close
-    coerced = adapter.validate_python(model_shaped | {"shape": [True, -5]})
-    assert coerced.shape == (1, -5)  # from_json would reject both entries
-
-    # __post_init__ invariants DO still run under pydantic construction
-    with pytest.raises(ValidationError, match="Extra fields"):
-        adapter.validate_python(
-            model_shaped | {"extra_fields": {"shape": {"must_understand": False}}}
+    def build_and_use() -> None:
+        adapter = TypeAdapter(ArrayMetadataModelV3)
+        adapter.rebuild(
+            force=True,
+            _types_namespace={
+                "JSONValue": JSONValue,
+                "ExtensionFieldV3": ExtensionFieldV3,
+                "MetadataV3": MetadataV3,
+                "ArrayMetadataV3": ArrayMetadataV3,
+                "NamedConfigModelV3": NamedConfigModelV3,
+                "MetadataFieldModelV3": NamedConfigModelV3,
+                "UNSET": UNSET,
+            },
         )
+        adapter.validate_python({})
+
+    with pytest.raises((AttributeError, PydanticSchemaGenerationError, PydanticUserError)):
+        build_and_use()
 
 
 # --- a first-class pydantic model, engine-backed (the pydantic-zarr pattern) --

packages/zarr-metadata/tests/test_public_api.py

@@ -56,7 +56,6 @@ def _group_rank(s: str) -> int:
     "MetadataValidationError",
     "ProblemKind",
     "UNSET",
-    "UnsetType",
     # v2 data-type encoding union
     "DataTypeMetadataV2",
     # Category B — codec canonical unions