feat: accept DPv2 missing values object with labels.

frictionless/schema/__init__.py

@@ -1,3 +1,4 @@
 from .field import Field
+from .missing_values import MissingValue, MissingValues
 from .schema import Schema
 from .types import *

frictionless/schema/__spec__/field/test_read.py

@@ -43,3 +43,20 @@ def test_field_read_cell_number_missingValues():
     assert field.read_cell("NA") == (None, None)
     assert field.read_cell("N/A") == (None, None)
     assert field.read_cell(0) == (None, None)
+
+
+def test_field_read_cell_object_missingValues():
+    descriptor = {
+        "name": "name",
+        "type": "string",
+        "missingValues": [
+            {"value": "", "label": "OMITTED"},
+            {"value": "-99", "label": "REFUSED"},
+        ],
+    }
+    field = Field.from_descriptor(descriptor)
+    assert field.read_cell("") == (None, None)
+    assert field.read_cell("-99") == (None, None)
+    assert field.read_cell("x") == ("x", None)
+    # serialization stays lossless
+    assert field.to_descriptor()["missingValues"] == descriptor["missingValues"]

frictionless/schema/__spec__/test_missing_values.py

@@ -0,0 +1,188 @@
+import pytest
+
+from frictionless import Field, FrictionlessException, Schema
+from frictionless.schema.missing_values import MissingValues
+
+# Read / query
+
+READ_CASES = [
+    # name, descriptor, value_strings, contains, not_contains, representation
+    (
+        "string-form",
+        ["", "NA", "N/A"],
+        ["", "NA", "N/A"],
+        ["", "NA", "N/A"],
+        ["x", "-"],
+        "",
+    ),
+    (
+        "object-form",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "", "label": "OMITTED"}],
+        ["-99", ""],
+        ["-99", ""],
+        ["x", "NA"],
+        "-99",
+    ),
+    (
+        "object-without-label",
+        [{"value": "-"}],
+        ["-"],
+        ["-"],
+        ["", "x"],
+        "-",
+    ),
+    (
+        # empty: no value converts to null, and the representation falls back
+        # to the spec default missing value ("")
+        "empty",
+        [],
+        [],
+        [],
+        ["", "x"],
+        "",
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "name, descriptor, value_strings, contains, not_contains, representation",
+    READ_CASES,
+    ids=[case[0] for case in READ_CASES],
+)
+def test_missing_values_query(
+    name, descriptor, value_strings, contains, not_contains, representation
+):
+    mv = MissingValues.from_descriptor(descriptor)
+    assert mv.value_strings() == value_strings
+    for value in contains:
+        assert value in mv
+    for value in not_contains:
+        assert value not in mv
+    assert mv.representation == representation
+
+
+# List-like interface (backward compatibility: consumers read List[str])
+
+LIST_LIKE_CASES = [
+    ("string-form", ["", "NA", "N/A"], ["", "NA", "N/A"]),
+    (
+        "object-form",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "", "label": "OMITTED"}],
+        ["-99", ""],
+    ),
+    ("empty", [], []),
+]
+
+
+@pytest.mark.parametrize(
+    "name, descriptor, expected",
+    LIST_LIKE_CASES,
+    ids=[case[0] for case in LIST_LIKE_CASES],
+)
+def test_missing_values_list_like(name, descriptor, expected):
+    mv = MissingValues.from_descriptor(descriptor)
+    assert mv == expected
+    assert list(mv) == expected
+    assert len(mv) == len(expected)
+    for index, value in enumerate(expected):
+        assert mv[index] == value
+
+
+# Serialization (lossless round-trip)
+
+ROUNDTRIP_CASES = [
+    ("string-form", ["", "NA"]),
+    ("object-form", [{"value": "-99", "label": "REFUSED"}]),
+    ("object-without-label", [{"value": "-"}]),
+    ("empty", []),
+]
+
+
+@pytest.mark.parametrize(
+    "name, descriptor",
+    ROUNDTRIP_CASES,
+    ids=[case[0] for case in ROUNDTRIP_CASES],
+)
+def test_missing_values_roundtrip(name, descriptor):
+    mv = MissingValues.from_descriptor(descriptor)
+    assert mv.to_descriptor() == descriptor
+
+
+# Validation (uniqueness of value and label)
+
+VALIDATION_CASES = [
+    ("valid-strings", ["", "NA"], []),
+    (
+        "valid-objects",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "", "label": "OMITTED"}],
+        [],
+    ),
+    (
+        "duplicate-value",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "-99", "label": "OMITTED"}],
+        ['missing value "-99" is not unique'],
+    ),
+    (
+        "duplicate-label",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "", "label": "REFUSED"}],
+        ['missing value label "REFUSED" is not unique'],
+    ),
+    (
+        "duplicate-value-and-label",
+        [{"value": "-99", "label": "REFUSED"}, {"value": "-99", "label": "REFUSED"}],
+        [
+            'missing value "-99" is not unique',
+            'missing value label "REFUSED" is not unique',
+        ],
+    ),
+    (
+        # labels left out do not collide with each other
+        "missing-labels-do-not-collide",
+        [{"value": "-99"}, {"value": "-1"}],
+        [],
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "name, descriptor, expected",
+    VALIDATION_CASES,
+    ids=[case[0] for case in VALIDATION_CASES],
+)
+def test_missing_values_validation_notes(name, descriptor, expected):
+    mv = MissingValues.from_descriptor(descriptor)
+    assert mv.validation_notes() == expected
+
+
+# Validation surfaces through Field/Schema metadata validation
+
+
+def test_field_missing_values_duplicate_value_is_invalid():
+    with pytest.raises(FrictionlessException) as excinfo:
+        Field.from_descriptor(
+            {
+                "name": "name",
+                "type": "string",
+                "missingValues": [
+                    {"value": "-99", "label": "REFUSED"},
+                    {"value": "-99", "label": "OMITTED"},
+                ],
+            }
+        )
+    notes = [reason.note for reason in excinfo.value.reasons]
+    assert 'missing value "-99" is not unique' in notes
+
+
+def test_schema_missing_values_duplicate_label_is_invalid():
+    with pytest.raises(FrictionlessException) as excinfo:
+        Schema.from_descriptor(
+            {
+                "fields": [{"name": "name", "type": "string"}],
+                "missingValues": [
+                    {"value": "-99", "label": "REFUSED"},
+                    {"value": "", "label": "REFUSED"},
+                ],
+            }
+        )
+    notes = [reason.note for reason in excinfo.value.reasons]
+    assert 'missing value label "REFUSED" is not unique' in notes

frictionless/schema/field.py

@@ -11,6 +11,7 @@
 from ..exception import FrictionlessException
 from ..metadata import Metadata
 from ..system import system
+from .missing_values import MISSING_VALUES_PROFILE, MissingValues
 
 if TYPE_CHECKING:
     from ..types import IDescriptor
@@ -50,12 +51,15 @@ class Field(Metadata):
     For example: "default","array" etc.
     """
 
-    missing_values: List[str] = attrs.field(
-        factory=settings.DEFAULT_MISSING_VALUES.copy
+    missing_values: MissingValues = attrs.field(
+        factory=lambda: MissingValues.from_descriptor(
+            settings.DEFAULT_MISSING_VALUES.copy()
+        ),
     )
     """
-    List of string values to be set as missing values in the field. If any of string in missing values
-    is found in the field value then it is set as None.
+    Values to be treated as missing values in the field. If any of them is found
+    in the field value then it is set as None. Accepts either the string form
+    (``["", "NA"]``) or the object form (``[{"value": "-99", "label": "REFUSED"}]``).
     """
 
     constraints: Dict[str, Any] = attrs.field(factory=dict)
@@ -92,6 +96,8 @@ def __setattr__(self, name: str, value: Any):  # type: ignore
         if name == "type":
             note = 'Use "schema.set_field_type()" to update the type of the field'
             raise FrictionlessException(errors.FieldError(note=note))
+        if name == "missing_values" and not isinstance(value, MissingValues):
+            value = MissingValues.from_descriptor(value)
         return super().__setattr__(name, value)  # type: ignore
 
     @property
@@ -168,12 +174,10 @@ def create_cell_writer(self) -> types.ICellWriter:
         value_writer = self.create_value_writer()
 
         # Create missing value
-        try:
-            missing_value = self.missing_values[0]
-            if not self.has_defined("missing_values") and self.schema:
-                missing_value = self.schema.missing_values[0]
-        except IndexError:
-            missing_value = settings.DEFAULT_MISSING_VALUES[0]
+        missing_values = self.missing_values
+        if not self.has_defined("missing_values") and self.schema:
+            missing_values = self.schema.missing_values
+        missing_value = missing_values.representation
 
         # Create writer
         def cell_writer(cell: Any, *, ignore_missing: bool = False):
@@ -209,10 +213,7 @@ def value_writer(cell: Any):
             "title": {"type": "string"},
             "description": {"type": "string"},
             "format": {"type": "string"},
-            "missingValues": {
-                "type": "array",
-                "items": {"type": "string"},
-            },
+            "missingValues": MISSING_VALUES_PROFILE,
             "constraints": {
                 "type": "object",
                 "properties": {
@@ -281,12 +282,24 @@ def metadata_validate(cls, descriptor: IDescriptor):  # type: ignore
                 )
                 yield errors.FieldError(note=note)
 
+        # Missing values
+        missing_values = descriptor.get("missingValues")
+        if missing_values is not None:
+            for note in MissingValues.from_descriptor(missing_values).validation_notes():
+                yield errors.FieldError(note=note)
+
         # Misleading
         for name in ["required"]:
             if name in descriptor:
                 note = f'"{name}" should be set as "constraints.{name}"'
                 yield errors.FieldError(note=note)
 
+    def metadata_export(self, *, exclude: List[str] = []) -> IDescriptor:
+        descriptor = super().metadata_export(exclude=exclude)
+        if "missingValues" in descriptor:
+            descriptor["missingValues"] = self.missing_values.to_descriptor()
+        return descriptor
+
 
 # Internal