Merge branch 'mlcommons:main' into docs/update-spec-draft-v2

Author: benjelloun

python/mlcroissant/mlcroissant/_src/core/constants.py

@@ -75,6 +75,7 @@ def ML_COMMONS(ctx) -> rdflib.Namespace:
 # ML_COMMONS.replace is understood as the `replace` method on the class Namespace.
 ML_COMMONS_REPLACE = lambda ctx: ML_COMMONS(ctx)["replace"]
 ML_COMMONS_SAMPLING_RATE = lambda ctx: ML_COMMONS(ctx).samplingRate
+ML_COMMONS_SD_VERSION = lambda ctx: ML_COMMONS(ctx).sdVersion
 ML_COMMONS_SEPARATOR = lambda ctx: ML_COMMONS(ctx).separator
 ML_COMMONS_SOURCE = lambda ctx: ML_COMMONS(ctx).source
 ML_COMMONS_SUB_FIELD = lambda ctx: ML_COMMONS(ctx).subField
@@ -187,6 +188,7 @@ def ML_COMMONS(ctx) -> rdflib.Namespace:
     ML_COMMONS_SAMPLING_RATE(ctx): "sampling_rate",
     ML_COMMONS_SEPARATOR(ctx): "separator",
     ML_COMMONS_SOURCE(ctx): "source",
+    ML_COMMONS_SD_VERSION(ctx): "sd_version",
     ML_COMMONS_TRANSFORM(ctx): "transforms",
     DCTERMS_CONFORMS_TO: "conforms_to",
     SCHEMA_ORG_CONTENT_SIZE: "content_size",

python/mlcroissant/mlcroissant/_src/structure_graph/nodes/metadata.py

@@ -177,6 +177,13 @@ class Metadata(Node):
         input_types=[SDO.Integer, SDO.Number, SDO.Text],
         url=constants.SCHEMA_ORG_VERSION,
     )
+    sd_version: str | None = mlc_dataclasses.jsonld_field(
+        cast_fn=cast_version,
+        default=None,
+        description="The version of the dataset metadata.",
+        input_types=[SDO.Integer, SDO.Number, SDO.Text],
+        url=constants.ML_COMMONS_SD_VERSION,
+    )
     distribution: list[FileObject | FileSet] = mlc_dataclasses.jsonld_field(
         cardinality="MANY",
         default_factory=list,

python/mlcroissant/mlcroissant/_src/structure_graph/nodes/metadata_test.py

@@ -66,6 +66,7 @@ def test_from_jsonld(conforms_to: CroissantVersion, version: Any):
         constants.SCHEMA_ORG_URL: "https://mlcommons.org",
         constants.SCHEMA_ORG_VERSION: version,
         constants.ML_COMMONS_IS_LIVE_DATASET(ctx): False,
+        constants.ML_COMMONS_SD_VERSION(ctx): "2.0.0",
     }
     metadata = Metadata.from_jsonld(ctx, jsonld)
     assert metadata.name == "foo"
@@ -81,6 +82,7 @@ def test_from_jsonld(conforms_to: CroissantVersion, version: Any):
     assert metadata.ctx.is_live_dataset == False
     assert metadata.url == "https://mlcommons.org"
     assert metadata.version == "1.0.0"
+    assert metadata.sd_version == "2.0.0"
     assert not ctx.issues.errors
     assert not ctx.issues.warnings