Include sc:Time support in mlcroissant (#912)

Author: ccl-core

datasets/1.1/huggingface-standard-chess-game-mini/data/0001.parquet

@@ -0,0 +1,321 @@
+{
+  "@context": {
+    "@language": "en",
+    "@vocab": "https://schema.org/",
+    "arrayShape": "cr:arrayShape",
+    "citeAs": "cr:citeAs",
+    "column": "cr:column",
+    "conformsTo": "dct:conformsTo",
+    "cr": "http://mlcommons.org/croissant/",
+    "data": {
+      "@id": "cr:data",
+      "@type": "@json"
+    },
+    "dataBiases": "cr:dataBiases",
+    "dataCollection": "cr:dataCollection",
+    "dataType": {
+      "@id": "cr:dataType",
+      "@type": "@vocab"
+    },
+    "dct": "http://purl.org/dc/terms/",
+    "extract": "cr:extract",
+    "field": "cr:field",
+    "fileProperty": "cr:fileProperty",
+    "fileObject": "cr:fileObject",
+    "fileSet": "cr:fileSet",
+    "format": "cr:format",
+    "includes": "cr:includes",
+    "isArray": "cr:isArray",
+    "isLiveDataset": "cr:isLiveDataset",
+    "jsonPath": "cr:jsonPath",
+    "key": "cr:key",
+    "md5": "cr:md5",
+    "parentField": "cr:parentField",
+    "path": "cr:path",
+    "personalSensitiveInformation": "cr:personalSensitiveInformation",
+    "recordSet": "cr:recordSet",
+    "references": "cr:references",
+    "regex": "cr:regex",
+    "repeated": "cr:repeated",
+    "replace": "cr:replace",
+    "sc": "https://schema.org/",
+    "separator": "cr:separator",
+    "source": "cr:source",
+    "subField": "cr:subField",
+    "transform": "cr:transform"
+  },
+  "@type": "sc:Dataset",
+  "distribution": [
+    {
+      "@type": "cr:FileSet",
+      "@id": "parquet-files-for-config-default",
+      "encodingFormat": "application/x-parquet",
+      "includes": "data/*.parquet"
+    }
+  ],
+  "recordSet": [
+    {
+      "@type": "cr:RecordSet",
+      "@id": "default",
+      "description": "Note: this is a smaller version of the original HuggingFace dataset. Lichess/standard-chess-games - 'default' subset.",
+      "field": [
+        {
+          "@type": "cr:Field",
+          "@id": "default/Event",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Event"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/Site",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Site"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/White",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "White"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/Black",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Black"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/Result",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Result"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/WhiteTitle",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "WhiteTitle"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/BlackTitle",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "BlackTitle"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/WhiteElo",
+          "dataType": "cr:Int16",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "WhiteElo"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/BlackElo",
+          "dataType": "cr:Int16",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "BlackElo"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/WhiteRatingDiff",
+          "dataType": "cr:Int16",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "WhiteRatingDiff"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/BlackRatingDiff",
+          "dataType": "cr:Int16",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "BlackRatingDiff"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/UTCDate",
+          "dataType": "sc:Date",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "UTCDate"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/UTCTime",
+          "dataType": "sc:Time",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "UTCTime"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/ECO",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "ECO"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/Opening",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Opening"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/Termination",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "Termination"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/TimeControl",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "TimeControl"
+            }
+          }
+        },
+        {
+          "@type": "cr:Field",
+          "@id": "default/movetext",
+          "dataType": "sc:Text",
+          "source": {
+            "fileSet": {
+              "@id": "parquet-files-for-config-default"
+            },
+            "extract": {
+              "column": "movetext"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "conformsTo": "http://mlcommons.org/croissant/1.1",
+  "name": "standard-chess-games",
+  "description": "\n[!CAUTION]\nThis dataset is still a work in progress and some breaking changes might occur.\n\n\n\t\n\t\t\n\t\tLichess Rated Standard Chess Games Dataset\n\t\n\n\n\t\n\t\t\n\t\tDataset Description\n\t\n\n6,771,826,271 standard rated games, played on lichess.org, updated monthly from the database dumps.\nThis version of the data is meant for data analysis. If you need PGN files you can find those here. That said, once you have a subset of interest, it is trivial to convert it back to PGN as shown in the Dataset Usage… See the full description on the dataset page: https://huggingface.co/datasets/Lichess/standard-chess-games.",
+  "keywords": [
+    "cc0-1.0",
+    "1B - 10B",
+    "parquet",
+    "Tabular",
+    "Text",
+    "Datasets",
+    "Dask",
+    "Croissant",
+    "Polars",
+    "🇺🇸 Region: US",
+    "chess",
+    "games",
+    "game",
+    "lichess",
+    "tabular"
+  ],
+  "license": "https://choosealicense.com/licenses/cc0-1.0/",
+  "url": "https://huggingface.co/datasets/Lichess/standard-chess-games"
+}

datasets/1.1/huggingface-standard-chess-game-mini/metadata.json

@@ -0,0 +1,2 @@
+{"default/Event": "Rated Classical game", "default/Site": "https://lichess.org/9tp6v4ps", "default/White": "Smok", "default/Black": "McCoy", "default/Result": "1-0", "default/WhiteTitle": "None", "default/BlackTitle": "None", "default/WhiteElo": "1452", "default/BlackElo": "1227", "default/WhiteRatingDiff": "5", "default/BlackRatingDiff": "-20", "default/UTCDate": "2013-01-31 00:00:00", "default/UTCTime": "23:01:50", "default/ECO": "B01", "default/Opening": "Scandinavian Defense", "default/Termination": "Time forfeit", "default/TimeControl": "480+0", "default/movetext": "1. e4 d5 2. exd5 e6 3. dxe6 fxe6 4. Qe2 Qd6 5. Nf3 Nc6 6. d4 Nf6 7. Bg5 h6 8. Bxf6 gxf6 9. Nc3 h5 10. d5 Nb4 11. Nh4 Nxd5 12. Nxd5 Qxd5 13. Qd3 Qe5+ 14. Be2 Qxb2 15. O-O Rg8 16. Qb5+ c6 17. Qxb2 1-0"}
+{"default/Event": "Rated Blitz game", "default/Site": "https://lichess.org/b9tstv2z", "default/White": "dvorak", "default/Black": "Kiriush", "default/Result": "1-0", "default/WhiteTitle": "None", "default/BlackTitle": "None", "default/WhiteElo": "1796", "default/BlackElo": "1876", "default/WhiteRatingDiff": "13", "default/BlackRatingDiff": "-14", "default/UTCDate": "2013-01-31 00:00:00", "default/UTCTime": "23:03:05", "default/ECO": "C55", "default/Opening": "Italian Game: Two Knights Defense, Perreux Variation", "default/Termination": "Normal", "default/TimeControl": "180+0", "default/movetext": "1. e4 e5 2. Nf3 Nc6 3. d4 exd4 4. Bc4 Nf6 5. Ng5 d5 6. exd5 Na5 7. Bb5+ c6 8. dxc6 bxc6 9. Ba4 h6 10. Nf3 Bg4 11. O-O Be7 12. h3 Bxf3 13. Qxf3 Rc8 14. Qg3 Nh5 15. Qg4 g6 16. Re1 Kf8 17. Bd2 Nc4 18. Bb3 Nxd2 19. Nxd2 Kg7 20. Qf3 Nf6 21. Ne4 Nxe4 22. Qxf7# 1-0"}

datasets/1.1/huggingface-standard-chess-game-mini/output/default.jsonl

@@ -133,6 +133,7 @@ def ML_COMMONS(ctx) -> rdflib.Namespace:
 SCHEMA_ORG_DATA_TYPE_IMAGE_OBJECT = namespace.SDO.ImageObject
 SCHEMA_ORG_DATA_TYPE_INTEGER = namespace.SDO.Integer
 SCHEMA_ORG_DATA_TYPE_TEXT = namespace.SDO.Text
+SCHEMA_ORG_DATA_TYPE_TIME = namespace.SDO.Time
 SCHEMA_ORG_DATA_TYPE_URL = namespace.SDO.URL
 SCHEMA_ORG_DESCRIPTION = namespace.SDO.description
 SCHEMA_ORG_DISTRIBUTION = namespace.SDO.distribution
@@ -263,4 +264,5 @@ class DataType:
     UINT64 = ML_COMMONS_V_1_0.UInt64
     SPLIT = ML_COMMONS_V_1_0.Split
     TEXT = namespace.SDO.Text
+    TIME = namespace.SDO.Time
     URL = namespace.SDO.URL

python/mlcroissant/mlcroissant/_src/core/constants.py

@@ -1,5 +1,7 @@
 """data_types module."""
 
+import datetime
+
 import numpy as np
 import pandas as pd
 from rdflib import term
@@ -35,6 +37,7 @@ def check_expected_type(issues: Issues, jsonld: Json, expected_type: str):
     DataType.INT32: np.int32,
     DataType.INT64: np.int64,
     DataType.TEXT: bytes,
+    DataType.TIME: datetime.time,
     DataType.URL: bytes,
     DataType.UINT8: np.uint8,
     DataType.UINT16: np.uint16,

python/mlcroissant/mlcroissant/_src/core/data_types.py

@@ -1,6 +1,7 @@
 """Field operation module."""
 
 import dataclasses
+import datetime
 import functools
 import io
 import logging
@@ -107,6 +108,15 @@ def _cast_value(ctx: Context, value: Any, data_type: type | term.URIRef | None):
         return [_cast_value(ctx=ctx, value=v, data_type=data_type) for v in value]
     elif data_type == bytes and not isinstance(value, bytes):
         return _to_bytes(value)
+    elif data_type == datetime.time:
+        if isinstance(value, str):
+            return datetime.datetime.strptime(value, "%H:%M:%S").time()
+        elif isinstance(value, datetime.time):
+            return value
+        else:
+            raise ValueError(
+                f"No special case for type: {type(value)} of data_type: {data_type}"
+            )
     else:
         return data_type(value)