Add a EncodingFormat for .arff files (#835)

Author: ccl-core

datasets/1.0/credit-g/metadata.json

@@ -0,0 +1,2 @@
+{"features/0-checking_status": "<0", "features/1-duration": 6.0, "features/2-credit_history": "critical/other existing credit", "features/3-purpose": "radio/tv", "features/4-credit_amount": 1169.0, "features/5-savings_status": "no known savings", "features/6-employment": ">=7", "features/7-installment_commitment": 4.0, "features/8-personal_status": "male single", "features/9-other_parties": "none", "features/10-residence_since": 4.0, "features/11-property_magnitude": "real estate", "features/12-age": 67.0, "features/13-other_payment_plans": "none", "features/14-housing": "own", "features/15-existing_credits": 2.0, "features/16-job": "skilled", "features/17-num_dependents": 1.0, "features/18-own_telephone": "yes", "features/19-foreign_worker": "yes", "features/20-class": "good"}
+{"features/0-checking_status": "0<=X<200", "features/1-duration": 48.0, "features/2-credit_history": "existing paid", "features/3-purpose": "radio/tv", "features/4-credit_amount": 5951.0, "features/5-savings_status": "<100", "features/6-employment": "1<=X<4", "features/7-installment_commitment": 2.0, "features/8-personal_status": "female div/dep/mar", "features/9-other_parties": "none", "features/10-residence_since": 2.0, "features/11-property_magnitude": "real estate", "features/12-age": 22.0, "features/13-other_payment_plans": "none", "features/14-housing": "own", "features/15-existing_credits": 1.0, "features/16-job": "skilled", "features/17-num_dependents": 1.0, "features/18-own_telephone": "none", "features/19-foreign_worker": "yes", "features/20-class": "bad"}

datasets/1.0/credit-g/output/data-file-description.jsonl

@@ -622,8 +622,8 @@ Most of the important properties needed to describe a `FileObject` are defined i
   <tr>
     <td><a href="https://schema.org/encodingFormat">sc:encodingFormat</a></td>
     <td><a href="http://schema.org/Text">Text</a></td>
-    <td>ONE</td>
-    <td>The format of the file, given as a mime type.</td>
+    <td>MANY</td>
+    <td>The formats of the file, given as a mime type. Unregistered or niche encoding and file formats can be indicated instead via the most appropriate URL, e.g. defining Web page or a Wikipedia/Wikidata entry.</td>
   </tr>
   <tr>
     <td><a href="https://schema.org/sameAs">sc:sameAs</a></td>

docs/croissant-spec-draft.md

@@ -179,7 +179,7 @@ def ML_COMMONS(ctx) -> rdflib.Namespace:
     SCHEMA_ORG_DATE_PUBLISHED: "date_published",
     SCHEMA_ORG_DESCRIPTION: "description",
     SCHEMA_ORG_DISTRIBUTION: "distribution",
-    SCHEMA_ORG_ENCODING_FORMAT: "encoding_format",
+    SCHEMA_ORG_ENCODING_FORMAT: "encoding_formats",
     SCHEMA_ORG_KEYWORDS: "keywords",
     SCHEMA_ORG_LICENSE: "license",
     SCHEMA_ORG_MD5: "md5",
@@ -209,9 +209,14 @@ def ML_COMMONS(ctx) -> rdflib.Namespace:
 class EncodingFormat:
     """Supported MIME Types in Croissant.
 
+    Unregistered or niche encoding and file formats can be indicated instead via the most
+    appropriate URL, e.g. defining Web page or a Wikipedia/Wikidata entry.
+    Supersedes fileFormat.
+
     We inherit the wrong naming `encodingFormat` from https://schema.org/encodingFormat.
     """
 
+    ARFF = "https://ml.cms.waikato.ac.nz/weka/arff.html"
     CSV = "text/csv"
     GIT = "git+https"
     JPG = "image/jpeg"

python/mlcroissant/mlcroissant/_src/core/constants.py

@@ -91,6 +91,11 @@ def librosa(cls) -> types.ModuleType:  # pylint: disable=invalid-name
         """Cached librosa module."""
         return _try_import("librosa", package_name="librosa")
 
+    @cached_class_property
+    def scipy(cls) -> types.ModuleType:  # pylint: disable=invalid-name
+        """Cached scipy module."""
+        return _try_import("scipy", package_name="scipy")
+
     @cached_class_property
     def torchdata_datapipes(cls) -> types.ModuleType:
         """Cached torchdata module."""

python/mlcroissant/mlcroissant/_src/core/optional.py

@@ -82,9 +82,9 @@ def _add_operations_for_file_object(
         operation = first_operation
         # Extract the file if needed
         if (
-            should_extract(node.encoding_format)
+            should_extract(node.encoding_formats)
             and isinstance(successor, (FileObject, FileSet))
-            and not should_extract(successor.encoding_format)
+            and not should_extract(successor.encoding_formats)
         ):
             operation = operation >> Extract(operations=operations, node=node)
         if isinstance(successor, FileSet):
@@ -93,7 +93,7 @@ def _add_operations_for_file_object(
                 >> FilterFiles(operations=operations, node=successor)
                 >> Concatenate(operations=operations, node=successor)
             )
-        if node.encoding_format and not should_extract(node.encoding_format):
+        if node.encoding_formats and not should_extract(node.encoding_formats):
             fields = tuple([
                 field for field in node.recursive_successors if isinstance(field, Field)
             ])
@@ -192,7 +192,10 @@ def from_nodes(cls, ctx: Context, metadata: Node) -> "OperationGraph":
         operations = Operations()
         for node in nx.topological_sort(ctx.graph):
             if isinstance(node, FileObject):
-                if node.encoding_format == EncodingFormat.GIT:
+                if (
+                    node.encoding_formats
+                    and EncodingFormat.GIT in node.encoding_formats
+                ):
                     _add_operations_for_git(operations, node, ctx.folder)
                 else:
                     _add_operations_for_file_object(operations, node, ctx.folder)

python/mlcroissant/mlcroissant/_src/operation_graph/graph.py

@@ -223,7 +223,10 @@ def call(self, *args) -> Path:
         del args  # unused
         filepath = get_download_filepath(self.node)
         if not filepath.exists():
-            if self.node.encoding_format == EncodingFormat.GIT:
+            if (
+                self.node.encoding_formats
+                and EncodingFormat.GIT in self.node.encoding_formats
+            ):
                 self._download_from_git(filepath)
             else:
                 self._download_from_http(filepath)

python/mlcroissant/mlcroissant/_src/operation_graph/operations/download.py

@@ -17,10 +17,12 @@
 from mlcroissant._src.structure_graph.nodes.file_object import FileObject
 
 
-def should_extract(encoding_format: str | None) -> bool:
+def should_extract(encoding_formats: list[str] | None) -> bool:
     """Whether the encoding format should be extracted (zip or tar)."""
+    if not encoding_formats:
+        return False
     return (
-        encoding_format == EncodingFormat.TAR or encoding_format == EncodingFormat.ZIP
+        EncodingFormat.TAR in encoding_formats or EncodingFormat.ZIP in encoding_formats
     )
 
 

python/mlcroissant/mlcroissant/_src/operation_graph/operations/extract.py

@@ -91,7 +91,7 @@ def test_readfield_with_subfields():
                 id="file_id",
                 content_url=csv_file,
                 sha256="None",
-                encoding_format="text/csv",
+                encoding_formats=["text/csv"],
             )
         ]
         fields = [
@@ -244,7 +244,7 @@ def test_extract_lines(separator):
                 id="file_id",
                 content_url=path,
                 sha256="None",
-                encoding_format="text/plain",
+                encoding_formats=["text/plain"],
             )
         ]
         fields = []

python/mlcroissant/mlcroissant/_src/operation_graph/operations/field_test.py

@@ -12,6 +12,7 @@
 from mlcroissant._src.core.constants import EncodingFormat
 from mlcroissant._src.core.git import download_git_lfs_file
 from mlcroissant._src.core.git import is_git_lfs_file
+from mlcroissant._src.core.optional import deps
 from mlcroissant._src.core.path import Path
 from mlcroissant._src.operation_graph.base_operation import Operation
 from mlcroissant._src.operation_graph.operations.download import is_url
@@ -21,6 +22,12 @@
 from mlcroissant._src.structure_graph.nodes.file_set import FileSet
 from mlcroissant._src.structure_graph.nodes.source import FileProperty
 
+try:
+    scipy = deps.scipy
+except ModuleNotFoundError:
+    scipy = None
+INSTALL_MESSAGE = "scipy is not installed and is a dependency."
+
 
 class ReadingMethod(enum.Enum):
     """Reading method derived from the fields that consume the FileObject/FileSet."""
@@ -82,65 +89,80 @@ class Read(Operation):
     folder: epath.Path
     fields: tuple[Field, ...]
 
-    def _read_file_content(self, encoding_format: str, file: Path) -> pd.DataFrame:
+    def _read_file_content(
+        self, encoding_formats: list[str], file: Path
+    ) -> pd.DataFrame:
         """Extracts the `source` file to `target`."""
         filepath = file.filepath
         if is_git_lfs_file(filepath):
             download_git_lfs_file(file)
         reading_method = _reading_method(self.node, self.fields)
+        if EncodingFormat.ARFF in encoding_formats:
+            if scipy is None:
+                raise NotImplementedError(INSTALL_MESSAGE)
+
+            data = scipy.io.arff.loadarff(filepath)
+            if not isinstance(data, list) or len(data) != 1:
+                raise ValueError(
+                    "The loaded data from scipy.io.arff does not have the expected"
+                    " shape (a list with one element). Please ensure the ARFF file is"
+                    " valid."
+                )
+            return pd.DataFrame(data[0])
 
         with filepath.open("rb") as file:
-            # TODO(https://github.com/mlcommons/croissant/issues/635).
-            if filepath.suffix == ".gz":
-                file = gzip.open(file, "rt", newline="")
-            if encoding_format == EncodingFormat.CSV:
-                return pd.read_csv(file)
-            elif encoding_format == EncodingFormat.TSV:
-                return pd.read_csv(file, sep="\t")
-            elif encoding_format == EncodingFormat.JSON:
-                json_content = json.load(file)
-                if reading_method == ReadingMethod.JSON:
-                    return parse_json_content(json_content, self.fields)
-                else:
-                    # Raw files are returned as a one-line pd.DataFrame.
-                    return pd.DataFrame({
-                        FileProperty.content: [json_content],
-                    })
-            elif encoding_format == EncodingFormat.JSON_LINES:
-                return pd.read_json(file, lines=True)
-            elif encoding_format == EncodingFormat.PARQUET:
-                try:
-                    df = pd.read_parquet(file)
-                    # Sometimes the author already set an index in Parquet, so we want
-                    # to reset it to always have the same format.
-                    df.reset_index(inplace=True)
-                    return df
-                except ImportError as e:
-                    raise ImportError(
-                        "Missing dependency to read Parquet files. pyarrow is not"
-                        " installed. Please, install `pip install"
-                        " mlcroissant[parquet]`."
-                    ) from e
-            elif encoding_format == EncodingFormat.TEXT:
-                if reading_method == ReadingMethod.LINES:
-                    return pd.read_csv(
-                        filepath, header=None, names=[FileProperty.lines]
-                    )
-                else:
+            for encoding_format in encoding_formats:
+                # TODO(https://github.com/mlcommons/croissant/issues/635).
+                if filepath.suffix == ".gz":
+                    file = gzip.open(file, "rt", newline="")
+                if encoding_format == EncodingFormat.CSV:
+                    return pd.read_csv(file)
+                elif encoding_format == EncodingFormat.TSV:
+                    return pd.read_csv(file, sep="\t")
+                elif encoding_format == EncodingFormat.JSON:
+                    json_content = json.load(file)
+                    if reading_method == ReadingMethod.JSON:
+                        return parse_json_content(json_content, self.fields)
+                    else:
+                        # Raw files are returned as a one-line pd.DataFrame.
+                        return pd.DataFrame({
+                            FileProperty.content: [json_content],
+                        })
+                elif encoding_format == EncodingFormat.JSON_LINES:
+                    return pd.read_json(file, lines=True)
+                elif encoding_format == EncodingFormat.PARQUET:
+                    try:
+                        df = pd.read_parquet(file)
+                        # Sometimes the author already set an index in Parquet, so we
+                        # want to reset it to always have the same format.
+                        df.reset_index(inplace=True)
+                        return df
+                    except ImportError as e:
+                        raise ImportError(
+                            "Missing dependency to read Parquet files. pyarrow is not"
+                            " installed. Please, install `pip install"
+                            " mlcroissant[parquet]`."
+                        ) from e
+                elif encoding_format == EncodingFormat.TEXT:
+                    if reading_method == ReadingMethod.LINES:
+                        return pd.read_csv(
+                            filepath, header=None, names=[FileProperty.lines]
+                        )
+                    else:
+                        return pd.DataFrame({
+                            FileProperty.content: [file.read()],
+                        })
+                elif (
+                    encoding_format == EncodingFormat.MP3
+                    or encoding_format == EncodingFormat.JPG
+                ):
                     return pd.DataFrame({
                         FileProperty.content: [file.read()],
                     })
-            elif (
-                encoding_format == EncodingFormat.MP3
-                or encoding_format == EncodingFormat.JPG
-            ):
-                return pd.DataFrame({
-                    FileProperty.content: [file.read()],
-                })
-            else:
-                raise ValueError(
-                    f"Unsupported encoding format for file: {encoding_format}"
-                )
+            raise ValueError(
+                f"None of the provided encoding formats: {encoding_format} for file"
+                f" {filepath} returned a valid pandas dataframe."
+            )
 
     def call(self, files: list[Path] | Path) -> pd.DataFrame:
         """See class' docstring."""
@@ -170,8 +192,8 @@ def call(self, files: list[Path] | Path) -> pd.DataFrame:
                     f'In node "{self.node.uuid}", file "{self.node.content_url}" is'
                     " either an invalid URL or an invalid path."
                 )
-            assert self.node.encoding_format, "Encoding format is not specified."
-            file_content = self._read_file_content(self.node.encoding_format, file)
+            assert self.node.encoding_formats, "Encoding format is not specified."
+            file_content = self._read_file_content(self.node.encoding_formats, file)
             if _should_append_line_numbers(self.fields):
                 file_content[FileProperty.lineNumbers] = range(len(file_content))
             file_content[FileProperty.filepath] = file.filepath