feat: Add a parquet uuid calculation (#3440)

* Calculate a uuid from parquet metadata, utilizing detailed info of the first and last row_groups plus the col_counts of all row_groups of the file or dataset

* pre-commit fixups

* test for parquet uuid

* try more selective key-value pairs for hashing, which might be less vulnerable to OS-specific effects

* Update testing hash and print uuid in case this still is insufficient for OS-agnostic uuid

* style: pre-commit fixes

* explicit importskip for pyarrow.parquet

* Debug commit, prints

* Debug commit, fail assertion

* style: pre-commit fixes

* Remove columns::statistics which contains a distinct_counts key whose value can be None or 0 depending on versions, sorting_columns also different and removed from this list

* Remove DEBUG print statements, but not a full revert since one change still desired

* Updated hash for parquet uuid

* style: pre-commit fixes

---------

Co-authored-by: Nick Manganelli <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ianna Osborne <[email protected]>
Co-authored-by: Iason Krommydas <[email protected]>

Author: 5 people

src/awkward/operations/ak_from_parquet.py

@@ -2,6 +2,9 @@
 
 from __future__ import annotations
 
+import hashlib
+import json
+
 import fsspec.parquet
 
 import awkward as ak
@@ -65,11 +68,8 @@ def from_parquet(
     See also #ak.to_parquet, #ak.metadata_from_parquet.
     """
 
-    parquet_columns, subform, actual_paths, fs, subrg, row_counts, meta = metadata(
-        path,
-        storage_options,
-        row_groups,
-        columns,
+    parquet_columns, subform, actual_paths, fs, subrg, row_counts, meta, uuid = (
+        metadata(path, storage_options, row_groups, columns, calculate_uuid=True)
     )
     return _load(
         actual_paths,
@@ -95,6 +95,7 @@ def metadata(
     columns=None,
     ignore_metadata=False,
     scan_files=True,
+    calculate_uuid=False,
 ):
     # early exit if missing deps
     pyarrow_parquet = awkward._connect.pyarrow.import_pyarrow_parquet("ak.from_parquet")
@@ -193,6 +194,43 @@ def metadata(
         list_indicator=list_indicator, column_prefix=column_prefix
     )
 
+    # generate hash from the col_counts, first row_group and last row_group to calculate approximate parquet uuid
+    uuid = None
+    if calculate_uuid:
+        uuids = [repr({"col_counts": col_counts})]
+        for row_group_index in (0, metadata.num_row_groups - 1):
+            row_group_info = metadata.row_group(row_group_index).to_dict()
+            for k, v in row_group_info.items():
+                # sorting columns, and columns::statistics have some version skew in underlying library
+                # with latter's 'distinct_counts' showing None vs 0 for example, so they're not used
+                if k in ["num_rows", "num_columns"]:
+                    uuids.append(repr({k: v}))
+                if k == "columns":
+                    for subitem in v:
+                        for subkey in subitem:
+                            if subkey not in [
+                                "file_offset",
+                                "file_path",
+                                "physical_type",
+                                "path_in_schema",
+                                "compression",
+                                "encodings",
+                                "total_compressed_size",
+                            ]:
+                                continue
+                            uuids.append(repr({subkey: subitem[subkey]}))
+        uuid = hashlib.sha256(json.dumps(",".join(uuids)).encode()).hexdigest()
+        return (
+            parquet_columns,
+            subform,
+            actual_paths,
+            fs,
+            subrg,
+            col_counts,
+            metadata,
+            uuid,
+        )
+
     return parquet_columns, subform, actual_paths, fs, subrg, col_counts, metadata
 
 

src/awkward/operations/ak_metadata_from_parquet.py

@@ -76,16 +76,25 @@ def _impl(
     path, storage_options, row_groups=None, ignore_metadata=False, scan_files=True
 ):
     results = ak.operations.ak_from_parquet.metadata(
-        path, storage_options, row_groups, None, ignore_metadata, scan_files
+        path,
+        storage_options,
+        row_groups,
+        None,
+        ignore_metadata,
+        scan_files,
+        calculate_uuid=True,
+    )
+    parquet_columns, subform, actual_paths, fs, subrg, col_counts, metadata, uuid = (
+        results
     )
-    parquet_columns, subform, actual_paths, fs, subrg, col_counts, metadata = results
 
     out = {
         "form": subform,
         "fs": fs,
         "paths": actual_paths,
         "col_counts": col_counts,
         "columns": parquet_columns,
+        "uuid": uuid,
     }
     if col_counts:
         out["num_rows"] = sum(col_counts)

tests/test_3440_calculate_parquet_uuid.py

@@ -0,0 +1,59 @@
+# BSD 3-Clause License; see https://github.com/scikit-hep/awkward/blob/main/LICENSE
+
+from __future__ import annotations
+
+import pathlib
+
+import pytest
+
+import awkward
+
+pytest.importorskip("pyarrow.parquet")
+
+metadata_from_parquet_submodule = pytest.importorskip(
+    "awkward.operations.ak_metadata_from_parquet"
+)
+metadata_from_parquet = metadata_from_parquet_submodule.metadata_from_parquet
+
+SAMPLES_DIR = pathlib.Path(__file__).parent / "samples"
+input = SAMPLES_DIR / "nullable-record-primitives.parquet"
+
+
+def test_parquet_uuid():
+    meta = metadata_from_parquet(input)
+    assert (
+        meta["uuid"]
+        == "582dabdb8c87bfa17bc930676ed26b8d4ab22a900f92357751dc380c41acb593"
+    )
+
+
+@pytest.mark.parametrize("calculate_uuid", [True, False])
+def test_return_tuple_with_or_without_uuid(calculate_uuid):
+    results = awkward.operations.ak_from_parquet.metadata(
+        input,
+        {},
+        None,
+        None,
+        False,
+        True,
+        calculate_uuid=calculate_uuid,
+    )
+    if calculate_uuid:
+        assert len(results) == 8, "Expected 8 items in the result tuple"
+        (
+            parquet_columns,
+            subform,
+            actual_paths,
+            fs,
+            subrg,
+            col_counts,
+            metadata,
+            uuid,
+        ) = results
+        assert uuid is not None, "UUID should be present when calculate_uuid is True"
+        print("uuid:", uuid)
+    else:
+        assert len(results) == 7, "Expected 7 items in the result tuple"
+        parquet_columns, subform, actual_paths, fs, subrg, col_counts, metadata = (
+            results
+        )