Move xxhash to the "performance" extra

Author: Dev-iL

docs/concepts/caching.rst

@@ -357,6 +357,21 @@ Data version
 
 Caching requires the ability to uniquely identify data (e.g., create a hash). By default, all Python primitive types (``int``, ``str``, ``dict``, etc.) are supported and more types can be added via extensions (e.g., ``pandas``). For types not explicitly supported, caching can still function by versioning the object's internal ``__dict__`` instead. However, this could be expensive to compute or less reliable than alternatives.
 
+Hashing backend
+~~~~~~~~~~~~~~~
+
+By default, hashing uses the standard library's ``hashlib.md5`` (chosen for speed, not for cryptographic security). Installing the optional `xxhash <https://github.com/ifduyue/python-xxhash>`_ dependency switches the backend to ``xxhash.xxh3_128``, which is substantially faster on buffer-bound paths (e.g. numpy arrays, polars DataFrames):
+
+.. code-block:: console
+
+    pip install "apache-hamilton[performance]"
+
+Both backends produce a 16-byte digest, so cache keys and ``data_version`` strings are unaffected in shape. However, the two algorithms produce different digests for the same input, so **installing or removing xxhash changes every ``data_version`` in your dataflow**, invalidating previously persisted caches. Cached results are just recomputed on the next run, at the cost of losing the benefit of the existing cache.
+
+.. note::
+
+    Because the resolved backend is process-wide, keep it consistent across the environments that share a cache (e.g. all workers writing to the same cache store) to avoid needless cache misses.
+
 Recursion depth
 ~~~~~~~~~~~~~~~
 

hamilton/caching/fingerprinting.py

@@ -33,14 +33,15 @@
 implementation should pass the `depth` parameter to prevent `RecursionError`.
 """
 
+from __future__ import annotations
+
 import base64
 import datetime
 import functools
 import logging
 import sys
 from collections.abc import Mapping, Sequence, Set
-
-import xxhash
+from typing import TYPE_CHECKING
 
 from hamilton.experimental import h_databackends
 
@@ -50,6 +51,31 @@
 except ImportError:
     NoneType = type(None)
 
+if TYPE_CHECKING:
+    from collections.abc import Buffer, Callable
+    from typing import Protocol
+
+    class Hash(Protocol):
+        def digest(self) -> bytes: ...
+
+
+hash_func: Callable[[str | Buffer], Hash]
+try:
+    # xxh3_128 produces a 16-byte digest (24 base64url chars, the same width as the
+    # md5 it replaces) while running substantially faster on buffer-bound paths.
+    # TODO(2.0): revisit once/if a dependency-free `apache-hamilton-core` package
+    # exists, so this optional performance dependency lands in the right tier.
+    import xxhash
+
+    hash_func = xxhash.xxh3_128
+except (ModuleNotFoundError, AttributeError):
+    # ModuleNotFoundError covers xxhash not being installed; AttributeError
+    # covers an xxhash older than 0.8.0 (which added xxh3_128).
+    # usedforsecurity=False avoids a ValueError on FIPS-mode Python; it
+    # doesn't change the digest.
+    import hashlib
+
+    hash_func = functools.partial(hashlib.md5, usedforsecurity=False)
 
 logger = logging.getLogger("hamilton.caching")
 
@@ -78,16 +104,20 @@ def _compact_hash(digest: bytes) -> str:
 
 
 def _hash_bytes(data: bytes) -> str:
-    """Hash raw bytes with the non-cryptographic xxh3_128 algorithm and
-    compact-encode the digest.
+    """Hash raw bytes and compact-encode the digest.
 
     All hashing in this module routes through this single helper so the
     underlying hashing algorithm can be changed in exactly one place.
-    xxh3_128 produces a 16-byte digest (24 base64url chars, the same width
-    as the md5 it replaces) while running substantially faster on
-    buffer-bound paths.
+
+    Uses the non-cryptographic xxh3_128 algorithm when the optional
+    ``xxhash`` package is installed (see the ``performance`` extra),
+    falling back to hashlib's md5 otherwise. Both produce a 16-byte digest
+    (24 base64url chars), so digest width is unaffected either way, but the
+    two algorithms don't produce the same bytes for the same input:
+    fingerprints are stable within an environment, not across installs with
+    a different backend.
     """
-    return _compact_hash(xxhash.xxh3_128(data).digest())
+    return _compact_hash(hash_func(data).digest())
 
 
 @functools.singledispatch

pyproject.toml

@@ -51,7 +51,6 @@ dependencies = [
     "pandas",
     "typing_extensions > 4.0.0",
     "typing_inspect",
-    "xxhash>=0.8.0",
 ]
 
 [project.optional-dependencies]
@@ -71,6 +70,10 @@ experiments = [
 lsp = ["apache-hamilton-lsp"]
 openlineage = ["openlineage-python"]
 pandera = ["pandera"]
+performance = [
+    # Internal performance boosters go here
+    "xxhash>=0.8.0",
+]
 pydantic = ["pydantic>=2.0"]
 pyspark = [
   # we have to run these dependencies because Spark does not check to ensure the right target was called
@@ -135,6 +138,7 @@ test = [
   "xgboost; python_version < '3.14'",
   "xlsx2csv", # for excel data loader
   "xlsxwriter",  # Excel export requires 'xlsxwriter'
+  "xxhash>=0.8.0", # exercises the high-performance fingerprinting path in tests
 ]
 docs = [
   {include-group = "dev"},

tests/caching/test_fingerprinting.py

@@ -20,13 +20,28 @@
 the original `hash_value()` and the `hash_primitive()` functions.
 """
 
+import functools
+import hashlib
+import importlib
+import sys
+import types
+
 import numpy as np
 import pandas as pd
 import pytest
 
 from hamilton.caching import fingerprinting
 
 
+@pytest.fixture
+def force_md5_backend(monkeypatch):
+    """Force the hashlib.md5 fallback backend, regardless of whether xxhash
+    is installed in the current environment."""
+    monkeypatch.setattr(
+        fingerprinting, "hash_func", functools.partial(hashlib.md5, usedforsecurity=False)
+    )
+
+
 def test_hash_none():
     fingerprint = fingerprinting.hash_value(None)
     assert fingerprint == "<none>"
@@ -226,6 +241,123 @@ def test_hash_numpy():
     assert fingerprint == expected_hash
 
 
+# ---------------------------------------------------------------------------
+# hashlib.md5 fallback backend
+#
+# These mirror the pinned tests above but force `hash_func` to the hashlib.md5
+# fallback (via the `force_md5_backend` fixture) so the fallback path used
+# when `xxhash` isn't installed is verified regardless of what's installed in
+# the environment running the suite. Expected digests are the pre-xxh3_128
+# values this module used before xxhash became the default backend.
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+@pytest.mark.parametrize(
+    ("obj", "expected_hash"),
+    [
+        ("hello-world", "L1Q1Kh6_t1atHO_H8RbBeA=="),
+        (17.31231, "mJPTpPyXDSZgU-u8NuztIQ=="),
+        (16474, "6MgAp1NbMW0ZZpe_8iKVsg=="),
+        (True, "J2eGynSuIpd5bwVQzO9VVg=="),
+        (b"\x951!\x89u=\xe6\xadG\xdf", "d1DufDgRQmqi9Kt4Z2PeUQ=="),
+    ],
+)
+def test_hash_primitive_md5_fallback(obj, expected_hash):
+    fingerprint = fingerprinting.hash_primitive(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+@pytest.mark.parametrize(
+    ("obj", "expected_hash"),
+    [
+        ([0, True, "hello-world"], "mlOjj4yeCrSDFSn5zgdEIg=="),
+        ((17.0, False, "world"), "BcRSGfyKeIOdym9B6TmAyQ=="),
+    ],
+)
+def test_hash_sequence_md5_fallback(obj, expected_hash):
+    fingerprint = fingerprinting.hash_sequence(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_ordered_mapping_md5_fallback():
+    obj = {0: True, "key": "value", 17.0: None}
+    expected_hash = "GyxyI9-pq-EJJvSAIN509g=="
+    fingerprint = fingerprinting.hash_mapping(obj, ignore_order=False)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_unordered_mapping_md5_fallback():
+    obj = {0: True, "key": "value", 17.0: None}
+    expected_hash = "cDuuL2eA3DaSWlWW3u7o9g=="
+    fingerprint = fingerprinting.hash_mapping(obj, ignore_order=True)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_set_md5_fallback():
+    obj = {0, True, "key", "value", 17.0, None}
+    expected_hash = "E_f_tjbi6qn7KL3NUCZayg=="
+    fingerprint = fingerprinting.hash_set(obj)
+    assert fingerprint == expected_hash
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_numpy_md5_fallback():
+    array = np.array([[0, 1], [2, 3]], dtype=np.int64)
+    expected_hash = "024zwZIcWy6r4dlX4AMTow=="
+    fingerprint = fingerprinting.hash_value(array)
+    assert fingerprint == expected_hash
+
+
+def test_hash_bytes_digest_width():
+    """Both backends produce a 16-byte digest (24 base64url chars), so swapping
+    the algorithm doesn't change the shape of cache keys / `data_version` strings.
+    """
+    assert len(fingerprinting._hash_bytes(b"x")) == 24
+
+
+@pytest.mark.usefixtures("force_md5_backend")
+def test_hash_bytes_digest_width_md5_fallback():
+    assert len(fingerprinting._hash_bytes(b"x")) == 24
+
+
+# ---------------------------------------------------------------------------
+# Import-time backend resolution
+#
+# These reload the module to actually exercise the try/except at import time
+# (as opposed to `force_md5_backend`, which only overrides the already-resolved
+# `hash_func`), then restore the module to its real, ambient-environment state
+# so later tests aren't affected.
+# ---------------------------------------------------------------------------
+
+
+def test_falls_back_when_xxhash_not_installed(monkeypatch):
+    """Simulate xxhash being absent: `import xxhash` raises ModuleNotFoundError."""
+    monkeypatch.setitem(sys.modules, "xxhash", None)
+    try:
+        reloaded = importlib.reload(fingerprinting)
+        assert reloaded.hash_func.func is hashlib.md5
+        assert reloaded.hash_func.keywords == {"usedforsecurity": False}
+    finally:
+        importlib.reload(fingerprinting)
+
+
+def test_falls_back_when_xxhash_lacks_xxh3_128(monkeypatch):
+    """Simulate an xxhash older than 0.8.0, which doesn't define xxh3_128."""
+    stub = types.ModuleType("xxhash")
+    monkeypatch.setitem(sys.modules, "xxhash", stub)
+    try:
+        reloaded = importlib.reload(fingerprinting)
+        assert reloaded.hash_func.func is hashlib.md5
+        assert reloaded.hash_func.keywords == {"usedforsecurity": False}
+    finally:
+        importlib.reload(fingerprinting)
+
+
 def test_hash_numpy_different_shapes_differ():
     """Arrays with the same raw bytes but different shapes must hash differently."""
     a = np.array([1, 2, 3, 4, 5, 6])