RFC/DEPR: deprecate yt.funcs.levenshtein_distance in favor of difflib.get_close_matches

Author: neutrinoceros

nose_ignores.txt

@@ -50,3 +50,4 @@
 --ignore-file=test_sph_pixelization_pytestonly\.py
 --ignore-file=test_time_series\.py
 --ignore-file=test_cf_radial_pytest\.py
+--ignore-file=test_levenshtein_pytest\.py

tests/tests.yaml

@@ -231,5 +231,6 @@ other_tests:
      - "--ignore-file=test_offaxisprojection_pytestonly\\.py"
      - "--ignore-file=test_sph_pixelization_pytestonly\\.py"
      - "--ignore-file=test_cf_radial_pytest\\.py"
+     - "--ignore-file=test_levenshtein_pytest\\.py"
   cookbook:
      - 'doc/source/cookbook/tests/test_cookbook.py'

yt/funcs.py

@@ -1414,6 +1414,12 @@ def levenshtein_distance(seq1, seq2, max_dist=None):
     as the number of edits goes above the value. This allows for an earlier break
     and speeds calculations up.
     """
+    issue_deprecation_warning(
+        "yt.funcs.levenshtein_distance is deprecated. "
+        "Please prefer difflib.get_close_matches.",
+        since="4.5.0",
+        stacklevel=3,
+    )
     size_x = len(seq1) + 1
     size_y = len(seq2) + 1
     if max_dist is None:

yt/loaders.py

@@ -10,6 +10,7 @@
 import types
 import warnings
 from collections.abc import Mapping
+from difflib import get_close_matches
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Union, cast
 from urllib.parse import urlsplit
@@ -23,7 +24,6 @@
 )
 from yt._typing import AnyFieldKey, AxisOrder, FieldKey
 from yt.data_objects.static_output import Dataset
-from yt.funcs import levenshtein_distance
 from yt.sample_data.api import lookup_on_disk_data
 from yt.utilities.decompose import decompose_array, get_psize
 from yt.utilities.exceptions import (
@@ -1513,10 +1513,7 @@ def _get_sample_data(
     known_names: list[str] = registry_table.dropna()["filename"].to_list()
     if topdir not in known_names:
         msg = f"'{topdir}' is not an available dataset."
-        lexical_distances: list[tuple[str, int]] = [
-            (name, levenshtein_distance(name, topdir)) for name in known_names
-        ]
-        suggestions: list[str] = [name for name, dist in lexical_distances if dist < 4]
+        suggestions = get_close_matches(topdir, known_names)
         if len(suggestions) == 1:
             msg += f" Did you mean '{suggestions[0]}' ?"
         elif suggestions:

yt/tests/test_funcs.py

@@ -3,13 +3,7 @@
 from nose.tools import assert_raises
 from numpy.testing import assert_equal
 
-from yt.funcs import (
-    just_one,
-    levenshtein_distance,
-    simple_download_file,
-    validate_axis,
-    validate_center,
-)
+from yt.funcs import just_one, simple_download_file, validate_axis, validate_center
 from yt.testing import fake_amr_ds
 from yt.units import YTArray, YTQuantity
 
@@ -71,41 +65,6 @@ def test_just_one():
         assert jo == expected
 
 
-def test_levenshtein():
-    assert_equal(levenshtein_distance("abcdef", "abcdef"), 0)
-
-    # Deletions / additions
-    assert_equal(levenshtein_distance("abcdef", "abcde"), 1)
-    assert_equal(levenshtein_distance("abcdef", "abcd"), 2)
-    assert_equal(levenshtein_distance("abcdef", "abc"), 3)
-
-    assert_equal(levenshtein_distance("abcdf", "abcdef"), 1)
-    assert_equal(levenshtein_distance("cdef", "abcdef"), 2)
-    assert_equal(levenshtein_distance("bde", "abcdef"), 3)
-
-    # Substitutions
-    assert_equal(levenshtein_distance("abcd", "abc_"), 1)
-    assert_equal(levenshtein_distance("abcd", "ab__"), 2)
-    assert_equal(levenshtein_distance("abcd", "a___"), 3)
-    assert_equal(levenshtein_distance("abcd", "____"), 4)
-
-    # Deletion + Substitutions
-    assert_equal(levenshtein_distance("abcd", "abc_z"), 2)
-    assert_equal(levenshtein_distance("abcd", "ab__zz"), 4)
-    assert_equal(levenshtein_distance("abcd", "a___zzz"), 6)
-    assert_equal(levenshtein_distance("abcd", "____zzzz"), 8)
-
-    # Max distance
-    assert_equal(levenshtein_distance("abcd", "", max_dist=0), 1)
-    assert_equal(levenshtein_distance("abcd", "", max_dist=3), 4)
-    assert_equal(levenshtein_distance("abcd", "", max_dist=10), 4)
-    assert_equal(levenshtein_distance("abcd", "", max_dist=1), 2)
-    assert_equal(levenshtein_distance("abcd", "a", max_dist=2), 3)
-    assert_equal(levenshtein_distance("abcd", "ad", max_dist=2), 2)
-    assert_equal(levenshtein_distance("abcd", "abd", max_dist=2), 1)
-    assert_equal(levenshtein_distance("abcd", "abcd", max_dist=2), 0)
-
-
 def test_simple_download_file():
     fn = simple_download_file("http://yt-project.org", "simple-download-file")
     try:

yt/tests/test_funcs_pytest.py

@@ -0,0 +1,49 @@
+import pytest
+
+from yt.funcs import levenshtein_distance
+
+
+@pytest.mark.parametrize(
+    "a, b, expected",
+    [
+        ("abcdef", "abcdef", 0),
+        # Deletions / additions
+        ("abcdef", "abcde", 1),
+        ("abcdef", "abcd", 2),
+        ("abcdef", "abc", 3),
+        ("abcdf", "abcdef", 1),
+        ("cdef", "abcdef", 2),
+        ("bde", "abcdef", 3),
+        # Substitutions
+        ("abcd", "abc_", 1),
+        ("abcd", "ab__", 2),
+        ("abcd", "a___", 3),
+        ("abcd", "____", 4),
+        # Deletion + Substitutions
+        ("abcd", "abc_z", 2),
+        ("abcd", "ab__zz", 4),
+        ("abcd", "a___zzz", 6),
+        ("abcd", "____zzzz", 8),
+    ],
+)
+def test_levenshtein(a, b, expected):
+    with pytest.deprecated_call():
+        assert levenshtein_distance(a, b) == expected
+
+
+@pytest.mark.parametrize(
+    "a, b, max_dist, expected",
+    [
+        ("abcd", "", 0, 1),
+        ("abcd", "", 3, 4),
+        ("abcd", "", 10, 4),
+        ("abcd", "", 1, 2),
+        ("abcd", "a", 2, 3),
+        ("abcd", "ad", 2, 2),
+        ("abcd", "abd", 2, 1),
+        ("abcd", "abcd", 2, 0),
+    ],
+)
+def test_levenshtein_with_max_dist(a, b, max_dist, expected):
+    with pytest.deprecated_call():
+        assert levenshtein_distance(a, b, max_dist=max_dist) == expected

yt/utilities/exceptions.py

@@ -1,5 +1,6 @@
 # We don't need to import 'exceptions'
 import os.path
+from difflib import get_close_matches
 
 from unyt.exceptions import UnitOperationError
 
@@ -97,51 +98,36 @@ def __init__(self, field, ds):
         self.ds = ds
 
     def _get_suggestions(self) -> list[FieldKey]:
-        from yt.funcs import levenshtein_distance
-
         field = self.field
         ds = self.ds
 
-        suggestions = {}
         if not isinstance(field, tuple):
             ftype, fname = None, field
         elif field[1] is None:
             ftype, fname = None, field[0]
         else:
             ftype, fname = field
 
-        # Limit the suggestions to a distance of 3 (at most 3 edits)
-        # This is very arbitrary, but is picked so that...
-        # - small typos lead to meaningful suggestions (e.g. `densty` -> `density`)
-        # - we don't suggest unrelated things (e.g. `pressure` -> `density` has a distance
-        #   of 6, we definitely do not want it)
-        # A threshold of 3 seems like a good middle point.
-        max_distance = 3
+        suggestions: list[FieldKey] = []
+        if ftype is not None:
+            fields_str: dict[str, FieldKey] = {
+                str(df).lower(): df for df in ds.derived_field_list
+            }
+            field_str = str(field).lower()
+            suggestions.extend(
+                fields_str[k] for k in get_close_matches(field_str, fields_str.keys())
+            )
 
-        # Suggest (ftype, fname), with alternative ftype
+        # Ensure we suggest (ftype, fname), with alternative ftype
         for ft, fn in ds.derived_field_list:
-            if fn.lower() == fname.lower() and (
-                ftype is None or ft.lower() != ftype.lower()
+            if (
+                fn.lower() == fname.lower()
+                and (ftype is None or ft.lower() != ftype.lower())
+                and (ft, fn) not in suggestions
             ):
-                suggestions[ft, fn] = 0
-
-        if ftype is not None:
-            # Suggest close matches using levenshtein distance
-            fields_str = {_: str(_).lower() for _ in ds.derived_field_list}
-            field_str = str(field).lower()
+                suggestions.insert(0, (ft, fn))
 
-            for (ft, fn), fs in fields_str.items():
-                distance = levenshtein_distance(field_str, fs, max_dist=max_distance)
-                if distance < max_distance:
-                    if (ft, fn) in suggestions:
-                        continue
-                    suggestions[ft, fn] = distance
-
-        # Return suggestions sorted by increasing distance (first are most likely)
-        return [
-            (ft, fn)
-            for (ft, fn), distance in sorted(suggestions.items(), key=lambda v: v[1])
-        ]
+        return suggestions
 
     def __str__(self):
         msg = f"Could not find field {self.field!r} in {self.ds}."