✨ use rapidfuzz again

Author: Joshix-1

Pipfile

@@ -14,7 +14,6 @@ defity = "<2"
 defusedxml = "<2"
 dill = "<2"
 ecs-logging = "==2.*"
-editdistancek-rs = "==1.*"
 elastic-apm = "==6.*"
 elastic-enterprise-search = "<8.16"
 elasticsearch = {version = "==8.15.*", extras = ["async"]}
@@ -40,6 +39,7 @@ python-geoip-geolite2-yplan = "==2019.1224"
 python-geoip-yplan = "==1.*"
 pytz = "*"
 PyYAML = "==6.*"
+rapidfuzz = "==3.*"
 redis = {version = "==5.*", extras = ["hiredis"]}
 regex = "==2024.9.11"
 setproctitle = "==1.*"

Pipfile.lock

@@ -17,12 +17,11 @@
 
 from collections.abc import Collection
 
-from editdistancek_rs import distance
 from hangman_solver import Language, read_words_with_length
 from typed_stream import Stream
 
 from ..utils.request_handler import APIRequestHandler, HTMLRequestHandler
-from ..utils.utils import ModuleInfo
+from ..utils.utils import ModuleInfo, bounded_edit_distance
 
 
 def get_module_info() -> ModuleInfo:
@@ -56,7 +55,9 @@ def find_solutions(word: str, ignore: Collection[str]) -> Stream[str]:
             )
         )
         .exclude(ignore.__contains__)
-        .filter(lambda test_word: distance(word, test_word, 2) == 1)
+        .filter(
+            lambda test_word: bounded_edit_distance(word, test_word, 2) == 1
+        )
     )
 
 

an_website/hangman_solver/wordgame_solver.py

@@ -18,10 +18,10 @@
 from dataclasses import dataclass
 from typing import cast
 
-from editdistancek_rs import distance
 from tornado.web import HTTPError, MissingArgumentError
 
 from ..utils.data_parsing import parse_args
+from ..utils.utils import bounded_edit_distance
 from .utils import (
     AUTHORS_CACHE,
     QUOTES_CACHE,
@@ -117,7 +117,7 @@ async def get_authors(author_name: str) -> list[Author | str]:
         *(
             author
             for author in AUTHORS_CACHE.values()
-            if distance(
+            if bounded_edit_distance(
                 author.name.lower(), author_name_lower, max_distance + 1
             )
             <= max_distance
@@ -168,7 +168,9 @@ async def get_quotes(quote_str: str) -> list[Quote | str]:
         *(
             quote
             for quote in QUOTES_CACHE.values()
-            if distance(quote.quote.lower(), lower_quote_str, max_distance + 1)
+            if bounded_edit_distance(
+                quote.quote.lower(), lower_quote_str, max_distance + 1
+            )
             <= max_distance
         ),
         fix_quote_str(quote_str),

an_website/quotes/create.py

@@ -59,11 +59,11 @@
 import elasticapm
 import regex
 from blake3 import blake3
-from editdistancek_rs import distance
 from elastic_transport import ApiError, TransportError
 from elasticsearch import AsyncElasticsearch
 from geoip import geolite2  # type: ignore[import-untyped]
 from openmoji_dist import VERSION as OPENMOJI_VERSION
+from rapidfuzz.distance.Levenshtein import distance
 from redis.asyncio import Redis
 from tornado.web import HTTPError, RequestHandler
 from typed_stream import Stream
@@ -300,6 +300,16 @@ def bool_to_str(val: bool) -> str:
     return "sure" if val else "nope"
 
 
+def bounded_edit_distance(s1: str, s2: str, /, k: int) -> int:
+    """Return a bounded edit distance between two strings.
+
+    k is the maximum number returned
+    """
+    if (dist := distance(s1, s2, score_cutoff=k)) == k + 1:
+        return k
+    return dist
+
+
 def country_code_to_flag(code: str) -> str:
     """Convert a two-letter ISO country code to a flag emoji."""
     return "".join(chr(ord(char) + 23 * 29 * 191) for char in code.upper())
@@ -526,7 +536,9 @@ def get_close_matches(  # based on difflib.get_close_matches
     result: list[tuple[float, str]] = []
     for possibility in possibilities:
         if max_dist := max(word_len, len(possibility)):
-            dist = distance(possibility, word, 1 + int(cutoff * max_dist))
+            dist = bounded_edit_distance(
+                possibility, word, 1 + int(cutoff * max_dist)
+            )
             if (ratio := dist / max_dist) <= cutoff:
                 bisect.insort(result, (ratio, possibility))
                 if len(result) > count:

an_website/utils/utils.py

@@ -118,7 +118,6 @@ defity==0.5.1
 defusedxml==0.7.1
 dunamai==1.23.0
 ecs-logging==2.2.0
-editdistancek-rs==1.0.0
 elastic-apm==6.23.0
 elastic-enterprise-search==8.11.0
 elastic-transport==8.17.0
@@ -148,6 +147,7 @@ pysocks==1.7.1
 python-geoip-geolite2-yplan==2019.1224
 python-geoip-yplan==1.2
 pytz==2024.2
+rapidfuzz==3.11.0
 redis==5.2.1
 regex==2024.9.11
 setproctitle==1.3.4

pip-constraints.txt

@@ -118,7 +118,6 @@ defity==0.5.1; python_version >= '3.10'
 defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 dunamai==1.23.0; python_version >= '3.5'
 ecs-logging==2.2.0; python_version >= '3.6'
-editdistancek-rs==1.0.0; python_version >= '3.12'
 elastic-apm==6.23.0; python_version >= '3.6' and python_version < '4'
 elastic-enterprise-search==8.11.0; python_version >= '3.6'
 elastic-transport==8.17.0; python_version >= '3.8'
@@ -148,6 +147,7 @@ pysocks==1.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2
 python-geoip-geolite2-yplan==2019.1224
 python-geoip-yplan==1.2
 pytz==2024.2
+rapidfuzz==3.11.0; python_version >= '3.9'
 redis[hiredis]==5.2.1; python_version >= '3.8'
 regex==2024.9.11; python_version >= '3.8'
 setproctitle==1.3.4; python_version >= '3.8'

pip-dev-requirements.txt

@@ -15,7 +15,6 @@ defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1,
 dill==0.3.9; python_version >= '3.8'
 dunamai==1.23.0; python_version >= '3.5'
 ecs-logging==2.2.0; python_version >= '3.6'
-editdistancek-rs==1.0.0; python_version >= '3.12'
 elastic-apm==6.23.0; python_version >= '3.6' and python_version < '4'
 elastic-enterprise-search==8.11.0; python_version >= '3.6'
 elastic-transport==8.17.0; python_version >= '3.8'
@@ -50,6 +49,7 @@ python-geoip-geolite2-yplan==2019.1224
 python-geoip-yplan==1.2
 pytz==2024.2
 pyyaml==6.0.2; python_version >= '3.8'
+rapidfuzz==3.11.0; python_version >= '3.9'
 redis[hiredis]==5.2.1; python_version >= '3.8'
 regex==2024.9.11; python_version >= '3.8'
 setproctitle==1.3.4; python_version >= '3.8'