Handle un-escape

Author: lpil

src/packages/override.gleam

@@ -32,8 +32,12 @@ pub fn is_ignored_package(name: String) -> Bool {
   }
 }
 
-/// Some words have common misspellings or associated words so we add those to
-/// the search to get all appropriate results.
+/// Some words have common alterative words so we add those to the search to
+/// get all appropriate results.
+///
+/// Common spelling mistakes and dialect variations are handled in
+/// `text_search.normalise` instead.
+///
 pub fn expand_search_term(term: String) -> List(String) {
   case term {
     "postgres" | "postgresql" -> ["postgres", "postgresql"]

src/packages/text_search.gleam

@@ -153,39 +153,53 @@ fn split_and_normalise_words(text: String) -> List(String) {
   |> string.replace("'", "")
   |> string.split(" ")
   |> list.filter(fn(word) { word != "" })
-  |> list.map(normalise_spelling)
+  |> normalise([])
 }
 
-fn normalise_spelling(word: String) -> String {
-  case word {
-    "analyze" -> "analyse"
-    "authorize" -> "authorise"
-    "behavior" -> "behaviour"
-    "categorize" -> "categorise"
-    "color" -> "colour"
-    "customization" -> "customisation"
-    "customize" -> "customise"
-    "honor" -> "honour"
-    "initialize" -> "initialise"
-    "labeled" -> "labelled"
-    "labor" -> "labour"
-    "license" -> "licence"
-    "modeling" -> "modelling"
-    "normalization" -> "normalisation"
-    "normalize" -> "normalise"
-    "optimization" -> "optimisation"
-    "optimize" -> "optimise"
-    "organize" -> "organise"
-    "parameterize" -> "parameterise"
-    "deserialization" -> "deserialisation"
-    "deserialize" -> "deserialise"
-    "serialization" -> "serialisation"
-    "serialize" -> "serialise"
-    "standardize" -> "standardise"
-    "summarize" -> "summarise"
-    "synchronize" -> "synchronise"
-    "tokenize" -> "tokenise"
-    _ -> word
+/// Normalise spelling, unifying different English dialects and correcting some
+/// spelling mistakes, to get more permissive search results.
+///
+fn normalise(words: List(String), out: List(String)) -> List(String) {
+  case words {
+    [] -> list.reverse(out)
+
+    // The 
+    ["un", "escape", ..words] -> normalise(words, ["unescape", ..out])
+    ["un", "escaping", ..words] -> normalise(words, ["unescaping", ..out])
+
+    [word, ..words] -> {
+      let word = case word {
+        "analyze" -> "analyse"
+        "authorize" -> "authorise"
+        "behavior" -> "behaviour"
+        "categorize" -> "categorise"
+        "color" -> "colour"
+        "customization" -> "customisation"
+        "customize" -> "customise"
+        "honor" -> "honour"
+        "initialize" -> "initialise"
+        "labeled" -> "labelled"
+        "labor" -> "labour"
+        "license" -> "licence"
+        "modeling" -> "modelling"
+        "normalization" -> "normalisation"
+        "normalize" -> "normalise"
+        "optimization" -> "optimisation"
+        "optimize" -> "optimise"
+        "organize" -> "organise"
+        "parameterize" -> "parameterise"
+        "deserialization" -> "deserialisation"
+        "deserialize" -> "deserialise"
+        "serialization" -> "serialisation"
+        "serialize" -> "serialise"
+        "standardize" -> "standardise"
+        "summarize" -> "summarise"
+        "synchronize" -> "synchronise"
+        "tokenize" -> "tokenise"
+        _ -> word
+      }
+      normalise(words, [word, ..out])
+    }
   }
 }
 

test/packages/text_search_test.gleam

@@ -236,3 +236,15 @@ pub fn lookup_ist_test() {
   let assert Ok(value) = text_search.lookup(index, "geology")
   assert value == [Found("geologist", 1)]
 }
+
+// Odysseus writes "unescape" as "un-escape"
+pub fn un_escape_test() {
+  let index = text_search.new()
+  let assert Ok(_) =
+    text_search.insert(index, "houdini", "🪄 Fast HTML escaping")
+  let assert Ok(_) = text_search.insert(index, "odysseus", "UN-escaping HTML")
+
+  assert text_search.lookup(index, "escape") == Ok([Found("houdini", 1)])
+  assert text_search.lookup(index, "un-escape") == Ok([Found("odysseus", 1)])
+  assert text_search.lookup(index, "unescape") == Ok([Found("odysseus", 1)])
+}