Add support for integrated suggest-as-you-type suggestions.

This patch aims to remove the govarnam dependency for search suggestions.

- Introduces a new trie-based in-memory word search mechanism which
  is populated using all words in the DB during load. Configurable with
  the `[suggestions]` config block. This works for all languages that
  are entered in the input box in their native Unicode forms.

  If no suggestions are found, it falls back to fulltext search on the DB.

- For languages (eg: Malayalam, Kannada) where the input can be Latin
  characters denoting phonetic strings (eg: "amma"), search + suggestions
  still work thanks to the new transliteration algorithm shipped in the
  bundled luka tokenizers. This works for any language. As long as
  tokenize() in the Lua tokenizers return tokens, whatever the input maybe.

  So, this patch introduces transliteration mapping for generating IndicPhone
  hashes for "Manglish" and "Kanglish" latin char phonetic inputs.

Author: knadh

Cargo.lock

@@ -40,6 +40,7 @@ bytes = "1"
 bincode = "1"
 rmp-serde = "1"
 md5 = "0.8"
+trie-rs = "0.4.2"
 
 # Use mimalloc for musl builds (musl's default malloc is very slow).
 [target.'cfg(target_env = "musl")'.dependencies]

Cargo.toml

@@ -65,6 +65,15 @@ max_disk_mb = 512
 dir = "/tmp/dictpress-cache"
 
 
+[suggestions]
+# Enable autocomplete suggestions for search queries on the frontend.
+# This loads all dictionary words into memory for fast lookup.
+enabled = false
+
+# Number of suggestions to return.
+num = 10
+
+
 ################################################################################
 
 

config.sample.toml

@@ -265,14 +265,13 @@ async function screenshotElement(element) {
       clearTimeout(debounce);
       return new Promise(resolve => {
         debounce = setTimeout(async () => {
-          const response = await fetch(`${_ROOT_URL}/atl/${langCode}/${val.toLowerCase()}`);
-          const data = await response.json();
+            const response = await fetch(`${_ROOT_URL}/api/suggestions/${langCode}/${val}`);
+            const data = await response.json();
 
-          const a = data.greedy_tokenized.map(item => item.word).slice(0, 3).sort((a, b) => a.length - b.length);
-          const b = data.dictionary_suggestions.map(item => item.word).slice(0, 6).sort((a, b) => a.length - b.length);
+            const suggestions = data.data.map(item => item.content[0]);
 
-          debounce = null;
-          resolve([...new Set(a.concat(b))]);
+            debounce = null;
+            resolve(suggestions);
         }, 50);
       });
     },

site/static/main.js

@@ -20,6 +20,7 @@ use crate::{
     cache::Cache,
     manager::Manager,
     models::{Dicts, LangMap},
+    suggestions::Suggestions,
 };
 
 pub type I18n = HashMap<String, String>;
@@ -30,6 +31,7 @@ pub struct Ctx {
     pub langs: LangMap,
     pub dicts: Dicts,
     pub cache: Option<Arc<Cache>>,
+    pub suggestions: Option<Arc<Suggestions>>,
 
     /// Admin templates (always loaded, embedded in binary).
     pub admin_tpl: Arc<Tera>,
@@ -72,6 +74,10 @@ pub struct Consts {
     pub glossary_max_per_page: i32,
     pub glossary_num_page_nums: i32,
 
+    // Suggestions settings.
+    pub suggestions_enabled: bool,
+    pub num_suggestions: i32,
+
     // Admin assets split by type for easier template rendering.
     pub admin_js_assets: Vec<String>,
     pub admin_css_assets: Vec<String>,
@@ -100,6 +106,9 @@ impl Default for Consts {
             glossary_max_per_page: 100,
             glossary_num_page_nums: 10,
 
+            suggestions_enabled: false,
+            num_suggestions: 10,
+
             admin_js_assets: Vec::new(),
             admin_css_assets: Vec::new(),
         }

src/handlers/mod.rs

@@ -7,7 +7,9 @@ use axum::{
 
 use super::{json, paginate, total_pages, ApiErr, ApiResp, Ctx, Result};
 use crate::cache::make_search_cache_key;
-use crate::models::{RelationsQuery, SearchQuery, SearchResults, STATUS_ENABLED};
+use crate::models::{
+    RelationsQuery, SearchQuery, SearchResults, StringArray, Suggestion, STATUS_ENABLED,
+};
 
 /// Search a dictionary with query in path (public API).
 pub async fn search(
@@ -158,3 +160,52 @@ pub async fn do_search(ctx: Arc<Ctx>, query: SearchQuery, is_admin: bool) -> Res
 
     Ok(results)
 }
+
+/// Suggestions endpoint for search word autocomplete.
+pub async fn get_suggestions(
+    State(ctx): State<Arc<Ctx>>,
+    Path((lang, q)): Path<(String, String)>,
+) -> Result<ApiResp<Vec<Suggestion>>> {
+    if q.is_empty() {
+        return Err(ApiErr::new("`q` is required", StatusCode::BAD_REQUEST));
+    }
+    if !ctx.langs.contains_key(&lang) {
+        return Err(ApiErr::new("unknown language", StatusCode::BAD_REQUEST));
+    }
+
+    // If suggestions are disable, return an empty array.
+    if !ctx.consts.suggestions_enabled {
+        return Ok(json(Vec::new()));
+    }
+
+    let limit = ctx.consts.num_suggestions;
+
+    // Try trie search first if suggestions are enabled.
+    let mut out: Vec<Suggestion> = if let Some(sugg) = &ctx.suggestions {
+        sugg.query(&lang, &q, limit as usize)
+            .into_iter()
+            .map(|w| Suggestion {
+                content: StringArray(vec![w]),
+            })
+            .collect()
+    } else {
+        Vec::new()
+    };
+
+    // If there are fewer than limit results, supplement with DB FTS search.
+    if out.len() < limit as usize {
+        let remaining = limit - out.len() as i32;
+        if let Ok(res) = ctx.mgr.get_suggestions(&lang, &q, remaining).await {
+            for s in res {
+                if !out.iter().any(|r| r.content.0 == s.content.0) {
+                    out.push(s);
+                    if out.len() >= limit as usize {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(json(out))
+}

src/handlers/search.rs

@@ -35,7 +35,8 @@ pub fn init_handlers(ctx: Arc<Ctx>) -> Router {
         .route(
             "/api/dictionary/entries/{guid}",
             get(entries::get_entry_by_guid),
-        );
+        )
+        .route("/api/suggestions/{lang}/{q}", get(search::get_suggestions));
 
     // Public submission routes (if enabled).
     let submit_routes = Router::new()

src/http.rs

@@ -1,5 +1,7 @@
 use crate::cache::{Cache, CacheConfig, CacheError};
+use crate::manager::Manager;
 use crate::models::{Config, Dicts, Lang, LangMap, DEFAULT_TOKENIZER};
+use crate::suggestions::Suggestions;
 use crate::tokenizer::Tokenizers;
 
 /// Initialize logger.
@@ -182,3 +184,20 @@ pub async fn cache(cfg: &CacheConfig) -> Result<Cache, CacheError> {
     );
     Cache::new(cfg).await
 }
+
+/// Initialize suggestions trie from database.
+pub async fn suggestions(
+    mgr: &Manager,
+    langs: &LangMap,
+) -> Result<Suggestions, Box<dyn std::error::Error>> {
+    let mut sugg = Suggestions::new();
+
+    for lang_id in langs.keys() {
+        let words = mgr.get_all_words(lang_id).await?;
+        let num = words.len();
+        sugg.build(lang_id, words);
+        log::info!("suggestions: loaded {} words for {}", num, lang_id);
+    }
+
+    Ok(sugg)
+}

src/init.rs

@@ -9,6 +9,7 @@ mod init;
 mod manager;
 mod models;
 mod sitemaps;
+mod suggestions;
 mod tokenizer;
 
 // Use mimalloc for musl builds (musl's default malloc is very slow).
@@ -226,6 +227,19 @@ async fn main() {
         None
     };
 
+    // Initialize trie suggestions if enabled.
+    let suggestions = if config.suggestions.enabled {
+        match init::suggestions(&mgr, &langs).await {
+            Ok(s) => Some(Arc::new(s)),
+            Err(e) => {
+                log::error!("error initializing suggestions module: {}", e);
+                None
+            }
+        }
+    } else {
+        None
+    };
+
     // Preload static files (JS & CSS) for bundling.
     let static_files = http::preload_static_files(&cli.site);
 
@@ -235,6 +249,7 @@ async fn main() {
         langs,
         dicts,
         cache,
+        suggestions,
         admin_tpl,
         site_tpl,
         site_path: cli.site.clone(),
@@ -263,6 +278,9 @@ async fn main() {
             glossary_max_per_page: config.glossary.max_per_page,
             glossary_num_page_nums: config.glossary.num_page_nums,
 
+            suggestions_enabled: config.suggestions.enabled,
+            num_suggestions: config.suggestions.num,
+
             // Split admin assets by file extension for template rendering.
             admin_js_assets: config
                 .app

src/main.rs

@@ -5,7 +5,7 @@ use sqlx::{sqlite::SqlitePool, Row};
 use crate::{
     models::{
         q, Comment, Dicts, Entry, GlossaryWord, LangMap, Relation, RelationsQuery, SearchQuery,
-        Stats, STATUS_ENABLED,
+        Stats, Suggestion, STATUS_ENABLED,
     },
     tokenizer::{Tokenizer, TokenizerError, Tokenizers},
 };
@@ -116,6 +116,33 @@ impl Manager {
         Ok((results, total))
     }
 
+    /// Get word suggestions for autocomplete.
+    pub async fn get_suggestions(
+        &self,
+        lang: &str,
+        query: &str,
+        limit: i32,
+    ) -> Result<Vec<Suggestion>, Error> {
+        if !self.langs.contains_key(lang) {
+            return Err(Error::UnknownLang(lang.to_string()));
+        }
+
+        let fts_query = self.to_fts_query(query, lang)?;
+        if fts_query.trim().is_empty() {
+            return Err(Error::Validation("invalid search query".to_string()));
+        }
+
+        let results: Vec<Suggestion> = sqlx::query_as(&q.search_words.query)
+            .bind(lang)
+            .bind(query)
+            .bind(&fts_query)
+            .bind(limit)
+            .fetch_all(&self.db)
+            .await?;
+
+        Ok(results)
+    }
+
     /// Load relations for a set of entries.
     /// rel_query.max_per_type: 0 = load all relations, >0 = limit relations per type per entry.
     /// rel_query.max_content_items: 0 = no truncation, >0 = truncate content array.
@@ -582,4 +609,19 @@ impl Manager {
         let stats: Stats = serde_json::from_str(&row.0).unwrap_or_default();
         Ok(stats)
     }
+
+    /// Get all normalized words for a given language (for building suggestions trie).
+    pub async fn get_all_words(&self, lang: &str) -> Result<Vec<String>, Error> {
+        let rows: Vec<(String,)> = sqlx::query_as(&q.get_all_words.query)
+            .bind(lang)
+            .fetch_all(&self.db)
+            .await?;
+
+        // Filter out empty strings and collect
+        Ok(rows
+            .into_iter()
+            .map(|(s,)| s.trim().to_string())
+            .filter(|s| !s.is_empty())
+            .collect())
+    }
 }