Merge branch 'rolling' into engines

Author: mergify[bot]

Cargo.lock

@@ -1,12 +1,14 @@
 [package]
 name = "websurfx"
-version = "1.24.9"
-edition = "2021"
+version = "1.24.11"
+edition = "2024"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
 license = "AGPL-3.0"
 
 [dependencies]
+rayon = "1.10.0"
+arc-swap = "1.7.1"
 reqwest = { version = "0.12.5", default-features = false, features = [
     "rustls-tls",
     "brotli",
@@ -39,6 +41,7 @@ fake-useragent = { version = "0.1.3", default-features = false }
 env_logger = { version = "0.11.6", default-features = false }
 log = { version = "0.4.21", default-features = false }
 mlua = { version = "0.10.3", features = [
+    "async",
     "luajit",
     "vendored",
 ], default-features = false }
@@ -81,12 +84,10 @@ stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
 thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
     "moby",
 ]}
-
 actix-multipart = { version = "0.7.2", default-features = false, features = [
     "derive",
     "tempfile",
 ]}
-itertools = {version = "0.14.0", default-features = false}
 
 [dev-dependencies]
 rusty-hook = { version = "^0.11.2", default-features = false }
@@ -102,6 +103,17 @@ lightningcss = { version = "1.0.0-alpha.57", default-features = false, features
 # Temporary fork with fix
 minify-js = { git = "https://github.com/RuairidhWilliamson/minify-js", branch = "master", version = "0.6.0", default-features = false} 
 
+[lints.rust]
+unsafe_code = "forbid"
+rust_2018_idioms = { level = "warn", priority = 1 }
+missing_docs = "deny"
+
+[lints.clippy]
+cognitive_complexity = "warn"
+panic = "forbid"
+missing_docs_in_private_items = "deny"
+perf = { level = "deny", priority = 1 }
+
 [profile.dev]
 opt-level = 0
 debug = true
@@ -125,7 +137,7 @@ panic = 'abort'
 incremental = false
 codegen-units = 1
 rpath = false
-strip = "symbols"
+strip = "debuginfo"
 
 [profile.profiling]
 inherits = "release"
@@ -176,8 +188,8 @@ inherits = "lpcb1"
 opt-level = "z"
 
 [features]
-use-synonyms-search = ["thesaurus/static"]
 default = ["memory-cache"]
+use-synonyms-search = ["thesaurus/static"]
 dhat-heap = ["dep:dhat"]
 memory-cache = ["dep:moka"]
 redis-cache = ["dep:redis", "dep:base64"]

Cargo.toml

@@ -7,9 +7,9 @@
 
 // ------- Imports -------
 use lightningcss::stylesheet::{MinifyOptions, ParserOptions, PrinterOptions, StyleSheet};
-use minify_js::{minify, Session, TopLevelMode};
+use minify_js::{Session, TopLevelMode, minify};
 use std::{
-    fs::{read_dir, read_to_string, File, OpenOptions},
+    fs::{File, OpenOptions, read_dir, read_to_string},
     io::{Read, Write},
 };
 

build.rs

@@ -1,32 +1,31 @@
 //! This module provides the functionality to scrape and gathers all the results from the upstream
 //! search engines and then removes duplicate results.
 
-use super::user_agent::random_user_agent;
-use crate::handler::{file_path, FileType};
+use crate::handler::{FileType, file_path};
 use crate::models::{
     aggregation::{EngineErrorInfo, SearchResult, SearchResults},
     engine::{EngineError, EngineHandler},
 };
 use crate::parser::Config;
 
 use error_stack::Report;
-use futures::stream::FuturesUnordered;
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use rayon::slice::ParallelSliceMut;
 use regex::Regex;
 use reqwest::{Client, ClientBuilder};
 use std::sync::Arc;
+use tokio::task::JoinSet;
 use tokio::{
     fs::File,
     io::{AsyncBufReadExt, BufReader},
-    task::JoinHandle,
     time::Duration,
 };
 
 /// A constant for holding the prebuilt Client globally in the app.
 static CLIENT: std::sync::OnceLock<Client> = std::sync::OnceLock::new();
 
 /// Aliases for long type annotations
-type FutureVec =
-    FuturesUnordered<JoinHandle<Result<Vec<(String, SearchResult)>, Report<EngineError>>>>;
+type FutureVec = JoinSet<Result<Vec<(String, SearchResult)>, Report<EngineError>>>;
 
 /// The function aggregates the scraped results from the user-selected upstream search engines.
 /// These engines can be chosen either from the user interface (UI) or from the configuration file.
@@ -72,6 +71,7 @@ pub async fn aggregate(
     config: &Config,
     upstream_search_engines: &[EngineHandler],
     safe_search: u8,
+    user_agent: &'static str,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let client = CLIENT.get_or_init(|| {
         let mut cb = ClientBuilder::new()
@@ -96,19 +96,17 @@ pub async fn aggregate(
         cb.build().unwrap()
     });
 
-    let user_agent: &str = random_user_agent();
-
     let mut names: Vec<&str> = Vec::with_capacity(0);
 
     // create tasks for upstream result fetching
-    let tasks: FutureVec = FutureVec::new();
+    let mut tasks: FutureVec = JoinSet::new();
 
     let query: Arc<String> = Arc::new(query.to_string());
     for engine_handler in upstream_search_engines {
         let (name, search_engine) = engine_handler.clone().into_name_engine();
         names.push(name);
         let query_partially_cloned = query.clone();
-        tasks.push(tokio::spawn(async move {
+        tasks.spawn(async move {
             search_engine
                 .results(
                     &query_partially_cloned,
@@ -118,16 +116,7 @@ pub async fn aggregate(
                     safe_search,
                 )
                 .await
-        }));
-    }
-
-    // get upstream responses
-    let mut responses = Vec::with_capacity(tasks.len());
-
-    for task in tasks {
-        if let Ok(result) = task.await {
-            responses.push(result)
-        }
+        });
     }
 
     // aggregate search results, removing duplicates and handling errors the upstream engines returned
@@ -142,60 +131,58 @@ pub async fn aggregate(
         ));
     };
 
-    for _ in 0..responses.len() {
-        let response = responses.pop().unwrap();
+    while let Some(Ok(response)) = tasks.join_next().await {
         let engine = names.pop().unwrap();
 
-        if result_map.is_empty() {
-            match response {
-                Ok(results) => result_map = results,
-                Err(error) => handle_error(&error, engine),
-            };
-            continue;
-        }
-
-        match response {
-            Ok(result) => {
-                result.into_iter().for_each(|(key, value)| {
-                    match result_map.iter().find(|(key_s, _)| key_s == &key) {
-                        Some(value) => value.1.to_owned().add_engines(engine),
-                        None => result_map.push((key, value)),
-                    };
-                });
+        if let Ok(result) = response {
+            for (key, value) in result {
+                if let Some(value) = result_map.iter().find(|(key_s, _)| key_s == &key) {
+                    value.1.to_owned().add_engines(engine)
+                } else {
+                    result_map.push((key, value))
+                }
             }
-            Err(error) => handle_error(&error, engine),
-        };
+        } else if let Err(error) = response {
+            handle_error(&error, engine)
+        }
     }
 
     if safe_search >= 3 {
         let mut blacklist_map: Vec<(String, SearchResult)> = Vec::new();
         filter_with_lists(
             &mut result_map,
             &mut blacklist_map,
-            file_path(FileType::BlockList)?,
+            &file_path(FileType::BlockList).await?,
         )
         .await?;
 
         filter_with_lists(
             &mut blacklist_map,
             &mut result_map,
-            file_path(FileType::AllowList)?,
+            &file_path(FileType::AllowList).await?,
         )
         .await?;
 
         drop(blacklist_map);
     }
 
-    let mut results: Box<[SearchResult]> = result_map
-        .into_iter()
-        .map(|(_, mut value)| {
-            if !value.url.contains("temu.com") {
-                value.calculate_relevance(query.as_str())
-            }
-            value
-        })
-        .collect();
-    sort_search_results(&mut results);
+    let results: Box<[SearchResult]> = tokio::task::spawn_blocking(move || {
+        let mut unsorted_results: Box<[SearchResult]> = result_map
+            .par_iter()
+            .cloned()
+            .map(|(_, mut value)| {
+                if !value.url.contains("temu.com") {
+                    value.calculate_relevance(query.as_str())
+                }
+                value
+            })
+            .collect();
+
+        sort_search_results(&mut unsorted_results);
+
+        unsorted_results
+    })
+    .await?;
 
     Ok(SearchResults::new(
         results,
@@ -231,17 +218,16 @@ pub async fn filter_with_lists(
         while idx < length {
             let ele = &map_to_be_filtered[idx];
             let ele_inner = &ele.1;
-            match re.is_match(&ele.0.to_lowercase())
+            // If the search result matches the regex pattern, move it from the original map to the resultant map
+            if re.is_match(&ele.0.to_lowercase())
                 || re.is_match(&ele_inner.title.to_lowercase())
                 || re.is_match(&ele_inner.description.to_lowercase())
             {
-                true => {
-                    // If the search result matches the regex pattern, move it from the original map to the resultant map
-                    resultant_map.push(map_to_be_filtered.swap_remove(idx));
-                    length -= 1;
-                }
-                false => idx += 1,
-            };
+                resultant_map.push(map_to_be_filtered.swap_remove(idx));
+                length -= 1;
+            } else {
+                idx += 1;
+            }
         }
     }
 
@@ -251,11 +237,12 @@ pub async fn filter_with_lists(
 /// Sorts  SearchResults by relevance score.
 /// <br> sort_unstable is used as its faster,stability is not an issue on our side.
 /// For reasons why, check out [`this`](https://rust-lang.github.io/rfcs/1884-unstable-sort.html)
+///
 ///  # Arguments
+///
 ///  * `results` - A mutable slice or Vec of SearchResults
-///  
 fn sort_search_results(results: &mut [SearchResult]) {
-    results.sort_unstable_by(|a, b| {
+    results.par_sort_unstable_by(|a, b| {
         use std::cmp::Ordering;
 
         b.relevance_score
@@ -311,12 +298,16 @@ mod tests {
         .await?;
 
         assert_eq!(resultant_map.len(), 2);
-        assert!(resultant_map
-            .iter()
-            .any(|(key, _)| key == "https://www.example.com"));
-        assert!(resultant_map
-            .iter()
-            .any(|(key, _)| key == "https://www.rust-lang.org/"));
+        assert!(
+            resultant_map
+                .iter()
+                .any(|(key, _)| key == "https://www.example.com")
+        );
+        assert!(
+            resultant_map
+                .iter()
+                .any(|(key, _)| key == "https://www.rust-lang.org/")
+        );
         assert_eq!(map_to_be_filtered.len(), 0);
 
         Ok(())
@@ -362,13 +353,17 @@ mod tests {
         .await?;
 
         assert_eq!(resultant_map.len(), 1);
-        assert!(resultant_map
-            .iter()
-            .any(|(key, _)| key == "https://www.example.com"));
+        assert!(
+            resultant_map
+                .iter()
+                .any(|(key, _)| key == "https://www.example.com")
+        );
         assert_eq!(map_to_be_filtered.len(), 1);
-        assert!(map_to_be_filtered
-            .iter()
-            .any(|(key, _)| key == "https://www.rust-lang.org/"));
+        assert!(
+            map_to_be_filtered
+                .iter()
+                .any(|(key, _)| key == "https://www.rust-lang.org/")
+        );
 
         Ok(())
     }

src/aggregator.rs

@@ -1,16 +1,16 @@
 use chacha20poly1305::{
-    consts::{B0, B1},
     ChaChaPoly1305,
+    consts::{B0, B1},
 };
 use std::sync::OnceLock;
 
 use chacha20::{
+    ChaChaCore,
     cipher::{
+        StreamCipherCoreWrapper,
         generic_array::GenericArray,
         typenum::{UInt, UTerm},
-        StreamCipherCoreWrapper,
     },
-    ChaChaCore,
 };
 
 /// The ChaCha20 core wrapped in a stream cipher for use in ChaCha20-Poly1305 authenticated encryption.