add support for multi-bytes utf8 strings in logs

Author: adarshsrivastava11

src/format/llm.rs

@@ -246,7 +246,15 @@ const MAX_TOKEN_LEN: usize = 50;
 /// Truncate a single string if longer than MAX_TOKEN_LEN.
 fn truncate_token(s: &str) -> String {
     if s.len() > MAX_TOKEN_LEN {
-        format!("{}...{}", &s[..20], &s[s.len() - 10..])
+        let mut head = 20;
+        while !s.is_char_boundary(head) {
+            head -= 1;
+        }
+        let mut tail = s.len() - 10;
+        while !s.is_char_boundary(tail) {
+            tail += 1;
+        }
+        format!("{}...{}", &s[..head], &s[tail..])
     } else {
         s.to_string()
     }

src/scoring.rs

@@ -101,7 +101,11 @@ const KEYWORD_PREFIX_LEN: usize = 100;
 /// deep in the template body. Returns the max weight found, or 1.0 if
 /// no keywords match.
 fn match_keyword_weight(template: &str, weights: &HashMap<&str, f64>) -> f64 {
-    let prefix = &template[..template.len().min(KEYWORD_PREFIX_LEN)];
+    let mut end = template.len().min(KEYWORD_PREFIX_LEN);
+    while !template.is_char_boundary(end) {
+        end -= 1;
+    }
+    let prefix = &template[..end];
     let tokens = tokenize_template(prefix);
     let mut max_weight = 1.0_f64;
     let mut found = false;