feat(core)!: more explicit handling of case-sensitivity in dictionaries

harper-asciidoc/tests/asciidoc_tests.rs

@@ -1,6 +1,8 @@
+use std::sync::Arc;
+
 use harper_asciidoc::AsciidocParser;
 use harper_core::linting::{LintGroup, Linter};
-use harper_core::spell::FstDictionary;
+use harper_core::spell::WordMap;
 use harper_core::{Dialect, Document};
 
 /// Creates a unit test checking Asciidoc source code parsing.
@@ -18,7 +20,7 @@ macro_rules! create_test {
                  );
 
                  let parser = AsciidocParser::default();
-                 let dict = FstDictionary::curated();
+                 let dict = Arc::new(WordMap::curated());
                  let document = Document::new(&source, &parser, &dict);
 
                  let mut linter = LintGroup::new_curated(dict, Dialect::American);

harper-brill/src/lib.rs

@@ -1,39 +1,37 @@
 use std::num::NonZero;
 use std::rc::Rc;
-use std::sync::{Arc, LazyLock};
+use std::sync::LazyLock;
 
 pub use harper_pos_utils::{
     BrillChunker, BrillTagger, BurnChunkerCpu, CachedChunker, Chunker, FreqDict, Tagger, UPOS,
 };
 
 const BRILL_TAGGER_SOURCE: &str = include_str!("../trained_tagger_model.json");
 
-static BRILL_TAGGER: LazyLock<Arc<BrillTagger<FreqDict>>> =
-    LazyLock::new(|| Arc::new(uncached_brill_tagger()));
+static BRILL_TAGGER: LazyLock<BrillTagger<FreqDict>> = LazyLock::new(uncached_brill_tagger);
 
 fn uncached_brill_tagger() -> BrillTagger<FreqDict> {
     serde_json::from_str(BRILL_TAGGER_SOURCE).unwrap()
 }
 
 /// Get a copy of a shared, lazily-initialized [`BrillTagger`]. There will be only one instance
 /// per-process.
-pub fn brill_tagger() -> Arc<BrillTagger<FreqDict>> {
-    (*BRILL_TAGGER).clone()
+pub fn brill_tagger() -> &'static BrillTagger<FreqDict> {
+    &BRILL_TAGGER
 }
 
 const BRILL_CHUNKER_SOURCE: &str = include_str!("../trained_chunker_model.json");
 
-static BRILL_CHUNKER: LazyLock<Arc<BrillChunker>> =
-    LazyLock::new(|| Arc::new(uncached_brill_chunker()));
+static BRILL_CHUNKER: LazyLock<BrillChunker> = LazyLock::new(uncached_brill_chunker);
 
 fn uncached_brill_chunker() -> BrillChunker {
     serde_json::from_str(BRILL_CHUNKER_SOURCE).unwrap()
 }
 
 /// Get a copy of a shared, lazily-initialized [`BrillChunker`]. There will be only one instance
 /// per-process.
-pub fn brill_chunker() -> Arc<BrillChunker> {
-    (*BRILL_CHUNKER).clone()
+pub fn brill_chunker() -> &'static BrillChunker {
+    &BRILL_CHUNKER
 }
 
 const BURN_CHUNKER_VOCAB: &[u8; 627993] = include_bytes!("../finished_chunker/vocab.json");

harper-cli/src/input/single_input.rs

@@ -41,7 +41,10 @@ pub(crate) trait SingleInputTrait: InputTrait {
         dictionary: &dyn Dictionary,
     ) -> anyhow::Result<(Document, Cow<'_, str>)> {
         let text = self.get_content()?;
-        Ok((Document::new(&text, &parser, &dictionary), text))
+        Ok((
+            Document::new(&text, &parser, dictionary.get_word_map()),
+            text,
+        ))
     }
 
     /// The parser that should be used to parse this input.

harper-cli/src/lint.rs

@@ -10,11 +10,12 @@ use hashbrown::HashMap;
 use rayon::prelude::*;
 
 use harper_core::{
+    CharString, Dialect, Document, Token, TokenKind,
     linting::{Lint, LintGroup, LintGroupConfig, LintKind},
     parsers::MarkdownOptions,
-    spell::{Dictionary, MergedDictionary, MutableDictionary},
+    remove_overlaps_map,
+    spell::{Dictionary, MergedDictionary, MutableDictionary, WordMapEntry},
     weirpack::Weirpack,
-    {Dialect, DictWordMetadata, Document, Token, TokenKind, remove_overlaps_map},
 };
 
 use crate::input::{
@@ -28,9 +29,9 @@ fn load_dict(path: &Path) -> anyhow::Result<MutableDictionary> {
     let str = fs::read_to_string(path)?;
 
     let mut dict = MutableDictionary::new();
-    dict.extend_words(
+    dict.extend(
         str.lines()
-            .map(|l| (l.chars().collect::<Vec<_>>(), DictWordMetadata::default())),
+            .map(|l| WordMapEntry::new(l.chars().collect::<CharString>())),
     );
 
     Ok(dict)
@@ -111,7 +112,7 @@ impl InputInfo<'_> {
 
 pub fn lint(
     markdown_options: MarkdownOptions,
-    curated_dictionary: Arc<dyn Dictionary>,
+    curated_dictionary: &'static dyn Dictionary,
     mut inputs: Vec<AnyInput>,
     mut lint_options: LintOptions,
     user_dict_path: PathBuf,
@@ -348,11 +349,11 @@ fn lint_one_input(
             Err(err) => eprintln!("{}", err),
             Ok((doc, source)) => {
                 // Create the Lint Group from which we will lint this input, using the combined dictionary and the specified dialect
-                let mut lint_group = LintGroup::new_curated(merged_dictionary.into(), *dialect);
+                let mut lint_group = LintGroup::new_curated(Arc::new(merged_dictionary), *dialect);
 
                 for pack in weirpacks {
-                    let mut pack_group = pack.to_lint_group()?;
-                    lint_group.merge_from(&mut pack_group);
+                    let pack_group = pack.to_lint_group()?;
+                    lint_group.merge_from(pack_group);
                 }
 
                 // Turn specified rules on or off

harper-cli/src/main.rs

@@ -1,12 +1,14 @@
 #![doc = include_str!("../README.md")]
 
-use harper_core::spell::{Dictionary, FstDictionary, MutableDictionary, WordId};
+use harper_core::spell::{
+    CanonicalWordId, CommonDictFuncs, Dictionary, FstDictionary, MutableDictionary, WordMap,
+};
 use hashbrown::HashMap;
 use std::collections::BTreeMap;
 use std::fs::File;
 use std::io::BufReader;
 use std::path::PathBuf;
-// use std::sync::Arc;
+use std::sync::Arc;
 use std::{fs, process};
 
 use anyhow::anyhow;
@@ -382,7 +384,9 @@ fn main() -> anyhow::Result<()> {
             ];
 
             for word in words {
-                let meta = curated_dictionary.get_word_metadata_str(&word);
+                let meta = curated_dictionary
+                    .get_word_exact_str(&word)
+                    .map(|word| &word.metadata);
                 let (flags, emojis) = meta.as_ref().map_or_else(
                     || (String::new(), String::new()),
                     |md| {
@@ -460,7 +464,7 @@ fn main() -> anyhow::Result<()> {
 
             if let Some((dict_word, dict_annot)) = &entry_in_dict {
                 println!("Old, from the dictionary:");
-                print_word_derivations(dict_word, dict_annot, &FstDictionary::curated());
+                print_word_derivations(dict_word, dict_annot, WordMap::curated());
             };
 
             if !annot.is_empty() {
@@ -471,7 +475,7 @@ fn main() -> anyhow::Result<()> {
                 )?;
 
                 println!("New, from you:");
-                print_word_derivations(&word, &annot, &dict);
+                print_word_derivations(&word, &annot, dict.get_word_map());
             }
 
             Ok(())
@@ -483,7 +487,7 @@ fn main() -> anyhow::Result<()> {
                 description: String,
             }
 
-            let linter = LintGroup::new_curated(curated_dictionary, Dialect::American);
+            let linter = LintGroup::new_curated(Arc::new(curated_dictionary), Dialect::American);
 
             let default_config: HashMap<String, bool> =
                 serde_json::from_str(&serde_json::to_string(&linter.config).unwrap()).unwrap();
@@ -878,7 +882,10 @@ fn main() -> anyhow::Result<()> {
             let mut processed_words = HashMap::new();
             let mut longest_word = 0;
             for word in curated_dictionary.words_iter() {
-                if let Some(metadata) = curated_dictionary.get_word_metadata(word) {
+                if let Some(metadata) = curated_dictionary
+                    .get_word_exact(word)
+                    .map(|word| &word.metadata)
+                {
                     let orth = metadata.orth_info;
                     let bits = orth.bits() & case_bitmask.bits();
 
@@ -994,14 +1001,19 @@ fn line_to_parts(line: &str) -> (String, String) {
     }
 }
 
-fn print_word_derivations(word: &str, annot: &str, dictionary: &impl Dictionary) {
+fn print_word_derivations(word: &str, annot: &str, dictionary: &WordMap) {
     println!("{word}/{annot}");
 
-    let id = WordId::from_word_str(word);
+    let id = CanonicalWordId::from_word_str(word);
 
-    let children = dictionary
-        .words_iter()
-        .filter(|e| dictionary.get_word_metadata(e).unwrap().derived_from == Some(id));
+    let children = dictionary.words_iter().filter(|e| {
+        dictionary
+            .get_word_exact(e)
+            .unwrap()
+            .metadata
+            .derived_from
+            .contains(id)
+    });
 
     println!(" - {word}");
 

harper-comments/tests/language_support.rs

@@ -1,9 +1,10 @@
 use std::path::Path;
+use std::sync::Arc;
 
 use harper_comments::CommentParser;
 use harper_core::linting::{LintGroup, Linter};
 use harper_core::parsers::MarkdownOptions;
-use harper_core::spell::FstDictionary;
+use harper_core::spell::WordMap;
 use harper_core::{Dialect, Document};
 
 /// Creates a unit test checking that the linting of a source file in
@@ -23,7 +24,7 @@ macro_rules! create_test {
                  );
 
                  let parser = CommentParser::new_from_filename(Path::new(filename), MarkdownOptions::default()).unwrap();
-                 let dict = FstDictionary::curated();
+                 let dict = Arc::new(WordMap::curated());
                  let document = Document::new(&source, &parser, &dict);
 
                  let mut linter = LintGroup::new_curated(dict, Dialect::American);

harper-core/Cargo.toml

@@ -34,6 +34,7 @@ harper-brill = { path = "../harper-brill", version = "1.0.0" }
 harper-thesaurus = { path = "../harper-thesaurus", version = "1.4.1", optional = true }
 bitflags = { version = "2.11.0", features = ["serde"] }
 trie-rs = "0.4.2"
+indexmap = "2.12.1"
 zip = { version = "8.0.0", default-features = false, features = ["deflate"] }
 regex = "1.12.3"
 

harper-core/README.md

@@ -16,6 +16,8 @@ If you would prefer to run Harper from inside a JavaScript runtime, [we have a p
 Here's what a full end-to-end linting pipeline could look like using `harper-core`.
 
 ```rust
+use std::sync::Arc;
+
 use harper_core::linting::{LintGroup, Linter};
 use harper_core::parsers::PlainEnglish;
 use harper_core::spell::FstDictionary;
@@ -26,7 +28,7 @@ let parser = PlainEnglish;
 
 let document = Document::new_curated(text, &parser);
 
-let dict = FstDictionary::curated();
+let dict = Arc::new(FstDictionary::curated());
 let mut linter = LintGroup::new_curated(dict, Dialect::American);
 
 let lints = linter.lint(&document);

harper-core/benches/parse_essay.rs

@@ -1,8 +1,11 @@
+use std::hint::black_box;
+use std::sync::Arc;
+
 use criterion::{Criterion, criterion_group, criterion_main};
+
 use harper_core::linting::{LintGroup, Linter};
-use harper_core::spell::FstDictionary;
+use harper_core::spell::{Dictionary, FstDictionary};
 use harper_core::{Dialect, Document};
-use std::hint::black_box;
 
 static ESSAY: &str = include_str!("./essay.md");
 
@@ -13,7 +16,7 @@ fn parse_essay(c: &mut Criterion) {
 }
 
 fn lint_essay(c: &mut Criterion) {
-    let dictionary = FstDictionary::curated();
+    let dictionary = Arc::new(FstDictionary::curated());
     let mut lint_set = LintGroup::new_curated(dictionary, Dialect::American);
     let document = Document::new_markdown_default_curated(black_box(ESSAY));
 
@@ -25,9 +28,10 @@ fn lint_essay(c: &mut Criterion) {
 fn lint_essay_uncached(c: &mut Criterion) {
     c.bench_function("lint_essay_uncached", |b| {
         b.iter(|| {
-            let dictionary = FstDictionary::curated();
+            let dictionary = Arc::new(FstDictionary::curated());
             let mut lint_set = LintGroup::new_curated(dictionary.clone(), Dialect::American);
-            let document = Document::new_markdown_default(black_box(ESSAY), &dictionary);
+            let document =
+                Document::new_markdown_default(black_box(ESSAY), dictionary.get_word_map());
             lint_set.lint(&document)
         })
     });

harper-core/dictionary.dict

@@ -8274,6 +8274,7 @@ PowerPoint/ONgV
 Powers/NOg
 Powhatan/NOg
 Poznan/Og
+Pr/                 # Praseodymium
 Prada/g
 Prado/Og
 Praetorian/Ng