Allow language models directory to be specified (#9)

This commit allows a language models directory to be specified from
which to load language models.

By default, language models will be loaded from the `Lingua/LanguageModels`
directory, relative to `AppContext.BaseDirectory`.

Author: russcam

src/Lingua/LanguageDetector.cs

@@ -8,8 +8,8 @@
 namespace Lingua;
 
 /// <summary>
-/// Detects language of given input text, and computes confidence values for every language considered possible
-/// for given input text.
+/// Detects language of a given input text and computes confidence values for every language considered possible
+/// for the given input text.
 /// </summary>
 public sealed partial class LanguageDetector
 {
@@ -85,6 +85,7 @@ public sealed partial class LanguageDetector
 	private readonly bool _isLowAccuracyModeEnabled;
 	private readonly Dictionary<Alphabet, Language> _oneLanguageAlphabets;
 	private readonly IEnumerable<Language> _languagesWithUniqueCharacters;
+	private readonly string _languageModelsDirectory;
 
 	private static readonly int[] LowAccuracyRange = [3];
 	private static readonly int[] HighAccuracyRange = [1, 2, 3, 4, 5];
@@ -95,14 +96,16 @@ internal LanguageDetector(
 		HashSet<Language> languages,
 		double minimumRelativeDistance = 0,
 		bool isEveryLanguageModelPreloaded = false,
-		bool isLowAccuracyModeEnabled = false)
+		bool isLowAccuracyModeEnabled = false,
+		string? languageModelsDirectory = null)
 	{
 		_languages = languages;
 		_minimumRelativeDistance = minimumRelativeDistance;
 		_isLowAccuracyModeEnabled = isLowAccuracyModeEnabled;
 		_oneLanguageAlphabets = AlphabetExtensions.AllSupportingExactlyOneLanguage()
 			.Where(a => languages.Contains(a.Value)).ToDictionary();
 		_languagesWithUniqueCharacters = languages.Where(l => !string.IsNullOrWhiteSpace(l.UniqueCharacters()));
+		_languageModelsDirectory = languageModelsDirectory ?? Path.Combine(AppContext.BaseDirectory, "Lingua", "LanguageModels");
 
 		if (isEveryLanguageModelPreloaded)
 			PreloadLanguageModels();
@@ -195,13 +198,13 @@ public IDictionary<Language, double> ComputeLanguageConfidenceValues(string text
 		Parallel.ForEach(ngramSizeRange, i =>
 		{
 			var testDataModel = TestLanguageModel.FromText(cleanedUpText, i);
-			var probabilities = ComputeLanguageProbabilities(testDataModel, filteredLanguages);
+			var probabilities = ComputeLanguageProbabilities(testDataModel, filteredLanguages, _languageModelsDirectory);
 			if (i == 1)
 			{
 				var unigramFilteredLanguages = _languages.Count > 0
 					? filteredLanguages.Where(f => _languages.Contains(f)).ToHashSet()
 					: filteredLanguages;
-				unigramCounts = CountUnigrams(testDataModel, unigramFilteredLanguages);
+				unigramCounts = CountUnigrams(testDataModel, unigramFilteredLanguages, _languageModelsDirectory);
 			}
 			allProbabilities[i - startValue] = probabilities;
 		});
@@ -314,14 +317,14 @@ private static Dictionary<Language, double> SumUpProbabilities(
 		return summedUpProbabilities;
 	}
 
-	private static Dictionary<Language, int> CountUnigrams(TestLanguageModel unigramLanguageModel, HashSet<Language> filteredLanguages)
+	private static Dictionary<Language, int> CountUnigrams(TestLanguageModel unigramLanguageModel, HashSet<Language> filteredLanguages, string languageModelDirectory)
 	{
 		var unigramCounts = new Dictionary<Language, int>();
 		foreach (var language in filteredLanguages)
 		{
 			foreach (var unigram in unigramLanguageModel.Ngrams)
 			{
-				var probability = LookupNgramProbability(language, unigram.AsSpan());
+				var probability = LookupNgramProbability(language, unigram.AsSpan(), languageModelDirectory);
 				if (probability > 0)
 					unigramCounts.IncrementCounter(language);
 			}
@@ -330,27 +333,27 @@ private static Dictionary<Language, int> CountUnigrams(TestLanguageModel unigram
 		return unigramCounts;
 	}
 
-	internal static Dictionary<Language, double> ComputeLanguageProbabilities(TestLanguageModel testModel, IReadOnlySet<Language> filteredLanguages)
+	internal static Dictionary<Language, double> ComputeLanguageProbabilities(TestLanguageModel testModel, IReadOnlySet<Language> filteredLanguages, string languageModelDirectory)
 	{
 		var probabilities = new Dictionary<Language, double>();
 		foreach (var language in filteredLanguages)
 		{
-			var sum = ComputeSumOfNgramProbabilities(language, testModel.Ngrams);
+			var sum = ComputeSumOfNgramProbabilities(language, testModel.Ngrams, languageModelDirectory);
 			if (sum < 0)
 				probabilities[language] = sum;
 		}
 
 		return probabilities;
 	}
 
-	internal static double ComputeSumOfNgramProbabilities(Language language, HashSet<Ngram> ngrams)
+	internal static double ComputeSumOfNgramProbabilities(Language language, HashSet<Ngram> ngrams, string languageModelDirectory)
 	{
 		var sum = 0d;
 		foreach (var ngram in ngrams)
 		{
 			foreach (var elem in ngram.LowerOrderNGrams())
 			{
-				var probability = LookupNgramProbability(language, elem);
+				var probability = LookupNgramProbability(language, elem, languageModelDirectory);
 				if (probability > 0)
 				{
 					sum += Math.Log(probability);
@@ -362,9 +365,9 @@ internal static double ComputeSumOfNgramProbabilities(Language language, HashSet
 		return sum;
 	}
 
-	internal static double LookupNgramProbability(Language language, ReadOnlySpan<char> ngram)
+	internal static double LookupNgramProbability(Language language, ReadOnlySpan<char> ngram, string languageModelDirectory)
 	{
-		var model = LoadLanguageModel(language, ngram.Length);
+		var model = LoadLanguageModel(language, ngram.Length, languageModelDirectory);
 #if NET9_0
 		var lookup = model.GetAlternateLookup<ReadOnlySpan<char>>();
 		return lookup.TryGetValue(ngram, out var result) ? result : 0;
@@ -374,7 +377,7 @@ internal static double LookupNgramProbability(Language language, ReadOnlySpan<ch
 #endif
 	}
 
-	private static FrozenDictionary<string, double> LoadLanguageModel(Language language, int ngramLength)
+	private static FrozenDictionary<string, double> LoadLanguageModel(Language language, int ngramLength, string languageModelDirectory)
 	{
 		var languageModels = ngramLength switch
 		{
@@ -387,7 +390,7 @@ private static FrozenDictionary<string, double> LoadLanguageModel(Language langu
 			_ => throw new ArgumentException($"unsupported ngram length detected: ${ngramLength}")
 		};
 
-		return LoadLanguageModels(languageModels, language, ngramLength);
+		return LoadLanguageModels(languageModels, language, ngramLength, languageModelDirectory);
 	}
 
 	private void PreloadLanguageModels()
@@ -405,33 +408,34 @@ private void PreloadLanguageModels()
 			switch (ngramLength)
 			{
 				case 1:
-					LoadLanguageModels(UnigramLanguageModels, language, ngramLength);
+					LoadLanguageModels(UnigramLanguageModels, language, ngramLength, _languageModelsDirectory);
 					break;
 				case 2:
-					LoadLanguageModels(BigramLanguageModels, language, ngramLength);
+					LoadLanguageModels(BigramLanguageModels, language, ngramLength, _languageModelsDirectory);
 					break;
 				case 3:
-					LoadLanguageModels(TrigramLanguageModels, language, ngramLength);
+					LoadLanguageModels(TrigramLanguageModels, language, ngramLength, _languageModelsDirectory);
 					break;
 				case 4:
-					LoadLanguageModels(QuadrigramLanguageModels, language, ngramLength);
+					LoadLanguageModels(QuadrigramLanguageModels, language, ngramLength, _languageModelsDirectory);
 					break;
 				case 5:
-					LoadLanguageModels(FivegramLanguageModels, language, ngramLength);
+					LoadLanguageModels(FivegramLanguageModels, language, ngramLength, _languageModelsDirectory);
 					break;
 			}
 		});
 	}
 
-	private static FrozenDictionary<string, double> LoadLanguageModels(ConcurrentDictionary<Language, Lazy<FrozenDictionary<string, double>>> languageModels, Language language, int ngramLength) =>
-		languageModels.GetOrAdd(language, static (l, nl) =>
-			new Lazy<FrozenDictionary<string, double>>(() => ReadLanguageModel(l, nl)), ngramLength).Value;
+	private static FrozenDictionary<string, double> LoadLanguageModels(ConcurrentDictionary<Language, Lazy<FrozenDictionary<string, double>>> languageModels, Language language, int ngramLength, string languageModelDirectory) =>
+		languageModels.GetOrAdd(language, static (l, arg) =>
+			new Lazy<FrozenDictionary<string, double>>(() =>
+				ReadLanguageModel(l, arg.ngramLength, arg.languageModelDirectory)), (ngramLength, languageModelDirectory)).Value;
 
-	private static FrozenDictionary<string, double> ReadLanguageModel(Language language, int ngramLength)
+	private static FrozenDictionary<string, double> ReadLanguageModel(Language language, int ngramLength, string languageModelDirectory)
 	{
 		var isoCode = language.IsoCode6391().ToString().ToLowerInvariant();
 		var nGramName = Ngram.GetNameByLength(ngramLength);
-		var file = Path.Combine("Lingua", "LanguageModels", isoCode, $"{nGramName}s.json.br");
+		var file = Path.Combine(languageModelDirectory, isoCode, $"{nGramName}s.json.br");
 
 		try
 		{

src/Lingua/LanguageDetectorBuilder.cs

@@ -9,6 +9,7 @@ public class LanguageDetectorBuilder
 	private double _minimumRelativeDistance;
 	private bool _isEveryLanguageModelPreloaded;
 	private bool _isLowAccuracyModeEnabled;
+	private string? _languageModelsDirectory;
 
 	private LanguageDetectorBuilder(HashSet<Language> languages) => _languages = languages;
 
@@ -96,7 +97,7 @@ public static LanguageDetectorBuilder FromLanguages(params Language[] languages)
 	/// <summary>
 	/// Sets the desired value for the minimum relative distance measure.
 	/// <para />
-	/// By default, *Lingua* returns the most likely language for a given
+	/// By default, Lingua returns the most likely language for a given
 	/// input text. However, there are certain words that are spelled the
 	/// same in more than one language. The word *prologue*, for instance,
 	/// is both a valid English and French word. Lingua would output either
@@ -127,7 +128,7 @@ public LanguageDetectorBuilder WithMinimumRelativeDistance(double distance)
 	/// <summary>
 	/// Preloads all language models when creating the instance of <see cref="Lingua.LanguageDetector"/>.
 	/// <para />
-	/// By default, *Lingua* uses lazy-loading to load only those language models
+	/// By default, Lingua uses lazy-loading to load only those language models
 	/// on demand which are considered relevant by the rule-based filter engine.
 	/// For web services, for instance, it is rather beneficial to preload all language
 	/// models into memory to avoid unexpected latency while waiting for the
@@ -142,7 +143,7 @@ public LanguageDetectorBuilder WithPreloadedLanguageModels()
 	/// <summary>
 	/// Disables the high accuracy mode in order to save memory and increase performance.
 	/// <para />
-	/// By default, *Lingua's* high detection accuracy comes at the cost of
+	/// By default, Lingua's high detection accuracy comes at the cost of
 	/// loading large language models into memory which might not be feasible
 	/// for systems running low on resources.
 	/// <para />
@@ -158,6 +159,39 @@ public LanguageDetectorBuilder WithLowAccuracyMode()
 		return this;
 	}
 
+	/// <summary>
+	/// The directory from which to load language models.
+	/// <para />
+	/// By default, Lingua loads language models from the directory "Lingua/LanguageModels" relative
+	/// to <see cref="AppContext.BaseDirectory"/>.
+	/// <para />
+	/// Lingua expects language models to be stored in the directory following directory structure:
+	/// <code>
+	/// &lt;languageModelsDirectory>/&lt;ISO639-1 code>/&lt;Ngram length name>.br.json
+	/// </code>
+	/// where
+	/// <list type="bullet">
+	/// <item>
+	/// &lt;languageModelsDirectory> is the directory specified by this method
+	/// </item>
+	/// <item>
+	/// &lt;ISO639-1 code> is a directory with the ISO 639-1 code of the language models stored in it
+	/// </item>
+	/// <item>
+	/// &lt;Ngram length name> is the name of the ngram length: unigrams, bigrams, trigrams, quadrigrams, fivegrams
+	/// </item>
+	/// </list>
+	/// </summary>
+	/// <exception cref="DirectoryNotFoundException">If the specified directory does not exist</exception>
+	public LanguageDetectorBuilder WithLanguageModelsDirectory(string languageModelsDirectory)
+	{
+		if (!Directory.Exists(languageModelsDirectory))
+			throw new DirectoryNotFoundException($"Directory '{languageModelsDirectory}' does not exist");
+
+		_languageModelsDirectory = languageModelsDirectory;
+		return this;
+	}
+
 	/// <summary>
 	/// Builds a new instance of <see cref="Lingua.LanguageDetector"/>.
 	/// </summary>
@@ -166,5 +200,6 @@ public LanguageDetector Build() =>
 		new(_languages,
 			_minimumRelativeDistance,
 			_isEveryLanguageModelPreloaded,
-			_isLowAccuracyModeEnabled);
+			_isLowAccuracyModeEnabled,
+			_languageModelsDirectory);
 }