Revert breaking changes and restore compatibility; update tests for Bigram and WordDictionary

Author: NehanPathan

src/Lucene.Net.Analysis.SmartCn/Hhmm/AbstractDictionary.cs

@@ -33,7 +33,9 @@ namespace Lucene.Net.Analysis.Cn.Smart.Hhmm
     internal abstract class AbstractDictionary
     {
         // LUCENENET specific: cached GB2312 encoding to avoid repeated calls to Encoding.GetEncoding("GB2312")
-        protected static readonly Encoding gb2312Encoding = Encoding.GetEncoding("GB2312");
+        protected static readonly Encoding gb2312Encoding = Encoding.GetEncoding("GB2312",
+    EncoderFallback.ExceptionFallback, DecoderFallback.ExceptionFallback);
+
 
         /// <summary>
         /// First Chinese Character in GB2312 (15 * 94)

src/Lucene.Net.Analysis.SmartCn/Hhmm/BigramDictionary.cs

@@ -258,11 +258,19 @@ private void Load(string dictRoot)
         /// <exception cref="IOException">If there is a low-level I/O error</exception>
         public virtual void LoadFromFile(string dctFilePath)
         {
+            int i, cnt, length, total = 0;
+
             // The file only counted 6763 Chinese characters plus 5 reserved slots 3756~3760.
             // The 3756th is used (as a header) to store information.
 
+            Span<int> buffer = stackalloc int[3];
+            string tmpword;
+
             // LUCENENET: Removed buffer and intBuffer arrays since BinaryReader handles reading values directly in a more type-safe and readable way.
             // LUCENENET specific - refactored constants for clarity
+
+            // The 3756th position (using 1-based counting) corresponds to index 3755 (using 0-based indexing)
+            // This matches the original Java implementation which used 3755 + GB2312_FIRST_CHAR in the condition
             const int HEADER_POSITION = 3755;
             const int MAX_VALID_LENGTH = 1000;
 
@@ -271,39 +279,43 @@ public virtual void LoadFromFile(string dctFilePath)
             using var reader = new BinaryReader(dctFile);
 
             // GB2312 characters 0 - 6768
-            for (int i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++)
+            for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++)
             {
 
-                string currentStr = GetCCByGB2312Id(i); 
-                int cnt;
+                string currentStr = GetCCByGB2312Id(i);
+                // if (i == 5231)
+                // System.out.println(i);
                 try
                 {
-                   cnt = reader.ReadInt32();  // LUCENENET: Use BinaryReader methods instead of ByteBuffer
+                    cnt = reader.ReadInt32();  // LUCENENET: Use BinaryReader to decode little endian instead of ByteBuffer, since this is the default in .NET
                 }
                 catch (EndOfStreamException)
                 {
-                    // Reached end of file
+                    // Test dictionary files contain fewer entries than production files
+                    // Breaking here is normal and expected behavior for test files
                     break;
                 }
 
                 if (cnt <= 0)
                 {
                     continue;
                 }
-
-                for (int j = 0; j < cnt; j++)
+                total += cnt;
+                int j = 0;
+                while (j < cnt)
                 {
-                    // LUCENENET: Use BinaryReader methods instead of ByteBuffer
-                    int frequency = reader.ReadInt32();
-                    int length = reader.ReadInt32();
-                    reader.ReadInt32();  // Skip handle value (unused)
+                    // LUCENENET: Use BinaryReader to decode little endian instead of ByteBuffer, since this is the default in .NET
+                    buffer[0] = reader.ReadInt32(); // frequency
+                    buffer[1] = reader.ReadInt32(); // length
+                    buffer[2] = reader.ReadInt32(); // Skip handle value (unused)
 
+                    length = buffer[1];
                     if (length > 0 && length <= MAX_VALID_LENGTH && dctFile.Position + length <= dctFile.Length)
                     {
-                        byte[] lchBuffer = reader.ReadBytes(length);  // LUCENENET: Use BinaryReader methods instead of ByteBuffer
+                        byte[] lchBuffer = reader.ReadBytes(length);  // LUCENENET: Use BinaryReader to decode little endian instead of ByteBuffer, since this is the default in .NET
 
                         //tmpword = new String(lchBuffer, "GB2312");
-                        string tmpword = gb2312Encoding.GetString(lchBuffer); // LUCENENET specific: use cached encoding instance from base class
+                        tmpword = gb2312Encoding.GetString(lchBuffer); // LUCENENET specific: use cached encoding instance from base class
                         //tmpword = Encoding.GetEncoding("hz-gb-2312").GetString(lchBuffer);
 
 
@@ -324,11 +336,13 @@ public virtual void LoadFromFile(string dctFilePath)
                                 // bigramStringTable[index] = tmpword;
 
                             }
-                            frequencyTable[index] += frequency;
+                            frequencyTable[index] += buffer[0];
                         }
                     }
+                    j++;
                 }
             }
+            // log.info("load dictionary done! " + dctFilePath + " total:" + total);
         }
         private int GetAvaliableIndex(long hashId, ReadOnlySpan<char> carray)
         {

src/Lucene.Net.Analysis.SmartCn/Hhmm/WordDictionary.cs

@@ -345,21 +345,26 @@ private void SaveToObj(FileInfo serialObj)
         /// <exception cref="IOException">If there is a low-level I/O error.</exception>
         private int LoadMainDataFromFile(string dctFilePath)
         {
-            int total = 0;
+            int i, cnt, length, total = 0;
 
             // The file only counted 6763 Chinese characters plus 5 reserved slots (3756~3760).
             // The 3756th is used (as a header) to store information.
 
+            Span<int> buffer = stackalloc int[3];
+
             // LUCENENET: Removed buffer and intBuffer arrays since BinaryReader handles reading values directly in a more type-safe and readable way.
             // LUCENENET: Use BinaryReader to simplify endian conversion and stream reading.
 
             using (var dctFile = new FileStream(dctFilePath, FileMode.Open, FileAccess.Read))
             using (var reader = new BinaryReader(dctFile))
             {
                 // GB2312 characters 0 - 6768
-                for (int i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++)
+                for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++)
                 {
-                    int cnt = reader.ReadInt32(); // LUCENENET: Use BinaryReader methods instead of ByteBuffer
+                    // if (i == 5231)
+                    // System.out.println(i);
+
+                    cnt = reader.ReadInt32(); // LUCENENET: Use BinaryReader to decode little endian instead of ByteBuffer, since this is the default in .NET
 
                     if (cnt <= 0)
                     {
@@ -371,16 +376,21 @@ private int LoadMainDataFromFile(string dctFilePath)
                     wordItem_charArrayTable[i] = new char[cnt][];
                     wordItem_frequencyTable[i] = new int[cnt];
                     total += cnt;
-
-                    for (int j = 0; j < cnt; j++)
+                    int j = 0;
+                    while (j < cnt)
                     {
-                        // LUCENENET: Use BinaryReader methods instead of ByteBuffer
-                        int frequency = reader.ReadInt32();
-                        int length = reader.ReadInt32();
-                        reader.ReadInt32(); // Skip handle (unused)
+                        // wordItemTable[i][j] = new WordItem();
+
+                        // LUCENENET: Use BinaryReader to decode little endian instead of ByteBuffer, since this is the default in .NET
+                        buffer[0] = reader.ReadInt32(); // frequency
+                        buffer[1] = reader.ReadInt32(); // length
+                        buffer[2] = reader.ReadInt32(); // handle
+
+                        // wordItemTable[i][j].frequency = buffer[0];
 
-                        wordItem_frequencyTable[i][j] = frequency;
+                        wordItem_frequencyTable[i][j] = buffer[0];
 
+                        length = buffer[1];
                         if (length > 0)
                         {
                             byte[] lchBuffer = reader.ReadBytes(length);
@@ -389,8 +399,11 @@ private int LoadMainDataFromFile(string dctFilePath)
                         }
                         else
                         {
+                            // wordItemTable[i][j].charArray = null;
                             wordItem_charArrayTable[i][j] = null;
                         }
+                        // System.out.println(indexTable[i].wordItems[j]);
+                        j++;
                     }
 
                     string str = GetCCByGB2312Id(i);

src/Lucene.Net.Analysis.SmartCn/Lucene.Net.Analysis.SmartCn.csproj

@@ -38,15 +38,13 @@
     <NoWarn>$(NoWarn);1591;1573</NoWarn>
   </PropertyGroup>
 
-
-
   <ItemGroup>
     <EmbeddedResource Include="Hhmm/*.mem" Label="Dict Test Data" />
     <EmbeddedResource Include="**/*.txt" Exclude="bin/**/*;obj/**/*" Label="Text Test Data" />
   </ItemGroup>
 
   <ItemGroup>
-	<ProjectReference Include="..\dotnet\Lucene.Net.ICU\Lucene.Net.ICU.csproj" />
+    <ProjectReference Include="..\dotnet\Lucene.Net.ICU\Lucene.Net.ICU.csproj" />
     <ProjectReference Include="..\Lucene.Net\Lucene.Net.csproj" />
     <ProjectReference Include="..\Lucene.Net.Analysis.Common\Lucene.Net.Analysis.Common.csproj" />
   </ItemGroup>
@@ -65,8 +63,7 @@
   </ItemGroup>
 
   <ItemGroup>
-  <InternalsVisibleTo Include="Lucene.Net.Tests.Analysis.SmartCn" />
-</ItemGroup>
-
+    <InternalsVisibleTo Include="Lucene.Net.Tests.Analysis.SmartCn" />
+  </ItemGroup>
 
 </Project>

src/Lucene.Net.Tests.Analysis.SmartCn/DictionaryTests.cs

@@ -0,0 +1,110 @@
+using J2N;
+using Lucene.Net.Analysis.Cn.Smart;
+using Lucene.Net.Analysis.Cn.Smart.Hhmm;
+using Lucene.Net.Attributes;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.IO;
+using Assert = Lucene.Net.TestFramework.Assert;
+
+namespace Lucene.Net.Analysis.Cn.Smart.Hhmm
+{
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    [LuceneNetSpecific]
+    public class TestBuildDictionary : LuceneTestCase
+    {
+        private DirectoryInfo tempDir;
+
+        public override void OneTimeSetUp()
+        {
+            base.OneTimeSetUp();
+            tempDir = CreateTempDir("smartcn-data");
+            AnalyzerProfile.ANALYSIS_DATA_DIR = tempDir.FullName;
+            using (var zipFileStream = typeof(TestBuildDictionary).FindAndGetManifestResourceStream("custom-dictionary-input.zip"))
+            {
+                TestUtil.Unzip(zipFileStream, tempDir);
+            }
+        }
+
+        public override void OneTimeTearDown()
+        {
+            AnalyzerProfile.ANALYSIS_DATA_DIR = null; // Ensure this test data is not loaded for other tests
+            base.OneTimeTearDown();
+        }
+
+        [Test]
+        public void TestBigramDictionary()
+        {
+            // First test - builds and loads dictionary from .dict file
+            BigramDictionary bigramDict = BigramDictionary.GetInstance();
+            CheckBigramDictionary(bigramDict);
+            
+            // Ensure .mem file was created
+            string memFile = System.IO.Path.Combine(tempDir.FullName,  "bigramdict.mem");
+            Assert.IsTrue(File.Exists(memFile), "Memory file should be created after first load");
+            
+            // Delete the original .dict file
+            string dictFile = System.IO.Path.Combine(tempDir.FullName,  "bigramdict.dct");
+            if (File.Exists(dictFile))
+            {
+                File.Delete(dictFile);
+            }
+            
+            // Second test - should load from .mem file now
+            bigramDict = BigramDictionary.GetInstance();
+            CheckBigramDictionary(bigramDict);
+        }
+        
+        private static void CheckBigramDictionary(BigramDictionary bigramDict)
+        {
+            Assert.AreEqual(10, bigramDict.GetFrequency("啊hello".AsSpan()), "Frequency for '啊hello' is incorrect.");
+            Assert.AreEqual(20, bigramDict.GetFrequency("阿world".AsSpan()), "Frequency for '阿world' is incorrect.");
+        }
+
+        [Test]
+        public void TestWordDictionary()
+        {
+            // First test - builds and loads dictionary from .dict file
+            WordDictionary wordDict = WordDictionary.GetInstance();
+            CheckWordDictionary(wordDict);
+            
+            // Ensure .mem file was created
+            string memFile = System.IO.Path.Combine(tempDir.FullName, "coredict.mem");
+            Assert.IsTrue(File.Exists(memFile), "Memory file should be created after first load");
+            
+            // Delete the original .dict file
+            string dictFile = System.IO.Path.Combine(tempDir.FullName, "coredict.dct");
+            if (File.Exists(dictFile))
+            {
+                File.Delete(dictFile);
+            }
+            
+            // Second test - should load from .mem file now
+            wordDict = WordDictionary.GetInstance();
+            CheckWordDictionary(wordDict);
+        }
+        
+        private static void CheckWordDictionary(WordDictionary wordDict)
+        {
+            Assert.AreEqual(30, wordDict.GetFrequency("尼".ToCharArray()), "Frequency for '尼' is incorrect.");
+            Assert.AreEqual(0, wordDict.GetFrequency("missing".ToCharArray()), "Expected frequency 0 for unknown word.");
+        }
+    }
+}