UnicodeUtil updates: TryUTF8toUTF16, ReadOnlySpan methods, #1024 (#1057)

* UnicodeUtil updates: TryUTF8toUTF16, ReadOnlySpan methods, #1024

* Add back array overloads; add unit test for TryUTF8toUTF16

* Fix comment formatting

* Remove offset/length parameters from Span-based methods, #1024

* Move ToCharArray methods to ObsoleteAPI for removal in 4.8.0 RC, #1024

* Add fallback version of UTF8toUTF16

* Throw ParseException on out of range in UTF8toUTF16, add more tests

* Use Utf8ToStringWithFallback in ToString and exception/logging message building

* Use FormatException instead of ParseException

* Add FormatException case to Utf8ToString() catch

* Fix exception type in XML docs in UnicodeUtil.NewString

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix TestUnicodeUtil: Use IOUtils.ENCODING_UTF_8_NO_BOM instead of non-existent CHARSET_UTF_8

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: paulirwin

src/Lucene.Net.Codecs/SimpleText/SimpleTextDocValuesReader.cs

@@ -411,10 +411,10 @@ public override int GetOrd(int docId)
                     SimpleTextUtil.ReadLine(_input, _scratch);
                     try
                     {
-                        // LUCNENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in.
+                        // LUCENENET: .NET doesn't have a way to specify a pattern with integer, but all of the standard ones are built in.
                         return int.Parse(_scratch.Utf8ToString(), NumberStyles.Integer, CultureInfo.InvariantCulture) - 1;
                     }
-                    catch (Exception pe) when (pe.IsParseException())
+                    catch (Exception pe) when (pe.IsParseException() || pe.IsNumberFormatException())
                     {
                         var e = new CorruptIndexException($"failed to parse ord (resource={_input})", pe);
                         throw e;

src/Lucene.Net.Codecs/SimpleText/SimpleTextUtil.cs

@@ -106,8 +106,9 @@ public static void CheckFooter(ChecksumIndexInput input)
 
             if (StringHelper.StartsWith(scratch, CHECKSUM) == false)
             {
+                // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
                 throw new CorruptIndexException("SimpleText failure: expected checksum line but got " +
-                                                scratch.Utf8ToString() + " (resource=" + input + ")");
+                                                scratch.Utf8ToStringWithFallback() + " (resource=" + input + ")");
             }
             var actualChecksum =
                 (new BytesRef(scratch.Bytes, CHECKSUM.Length, scratch.Length - CHECKSUM.Length)).Utf8ToString();

src/Lucene.Net.Facet/SortedSet/DefaultSortedSetDocValuesReaderState.cs

@@ -79,7 +79,8 @@ public DefaultSortedSetDocValuesReaderState(IndexReader reader, string field = F
                 string[] components = FacetsConfig.StringToPath(spare.Utf8ToString());
                 if (components.Length != 2)
                 {
-                    throw new ArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.ToString(components) + " " + spare.Utf8ToString());
+                    // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
+                    throw new ArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.ToString(components) + " " + spare.Utf8ToStringWithFallback());
                 }
                 if (!components[0].Equals(lastDim, StringComparison.Ordinal))
                 {

src/Lucene.Net.Grouping/AbstractGroupFacetCollector.cs

@@ -278,7 +278,7 @@ public override int GetHashCode()
             public override string ToString()
             {
                 return "FacetEntry{" +
-                    "value=" + value.Utf8ToString() +
+                    "value=" + value.Utf8ToStringWithFallback() + // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
                     ", count=" + count +
                     '}';
             }

src/Lucene.Net.Join/TermsIncludingScoreQuery.cs

@@ -314,7 +314,7 @@ internal Explanation Explain(int target) // LUCENENET NOTE: changed accessibilit
                 } while (docId != DocIdSetIterator.NO_MORE_DOCS);
 
                 return new ComplexExplanation(true, outerInstance._scores[outerInstance._ords[_scoreUpto]],
-                    "Score based on join value " + _termsEnum.Term.Utf8ToString());
+                    "Score based on join value " + _termsEnum.Term.Utf8ToStringWithFallback()); // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
             }
         }
 

src/Lucene.Net.Misc/Misc/TermStats.cs

@@ -45,7 +45,8 @@ internal string GetTermText()
 
         public override string ToString()
         {
-            return ("TermStats: Term=" + TermText.Utf8ToString() + " DocFreq=" + DocFreq + " TotalTermFreq=" + TotalTermFreq);
+            // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
+            return "TermStats: Term=" + TermText.Utf8ToStringWithFallback() + " DocFreq=" + DocFreq + " TotalTermFreq=" + TotalTermFreq;
         }
     }
 }

src/Lucene.Net.Queries/TermsFilter.cs

@@ -319,7 +319,7 @@ public override string ToString()
                     }
                     first = false;
                     builder.Append(current.field).Append(':');
-                    builder.Append(spare.Utf8ToString());
+                    builder.Append(spare.Utf8ToStringWithFallback()); // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
                 }
             }
 

src/Lucene.Net.Suggest/Suggest/Fst/FSTCompletion.cs

@@ -59,7 +59,8 @@ internal Completion(BytesRef key, int bucket)
 
             public override string ToString()
             {
-                return Utf8.Utf8ToString() + "/" + Bucket.ToString("0.0", CultureInfo.InvariantCulture);
+                // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
+                return Utf8.Utf8ToStringWithFallback() + "/" + Bucket.ToString("0.0", CultureInfo.InvariantCulture);
             }
 
             /// <seealso cref="BytesRef.CompareTo(object)"></seealso>

src/Lucene.Net.Tests/Util/TestUnicodeUtil.cs

@@ -326,5 +326,54 @@ public virtual void TestUTF8UTF16CharsRef()
                 Assert.AreEqual(cRef.ToString(), unicode);
             }
         }
+
+        [Test]
+        [LuceneNetSpecific]
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3 }, true)] // ca�, start of 2-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xe3 }, true)] // ca�, start of 3-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xf3 }, true)] // ca�, start of 4-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3, 0xb1, 0x6f, 0x6e }, false)] // cañon
+        public void TestUTF8toUTF16Exception(byte[] invalidUtf8, bool shouldThrow)
+        {
+            var scratch = new CharsRef();
+
+            if (shouldThrow)
+            {
+                Assert.Throws<FormatException>(() => UnicodeUtil.UTF8toUTF16(invalidUtf8, scratch));
+            }
+            else
+            {
+                UnicodeUtil.UTF8toUTF16(invalidUtf8, scratch);
+            }
+        }
+
+        [Test]
+        [LuceneNetSpecific] // this is a Lucene.NET specific method
+        [Repeat(100)]
+        public void TestTryUTF8toUTF16()
+        {
+            string unicode = TestUtil.RandomRealisticUnicodeString(Random);
+            var utf8 = new BytesRef(IOUtils.ENCODING_UTF_8_NO_BOM.GetBytes(unicode));
+
+            bool success = UnicodeUtil.TryUTF8toUTF16(utf8, out var chars);
+
+            Assert.IsTrue(success);
+            Assert.AreEqual(unicode, chars?.ToString());
+        }
+
+        [Test]
+        [LuceneNetSpecific] // this is a Lucene.NET specific method
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3 }, "ca\ufffd")] // ca�, start of 2-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xe3 }, "ca\ufffd")] // ca�, start of 3-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xf3 }, "ca\ufffd")] // ca�, start of 4-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3, 0xb1, 0x6f, 0x6e }, "cañon")]
+        public void TestUTF8toUTF16WithFallback(byte[] utf8, string expected)
+        {
+            var scratch = new CharsRef();
+
+            UnicodeUtil.UTF8toUTF16WithFallback(utf8, scratch);
+
+            Assert.AreEqual(expected, scratch.ToString());
+        }
     }
 }

src/Lucene.Net/Codecs/BlockTreeTermsWriter.cs

@@ -440,7 +440,8 @@ public PendingTerm(BytesRef term, BlockTermState state)
 
             public override string ToString()
             {
-                return Term.Utf8ToString();
+                // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
+                return Term.Utf8ToStringWithFallback();
             }
         }
 
@@ -468,20 +469,33 @@ public PendingBlock(BytesRef prefix, long fp, bool hasTerms, bool isFloor, int f
 
             public override string ToString()
             {
-                return "BLOCK: " + Prefix.Utf8ToString();
+                // LUCENENET specific - use Utf8ToStringWithFallback() to handle invalid UTF-8 bytes
+                return $"BLOCK: {Prefix.Utf8ToStringWithFallback()}";
+            }
+
+            #nullable enable
+            public bool TryToString([NotNullWhen(true)] out string? result)
+            {
+                if (Prefix.TryUtf8ToString(out string? prefixString))
+                {
+                    result = $"BLOCK: {prefixString}";
+                    return true;
+                }
+
+                result = null;
+                return false;
             }
 
             // LUCENENET specific - to keep the Debug.Assert statement from throwing exceptions
             // because of invalid UTF8 code in Prefix, we have a wrapper class that falls back
             // to using PendingBlock.Prefix.ToString() if PendingBlock.ToString() errors.
             // This struct defers formatting the string until it is actually used as a parameter
             // in string.Format().
-            private struct PendingBlocksFormatter // For assert
+            private readonly struct PendingBlocksFormatter // For assert
             {
-#pragma warning disable IDE0044 // Add readonly modifier
-                private IList<PendingBlock> blocks;
-#pragma warning restore IDE0044 // Add readonly modifier
-                public PendingBlocksFormatter(IList<PendingBlock> blocks)
+                private readonly IList<PendingBlock>? blocks;
+
+                public PendingBlocksFormatter(IList<PendingBlock>? blocks)
                 {
                     this.blocks = blocks; // May be null
                 }
@@ -500,17 +514,17 @@ public override string ToString() // For assert
                     it.MoveNext();
                     while (true)
                     {
-                        var e = it.Current;
+                        var e = it.Current ?? throw new InvalidOperationException("Expected a non-null value in the enumerator due to Count check above.");
                         // There is a chance that the Prefix will contain invalid UTF8,
                         // so we catch that and use the alternative way of displaying it
-                        try
+                        if (e.TryToString(out string? eString))
                         {
-                            sb.Append(e.ToString());
+                            sb.Append(eString);
                         }
-                        catch (IndexOutOfRangeException)
+                        else
                         {
                             sb.Append("BLOCK: ");
-                            sb.Append(e.Prefix.ToString());
+                            sb.Append(e.Prefix);
                         }
                         if (!it.MoveNext())
                         {
@@ -520,6 +534,7 @@ public override string ToString() // For assert
                     }
                 }
             }
+            #nullable restore
 
             public void CompileIndex(IList<PendingBlock> floorBlocks, RAMOutputStream scratchBytes)
             {