More BytesRef/UnicodeUtil UTF-8-to-string fixes, #1024 (#1171)

* UnicodeUtil: Change UTF8toUTF16 to throw DecoderFallbackException, #1024

Changed UTF8toUTF16 method to throw DecoderFallbackException instead of FormatException
when invalid UTF-8 is encountered. This aligns with .NET conventions where
DecoderFallbackException is the appropriate exception type for character decoding
issues (equivalent to Java's CharacterCodingException).

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

* BytesRef: Use Utf8ToStringWithFallback in DebuggerDisplay, #1024

Changed DebuggerDisplay attribute to use Utf8ToStringWithFallback() instead of
Utf8ToString() to prevent exceptions when debugging BytesRef instances that
contain invalid UTF-8 sequences.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

* BytesRef: Show "Invalid UTF-8" label and string-before-bytes in DebuggerDisplay, #1024

Address NightOwl888's review on #1171. The prior DebuggerDisplay used
Utf8ToStringWithFallback, which silently substitutes U+FFFD for malformed
input — indistinguishable from a real U+FFFD in the data. Switch to a
private DebuggerDisplay property that uses TryUtf8ToString, shows the
literal "Invalid UTF-8" on decode failure, and puts the decoded string
before the raw bytes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* UnicodeUtil: Fix UTF8toUTF16WithFallback byte[] overload to use fallback path, #1024

The byte[]/offset/length overload was delegating to the throwing
UTF8toUTF16 variant, so callers like BytesRef.Utf8ToStringWithFallback()
would throw DecoderFallbackException on invalid UTF-8 instead of
substituting U+FFFD as the method name and docs promise. Delegate to
the ReadOnlySpan<byte> fallback overload and add a regression test
covering the 4-arg signature with a non-zero offset.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: paulirwin

src/Lucene.Net.Tests/Util/TestBytesRef.cs

@@ -3,6 +3,8 @@
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using Assert = Lucene.Net.TestFramework.Assert;
@@ -612,5 +614,64 @@ public static void Test_CopyChars_UnpairedSurrogates()
         }
 
         #endregion
+
+        #region DebuggerDisplay
+
+        private static string GetDebuggerDisplay(BytesRef br)
+        {
+            var prop = typeof(BytesRef).GetProperty("DebuggerDisplay", BindingFlags.Instance | BindingFlags.NonPublic);
+            Assert.IsNotNull(prop, "BytesRef.DebuggerDisplay private property not found");
+            return (string)prop.GetValue(br);
+        }
+
+        [Test]
+        [LuceneNetSpecific]
+        public static void Test_DebuggerDisplayAttribute_ReferencesExistingMember()
+        {
+            // Guards against typos/renames in the [DebuggerDisplay] expression.
+            var attr = typeof(BytesRef).GetCustomAttribute<DebuggerDisplayAttribute>();
+            Assert.IsNotNull(attr);
+            Assert.AreEqual("{DebuggerDisplay,nq}", attr.Value);
+
+            var prop = typeof(BytesRef).GetProperty("DebuggerDisplay", BindingFlags.Instance | BindingFlags.NonPublic);
+            Assert.IsNotNull(prop);
+        }
+
+        [Test]
+        [LuceneNetSpecific]
+        public static void Test_DebuggerDisplay_ValidUtf8_ShowsStringThenBytes()
+        {
+            var br = new BytesRef("abc");
+            Assert.AreEqual("abc [61 62 63]", GetDebuggerDisplay(br));
+        }
+
+        [Test]
+        [LuceneNetSpecific]
+        public static void Test_DebuggerDisplay_InvalidUtf8_ShowsInvalidLabel()
+        {
+            // 0xC3 starts a 2-byte UTF-8 sequence but the continuation byte is missing.
+            var br = new BytesRef(new byte[] { 0xC3 });
+            Assert.AreEqual("Invalid UTF-8 [c3]", GetDebuggerDisplay(br));
+        }
+
+        [Test]
+        [LuceneNetSpecific]
+        public static void Test_DebuggerDisplay_EmbeddedFFFD_IsNotMistakenForInvalid()
+        {
+            // U+FFFD encoded as legitimate UTF-8 (EF BF BD) must round-trip, not be reported as invalid.
+            var br = new BytesRef("\uFFFD");
+            Assert.AreEqual("\uFFFD [ef bf bd]", GetDebuggerDisplay(br));
+        }
+
+        [Test]
+        [LuceneNetSpecific]
+        public static void Test_DebuggerDisplay_RespectsOffsetAndLength()
+        {
+            var bytes = new[] { (byte)'a', (byte)'b', (byte)'c', (byte)'d' };
+            var br = new BytesRef(bytes, 1, 2); // "bc"
+            Assert.AreEqual("bc [62 63]", GetDebuggerDisplay(br));
+        }
+
+        #endregion
     }
 }

src/Lucene.Net.Tests/Util/TestUnicodeUtil.cs

@@ -3,6 +3,7 @@
 using Lucene.Net.Attributes;
 using NUnit.Framework;
 using System;
+using System.Text;
 using Assert = Lucene.Net.TestFramework.Assert;
 
 namespace Lucene.Net.Util
@@ -339,7 +340,7 @@ public void TestUTF8toUTF16Exception(byte[] invalidUtf8, bool shouldThrow)
 
             if (shouldThrow)
             {
-                Assert.Throws<FormatException>(() => UnicodeUtil.UTF8toUTF16(invalidUtf8, scratch));
+                Assert.Throws<DecoderFallbackException>(() => UnicodeUtil.UTF8toUTF16(invalidUtf8, scratch));
             }
             else
             {
@@ -376,6 +377,25 @@ public void TestUTF8toUTF16WithFallback(byte[] utf8, string expected)
             Assert.AreEqual(expected, scratch.ToString());
         }
 
+        [Test]
+        [LuceneNetSpecific]
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3 }, "ca\ufffd")] // ca�, start of 2-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xe3 }, "ca\ufffd")] // ca�, start of 3-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xf3 }, "ca\ufffd")] // ca�, start of 4-byte sequence
+        [TestCase(new byte[] { 0x63, 0x61, 0xc3, 0xb1, 0x6f, 0x6e }, "cañon")]
+        public void TestUTF8toUTF16WithFallback_ByteArrayOverload(byte[] utf8, string expected)
+        {
+            // Regression: the 4-arg byte[] overload must delegate to the fallback implementation,
+            // not the throwing one. Exercise it via a non-zero offset to confirm the arguments flow through.
+            var scratch = new CharsRef();
+            var padded = new byte[utf8.Length + 2];
+            Array.Copy(utf8, 0, padded, 2, utf8.Length);
+
+            UnicodeUtil.UTF8toUTF16WithFallback(padded, 2, utf8.Length, scratch);
+
+            Assert.AreEqual(expected, scratch.ToString());
+        }
+
         [Test]
         [LuceneNetSpecific]
         [Repeat(100)]

src/Lucene.Net/Util/BytesRef.cs

@@ -45,7 +45,7 @@ namespace Lucene.Net.Util
     [Serializable]
 #endif
     // LUCENENET specific: Not implementing ICloneable per Microsoft's recommendation
-    [DebuggerDisplay("{ToString()} {Utf8ToString()}")]
+    [DebuggerDisplay("{DebuggerDisplay,nq}")]
     public sealed class BytesRef : IComparable<BytesRef>, IComparable, IEquatable<BytesRef> // LUCENENET specific - implemented IComparable for FieldComparator, IEquatable<BytesRef>
     {
         /// <summary>
@@ -317,6 +317,11 @@ public bool TryUtf8ToString([NotNullWhen(true)] out string? result)
         }
         #nullable restore
 
+        // LUCENENET specific: "Invalid UTF-8" disambiguates a decode failure from a legitimate U+FFFD in the data.
+        [SuppressMessage("CodeQuality", "IDE0051:Remove unused private members", Justification = "Referenced by DebuggerDisplay attribute")]
+        private string DebuggerDisplay
+            => $"{(TryUtf8ToString(out var s) ? s : "Invalid UTF-8")} {ToString()}";
+
         /// <summary>
         /// Returns hex encoded bytes, eg [0x6c 0x75 0x63 0x65 0x6e 0x65] </summary>
         public override string ToString()

src/Lucene.Net/Util/UnicodeUtil.cs

@@ -975,7 +975,7 @@ public static string ToHexString(string s)
         /// it doesn't provide enough space to hold the worst case of each byte becoming a UTF-16 codepoint.
         /// <para/>
         /// NOTE: Full characters are read, even if this reads past the length passed (and
-        /// can result in an <see cref="FormatException"/> if invalid UTF-8 is passed).
+        /// can result in a <see cref="DecoderFallbackException"/> if invalid UTF-8 is passed).
         /// Explicit checks for valid UTF-8 are not performed.
         /// </summary>
         /// <seealso cref="UTF8toUTF16(ReadOnlySpan{byte}, CharsRef)"/>
@@ -990,7 +990,7 @@ public static void UTF8toUTF16(byte[] utf8, int offset, int length, CharsRef cha
         /// it doesn't provide enough space to hold the worst case of each byte becoming a UTF-16 codepoint.
         /// <para/>
         /// NOTE: Full characters are read, even if this reads past the length passed (and
-        /// can result in an <see cref="FormatException"/> if invalid UTF-8 is passed).
+        /// can result in a <see cref="DecoderFallbackException"/> if invalid UTF-8 is passed).
         /// Explicit checks for valid UTF-8 are not performed.
         /// </summary>
         /// <remarks>
@@ -1015,15 +1015,15 @@ public static void UTF8toUTF16(ReadOnlySpan<byte> utf8, CharsRef chars)
                 {
                     if (utf8.Length <= i)
                     {
-                        throw new FormatException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
+                        throw new DecoderFallbackException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
                     }
                     @out[out_offset++] = (char)(((b & 0x1f) << 6) + (utf8[i++] & 0x3f));
                 }
                 else if (b < 0xf0)
                 {
                     if (utf8.Length <= i + 1)
                     {
-                        throw new FormatException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
+                        throw new DecoderFallbackException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
                     }
                     @out[out_offset++] = (char)(((b & 0xf) << 12) + ((utf8[i] & 0x3f) << 6) + (utf8[i + 1] & 0x3f));
                     i += 2;
@@ -1032,7 +1032,7 @@ public static void UTF8toUTF16(ReadOnlySpan<byte> utf8, CharsRef chars)
                 {
                     if (utf8.Length <= i + 2)
                     {
-                        throw new FormatException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
+                        throw new DecoderFallbackException($"Invalid UTF-8 starting at [{b:x2}] at offset {i - 1}");
                     }
                     if (Debugging.AssertsEnabled) Debugging.Assert(b < 0xf8, "b = 0x{0:x}", b);
                     int ch = ((b & 0x7) << 18) + ((utf8[i] & 0x3f) << 12) + ((utf8[i + 1] & 0x3f) << 6) + (utf8[i + 2] & 0x3f);
@@ -1065,7 +1065,7 @@ public static void UTF8toUTF16(ReadOnlySpan<byte> utf8, CharsRef chars)
         // TODO: broken if chars.offset != 0
         public static void UTF8toUTF16WithFallback(byte[] utf8, int offset, int length, CharsRef chars)
         {
-            UTF8toUTF16(utf8.AsSpan(offset, length), chars);
+            UTF8toUTF16WithFallback(utf8.AsSpan(offset, length), chars);
         }
 
         /// <summary>