Syntax highlight using ReadOnlyMemory<char> for reduced allocations

Directory.Build.props

@@ -4,7 +4,7 @@
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <AvaloniaVersion>11.0.0</AvaloniaVersion>
     <AvaloniaSampleVersion>11.3.0</AvaloniaSampleVersion>
-    <TextMateSharpVersion>1.0.66</TextMateSharpVersion>
+    <TextMateSharpVersion>2.0.2</TextMateSharpVersion>
     <VersionSuffix>beta</VersionSuffix>
 
     <PublishRepositoryUrl>true</PublishRepositoryUrl>

azure-pipelines.yml

@@ -6,6 +6,12 @@ jobs:
     vmImage: 'windows-2022'
   steps:
 
+  - task: UseDotNet@2
+    displayName: 'Install .NET 8 SDK'
+    inputs:
+      packageType: 'sdk'
+      version: '8.x'
+
   - task: DotNetCoreCLI@2
     inputs:
       command: 'test'

src/AvaloniaEdit.TextMate/AvaloniaEdit.TextMate.csproj

@@ -12,8 +12,8 @@
 
   <ItemGroup>
     <PackageReference Include="Avalonia" Version="$(AvaloniaVersion)" />
-    <PackageReference Include="TextMateSharp" Version="1.0.70" />
-    <PackageReference Include="TextMateSharp.Grammars" Version="1.0.70" />
+    <PackageReference Include="TextMateSharp" Version="$(TextMateSharpVersion)" />
+    <PackageReference Include="TextMateSharp.Grammars" Version="$(TextMateSharpVersion)" />
   </ItemGroup>
 
 </Project>

src/AvaloniaEdit.TextMate/DocumentSnapshot.cs

@@ -56,6 +56,15 @@ public string GetLineTextIncludingTerminator(int lineIndex)
             }
         }
 
+        public ReadOnlyMemory<char> GetLineTextIncludingTerminatorAsMemory(int lineIndex)
+        {
+            lock (_lock)
+            {
+                var lineRange = _lineRanges[lineIndex];
+                return _textSource.GetTextAsMemory(lineRange.Offset, lineRange.TotalLength);
+            }
+        }
+
         public string GetLineTerminator(int lineIndex)
         {
             lock (_lock)

src/AvaloniaEdit.TextMate/TextEditorModel.cs

@@ -4,7 +4,7 @@
 
 using AvaloniaEdit.Document;
 using AvaloniaEdit.Rendering;
-
+using TextMateSharp.Grammars;
 using TextMateSharp.Model;
 
 #if NET6_0_OR_GREATER
@@ -69,9 +69,9 @@ public override int GetNumberOfLines()
             return _documentSnapshot.LineCount;
         }
 
-        public override string GetLineText(int lineIndex)
+        public override LineText GetLineTextIncludingTerminators(int lineIndex)
         {
-            return _documentSnapshot.GetLineText(lineIndex);
+            return _documentSnapshot.GetLineTextIncludingTerminatorAsMemory(lineIndex);
         }
 
         public override int GetLineLength(int lineIndex)

src/AvaloniaEdit/Document/ITextSource.cs

@@ -88,7 +88,15 @@ public interface ITextSource
 		/// <remarks>This is the same as Text.Substring, but is more efficient because
 		///  it doesn't require creating a String object for the whole document.</remarks>
 		string GetText(int offset, int length);
-
+
+		/// <summary>
+		/// Retrieves the text for a portion of the document as ReadOnlyMemory&lt;char&gt;.
+		/// </summary>
+		/// <exception cref="ArgumentOutOfRangeException">offset or length is outside the valid range.</exception>
+		/// <remarks>This method may be more efficient than GetText(int, int) because it can avoid allocating a new string
+		///  in some implementations.</remarks>
+		ReadOnlyMemory<char> GetTextAsMemory(int offset, int length);
+
 		/// <summary>
 		/// Retrieves the text for a portion of the document.
 		/// </summary>
@@ -300,7 +308,13 @@ public string GetText(int offset, int length)
 		{
 			return Text.Substring(offset, length);
 		}
-
+
+		/// <inheritdoc/>
+		public ReadOnlyMemory<char> GetTextAsMemory(int offset, int length)
+		{
+			return Text.AsMemory(offset, length);
+		}
+
 		/// <inheritdoc/>
 		public string GetText(ISegment segment)
 		{

src/AvaloniaEdit/Document/RopeTextSource.cs

@@ -76,7 +76,13 @@ public string GetText(int offset, int length)
 		{
 			return _rope.ToString(offset, length);
 		}
-
+
+		/// <inheritdoc/>
+		public ReadOnlyMemory<char> GetTextAsMemory(int offset, int length)
+		{
+			return _rope.GetMemory(offset, length);
+		}
+
 		/// <inheritdoc/>
 		public string GetText(ISegment segment)
 		{

src/AvaloniaEdit/Document/TextDocument.cs

@@ -129,6 +129,13 @@ public string GetText(int offset, int length)
             return _rope.ToString(offset, length);
         }
 
+        /// <inheritdoc/>
+        public ReadOnlyMemory<char> GetTextAsMemory(int offset, int length)
+        {
+            VerifyAccess();
+            return _rope.GetMemory(offset, length);
+        }
+
         private Thread ownerThread = Thread.CurrentThread;
 
 		/// <summary>

src/AvaloniaEdit/Utils/CharRope.cs

@@ -64,6 +64,46 @@ public static string ToString(this Rope<char> rope, int startIndex, int length)
 			return new string(buffer);
 #endif
 		}
+
+		/// <summary>
+		/// Retrieves the text for a portion of the rope as ReadOnlyMemory.
+		/// Runs in O(lg N + M), where M=<paramref name="length"/>.
+		/// </summary>
+		/// <exception cref="ArgumentOutOfRangeException">offset or length is outside the valid range.</exception>
+		/// <remarks>
+		/// This method counts as a read access and may be called concurrently to other read accesses.
+		/// When the requested range falls entirely within a single leaf node, this method returns
+		/// a slice of the internal buffer without allocation. Otherwise, a new buffer is allocated.
+		/// </remarks>
+		public static ReadOnlyMemory<char> GetMemory(this Rope<char> rope, int startIndex, int length)
+		{
+			if (rope == null)
+				throw new ArgumentNullException(nameof(rope));
+
+			if (length < 0)
+				throw new ArgumentOutOfRangeException(nameof(length), length, "Value must be >= 0");
+
+			if (length == 0)
+				return ReadOnlyMemory<char>.Empty;
+
+			rope.VerifyRange(startIndex, length);
+
+			// Try to get a zero-allocation slice if the range fits within a single leaf node
+			var entry = rope.FindNodeUsingCache(startIndex).PeekOrDefault();
+			int offsetWithinNode = startIndex - entry.NodeStartIndex;
+
+			// Check if the entire requested range fits within this leaf node
+			if (offsetWithinNode + length <= entry.Node.Length)
+			{
+				// Return a slice of the existing buffer - no allocation needed
+				return new ReadOnlyMemory<char>(entry.Node.Contents, offsetWithinNode, length);
+			}
+
+			// Range spans multiple nodes - must allocate and copy
+			char[] buffer = new char[length];
+			rope.CopyTo(startIndex, buffer, 0, length);
+			return new ReadOnlyMemory<char>(buffer);
+		}
 
 		/// <summary>
 		/// Retrieves the text for a portion of the rope and writes it to the specified text writer.

test/AvaloniaEdit.Tests/AvaloniaEdit.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>

test/AvaloniaEdit.Tests/TextMate/TextEditorModelTests.cs

@@ -1,8 +1,11 @@
-using AvaloniaEdit.Document;
+using System;
+using Avalonia.Controls.Shapes;
+using AvaloniaEdit.Document;
 using AvaloniaEdit.Rendering;
 using AvaloniaEdit.TextMate;
 
 using NUnit.Framework;
+using TextMateSharp.Grammars;
 
 namespace AvaloniaEdit.Tests.TextMate
 {
@@ -34,9 +37,9 @@ public void Lines_Should_Have_Valid_Content()
 
             document.Text = "puppy\npussy\nbirdie";
 
-            Assert.AreEqual("puppy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("pussy", textEditorModel.GetLineText(1));
-            Assert.AreEqual("birdie", textEditorModel.GetLineText(2));
+            AssertLinesAreEqual("puppy\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("pussy\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("birdie", textEditorModel.GetLineTextIncludingTerminators(2));
         }
 
         [Test]
@@ -52,9 +55,9 @@ public void Editing_Line_Should_Update_The_Line_Content()
 
             document.Insert(0, "cutty ");
 
-            Assert.AreEqual("cutty puppy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("pussy", textEditorModel.GetLineText(1));
-            Assert.AreEqual("birdie", textEditorModel.GetLineText(2));
+            AssertLinesAreEqual("cutty puppy\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("pussy\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("birdie", textEditorModel.GetLineTextIncludingTerminators(2));
         }
 
         [Test]
@@ -86,10 +89,10 @@ public void Inserting_Line_Should_Update_The_Line_Ranges()
 
             document.Insert(0, "lion\n");
 
-            Assert.AreEqual("lion", textEditorModel.GetLineText(0));
-            Assert.AreEqual("puppy", textEditorModel.GetLineText(1));
-            Assert.AreEqual("pussy", textEditorModel.GetLineText(2));
-            Assert.AreEqual("birdie", textEditorModel.GetLineText(3));
+            AssertLinesAreEqual("lion\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("puppy\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("pussy\n", textEditorModel.GetLineTextIncludingTerminators(2));
+            AssertLinesAreEqual("birdie", textEditorModel.GetLineTextIncludingTerminators(3));
         }
 
         [Test]
@@ -107,8 +110,8 @@ public void Removing_Line_Should_Update_The_Line_Ranges()
                 document.Lines[0].Offset,
                 document.Lines[0].TotalLength);
 
-            Assert.AreEqual("pussy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("birdie", textEditorModel.GetLineText(1));
+            AssertLinesAreEqual("pussy\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("birdie", textEditorModel.GetLineTextIncludingTerminators(1));
         }
 
         [Test]
@@ -126,8 +129,8 @@ public void Removing_Line_With_LFCR_Should_Update_The_Line_Ranges()
                 document.Lines[0].Offset,
                 document.Lines[0].TotalLength);
 
-            Assert.AreEqual("pussy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("birdie", textEditorModel.GetLineText(1));
+            AssertLinesAreEqual("pussy\r\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("birdie", textEditorModel.GetLineTextIncludingTerminators(1));
         }
 
         [Test]
@@ -219,7 +222,7 @@ public void Replace_Text_Of_Same_Length_Should_Update_Line_Content()
 
             document.Replace(0, 1, "P");
 
-            Assert.AreEqual("Puppy", textEditorModel.GetLineText(0));
+            AssertLinesAreEqual("Puppy\n", textEditorModel.GetLineTextIncludingTerminators(0));
         }
 
         [Test]
@@ -235,7 +238,7 @@ public void Replace_Text_Of_Same_Length_With_CR_Should_Update_Line_Content()
 
             document.Replace(0, 1, "\n");
 
-            Assert.AreEqual("", textEditorModel.GetLineText(0));
+            AssertLinesAreEqual("\n", textEditorModel.GetLineTextIncludingTerminators(0));
         }
 
         [Test]
@@ -252,7 +255,7 @@ public void Remove_Document_Text_Should_Update_Line_Contents()
 
             document.Text = string.Empty;
             Assert.AreEqual(1, textEditorModel.GetNumberOfLines());
-            Assert.AreEqual(string.Empty, textEditorModel.GetLineText(0));
+            AssertLinesAreEqual(string.Empty, textEditorModel.GetLineTextIncludingTerminators(0));
         }
 
         [Test]
@@ -270,10 +273,10 @@ public void Replace_Document_Text_Should_Update_Line_Contents()
             document.Text = "one\ntwo\nthree\nfour";
             Assert.AreEqual(4, textEditorModel.GetNumberOfLines());
 
-            Assert.AreEqual("one", textEditorModel.GetLineText(0));
-            Assert.AreEqual("two", textEditorModel.GetLineText(1));
-            Assert.AreEqual("three", textEditorModel.GetLineText(2));
-            Assert.AreEqual("four", textEditorModel.GetLineText(3));
+            AssertLinesAreEqual("one\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("two\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("three\n", textEditorModel.GetLineTextIncludingTerminators(2));
+            AssertLinesAreEqual("four", textEditorModel.GetLineTextIncludingTerminators(3));
         }
 
         [Test]
@@ -311,9 +314,9 @@ public void Batch_Document_Changes_Should_Invalidate_Lines()
             Assert.IsNull(textEditorModel.InvalidRange,
                 "InvalidRange should be null");
 
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(1));
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(2));
+            AssertLinesAreEqual("*puppy\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("*puppy\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("*puppy", textEditorModel.GetLineTextIncludingTerminators(2));
         }
 
         [Test]
@@ -355,9 +358,14 @@ public void Nested_Batch_Document_Changes_Should_Invalidate_Lines()
             Assert.IsNull(textEditorModel.InvalidRange,
                 "InvalidRange should be null");
 
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(0));
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(1));
-            Assert.AreEqual("*puppy", textEditorModel.GetLineText(2));
+            AssertLinesAreEqual("*puppy\n", textEditorModel.GetLineTextIncludingTerminators(0));
+            AssertLinesAreEqual("*puppy\n", textEditorModel.GetLineTextIncludingTerminators(1));
+            AssertLinesAreEqual("*puppy", textEditorModel.GetLineTextIncludingTerminators(2));
+        }
+
+        private static void AssertLinesAreEqual(LineText expected, LineText actual)
+        {
+            Assert.AreEqual(expected, actual);
         }
     }
 }