Improve nav-to fuzzy pre-filtering with q-gram bigram bitset and corrected length thresholds

src/EditorFeatures/CSharpTest/NavigateTo/NavigateToTests.cs

@@ -1792,5 +1792,114 @@ await TestAsync(testHost, composition, content, async w =>
             VerifyNavigateToResultItem(item, "Goo", "[|Goo|]", PatternMatchKind.Exact, NavigateToItemKind.Property, Glyph.PropertyPublic);
         });
     }
+
+    #region Pre-filter integration tests
+    // These tests verify that the NavigateToSearchInfo pre-filter (hump bigrams, trigram filter,
+    // length bitset, and the split fuzzy/non-fuzzy signal) correctly allows matches through the
+    // full NavigateTo pipeline. Exhaustive unit tests for each individual filter are in
+    // TopLevelSyntaxTreeIndexTests (src/Workspaces/CoreTest/FindSymbols/TopLevelSyntaxTreeIndexTests.cs).
+
+    /// <summary>
+    /// Verifies that a CamelCase pattern passes the hump bigram filter and produces a match.
+    /// The pre-filter stores ordered pairs of adjacent hump initials (e.g., "GB" for GooBar)
+    /// and the DP checks against them. "GB" should match "GooBar" as CamelCaseExact.
+    /// </summary>
+    [Theory, CombinatorialData]
+    public Task PreFilter_CamelCaseHumpBigram(TestHost testHost, Composition composition)
+        => TestAsync(
+            testHost, composition, """
+            class GooBar
+            {
+            }
+            """, async w =>
+            {
+                var item = (await _aggregator.GetItemsAsync("GB")).Single(x => x.Kind != "Method");
+                VerifyNavigateToResultItem(item, "GooBar", "[|G|]oo[|B|]ar", PatternMatchKind.CamelCaseExact, NavigateToItemKind.Class, Glyph.ClassInternal);
+            });
+
+    /// <summary>
+    /// Verifies that an all-lowercase pattern passes the hump prefix filter via the DP algorithm.
+    /// The pre-filter stores lowercased prefixes of each hump (e.g., "g", "go", "goo" from "Goo").
+    /// The DP tries to split the all-lowercase pattern into segments that are each a valid prefix
+    /// of some hump. "goo" matches the "Goo" hump prefix, so the filter lets it through.
+    /// PatternMatcher returns Prefix (not CamelCasePrefix) because "goo" is a literal
+    /// case-insensitive prefix of "GooBar" and literal prefix takes priority.
+    /// </summary>
+    [Theory, CombinatorialData]
+    public Task PreFilter_AllLowercaseDPHumpPrefix(TestHost testHost, Composition composition)
+        => TestAsync(
+            testHost, composition, """
+            class GooBar
+            {
+            }
+            """, async w =>
+            {
+                var item = (await _aggregator.GetItemsAsync("goo")).Single(x => x.Kind != "Method");
+                VerifyNavigateToResultItem(item, "GooBar", "[|Goo|]Bar", PatternMatchKind.Prefix, NavigateToItemKind.Class, Glyph.ClassInternal);
+            });
+
+    /// <summary>
+    /// Verifies that a lowercase substring pattern passes the trigram filter and produces a match.
+    /// The pre-filter stores all 3-char sliding windows of the lowercased symbol name. The pattern
+    /// "line" has trigrams "lin" and "ine", both present in "readline", so the filter lets it through.
+    /// The PatternMatcher returns <see cref="PatternMatchKind.Fuzzy"/> because "line"
+    /// is all-lowercase and lands at a non-word-boundary in "Readline". NavigateToMatchKind has no
+    /// dedicated bucket for this, so it maps to Fuzzy (see s_kindPairs). Exhaustive coverage of the
+    /// underlying PatternMatchKind is in NavigateToSearchIndexTests.
+    /// </summary>
+    [Theory, CombinatorialData]
+    public Task PreFilter_TrigramSubstring(TestHost testHost, Composition composition)
+        => TestAsync(
+            testHost, composition, """
+            class C
+            {
+                void Readline() { }
+            }
+            """, async w =>
+            {
+                var item = (await _aggregator.GetItemsAsync("line")).Single();
+                VerifyNavigateToResultItem(item, "Readline", "Read[|line|]()", PatternMatchKind.Fuzzy, NavigateToItemKind.Method, Glyph.MethodPrivate);
+            });
+
+    /// <summary>
+    /// Verifies that a fuzzy match is found when the length bitset check passes (symbol length
+    /// within ±2 of pattern length). With the split fuzzy/non-fuzzy pre-filtering, the length
+    /// check sets 'allowFuzzyMatching', enabling the PatternMatcher's edit-distance computation.
+    /// "ToEror" (length 6) fuzzy-matches "ToError" (length 7), delta=1, within ±2.
+    /// </summary>
+    [Theory, CombinatorialData]
+    public Task PreFilter_FuzzyMatchEnabledByLengthCheck(TestHost testHost, Composition composition)
+        => TestAsync(
+            testHost, composition, """
+            class C
+            {
+                public void ToError() { }
+            }
+            """, async w =>
+            {
+                var item = (await _aggregator.GetItemsAsync("ToEror")).Single();
+                VerifyNavigateToResultItem(item, "ToError", "ToError()", PatternMatchKind.Fuzzy, NavigateToItemKind.Method, Glyph.MethodPublic);
+            });
+
+    /// <summary>
+    /// Verifies that a pattern completely unrelated to any symbol in the document produces no
+    /// results. All three pre-filter checks (hump, trigram, length) must fail for the document
+    /// to be skipped entirely. "XyzXyzXyzXyz" shares no hump structure, no trigrams, and has
+    /// length 12 which is far from "GooBar" (length 6).
+    /// </summary>
+    [Theory, CombinatorialData]
+    public Task PreFilter_NoMatchWhenAllChecksFail(TestHost testHost, Composition composition)
+        => TestAsync(
+            testHost, composition, """
+            class GooBar
+            {
+            }
+            """, async w =>
+            {
+                var items = await _aggregator.GetItemsAsync("XyzXyzXyzXyz");
+                Assert.Empty(items);
+            });
+
+    #endregion
 }
 #pragma warning restore CS0618 // MatchKind is obsolete

src/EditorFeatures/Test/Utilities/PatternMatcherTests.cs

@@ -13,7 +13,6 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Collections;
 using Microsoft.CodeAnalysis.PatternMatching;
-using Microsoft.CodeAnalysis.Shared.Collections;
 using Microsoft.CodeAnalysis.Shared.Utilities;
 using Microsoft.CodeAnalysis.Text;
 using Roslyn.Test.Utilities;
@@ -479,7 +478,7 @@ private static IEnumerable<PatternMatch> TryMatchMultiWordPattern(string candida
         MarkupTestFile.GetSpans(candidate, out candidate, out var expectedSpans);
 
         using var matches = TemporaryArray<PatternMatch>.Empty;
-        PatternMatcher.CreatePatternMatcher(pattern, includeMatchedSpans: true).AddMatches(candidate, ref matches.AsRef());
+        PatternMatcher.CreatePatternMatcher(pattern, includeMatchedSpans: true, allowFuzzyMatching: false).AddMatches(candidate, ref matches.AsRef());
 
         if (matches.Count == 0)
         {

src/Features/Core/Portable/NavigateTo/AbstractNavigateToSearchService.CachedDocumentSearch.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -22,6 +22,7 @@
 namespace Microsoft.CodeAnalysis.NavigateTo;
 
 using CachedIndexMap = ConcurrentDictionary<(IChecksummedPersistentStorageService service, DocumentKey documentKey, StringTable stringTable), AsyncLazy<TopLevelSyntaxTreeIndex?>>;
+using CachedFilterIndexMap = ConcurrentDictionary<(IChecksummedPersistentStorageService service, DocumentKey documentKey, StringTable stringTable), AsyncLazy<NavigateToSearchIndex?>>;
 
 internal abstract partial class AbstractNavigateToSearchService
 {
@@ -32,6 +33,12 @@ internal abstract partial class AbstractNavigateToSearchService
     /// </summary>
     private static CachedIndexMap? s_cachedIndexMap = [];
 
+    /// <summary>
+    /// Cached map from document key to the (potentially stale) lightweight filter index.  Loaded first
+    /// to quickly reject documents before loading the much larger <see cref="TopLevelSyntaxTreeIndex"/>.
+    /// </summary>
+    private static CachedFilterIndexMap? s_cachedFilterIndexMap = [];
+
     /// <summary>
     /// String table we use to dedupe common values while deserializing <see cref="SyntaxTreeIndex"/>s.  Once the 
     /// full solution is available, this will be dropped (set to <see langword="null"/>) to release all cached data.
@@ -43,16 +50,19 @@ private static void ClearCachedData()
         // Volatiles are technically not necessary due to automatic fencing of reference-type writes.  However,
         // i prefer the explicitness here as we are reading and writing these fields from different threads.
         Volatile.Write(ref s_cachedIndexMap, null);
+        Volatile.Write(ref s_cachedFilterIndexMap, null);
         Volatile.Write(ref s_stringTable, null);
     }
 
     private static bool ShouldSearchCachedDocuments(
         [NotNullWhen(true)] out CachedIndexMap? cachedIndexMap,
+        [NotNullWhen(true)] out CachedFilterIndexMap? cachedFilterIndexMap,
         [NotNullWhen(true)] out StringTable? stringTable)
     {
         cachedIndexMap = Volatile.Read(ref s_cachedIndexMap);
+        cachedFilterIndexMap = Volatile.Read(ref s_cachedFilterIndexMap);
         stringTable = Volatile.Read(ref s_stringTable);
-        return cachedIndexMap != null && stringTable != null;
+        return cachedIndexMap != null && cachedFilterIndexMap != null && stringTable != null;
     }
 
     public async Task SearchCachedDocumentsAsync(
@@ -107,7 +117,7 @@ public static async Task SearchCachedDocumentsInCurrentProcessAsync(
         CancellationToken cancellationToken)
     {
         // Quick abort if OOP is now fully loaded.
-        if (!ShouldSearchCachedDocuments(out _, out _))
+        if (!ShouldSearchCachedDocuments(out _, out _, out _))
             return;
 
         var (patternName, patternContainer) = PatternMatcher.GetNameAndContainer(searchPattern);
@@ -143,21 +153,46 @@ await Parallel.ForEachAsync(
                 cancellationToken,
                 async (documentKey, cancellationToken) =>
                 {
-                    var index = await GetIndexAsync(storageService, documentKey, cancellationToken).ConfigureAwait(false);
+                    // First, load the lightweight filter index to check if this document could possibly match.
+                    var filterIndex = await GetFilterIndexAsync(storageService, documentKey, cancellationToken).ConfigureAwait(false);
+                    if (filterIndex is null || !filterIndex.CouldContainNavigateToMatch(patternName, patternContainer, out var nameMatchKinds))
+                        return;
+
+                    // The filter passed — now load the full index with all declared symbols.
+                    var index = await GetFullIndexAsync(storageService, documentKey, cancellationToken).ConfigureAwait(false);
                     if (index == null)
                         return;
 
                     ProcessIndex(
                         documentKey, document: null, patternName, patternContainer, declaredSymbolInfoKindsSet,
-                        index, linkedIndices: null, onItemFound, cancellationToken);
+                        nameMatchKinds, index, linkedIndices: null, onItemFound, cancellationToken);
                 }).ConfigureAwait(false);
 
             // done with project.  Let the host know.
             await onProjectCompleted().ConfigureAwait(false);
         }
     }
 
-    private static Task<TopLevelSyntaxTreeIndex?> GetIndexAsync(
+    private static async Task<NavigateToSearchIndex?> GetFilterIndexAsync(
+        IChecksummedPersistentStorageService storageService,
+        DocumentKey documentKey,
+        CancellationToken cancellationToken)
+    {
+        if (cancellationToken.IsCancellationRequested)
+            return null;
+
+        if (!ShouldSearchCachedDocuments(out _, out var cachedFilterIndexMap, out var stringTable))
+            return null;
+
+        var asyncLazy = cachedFilterIndexMap.GetOrAdd(
+            (storageService, documentKey, stringTable),
+            static t => AsyncLazy.Create(static (t, c) =>
+                NavigateToSearchIndex.LoadAsync(t.service, t.documentKey, checksum: null, t.stringTable, c),
+                arg: t));
+        return await asyncLazy.GetValueAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private static Task<TopLevelSyntaxTreeIndex?> GetFullIndexAsync(
         IChecksummedPersistentStorageService storageService,
         DocumentKey documentKey,
         CancellationToken cancellationToken)
@@ -167,7 +202,7 @@ await Parallel.ForEachAsync(
 
         // Retrieve the string table we use to dedupe strings.  If we can't get it, that means the solution has 
         // fully loaded and we've switched over to normal navto lookup.
-        if (!ShouldSearchCachedDocuments(out var cachedIndexMap, out var stringTable))
+        if (!ShouldSearchCachedDocuments(out var cachedIndexMap, out _, out var stringTable))
             return SpecializedTasks.Null<TopLevelSyntaxTreeIndex>();
 
         // Add the async lazy to compute the index for this document.  Or, return the existing cached one if already

src/Features/Core/Portable/NavigateTo/AbstractNavigateToSearchService.InProcess.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
@@ -35,6 +35,10 @@ internal abstract partial class AbstractNavigateToSearchService
             (PatternMatchKind.CamelCaseSubstring, NavigateToMatchKind.CamelCaseSubstring),
             (PatternMatchKind.CamelCaseNonContiguousSubstring, NavigateToMatchKind.CamelCaseNonContiguousSubstring),
             (PatternMatchKind.Fuzzy, NavigateToMatchKind.Fuzzy),
+
+            // LowercaseSubstring is the weakest non-fuzzy PatternMatchKind (an all-lowercase pattern found
+            // inside a candidate at a non-word-boundary, e.g. "line" in "Readline"). NavigateToMatchKind has
+            // no dedicated bucket for it, so we map it to Fuzzy as the closest available quality tier.
             (PatternMatchKind.LowercaseSubstring, NavigateToMatchKind.Fuzzy),
         ];
 
@@ -49,8 +53,13 @@ private static async ValueTask SearchSingleDocumentAsync(
         if (cancellationToken.IsCancellationRequested)
             return;
 
-        // Get the index for the file we're searching, as well as for its linked siblings.  We'll use the latter to add
-        // the information to a symbol about all the project TFMs is can be found in.
+        // First, load the lightweight filter index to check if this document could possibly match.
+        // This avoids loading the much larger TopLevelSyntaxTreeIndex for non-matching documents.
+        var filterIndex = await NavigateToSearchIndex.GetRequiredIndexAsync(document, cancellationToken).ConfigureAwait(false);
+        if (!filterIndex.CouldContainNavigateToMatch(patternName, patternContainer, out var allowFuzzyMatching))
+            return;
+
+        // The filter passed — now load the full index with all declared symbols.
         var index = await TopLevelSyntaxTreeIndex.GetRequiredIndexAsync(document, cancellationToken).ConfigureAwait(false);
         using var _ = ArrayBuilder<(TopLevelSyntaxTreeIndex, ProjectId)>.GetInstance(out var linkedIndices);
 
@@ -63,7 +72,7 @@ private static async ValueTask SearchSingleDocumentAsync(
 
         ProcessIndex(
             DocumentKey.ToDocumentKey(document), document, patternName, patternContainer, kinds,
-            index, linkedIndices, onItemFound, cancellationToken);
+            allowFuzzyMatching, index, linkedIndices, onItemFound, cancellationToken);
     }
 
     private static void ProcessIndex(
@@ -72,6 +81,7 @@ private static void ProcessIndex(
         string patternName,
         string? patternContainer,
         DeclaredSymbolInfoKindSet kinds,
+        bool allowFuzzyMatching,
         TopLevelSyntaxTreeIndex index,
         ArrayBuilder<(TopLevelSyntaxTreeIndex, ProjectId)>? linkedIndices,
         Action<RoslynNavigateToItem> onItemFound,
@@ -80,12 +90,8 @@ private static void ProcessIndex(
         if (cancellationToken.IsCancellationRequested)
             return;
 
-        var containerMatcher = patternContainer != null
-            ? PatternMatcher.CreateDotSeparatedContainerMatcher(patternContainer, includeMatchedSpans: true)
-            : null;
-
-        using var nameMatcher = PatternMatcher.CreatePatternMatcher(patternName, includeMatchedSpans: true, allowFuzzyMatching: true);
-        using var _1 = containerMatcher;
+        using var containerMatcher = PatternMatcher.CreateDotSeparatedContainerMatcher(patternContainer, includeMatchedSpans: true);
+        using var nameMatcher = PatternMatcher.CreatePatternMatcher(patternName, includeMatchedSpans: true, allowFuzzyMatching);
 
         foreach (var declaredSymbolInfo in index.DeclaredSymbolInfos)
         {