Skip to content

Commit 86d7a49

Browse files
markwallace-microsoftalzareicrickmanCopilot
authored
.Net: Feature text search linq (#13384)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [ ] The code builds clean without any errors or warnings - [ ] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [ ] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄 --------- Co-authored-by: Alexander Zarei <azarei@ece.ubc.ca> Co-authored-by: Alexander Zarei <alzarei@users.noreply.github.com> Co-authored-by: Chris <66376200+crickman@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent e901020 commit 86d7a49

File tree

47 files changed

+5475
-222
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+5475
-222
lines changed

‎docs/decisions/0073-linq-based-text-search-filtering.md‎

Lines changed: 567 additions & 0 deletions
Large diffs are not rendered by default.

‎dotnet/samples/Concepts/RAG/Bing_RagWithTextSearch.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
// Copyright (c) Microsoft. All rights reserved.
2+
23
using Microsoft.SemanticKernel;
34
using Microsoft.SemanticKernel.Data;
45
using Microsoft.SemanticKernel.Plugins.Web.Bing;
@@ -133,6 +134,7 @@ Include citations to and the date of the relevant information where it is refere
133134
));
134135
}
135136

137+
#pragma warning disable CS0618 // Suppress obsolete warnings for legacy TextSearchOptions/TextSearchFilter usage
136138
/// <summary>
137139
/// Show how to create a default <see cref="KernelPlugin"/> from an <see cref="ITextSearch"/> and use it to
138140
/// add grounding context to a Handlebars prompt that include full web pages.
@@ -183,4 +185,5 @@ Include citations to the relevant information where it is referenced in the resp
183185
promptTemplateFactory: promptTemplateFactory
184186
));
185187
}
188+
#pragma warning restore CS0618
186189
}

‎dotnet/samples/Concepts/Search/Bing_FunctionCallingWithTextSearch.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ public async Task FunctionCallingWithBingTextSearchIncludingCitationsAsync()
6666
Console.WriteLine(await kernel.InvokePromptAsync("What is the Semantic Kernel? Include citations to the relevant information where it is referenced in the response.", arguments));
6767
}
6868

69+
#pragma warning disable CS0618 // Suppress obsolete warnings for legacy TextSearchOptions/TextSearchFilter usage
6970
/// <summary>
7071
/// Show how to create a default <see cref="KernelPlugin"/> from an <see cref="BingTextSearch"/> and use it with
7172
/// function calling to have the LLM include grounding context from the Microsoft Dev Blogs site in it's response.
@@ -143,4 +144,5 @@ private static KernelFunction CreateSearchBySite(BingTextSearch textSearch, Text
143144

144145
return textSearch.CreateSearch(options);
145146
}
147+
#pragma warning restore CS0618
146148
}

‎dotnet/samples/Concepts/Search/Bing_TextSearch.cs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace Search;
1111
/// </summary>
1212
public class Bing_TextSearch(ITestOutputHelper output) : BaseTest(output)
1313
{
14+
#pragma warning disable CS0618 // Suppress obsolete warnings for legacy TextSearchOptions/TextSearchFilter usage
1415
/// <summary>
1516
/// Show how to create a <see cref="BingTextSearch"/> and use it to perform a text search.
1617
/// </summary>
@@ -118,6 +119,87 @@ public async Task UsingBingTextSearchWithASiteFilterAsync()
118119
WriteHorizontalRule();
119120
}
120121
}
122+
#pragma warning restore CS0618
123+
124+
/// <summary>
125+
/// Show how to use enhanced LINQ filtering with BingTextSearch for type-safe searches.
126+
/// </summary>
127+
[Fact]
128+
public async Task UsingBingTextSearchWithLinqFilteringAsync()
129+
{
130+
// Create a logging handler to output HTTP requests and responses
131+
LoggingHandler handler = new(new HttpClientHandler(), this.Output);
132+
using HttpClient httpClient = new(handler);
133+
134+
// Create an ITextSearch<BingWebPage> instance for type-safe LINQ filtering
135+
ITextSearch<BingWebPage> textSearch = new BingTextSearch(apiKey: TestConfiguration.Bing.ApiKey, options: new() { HttpClient = httpClient });
136+
137+
var query = "Semantic Kernel AI";
138+
139+
// Example 1: Filter by language (English only)
140+
Console.WriteLine("——— Example 1: Language Filter (English) ———\n");
141+
var languageOptions = new TextSearchOptions<BingWebPage>
142+
{
143+
Top = 2,
144+
Filter = page => page.Language == "en"
145+
};
146+
var languageResults = await textSearch.SearchAsync(query, languageOptions);
147+
await foreach (string result in languageResults.Results)
148+
{
149+
Console.WriteLine(result);
150+
WriteHorizontalRule();
151+
}
152+
153+
// Example 2: Filter by family-friendly content
154+
Console.WriteLine("\n——— Example 2: Family Friendly Filter ———\n");
155+
var familyFriendlyOptions = new TextSearchOptions<BingWebPage>
156+
{
157+
Top = 2,
158+
Filter = page => page.IsFamilyFriendly == true
159+
};
160+
var familyFriendlyResults = await textSearch.SearchAsync(query, familyFriendlyOptions);
161+
await foreach (string result in familyFriendlyResults.Results)
162+
{
163+
Console.WriteLine(result);
164+
WriteHorizontalRule();
165+
}
166+
167+
// Example 3: Compound AND filtering (language + family-friendly)
168+
Console.WriteLine("\n——— Example 3: Compound Filter (English + Family Friendly) ———\n");
169+
var compoundOptions = new TextSearchOptions<BingWebPage>
170+
{
171+
Top = 2,
172+
Filter = page => page.Language == "en" && page.IsFamilyFriendly == true
173+
};
174+
var compoundResults = await textSearch.GetSearchResultsAsync(query, compoundOptions);
175+
await foreach (BingWebPage page in compoundResults.Results)
176+
{
177+
Console.WriteLine($"Name: {page.Name}");
178+
Console.WriteLine($"Snippet: {page.Snippet}");
179+
Console.WriteLine($"Language: {page.Language}");
180+
Console.WriteLine($"Family Friendly: {page.IsFamilyFriendly}");
181+
WriteHorizontalRule();
182+
}
183+
184+
// Example 4: Complex compound filtering with nullable checks
185+
Console.WriteLine("\n——— Example 4: Complex Compound Filter (Language + Site + Family Friendly) ———\n");
186+
var complexOptions = new TextSearchOptions<BingWebPage>
187+
{
188+
Top = 2,
189+
Filter = page => page.Language == "en" &&
190+
page.IsFamilyFriendly == true &&
191+
page.DisplayUrl != null && page.DisplayUrl.Contains("microsoft")
192+
};
193+
var complexResults = await textSearch.GetSearchResultsAsync(query, complexOptions);
194+
await foreach (BingWebPage page in complexResults.Results)
195+
{
196+
Console.WriteLine($"Name: {page.Name}");
197+
Console.WriteLine($"Display URL: {page.DisplayUrl}");
198+
Console.WriteLine($"Language: {page.Language}");
199+
Console.WriteLine($"Family Friendly: {page.IsFamilyFriendly}");
200+
WriteHorizontalRule();
201+
}
202+
}
121203

122204
#region private
123205
/// <summary>

‎dotnet/samples/Concepts/Search/Google_TextSearch.cs‎

Lines changed: 113 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace Search;
1212
/// </summary>
1313
public class Google_TextSearch(ITestOutputHelper output) : BaseTest(output)
1414
{
15+
#pragma warning disable CS0618 // Suppress obsolete warnings for legacy TextSearchOptions/TextSearchFilter usage
1516
/// <summary>
1617
/// Show how to create a <see cref="GoogleTextSearch"/> and use it to perform a text search.
1718
/// </summary>
@@ -26,7 +27,7 @@ public async Task UsingGoogleTextSearchAsync()
2627
var query = "What is the Semantic Kernel?";
2728

2829
// Search and return results as string items
29-
KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new() { Top = 4, Skip = 0 });
30+
KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 4, Skip = 0 });
3031
Console.WriteLine("——— String Results ———\n");
3132
await foreach (string result in stringResults.Results)
3233
{
@@ -35,7 +36,7 @@ public async Task UsingGoogleTextSearchAsync()
3536
}
3637

3738
// Search and return results as TextSearchResult items
38-
KernelSearchResults<TextSearchResult> textResults = await textSearch.GetTextSearchResultsAsync(query, new() { Top = 4, Skip = 4 });
39+
KernelSearchResults<TextSearchResult> textResults = await textSearch.GetTextSearchResultsAsync(query, new TextSearchOptions { Top = 4, Skip = 4 });
3940
Console.WriteLine("\n——— Text Search Results ———\n");
4041
await foreach (TextSearchResult result in textResults.Results)
4142
{
@@ -46,7 +47,7 @@ public async Task UsingGoogleTextSearchAsync()
4647
}
4748

4849
// Search and return results as Google.Apis.CustomSearchAPI.v1.Data.Result items
49-
KernelSearchResults<object> fullResults = await textSearch.GetSearchResultsAsync(query, new() { Top = 4, Skip = 8 });
50+
KernelSearchResults<object> fullResults = await textSearch.GetSearchResultsAsync(query, new TextSearchOptions { Top = 4, Skip = 8 });
5051
Console.WriteLine("\n——— Google Web Page Results ———\n");
5152
await foreach (Google.Apis.CustomSearchAPI.v1.Data.Result result in fullResults.Results)
5253
{
@@ -74,7 +75,7 @@ public async Task UsingGoogleTextSearchWithACustomMapperAsync()
7475
var query = "What is the Semantic Kernel?";
7576

7677
// Search with TextSearchResult textResult type
77-
KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new() { Top = 2, Skip = 0 });
78+
KernelSearchResults<string> stringResults = await textSearch.SearchAsync(query, new TextSearchOptions { Top = 2, Skip = 0 });
7879
Console.WriteLine("--- Serialized JSON Results ---");
7980
await foreach (string result in stringResults.Results)
8081
{
@@ -106,6 +107,114 @@ public async Task UsingGoogleTextSearchWithASiteSearchFilterAsync()
106107
Console.WriteLine(new string('-', HorizontalRuleLength));
107108
}
108109
}
110+
#pragma warning restore CS0618
111+
112+
/// <summary>
113+
/// Show how to use enhanced LINQ filtering with GoogleTextSearch including Contains, NOT, FileType, and compound AND expressions.
114+
/// </summary>
115+
[Fact]
116+
public async Task UsingGoogleTextSearchWithEnhancedLinqFilteringAsync()
117+
{
118+
// Create an ITextSearch<GoogleWebPage> instance using Google search
119+
var textSearch = new GoogleTextSearch(
120+
initializer: new() { ApiKey = TestConfiguration.Google.ApiKey, HttpClientFactory = new CustomHttpClientFactory(this.Output) },
121+
searchEngineId: TestConfiguration.Google.SearchEngineId);
122+
123+
var query = "Semantic Kernel AI";
124+
125+
// Example 1: Simple equality filtering
126+
Console.WriteLine("——— Example 1: Equality Filter (DisplayLink) ———\n");
127+
var equalityOptions = new TextSearchOptions<GoogleWebPage>
128+
{
129+
Top = 2,
130+
Skip = 0,
131+
Filter = page => page.DisplayLink == "microsoft.com"
132+
};
133+
var equalityResults = await textSearch.SearchAsync(query, equalityOptions);
134+
await foreach (string result in equalityResults.Results)
135+
{
136+
Console.WriteLine(result);
137+
Console.WriteLine(new string('—', HorizontalRuleLength));
138+
}
139+
140+
// Example 2: Contains filtering
141+
Console.WriteLine("\n——— Example 2: Contains Filter (Title) ———\n");
142+
var containsOptions = new TextSearchOptions<GoogleWebPage>
143+
{
144+
Top = 2,
145+
Skip = 0,
146+
Filter = page => page.Title != null && page.Title.Contains("AI")
147+
};
148+
var containsResults = await textSearch.SearchAsync(query, containsOptions);
149+
await foreach (string result in containsResults.Results)
150+
{
151+
Console.WriteLine(result);
152+
Console.WriteLine(new string('—', HorizontalRuleLength));
153+
}
154+
155+
// Example 3: NOT Contains filtering (exclusion)
156+
Console.WriteLine("\n——— Example 3: NOT Contains Filter (Exclude 'deprecated') ———\n");
157+
var notContainsOptions = new TextSearchOptions<GoogleWebPage>
158+
{
159+
Top = 2,
160+
Skip = 0,
161+
Filter = page => page.Title != null && !page.Title.Contains("deprecated")
162+
};
163+
var notContainsResults = await textSearch.SearchAsync(query, notContainsOptions);
164+
await foreach (string result in notContainsResults.Results)
165+
{
166+
Console.WriteLine(result);
167+
Console.WriteLine(new string('—', HorizontalRuleLength));
168+
}
169+
170+
// Example 4: FileFormat filtering
171+
Console.WriteLine("\n——— Example 4: FileFormat Filter (PDF files) ———\n");
172+
var fileFormatOptions = new TextSearchOptions<GoogleWebPage>
173+
{
174+
Top = 2,
175+
Skip = 0,
176+
Filter = page => page.FileFormat == "pdf"
177+
};
178+
var fileFormatResults = await textSearch.SearchAsync(query, fileFormatOptions);
179+
await foreach (string result in fileFormatResults.Results)
180+
{
181+
Console.WriteLine(result);
182+
Console.WriteLine(new string('—', HorizontalRuleLength));
183+
}
184+
185+
// Example 5: Compound AND filtering (multiple conditions)
186+
Console.WriteLine("\n——— Example 5: Compound AND Filter (Title + Site) ———\n");
187+
var compoundOptions = new TextSearchOptions<GoogleWebPage>
188+
{
189+
Top = 2,
190+
Skip = 0,
191+
Filter = page => page.Title != null && page.Title.Contains("Semantic") &&
192+
page.DisplayLink != null && page.DisplayLink.Contains("microsoft")
193+
};
194+
var compoundResults = await textSearch.SearchAsync(query, compoundOptions);
195+
await foreach (string result in compoundResults.Results)
196+
{
197+
Console.WriteLine(result);
198+
Console.WriteLine(new string('—', HorizontalRuleLength));
199+
}
200+
201+
// Example 6: Complex compound filtering (equality + contains + exclusion)
202+
Console.WriteLine("\n——— Example 6: Complex Compound Filter (FileFormat + Contains + NOT Contains) ———\n");
203+
var complexOptions = new TextSearchOptions<GoogleWebPage>
204+
{
205+
Top = 2,
206+
Skip = 0,
207+
Filter = page => page.FileFormat == "pdf" &&
208+
page.Title != null && page.Title.Contains("AI") &&
209+
page.Snippet != null && !page.Snippet.Contains("deprecated")
210+
};
211+
var complexResults = await textSearch.SearchAsync(query, complexOptions);
212+
await foreach (string result in complexResults.Results)
213+
{
214+
Console.WriteLine(result);
215+
Console.WriteLine(new string('—', HorizontalRuleLength));
216+
}
217+
}
109218

110219
#region private
111220
private const int HorizontalRuleLength = 80;

‎dotnet/samples/Concepts/Search/Tavily_TextSearch.cs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace Search;
1111
/// </summary>
1212
public class Tavily_TextSearch(ITestOutputHelper output) : BaseTest(output)
1313
{
14+
#pragma warning disable CS0618 // Suppress obsolete warnings for legacy TextSearchOptions/TextSearchFilter usage
1415
/// <summary>
1516
/// Show how to create a <see cref="TavilyTextSearch"/> and use it to perform a text search.
1617
/// </summary>
@@ -181,6 +182,87 @@ public async Task UsingTavilyTextSearchWithAnIncludeDomainFilterAsync()
181182
WriteHorizontalRule();
182183
}
183184
}
185+
#pragma warning restore CS0618
186+
187+
/// <summary>
188+
/// Show how to use enhanced LINQ filtering with TavilyTextSearch for type-safe searches with Title.Contains() support.
189+
/// </summary>
190+
[Fact]
191+
public async Task UsingTavilyTextSearchWithLinqFilteringAsync()
192+
{
193+
// Create a logging handler to output HTTP requests and responses
194+
LoggingHandler handler = new(new HttpClientHandler(), this.Output);
195+
using HttpClient httpClient = new(handler);
196+
197+
// Create an ITextSearch<TavilyWebPage> instance for type-safe LINQ filtering
198+
ITextSearch<TavilyWebPage> textSearch = new TavilyTextSearch(apiKey: TestConfiguration.Tavily.ApiKey, options: new() { HttpClient = httpClient });
199+
200+
var query = "Semantic Kernel AI";
201+
202+
// Example 1: Filter results by title content using Contains
203+
Console.WriteLine("——— Example 1: Title Contains Filter ———\n");
204+
var titleContainsOptions = new TextSearchOptions<TavilyWebPage>
205+
{
206+
Top = 2,
207+
Filter = page => page.Title != null && page.Title.Contains("Kernel")
208+
};
209+
var titleResults = await textSearch.SearchAsync(query, titleContainsOptions);
210+
await foreach (string result in titleResults.Results)
211+
{
212+
Console.WriteLine(result);
213+
WriteHorizontalRule();
214+
}
215+
216+
// Example 2: Compound AND filtering (title contains + NOT contains)
217+
Console.WriteLine("\n——— Example 2: Compound Filter (Title Contains + Exclusion) ———\n");
218+
var compoundOptions = new TextSearchOptions<TavilyWebPage>
219+
{
220+
Top = 2,
221+
Filter = page => page.Title != null && page.Title.Contains("AI") &&
222+
page.Content != null && !page.Content.Contains("deprecated")
223+
};
224+
var compoundResults = await textSearch.SearchAsync(query, compoundOptions);
225+
await foreach (string result in compoundResults.Results)
226+
{
227+
Console.WriteLine(result);
228+
WriteHorizontalRule();
229+
}
230+
231+
// Example 3: Get full results with LINQ filtering
232+
Console.WriteLine("\n——— Example 3: Full Results with Title Filter ———\n");
233+
var fullResultsOptions = new TextSearchOptions<TavilyWebPage>
234+
{
235+
Top = 2,
236+
Filter = page => page.Title != null && page.Title.Contains("Semantic")
237+
};
238+
var fullResults = await textSearch.GetSearchResultsAsync(query, fullResultsOptions);
239+
await foreach (TavilyWebPage page in fullResults.Results)
240+
{
241+
Console.WriteLine($"Title: {page.Title}");
242+
Console.WriteLine($"Content: {page.Content}");
243+
Console.WriteLine($"URL: {page.Url}");
244+
Console.WriteLine($"Score: {page.Score}");
245+
WriteHorizontalRule();
246+
}
247+
248+
// Example 4: Complex compound filtering with multiple conditions
249+
Console.WriteLine("\n——— Example 4: Complex Compound Filter (Title + Content + URL) ———\n");
250+
var complexOptions = new TextSearchOptions<TavilyWebPage>
251+
{
252+
Top = 2,
253+
Filter = page => page.Title != null && page.Title.Contains("Kernel") &&
254+
page.Content != null && page.Content.Contains("AI") &&
255+
page.Url != null && page.Url.ToString().Contains("microsoft")
256+
};
257+
var complexResults = await textSearch.GetSearchResultsAsync(query, complexOptions);
258+
await foreach (TavilyWebPage page in complexResults.Results)
259+
{
260+
Console.WriteLine($"Title: {page.Title}");
261+
Console.WriteLine($"URL: {page.Url}");
262+
Console.WriteLine($"Score: {page.Score}");
263+
WriteHorizontalRule();
264+
}
265+
}
184266

185267
#region private
186268
/// <summary>

0 commit comments

Comments
 (0)