Skip to content

Commit 898cc5f

Browse files
maumarmaumar
committed
Cosmos Full Text Search support
- Adding model building API to configure property as full-text search enabled, as well as setup the index for it, - Adding model validation (e.g. FTS index not matching FTS property), - Adding EF.Functions stubs and translations for FullTextContains, FullTextContainsAll, FullTextContainsAny, FullTextScore and RRF (for hybrid), - Adding logic in SelectExpression to produce ORDER BY RANK when necessary, - Adding validation when attempting to mix with ORDER BY RANK with regular ORDER BY, - Rewrite OFFSET/LIMIT from parameter to constant when ORDER BY RANK is present. outstanding work: - support for FTS Container building using Azure.ResourceManager.CosmosDb (currently blocked) - support for Owned types (adjust model in tests and fix paths) - this also needs to happen for vector search - add model building support for default language, - add more tests for hybrid search, - add more model validation tests for invalid scenarios (index on multiple columns, FTS on non-string etc) - add xml docs, - clean up exception messages and put them in a resource file, Fixes #35476 Fixes #35853 Fixes #35867
1 parent ce01986 commit 898cc5f

29 files changed

+1573
-21
lines changed

Diff for: ‎Directory.Packages.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
<PackageVersion Include="Microsoft.DotNet.Build.Tasks.Templating" Version="$(MicrosoftDotNetBuildTasksTemplatingVersion)" />
3838

3939
<!-- Azure SDK for .NET dependencies -->
40-
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.46.0" />
40+
<PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.49.0-preview.0" />
4141

4242
<!-- SQL Server dependencies -->
4343
<PackageVersion Include="Microsoft.Data.SqlClient" Version="6.0.1" />

Diff for: ‎src/EFCore.Cosmos/EFCore.Cosmos.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
<NoWarn>$(NoWarn);EF9101</NoWarn> <!-- Metrics is experimental -->
1313
<NoWarn>$(NoWarn);EF9102</NoWarn> <!-- Paging is experimental -->
1414
<NoWarn>$(NoWarn);EF9103</NoWarn> <!-- Vector search is experimental -->
15+
<NoWarn>$(NoWarn);EF9104</NoWarn> <!-- Full-text search is experimental -->
1516
</PropertyGroup>
1617

1718
<ItemGroup>

Diff for: ‎src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs

+35
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,41 @@ public static T CoalesceUndefined<T>(
5252
T expression2)
5353
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(CoalesceUndefined)));
5454

55+
/// <summary>
56+
/// TODO
57+
/// </summary>
58+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
59+
public static bool FullTextContains(this DbFunctions _, string property, string keyword)
60+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContains)));
61+
62+
/// <summary>
63+
/// TODO
64+
/// </summary>
65+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
66+
public static bool FullTextContainsAll(this DbFunctions _, string property, params string[] keywords)
67+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAll)));
68+
69+
/// <summary>
70+
/// TODO
71+
/// </summary>
72+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
73+
public static bool FullTextContainsAny(this DbFunctions _, string property, params string[] keywords)
74+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAny)));
75+
76+
/// <summary>
77+
/// TODO
78+
/// </summary>
79+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
80+
public static double FullTextScore(this DbFunctions _, string property, string[] keywords)
81+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextScore)));
82+
83+
/// <summary>
84+
/// TODO
85+
/// </summary>
86+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
87+
public static double Rrf(this DbFunctions _, params double[] functions)
88+
=> throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(Rrf)));
89+
5590
/// <summary>
5691
/// Returns the distance between two vectors, using the distance function and data type defined using
5792
/// <see

Diff for: ‎src/EFCore.Cosmos/Extensions/CosmosIndexBuilderExtensions.cs

+80
Original file line numberDiff line numberDiff line change
@@ -96,4 +96,84 @@ public static bool CanSetVectorIndexType(
9696
VectorIndexType? indexType,
9797
bool fromDataAnnotation = false)
9898
=> indexBuilder.CanSetAnnotation(CosmosAnnotationNames.VectorIndexType, indexType, fromDataAnnotation);
99+
100+
/// <summary>
101+
/// Configures the index as a full-text index.
102+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
103+
/// </summary>
104+
/// <remarks>
105+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
106+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
107+
/// </remarks>
108+
/// <param name="indexBuilder">The builder for the index being configured.</param>
109+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
110+
/// <returns>A builder to further configure the index.</returns>
111+
public static IndexBuilder ForFullText(this IndexBuilder indexBuilder, bool? value = true)
112+
{
113+
indexBuilder.Metadata.SetFullTextIndex(value);
114+
115+
return indexBuilder;
116+
}
117+
118+
/// <summary>
119+
/// Configures the index as a full-text index.
120+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
121+
/// </summary>
122+
/// <remarks>
123+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
124+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
125+
/// </remarks>
126+
/// <param name="indexBuilder">The builder for the index being configured.</param>
127+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
128+
/// <returns>A builder to further configure the index.</returns>
129+
public static IndexBuilder<TEntity> ForFullText<TEntity>(
130+
this IndexBuilder<TEntity> indexBuilder,
131+
bool? value = true)
132+
=> (IndexBuilder<TEntity>)ForFullText((IndexBuilder)indexBuilder, value);
133+
134+
/// <summary>
135+
/// Configures the index as a full-text index.
136+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
137+
/// </summary>
138+
/// <remarks>
139+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
140+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
141+
/// </remarks>
142+
/// <param name="indexBuilder">The builder for the index being configured.</param>
143+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
144+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
145+
/// <returns>
146+
/// The same builder instance if the configuration was applied,
147+
/// <see langword="null" /> otherwise.
148+
/// </returns>
149+
public static IConventionIndexBuilder? ForFullText(
150+
this IConventionIndexBuilder indexBuilder,
151+
bool? value,
152+
bool fromDataAnnotation = false)
153+
{
154+
if (indexBuilder.CanSetFullTextIndex(fromDataAnnotation))
155+
{
156+
indexBuilder.Metadata.SetFullTextIndex(value, fromDataAnnotation);
157+
return indexBuilder;
158+
}
159+
160+
return null;
161+
}
162+
163+
/// <summary>
164+
/// Returns a value indicating whether the index can be configured as a Full-text index.
165+
/// </summary>
166+
/// <remarks>
167+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
168+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
169+
/// </remarks>
170+
/// <param name="indexBuilder">The builder for the index being configured.</param>
171+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
172+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
173+
/// <returns><see langword="true" /> if the index can be configured as a Full-text index.</returns>
174+
public static bool CanSetFullTextIndex(
175+
this IConventionIndexBuilder indexBuilder,
176+
bool? value,
177+
bool fromDataAnnotation = false)
178+
=> indexBuilder.CanSetAnnotation(CosmosAnnotationNames.FullTextIndex, value, fromDataAnnotation);
99179
}

Diff for: ‎src/EFCore.Cosmos/Extensions/CosmosIndexExtensions.cs

+45
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,49 @@ public static void SetVectorIndexType(this IMutableIndex index, VectorIndexType?
6161
/// <returns>The <see cref="ConfigurationSource" /> for whether the index is clustered.</returns>
6262
public static ConfigurationSource? GetVectorIndexTypeConfigurationSource(this IConventionIndex property)
6363
=> property.FindAnnotation(CosmosAnnotationNames.VectorIndexType)?.GetConfigurationSource();
64+
65+
/// <summary>
66+
/// Returns the value indicating whether the index is configured for Full-text search.
67+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
68+
/// </summary>
69+
/// <param name="index">The index.</param>
70+
/// <returns>The index type to use, or <see langword="null" /> if none is set.</returns>
71+
public static bool? IsFullTextIndex(this IReadOnlyIndex index)
72+
=> (index is RuntimeIndex)
73+
? throw new InvalidOperationException(CoreStrings.RuntimeModelMissingData)
74+
: (bool?)index[CosmosAnnotationNames.FullTextIndex];
75+
76+
/// <summary>
77+
/// Configures the index for Full-text search.
78+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
79+
/// </summary>
80+
/// <param name="index">The index.</param>
81+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
82+
public static void SetFullTextIndex(this IMutableIndex index, bool? value)
83+
=> index.SetAnnotation(CosmosAnnotationNames.FullTextIndex, value);
84+
85+
/// <summary>
86+
/// Configures the index for Full-text search.
87+
/// See <see href="https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search">Full-text search in Azure Cosmos DB for NoSQL</see> for more information.
88+
/// </summary>
89+
/// <param name="index">The index.</param>
90+
/// <param name="value">The value indicating whether the index is configured for Full-text search.</param>
91+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
92+
/// <returns>The configured value.</returns>
93+
public static string? SetFullTextIndex(
94+
this IConventionIndex index,
95+
bool? value,
96+
bool fromDataAnnotation = false)
97+
=> (string?)index.SetAnnotation(
98+
CosmosAnnotationNames.FullTextIndex,
99+
value,
100+
fromDataAnnotation)?.Value;
101+
102+
/// <summary>
103+
/// Returns the <see cref="ConfigurationSource" /> for whether the <see cref="IsFullTextIndex" />.
104+
/// </summary>
105+
/// <param name="property">The property.</param>
106+
/// <returns>The <see cref="ConfigurationSource" /> for whether the index is clustered.</returns>
107+
public static ConfigurationSource? IsFullTextIndexConfigurationSource(this IConventionIndex property)
108+
=> property.FindAnnotation(CosmosAnnotationNames.FullTextIndex)?.GetConfigurationSource();
64109
}

Diff for: ‎src/EFCore.Cosmos/Extensions/CosmosPropertyBuilderExtensions.cs

+91
Original file line numberDiff line numberDiff line change
@@ -243,4 +243,95 @@ private static CosmosVectorType CreateVectorType(DistanceFunction distanceFuncti
243243
? new CosmosVectorType(distanceFunction, dimensions)
244244
: throw new ArgumentException(
245245
CoreStrings.InvalidEnumValue(distanceFunction, nameof(distanceFunction), typeof(DistanceFunction)));
246+
247+
/// <summary>
248+
/// Configures the property to be used with a full-text search.
249+
/// </summary>
250+
/// <remarks>
251+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
252+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
253+
/// </remarks>
254+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
255+
/// <param name="language">The language for the full-text search.</param>
256+
/// <returns>The same builder instance so that multiple calls can be chained.</returns>
257+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
258+
public static PropertyBuilder IsFullText(
259+
this PropertyBuilder propertyBuilder,
260+
string language = "en-US")
261+
{
262+
propertyBuilder.Metadata.SetFullTextSearchLanguage(language);
263+
return propertyBuilder;
264+
}
265+
266+
/// <summary>
267+
/// Configures the property to enable Full-text search for Azure Cosmos DB using a specified language.
268+
/// </summary>
269+
/// <remarks>
270+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
271+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
272+
/// </remarks>
273+
/// <remarks>
274+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
275+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
276+
/// </remarks>
277+
/// <typeparam name="TProperty">The type of the property being configured.</typeparam>
278+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
279+
/// <param name="language">The language for the full-text search.</param>
280+
/// <returns>The same builder instance so that multiple calls can be chained.</returns>
281+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
282+
public static PropertyBuilder<TProperty> IsFullText<TProperty>(
283+
this PropertyBuilder<TProperty> propertyBuilder,
284+
string language = "en-US")
285+
=> (PropertyBuilder<TProperty>)IsFullText((PropertyBuilder)propertyBuilder, language);
286+
287+
/// <summary>
288+
/// Configures the property to enable Full-text search for Azure Cosmos DB using a specified language.
289+
/// </summary>
290+
/// <remarks>
291+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
292+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
293+
/// </remarks>
294+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
295+
/// <param name="language">The language for the full-text search.</param>
296+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
297+
/// <returns>
298+
/// The same builder instance if the configuration was applied,
299+
/// <see langword="null" /> otherwise.
300+
/// </returns>
301+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
302+
public static IConventionPropertyBuilder? IsFullText(
303+
this IConventionPropertyBuilder propertyBuilder,
304+
string language,
305+
bool fromDataAnnotation = false)
306+
{
307+
if (!propertyBuilder.CanSetIsFullText(language, fromDataAnnotation))
308+
{
309+
return null;
310+
}
311+
312+
propertyBuilder.Metadata.SetFullTextSearchLanguage(language, fromDataAnnotation);
313+
314+
return propertyBuilder;
315+
}
316+
317+
/// <summary>
318+
/// Returns a value indicating whether the Full-text search language can be set.
319+
/// </summary>
320+
/// <remarks>
321+
/// See <see href="https://aka.ms/efcore-docs-modeling">Modeling entity types and relationships</see>, and
322+
/// <see href="https://aka.ms/efcore-docs-cosmos">Accessing Azure Cosmos DB with EF Core</see> for more information and examples.
323+
/// </remarks>
324+
/// <param name="propertyBuilder">The builder for the property being configured.</param>
325+
/// <param name="language">The language for the full-text search.</param>
326+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
327+
/// <returns><see langword="true" /> if the vector type can be set.</returns>
328+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
329+
public static bool CanSetIsFullText(
330+
this IConventionPropertyBuilder propertyBuilder,
331+
string language,
332+
bool fromDataAnnotation = false)
333+
=> propertyBuilder.CanSetAnnotation(
334+
CosmosAnnotationNames.FullTextSearchLanguage,
335+
language,
336+
fromDataAnnotation);
246337
}

Diff for: ‎src/EFCore.Cosmos/Extensions/CosmosPropertyExtensions.cs

+46
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,50 @@ public static void SetVectorType(this IMutableProperty property, CosmosVectorTyp
128128
[Experimental(EFDiagnostics.CosmosVectorSearchExperimental)]
129129
public static ConfigurationSource? GetVectorTypeConfigurationSource(this IConventionProperty property)
130130
=> property.FindAnnotation(CosmosAnnotationNames.VectorType)?.GetConfigurationSource();
131+
132+
/// <summary>
133+
/// Returns the full-text search language defined for this property.
134+
/// </summary>
135+
/// <param name="property">The property.</param>
136+
/// <returns>Returns the definition of the vector stored in this property.</returns>
137+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
138+
public static string? GetFullTextSearchLanguage(this IReadOnlyProperty property)
139+
=> (string?)property[CosmosAnnotationNames.FullTextSearchLanguage];
140+
141+
/// <summary>
142+
/// Sets the full-text search language defined for this property.
143+
/// </summary>
144+
/// <param name="property">The property.</param>
145+
/// <param name="language">The full-text search language for the property.</param>
146+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
147+
public static void SetFullTextSearchLanguage(this IMutableProperty property, string? language)
148+
=> property.SetOrRemoveAnnotation(CosmosAnnotationNames.FullTextSearchLanguage, language);
149+
150+
/// <summary>
151+
/// Sets the definition of the vector stored in this property.
152+
/// </summary>
153+
/// <param name="property">The property.</param>
154+
/// <param name="language">The full-text search language for the property.</param>
155+
/// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
156+
/// <returns>The configured value.</returns>
157+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
158+
public static string? SetFullTextSearchLanguage(
159+
this IConventionProperty property,
160+
string? language,
161+
bool fromDataAnnotation = false)
162+
=> (string?)property.SetOrRemoveAnnotation(
163+
CosmosAnnotationNames.FullTextSearchLanguage,
164+
language,
165+
fromDataAnnotation)?.Value;
166+
167+
/// <summary>
168+
/// Gets the <see cref="ConfigurationSource" /> for the definition of the full-text search language for this property.
169+
/// </summary>
170+
/// <param name="property">The property.</param>
171+
/// <returns>
172+
/// The <see cref="ConfigurationSource" /> for the definition of full-text-search language for this property.
173+
/// </returns>
174+
[Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
175+
public static ConfigurationSource? GetFullTextSearchLanguageConfigurationSource(this IConventionProperty property)
176+
=> property.FindAnnotation(CosmosAnnotationNames.FullTextSearchLanguage)?.GetConfigurationSource();
131177
}

Diff for: ‎src/EFCore.Cosmos/Infrastructure/Internal/CosmosModelValidator.cs

+18
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,24 @@ protected virtual void ValidateIndexes(
570570
index.Properties[0].Name));
571571
}
572572
}
573+
else if (index.FindAnnotation(CosmosAnnotationNames.FullTextIndex) != null)
574+
{
575+
if (index.Properties.Count > 1)
576+
{
577+
throw new InvalidOperationException(
578+
CosmosStrings.CompositeFullTextIndex(
579+
entityType.DisplayName(),
580+
string.Join(",", index.Properties.Select(e => e.Name))));
581+
}
582+
583+
if (index.Properties[0].FindAnnotation(CosmosAnnotationNames.FullTextSearchLanguage) == null)
584+
{
585+
throw new InvalidOperationException(
586+
CosmosStrings.FullTextIndexOnNonFullTextProperty(
587+
entityType.DisplayName(),
588+
index.Properties[0].Name));
589+
}
590+
}
573591
else
574592
{
575593
throw new InvalidOperationException(

0 commit comments

Comments
 (0)