[release/9.0-staging] Cosmos Full Text Search support (query part) (#35909)

* Limited port of #35868

Fixes #35476
Fixes #35853 (need to fix this one, otherwise vector translator will try to translate full text methods and fail)

Description
This PR enables full-text search queries using EF Core 9 when targeting Azure Cosmos Db. This is one of the flagship new features for Cosmos and the aim here is to help with it's adoption. This is very limited port of the full feature we are adding in EF 10 Preview 4. We are only adding querying capabilities: function stubs for FullTextContains, FullTextContainsAll, FullTextContainsAny, FullTextScore and RRF as well as logic translating these signatures to built-in Cosmos functions.
No model building or data manipulation - containers need to be created outside EF (using Cosmos SDK or in the Data Explorer).

Customer impact
Customers will be able to use the upcoming full text search capabilities when working with Azure Cosmos Db without the need to upgrade to EF 10 preview.

How found
Partner team ask.

Regression
No

Testing
Extensively tested on EF 10, manual testing on EF9. End-to-end testing is not possible because we can't create containers programmatically (no support for it inside EF Core itself, and the Cosmos SDK which supports it is currently only available in beta, so we can't take dependency on it). Instead, we created containers and data using EF 10, ported all the query tests from EF 10 and ran them using the EF9 bits.

Risk
Low. Code here is purely additive and actually localized to only a handful of places in the code: validation in ApplyOrdering/AppendOrdering, FTS method translator, parameter inliner and sql generator. Feature is marked as experimental and quirks have been added.

* fix FTContainsAll/Any for constants,
change TFScore signature to accept params,
fixed SqlFunctionExpression to use the new ctor internally,
added mandatory property overrides for FragmentExpression (vector search)
fixed parameter inliner to not match parameter of type string, but only string[] when processing ContainsAll/Any

Author: maumar

Diff for: src/EFCore.Analyzers/EFDiagnostics.cs

@@ -19,4 +19,5 @@ public static class EFDiagnostics
     public const string MetricsExperimental = "EF9101";
     public const string PagingExperimental = "EF9102";
     public const string CosmosVectorSearchExperimental = "EF9103";
+    public const string CosmosFullTextSearchExperimental = "EF9104";
 }

Diff for: src/EFCore.Cosmos/EFCore.Cosmos.csproj

@@ -12,6 +12,7 @@
     <NoWarn>$(NoWarn);EF9101</NoWarn> <!-- Metrics is experimental -->
     <NoWarn>$(NoWarn);EF9102</NoWarn> <!-- Paging is experimental -->
     <NoWarn>$(NoWarn);EF9103</NoWarn> <!-- Vector search is experimental -->
+    <NoWarn>$(NoWarn);EF9104</NoWarn> <!-- Full-text search is experimental -->
   </PropertyGroup>
 
   <ItemGroup>

Diff for: src/EFCore.Cosmos/Extensions/CosmosDbFunctionsExtensions.cs

@@ -52,6 +52,60 @@ public static T CoalesceUndefined<T>(
         T expression2)
         => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(CoalesceUndefined)));
 
+    /// <summary>
+    ///     Checks if the specified property contains the given keyword using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keyword">The keyword to search for.</param>
+    /// <returns><see langword="true" /> if the property contains the keyword; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContains(this DbFunctions _, string property, string keyword)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContains)));
+
+    /// <summary>
+    ///     Checks if the specified property contains all the given keywords using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keywords">The keywords to search for.</param>
+    /// <returns><see langword="true" /> if the property contains all the keywords; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContainsAll(this DbFunctions _, string property, params string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAll)));
+
+    /// <summary>
+    ///     Checks if the specified property contains any of the given keywords using full-text search.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to search.</param>
+    /// <param name="keywords">The keywords to search for.</param>
+    /// <returns><see langword="true" /> if the property contains any of the keywords; otherwise, <see langword="false" />.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static bool FullTextContainsAny(this DbFunctions _, string property, params string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextContainsAny)));
+
+    /// <summary>
+    ///     Returns the full-text search score for the specified property and keywords.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="property">The property to score.</param>
+    /// <param name="keywords">The keywords to score by.</param>
+    /// <returns>The full-text search score.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static double FullTextScore(this DbFunctions _, string property, params string[] keywords)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(FullTextScore)));
+
+    /// <summary>
+    ///     Combines scores provided by two or more specified functions.
+    /// </summary>
+    /// <param name="_">The <see cref="DbFunctions" /> instance.</param>
+    /// <param name="functions">The functions to compute the score for.</param>
+    /// <returns>The combined score.</returns>
+    [Experimental(EFDiagnostics.CosmosFullTextSearchExperimental)]
+    public static double Rrf(this DbFunctions _, params double[] functions)
+        => throw new InvalidOperationException(CoreStrings.FunctionOnClient(nameof(Rrf)));
+
     /// <summary>
     ///     Returns the distance between two vectors, using the distance function and data type defined using
     ///     <see

Diff for: src/EFCore.Cosmos/Properties/CosmosStrings.Designer.cs

@@ -283,6 +283,15 @@
   <data name="OneOfTwoValuesMustBeSet" xml:space="preserve">
     <value>Exactly one of '{param1}' or '{param2}' must be set.</value>
   </data>
+  <data name="OrderByDescendingScoringFunction" xml:space="preserve">
+    <value>Ordering based on scoring function is not supported inside '{orderByDescending}'. Use '{orderBy}' instead.</value>
+  </data>
+  <data name="OrderByMultipleScoringFunctionWithoutRrf" xml:space="preserve">
+    <value>Only one ordering using scoring function is allowed. Use 'EF.Functions.{rrf}' method to combine multiple scoring functions.</value>
+  </data>
+  <data name="OrderByScoringFunctionMixedWithRegularOrderby" xml:space="preserve">
+    <value>Ordering using a scoring function is mutually exclusive with other forms of ordering.</value>
+  </data>
   <data name="OrphanedNestedDocument" xml:space="preserve">
     <value>The entity of type '{entityType}' is mapped as a part of the document mapped to '{missingEntityType}', but there is no tracked entity of this type with the corresponding key value. Consider using 'DbContextOptionsBuilder.EnableSensitiveDataLogging' to see the key values.</value>
   </data>

Diff for: src/EFCore.Cosmos/Properties/CosmosStrings.resx

@@ -36,7 +36,8 @@ public CosmosMethodCallTranslatorProvider(
             new CosmosRegexTranslator(sqlExpressionFactory),
             new CosmosStringMethodTranslator(sqlExpressionFactory),
             new CosmosTypeCheckingTranslator(sqlExpressionFactory),
-            new CosmosVectorSearchTranslator(sqlExpressionFactory, typeMappingSource)
+            new CosmosVectorSearchTranslator(sqlExpressionFactory, typeMappingSource),
+            new CosmosFullTextSearchTranslator(sqlExpressionFactory, typeMappingSource)
             //new LikeTranslator(sqlExpressionFactory),
             //new EnumHasFlagTranslator(sqlExpressionFactory),
             //new GetValueOrDefaultTranslator(sqlExpressionFactory),

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosMethodCallTranslatorProvider.cs

@@ -14,6 +14,9 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
 /// </summary>
 public class CosmosQuerySqlGenerator(ITypeMappingSource typeMappingSource) : SqlExpressionVisitor
 {
+    private static readonly bool UseOldBehavior35476 =
+          AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue35476", out var enabled35476) && enabled35476;
+
     private readonly IndentedStringBuilder _sqlBuilder = new();
     private IReadOnlyDictionary<string, object> _parameterValues = null!;
     private List<SqlParameter> _sqlParameters = null!;
@@ -341,6 +344,15 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
         {
             _sqlBuilder.AppendLine().Append("ORDER BY ");
 
+            var orderByScoringFunction = selectExpression.Orderings is [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }];
+            if (!UseOldBehavior35476 && orderByScoringFunction)
+            {
+                _sqlBuilder.Append("RANK ");
+            }
+
+            Check.DebugAssert(UseOldBehavior35476 || orderByScoringFunction || selectExpression.Orderings.All(x => x.Expression is not SqlFunctionExpression { IsScoringFunction: true }),
+                "Scoring function can only appear as first (and only) ordering, or not at all.");
+
             GenerateList(selectExpression.Orderings, e => Visit(e));
         }
 

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosQuerySqlGenerator.cs

@@ -9,48 +9,129 @@ namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
 
 public partial class CosmosShapedQueryCompilingExpressionVisitor
 {
-    private sealed class InExpressionValuesExpandingExpressionVisitor(
+    private static readonly bool UseOldBehavior35476 =
+          AppContext.TryGetSwitch("Microsoft.EntityFrameworkCore.Issue35476", out var enabled35476) && enabled35476;
+
+    private sealed class ParameterInliner(
         ISqlExpressionFactory sqlExpressionFactory,
         IReadOnlyDictionary<string, object> parametersValues)
         : ExpressionVisitor
     {
         protected override Expression VisitExtension(Expression expression)
         {
-            if (expression is InExpression inExpression)
+            if (!UseOldBehavior35476)
             {
-                IReadOnlyList<SqlExpression> values;
+                expression = base.VisitExtension(expression);
+            }
 
-                switch (inExpression)
+            switch (expression)
+            {
+                // Inlines array parameter of InExpression, transforming: 'item IN (@valuesArray)' to: 'item IN (value1, value2)'
+                case InExpression inExpression:
                 {
-                    case { Values: IReadOnlyList<SqlExpression> values2 }:
-                        values = values2;
-                        break;
-
-                    // TODO: IN with subquery (return immediately, nothing to do here)
+                    IReadOnlyList<SqlExpression> values;
 
-                    case { ValuesParameter: SqlParameterExpression valuesParameter }:
+                    switch (inExpression)
                     {
-                        var typeMapping = valuesParameter.TypeMapping;
-                        var mutableValues = new List<SqlExpression>();
-                        foreach (var value in (IEnumerable)parametersValues[valuesParameter.Name])
+                        case { Values: IReadOnlyList<SqlExpression> values2 }:
+                            values = values2;
+                            break;
+
+                        // TODO: IN with subquery (return immediately, nothing to do here)
+
+                        case { ValuesParameter: SqlParameterExpression valuesParameter }:
                         {
-                            mutableValues.Add(sqlExpressionFactory.Constant(value, value?.GetType() ?? typeof(object), typeMapping));
+                            var typeMapping = valuesParameter.TypeMapping;
+                            var mutableValues = new List<SqlExpression>();
+                            foreach (var value in (IEnumerable)parametersValues[valuesParameter.Name])
+                            {
+                                mutableValues.Add(sqlExpressionFactory.Constant(value, value?.GetType() ?? typeof(object), typeMapping));
+                            }
+
+                            values = mutableValues;
+                            break;
                         }
 
-                        values = mutableValues;
-                        break;
+                        default:
+                            throw new UnreachableException();
                     }
 
-                    default:
-                        throw new UnreachableException();
+                    return values.Count == 0
+                        ? sqlExpressionFactory.ApplyDefaultTypeMapping(sqlExpressionFactory.Constant(false))
+                        : sqlExpressionFactory.In((SqlExpression)Visit(inExpression.Item), values);
                 }
 
-                return values.Count == 0
-                    ? sqlExpressionFactory.ApplyDefaultTypeMapping(sqlExpressionFactory.Constant(false))
-                    : sqlExpressionFactory.In((SqlExpression)Visit(inExpression.Item), values);
-            }
+                // Converts Offset and Limit parameters to constants when ORDER BY RANK is detected in the SelectExpression (i.e. we order by scoring function)
+                // Cosmos only supports constants in Offset and Limit for this scenario currently (ORDER BY RANK limitation)
+                case SelectExpression { Orderings: [{ Expression: SqlFunctionExpression { IsScoringFunction: true } }], Limit: var limit, Offset: var offset } hybridSearch
+                    when !UseOldBehavior35476 && (limit is SqlParameterExpression || offset is SqlParameterExpression):
+                {
+                    if (hybridSearch.Limit is SqlParameterExpression limitPrm)
+                    {
+                        hybridSearch.ApplyLimit(
+                            sqlExpressionFactory.Constant(
+                                parametersValues[limitPrm.Name],
+                                limitPrm.TypeMapping));
+                    }
+
+                    if (hybridSearch.Offset is SqlParameterExpression offsetPrm)
+                    {
+                        hybridSearch.ApplyOffset(
+                            sqlExpressionFactory.Constant(
+                                parametersValues[offsetPrm.Name],
+                                offsetPrm.TypeMapping));
+                    }
+
+                    return base.VisitExtension(expression);
+                }
 
-            return base.VisitExtension(expression);
+                // Inlines array parameter of full-text functions, transforming FullTextContainsAll(x, @keywordsArray) to FullTextContainsAll(x, keyword1, keyword2)) 
+                case SqlFunctionExpression
+                {
+                    Name: "FullTextContainsAny" or "FullTextContainsAll",
+                    Arguments: [var property, SqlParameterExpression { TypeMapping: { ElementTypeMapping: var elementTypeMapping }, Type: Type type } keywords]
+                } fullTextContainsAllAnyFunction
+                when !UseOldBehavior35476 && type == typeof(string[]):
+                {
+                    var keywordValues = new List<SqlExpression>();
+                    foreach (var value in (IEnumerable)parametersValues[keywords.Name])
+                    {
+                        keywordValues.Add(sqlExpressionFactory.Constant(value, typeof(string), elementTypeMapping));
+                    }
+
+                    return sqlExpressionFactory.Function(
+                        fullTextContainsAllAnyFunction.Name,
+                        [property, .. keywordValues],
+                        fullTextContainsAllAnyFunction.Type,
+                        fullTextContainsAllAnyFunction.TypeMapping);
+                }
+
+                // Inlines array parameter of full-text score, transforming FullTextScore(x, @keywordsArray) to FullTextScore(x, [keyword1, keyword2])) 
+                case SqlFunctionExpression
+                {
+                    Name: "FullTextScore",
+                    IsScoringFunction: true,
+                    Arguments: [var property, SqlParameterExpression { TypeMapping: { ElementTypeMapping: not null } typeMapping } keywords]
+                } fullTextScoreFunction
+                when !UseOldBehavior35476:
+                {
+                    var keywordValues = new List<string>();
+                    foreach (var value in (IEnumerable)parametersValues[keywords.Name])
+                    {
+                        keywordValues.Add((string)value);
+                    }
+
+                    return new SqlFunctionExpression(
+                        fullTextScoreFunction.Name,
+                        isScoringFunction: true,
+                        [property, sqlExpressionFactory.Constant(keywordValues, typeMapping)],
+                        fullTextScoreFunction.Type,
+                        fullTextScoreFunction.TypeMapping);
+                }
+
+                default:
+                    return expression;
+            }
         }
     }
 }

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.InExpressionValuesExpandingExpressionVisitor.cs

@@ -75,7 +75,7 @@ public IAsyncEnumerator<CosmosPage<T>> GetAsyncEnumerator(CancellationToken canc
 
         private CosmosSqlQuery GenerateQuery()
             => _querySqlGeneratorFactory.Create().GetSqlQuery(
-                (SelectExpression)new InExpressionValuesExpandingExpressionVisitor(
+                (SelectExpression)new ParameterInliner(
                         _sqlExpressionFactory,
                         _cosmosQueryContext.ParameterValues)
                     .Visit(_selectExpression),

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.PagingQueryingEnumerable.cs

@@ -71,7 +71,7 @@ IEnumerator IEnumerable.GetEnumerator()
 
         private CosmosSqlQuery GenerateQuery()
             => _querySqlGeneratorFactory.Create().GetSqlQuery(
-                (SelectExpression)new InExpressionValuesExpandingExpressionVisitor(
+                (SelectExpression)new ParameterInliner(
                         _sqlExpressionFactory,
                         _cosmosQueryContext.ParameterValues)
                     .Visit(_selectExpression),

Diff for: src/EFCore.Cosmos/Query/Internal/CosmosShapedQueryCompilingExpressionVisitor.QueryingEnumerable.cs

@@ -23,6 +23,14 @@ public class FragmentExpression(string fragment) : Expression, IPrintableExpress
     /// </summary>
     public virtual string Fragment { get; } = fragment;
 
+    /// <inheritdoc />
+    public override ExpressionType NodeType
+        => base.NodeType;
+
+    /// <inheritdoc />
+    public override Type Type
+        => typeof(object);
+
     /// <inheritdoc />
     protected override Expression VisitChildren(ExpressionVisitor visitor)
         => this;