.Net: Vector store abstractions hybrid search ADR

dotnet/Directory.Packages.props

@@ -120,7 +120,7 @@
     <PackageVersion Include="YamlDotNet" Version="15.3.0" />
     <PackageVersion Include="Fluid.Core" Version="2.11.1" />
     <!-- Memory stores -->
-    <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.45.2" />
+    <PackageVersion Include="Microsoft.Azure.Cosmos" Version="3.48.0-preview.0" />
     <PackageVersion Include="Pgvector" Version="0.2.0" />
     <PackageVersion Include="NRedisStack" Version="0.12.0" />
     <PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />

dotnet/src/Connectors/Connectors.AzureCosmosDBMongoDB.UnitTests/AzureCosmosDBMongoDBVectorStoreRecordCollectionTests.cs

@@ -568,8 +568,6 @@ public async Task VectorizedSearchThrowsExceptionWithInvalidVectorTypeAsync(obje
     }
 
     [Theory]
-    [InlineData(null, "TestEmbedding1", 1, 1)]
-    [InlineData("", "TestEmbedding1", 2, 2)]
     [InlineData("TestEmbedding1", "TestEmbedding1", 3, 3)]
     [InlineData("TestEmbedding2", "test_embedding_2", 4, 4)]
     public async Task VectorizedSearchUsesValidQueryAsync(

dotnet/src/Connectors/Connectors.AzureCosmosDBNoSQL.UnitTests/AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilderTests.cs

@@ -35,16 +35,18 @@ public void BuildSearchQueryByDefaultReturnsValidQueryDefinition()
             .EqualTo("TestProperty2", "test-value-2")
             .AnyTagEqualTo("TestProperty3", "test-value-3");
 
-        var searchOptions = new VectorSearchOptions { Filter = filter, Skip = 5, Top = 10 };
-
         // Act
         var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
             vector,
+            null,
             fields,
             this._storagePropertyNames,
             vectorPropertyName,
+            null,
             ScorePropertyName,
-            searchOptions);
+            filter,
+            10,
+            5);
 
         var queryText = queryDefinition.QueryText;
         var queryParameters = queryDefinition.GetQueryParameters();
@@ -54,22 +56,16 @@ public void BuildSearchQueryByDefaultReturnsValidQueryDefinition()
         Assert.Contains("FROM x", queryText);
         Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
         Assert.Contains("ORDER BY VectorDistance(x.test_property_1, @vector)", queryText);
-        Assert.Contains("OFFSET @offset LIMIT @limit", queryText);
+        Assert.Contains("OFFSET 5 LIMIT 10", queryText);
 
         Assert.Equal("@vector", queryParameters[0].Name);
         Assert.Equal(vector, queryParameters[0].Value);
 
-        Assert.Equal("@offset", queryParameters[1].Name);
-        Assert.Equal(5, queryParameters[1].Value);
-
-        Assert.Equal("@limit", queryParameters[2].Name);
-        Assert.Equal(10, queryParameters[2].Value);
-
-        Assert.Equal("@cv0", queryParameters[3].Name);
-        Assert.Equal("test-value-2", queryParameters[3].Value);
+        Assert.Equal("@cv0", queryParameters[1].Name);
+        Assert.Equal("test-value-2", queryParameters[1].Value);
 
-        Assert.Equal("@cv1", queryParameters[4].Name);
-        Assert.Equal("test-value-3", queryParameters[4].Value);
+        Assert.Equal("@cv1", queryParameters[2].Name);
+        Assert.Equal("test-value-3", queryParameters[2].Value);
     }
 
     [Fact]
@@ -84,39 +80,38 @@ public void BuildSearchQueryWithoutOffsetReturnsQueryDefinitionWithTopParameter(
             .EqualTo("TestProperty2", "test-value-2")
             .AnyTagEqualTo("TestProperty3", "test-value-3");
 
-        var searchOptions = new VectorSearchOptions { Filter = filter, Top = 10 };
-
         // Act
         var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
             vector,
+            null,
             fields,
             this._storagePropertyNames,
             vectorPropertyName,
+            null,
             ScorePropertyName,
-            searchOptions);
+            filter,
+            10,
+            0);
 
         var queryText = queryDefinition.QueryText;
         var queryParameters = queryDefinition.GetQueryParameters();
 
         // Assert
-        Assert.Contains("SELECT TOP @top x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
+        Assert.Contains("SELECT TOP 10 x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
         Assert.Contains("FROM x", queryText);
         Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
         Assert.Contains("ORDER BY VectorDistance(x.test_property_1, @vector)", queryText);
 
-        Assert.DoesNotContain("OFFSET @offset LIMIT @limit", queryText);
+        Assert.DoesNotContain("OFFSET 0 LIMIT 10", queryText);
 
         Assert.Equal("@vector", queryParameters[0].Name);
         Assert.Equal(vector, queryParameters[0].Value);
 
-        Assert.Equal("@top", queryParameters[1].Name);
-        Assert.Equal(10, queryParameters[1].Value);
-
-        Assert.Equal("@cv0", queryParameters[2].Name);
-        Assert.Equal("test-value-2", queryParameters[2].Value);
+        Assert.Equal("@cv0", queryParameters[1].Name);
+        Assert.Equal("test-value-2", queryParameters[1].Value);
 
-        Assert.Equal("@cv1", queryParameters[3].Name);
-        Assert.Equal("test-value-3", queryParameters[3].Value);
+        Assert.Equal("@cv1", queryParameters[2].Name);
+        Assert.Equal("test-value-3", queryParameters[2].Value);
     }
 
     [Fact]
@@ -129,17 +124,19 @@ public void BuildSearchQueryWithInvalidFilterThrowsException()
 
         var filter = new VectorSearchFilter().EqualTo("non-existent-property", "test-value-2");
 
-        var searchOptions = new VectorSearchOptions { Filter = filter, Skip = 5, Top = 10 };
-
         // Act & Assert
         Assert.Throws<InvalidOperationException>(() =>
             AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
                 vector,
+                null,
                 fields,
                 this._storagePropertyNames,
                 vectorPropertyName,
+                null,
                 ScorePropertyName,
-                searchOptions));
+                filter,
+                10,
+                5));
     }
 
     [Fact]
@@ -150,31 +147,28 @@ public void BuildSearchQueryWithoutFilterDoesNotContainWhereClause()
         var vectorPropertyName = "test_property_1";
         var fields = this._storagePropertyNames.Values.ToList();
 
-        var searchOptions = new VectorSearchOptions { Skip = 5, Top = 10 };
-
         // Act
         var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
             vector,
+            null,
             fields,
             this._storagePropertyNames,
             vectorPropertyName,
+            null,
             ScorePropertyName,
-            searchOptions);
+            null,
+            10,
+            5);
 
         var queryText = queryDefinition.QueryText;
         var queryParameters = queryDefinition.GetQueryParameters();
 
         // Assert
         Assert.DoesNotContain("WHERE", queryText);
+        Assert.Contains("OFFSET 5 LIMIT 10", queryText);
 
         Assert.Equal("@vector", queryParameters[0].Name);
         Assert.Equal(vector, queryParameters[0].Value);
-
-        Assert.Equal("@offset", queryParameters[1].Name);
-        Assert.Equal(5, queryParameters[1].Value);
-
-        Assert.Equal("@limit", queryParameters[2].Name);
-        Assert.Equal(10, queryParameters[2].Value);
     }
 
     [Fact]
@@ -211,4 +205,51 @@ public void BuildSelectQueryByDefaultReturnsValidQueryDefinition()
         Assert.Equal("@pk0", queryParameters[1].Name);
         Assert.Equal("partition_key", queryParameters[1].Value);
     }
+
+    [Fact]
+    public void BuildSearchQueryWithHybridFieldsReturnsValidHybridQueryDefinition()
+    {
+        // Arrange
+        var vector = new ReadOnlyMemory<float>([1f, 2f, 3f]);
+        var keywordText = "hybrid";
+        var vectorPropertyName = "test_property_1";
+        var textPropertyName = "test_property_2";
+        var fields = this._storagePropertyNames.Values.ToList();
+
+        var filter = new VectorSearchFilter()
+            .EqualTo("TestProperty2", "test-value-2")
+            .AnyTagEqualTo("TestProperty3", "test-value-3");
+
+        // Act
+        var queryDefinition = AzureCosmosDBNoSQLVectorStoreCollectionQueryBuilder.BuildSearchQuery(
+            vector,
+            [keywordText],
+            fields,
+            this._storagePropertyNames,
+            vectorPropertyName,
+            textPropertyName,
+            ScorePropertyName,
+            filter,
+            10,
+            5);
+
+        var queryText = queryDefinition.QueryText;
+        var queryParameters = queryDefinition.GetQueryParameters();
+
+        // Assert
+        Assert.Contains("SELECT x.test_property_1,x.test_property_2,x.test_property_3,VectorDistance(x.test_property_1, @vector) AS TestScore", queryText);
+        Assert.Contains("FROM x", queryText);
+        Assert.Contains("WHERE x.test_property_2 = @cv0 AND ARRAY_CONTAINS(x.test_property_3, @cv1)", queryText);
+        Assert.Contains("ORDER BY RANK RRF(VectorDistance(x.test_property_1, @vector), FullTextScore(x.test_property_2, [\"hybrid\"]))", queryText);
+        Assert.Contains("OFFSET 5 LIMIT 10", queryText);
+
+        Assert.Equal("@vector", queryParameters[0].Name);
+        Assert.Equal(vector, queryParameters[0].Value);
+
+        Assert.Equal("@cv0", queryParameters[1].Name);
+        Assert.Equal("test-value-2", queryParameters[1].Value);
+
+        Assert.Equal("@cv1", queryParameters[2].Name);
+        Assert.Equal("test-value-3", queryParameters[2].Value);
+    }
 }

dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs

@@ -23,7 +23,10 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch;
 /// </summary>
 /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
 #pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public sealed class AzureAISearchVectorStoreRecordCollection<TRecord> : IVectorStoreRecordCollection<string, TRecord>, IVectorizableTextSearch<TRecord>
+public sealed class AzureAISearchVectorStoreRecordCollection<TRecord> :
+    IVectorStoreRecordCollection<string, TRecord>,
+    IVectorizableTextSearch<TRecord>,
+    IKeywordHybridSearch<TRecord>
 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix
 {
     /// <summary>The name of this database for telemetry purposes.</summary>
@@ -68,6 +71,9 @@ public sealed class AzureAISearchVectorStoreRecordCollection<TRecord> : IVectorS
     /// <summary>The default options for vector search.</summary>
     private static readonly VectorData.VectorSearchOptions s_defaultVectorSearchOptions = new();
 
+    /// <summary>The default options for hybrid vector search.</summary>
+    private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new();
+
     /// <summary>Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service.</summary>
     private readonly SearchIndexClient _searchIndexClient;
 
@@ -316,25 +322,16 @@ public async IAsyncEnumerable<string> UpsertBatchAsync(IEnumerable<TRecord> reco
     /// <inheritdoc />
     public Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(TVector vector, VectorData.VectorSearchOptions? options = null, CancellationToken cancellationToken = default)
     {
-        Verify.NotNull(vector);
-
-        if (this._propertyReader.FirstVectorPropertyName is null)
-        {
-            throw new InvalidOperationException("The collection does not have any vector fields, so vector search is not possible.");
-        }
-
-        if (vector is not ReadOnlyMemory<float> floatVector)
-        {
-            throw new NotSupportedException($"The provided vector type {vector.GetType().FullName} is not supported by the Azure AI Search connector.");
-        }
+        var floatVector = VerifyVectorParam(vector);
 
         // Resolve options.
         var internalOptions = options ?? s_defaultVectorSearchOptions;
-        string? vectorFieldName = this.ResolveVectorFieldName(internalOptions.VectorPropertyName);
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrSingle(internalOptions.VectorPropertyName);
+        var vectorPropertyName = this._propertyReader.GetJsonPropertyName(vectorProperty!.DataModelPropertyName);
 
         // Configure search settings.
         var vectorQueries = new List<VectorQuery>();
-        vectorQueries.Add(new VectorizedQuery(floatVector) { KNearestNeighborsCount = internalOptions.Top, Fields = { vectorFieldName } });
+        vectorQueries.Add(new VectorizedQuery(floatVector) { KNearestNeighborsCount = internalOptions.Top, Fields = { vectorPropertyName } });
         var filterString = AzureAISearchVectorStoreCollectionSearchMapping.BuildFilterString(internalOptions.Filter, this._propertyReader.JsonPropertyNamesMap);
 
         // Build search options.
@@ -370,11 +367,12 @@ public Task<VectorSearchResults<TRecord>> VectorizableTextSearchAsync(string sea
 
         // Resolve options.
         var internalOptions = options ?? s_defaultVectorSearchOptions;
-        string? vectorFieldName = this.ResolveVectorFieldName(internalOptions.VectorPropertyName);
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrSingle(internalOptions.VectorPropertyName);
+        var vectorPropertyName = this._propertyReader.GetJsonPropertyName(vectorProperty!.DataModelPropertyName);
 
         // Configure search settings.
         var vectorQueries = new List<VectorQuery>();
-        vectorQueries.Add(new VectorizableTextQuery(searchText) { KNearestNeighborsCount = internalOptions.Top, Fields = { vectorFieldName } });
+        vectorQueries.Add(new VectorizableTextQuery(searchText) { KNearestNeighborsCount = internalOptions.Top, Fields = { vectorPropertyName } });
         var filterString = AzureAISearchVectorStoreCollectionSearchMapping.BuildFilterString(internalOptions.Filter, this._propertyReader.JsonPropertyNamesMap);
 
         // Build search options.
@@ -398,6 +396,48 @@ public Task<VectorSearchResults<TRecord>> VectorizableTextSearchAsync(string sea
         return this.SearchAndMapToDataModelAsync(null, searchOptions, internalOptions.IncludeVectors, cancellationToken);
     }
 
+    /// <inheritdoc />
+    public Task<VectorSearchResults<TRecord>> HybridSearchAsync<TVector>(TVector vector, ICollection<string> keywords, HybridSearchOptions? options = null, CancellationToken cancellationToken = default)
+    {
+        Verify.NotNull(keywords);
+        var floatVector = VerifyVectorParam(vector);
+
+        // Resolve options.
+        var internalOptions = options ?? s_defaultKeywordVectorizedHybridSearchOptions;
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrSingle(internalOptions.VectorPropertyName);
+        var vectorPropertyName = this._propertyReader.GetJsonPropertyName(vectorProperty.DataModelPropertyName);
+        var textDataProperty = this._propertyReader.GetFullTextDataPropertyOrSingle(internalOptions.AdditionalPropertyName);
+        var textDataPropertyName = this._propertyReader.GetJsonPropertyName(textDataProperty.DataModelPropertyName);
+
+        // Configure search settings.
+        var vectorQueries = new List<VectorQuery>();
+        vectorQueries.Add(new VectorizedQuery(floatVector) { KNearestNeighborsCount = internalOptions.Top, Fields = { vectorPropertyName } });
+        var filterString = AzureAISearchVectorStoreCollectionSearchMapping.BuildFilterString(internalOptions.Filter, this._propertyReader.JsonPropertyNamesMap);
+
+        // Build search options.
+        var searchOptions = new SearchOptions
+        {
+            VectorSearch = new(),
+            Size = internalOptions.Top,
+            Skip = internalOptions.Skip,
+            Filter = filterString,
+            IncludeTotalCount = internalOptions.IncludeTotalCount,
+        };
+        searchOptions.VectorSearch.Queries.AddRange(vectorQueries);
+        searchOptions.SearchFields.Add(textDataPropertyName);
+
+        // Filter out vector fields if requested.
+        if (!internalOptions.IncludeVectors)
+        {
+            searchOptions.Select.Add(this._propertyReader.KeyPropertyJsonName);
+            searchOptions.Select.AddRange(this._propertyReader.DataPropertyJsonNames);
+        }
+
+        var keywordsCombined = string.Join(" ", keywords);
+
+        return this.SearchAndMapToDataModelAsync(keywordsCombined, searchOptions, internalOptions.IncludeVectors, cancellationToken);
+    }
+
     /// <summary>
     /// Get the document with the given key and map it to the data model using the configured mapper type.
     /// </summary>
@@ -556,31 +596,6 @@ private GetDocumentOptions ConvertGetDocumentOptions(GetRecordOptions? options)
         return innerOptions;
     }
 
-    /// <summary>
-    /// Resolve the vector field name to use for a search by using the storage name for the field name from options
-    /// if available, and falling back to the first vector field name if not.
-    /// </summary>
-    /// <param name="optionsVectorFieldName">The vector field name provided via options.</param>
-    /// <returns>The resolved vector field name.</returns>
-    /// <exception cref="InvalidOperationException">Thrown if the provided field name is not a valid field name.</exception>
-    private string ResolveVectorFieldName(string? optionsVectorFieldName)
-    {
-        string? vectorFieldName;
-        if (!string.IsNullOrWhiteSpace(optionsVectorFieldName))
-        {
-            if (!this._propertyReader.JsonPropertyNamesMap.TryGetValue(optionsVectorFieldName!, out vectorFieldName))
-            {
-                throw new InvalidOperationException($"The collection does not have a vector field named '{optionsVectorFieldName}'.");
-            }
-        }
-        else
-        {
-            vectorFieldName = this._propertyReader.FirstVectorPropertyJsonName;
-        }
-
-        return vectorFieldName!;
-    }
-
     /// <summary>
     /// Get a document with the given key, and return null if it is not found.
     /// </summary>
@@ -638,4 +653,16 @@ private async Task<T> RunOperationAsync<T>(string operationName, Func<Task<T>> o
             };
         }
     }
+
+    private static ReadOnlyMemory<float> VerifyVectorParam<TVector>(TVector vector)
+    {
+        Verify.NotNull(vector);
+
+        if (vector is not ReadOnlyMemory<float> floatVector)
+        {
+            throw new NotSupportedException($"The provided vector type {vector.GetType().FullName} is not supported by the Azure AI Search connector.");
+        }
+
+        return floatVector;
+    }
 }