From dc05a71de1ef09acc34c5d5f8cb591368fefb847 Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Fri, 25 Oct 2024 08:55:51 -0700 Subject: [PATCH 1/2] Initial commit --- Microsoft.Azure.Cosmos.sln | 6 +- .../src/Fluent/Settings/ContainerBuilder.cs | 5 + .../CosmosQueryExecutionContextFactory.cs | 34 +-- .../Core/QueryPlan/QueryPlanRetriever.cs | 5 +- .../Query/v3Query/CosmosQueryClientCore.cs | 11 +- .../Fluent/ContainerSettingsTests.cs | 4 +- .../Query/HybridRankingQueryTests.cs | 209 ++++++++++++++++++ 7 files changed, 248 insertions(+), 26 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index b1d77052bf..60723d2aa9 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.11.35327.3 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -34,7 +34,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Encr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FaultInjection", "Microsoft.Azure.Cosmos\FaultInjection\src\FaultInjection.csproj", "{021DDC27-02EF-42C4-9A9E-AA600833C2EE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests", "Microsoft.Azure.Cosmos.Encryption.Custom\tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj", "{CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests", "Microsoft.Azure.Cosmos.Encryption.Custom\tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj", "{CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/Microsoft.Azure.Cosmos/src/Fluent/Settings/ContainerBuilder.cs b/Microsoft.Azure.Cosmos/src/Fluent/Settings/ContainerBuilder.cs index c5f3e65536..5cac70f27f 100644 --- a/Microsoft.Azure.Cosmos/src/Fluent/Settings/ContainerBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/Fluent/Settings/ContainerBuilder.cs @@ -270,6 +270,11 @@ public async Task CreateIfNotExistsAsync( containerProperties.VectorEmbeddingPolicy = this.vectorEmbeddingPolicy; } + if (this.fullTextPolicy != null) + { + containerProperties.FullTextPolicy = this.fullTextPolicy; + } + return containerProperties; } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs index fa98aa209e..892190964e 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Pipeline/CosmosQueryExecutionContextFactory.cs @@ -575,7 +575,7 @@ private static async Task GetPartitionedQueryExec CancellationToken cancellationToken) { PartitionedQueryExecutionInfo partitionedQueryExecutionInfo; - if (cosmosQueryContext.QueryClient.BypassQueryParsing()) + //if (true) { // For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop, so need to bypass in that case. // We are also now bypassing this for 32 bit host process running even on Windows as there are many 32 bit apps that will not work without this @@ -588,22 +588,22 @@ private static async Task GetPartitionedQueryExec trace, cancellationToken); } - else - { - Documents.PartitionKeyDefinition partitionKeyDefinition = GetPartitionKeyDefinition(inputParameters, containerQueryProperties); - - partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanWithServiceInteropAsync( - cosmosQueryContext.QueryClient, - inputParameters.SqlQuerySpec, - cosmosQueryContext.ResourceTypeEnum, - partitionKeyDefinition, - containerQueryProperties.VectorEmbeddingPolicy, - inputParameters.PartitionKey != null, - containerQueryProperties.GeospatialType, - cosmosQueryContext.UseSystemPrefix, - trace, - cancellationToken); - } + //else + //{ + // Documents.PartitionKeyDefinition partitionKeyDefinition = GetPartitionKeyDefinition(inputParameters, containerQueryProperties); + + // partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanWithServiceInteropAsync( + // cosmosQueryContext.QueryClient, + // inputParameters.SqlQuerySpec, + // cosmosQueryContext.ResourceTypeEnum, + // partitionKeyDefinition, + // containerQueryProperties.VectorEmbeddingPolicy, + // inputParameters.PartitionKey != null, + // containerQueryProperties.GeospatialType, + // cosmosQueryContext.UseSystemPrefix, + // trace, + // cancellationToken); + //} return partitionedQueryExecutionInfo; } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPlanRetriever.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPlanRetriever.cs index 3b5d620837..c072d129bb 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPlanRetriever.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPlan/QueryPlanRetriever.cs @@ -148,8 +148,9 @@ public static Task GetQueryPlanThroughGatewayAsyn ResourceType.Document, OperationType.QueryPlan, sqlQuerySpec, - partitionKey, - GetSupportedQueryFeaturesString(isNonStreamingOrderByQueryFeatureDisabled), + partitionKey, + // GetSupportedQueryFeaturesString(isNonStreamingOrderByQueryFeatureDisabled), + "Aggregate, Distinct, GroupBy, MultipleAggregates, MultipleOrderBy, OffsetAndLimit, OrderBy, Top, NonValueAggregate, DCount, NonStreamingOrderBy, CountIf, HybridSearch", trace, cancellationToken); } diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs index 770e12a984..ee3da0de9e 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/CosmosQueryClientCore.cs @@ -207,8 +207,15 @@ public override async Task ExecuteQueryPlanReques cancellationToken: cancellationToken)) { // Syntax exception are argument exceptions and thrown to the user. - message.EnsureSuccessStatusCode(); - partitionedQueryExecutionInfo = this.clientContext.SerializerCore.FromStream(message.Content); + message.EnsureSuccessStatusCode(); + + //using (var streamReader = new StreamReader(message.Content)) + //{ + // string result = streamReader.ReadToEnd(); + // Console.WriteLine(result); + //} + //message.Content.Seek(0, SeekOrigin.Begin); + partitionedQueryExecutionInfo = this.clientContext.SerializerCore.FromStream(message.Content); } return partitionedQueryExecutionInfo; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs index ccae6e036c..29ac4929c9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs @@ -683,7 +683,7 @@ await this.database.DefineContainer(containerName, partitionKeyPath) } [TestMethod] - [Ignore("This test will be enabled once the full text search changes are made available into the public emulator.")] + // [Ignore("This test will be enabled once the full text search changes are made available into the public emulator.")] public async Task TestFullTextSearchPolicy() { string fullTextPath1 = "/fts1", fullTextPath2 = "/fts2", fullTextPath3 = "/fts3"; @@ -749,7 +749,7 @@ await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath Assert.AreEqual(fullTextPaths.Count, containerSettings.IndexingPolicy.FullTextIndexes.Count()); Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[0].Path); Assert.AreEqual(fullTextPath2, containerSettings.IndexingPolicy.FullTextIndexes[1].Path); - Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[2].Path); + Assert.AreEqual(fullTextPath3, containerSettings.IndexingPolicy.FullTextIndexes[2].Path); } finally { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs new file mode 100644 index 0000000000..f07b8f3dcf --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs @@ -0,0 +1,209 @@ +namespace Microsoft.Azure.Cosmos.Query +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading.Tasks; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class HybridRankingQueryTests + { + private CosmosClient CreateCosmosClient(bool local) + { + return local ? + new CosmosClient( + "https://localhost:8081", + "") : + new CosmosClient( + "https://hybridranktest.documents.azure.com:443/", + "", + new CosmosClientOptions() { ConnectionMode = ConnectionMode.Gateway }); + } + + [TestMethod] + public async Task CreateCollectionIndexOnly() + { + string fullTextPath1 = "/fts1"; + Database databaseForVectorEmbedding = await this.CreateCosmosClient(local: false).CreateDatabaseAsync("fullTextSearchDB", + cancellationToken: default); + + Collection fullTextPaths = new Collection() + { + new FullTextPath() + { + Path = fullTextPath1, + Language = "en-US", + } + }; + + string containerName = "fullTextContainerTestIndexOnly"; + string partitionKeyPath = "/pk"; + + ContainerResponse containerResponse = + await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath) + .WithIndexingPolicy() + .WithFullTextIndex() + .Path(fullTextPath1) + .Attach() + .Attach() + .CreateAsync(); + + Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode); + Assert.AreEqual(containerName, containerResponse.Resource.Id); + Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First()); + ContainerProperties containerSettings = containerResponse.Resource; + + // Validate FullText Paths. + Assert.IsNotNull(containerSettings.FullTextPolicy); + Assert.IsNotNull(containerSettings.FullTextPolicy.FullTextPaths); + Assert.AreEqual(fullTextPaths.Count, containerSettings.FullTextPolicy.FullTextPaths.Count()); + Assert.IsTrue(fullTextPaths.OrderBy(x => x.Path).SequenceEqual(containerSettings.FullTextPolicy.FullTextPaths.OrderBy(x => x.Path))); + + // Validate Full Text Indexes. + Assert.IsNotNull(containerSettings.IndexingPolicy.FullTextIndexes); + Assert.AreEqual(fullTextPaths.Count, containerSettings.IndexingPolicy.FullTextIndexes.Count()); + Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[0].Path); + } + + [TestMethod] + public async Task CreateCollectionBothPolicyAndIndex() + { + string fullTextPath1 = "/fts1"; + Database databaseForVectorEmbedding = await this.CreateCosmosClient(local: false).CreateDatabaseAsync("fullTextSearchDB", + cancellationToken: default); + + Collection fullTextPaths = new Collection() + { + new FullTextPath() + { + Path = fullTextPath1, + Language = "en-US", + } + }; + + string containerName = "fullTextContainerTestPolicyAndIndex"; + string partitionKeyPath = "/pk"; + + ContainerResponse containerResponse = + await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath) + .WithFullTextPolicy( + defaultLanguage: "en-US", + fullTextPaths: fullTextPaths) + .Attach() + .WithIndexingPolicy() + .WithFullTextIndex() + .Path(fullTextPath1) + .Attach() + .Attach() + .CreateAsync(); + + Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode); + Assert.AreEqual(containerName, containerResponse.Resource.Id); + Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First()); + ContainerProperties containerSettings = containerResponse.Resource; + + // Validate FullText Paths. + Assert.IsNotNull(containerSettings.FullTextPolicy); + Assert.IsNotNull(containerSettings.FullTextPolicy.FullTextPaths); + Assert.AreEqual(fullTextPaths.Count, containerSettings.FullTextPolicy.FullTextPaths.Count()); + Assert.IsTrue(fullTextPaths.OrderBy(x => x.Path).SequenceEqual(containerSettings.FullTextPolicy.FullTextPaths.OrderBy(x => x.Path))); + + // Validate Full Text Indexes. + Assert.IsNotNull(containerSettings.IndexingPolicy.FullTextIndexes); + Assert.AreEqual(fullTextPaths.Count, containerSettings.IndexingPolicy.FullTextIndexes.Count()); + Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[0].Path); + } + + [TestMethod] + public async Task CreateCollectionPolicyOnly() + { + string fullTextPath1 = "/fts1", fullTextPath2 = "/fts2", fullTextPath3 = "/fts3"; + Database databaseForVectorEmbedding = await this.CreateCosmosClient(local: false).CreateDatabaseAsync("fullTextSearchDBPolicyOnly", + cancellationToken: default); + + Collection fullTextPaths = new Collection() + { + new FullTextPath() + { + Path = fullTextPath1, + Language = "en-US", + }, + new FullTextPath() + { + Path = fullTextPath2, + Language = "en-US", + }, + new FullTextPath() + { + Path = fullTextPath3, + Language = "en-US", + }, + }; + + string containerName = "fullTextContainerTestPolicyOnly"; + string partitionKeyPath = "/pk"; + + ContainerResponse containerResponse = + await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath) + .WithFullTextPolicy( + defaultLanguage: "en-US", + fullTextPaths: fullTextPaths) + .Attach() + //.WithIndexingPolicy() + // .WithFullTextIndex() + // .Path(fullTextPath1) + // .Attach() + // .WithFullTextIndex() + // .Path(fullTextPath2) + // .Attach() + // .WithFullTextIndex() + // .Path(fullTextPath3) + // .Attach() + //.Attach() + .CreateAsync(); + + Assert.AreEqual(HttpStatusCode.Created, containerResponse.StatusCode); + Assert.AreEqual(containerName, containerResponse.Resource.Id); + Assert.AreEqual(partitionKeyPath, containerResponse.Resource.PartitionKey.Paths.First()); + ContainerProperties containerSettings = containerResponse.Resource; + + // Validate FullText Paths. + //Assert.IsNotNull(containerSettings.FullTextPolicy); + //Assert.IsNotNull(containerSettings.FullTextPolicy.FullTextPaths); + //Assert.AreEqual(fullTextPaths.Count, containerSettings.FullTextPolicy.FullTextPaths.Count()); + //Assert.IsTrue(fullTextPaths.OrderBy(x => x.Path).SequenceEqual(containerSettings.FullTextPolicy.FullTextPaths.OrderBy(x => x.Path))); + + // Validate Full Text Indexes. + //Assert.IsNotNull(containerSettings.IndexingPolicy.FullTextIndexes); + //Assert.AreEqual(fullTextPaths.Count, containerSettings.IndexingPolicy.FullTextIndexes.Count()); + //Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[0].Path); + //Assert.AreEqual(fullTextPath2, containerSettings.IndexingPolicy.FullTextIndexes[1].Path); + //Assert.AreEqual(fullTextPath3, containerSettings.IndexingPolicy.FullTextIndexes[2].Path); + } + + [TestMethod] + public async Task HybridRankingQuery() + { + CosmosClient client = this.CreateCosmosClient(local: false); + Container container = client.GetContainer("HybridRankTesting", "arxiv-ada2-15properties-1536dimensions-100documents"); + FeedIterator iterator = container.GetItemQueryIterator( + queryText: @"SELECT TOP 10 c.id FROM c ORDER BY RANK FullTextScore(c.text, ['quantum'])" + ); + List results = new(); + while (iterator.HasMoreResults) + { + FeedResponse page = await iterator.ReadNextAsync(); + results.AddRange(page); + } + + foreach (dynamic item in results) + { + Console.WriteLine(item); + } + } + } +} From d42149f0adbdfc01b97f5462e263bcb19e5d91e0 Mon Sep 17 00:00:00 2001 From: "REDMOND\\adityasa" Date: Fri, 1 Nov 2024 11:50:22 -0700 Subject: [PATCH 2/2] update --- .../Query/HybridRankingQueryTests.cs | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs index f07b8f3dcf..5e8137044d 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/HybridRankingQueryTests.cs @@ -14,14 +14,6 @@ public class HybridRankingQueryTests { private CosmosClient CreateCosmosClient(bool local) { - return local ? - new CosmosClient( - "https://localhost:8081", - "") : - new CosmosClient( - "https://hybridranktest.documents.azure.com:443/", - "", - new CosmosClientOptions() { ConnectionMode = ConnectionMode.Gateway }); } [TestMethod] @@ -69,6 +61,32 @@ await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath Assert.AreEqual(fullTextPath1, containerSettings.IndexingPolicy.FullTextIndexes[0].Path); } + [TestMethod] + public async Task CreatePolicyAndIdnexOnExistingContainer() + { + string path = "/abstract"; + CosmosClient client = this.CreateCosmosClient(local: false); + Container container = client.GetContainer("HybridRankTesting", "arxiv-250kdocuments-index"); + ContainerResponse response = await container.ReadContainerAsync(); + ContainerProperties containerProperties = response.Resource; + containerProperties.FullTextPolicy = new FullTextPolicy + { + DefaultLanguage = "en-US", + FullTextPaths = new Collection + { + new FullTextPath + { + Path = path, + Language = "en-US", + } + } + }; + containerProperties.IndexingPolicy.FullTextIndexes.Add(new FullTextIndexPath { Path = path }); + + ContainerResponse containerResponse = await container.ReplaceContainerAsync(containerProperties); + Assert.IsTrue(containerResponse.StatusCode == HttpStatusCode.OK); + } + [TestMethod] public async Task CreateCollectionBothPolicyAndIndex() {