[Thin Client Integration] Benchmark tests: Adds tests for thinclient mode dataplane scenarios.

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs

@@ -29,7 +29,17 @@ public class BenchmarkConfig
 
         [Option('k', Required = true, HelpText = "Cosmos account master key")]
         [JsonIgnore]
-        public string Key { get; set; }
+        public string Key { get; set; }
+
+        [Option(Required = false, HelpText = "ThinClient Cosmos account end point")]
+        public string ThinClientEndPoint { get; set; }
+
+        [Option(Required = false, HelpText = "ThinClient Cosmos account master key")]
+        [JsonIgnore]
+        public string ThinClientKey { get; set; }
+
+        [Option(Required = false, HelpText = "ThinClient enabled")]
+        public bool IsThinClientEnabled { get; set; }
 
         [Option(Required = false, HelpText = "Workload Name, it will override the workloadType value in published results")]
         public string WorkloadName { get; set; }
@@ -209,7 +219,7 @@ private string GetUserAgentPrefix()
             return this.WorkloadName ?? this.WorkloadType ?? BenchmarkConfig.UserAgentSuffix;
         }
 
-        internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKey)
+        internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient()
         {
             // Overwrite the default timespan if configured
             if(this.TelemetryScheduleInSec > 0)
@@ -222,7 +232,8 @@ internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKe
                 ApplicationName = this.GetUserAgentPrefix(),
                 MaxRetryAttemptsOnRateLimitedRequests = 0,
                 MaxRequestsPerTcpConnection = this.MaxRequestsPerTcpConnection,
-                MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
+                MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
+                ConnectionMode = this.IsThinClientEnabled ? Microsoft.Azure.Cosmos.ConnectionMode.Gateway: Microsoft.Azure.Cosmos.ConnectionMode.Direct,
                 CosmosClientTelemetryOptions = new Microsoft.Azure.Cosmos.CosmosClientTelemetryOptions()
                 {
                     DisableSendingMetricsToService = !this.EnableTelemetry,
@@ -239,11 +250,11 @@ internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKe
             if (!string.IsNullOrWhiteSpace(this.ConsistencyLevel))
             {
                 clientOptions.ConsistencyLevel = (Microsoft.Azure.Cosmos.ConsistencyLevel)Enum.Parse(typeof(Microsoft.Azure.Cosmos.ConsistencyLevel), this.ConsistencyLevel, ignoreCase: true);
-            }
+            }
 
             return new Microsoft.Azure.Cosmos.CosmosClient(
-                        this.EndPoint,
-                        accountKey,
+                        this.IsThinClientEnabled ? this.ThinClientEndPoint : this.EndPoint,
+                        this.IsThinClientEnabled ? this.ThinClientKey : this.Key,
                         clientOptions);
         }
 

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs

@@ -34,7 +34,9 @@ public static async Task Main(string[] args)
         {
             try
             {
-                BenchmarkConfig config = BenchmarkConfig.From(args);
+                BenchmarkConfig config = BenchmarkConfig.From(args);
+                Environment.SetEnvironmentVariable("AZURE_COSMOS_THIN_CLIENT_ENABLED", config.IsThinClientEnabled.ToString());
+
                 await AddAzureInfoToRunSummary();
 
                 MeterProvider meterProvider = BuildMeterProvider(config);
@@ -70,7 +72,9 @@ public static async Task Main(string[] args)
                 Utility.TeeTraceInformation("Exception ocured:" + e.ToString());
             }
             finally
-            {
+            {
+                Environment.SetEnvironmentVariable("AZURE_COSMOS_THIN_CLIENT_ENABLED", "False");
+
                 Utility.TeeTraceInformation($"{nameof(CosmosBenchmark)} completed successfully.");
                 if (Debugger.IsAttached)
                 {
@@ -142,9 +146,10 @@ private static async Task AddAzureInfoToRunSummary()
         /// </summary>
         /// <returns>a Task object.</returns>
         private async Task<RunSummary> ExecuteAsync(BenchmarkConfig config)
-        {
-            // V3 SDK client initialization
-            using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key))
+        {
+            // V3 SDK client initialization
+
+            using (CosmosClient cosmosClient = config.CreateCosmosClient())
             {
                 Microsoft.Azure.Cosmos.Database database = cosmosClient.GetDatabase(config.Database);
                 if (config.CleanupOnStart)
@@ -219,8 +224,8 @@ private async Task<RunSummary> ExecuteAsync(BenchmarkConfig config)
                     Utility.TeeTraceInformation("Publishing results");
                     runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics();
                     await this.PublishResults(
-                        config, 
-                        runSummary, 
+                        config,
+                        runSummary,
                         cosmosClient);
                 }
 

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs

@@ -28,7 +28,7 @@ public RunSummary(
         public string Commit => this.BenchmarkConfig.CommitId;
         public string CommitDate => this.BenchmarkConfig.CommitDate;
         public string CommitTime => this.BenchmarkConfig.CommitTime;
-
+        public bool IsThinClient => this.BenchmarkConfig.IsThinClientEnabled;
         public string Remarks { get; set; }
         public string Date { get; }
         public string Time { get; }
@@ -59,7 +59,10 @@ public RunSummary(
         [JsonProperty]
         public static string Location { get; set; }
         [JsonProperty]
-        public static JObject AzureVmInfo { get; set; }
+        public static JObject AzureVmInfo { get; set; }
+
+        public string ThinClientAccountName => this.BenchmarkConfig.ThinClientEndPoint;
+        public string ThinClientAccountKey => this.BenchmarkConfig.ThinClientKey;
 
         public double Top10PercentAverageRps { get; set; }
         public double Top20PercentAverageRps { get; set; }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/loop.sh

@@ -8,6 +8,11 @@ export PL=18
 export ACCOUNT_ENDPOINT=
 export ACCOUNT_KEY=
 
+#Thinclient configurations
+export THINCLIENT_ENDPOINT=
+export THINCLIENT_KEY=
+export THINCLIENT_ENABLED=
+
 #Loop forever
 i=0
 while :

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/run.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 if [ -z "$ACCOUNT_ENDPOINT" ]
 then
     echo "Missing ACCOUNT_ENDPOINT"
@@ -24,12 +25,6 @@ then
     exit -1
 fi
 
-if [ -z "$TELEMETRY_ENDPOINT" ]
-then
-    echo "Missing TELEMETRY_ENDPOINT"
-    exit -1
-fi
-
 if [ -z "$INCLUDE_QUERY" ]
 then
     echo "Missing INCLUDE_QUERY"
@@ -46,37 +41,92 @@ echo $COMMIT_DATE
 echo $COMMIT_TIME
 echo $BRANCH_NAME
 
-# Client telemetry disabled ReadStreamExistsV3
-dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
-sleep 10 #Wait
+if [ "$THINCLIENT_ENABLED" = true ]; then
+    # ThinClient operations
+    if [ -z "$THINCLIENT_ENDPOINT" ]
+    then
+        echo "Missing THINCLIENT_ENDPOINT"
+        exit -1
+    fi
 
-# Client telemetry enabled ReadStreamExistsV3. This is needed to see the impact of client telemetry. 
-dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --WorkloadName ReadStreamExistsV3WithTelemetry  --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
-sleep 10 #Wait
+    if [ -z "$THINCLIENT_KEY" ]
+    then
+        echo "Missing THINCLIENT_KEY"
+        exit -1
+    fi
 
-# Open telemetry enabled ReadStreamExistsV3. This is needed to see the impact of distributed tracing (without listener) 
-dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --WorkloadName ReadStreamExistsV3WithDistributedTracingWOListener  --enableDistributedTracing --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
-sleep 10 #Wait
+    for WORKLOAD_NAME in \
+        CreateItemV3BenchmarkOperation \
+        CreateItemStreamV3BenchmarkOperation \
+        ReadItemV3BenchmarkOperation \
+        ReadItemStreamV3BenchmarkOperation \
+        ReplaceItemV3BenchmarkOperation \
+        ReplaceItemStreamV3BenchmarkOperation \
+        UpsertItemV3BenchmarkOperation \
+        UpsertItemStreamV3BenchmarkOperation \
+        DeleteItemV3BenchmarkOperation \
+        DeleteItemStreamV3BenchmarkOperation
+    do
+        dotnet run -c Release \
+            -- -n 2000000 \
+            -w $WORKLOAD_NAME \
+            --pl $PL \
+            -e $ACCOUNT_ENDPOINT \
+            -k $ACCOUNT_KEY \
+            --thinclientendpoint $THINCLIENT_ENDPOINT \
+            --thinclientkey $THINCLIENT_KEY \
+            --isthinclientenabled $THINCLIENT_ENABLED \
+            --enablelatencypercentiles \
+            --disablecoresdklogging \
+            --publishresults \
+            --resultspartitionkeyvalue $RESULTS_PK \
+            --commitid $COMMIT_ID \
+            --commitdate $COMMIT_DATE \
+            --committime $COMMIT_TIME \
+            --branchname $BRANCH_NAME \
+            --database testdb \
+            --container testcol \
+            --partitionkeypath /pk
+        sleep 10
+    done
+else
+    if [ -z "$TELEMETRY_ENDPOINT" ]
+    then
+        echo "Missing TELEMETRY_ENDPOINT"
+        exit -1
+    fi
+    # Client telemetry disabled ReadStreamExistsV3
+    dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+    sleep 10 #Wait
 
-#Point read operations
-for WORKLOAD_NAME in ReadNotExistsV3 ReadTExistsV3 ReadStreamExistsWithDiagnosticsV3
-do
-    dotnet run -c Release  -- -n 2000000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+    # Client telemetry enabled ReadStreamExistsV3. This is needed to see the impact of client telemetry. 
+    dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --WorkloadName ReadStreamExistsV3WithTelemetry  --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
     sleep 10 #Wait
-done
 
-#Insert operation
-dotnet run -c Release  -- -n 2000000 -w InsertV3 --pl 30 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 1 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
-sleep 45 #Wait
+    # Open telemetry enabled ReadStreamExistsV3. This is needed to see the impact of distributed tracing (without listener) 
+    dotnet run -c Release  -- -n 2000000 -w ReadStreamExistsV3 --WorkloadName ReadStreamExistsV3WithDistributedTracingWOListener  --enableDistributedTracing --tcp 10 --pl $PL -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+    sleep 10 #Wait
 
-if [ "$INCLUDE_QUERY" = true ]
-then
-    #Query operations
-    # n value is lowered to 200000 because queries are significantly slower. This prevents the runs from taking to long.
-    # pl is 16 because 18 was casuing a small amount of thorrtles.
-    for WORKLOAD_NAME in ReadFeedStreamV3 QueryTSinglePkV3 QueryTSinglePkOrderByWithPaginationV3 QueryTSinglePkOrderByFullDrainV3 QueryTCrossPkV3 QueryTCrossPkOrderByWithPaginationV3 QueryTCrossPkOrderByFullDrainV3 QueryStreamSinglePkV3 QueryStreamSinglePkOrderByWithPaginationV3 QueryStreamSinglePkOrderByFullDrainV3 QueryStreamCrossPkV3 QueryStreamCrossPkOrderByWithPaginationV3 QueryStreamCrossPkOrderByFullDrainV3
+    #Point read operations
+    for WORKLOAD_NAME in ReadNotExistsV3 ReadTExistsV3 ReadStreamExistsWithDiagnosticsV3
     do
-        dotnet run -c Release  -- -n 200000 -w $WORKLOAD_NAME --pl 16 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+        dotnet run -c Release  -- -n 2000000 -w $WORKLOAD_NAME --pl $PL --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
         sleep 10 #Wait
     done
+
+    #Insert operation
+    dotnet run -c Release  -- -n 2000000 -w InsertV3 --pl 30 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 1 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY  --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+    sleep 45 #Wait
+
+    if [ "$INCLUDE_QUERY" = true ]
+    then
+        #Query operations
+        # n value is lowered to 200000 because queries are significantly slower. This prevents the runs from taking to long.
+        # pl is 16 because 18 was casuing a small amount of thorrtles.
+        for WORKLOAD_NAME in ReadFeedStreamV3 QueryTSinglePkV3 QueryTSinglePkOrderByWithPaginationV3 QueryTSinglePkOrderByFullDrainV3 QueryTCrossPkV3 QueryTCrossPkOrderByWithPaginationV3 QueryTCrossPkOrderByFullDrainV3 QueryStreamSinglePkV3 QueryStreamSinglePkOrderByWithPaginationV3 QueryStreamSinglePkOrderByFullDrainV3 QueryStreamCrossPkV3 QueryStreamCrossPkOrderByWithPaginationV3 QueryStreamCrossPkOrderByFullDrainV3
+        do
+            dotnet run -c Release  -- -n 200000 -w $WORKLOAD_NAME --pl 16 --enableTelemetry --telemetryScheduleInSec 60 --telemetryEndpoint $TELEMETRY_ENDPOINT --tcp 10 -e $ACCOUNT_ENDPOINT -k $ACCOUNT_KEY --enablelatencypercentiles --disablecoresdklogging --publishresults --resultspartitionkeyvalue $RESULTS_PK --commitid $COMMIT_ID --commitdate $COMMIT_DATE --committime $COMMIT_TIME  --branchname $BRANCH_NAME --database testdb --container testcol --partitionkeypath /pk 
+            sleep 10 #Wait
+        done
+    fi
 fi

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/CreateItemStreamV3BenchmarkOperation.cs

@@ -0,0 +1,63 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Threading.Tasks;
+    using Microsoft.Azure.Cosmos;
+
+    internal class CreateItemStreamV3BenchmarkOperation : IBenchmarkOperation
+    {
+        private readonly Container container;
+        private readonly string partitionKeyPath;
+        private readonly Dictionary<string, object> sampleJObject;
+        private readonly string databaseName;
+        private readonly string containerName;
+
+        public CreateItemStreamV3BenchmarkOperation(
+            CosmosClient cosmosClient,
+            string dbName,
+            string containerName,
+            string partitionKeyPath,
+            string sampleJson)
+        {
+            this.databaseName = dbName;
+            this.containerName = containerName;
+            this.container = cosmosClient.GetContainer(this.databaseName, this.containerName);
+            this.partitionKeyPath = partitionKeyPath.Replace("/", "");
+            this.sampleJObject = JsonHelper.Deserialize<Dictionary<string, object>>(sampleJson);
+        }
+
+        public BenchmarkOperationType OperationType => BenchmarkOperationType.Insert;
+
+        public Task PrepareAsync()
+        {
+            this.sampleJObject["id"] = Guid.NewGuid().ToString();
+            this.sampleJObject[this.partitionKeyPath] = Guid.NewGuid().ToString();
+            return Task.CompletedTask;
+        }
+
+        public async Task<OperationResult> ExecuteOnceAsync()
+        {
+            PartitionKey partitionKey = new PartitionKey(this.sampleJObject[this.partitionKeyPath].ToString());
+            using (MemoryStream input = JsonHelper.ToStream(this.sampleJObject))
+            {
+                ResponseMessage itemResponse = await this.container.CreateItemStreamAsync(input, partitionKey);
+
+                return new OperationResult
+                {
+                    DatabseName = this.databaseName,
+                    ContainerName = this.containerName,
+                    OperationType = this.OperationType,
+                    RuCharges = itemResponse.Headers.RequestCharge,
+                    CosmosDiagnostics = itemResponse.Diagnostics,
+                    LazyDiagnostics = () => itemResponse.Diagnostics.ToString(),
+                };
+            }
+        }
+    }
+}