Add Dotnet Benchmark Runner

.github/workflows/ci.yml

@@ -33,6 +33,13 @@ jobs:
     - name: Lint s3-benchrunner-rust
       run: python scripts/lint.py rust
 
+    - uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: '8.0'
+
+    - name: Lint s3-benchrunner-dotnet
+      run: python scripts/lint.py dotnet
+
     - name: Check that workload.run.json files are up to date
       # build workloads and see if any files change
       run: |
@@ -50,6 +57,7 @@ jobs:
           - java
           - python
           - rust
+          - dotnet
     steps:
     - uses: actions/checkout@v4
 
@@ -68,5 +76,10 @@ jobs:
     - if: matrix.runner == 'cpp'
       run: sudo apt update && sudo apt install libcurl4-openssl-dev
 
+    - if: matrix.runner == 'dotnet'
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: '8.0'
+
     - name: Build
       run: python scripts/build-runner.py --lang ${{ matrix.runner }} --build-dir ${{runner.temp}}/build

.gitignore

@@ -1,5 +1,6 @@
 # Created by https://www.toptal.com/developers/gitignore/api/c++,python,node,java,visualstudiocode,intellij,pycharm,clion,macos,windows,visualstudio
 # Edit at https://www.toptal.com/developers/gitignore?templates=c++,python,node,java,visualstudiocode,intellij,pycharm,clion,macos,windows,visualstudio
+*.dll
 
 ### C++ ###
 # Prerequisites

README.md

@@ -16,6 +16,7 @@ This project is under active development and subject to change.
     *   JDK17+ (e.g. corretto, openjdk)
     *   Maven
     *   Python C extension headers and libraries (e.g. python3-devel)
+    *   .NET 8.0 SDK
 
 To benchmark **ALL** the workloads, your machine needs 300+ GiB of disk space available,
 and fast enough internet to upload a terabyte to S3 within your lifetime.
@@ -28,9 +29,13 @@ Your machine must have AWS credentials, with permission to read and write to an
 First, clone this repo.
 
 Then install the [requirements](#requirements) listed above.
-On Amazon Linux 2023, you can simply run this script:
+On Amazon Linux 2023, you can run these scripts:
 ```sh
+# For C/C++/Java/Python/Rust dependencies
 ./aws-crt-s3-benchmarks/scripts/install-tools-AL2023.py
+
+# For .NET SDK dependencies
+./aws-crt-s3-benchmarks/scripts/install-tools-AL2023-dotnet.py
 ```
 
 Then, install packages needed by the python scripts:
@@ -86,20 +91,21 @@ Here are the IDs used for various S3 Clients, and the runner you must build to b
 | `sdk-cpp-client-classic` | [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp) with (non-CRT) S3Client | `cpp` | [runners/s3-benchrunner-cpp](runners/s3-benchrunner-cpp/) |
 | `sdk-cpp-tm-classic` | [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp) with (non-CRT) TransferManager | `cpp` | [runners/s3-benchrunner-cpp](runners/s3-benchrunner-cpp/) |
 | `sdk-rust-tm` | [aws-s3-transfer-manager-rs](https://github.com/awslabs/aws-s3-transfer-manager-rs/) | `rust` | [runners/s3-benchrunner-rust](runners/s3-benchrunner-rust/) |
+| `sdk-dotnet-tm` | [AWS SDK for .NET](https://github.com/aws/aws-sdk-net) TransferUtility | `dotnet` | [runners/s3-benchrunner-dotnet](runners/s3-benchrunner-dotnet/) |
 
 ### Build a Runner
 
 You must build a "runner" for the S3 client you'll be benchmarking. For example, build [runners/s3-benchrunner-python](runners/s3-benchrunner-python/) to benchmark aws-crt-python, boto3, or AWS CLI.
 
 Run `scripts/build-runner.py`:
 ```sh
-usage: build-runner.py [-h] --lang {c,python,java} --build-dir BUILD_DIR [--branch BRANCH]
+usage: build-runner.py [-h] --lang {c,python,java,rust,dotnet} --build-dir BUILD_DIR [--branch BRANCH]
 
 Build a runner and its dependencies
 
 optional arguments:
   -h, --help            show this help message and exit
-  --lang {c,python,java}
+  --lang {c,python,java,rust,dotnet}
                         Build s3-benchrunner-<lang>
   --build-dir BUILD_DIR
                         Root dir for build artifacts

aws-crt-s3-benchmarks.sln

@@ -0,0 +1,32 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.2.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "runners", "runners", "{E6522D2C-A0D2-0AFB-6208-EAE0F209FD8A}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "s3-benchrunner-dotnet", "s3-benchrunner-dotnet", "{FAB3396C-FDED-B50E-2D20-EB9DA31CC016}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "S3BenchRunner", "runners\s3-benchrunner-dotnet\S3BenchRunner\S3BenchRunner.csproj", "{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{FAB3396C-FDED-B50E-2D20-EB9DA31CC016} = {E6522D2C-A0D2-0AFB-6208-EAE0F209FD8A}
+		{93E9F6E2-DEE8-12C6-B301-E3BD3EA41962} = {FAB3396C-FDED-B50E-2D20-EB9DA31CC016}
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {D6695669-2DAE-4CA2-93EE-A0A02345AC63}
+	EndGlobalSection
+EndGlobal

cdk/cdk.json

@@ -1,5 +1,5 @@
 {
-  "app": "python3 app.py",
+  "app": "python app.py",
   "watch": {
     "include": [
       "**"
@@ -11,7 +11,9 @@
       "source.bat",
       "**/__init__.py",
       "**/__pycache__",
-      "tests"
+      "tests",
+      "cdk.out",
+      "source.bat"
     ]
   },
   "context": {

cdk/per-instance-job.Dockerfile

@@ -1,12 +1,26 @@
 FROM amazonlinux:2023
 
-RUN dnf install -y git
+RUN dnf install -y git tar findutils libicu
+
+RUN dnf install -y python3-pip \
+    && python3 -m pip install boto3
 
 # Installing rustup is a pain, because you need to modify the shell afterwards.
 # Easier to just do it here, vs later via install-tools-AL2023.py
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
 
+# Install .NET SDK using the official install script
+RUN curl -L https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh && \
+    chmod +x ./dotnet-install.sh && \
+    ./dotnet-install.sh --version latest && \
+    rm dotnet-install.sh
+
+# Add .NET tools to PATH
+ENV PATH="/root/.dotnet/tools:${PATH}"
+ENV DOTNET_ROOT="/root/.dotnet"
+ENV PATH="${DOTNET_ROOT}:${PATH}"
+
 # s3_benchmarks/__init__.py is shared by CDK Stack and Batch jobs
 COPY s3_benchmarks/__init__.py /s3_benchmarks/
 

cdk/per-instance-job.py

@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import tempfile
+import boto3
 
 import s3_benchmarks
 
@@ -84,7 +85,7 @@ def run(cmd_args: list[str], check=True):
     print(f"Using tmp dir: {tmp_dir}")
 
     # git clone aws-crt-s3-benchmarks
-    run(['git', 'clone', 'https://github.com/awslabs/aws-crt-s3-benchmarks.git'])
+    run(['git', 'clone', 'https://github.com/GarrettBeatty/aws-crt-s3-benchmarks.git'])
     benchmarks_dir = Path('aws-crt-s3-benchmarks')
 
     # if branch specified, try to check it out
@@ -99,6 +100,30 @@ def run(cmd_args: list[str], check=True):
         run([sys.executable,
             str(benchmarks_dir/'scripts/install-tools-AL2023.py')])
 
+        # install .NET tools if any .NET client is being used
+        if any(client.startswith('sdk-dotnet') for client in args.s3_clients):
+            run([sys.executable,
+                str(benchmarks_dir/'scripts/install-tools-AL2023-dotnet.py')])
+
+            # Download custom S3 DLL using boto3
+            # only use this is you want to reference a local custom S3 DLL
+            # print("Downloading custom S3 DLL...")
+            # benchmark_dir = benchmarks_dir/'runners'/'s3-benchrunner-dotnet'/'S3BenchRunner'
+            # dll_path = benchmark_dir/'AWSSDK.S3.dll'
+            # try:
+            #     s3_client = boto3.client('s3', region_name=args.region)
+            #     s3_client.download_file(
+            #         's3dllgarrett',
+            #         'AWSSDK.S3.dll',
+            #         str(dll_path)
+            #     )
+            #     os.chmod(str(dll_path), 0o644)
+            #     print("Custom S3 DLL downloaded successfully and permissions set")
+            # except Exception as e:
+            #     print(f"Error downloading custom S3 DLL: {e}")
+            #     print("Please ensure the DLL exists in the s3dllgarrett bucket and the instance has proper permissions")
+            #     sys.exit(1)
+
         # install python packages
         run([sys.executable, '-m', 'pip', 'install', '-r',
             str(benchmarks_dir/'scripts/requirements.txt')])

cdk/s3_benchmarks/s3_benchmarks_stack.py

@@ -49,6 +49,7 @@ class S3ClientProps:
     'crt-python': S3ClientProps(color=cloudwatch.Color.BLUE),
     'cli-crt': S3ClientProps(color=cloudwatch.Color.PURPLE),
     'boto3-crt': S3ClientProps(color=cloudwatch.Color.PINK),
+    'sdk-dotnet-tm': S3ClientProps(color='#512bd4'),  # .NET purple
 }
 
 # The "default" set of workloads to benchmark.

runners/s3-benchrunner-dotnet/README.md

@@ -0,0 +1,84 @@
+# S3 Benchmark Runner for .NET SDK
+
+This benchmark runner tests the AWS SDK for .NET's TransferUtility implementation.
+
+## Requirements
+
+- .NET 8.0 SDK
+- AWS credentials configured with S3 access
+
+## Building
+
+From the root of this directory:
+
+```bash
+cd S3BenchRunner
+dotnet build -c Release
+```
+
+## Running
+
+The runner expects to be executed from the directory containing the files to upload/download. It follows the standard command line interface used by all benchmark runners:
+
+```bash
+dotnet run -c Release -- sdk-dotnet-tm WORKLOAD BUCKET REGION TARGET_THROUGHPUT WITH_RESPONSE_APIS
+```
+
+Arguments:
+- `sdk-dotnet-tm`: The only supported S3 client ID (current TransferUtility implementation)
+- `WORKLOAD`: Path to workload .run.json file
+- `BUCKET`: S3 bucket name
+- `REGION`: AWS region (e.g., us-west-2)
+- `TARGET_THROUGHPUT`: Target throughput in Gbps (floating point). This parameter not used for now.
+- `WITH_RESPONSE_APIS`: Boolean flag (true/false) to use WithResponse API variants (default: false)
+
+Examples:
+```bash
+# Using regular APIs (default behavior)
+dotnet run -c Release -- sdk-dotnet-tm workloads/download-1MB-1.run.json my-test-bucket us-west-2 100.0 false
+
+# Using WithResponse APIs
+dotnet run -c Release -- sdk-dotnet-tm workloads/download-1MB-1.run.json my-test-bucket us-west-2 100.0 true
+```
+
+## Output
+
+Results are written to stdout in a user-friendly format:
+```
+Run:N Secs:X.XXXXXX Gb/s:X.XXXXXX
+```
+
+Where:
+- N: Run number (1 to maxRepeatCount)
+- X.XXXXXX: Values with 6 decimal precision
+- Secs: Duration of executing all tasks in the workload
+- Gb/s: Throughput in gigabits per second (based on total bytes transferred)
+
+For workloads with multiple tasks:
+- Tasks are executed in parallel using async/await
+- Each run executes all tasks concurrently
+- Reports total time and aggregate throughput for the run
+
+File Handling:
+- When filesOnDisk is true (default):
+  * Downloads write to local files
+  * Uploads read from local files
+  * Files are cleaned up between runs
+- When filesOnDisk is false:
+  * Downloads write to /dev/null
+  * Uploads use random data from memory
+  * No files are created on disk
+
+API Variants:
+- When WITH_RESPONSE_APIS is false (default):
+  * Uses `DownloadAsync`, `UploadAsync`, and `OpenStreamAsync` methods
+- When WITH_RESPONSE_APIS is true:
+  * Uses `DownloadWithResponseAsync`, `UploadWithResponseAsync`, and `OpenStreamWithResponseAsync` methods
+  * These variants return response objects with additional metadata
+
+Example output:
+```
+Run:1 Secs:0.056775 Gb/s:0.009235
+Run:2 Secs:0.027504 Gb/s:0.019063
+Run:3 Secs:0.057251 Gb/s:0.009158
+```

runners/s3-benchrunner-dotnet/S3BenchRunner/BenchmarkRunner.cs

@@ -0,0 +1,44 @@
+using S3BenchRunner.Models;
+
+namespace S3BenchRunner;
+
+public abstract class BenchmarkRunner
+{
+    protected WorkloadConfig Config { get; }
+    protected string Bucket { get; }
+    protected string Region { get; }
+
+    protected BenchmarkRunner(WorkloadConfig config, string bucket, string region)
+    {
+        Config = config;
+        Bucket = bucket;
+        Region = region;
+    }
+
+    public abstract Task RunAsync();
+
+    public void PrepareRun()
+    {
+        // Preparation work between runs
+        foreach (var task in Config.Tasks)
+        {
+            if (task.Action == "download")
+            {
+                var path = task.LocalPath;
+                var directory = Path.GetDirectoryName(path);
+
+                // Create directory if it doesn't exist
+                if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory))
+                {
+                    Directory.CreateDirectory(directory);
+                }
+
+                // Delete file if it exists
+                if (File.Exists(path))
+                {
+                    File.Delete(path);
+                }
+            }
+        }
+    }
+}