diff --git a/PerformanceTests/Package.swift b/PerformanceTests/Package.swift new file mode 100644 index 00000000000..6da6275ca43 --- /dev/null +++ b/PerformanceTests/Package.swift @@ -0,0 +1,19 @@ +// swift-tools-version: 5.9 + +import PackageDescription + +let package = Package( + name: "SDKWorkbench", + platforms: [.macOS(.v12), .iOS(.v15)], + dependencies: [ + .package(name: "aws-sdk-swift", path: ".."), + ], + targets: [ + .executableTarget( + name: "PerformanceTestRunner", + dependencies: [ + .product(name: "AWSSTS", package: "aws-sdk-swift"), + ] + ), + ] +) diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Dimension.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Dimension.swift new file mode 100644 index 00000000000..730d77cff2c --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Dimension.swift @@ -0,0 +1,14 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +struct Dimension: Codable { + /// The name of the dimension. MUST comply with CloudWatch constraints on dimension name. + let name: String + + /// The value of the dimension. MUST comply with CloudWatch constraints on dimension value. + let value: String +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTest.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTest.swift new file mode 100644 index 00000000000..66d97290b9f --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTest.swift @@ -0,0 +1,14 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +protocol PerformanceTest { + var name: String { get } + var description: String { get } + var unit: Unit { get } + var dimensions: [Dimension] { get } + var test: () async throws -> Double { get } +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestReport.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestReport.swift new file mode 100644 index 00000000000..e72c49c5f2e --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestReport.swift @@ -0,0 +1,37 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import Foundation + +/// Represents a report containing the results of performance tests. +struct PerformanceTestReport: Codable { + /// The product ID of the SDK. + var productId: String = "AWS SDK for Swift" + + /// The version of the SDK. Optional. + let sdkVersion: String? + + /// The git commit ID. + let commitId: String + + /// An array of result objects, representing the results of each test that was run. + let results: [PerformanceTestResult] +} + +extension PerformanceTestReport { + func printFormatted() { + let jsonEncoder = JSONEncoder() + jsonEncoder.outputFormatting = .prettyPrinted + + if let jsonData = try? jsonEncoder.encode(self), + let jsonString = String(data: jsonData, encoding: .utf8) { + print(jsonString) + } else { + print("Error encoding JSON") + } + } +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestResult.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestResult.swift new file mode 100644 index 00000000000..f373f6039b5 --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/PerformanceTestResult.swift @@ -0,0 +1,33 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +/// Represents a single performance test result. +struct PerformanceTestResult: Codable { + /// The name of the benchmark. + /// Examples of acceptable values: `"crc32c.jdk.small"`, `"bundle.size"` + let name: String + + /// A description of the benchmark. + let description: String + + /// The unit of the measurements. + let unit: String + + /// The date the test was completed in Unix epoch seconds. + let date: Int + + /// The raw measurements taken for the test. Elements are IEEE 754 double-precision numbers. + let measurements: [Double] + + /// Used to specify additional dimensions for publishing this metric. Optional. + let dimensions: [Dimension]? + + /// Whether metrics for this result should be published to CloudWatch. Optional. + /// + /// By default, metrics will always be published unless this value is false. + var publishToCloudWatch: Bool? = nil +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Unit.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Unit.swift new file mode 100644 index 00000000000..c3860833ce0 --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Model/Unit.swift @@ -0,0 +1,21 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +enum Unit: String { + case operationsPerSecond = "Operations/Second" + case bitsPerSecond = "Bits/Second" + case megabitsPerSecond = "Megabits/Second" + case gigabitsPerSecond = "Gigabits/Second" + case seconds = "Seconds" + case milliseconds = "Milliseconds" + case microseconds = "Microseconds" + case nanoseconds = "Nanoseconds" + case bytes = "Bytes" + case megabytes = "Megabytes" + case gigabytes = "Gigabytes" + case percentage = "Percentage" +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Runners/PerformanceTestRunner.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Runners/PerformanceTestRunner.swift new file mode 100644 index 00000000000..e09cf88bce5 --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Runners/PerformanceTestRunner.swift @@ -0,0 +1,34 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import Foundation + +struct PerformanceTestRunner { + + var iterations: Int + + /// Executes the given test multiple times + func runTest( + _ perfTest: PerformanceTest + ) async throws -> PerformanceTestResult { + var measurements: [Double] = [] + + for _ in 0.. String { + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + + let outputPipe = Pipe() + process.standardOutput = outputPipe + process.standardError = Pipe() + + do { + try process.run() + process.waitUntilExit() + let outputData = outputPipe.fileHandleForReading.readDataToEndOfFile() + return String(data: outputData, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + } catch { + print("ProcessRunner Error: \(error.localizedDescription)") + return "" + } + } + + /// Retrieves the current Git commit ID. + static func getGitCommitId() -> String { + return ProcessRunner.runProcess(arguments: ["git", "rev-parse", "HEAD"]) + } + + /// Retrieves the SDK version from a file. + static func getSdkVersion() -> String { + let version = ProcessRunner.runProcess(arguments: ["cat", "../../aws-sdk-swift/Package.version"]) + return version.trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Core/Util/OperatingSystem.swift b/PerformanceTests/Sources/PerformanceTestRunner/Core/Util/OperatingSystem.swift new file mode 100644 index 00000000000..a8eb15be044 --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Core/Util/OperatingSystem.swift @@ -0,0 +1,22 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +enum OperatingSystem: String { + case macOS = "macOS" + case linux = "linux" + case unknownOS = "unknown" + + static var current: OperatingSystem { + #if os(macOS) + return .macOS + #elseif os(Linux) + return .linux + #else + return .unknownOS + #endif + } +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/PerformanceMain.swift b/PerformanceTests/Sources/PerformanceTestRunner/PerformanceMain.swift new file mode 100644 index 00000000000..1bc950139de --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/PerformanceMain.swift @@ -0,0 +1,37 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import Foundation +import ClientRuntime + +// MARK: - Main Entry Point +@main +struct PerformanceMain { + static func main() async throws { + let runner = PerformanceTestRunner(iterations: 5) + let commitId = ProcessRunner.getGitCommitId() + let sdkVersion = ProcessRunner.getSdkVersion() + + // Add more tests here + let performanceTests = [ + AWSSTSGetCallerIdentity() + ] + + var results: [PerformanceTestResult] = [] + for test in performanceTests { + let result = try await runner.runTest(test) + results.append(result) + } + + let report = PerformanceTestReport( + sdkVersion: sdkVersion, + commitId: commitId, + results: results + ) + report.printFormatted() + } +} diff --git a/PerformanceTests/Sources/PerformanceTestRunner/Tests/AWSSTSTests.swift b/PerformanceTests/Sources/PerformanceTestRunner/Tests/AWSSTSTests.swift new file mode 100644 index 00000000000..861f074ac74 --- /dev/null +++ b/PerformanceTests/Sources/PerformanceTestRunner/Tests/AWSSTSTests.swift @@ -0,0 +1,30 @@ +// +// Copyright Amazon.com Inc. or its affiliates. +// All Rights Reserved. +// +// SPDX-License-Identifier: Apache-2.0 +// + +import AWSSTS +import Foundation + +struct AWSSTSGetCallerIdentity: PerformanceTest { + let name = "sts.getcalleridentity.latency" + + let description = "The total time between initiating a GetCallerIdentity and reading the last byte of the object." + + let unit = Unit.milliseconds + + let dimensions = [ + Dimension(name: "OS", value: OperatingSystem.current.rawValue), + ] + + let test = getCallerIdentity + + static func getCallerIdentity() async throws -> Double { + let start = Date() + let client = try await STSClient() + _ = try await client.getCallerIdentity(input: .init()) + return Date().timeIntervalSince(start) * 1000 // Convert seconds to milliseconds + } +}