Skip to content

Commit 93e2ec9

Browse files
ianbotsfianbotsf
authored
chore: refactor S3 end-to-end tests to use deterministic names and include more debug logs (#1834)
1 parent 84a2e32 commit 93e2ec9

12 files changed

Lines changed: 401 additions & 209 deletions

File tree

‎services/kinesis/e2eTest/src/KinesisSubscribeToShardTest.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ class KinesisSubscribeToShardTest {
7070
streamArn = dataStreamArn
7171
}.shards?.single()!!.shardId
7272

73-
withAllEngines { engine ->
73+
withAllEngines { context ->
7474
client.withConfig {
75-
httpClient = engine
75+
httpClient = context.engine
7676
}.use { clientWithTestEngine ->
7777
clientWithTestEngine.subscribeToShard(
7878
SubscribeToShardRequest {

‎services/polly/e2eTest/src/PollyPresignerTest.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ class PollyPresignerTest {
3333
val client = PollyClient { region = "us-east-1" }
3434
val presignedRequest = client.presignSynthesizeSpeech(unsignedRequest, 10.seconds)
3535

36-
withAllEngines { engine ->
37-
val httpClient = SdkHttpClient(engine)
36+
withAllEngines { context ->
37+
val httpClient = SdkHttpClient(context.engine)
3838

3939
val call = httpClient.call(presignedRequest)
4040
call.complete()
4141

42-
assertEquals(200, call.response.status.value, "presigned polly request failed for engine: $engine")
42+
assertEquals(200, call.response.status.value, "presigned Polly request failed for ${context.name} engine")
4343
}
4444
}
4545
}

‎services/s3/e2eTest/src/ConnectionResetTest.kt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,12 @@
44
*/
55
package aws.sdk.kotlin.e2etest
66

7-
import aws.sdk.kotlin.services.s3.S3Client
87
import aws.sdk.kotlin.services.s3.model.PutObjectRequest
98
import aws.smithy.kotlin.runtime.content.ByteStream
109
import aws.smithy.kotlin.runtime.http.HttpException
11-
import aws.smithy.kotlin.runtime.util.Uuid
1210
import kotlinx.coroutines.async
1311
import kotlinx.coroutines.awaitAll
1412
import kotlinx.coroutines.delay
15-
import kotlinx.coroutines.invoke
1613
import kotlinx.coroutines.runBlocking
1714
import org.junit.jupiter.api.AfterAll
1815
import org.junit.jupiter.api.BeforeAll
@@ -27,27 +24,30 @@ import kotlin.time.Duration.Companion.seconds
2724
*/
2825
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
2926
class ConnectionResetTest {
30-
private val client = S3Client {
31-
region = S3TestUtils.DEFAULT_REGION
27+
private val client = S3TestUtils.createClient()
28+
29+
companion object {
30+
private const val CONN_TEST_CONCURRENCY = 10
3231
}
3332

3433
private lateinit var testBucket: String
3534

3635
@BeforeAll
3736
fun createResources(): Unit = runBlocking {
38-
testBucket = S3TestUtils.getTestBucket(client)
37+
testBucket = S3TestUtils.createTestBucket(client, "conn-reset")
3938
}
4039

4140
@AfterAll
4241
fun cleanup() = runBlocking {
43-
S3TestUtils.deleteBucketAndAllContents(client, testBucket)
42+
S3TestUtils.deleteBucket(client, testBucket)
43+
client.close()
4444
}
4545

4646
@Test
4747
fun testConnectionResetDoesntThrow(): Unit = runBlocking {
4848
// Launch multiple coroutines to populate connection pool
49-
val jobs = (1..10).map {
50-
async { client.putTestObject() }
49+
val jobs = (1..CONN_TEST_CONCURRENCY).map { index ->
50+
async { putTestObject(index) }
5151
}
5252
jobs.awaitAll()
5353
// Connections are now idle in the pool
@@ -56,13 +56,13 @@ class ConnectionResetTest {
5656
delay(7.seconds)
5757

5858
// Try to re-use a connection
59-
client.putTestObject()
59+
putTestObject(CONN_TEST_CONCURRENCY + 1)
6060
}
6161

62-
suspend fun S3Client.putTestObject() {
62+
suspend fun putTestObject(index: Int) {
6363
val putObjectRequest = PutObjectRequest {
6464
bucket = testBucket
65-
key = Uuid.random().toString()
65+
key = "conn-reset-test-object-$index"
6666
body = ByteStream.fromString("Content")
6767
}
6868

‎services/s3/e2eTest/src/MultiRegionAccessPointTest.kt‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package aws.sdk.kotlin.e2etest
6+
7+
import aws.sdk.kotlin.services.s3.S3Client
8+
import aws.sdk.kotlin.services.s3.deleteObject
9+
import aws.sdk.kotlin.services.s3.putObject
10+
import aws.sdk.kotlin.services.s3.withConfig
11+
import aws.sdk.kotlin.services.s3control.*
12+
import aws.sdk.kotlin.services.s3control.model.Region
13+
import aws.sdk.kotlin.services.s3control.paginators.listMultiRegionAccessPointsPaginated
14+
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner
15+
import aws.smithy.kotlin.runtime.auth.awssigning.DefaultAwsSigner
16+
import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner
17+
import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme
18+
import kotlinx.coroutines.delay
19+
import kotlinx.coroutines.flow.toList
20+
import kotlinx.coroutines.runBlocking
21+
import kotlinx.coroutines.withTimeout
22+
import org.junit.jupiter.api.AfterAll
23+
import org.junit.jupiter.api.BeforeAll
24+
import org.junit.jupiter.api.TestInstance
25+
import org.junit.jupiter.params.ParameterizedTest
26+
import org.junit.jupiter.params.provider.Arguments
27+
import org.junit.jupiter.params.provider.MethodSource
28+
import java.util.stream.Stream
29+
import kotlin.time.Duration
30+
import kotlin.time.Duration.Companion.minutes
31+
import kotlin.time.Duration.Companion.seconds
32+
33+
private const val TEST_OBJECT_KEY = "test.txt"
34+
35+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
36+
class MultiRegionAccessPointTest {
37+
private lateinit var s3West: S3Client
38+
private lateinit var s3East: S3Client
39+
private lateinit var s3Control: S3ControlClient
40+
41+
private lateinit var accountId: String
42+
private lateinit var multiRegionAccessPointName: String
43+
private lateinit var multiRegionAccessPointArn: String
44+
private lateinit var usWestBucket: String
45+
private lateinit var usEastBucket: String
46+
47+
@BeforeAll
48+
fun setup(): Unit = runBlocking {
49+
s3West = S3TestUtils.createClient { region = "us-west-2" }
50+
s3East = S3TestUtils.createClient { region = "us-east-2" }
51+
s3Control = S3ControlClient { region = "us-west-2" }
52+
53+
accountId = S3TestUtils.getAccountId()
54+
usWestBucket = S3TestUtils.createTestBucket(s3West, "mrap-west")
55+
usEastBucket = S3TestUtils.createTestBucket(s3East, "mrap-east")
56+
57+
multiRegionAccessPointName = "s3-test-mrap-${S3TestUtils.testRunId}"
58+
multiRegionAccessPointArn = s3Control.createMultiRegionAccessPoint(
59+
multiRegionAccessPointName,
60+
accountId,
61+
listOf(usWestBucket, usEastBucket),
62+
)
63+
}
64+
65+
@AfterAll
66+
fun cleanup(): Unit = runBlocking {
67+
s3Control.deleteMultiRegionAccessPoint(multiRegionAccessPointName, accountId)
68+
69+
val resp = s3Control.listMultiRegionAccessPointsPaginated {
70+
accountId = this@MultiRegionAccessPointTest.accountId
71+
}.toList().flatMap { it.accessPoints.orEmpty() }
72+
73+
val mrapManifest = buildString {
74+
appendLine("Existing multi-region access points for account ID $accountId (${resp.size}):")
75+
resp.forEach { accessPoint ->
76+
appendLine("* ${accessPoint.name}:")
77+
appendLine(" * Alias: ${accessPoint.alias}")
78+
appendLine(" * Created: ${accessPoint.createdAt}")
79+
appendLine(" * Status: ${accessPoint.status}")
80+
81+
val regions = accessPoint.regions.orEmpty()
82+
appendLine(" * Regions (${regions.size}):")
83+
84+
regions.forEach { region ->
85+
appendLine(" * ${region.region}: ${region.bucket} (account ID ${region.bucketAccountId})")
86+
}
87+
}
88+
}
89+
print(mrapManifest)
90+
91+
S3TestUtils.deleteBucket(s3West, usWestBucket)
92+
S3TestUtils.deleteBucket(s3East, usEastBucket)
93+
94+
s3West.close()
95+
s3East.close()
96+
s3Control.close()
97+
}
98+
99+
@ParameterizedTest
100+
@MethodSource("signerProvider")
101+
fun testMultiRegionAccessPointOperation(signer: AwsSigner): Unit = runBlocking {
102+
println("Testing multi-region access point operations with $signer")
103+
104+
val s3SigV4a = s3West.withConfig {
105+
authSchemes = listOf(SigV4AsymmetricAuthScheme(signer))
106+
}
107+
108+
s3SigV4a.putObject {
109+
bucket = multiRegionAccessPointArn
110+
key = TEST_OBJECT_KEY
111+
}
112+
113+
s3SigV4a.deleteObject {
114+
bucket = multiRegionAccessPointArn
115+
key = TEST_OBJECT_KEY
116+
}
117+
}
118+
119+
fun signerProvider(): Stream<Arguments> = Stream.of(
120+
Arguments.of(DefaultAwsSigner),
121+
Arguments.of(CrtAwsSigner),
122+
)
123+
}
124+
125+
/**
126+
* Create a multi-region access point named [name] in account [accountId] with [buckets] buckets.
127+
* @return the ARN of the multi-region access point that was created
128+
*/
129+
private suspend fun S3ControlClient.createMultiRegionAccessPoint(
130+
name: String,
131+
accountId: String,
132+
buckets: List<String>,
133+
): String {
134+
println("Creating multi-region access point: $name")
135+
136+
val requestTokenArn = checkNotNull(
137+
createMultiRegionAccessPoint {
138+
this.accountId = accountId
139+
details {
140+
this.name = name
141+
this.regions = buckets.map { Region { bucket = it } }
142+
}
143+
}.requestTokenArn,
144+
) { "createMultiRegionAccessPoint requestTokenArn was unexpectedly null" }
145+
146+
waitUntilOperationCompletes("createMultiRegionAccessPoint", accountId, requestTokenArn, 10.minutes)
147+
148+
return getMultiRegionAccessPointArn(name, accountId)
149+
}
150+
151+
private suspend fun S3ControlClient.getMultiRegionAccessPointArn(
152+
name: String,
153+
accountId: String,
154+
): String = getMultiRegionAccessPoint {
155+
this.name = name
156+
this.accountId = accountId
157+
}.accessPoint?.alias?.let {
158+
"arn:aws:s3::$accountId:accesspoint/$it"
159+
} ?: throw IllegalStateException("Failed to get ARN for multi-region access point $name")
160+
161+
private suspend fun S3ControlClient.deleteMultiRegionAccessPoint(
162+
name: String,
163+
accountId: String,
164+
) {
165+
println("Deleting multi-region access point $name")
166+
167+
val requestTokenArn = checkNotNull(
168+
deleteMultiRegionAccessPoint {
169+
this.accountId = accountId
170+
details {
171+
this.name = name
172+
}
173+
}.requestTokenArn,
174+
) { "deleteMultiRegionAccessPoint requestTokenArn was unexpectedly null" }
175+
176+
waitUntilOperationCompletes("deleteMultiRegionAccessPoint", accountId, requestTokenArn, 5.minutes)
177+
}
178+
179+
/**
180+
* Continuously poll the status of [requestTokenArn] until its status is "SUCCEEDED" or [timeout] duration has passed.
181+
*/
182+
private suspend fun S3ControlClient.waitUntilOperationCompletes(
183+
operation: String,
184+
accountId: String,
185+
requestTokenArn: String,
186+
timeout: Duration,
187+
) = withTimeout(timeout) {
188+
var status: String? = null
189+
190+
while (true) {
191+
val response = describeMultiRegionAccessPointOperation {
192+
this.accountId = accountId
193+
this.requestTokenArn = requestTokenArn
194+
}
195+
when (val latestStatus = response.asyncOperation?.requestStatus) {
196+
"SUCCEEDED" -> {
197+
println("$operation operation succeeded.")
198+
return@withTimeout
199+
}
200+
"FAILED" -> {
201+
val code = response.asyncOperation?.responseDetails?.errorDetails?.code
202+
val message = response.asyncOperation?.responseDetails?.errorDetails?.message
203+
throw IllegalStateException("$operation operation failed. Code: $code. Message: $message")
204+
}
205+
else -> {
206+
if (status == null || latestStatus != status) {
207+
println("Waiting for $operation to complete. Status: $latestStatus ")
208+
status = latestStatus
209+
}
210+
}
211+
}
212+
213+
delay(10.seconds) // Avoid constant status checks
214+
}
215+
}

‎services/s3/e2eTest/src/PaginatorTest.kt‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
*/
55
package aws.sdk.kotlin.e2etest
66

7-
import aws.sdk.kotlin.services.s3.S3Client
87
import aws.sdk.kotlin.services.s3.abortMultipartUpload
98
import aws.sdk.kotlin.services.s3.createMultipartUpload
109
import aws.sdk.kotlin.services.s3.model.CompletedPart
@@ -24,20 +23,19 @@ import kotlin.time.Duration.Companion.seconds
2423

2524
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
2625
class PaginatorTest {
27-
private val client = S3Client {
28-
region = S3TestUtils.DEFAULT_REGION
29-
}
26+
private val client = S3TestUtils.createClient()
3027

3128
private lateinit var testBucket: String
3229

3330
@BeforeAll
3431
fun createResources(): Unit = runBlocking {
35-
testBucket = S3TestUtils.getTestBucket(client)
32+
testBucket = S3TestUtils.createTestBucket(client, "pagination")
3633
}
3734

3835
@AfterAll
3936
fun cleanup() = runBlocking {
40-
S3TestUtils.deleteBucketAndAllContents(client, testBucket)
37+
S3TestUtils.deleteBucket(client, testBucket)
38+
client.close()
4139
}
4240

4341
// ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly.

0 commit comments

Comments
 (0)