Merge branch 'main' into crt_updates

sbSteveK · sbSteveK · commit 1a0a0e42e1cc · 2025-11-03T09:03:56.000-08:00
diff --git a/.builder/actions/localhost_test.py b/.builder/actions/localhost_test.py
@@ -1,19 +1,56 @@
 import Builder
+import subprocess
+import socket
 import sys
+import time
 import os
 
 
 class LocalhostTest(Builder.Action):
 
+    def start(self, env):
+        python = sys.executable
+        venv_path = os.path.join(env.root_dir,'crt','aws-c-http','tests','mock_server', '.venv')
+
+        result = env.shell.exec(python, '-m', 'venv', venv_path)
+        if result.returncode != 0:
+            print("Could not start a virtual environment. The localhost integration tests will fail.", file=sys.stderr)
+            return
+        
+        python = os.path.join(venv_path, "bin", "python")
+        
+        result = env.shell.exec(python, '-m', 'pip', 'install', 'h11', 'h2', 'trio')
+        if result.returncode != 0:
+            print("Could not install python HTTP dependencies. The localhost integration tests will fail.", file=sys.stderr)
+            return
+        
+        server_dir = os.path.join(env.root_dir,'crt','aws-c-http','tests','mock_server')
+        
+        p1 = subprocess.Popen([python, "h2tls_mock_server.py"], cwd=server_dir)
+        p2 = subprocess.Popen([python, "h2non_tls_server.py"], cwd=server_dir)
+        p3 = subprocess.Popen([python, "h11mock_server.py"], cwd=server_dir)
+        
+        # Wait for servers to be ready
+        ports = [3443, 3280, 8082, 8081]
+        for port in ports:
+            for attempt in range(30):
+                try:
+                    with socket.create_connection(("localhost", port), timeout=1):
+                        print(f"Server on port {port} is ready")
+                        break
+                except (socket.error, ConnectionRefusedError, OSError):
+                    if attempt == 29:
+                        print(f"ERROR: Server on port {port} failed to start", file=sys.stderr)
+                    time.sleep(1)
+
     def run(self, env):
+        self.start(env)
         env.shell.setenv('AWS_CRT_MEMORY_TRACING', '2')
-        actions = []
-        if os.system("mvn -Dtest=Http2ClientLocalHostTest test -DredirectTestOutputToFile=true -DforkCount=0 \
+
+        if os.system("mvn test -DredirectTestOutputToFile=true -DforkCount=0 \
             -Daws.crt.memory.tracing=2 \
             -Daws.crt.debugnative=true \
-            -Daws.crt.log.level=Debug \
+            -Daws.crt.aws_trace_log_per_test \
             -Daws.crt.localhost=true"):
             # Failed
             actions.append("exit 1")
-
-        return Builder.Script(actions, name='aws-crt-java-test')
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -436,16 +436,11 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Configure local host
-      run: |
-        python3 -m pip install h2
-        cd crt/aws-c-http/tests/py_localhost/
-        python3 server.py &
-        python3 non_tls_server.py &
     - name: Build and test
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-        python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream
+        python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant=localhost
+
 
   localhost-test-macos:
     runs-on: macos-14 # latest
@@ -458,19 +453,11 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Configure local host
-      run: |
-        python3 -m venv .venv
-        source .venv/bin/activate
-        python3 -m pip install h2
-        cd crt/aws-c-http/tests/py_localhost/
-        python3 server.py &
-        python3 non_tls_server.py &
     - name: Build and test
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder localhost-test -p ${{ env.PACKAGE_NAME }} --spec=downstream
+        ./builder build -p ${{ env.PACKAGE_NAME }} --variant=localhost
 
   localhost-test-win:
     runs-on: windows-2025 # latest
@@ -483,17 +470,10 @@ jobs:
       uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Configure local host
-      run: |
-        python -m pip install h2
     - name: Build and test
       run: |
-        cd crt/aws-c-http/tests/py_localhost/
-        Start-Process -NoNewWindow python .\server.py
-        Start-Process -NoNewWindow python .\non_tls_server.py
-        cd ../../../../
         python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
-        python builder.pyz localhost-test -p ${{ env.PACKAGE_NAME }} downstream
+        python builder.pyz build -p ${{ env.PACKAGE_NAME }} --variant=localhost
 
   GraalVM:
     runs-on: ${{ matrix.os }}
diff --git a/builder.json b/builder.json
@@ -162,12 +162,14 @@
     "variants":{
         "graalvm": {
             "!packages": [],
-            "!imports": [
-            ],
+            "!imports": [],
             "!test_env": {
                 "AWS_GRAALVM_CI": true
             },
             "!build_env": {}
+        },
+        "localhost": {
+            "!test_steps": ["localhost-test"]
         }
     }
 }
diff --git a/crt/aws-c-http b/crt/aws-c-http
@@ -1 +1 @@
-Subproject commit e638eb00423029d422aea97fcfadeb68890a2f5c
+Subproject commit 07302aa4a2892adbbf95ee6d458db3bb240030d3
diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientConnectionTest.java
@@ -28,17 +28,11 @@ public class Http2ClientConnectionTest extends HttpClientTestFixture {
 
     private final static int MAX_TEST_RETRIES = 5;
     private final static int TEST_RETRY_SLEEP_MILLIS = 2000;
-    private final static String HOST = "https://postman-echo.com";
+    // crt/aws-c-http/tests/mock_server includes a readme on how the server can be run locally for testing.
+    private final static String HOST = "https://localhost:3443";
     private final static HttpVersion EXPECTED_VERSION = HttpVersion.HTTP_2;
 
-    @Before
-    public void setUp() {
-        // postman-echo.com in now requires TLS1.3,
-        // but our Mac implementation doesn't support TLS1.3 yet.
-        // The work has been planned to Dec. 2025 to support TLS1.3,
-        // so disable the test for now. And reenable it afterward
-        skipIfMac();
-    }
+
 
     private void doHttp2ConnectionGetVersionTest() {
         try {
@@ -66,6 +60,7 @@ private void doHttp2ConnectionGetVersionTest() {
     public void testHttp2ConnectionGetVersion() throws Exception {
         skipIfAndroid();
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionGetVersionTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -104,6 +99,7 @@ private void doHttp2ConnectionUpdateSettingsTest() {
     public void testHttp2ConnectionUpdateSettings() throws Exception {
         skipIfAndroid();
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionUpdateSettingsTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -139,6 +135,7 @@ public void testHttp2ConnectionUpdateSettingsEmpty() throws Exception {
         skipIfAndroid();
         /* empty settings is allowed to send */
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionUpdateSettingsEmptyTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -176,6 +173,7 @@ private void doHttp2ConnectionPingTest() {
     public void testHttp2ConnectionPing() throws Exception {
         skipIfAndroid();
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionPingTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -220,6 +218,7 @@ private void doHttp2ConnectionPingExceptionPingDataLengthTest() {
     public void testHttp2ConnectionPingExceptionPingDataLength() throws Exception {
         skipIfAndroid();
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionPingExceptionPingDataLengthTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -258,6 +257,7 @@ public void testHttp2ConnectionSendGoAway() throws Exception {
          * for functionality
          */
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionSendGoAwayTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
@@ -296,6 +296,7 @@ public void testHttp2ConnectionUpdateConnectionWindow() throws Exception {
          * for functionality
          */
         skipIfNetworkUnavailable();
+        skipIfLocalhostUnavailable();
 
         TestUtils.doRetryableTest(this::doHttp2ConnectionUpdateConnectionWindowTest, TestUtils::isRetryableTimeout, MAX_TEST_RETRIES, TEST_RETRY_SLEEP_MILLIS);
 
diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2ClientLocalHostTest.java
@@ -17,6 +17,7 @@
 
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
 import org.junit.Test;
 import software.amazon.awssdk.crt.CRT;
 import software.amazon.awssdk.crt.CrtResource;
@@ -43,7 +44,7 @@
 import software.amazon.awssdk.crt.Log;
 
 public class Http2ClientLocalHostTest extends HttpClientTestFixture {
-
+    // crt/aws-c-http/tests/mock_server includes a readme on how the server can be run locally for testing.
     private static final int LOCAL_HTTPS_PORT = 3443;
     private static final int LOCAL_HTTP_PORT = 3280;
 
@@ -182,7 +183,7 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) {
     public void testParallelRequestsStressWithBody() throws Exception {
         skipIfAndroid();
         skipIfLocalhostUnavailable();
-        URI uri = new URI(String.format("https://localhost:%d/uploadTest", LOCAL_HTTPS_PORT));
+        URI uri = new URI(String.format("https://localhost:%d/echo", LOCAL_HTTPS_PORT));
         try (Http2StreamManager streamManager = createStreamManager(uri, 100)) {
             int numberToAcquire = 500 * 100;
             if (CRT.getOSIdentifier() == "linux") {
@@ -201,6 +202,7 @@ public void testParallelRequestsStressWithBody() throws Exception {
             final AtomicInteger numStreamsFailures = new AtomicInteger(0);
             for (int i = 0; i < numberToAcquire; i++) {
                 Http2Request request = createHttp2Request("PUT", uri, bodyLength);
+                request.addHeader(new HttpHeader("x-upload-test", "true"));
 
                 final CompletableFuture<Void> requestCompleteFuture = new CompletableFuture<Void>();
                 final int expectedLength = bodyLength;
@@ -217,12 +219,11 @@ public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int
                     @Override
                     public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){
                         String bodyString = new String(bodyBytesIn);
-                        int receivedLength = Integer.parseInt(bodyString);
-
-                        Assert.assertTrue(receivedLength == expectedLength);
-                        if(receivedLength!=expectedLength) {
-                            numStreamsFailures.incrementAndGet();
-                        }
+                        // Parse {"bytes": 123456} manually
+                        int start = bodyString.indexOf("\"bytes\":") + 8;
+                        int end = bodyString.indexOf("}", start);
+                        String bytesStr = bodyString.substring(start, end).trim();
+                        long receivedLength = Long.parseLong(bytesStr);
                         return bodyString.length();
                     }
 
@@ -252,10 +253,9 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) {
     @Test
     public void testRequestsUploadStress() throws Exception {
         skipIfAndroid();
-        /* Test that upload a 2.5GB data from local server (0.25GB for linux) */
         skipIfLocalhostUnavailable();
-
-        URI uri = new URI(String.format("https://localhost:%d/uploadTest", LOCAL_HTTPS_PORT));
+        /* Test that upload a 2.5GB data from local server (0.25GB for linux) */
+        URI uri = new URI(String.format("https://localhost:%d/echo", LOCAL_HTTPS_PORT));
         try (Http2StreamManager streamManager = createStreamManager(uri, 100)) {
             long bodyLength = 2500000000L;
             if (CRT.getOSIdentifier() == "linux") {
@@ -269,21 +269,26 @@ public void testRequestsUploadStress() throws Exception {
             }
 
             Http2Request request = createHttp2Request("PUT", uri, bodyLength);
+            request.addHeader(new HttpHeader("x-upload-test", "true"));
 
             final CompletableFuture<Void> requestCompleteFuture = new CompletableFuture<Void>();
             final long expectedLength = bodyLength;
             CompletableFuture<Http2Stream> acquireCompleteFuture  = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() {
                 @Override
                 public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType,
-                        HttpHeader[] nextHeaders) {
+                                              HttpHeader[] nextHeaders) {
 
                     Assert.assertTrue(responseStatusCode == 200);
                 }
 
                 @Override
                 public int onResponseBody(HttpStreamBase stream, byte[] bodyBytesIn){
                     String bodyString = new String(bodyBytesIn);
-                    long receivedLength = Long.parseLong(bodyString);
+                    // Parse {"bytes": 123456} manually
+                    int start = bodyString.indexOf("\"bytes\":") + 8;
+                    int end = bodyString.indexOf("}", start);
+                    String bytesStr = bodyString.substring(start, end).trim();
+                    long receivedLength = Long.parseLong(bytesStr);
                     Assert.assertTrue(receivedLength == expectedLength);
                     return bodyString.length();
                 }
@@ -307,20 +312,21 @@ public void onResponseComplete(HttpStreamBase stream, int errorCode) {
     @Test
     public void testRequestsDownloadStress() throws Exception {
         skipIfAndroid();
-        /* Test that download a 2.5GB data from local server */
         skipIfLocalhostUnavailable();
-        URI uri = new URI(String.format("https://localhost:%d/downloadTest", LOCAL_HTTPS_PORT));
+        /* Test that download a 2.5GB data from local server */
+        URI uri = new URI(String.format("https://localhost:%d/echo", LOCAL_HTTPS_PORT));
         try (Http2StreamManager streamManager = createStreamManager(uri, 100)) {
             long bodyLength = 2500000000L;
 
             Http2Request request = createHttp2Request("GET", uri, 0);
+            request.addHeader(new HttpHeader("x-repeat-data", String.valueOf(bodyLength)));
 
             final CompletableFuture<Void> requestCompleteFuture = new CompletableFuture<Void>();
             final AtomicLong receivedLength = new AtomicLong(0);
             CompletableFuture<Http2Stream> acquireCompleteFuture  = streamManager.acquireStream(request, new HttpStreamBaseResponseHandler() {
                 @Override
                 public void onResponseHeaders(HttpStreamBase stream, int responseStatusCode, int blockType,
-                        HttpHeader[] nextHeaders) {
+                                              HttpHeader[] nextHeaders) {
 
                     Assert.assertTrue(responseStatusCode == 200);
                 }
diff --git a/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/Http2RequestResponseTest.java
diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java b/src/test/java/software/amazon/awssdk/crt/test/HttpClientTestFixture.java
diff --git a/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java b/src/test/java/software/amazon/awssdk/crt/test/HttpRequestResponseTest.java

Original file line number	Diff line number	Diff line change
`@@ -162,12 +162,14 @@`
`162`	`162`	`"variants":{`
`163`	`163`	`"graalvm": {`
`164`	`164`	`"!packages": [],`
`165`		`- "!imports": [`
`166`		`- ],`
	`165`	`+ "!imports": [],`
`167`	`166`	`"!test_env": {`
`168`	`167`	`"AWS_GRAALVM_CI": true`
`169`	`168`	`},`
`170`	`169`	`"!build_env": {}`
	`170`	`+ },`
	`171`	`+ "localhost": {`
	`172`	`+ "!test_steps": ["localhost-test"]`
`171`	`173`	`}`
`172`	`174`	`}`
`173`	`175`	`}`