Merge branch 'main' into cmake-modules

Author: sfodagain

.builder/actions/crt_size_check.py

@@ -11,7 +11,7 @@ def run(self, env):
         # Maximum package size (for current platform) in bytes
         # NOTE: if you increase this, you might also need to increase the
         # limit in continuous-delivery/pack.sh
-        max_size = 8_250_000
+        max_size = 9_000_000
         # size of current folder
         folder_size = 0
         # total size in bytes

.github/workflows/ci.yml

@@ -21,7 +21,7 @@ permissions:
 
 jobs:
   linux-compat:
-    runs-on: ubuntu-24.04 # latest
+    runs-on: ubuntu-22.04 # temporarily downgrade to old ubuntu until https://github.com/actions/runner-images/issues/11471 resolves
     strategy:
       fail-fast: false
       matrix:
@@ -65,30 +65,24 @@ jobs:
         docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
 
-  linux-musl-armv7:
-    runs-on: ubuntu-24.04 # latest
+  linux-musl-armv8:
+    runs-on: codebuild-aws-crt-nodejs-arm64-${{ github.run_id }}-${{ github.run_attempt }}-arm-3.0-large
     strategy:
       fail-fast: false
-      matrix:
-        image:
-          - alpine-3.16-x64
     steps:
     - uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ env.CRT_CI_ROLE }}
         aws-region: ${{ env.AWS_DEFAULT_REGION }}
-    - name: Install qemu/docker
-      run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
     - name: Checkout Sources
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
       with:
         submodules: true
     # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
     - name: Build ${{ env.PACKAGE_NAME }}
       run: |
-        export AWS_CRT_ARCH=armv7
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
-        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }}
+        ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-arm64 build -p ${{ env.PACKAGE_NAME }}
 
   linux-compiler-compat:
     runs-on: ubuntu-24.04 # latest

continuous-delivery/build-binaries-linux-musl-arm64.yml

@@ -0,0 +1,17 @@
+version: 0.2
+phases:
+  install:
+    commands:
+  build:
+    commands:
+      - mkdir linux-arm64-musl
+      - cd aws-crt-nodejs
+      - ./continuous-delivery/generic-linux-build.sh aws-crt-alpine-3.16-arm64
+      - cp -r dist/bin/linux-arm64-musl/* ../linux-arm64-musl/
+
+  post_build:
+    commands:
+
+artifacts:
+  files:
+    - 'linux-arm64-musl/**/*'

continuous-delivery/generic-linux-build.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -ex
+
+IMAGE_NAME=$1
+shift
+
+# Pry the builder version this CRT is using out of ci.yml
+BUILDER_VERSION=$(cat .github/workflows/ci.yml | grep 'BUILDER_VERSION:' | sed 's/\s*BUILDER_VERSION:\s*\(.*\)/\1/')
+echo "Using builder version ${BUILDER_VERSION}"
+
+aws ecr get-login-password | docker login 123124136734.dkr.ecr.us-east-1.amazonaws.com -u AWS --password-stdin
+export DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/${IMAGE_NAME}:${BUILDER_VERSION}
+
+export BRANCH_TAG=$(git describe --tags)
+docker run --mount type=bind,src=$(pwd),dst=/root/aws-crt-nodejs --env CXXFLAGS $DOCKER_IMAGE --version=${BUILDER_VERSION} build -p aws-crt-nodejs --branch ${BRANCH_TAG} run_tests=false
+docker container prune -f

continuous-delivery/pack.sh

@@ -39,7 +39,7 @@ mkdir $UNZIP
 tar -xf aws-crt-$CURRENT_TAG.tgz -C $UNZIP
 PACK_FILE_SIZE_KB=$(du -sk $UNZIP | awk '{print $1}')
 echo "Current package size: ${PACK_FILE_SIZE_KB}"
-if expr $PACK_FILE_SIZE_KB \> "$((27000))" ; then
+if expr $PACK_FILE_SIZE_KB \> "$((33000))" ; then
     # the package size is too large, return -1
     echo "Package size is too large!"
     exit -1

lib/common/mqtt.spec.ts

@@ -129,13 +129,26 @@ test_env.conditional_test(test_env.AWS_IOT_ENV.mqtt311_is_valid_iot_cred())('MQT
     const onMessage = once(connectionWaitingForWill, 'message');
     await connectionWaitingForWill.subscribe(willTopic, QoS.AtLeastOnce);
 
+    // pause for a couple of seconds to try and minimize chance for a service-side race
+    await new Promise(resolve => setTimeout(resolve, 2000));
+
     // The third connection that will cause the first one to be disconnected because it has the same client ID.
     const connectionDuplicate = await makeConnection(undefined, client_id);
-    const onConnectDuplicate = once(connectionDuplicate, 'connect');
+
     const onDisconnectDuplicate = once(connectionDuplicate, 'disconnect');
-    await connectionDuplicate.connect()
-    const connectDuplicateResult = (await onConnectDuplicate)[0];
-    expect(connectDuplicateResult).toBeFalsy(); /* session present */
+
+    // Rarely, IoT Core disconnects the new connection and not the existing one, so retry in that case
+    let continueConnecting = true;
+    while (continueConnecting) {
+        try {
+            const onConnectDuplicate = once(connectionDuplicate, 'connect');
+            await connectionDuplicate.connect();
+            await onConnectDuplicate;
+            continueConnecting = false;
+        } catch (err) {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+    }
 
     // The second connection should receive Will message after the first connection was kicked out.
     const messageReceivedArgs = (await onMessage);

lib/native/mqtt5.spec.ts

@@ -635,7 +635,7 @@ test_utils.conditional_test(test_utils.ClientEnvironmentalConfig.hasIotCoreEnvir
         payload: testPayload
     });
 
-    await setTimeout(()=>{}, 2000);
+    await new Promise(resolve => setTimeout(resolve, 2000));
 
     statistics = client.getOperationalStatistics();
     expect(statistics.incompleteOperationCount).toBeLessThanOrEqual(0);

source/http_connection.c

@@ -149,9 +149,12 @@ struct http_connection_binding {
 /* finalizer called when node cleans up this object */
 static void s_http_connection_from_manager_binding_finalize(napi_env env, void *finalize_data, void *finalize_hint) {
     (void)finalize_hint;
-    (void)env;
     struct http_connection_binding *binding = finalize_data;
 
+    if (binding->node_external != NULL) {
+        napi_delete_reference(env, binding->node_external);
+    }
+
     /* no release call, the http_client_connection_manager has already released it */
     aws_mem_release(binding->allocator, binding);
 }

source/http_connection_manager.c

@@ -27,8 +27,10 @@ struct aws_http_connection_manager *aws_napi_get_http_connection_manager(
 
 static void s_http_connection_manager_finalize(napi_env env, void *finalize_data, void *finalize_hint) {
     (void)finalize_hint;
-    (void)env;
     struct http_connection_manager_binding *binding = finalize_data;
+    if (binding->node_external != NULL) {
+        napi_delete_reference(env, binding->node_external);
+    }
     aws_mem_release(binding->allocator, binding);
 }
 

source/http_message.c

@@ -79,29 +79,29 @@ napi_status aws_napi_http_message_bind(napi_env env, napi_value exports) {
 
 struct http_request_binding {
     struct aws_http_message *native;
-    struct aws_allocator *allocator;
 
     napi_ref node_headers;
 };
 
-/* Need a special finalizer to avoid releasing a request object we don't own */
-static void s_napi_wrapped_http_request_finalize(napi_env env, void *finalize_data, void *finalize_hint) {
-    (void)env;
+static void s_napi_http_request_finalize(napi_env env, void *finalize_data, void *finalize_hint) {
     (void)finalize_hint;
-
     struct http_request_binding *binding = finalize_data;
-    struct aws_allocator *allocator = binding->allocator;
+    struct aws_allocator *allocator = aws_napi_get_allocator();
+
+    if (binding->node_headers != NULL) {
+        napi_delete_reference(env, binding->node_headers);
+    }
 
+    aws_http_message_release(binding->native);
     aws_mem_release(allocator, binding);
 }
 
 napi_status aws_napi_http_message_wrap(napi_env env, struct aws_http_message *message, napi_value *result) {
 
     struct http_request_binding *binding =
         aws_mem_calloc(aws_napi_get_allocator(), 1, sizeof(struct http_request_binding));
-    binding->native = message;
-    binding->allocator = aws_napi_get_allocator();
-    return aws_napi_wrap(env, &s_request_class_info, binding, s_napi_wrapped_http_request_finalize, result);
+    binding->native = aws_http_message_acquire(message);
+    return aws_napi_wrap(env, &s_request_class_info, binding, s_napi_http_request_finalize, result);
 }
 
 struct aws_http_message *aws_napi_http_message_unwrap(napi_env env, napi_value js_object) {
@@ -115,20 +115,6 @@ struct aws_http_message *aws_napi_http_message_unwrap(napi_env env, napi_value j
  * Constructor
  **********************************************************************************************************************/
 
-static void s_napi_http_request_finalize(napi_env env, void *finalize_data, void *finalize_hint) {
-    (void)env;
-
-    struct http_request_binding *binding = finalize_data;
-    struct aws_allocator *allocator = finalize_hint;
-
-    if (binding->node_headers != NULL) {
-        napi_delete_reference(env, binding->node_headers);
-    }
-
-    aws_http_message_destroy(binding->native);
-    aws_mem_release(allocator, binding);
-}
-
 static napi_value s_request_constructor(napi_env env, const struct aws_napi_callback_info *cb_info) {
 
     struct aws_allocator *alloc = aws_napi_get_allocator();
@@ -167,7 +153,7 @@ static napi_value s_request_constructor(napi_env env, const struct aws_napi_call
     }
 
     napi_value node_this = cb_info->native_this;
-    AWS_NAPI_CALL(env, napi_wrap(env, node_this, binding, s_napi_http_request_finalize, alloc, NULL), {
+    AWS_NAPI_CALL(env, napi_wrap(env, node_this, binding, s_napi_http_request_finalize, NULL, NULL), {
         napi_throw_error(env, NULL, "Failed to wrap HttpRequest");
         goto cleanup;
     });
@@ -177,7 +163,7 @@ static napi_value s_request_constructor(napi_env env, const struct aws_napi_call
 cleanup:
     if (binding) {
         if (binding->native) {
-            aws_http_message_destroy(binding->native);
+            aws_http_message_release(binding->native);
         }
         aws_mem_release(alloc, binding);
     }
@@ -238,15 +224,16 @@ static napi_value s_request_headers_get(napi_env env, void *native_this) {
 
     napi_value result = NULL;
     if (binding->node_headers) {
+        /* This HTTP request already has a reference to the Node object for the headers */
         AWS_NAPI_ENSURE(env, napi_get_reference_value(env, binding->node_headers, &result));
     } else {
+        /* There is no Node object for these headers.
+         * Create a Node object, and a reference to it, and store that reference on this HTTP request */
         struct aws_http_headers *headers = aws_http_message_get_headers(binding->native);
         AWS_NAPI_ENSURE(env, aws_napi_http_headers_wrap(env, headers, &result));
+        AWS_NAPI_ENSURE(env, napi_create_reference(env, result, 1, &binding->node_headers));
     }
 
-    /* Store the value for later */
-    AWS_NAPI_ENSURE(env, napi_create_reference(env, result, 1, &binding->node_headers));
-
     return result;
 }
 

source/http_stream.c

@@ -74,7 +74,7 @@ static void s_on_response_call(napi_env env, napi_value on_response, void *conte
     }
 
     /* clean up the response buffer */
-    aws_http_message_destroy(binding->response);
+    aws_http_message_release(binding->response);
     binding->response = NULL;
 }
 

test/mqtt5.ts

@@ -402,7 +402,7 @@ export async function subPubUnsubTest(client: mqtt5.Mqtt5Client, qos: mqtt5.QoS,
         payload: testPayload
     });
 
-    await setTimeout(()=>{}, 2000);
+    await new Promise(resolve => setTimeout(resolve, 2000));
 
     client.stop();
     await stopped;
@@ -434,6 +434,9 @@ export async function willTest(publisher: mqtt5.Mqtt5Client, subscriber: mqtt5.M
         throw new CrtError("doh");
     }
 
+    // pause to minimize eventual consistency race condition possibility
+    await new Promise(resolve => setTimeout(resolve, 1000));
+
     publisher.stop({
         reasonCode: mqtt5.DisconnectReasonCode.DisconnectWithWillMessage
     });
@@ -659,8 +662,6 @@ export async function doSharedSubscriptionsTest(publisher: mqtt5.Mqtt5Client, su
     let messagesReceived : number = 0;
     subscriberMessages.forEach(v => {
         messagesReceived += v;
-        // Each subscriber should receive a portion of messages.
-        expect(v).toBeGreaterThan(0);
     });
     expect(messagesReceived).toEqual(messagesNumber);