diff --git a/.github/workflows/ci-android.yml b/.github/workflows/ci-android.yml new file mode 100644 index 000000000..7c95586b4 --- /dev/null +++ b/.github/workflows/ci-android.yml @@ -0,0 +1,143 @@ +name: CI Android + +on: + push: + branches-ignore: + - 'main' + - 'docs' + +# cancel in-progress builds after a new commit +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILDER_VERSION: v0.9.84 + BUILDER_SOURCE: releases + BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net + PACKAGE_NAME: aws-crt-java + RUN: ${{ github.run_id }}-${{ github.run_number }} + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + AWS_DEVICE_FARM_REGION: us-west-2 # Device Farm only available in us-west-2 region + +permissions: + id-token: write # This is required for requesting the JWT + +jobs: + android: + # ubuntu-24.04 comes with Android tooling + name: Android + runs-on: ubuntu-24.04 # latest + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + # Setup JDK 17 + - name: set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + # Ensure Gradle uses this JDK (important when toolchains are present) + - name: Point Gradle at JDK 17 + run: echo "ORG_GRADLE_JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV + + - name: Mirror ANDROID_HOME → ANDROID_SDK_ROOT + run: echo "ANDROID_SDK_ROOT=$ANDROID_HOME" >> "$GITHUB_ENV" + + # Install required Android components & accept licenses (NDK r28 + compileSdk 35) + - name: Install Android SDK packages + shell: bash + run: | + set -euo pipefail + SDKROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}" + SDKMANAGER="$SDKROOT/cmdline-tools/latest/bin/sdkmanager" + + # Some images have cmdline-tools already; ensure path exists + if [[ ! -x "$SDKMANAGER" ]]; then + echo "cmdline-tools not found; installing…" + mkdir -p "$SDKROOT/cmdline-tools" + curl -sSL -o /tmp/clt.zip https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip + sudo unzip -q /tmp/clt.zip -d "$SDKROOT/cmdline-tools" + sudo mv "$SDKROOT/cmdline-tools/cmdline-tools" "$SDKROOT/cmdline-tools/latest" + SDKMANAGER="$SDKROOT/cmdline-tools/latest/bin/sdkmanager" + fi + + echo "Installing platform-tools, Build-Tools 35, platform android-35, NDK r28, and CMake…" + # Avoid 'yes' SIGPIPE killing the step: turn off pipefail just for these lines + set +o pipefail + yes | sudo "$SDKMANAGER" --install \ + "platform-tools" \ + "build-tools;35.0.0" \ + "platforms;android-35" \ + "ndk;28.0.12433566" \ + "cmake;3.22.1" + + yes | "$SDKMANAGER" --licenses + set -o pipefail + + # Quick sanity prints (non-fatal) + "$SDKROOT/ndk/28.0.12433566/ndk-build" -v >/dev/null 2>&1 || true + "$SDKMANAGER" --list | sed -n '1,80p' || true + + # Build and publish locally for the test app to find the SNAPSHOT version + - name: Build ${{ env.PACKAGE_NAME }} + run: | + # Manually set -Xmx (max heap size) to something huge (tested 2g and that works, but why not go bigger). + # Only in CI, gradle daemon runs out of memory during "lintAnalyzeDebug" task, unless you specify it this way. + # You'd think Java's default of 25% RAM (ubuntu24 runner has 12g, so max 4g) would be sufficient, but no. + # You'd think setting -Xmx via gradle.properties would help, but no. + ./gradlew :android:crt:build -Dorg.gradle.jvmargs="-Xmx8g" + ./gradlew -PnewVersion="1.0.0-SNAPSHOT" :android:crt:publishToMavenLocal + # Setup files required by test app for Device Farm testing + - name: Setup Android Test Files + run: | + cd src/test/android/testapp/src/main/assets + python3 -m pip install boto3 + python3 ./android_file_creation.py + + - name: Set Android keystore home + run: | + echo "ANDROID_SDK_HOME=$GITHUB_WORKSPACE/.android-home" >> "$GITHUB_ENV" + echo "ANDROID_PREFS_ROOT=$GITHUB_WORKSPACE/.android-home" >> "$GITHUB_ENV" + mkdir -p "$GITHUB_WORKSPACE/.android-home/.android" + + - name: Create debug keystore + run: | + keytool -genkeypair \ + -keystore "$ANDROID_SDK_HOME/.android/debug.keystore" \ + -storepass android -keypass android \ + -alias androiddebugkey \ + -dname "CN=Android Debug,O=Android,C=US" \ + -keyalg RSA -keysize 2048 -validity 14000 + + - name: Build Test App + run: | + cd src/test/android/testapp + ../../../../gradlew assembledebug + ../../../../gradlew assembleAndroidTest + - name: Device Farm Tests Highly Available + run: | + echo "Running Device Farm Python Script" + python3 ./.github/workflows/run_android_ci.py \ + --run_id ${{ github.run_id }} \ + --run_attempt ${{ github.run_attempt }} \ + --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool highly_available + - name: Device Farm Tests Android 8.0.0 + run: | + echo "Running Device Farm Python Script" + python3 ./.github/workflows/run_android_ci.py \ + --run_id ${{ github.run_id }} \ + --run_attempt ${{ github.run_attempt }} \ + --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArnAndroid8" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ + --device_pool android_8 \ No newline at end of file diff --git a/.github/workflows/ci-slow.yml b/.github/workflows/ci-slow.yml new file mode 100644 index 000000000..09a8f11b0 --- /dev/null +++ b/.github/workflows/ci-slow.yml @@ -0,0 +1,66 @@ +name: CI Slow + +on: + push: + branches-ignore: + - 'main' + - 'docs' + +# cancel in-progress builds after a new commit +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILDER_VERSION: v0.9.84 + BUILDER_SOURCE: releases + BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net + PACKAGE_NAME: aws-crt-java + RUN: ${{ github.run_id }}-${{ github.run_number }} + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} + AWS_DEFAULT_REGION: us-east-1 + AWS_REGION: us-east-1 + +permissions: + id-token: write # This is required for requesting the JWT + +jobs: + linux-musl-arm: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + image: + - alpine-3.16-armv7 + - alpine-3.16-arm64 + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-duration-seconds: 14400 # these tests run slow and easily reach default cred expiry, hence change expiry to 4hrs + - name: Install qemu/docker + run: docker run --privileged --rm tonistiigi/binfmt --install all + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + + raspberry: + runs-on: ubuntu-24.04 # latest + strategy: + fail-fast: false + matrix: + image: + - raspbian-bullseye + steps: + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Install qemu/docker + run: docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7 + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 074f6afc1..f94c5078b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,46 +163,6 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - linux-musl-arm: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - image: - - alpine-3.16-armv7 - - alpine-3.16-arm64 - steps: - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - role-duration-seconds: 14400 # these tests run slow and easily reach default cred expiry, hence change expiry to 4hrs - - name: Install qemu/docker - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - - raspberry: - runs-on: ubuntu-24.04 # latest - strategy: - fail-fast: false - matrix: - image: - - raspbian-bullseye - steps: - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: Install qemu/docker - run: docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7 - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - windows: runs-on: windows-2025 # latest steps: @@ -277,122 +237,6 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream python3 codebuild/macos_compatibility_check.py - android: - # ubuntu-24.04 comes with Android tooling - name: Android - runs-on: ubuntu-24.04 # latest - steps: - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - name: Checkout Sources - uses: actions/checkout@v4 - with: - submodules: true - # Setup JDK 17 - - name: set up JDK 17 - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - # Ensure Gradle uses this JDK (important when toolchains are present) - - name: Point Gradle at JDK 17 - run: echo "ORG_GRADLE_JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV - - - name: Mirror ANDROID_HOME → ANDROID_SDK_ROOT - run: echo "ANDROID_SDK_ROOT=$ANDROID_HOME" >> "$GITHUB_ENV" - - # Install required Android components & accept licenses (NDK r28 + compileSdk 35) - - name: Install Android SDK packages - shell: bash - run: | - set -euo pipefail - SDKROOT="${ANDROID_SDK_ROOT:-$ANDROID_HOME}" - SDKMANAGER="$SDKROOT/cmdline-tools/latest/bin/sdkmanager" - - # Some images have cmdline-tools already; ensure path exists - if [[ ! -x "$SDKMANAGER" ]]; then - echo "cmdline-tools not found; installing…" - mkdir -p "$SDKROOT/cmdline-tools" - curl -sSL -o /tmp/clt.zip https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip - sudo unzip -q /tmp/clt.zip -d "$SDKROOT/cmdline-tools" - sudo mv "$SDKROOT/cmdline-tools/cmdline-tools" "$SDKROOT/cmdline-tools/latest" - SDKMANAGER="$SDKROOT/cmdline-tools/latest/bin/sdkmanager" - fi - - echo "Installing platform-tools, Build-Tools 35, platform android-35, NDK r28, and CMake…" - # Avoid 'yes' SIGPIPE killing the step: turn off pipefail just for these lines - set +o pipefail - yes | sudo "$SDKMANAGER" --install \ - "platform-tools" \ - "build-tools;35.0.0" \ - "platforms;android-35" \ - "ndk;28.0.12433566" \ - "cmake;3.22.1" - - yes | "$SDKMANAGER" --licenses - set -o pipefail - - # Quick sanity prints (non-fatal) - "$SDKROOT/ndk/28.0.12433566/ndk-build" -v >/dev/null 2>&1 || true - "$SDKMANAGER" --list | sed -n '1,80p' || true - - # Build and publish locally for the test app to find the SNAPSHOT version - - name: Build ${{ env.PACKAGE_NAME }} - run: | - # Manually set -Xmx (max heap size) to something huge (tested 2g and that works, but why not go bigger). - # Only in CI, gradle daemon runs out of memory during "lintAnalyzeDebug" task, unless you specify it this way. - # You'd think Java's default of 25% RAM (ubuntu24 runner has 12g, so max 4g) would be sufficient, but no. - # You'd think setting -Xmx via gradle.properties would help, but no. - ./gradlew :android:crt:build -Dorg.gradle.jvmargs="-Xmx8g" - ./gradlew -PnewVersion="1.0.0-SNAPSHOT" :android:crt:publishToMavenLocal - # Setup files required by test app for Device Farm testing - - name: Setup Android Test Files - run: | - cd src/test/android/testapp/src/main/assets - python3 -m pip install boto3 - python3 ./android_file_creation.py - - - name: Set Android keystore home - run: | - echo "ANDROID_SDK_HOME=$GITHUB_WORKSPACE/.android-home" >> "$GITHUB_ENV" - echo "ANDROID_PREFS_ROOT=$GITHUB_WORKSPACE/.android-home" >> "$GITHUB_ENV" - mkdir -p "$GITHUB_WORKSPACE/.android-home/.android" - - - name: Create debug keystore - run: | - keytool -genkeypair \ - -keystore "$ANDROID_SDK_HOME/.android/debug.keystore" \ - -storepass android -keypass android \ - -alias androiddebugkey \ - -dname "CN=Android Debug,O=Android,C=US" \ - -keyalg RSA -keysize 2048 -validity 14000 - - - name: Build Test App - run: | - cd src/test/android/testapp - ../../../../gradlew assembledebug - ../../../../gradlew assembleAndroidTest - - name: Device Farm Tests Highly Available - run: | - echo "Running Device Farm Python Script" - python3 ./.github/workflows/run_android_ci.py \ - --run_id ${{ github.run_id }} \ - --run_attempt ${{ github.run_attempt }} \ - --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool highly_available - - name: Device Farm Tests Android 8.0.0 - run: | - echo "Running Device Farm Python Script" - python3 ./.github/workflows/run_android_ci.py \ - --run_id ${{ github.run_id }} \ - --run_attempt ${{ github.run_attempt }} \ - --project_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/ProjectArn" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool_arn $(aws secretsmanager get-secret-value --region us-east-1 --secret-id "ci/DeviceFarm/DevicePoolArnAndroid8" --query "SecretString" | cut -f5 -d\" | cut -f1 -d'\') \ - --device_pool android_8 - # check that docs can still build check-docs: runs-on: ubuntu-22.04 # use same version as docs.yml diff --git a/src/native/crt.c b/src/native/crt.c index 463f31e34..c0c050497 100644 --- a/src/native/crt.c +++ b/src/native/crt.c @@ -54,6 +54,15 @@ struct aws_allocator *aws_jni_get_allocator(void) { return s_allocator; } +/* +Dispatch queue threads are handled differently than other types as we do not create and manage them. This is set during +creation of an Event Loop Group to control attach/detach to/from JVM behavior of threads. +*/ +static bool s_dispatch_queue_threads = false; +void aws_jni_set_dispatch_queue_threads(bool is_dispatch_queue) { + s_dispatch_queue_threads = is_dispatch_queue; +} + static void s_detach_jvm_from_thread(void *user_data) { AWS_LOGF_DEBUG(AWS_LS_COMMON_GENERAL, "s_detach_jvm_from_thread invoked"); JavaVM *jvm = user_data; @@ -63,7 +72,6 @@ static void s_detach_jvm_from_thread(void *user_data) { JNIEnv *env = aws_jni_acquire_thread_env(jvm); if (env != NULL) { (*jvm)->DetachCurrentThread(jvm); - aws_jni_release_thread_env(jvm, env); /********** JNI ENV RELEASE **********/ } @@ -76,7 +84,9 @@ static JNIEnv *s_aws_jni_get_thread_env(JavaVM *jvm) { void *env = NULL; #endif if ((*jvm)->GetEnv(jvm, (void **)&env, JNI_VERSION_1_6) == JNI_EDETACHED) { - AWS_LOGF_DEBUG(AWS_LS_COMMON_GENERAL, "s_aws_jni_get_thread_env returned detached, attaching"); + if (!s_dispatch_queue_threads) { + AWS_LOGF_DEBUG(AWS_LS_COMMON_GENERAL, "s_aws_jni_get_thread_env returned detached, attaching"); + } struct aws_string *thread_name = NULL; if (aws_thread_current_name(aws_jni_get_allocator(), &thread_name)) { @@ -109,9 +119,16 @@ static JNIEnv *s_aws_jni_get_thread_env(JavaVM *jvm) { fprintf(stderr, "Unrecoverable AttachCurrentThreadAsDaemon failed, JNI error code is %d\n", (int)result); return NULL; } - /* This should only happen in event loop threads, the JVM main thread attachment is - * managed by the JVM, so we only need to clean up event loop thread attachments */ - AWS_FATAL_ASSERT(AWS_OP_SUCCESS == aws_thread_current_at_exit(s_detach_jvm_from_thread, (void *)jvm)); + + /* + Dispatch Queue threads are managed by Apple's GCD and thus can't have an at_exit callback assigned. We manually + detatch dispatch queue threads from the JVM during `aws_jni_release_thread_env()` to insure cleanup. + */ + if (!s_dispatch_queue_threads) { + /* This should only happen in event loop threads, the JVM main thread attachment is + * managed by the JVM, so we only need to clean up event loop thread attachments */ + AWS_FATAL_ASSERT(AWS_OP_SUCCESS == aws_thread_current_at_exit(s_detach_jvm_from_thread, (void *)jvm)); + } } return env; @@ -239,6 +256,15 @@ void aws_jni_release_thread_env(JavaVM *jvm, JNIEnv *env) { (void)env; if (env != NULL) { + /* + Dispatch Queue threads must be manually detached after they're used instead of depending + on a thread exit callback due to the threads not being managed by the CRT and thus, their + lifetimes not trackable outside the context of their immediate use. + */ + if (s_dispatch_queue_threads) { + (*jvm)->DetachCurrentThread(jvm); + } + aws_rw_lock_runlock(&s_jvm_table_lock); } } diff --git a/src/native/crt.h b/src/native/crt.h index 2a7355394..3c5f26f1e 100644 --- a/src/native/crt.h +++ b/src/native/crt.h @@ -239,6 +239,11 @@ JNIEnv *aws_jni_acquire_thread_env(JavaVM *jvm); ******************************************************************************/ void aws_jni_release_thread_env(JavaVM *jvm, JNIEnv *env); +/******************************************************************************* + * aws_jni_set_dispatch_queue_threads - Sets whether the current event loop group uses dispatch queue threads + ******************************************************************************/ +void aws_jni_set_dispatch_queue_threads(bool is_dispatch_queue); + /******************************************************************************* * aws_jni_new_crt_exception_from_error_code - Creates a new jobject from the aws * error code, which is the type of software/amazon/awssdk/crt/CrtRuntimeException. diff --git a/src/native/event_loop_group.c b/src/native/event_loop_group.c index a5a5495e1..9dbb3ccf7 100644 --- a/src/native/event_loop_group.c +++ b/src/native/event_loop_group.c @@ -107,6 +107,10 @@ jlong JNICALL Java_software_amazon_awssdk_crt_io_EventLoopGroup_eventLoopGroupNe goto on_error; } + if (aws_event_loop_group_get_type(elg) == AWS_EVENT_LOOP_DISPATCH_QUEUE) { + aws_jni_set_dispatch_queue_threads(true); + } + callback_data->java_event_loop_group = (*env)->NewGlobalRef(env, elg_jobject); return (jlong)elg; @@ -154,6 +158,10 @@ jlong JNICALL Java_software_amazon_awssdk_crt_io_EventLoopGroup_eventLoopGroupNe goto on_error; } + if (aws_event_loop_group_get_type(elg) == AWS_EVENT_LOOP_DISPATCH_QUEUE) { + aws_jni_set_dispatch_queue_threads(true); + } + callback_data->java_event_loop_group = (*env)->NewGlobalRef(env, elg_jobject); return (jlong)elg;