integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml

name: Mobile Integration Tests (LLM)

on:
  workflow_call:
    inputs:
      ref:
        description: "Git ref to checkout"
        type: string
        required: false
      repository:
        description: "Repository to checkout"
        type: string
        required: false
      qvac_perf_runs:
        description: "Override QVAC_PERF_RUNS (number of counted iterations per perf test). Empty = test default."
        type: string
        required: false
        default: ""
      qvac_perf_warmup_runs:
        description: "Override QVAC_PERF_WARMUP_RUNS (number of warmup iterations per perf test). Empty = test default."
        type: string
        required: false
        default: ""
  workflow_dispatch:
    inputs:
      ref:
        description: "Git ref (branch/tag/SHA) to test"
        type: string
        required: false
        default: main
      package:
        description: "Full NPM package spec to test (default: @qvac/llm-llamacpp@latest)"
        type: string
        required: true
        default: "@qvac/llm-llamacpp@latest"
      qvac_perf_runs:
        description: "Override QVAC_PERF_RUNS (number of counted iterations per perf test). Empty = test default."
        type: string
        required: false
        default: ""
      qvac_perf_warmup_runs:
        description: "Override QVAC_PERF_WARMUP_RUNS (number of warmup iterations per perf test). Empty = test default."
        type: string
        required: false
        default: ""

env:
  NODE_VERSION: "lts/*"
  ADDON_NAME: "@qvac/llm-llamacpp"
  PREBUILD_ARTIFACT_PREFIX: "llama-cpp-" # Prefix for prebuild artifacts (empty string for generic patterns)
  TEST_FRAMEWORK_REF: "main" # Branch/tag of qvac-test-addon-mobile framework
  APP_BUNDLE_ID: "io.tether.test.qvac" # Bundle ID for the test app (same for all addons)
  ADDON_WORKDIR: "addon/packages/qvac-lib-infer-llamacpp-llm"

jobs:
  build-and-test:
    name: Build ${{ matrix.platform }} and Run E2E Tests
    runs-on: ${{ matrix.runner }}
    environment: release
    timeout-minutes: 120
    continue-on-error: true # Don't block PR merges if tests fail
    permissions:
      contents: read
      packages: read
      pull-requests: write # Allow commenting on PRs
      id-token: write
    strategy:
      fail-fast: false
      matrix:
        include:
          - platform: Android
            os: ubuntu-24.04
            runner: ai-run-linux
          - platform: iOS
            os: macos-14
            runner: macos-14

    steps:
      # Validate input data to prevent "Artifact Poisoning"
      - name: Validate Dispatch Inputs
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.package
        run: |
          if [[ ! "${{ github.event.inputs.package }}" =~ ^@qvac/ ]]; then
            echo "::error::Invalid package scope. Only @qvac/* is allowed."
            exit 1
          fi

      # Free up disk space on Ubuntu runner to prevent "No space left on device" errors
      - name: Free up disk space
        if: matrix.platform == 'Android'
        run: |
          echo "Disk space before cleanup:"
          df -h
          # Remove unnecessary software to free up disk space (|| true to handle self-hosted runners)
          sudo rm -rf /usr/share/dotnet || true
          sudo rm -rf /opt/ghc || true
          sudo rm -rf /opt/hostedtoolcache/CodeQL || true
          sudo rm -rf /opt/hostedtoolcache/go || true
          sudo rm -rf /opt/hostedtoolcache/Python || true
          sudo rm -rf /opt/hostedtoolcache/Ruby || true
          sudo rm -rf /usr/local/lib/android/sdk/ndk || true
          sudo rm -rf /usr/local/share/boost || true
          sudo rm -rf /usr/share/swift || true
          sudo docker image prune --all --force || true
          # Clean APT cache
          sudo apt-get clean || true
          echo "Disk space after cleanup:"
          df -h

      - name: Checkout addon repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
        with:
          repository: ${{ inputs.repository || github.repository }}
          ref: ${{ inputs.ref || github.ref }}
          token: ${{ secrets.PAT_TOKEN }}
          path: addon
          fetch-depth: 0

      - name: Checkout mobile test framework
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
        with:
          repository: tetherto/qvac-test-addon-mobile
          ref: ${{ env.TEST_FRAMEWORK_REF }}
          token: ${{ secrets.PAT_TOKEN }}
          path: test-framework
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # 6.3.0
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Install global dependencies
        run: |
          echo "Installing global dependencies..."
          npm install -g @expo/cli@latest

      - name: Download Android prebuilds (from artifacts)
        if: matrix.platform == 'Android' && !inputs.package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
        with:
          path: ${{ env.ADDON_WORKDIR }}/prebuilds
          pattern: ${{ env.PREBUILD_ARTIFACT_PREFIX }}android-*
          merge-multiple: true
        continue-on-error: true

      - name: Download iOS prebuilds (from artifacts)
        if: matrix.platform == 'iOS' && !inputs.package
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
        with:
          path: ${{ env.ADDON_WORKDIR }}/prebuilds
          pattern: ${{ env.PREBUILD_ARTIFACT_PREFIX }}ios-*
          merge-multiple: true
        continue-on-error: true

      - name: Download prebuilds from package
        if: inputs.package
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          PACKAGE_SPEC="${{ inputs.package }}"
          echo "📦 Downloading $PACKAGE_SPEC from npm for manual trigger..."

          PACKAGE_NAME="${PACKAGE_SPEC%@*}"

          if ! npm pack "$PACKAGE_SPEC" --ignore-scripts; then
            echo "ERROR: Failed to download $PACKAGE_SPEC from npm"
            echo "Please check that the package exists at https://www.npmjs.com/package/$PACKAGE_NAME"
            exit 1
          fi

          # Extract the tarball (pattern matches any addon name)
          tar -xzf *.tgz

          # Validate prebuilds directory exists
          if [ ! -d "package/prebuilds" ]; then
            echo "ERROR: No prebuilds directory found in package"
            echo "The downloaded package may not contain prebuilt binaries"
            exit 1
          fi

          # Move prebuilds to expected location
          mv package/prebuilds ./prebuilds

          # Cleanup
          rm -rf package *.tgz

          echo "✅ Prebuilds downloaded from npm:"
          ls -la prebuilds/

      - name: Verify and prepare prebuilds
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          echo "Checking for prebuilds..."
          echo "Current directory: $(pwd)"
          if [ -d "prebuilds" ] && [ "$(ls -A prebuilds)" ]; then
            echo "✅ Prebuilds found from artifacts:"
            ls -la prebuilds/
          else
            echo "❌ ERROR: No prebuilds found!"
            echo "   This workflow requires prebuilds to be available."
            echo "   Either:"
            echo "   1. Run this workflow after prebuild job completes"
            echo "   2. Or commit prebuilds to the repository"
            exit 1
          fi

          # Copy mobile prebuilds if needed
          if npm run mobile:copy-prebuilds 2>/dev/null; then
            echo "✅ Mobile prebuilds prepared"
          else
            echo "⚠️  mobile:copy-prebuilds script not available or failed"
          fi

      - name: Remove desktop prebuilds to save disk space
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          echo "Removing desktop prebuilds to save disk space (keeping Android + iOS)..."
          echo "Before cleanup:"
          du -sh prebuilds/* 2>/dev/null || true

          # Remove desktop prebuilds only (not needed for mobile tests)
          rm -rf prebuilds/darwin-* prebuilds/win32-* prebuilds/linux-* 2>/dev/null || true

          echo "After cleanup (Android + iOS only):"
          du -sh prebuilds/* 2>/dev/null || true
          df -h

      - name: Verify test files exist
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          echo "Verifying addon has mobile tests..."

          if [ ! -d "test/mobile" ]; then
            echo "❌ ERROR: test/mobile directory not found!"
            echo ""
            echo "This workflow requires the addon to have mobile tests at:"
            echo "  test/mobile/"
            echo ""
            echo "Please create this directory with your test files."
            echo "See qvac-test-addon-mobile README for test file format."
            exit 1
          fi

          # Check for .cjs test files
          CJS_COUNT=$(find test/mobile -name "*.cjs" -type f | wc -l)
          if [ "$CJS_COUNT" -eq 0 ]; then
            echo "❌ ERROR: No .cjs test files found in test/mobile!"
            exit 1
          fi

          echo "✅ Mobile test files found:"
          ls -la test/mobile/*.cjs

          # Check if testAssets exists
          if [ -d "test/mobile/testAssets" ]; then
            echo ""
            echo "✅ Test assets found:"
            ls -lah test/mobile/testAssets/
          else
            echo ""
            echo "ℹ️  No testAssets directory (this is optional)"
          fi

      - name: Install Ninja build tool
        if: matrix.platform == 'iOS'
        run: |
          echo "📦 Installing Ninja build system..."
          brew install ninja
          ninja --version
          echo "✅ Ninja installed successfully"

      - name: Install addon dependencies
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          echo "Installing addon dependencies..."
          npm install --ignore-scripts

      - name: Validate mobile tests are up-to-date
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: npm run test:mobile:validate

      - name: Pack addon
        working-directory: ${{ env.ADDON_WORKDIR }}
        run: |
          echo "Packing addon..."
          if npm run build:pack 2>/dev/null; then
            echo "✅ Addon packed using build:pack script"
          else
            echo "📦 Using npm pack directly..."
            mkdir -p dist
            npm pack --pack-destination dist --ignore-scripts
          fi

          # Verify pack file exists
          PACK_FILE=$(ls dist/*.tgz | head -1)
          if [ -f "$PACK_FILE" ]; then
            SIZE=$(du -h "$PACK_FILE" | cut -f1)
            echo "✅ Pack file created: $PACK_FILE (Size: $SIZE)"
          else
            echo "❌ Pack file not found in dist/"
            exit 1
          fi

      - name: Setup test framework dependencies
        working-directory: test-framework
        run: |
          echo "Setting up mobile test framework..."
          npm install --ignore-scripts
          echo "✅ Test framework dependencies installed"

      - name: Build test app with addon
        working-directory: test-framework
        run: |
          echo "Building test app with addon..."
          echo "This will:"
          echo "  1. Install the addon package"
          echo "  2. Extract test code from addon's test/mobile/ directory"
          echo "  3. Auto-detect and order test files by dependencies"
          echo "  4. Generate backend.cjs with test functions"
          echo "  5. Generate e2e tests for each test function"
          echo "  6. Copy testAssets if available"
          echo "  7. Bundle the app"
          echo ""

          ADDON_PATH="${{ github.workspace }}/${{ env.ADDON_WORKDIR }}"
          npm run build "$ADDON_PATH" "$ADDON_PATH/test/mobile"

          echo ""
          echo "✅ Test app built successfully"

          # Verify critical files were generated
          if [ ! -f "backend/backend.cjs" ]; then
            echo "❌ ERROR: backend/backend.cjs was not generated!"
            exit 1
          fi

          if [ ! -f "e2e/tests/app.test.js" ]; then
            echo "❌ ERROR: e2e/tests/app.test.js was not generated!"
            exit 1
          fi

          if [ ! -f "backend/app.bundle" ]; then
            echo "❌ ERROR: backend/app.bundle was not created!"
            exit 1
          fi

          echo "✅ All required files generated successfully"

          # Show what tests were extracted
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "EXTRACTED TEST FUNCTIONS:"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          if [ -f "app/testConfig.js" ]; then
            cat app/testConfig.js
          fi
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

      - name: Display build summary
        if: always()
        working-directory: test-framework
        run: |
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "📊 BUILD SUMMARY"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""
          echo "Platform: ${{ matrix.platform }}"
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo "Package: ${{ inputs.package }}"
          else
            echo "Addon: ${{ env.ADDON_NAME }} (from PR artifacts)"
          fi
          echo ""
          echo "Generated Files:"
          echo "  backend/backend.cjs: $([ -f backend/backend.cjs ] && echo '✅' || echo '❌')"
          echo "  backend/app.bundle: $([ -f backend/app.bundle ] && echo '✅' || echo '❌')"
          echo "  app/testConfig.js: $([ -f app/testConfig.js ] && echo '✅' || echo '❌')"
          echo "  app/assetManifest.js: $([ -f app/assetManifest.js ] && echo '✅' || echo '❌')"
          echo "  e2e/tests/app.test.js: $([ -f e2e/tests/app.test.js ] && echo '✅' || echo '❌')"
          echo ""
          echo "Test Assets:"
          if [ -d "testAssets" ]; then
            ASSET_COUNT=$(find testAssets -type f | wc -l)
            echo "  ✅ $ASSET_COUNT file(s) in testAssets/"
          else
            echo "  ℹ️  No testAssets (optional)"
          fi
          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

      # Android-specific steps
      - name: Set up JDK 17
        if: matrix.platform == 'Android'
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # 5.2.0
        with:
          java-version: 17
          distribution: temurin

      - name: Setup Android SDK
        if: matrix.platform == 'Android'
        uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # 3.2.2

      - name: Generate Android project
        if: matrix.platform == 'Android'
        working-directory: test-framework
        run: |
          echo "Generating Android project with Expo..."
          npx expo prebuild --platform android --clean

      - name: Build Android APK
        if: matrix.platform == 'Android'
        id: build_apk
        working-directory: test-framework
        run: |
          echo "Building Android APK for Device Farm..."
          export JAVA_HOME=$JAVA_HOME_17_X64

          # Bundle JavaScript
          echo "Bundling JavaScript code..."
          npm run bundle

          if [ $? -ne 0 ]; then
            echo "❌ Bundle failed"
            exit 1
          fi

          echo "✅ Bundle completed successfully"

          # Build RELEASE APK (not debug) to ensure JS bundle is included
          # Debug builds skip bundling by default and try to connect to Metro
          # Release builds embed the JS bundle in the APK
          cd android
          echo "Building APK with Gradle (RELEASE with embedded JS bundle)..."
          ./gradlew assembleRelease \
            -PreactNativeArchitectures=arm64-v8a \
            --no-daemon \
            --no-build-cache \
            --stacktrace
          cd ..

          # Find the APK (look for release)
          APK_PATH=$(find android/app/build/outputs/apk -name "*.apk" | grep "release" | grep -v "unaligned" | head -1)

          if [ -f "$APK_PATH" ]; then
            # Convert to absolute path
            APK_ABSOLUTE_PATH="${{ github.workspace }}/test-framework/$APK_PATH"
            SIZE=$(du -h "$APK_PATH" | cut -f1)
            echo "✅ APK built successfully: $APK_PATH (Size: $SIZE)"
            echo "apk_path=$APK_ABSOLUTE_PATH" >> $GITHUB_OUTPUT
            echo "app_type=ANDROID_APP" >> $GITHUB_OUTPUT
            echo "app_name=test-app-${{ matrix.platform }}.apk" >> $GITHUB_OUTPUT

            # Clean up build intermediates to free disk space
            echo "Cleaning up build intermediates..."
            rm -rf android/app/build/intermediates
            rm -rf android/.gradle
            df -h
          else
            echo "❌ APK file not found"
            echo "Searching in android/app/build/outputs/apk:"
            find android/app/build/outputs/apk -type f 2>/dev/null || echo "Directory not found"
            exit 1
          fi

      # iOS-specific steps
      - name: Set up Xcode version
        if: matrix.platform == 'iOS'
        run: |
          echo "Available Xcode versions:"
          ls /Applications | grep Xcode || echo "No Xcode apps found"

          echo ""
          echo "Current Xcode (before switch):"
          xcodebuild -version

          # React Native requires Xcode >= 16.1
          # Use Xcode 16.1 (has iOS 18.1 SDK which is stable and pre-installed)
          if [ -d "/Applications/Xcode_16.1.app" ]; then
            echo ""
            echo "✅ Switching to Xcode 16.1..."
            sudo xcode-select -s /Applications/Xcode_16.1.app
          elif [ -d "/Applications/Xcode_16.1.0.app" ]; then
            echo ""
            echo "✅ Switching to Xcode 16.1.0..."
            sudo xcode-select -s /Applications/Xcode_16.1.0.app
          elif [ -d "/Applications/Xcode_16.2.app" ]; then
            echo ""
            echo "⚠️  Using Xcode 16.2 (16.1 not found)..."
            sudo xcode-select -s /Applications/Xcode_16.2.app
          else
            echo ""
            echo "❌ ERROR: No suitable Xcode version found (need >= 16.1)"
            exit 1
          fi

          echo ""
          echo "Current Xcode (after switch):"
          xcodebuild -version

          echo ""
          echo "Available iOS SDKs:"
          xcodebuild -showsdks | grep -i ios

      - name: Install CocoaPods
        if: matrix.platform == 'iOS'
        run: |
          sudo gem install cocoapods
          pod --version

      - name: Create Keychain and Import Certificate
        if: matrix.platform == 'iOS'
        env:
          BUILD_CERTIFICATE_BASE64: ${{ secrets.TEST_APP_APPLE_DISTRIBUTION_CERTIFICATE  }}
          P12_PASSWORD: ${{ secrets.APPLE_P12_PASSWORD }}
          BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.TEST_APP_APPLE_PROVISIONING_PROFILE  }}
          KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
        run: |
          CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
          PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision
          KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db

          echo -n "$BUILD_CERTIFICATE_BASE64" | base64 --decode -o $CERTIFICATE_PATH
          echo -n "$BUILD_PROVISION_PROFILE_BASE64" | base64 --decode -o $PP_PATH

          security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
          security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

          security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
          security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
          security list-keychain -d user -s $KEYCHAIN_PATH

          # Extract UUID first, then copy with UUID as filename
          PP_UUID=$(/usr/libexec/PlistBuddy -c 'Print :UUID' /dev/stdin <<< $(security cms -D -i $PP_PATH))
          echo "PP_UUID=$PP_UUID" >> $GITHUB_ENV
          echo "Provisioning Profile UUID: $PP_UUID"

          # Copy provisioning profile with UUID as filename
          mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
          cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision

          security find-identity -p codesigning -v

      - name: Verify provisioning profile
        if: matrix.platform == 'iOS'
        run: |
          echo "🔍 Verifying provisioning profile..."
          echo "PP_UUID: $PP_UUID"

          PP_FILE=~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision
          if [ ! -f "$PP_FILE" ]; then
            echo "❌ Provisioning profile file not found at: $PP_FILE"
            ls -la ~/Library/MobileDevice/Provisioning\ Profiles/
            exit 1
          fi

          echo "📋 Provisioning Profile Details:"
          security cms -D -i "$PP_FILE" > /tmp/profile.plist

          PROFILE_NAME=$(/usr/libexec/PlistBuddy -c "Print :Name" /tmp/profile.plist 2>/dev/null || echo "Unknown")
          PROFILE_BUNDLE_ID=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:application-identifier" /tmp/profile.plist 2>/dev/null || echo "Unknown")
          PROFILE_TEAM_ID=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:com.apple.developer.team-identifier" /tmp/profile.plist 2>/dev/null || echo "Unknown")

          # Detect profile type (Development, Ad Hoc, App Store, Enterprise)
          HAS_DEVICES=$(/usr/libexec/PlistBuddy -c "Print :ProvisionedDevices" /tmp/profile.plist 2>/dev/null && echo "yes" || echo "no")
          PROVISIONS_ALL=$(/usr/libexec/PlistBuddy -c "Print :ProvisionsAllDevices" /tmp/profile.plist 2>/dev/null || echo "false")
          HAS_GET_TASK_ALLOW=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:get-task-allow" /tmp/profile.plist 2>/dev/null || echo "false")

          if [[ "$PROVISIONS_ALL" == "true" ]]; then
            PROFILE_TYPE="Enterprise"
            EXPORT_METHOD="enterprise"
          elif [[ "$HAS_DEVICES" == "yes" && "$HAS_GET_TASK_ALLOW" == "true" ]]; then
            PROFILE_TYPE="Development"
            EXPORT_METHOD="development"
          elif [[ "$HAS_DEVICES" == "yes" && "$HAS_GET_TASK_ALLOW" == "false" ]]; then
            PROFILE_TYPE="Ad Hoc"
            EXPORT_METHOD="ad-hoc"
          else
            PROFILE_TYPE="App Store"
            EXPORT_METHOD="app-store"
          fi

          echo "  Name: $PROFILE_NAME"
          echo "  Type: $PROFILE_TYPE"
          echo "  Export Method: $EXPORT_METHOD"
          echo "  Application ID: $PROFILE_BUNDLE_ID"
          echo "  Team ID: $PROFILE_TEAM_ID"
          echo "  Expected Bundle ID: ${{ env.APP_BUNDLE_ID }}"

          # Save export method for next step
          echo "EXPORT_METHOD=$EXPORT_METHOD" >> $GITHUB_ENV

          # Extract just the bundle ID part (remove team prefix)
          BUNDLE_ID_ONLY=$(echo "$PROFILE_BUNDLE_ID" | sed 's/^[^.]*\.//')

          if [[ "$BUNDLE_ID_ONLY" != "${{ env.APP_BUNDLE_ID }}" ]]; then
            echo ""
            echo "❌ ERROR: Provisioning profile bundle ID mismatch!"
            echo "   Profile has: $BUNDLE_ID_ONLY"
            echo "   Expected: ${{ env.APP_BUNDLE_ID }}"
            echo ""
            echo "The provisioning profile was created for a different bundle identifier."
            echo "Please create a new provisioning profile for: ${{ env.APP_BUNDLE_ID }}"
            exit 1
          fi

          echo "✅ Provisioning profile matches expected bundle ID"

      - name: Generate iOS project
        if: matrix.platform == 'iOS'
        working-directory: test-framework
        run: |
          echo "Generating iOS project with Expo..."
          npx expo prebuild --platform ios --clean

      - name: Install iOS dependencies
        if: matrix.platform == 'iOS'
        working-directory: test-framework/ios
        run: |
          echo "Installing CocoaPods dependencies..."
          pod install --repo-update

      - name: Build and Archive iOS App
        if: matrix.platform == 'iOS'
        id: build_ios
        working-directory: test-framework
        run: |
          echo "Building iOS app for Device Farm..."

          # Bundle JavaScript first
          echo "Bundling JavaScript code..."
          npm run bundle

          if [ $? -ne 0 ]; then
            echo "❌ Bundle failed"
            exit 1
          fi

          echo "✅ Bundle completed successfully"

          # Get scheme name
          cd ios
          SCHEME_NAME=$(xcodebuild -list | grep -A 1 "Schemes:" | grep -v "Schemes:" | head -1 | xargs)
          echo "Detected scheme: $SCHEME_NAME"

          # Debug: Check bundle identifier in project
          echo "🔍 Checking project configuration..."
          BUNDLE_ID=$(xcodebuild -showBuildSettings -workspace $SCHEME_NAME.xcworkspace -scheme "$SCHEME_NAME" -configuration Release -destination "generic/platform=iOS" 2>/dev/null | grep PRODUCT_BUNDLE_IDENTIFIER | head -1 | awk '{print $3}')
          echo "Bundle Identifier in project: $BUNDLE_ID"

          if [[ "$BUNDLE_ID" != "${{ env.APP_BUNDLE_ID }}" ]]; then
            echo "⚠️  Warning: Bundle ID mismatch in Xcode project!"
            echo "   Expected: ${{ env.APP_BUNDLE_ID }}"
            echo "   Found: $BUNDLE_ID"
          fi

          # Debug: Check provisioning profile
          echo "🔍 Provisioning profile UUID: $PP_UUID"
          security cms -D -i ~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision | grep -A 5 "application-identifier\|Name\|TeamIdentifier" | head -20 || echo "Could not read profile details"

          # Archive for iOS device
          xcodebuild -workspace $SCHEME_NAME.xcworkspace \
            -scheme "$SCHEME_NAME" \
            -sdk iphoneos \
            -configuration Release \
            -destination "generic/platform=iOS" \
            -archivePath $RUNNER_TEMP/$SCHEME_NAME.xcarchive \
            CODE_SIGN_STYLE=Manual \
            PROVISIONING_PROFILE_SPECIFIER="$PP_UUID" \
            CODE_SIGN_IDENTITY="Apple Distribution" \
            DEVELOPMENT_TEAM="${{ secrets.APPLE_TEAM_ID }}" \
            clean archive

      - name: Export IPA
        if: matrix.platform == 'iOS'
        id: export_ipa
        working-directory: test-framework/ios
        run: |
          SCHEME_NAME=$(xcodebuild -list | grep -A 1 "Schemes:" | grep -v "Schemes:" | head -1 | xargs)

          # Create export options using auto-detected export method
          # The EXPORT_METHOD was determined in the "Verify provisioning profile" step
          echo "📦 Using export method: $EXPORT_METHOD"

          EXPORT_OPTS_PATH=$RUNNER_TEMP/ExportOptions.plist
          cat > $EXPORT_OPTS_PATH << EOF
          <?xml version="1.0" encoding="UTF-8"?>
          <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
          <plist version="1.0">
          <dict>
            <key>method</key>
            <string>$EXPORT_METHOD</string>
            <key>teamID</key>
            <string>${{ secrets.APPLE_TEAM_ID }}</string>
            <key>signingStyle</key>
            <string>manual</string>
            <key>provisioningProfiles</key>
            <dict>
              <key>${{ env.APP_BUNDLE_ID }}</key>
              <string>$PP_UUID</string>
            </dict>
          </dict>
          </plist>
          EOF

          echo "📋 Export options:"
          cat $EXPORT_OPTS_PATH

          xcodebuild -exportArchive \
            -archivePath $RUNNER_TEMP/$SCHEME_NAME.xcarchive \
            -exportOptionsPlist $EXPORT_OPTS_PATH \
            -exportPath $RUNNER_TEMP/build

          IPA_FILE=$(find $RUNNER_TEMP/build -name "*.ipa" | head -1)
          if [ -f "$IPA_FILE" ]; then
            echo "✅ IPA exported: $IPA_FILE"
            echo "apk_path=$IPA_FILE" >> $GITHUB_OUTPUT
            echo "app_type=IOS_APP" >> $GITHUB_OUTPUT
            echo "app_name=test-app-${{ matrix.platform }}.ipa" >> $GITHUB_OUTPUT
          else
            echo "❌ IPA file not found"
            exit 1
          fi

      - name: Configure AWS credentials via OIDC
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-west-2
          role-session-name: device-farm-upload
          role-duration-seconds: 7200 # 2hrs for device farm tests

      - name: Upload App to Device Farm
        id: upload_app
        run: |
          if [ "${{ matrix.platform }}" == "Android" ]; then
            APP_PATH="${{ steps.build_apk.outputs.apk_path }}"
            APP_TYPE="${{ steps.build_apk.outputs.app_type }}"
            APP_NAME="${{ steps.build_apk.outputs.app_name }}"
          else
            APP_PATH="${{ steps.export_ipa.outputs.apk_path }}"
            APP_TYPE="${{ steps.export_ipa.outputs.app_type }}"
            APP_NAME="${{ steps.export_ipa.outputs.app_name }}"
          fi

          echo "📤 Uploading app to AWS Device Farm..."
          UPLOAD_RESPONSE=$(aws devicefarm create-upload \
            --project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
            --name "$APP_NAME" \
            --type "$APP_TYPE" \
            --output json)

          if [ $? -ne 0 ]; then
            echo "❌ Error creating upload in Device Farm"
            echo "Response: $UPLOAD_RESPONSE"
            exit 1
          fi

          APP_UPLOAD_URL=$(echo $UPLOAD_RESPONSE | jq -r '.upload.url')
          APP_UPLOAD_ARN=$(echo $UPLOAD_RESPONSE | jq -r '.upload.arn')
          echo "app_upload_arn=$APP_UPLOAD_ARN" >> $GITHUB_OUTPUT
          echo "App upload ARN: $APP_UPLOAD_ARN"

          echo "Uploading app file: $APP_PATH"
          curl -T "$APP_PATH" "$APP_UPLOAD_URL"

          if [ $? -ne 0 ]; then
            echo "❌ Error uploading app file using curl"
            exit 1
          fi

          # Wait for processing
          echo "⏳ Waiting for upload to be processed..."
          MAX_ATTEMPTS=30
          ATTEMPT=1
          while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
            STATUS=$(aws devicefarm get-upload --arn "$APP_UPLOAD_ARN" --query "upload.status" --output text)
            echo "Status (attempt $ATTEMPT/$MAX_ATTEMPTS): $STATUS"

            if [ "$STATUS" = "SUCCEEDED" ]; then
              echo "✅ App upload successful"
              break
            fi

            if [ "$STATUS" = "FAILED" ]; then
              echo "❌ Upload failed"
              aws devicefarm get-upload --arn "$APP_UPLOAD_ARN"
              exit 1
            fi

            sleep 10
            ATTEMPT=$((ATTEMPT + 1))
          done

      - name: Verify test package generation
        working-directory: test-framework/e2e
        run: |
          echo "Verifying e2e test package..."

          if [ ! -f "package.json" ]; then
            echo "❌ ERROR: e2e/package.json not found!"
            exit 1
          fi

          if [ ! -f "tests/app.test.js" ]; then
            echo "❌ ERROR: e2e/tests/app.test.js not found!"
            exit 1
          fi

          echo "✅ E2E test files verified"
          echo ""
          echo "Test package contents:"
          ls -la
          echo ""
          echo "Test files:"
          ls -la tests/

      - name: Package and Upload Test Package
        id: upload_test_package
        working-directory: test-framework
        run: |
          echo "📦 Packaging e2e tests..."
          cd e2e

          # Install dependencies before packing
          npm install --ignore-scripts

          # Create tarball
          npm pack --ignore-scripts

          # Create zip with test files only (no node_modules - will be installed on Device Farm)
          ZIP_NAME="e2e-tests-${{ matrix.platform }}.zip"
          zip -r "$ZIP_NAME" \
            package.json \
            tests/ \
            *.tgz

          echo "📦 Package contents (excluding node_modules):"
          unzip -l "$ZIP_NAME" | head -20

          # Verify zip was created
          if [ ! -f "$ZIP_NAME" ]; then
            echo "❌ ERROR: Failed to create test package zip"
            exit 1
          fi

          SIZE=$(du -h "$ZIP_NAME" | cut -f1)
          echo "✅ Test package created: $ZIP_NAME (Size: $SIZE)"

          mv "$ZIP_NAME" "$GITHUB_WORKSPACE/"

          # Upload test package to AWS Device Farm
          echo "📤 Uploading test package to AWS Device Farm..."
          UPLOAD_RESPONSE=$(aws devicefarm create-upload \
            --project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
            --name "$ZIP_NAME" \
            --type "APPIUM_NODE_TEST_PACKAGE" \
            --output json)

          if [ $? -ne 0 ]; then
            echo "❌ Error creating test package upload in Device Farm"
            echo "Response: $UPLOAD_RESPONSE"
            exit 1
          fi

          TEST_UPLOAD_URL=$(echo $UPLOAD_RESPONSE | jq -r '.upload.url')
          TEST_UPLOAD_ARN=$(echo $UPLOAD_RESPONSE | jq -r '.upload.arn')
          echo "test_package_upload_arn=$TEST_UPLOAD_ARN" >> $GITHUB_OUTPUT
          echo "Test package upload ARN: $TEST_UPLOAD_ARN"

          echo "Uploading to: $TEST_UPLOAD_URL"
          curl -T "$GITHUB_WORKSPACE/$ZIP_NAME" "$TEST_UPLOAD_URL"

          if [ $? -ne 0 ]; then
            echo "❌ Error uploading test package using curl"
            exit 1
          fi

          # Wait for processing
          echo "⏳ Waiting for test package to be processed..."
          MAX_ATTEMPTS=30
          ATTEMPT=1
          while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
            STATUS=$(aws devicefarm get-upload --arn "$TEST_UPLOAD_ARN" --query "upload.status" --output text)
            echo "Test package status (attempt $ATTEMPT/$MAX_ATTEMPTS): $STATUS"

            if [ "$STATUS" = "SUCCEEDED" ]; then
              echo "✅ Test package upload successful"
              break
            fi

            if [ "$STATUS" = "FAILED" ]; then
              echo "❌ Test package upload failed"
              aws devicefarm get-upload --arn "$TEST_UPLOAD_ARN"
              exit 1
            fi

            sleep 10
            ATTEMPT=$((ATTEMPT + 1))
          done

          if [ $ATTEMPT -gt $MAX_ATTEMPTS ]; then
            echo "❌ Timeout waiting for test package processing"
            exit 1
          fi

      # NOTE: Everything below remains unchanged from your source workflow.
      # The only monorepo-related change in this entire file is that "addon" operations
      # now target addon/packages/qvac-lib-infer-llamacpp-llm via env.ADDON_WORKDIR.

      - name: Create and Upload Test Spec
        id: upload_test_spec
        run: |
          echo "📝 Creating test spec for custom environment mode..."
          echo "Platform: ${{ matrix.platform }}"
          
          if [ "${{ matrix.platform }}" == "Android" ]; then
            PLATFORM="Android"
            AUTOMATION="UiAutomator2"
            HOST_LINE="android_test_host: amazon_linux_2"
            BUNDLE_ID="${{ env.APP_BUNDLE_ID }}"
            WDIO_CONFIG='exports.config={runner:"local",hostname:"127.0.0.1",port:4723,path:"/wd/hub",specs:["*.spec.js","*.test.js"],maxInstances:1,bail:0,capabilities:[{platformName:"Android","appium:automationName":"UiAutomator2","appium:appPackage":"'${{ env.APP_BUNDLE_ID }}'","appium:appActivity":"'${{ env.APP_BUNDLE_ID }}'.MainActivity","appium:newCommandTimeout":300,"appium:autoGrantPermissions":true,"appium:autoAcceptAlerts":true,"appium:noReset":true,"appium:dontStopAppOnReset":true,"appium:forceAppLaunch":false}],logLevel:"debug",waitforTimeout:120000,connectionRetryTimeout:30000,connectionRetryCount:3,services:[],framework:"mocha",reporters:["spec"],mochaOpts:{ui:"bdd",timeout:1800000,grep:"__MOCHA_GREP__"},before:async function(capabilities,specs,browser){const BUNDLE_ID="'${{ env.APP_BUNDLE_ID }}'";const TEST_FILTER="__TEST_FILTER__";const QVAC_PERF_RUNS_VALUE="__QVAC_PERF_RUNS__";const QVAC_PERF_WARMUP_RUNS_VALUE="__QVAC_PERF_WARMUP_RUNS__";global.appCrashed=false;global.checkAppCrash=async(stage)=>{try{const state=await browser.queryAppState(BUNDLE_ID);console.log("["+stage+"] App state: "+state+" (4=foreground,3=background,1=not running)");if(state<3){console.error("\\n🛑 APP CRASHED at "+stage+"! State="+state);console.error("Check device logs for BareKit/native errors.\\n");global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);}return state;}catch(e){console.log("["+stage+"] queryAppState error: "+e.message);return-1;}};console.log("Checking initial app state...");await global.checkAppCrash("startup");console.log("Waiting for app to initialize...");await browser.pause(5000);await global.checkAppCrash("after-pause");const initText=await browser.$("android=new UiSelector().textContains(\"INITIALIZED\")");await initText.waitForDisplayed({timeout:60000});await global.checkAppCrash("after-init");if(TEST_FILTER!=="__TEST_FILTER__"){try{const b64=Buffer.from(TEST_FILTER).toString("base64");await browser.pushFile("/data/local/tmp/testFilter.txt",b64);console.log("Pushed test filter: "+TEST_FILTER);}catch(e){console.log("pushFile failed: "+e.message);}}if(QVAC_PERF_RUNS_VALUE.length>0||QVAC_PERF_WARMUP_RUNS_VALUE.length>0){try{var perfCfg="QVAC_PERF_RUNS="+QVAC_PERF_RUNS_VALUE+"\\nQVAC_PERF_WARMUP_RUNS="+QVAC_PERF_WARMUP_RUNS_VALUE+"\\n";var pcb64=Buffer.from(perfCfg).toString("base64");await browser.pushFile("/data/local/tmp/qvacPerfConfig.txt",pcb64);console.log("Pushed perf config: "+perfCfg.replace(/\\n/g," "));}catch(e){console.log("perfConfig pushFile failed: "+e.message);}}console.log("App initialized, clicking Run Automated Tests...");const button=await browser.$("android=new UiSelector().textContains(\"Run Automated Tests\")");await button.waitForDisplayed({timeout:15000});await button.click();console.log("Button clicked!");await browser.pause(5000);await global.checkAppCrash("after-click");global.crashMonitor=setInterval(async()=>{if(global.appCrashed)return;try{const s=await browser.queryAppState(BUNDLE_ID);if(s<3){console.error("\\n🛑 BACKGROUND CRASH DETECTED! App state="+s);global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);}}catch(e){}},15000);},after:async function(){if(global.crashMonitor)clearInterval(global.crashMonitor);},afterTest:async function(test,context,{error}){if(global.appCrashed)return;await global.checkAppCrash("after-test:"+test.title);}};'
          else
            PLATFORM="iOS"
            AUTOMATION="XCUITest"
            HOST_LINE="ios_test_host: macos_sequoia"
            BUNDLE_ID="${{ env.APP_BUNDLE_ID }}"
            WDIO_CONFIG='exports.config={runner:"local",hostname:"127.0.0.1",port:4723,path:"/wd/hub",specs:["*.spec.js","*.test.js"],maxInstances:1,bail:0,capabilities:[{platformName:"iOS","appium:automationName":"XCUITest","appium:bundleId":"'${{ env.APP_BUNDLE_ID }}'","appium:newCommandTimeout":300,"appium:noReset":true,"appium:forceAppLaunch":false,"appium:usePrebuiltWDA":true,"appium:wdaLocalPort":8100,"appium:showIOSLog":true,"appium:realDeviceLogger":"/usr/local/lib/node_modules/appium/node_modules/deviceconsole/deviceconsole"}],logLevel:"debug",waitforTimeout:120000,connectionRetryTimeout:30000,connectionRetryCount:3,services:[],framework:"mocha",reporters:["spec"],mochaOpts:{ui:"bdd",timeout:1800000,grep:"__MOCHA_GREP__"},before:async function(capabilities,specs,browser){const BUNDLE_ID="'${{ env.APP_BUNDLE_ID }}'";const TEST_FILTER="__TEST_FILTER__";const QVAC_PERF_RUNS_VALUE="__QVAC_PERF_RUNS__";const QVAC_PERF_WARMUP_RUNS_VALUE="__QVAC_PERF_WARMUP_RUNS__";global.appCrashed=false;global.flushBareLog=async function(reason){try{var _h=require("http");var lb64=await new Promise(function(ok,fail){var bd=JSON.stringify({path:"@"+BUNDLE_ID+":documents/bare_console.log"});var rq=_h.request({hostname:"127.0.0.1",port:4723,path:"/wd/hub/session/"+browser.sessionId+"/appium/device/pull_file",method:"POST",headers:{"Content-Type":"application/json","Content-Length":Buffer.byteLength(bd)}},function(rs){var d="";rs.on("data",function(c){d+=c;});rs.on("end",function(){try{ok(JSON.parse(d).value);}catch(e){fail(e);}});});rq.on("error",fail);rq.write(bd);rq.end();});var logTxt=Buffer.from(lb64,"base64").toString();var logDir=process.env.DEVICEFARM_LOG_DIR||".";require("fs").writeFileSync(logDir+"/bare_console.log",logTxt);console.log("[bare-log] "+reason+" flush ok ("+logTxt.length+" bytes)");}catch(e){console.log("[bare-log] "+reason+" flush failed: "+e.message);}};global.checkAppCrash=async(stage)=>{try{const state=await browser.queryAppState(BUNDLE_ID);console.log("["+stage+"] App state: "+state+" (4=foreground,3=background,1=not running)");if(state<3){console.error("\\n🛑 APP CRASHED at "+stage+"! State="+state);console.error("Check device logs for BareKit/native errors.\\n");global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);try{await browser.pause(1500);await Promise.race([global.flushBareLog("crash-"+stage),new Promise(function(_,rj){setTimeout(function(){rj(new Error("bare-log flush timed out"));},3000);})]);}catch(_){}}return state;}catch(e){console.log("["+stage+"] queryAppState error: "+e.message);return-1;}};console.log("Checking initial app state...");await global.checkAppCrash("startup");console.log("Waiting for app to initialize...");await browser.pause(5000);await global.checkAppCrash("after-pause");const initText=await browser.$("-ios predicate string:label CONTAINS \"INITIALIZED\"");await initText.waitForDisplayed({timeout:60000});await global.checkAppCrash("after-init");if(TEST_FILTER!=="__TEST_FILTER__"){try{const b64=Buffer.from(TEST_FILTER).toString("base64");await browser.pushFile("@"+BUNDLE_ID+":documents/testFilter.txt",b64);console.log("Pushed test filter: "+TEST_FILTER);}catch(e){console.log("pushFile failed: "+e.message);}}if(QVAC_PERF_RUNS_VALUE.length>0||QVAC_PERF_WARMUP_RUNS_VALUE.length>0){try{var perfCfg="QVAC_PERF_RUNS="+QVAC_PERF_RUNS_VALUE+"\\nQVAC_PERF_WARMUP_RUNS="+QVAC_PERF_WARMUP_RUNS_VALUE+"\\n";var pcb64=Buffer.from(perfCfg).toString("base64");await browser.pushFile("@"+BUNDLE_ID+":documents/qvacPerfConfig.txt",pcb64);console.log("Pushed perf config: "+perfCfg.replace(/\\n/g," "));}catch(e){console.log("perfConfig pushFile failed: "+e.message);}}console.log("App initialized, clicking Run Automated Tests...");const button=await browser.$("-ios predicate string:label CONTAINS \"Run Automated Tests\"");await button.waitForDisplayed({timeout:15000});await button.click();console.log("Button clicked!");await browser.pause(5000);await global.checkAppCrash("after-click");global.crashMonitor=setInterval(async()=>{if(global.appCrashed)return;try{const s=await browser.queryAppState(BUNDLE_ID);if(s<3){console.error("\\n🛑 BACKGROUND CRASH DETECTED! App state="+s);global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);try{await browser.pause(1500);await Promise.race([global.flushBareLog("crash-bg"),new Promise(function(_,rj){setTimeout(function(){rj(new Error("bare-log flush timed out"));},3000);})]);}catch(_){}}}catch(e){}},15000);},after:async function(){if(global.crashMonitor)clearInterval(global.crashMonitor);console.log("[bare-log] Waiting for log flush...");await browser.pause(3000);if(global.flushBareLog)await global.flushBareLog("after");},afterTest:async function(test,context,{error}){if(global.appCrashed)return;await global.checkAppCrash("after-test:"+test.title);}};'
          fi
          
          WDIO_B64=$(echo "$WDIO_CONFIG" | base64 | tr -d '\n')

          generate_spec() {
            local output_file="$1"
            local wdio_b64="$2"
            {
              printf 'version: 0.1\n'
              if [ -n "$HOST_LINE" ]; then
                printf '%s\n' "$HOST_LINE"
              fi
              printf '\nphases:\n  install:\n    commands:\n'
              printf '      - echo "Setting up Node.js environment..."\n'
              printf '      - export NVM_DIR=$HOME/.nvm\n'
              printf '      - . $NVM_DIR/nvm.sh 2>/dev/null || true\n'
              printf '      - nvm install 18 2>/dev/null || true\n'
              printf '      - nvm use 18 2>/dev/null || true\n'
              printf '      - node --version || echo "Using system node"\n'
              printf '\n  pre_test:\n    commands:\n'
              printf '      - echo "Setting up test environment..."\n'
              printf '      - cd $DEVICEFARM_TEST_PACKAGE_PATH\n'
              printf '      - ls -la\n'
              printf '      - echo "Installing dependencies (clean install)..."\n'
              printf '      - rm -rf node_modules package-lock.json 2>/dev/null || true\n'
              printf '      - npm install --legacy-peer-deps 2>&1\n'
              printf '      - echo "Verifying wdio installation..."\n'
              printf '      - ls -la node_modules/.bin/ | grep wdio || echo "wdio not found in .bin"\n'
              printf '      - node node_modules/@wdio/cli/bin/wdio.js --version || echo "wdio version check failed"\n'
              printf '      - echo "Creating wdio config for Device Farm..."\n'
              printf '      - echo "%s" | base64 -d > tests/wdio.config.devicefarm.js\n' "$wdio_b64"
              printf '      - cat tests/wdio.config.devicefarm.js\n'
              if [ "${{ matrix.platform }}" == "iOS" ]; then
                printf '      - echo "Configuring WebDriverAgent for iOS..."\n'
                printf '      - export DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH=$DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH_V9\n'
                printf '      - echo "WDA Path: $DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH"\n'
              fi
              printf '      - echo "Starting Appium server..."\n'
              printf '      - export APPIUM_BASE_PATH=/wd/hub\n'
              printf '      - |\n'
              printf '        appium --base-path=$APPIUM_BASE_PATH --log-timestamp \\\n'
              printf '          --log-no-colors --relaxed-security --default-capabilities \\\n'
              printf '          "{\\"appium:deviceName\\": \\"$DEVICEFARM_DEVICE_NAME\\", \\\n'
              printf '          \\"platformName\\": \\"$DEVICEFARM_DEVICE_PLATFORM_NAME\\", \\\n'
              printf '          \\"appium:app\\": \\"$DEVICEFARM_APP_PATH\\", \\\n'
              printf '          \\"appium:udid\\":\\"$DEVICEFARM_DEVICE_UDID\\", \\\n'
              printf '          \\"appium:platformVersion\\": \\"$DEVICEFARM_DEVICE_OS_VERSION\\", \\\n'
              printf '          \\"appium:chromedriverExecutableDir\\": \\"$DEVICEFARM_CHROMEDRIVER_EXECUTABLE_DIR\\", \\\n'
              printf '          \\"appium:wdaLocalPort\\": 8100, \\\n'
              printf '          \\"appium:derivedDataPath\\": \\"$DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH\\", \\\n'
              printf '          \\"appium:usePrebuiltWDA\\": true, \\\n'
              printf '          \\"appium:automationName\\": \\"%s\\"}" \\\n' "$AUTOMATION"
              printf '          >> $DEVICEFARM_LOG_DIR/appium.log 2>&1 &\n'
              printf '      - echo "Waiting for Appium to be ready (max 30 seconds)..."\n'
              printf '      - |\n'
              printf '        appium_initialization_time=0\n'
              printf '        until curl --silent --fail "http://0.0.0.0:4723${APPIUM_BASE_PATH}/status"; do\n'
              printf '          if [[ $appium_initialization_time -gt 30 ]]; then\n'
              printf '            echo "Appium did not start within 30 seconds. Exiting..."\n'
              printf '            cat $DEVICEFARM_LOG_DIR/appium.log\n'
              printf '            exit 1\n'
              printf '          fi\n'
              printf '          appium_initialization_time=$((appium_initialization_time + 1))\n'
              printf '          echo "Waiting for Appium to start on port 4723 (${appium_initialization_time}s/30s)..."\n'
              printf '          sleep 1\n'
              printf '        done\n'
              printf '      - echo "Appium server is ready!"\n'
              printf '      - curl -s http://0.0.0.0:4723${APPIUM_BASE_PATH}/status || echo "Status check failed"\n'
              printf '\n  test:\n    commands:\n'
              printf '      - echo "Running WebDriverIO tests..."\n'
              printf '      - cd $DEVICEFARM_TEST_PACKAGE_PATH\n'
              printf '      - ps aux | grep appium | grep -v grep || echo "Appium process not found"\n'
              printf '      - curl -s http://127.0.0.1:4723/wd/hub/status || echo "Appium status check failed"\n'
              printf '      - node node_modules/@wdio/cli/bin/wdio.js run tests/wdio.config.devicefarm.js\n'
              printf '\n  post_test:\n    commands:\n'
              printf '      - echo "Test completed"\n'
              if [ "${{ matrix.platform }}" == "iOS" ]; then
                printf '      - echo ""\n'
                printf '      - echo "iOS Device Console Logs"\n'
                printf '      - |\n'
                printf '        if [ -f "$DEVICEFARM_LOG_DIR/device_console.log" ]; then\n'
                printf '          grep -i "bare\\|console\\|model\\|embedding\\|test\\|error" "$DEVICEFARM_LOG_DIR/device_console.log" || echo "No matching logs"\n'
                printf '        else\n'
                printf '          echo "No device_console.log found"\n'
                printf '        fi\n'
                printf '      - ls -lh $DEVICEFARM_LOG_DIR/ || true\n'
              fi
              printf '\nartifacts:\n  - $DEVICEFARM_LOG_DIR\n'
            } > "$output_file"
            echo "Generated: $output_file"
          }

          upload_spec() {
            local spec_file="$1"
            local label="$2"
            local RESP=$(aws devicefarm create-upload \
              --project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
              --name "$spec_file" \
              --type "APPIUM_NODE_TEST_SPEC" \
              --output json)
            local URL=$(echo $RESP | jq -r '.upload.url')
            local ARN=$(echo $RESP | jq -r '.upload.arn')
            curl -T "$spec_file" "$URL"
            local MAX=20; local A=1
            while [ $A -le $MAX ]; do
              local ST=$(aws devicefarm get-upload --arn "$ARN" --query "upload.status" --output text)
              if [ "$ST" = "SUCCEEDED" ]; then echo "$ARN"; return 0; fi
              if [ "$ST" = "FAILED" ]; then echo "$label spec upload failed" >&2; exit 1; fi
              sleep 5; A=$((A + 1))
            done
            echo "$label spec upload timed out after $MAX attempts" >&2; exit 1
          }

          QVAC_PERF_RUNS_INPUT="${{ inputs.qvac_perf_runs }}"
          QVAC_PERF_WARMUP_RUNS_INPUT="${{ inputs.qvac_perf_warmup_runs }}"
          make_split() {
            local pattern="$1" output="$2"
            local cfg
            cfg=$(echo "$WDIO_CONFIG" | sed "s#__MOCHA_GREP__#$pattern#")
            cfg=$(echo "$cfg" | sed "s#__TEST_FILTER__#$pattern#")
            cfg=$(echo "$cfg" | sed "s#__QVAC_PERF_RUNS__#$QVAC_PERF_RUNS_INPUT#")
            cfg=$(echo "$cfg" | sed "s#__QVAC_PERF_WARMUP_RUNS__#$QVAC_PERF_WARMUP_RUNS_INPUT#")
            local b64
            b64=$(echo "$cfg" | base64 | tr -d '\n')
            generate_spec "$output" "$b64"
          }

          GROUPS_JSON="${{ github.workspace }}/${{ env.ADDON_WORKDIR }}/test/mobile/test-groups.json"

          if [ "${{ matrix.platform }}" == "Android" ]; then
            ANDROID_GROUP_A=$(jq -r '.android.groupA | join("|")' "$GROUPS_JSON")
            ANDROID_GROUP_B=$(jq -r '.android.groupB | join("|")' "$GROUPS_JSON")

            echo "Android test split (2 groups):"
            echo "  Group A: $ANDROID_GROUP_A"
            echo "  Group B: $ANDROID_GROUP_B"

            make_split "$ANDROID_GROUP_A" testspec-android-a.yml
            make_split "$ANDROID_GROUP_B" testspec-android-b.yml

            SPEC_ARN_A=$(upload_spec testspec-android-a.yml "Android-A")
            echo "test_spec_arn=$SPEC_ARN_A" >> $GITHUB_OUTPUT
            SPEC_ARN_B=$(upload_spec testspec-android-b.yml "Android-B")
            echo "test_spec_arn_2=$SPEC_ARN_B" >> $GITHUB_OUTPUT
          else
            IOS_GREP_HEAVY_1=$(jq -r '.ios.heavy1 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_2=$(jq -r '.ios.heavy2 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_3=$(jq -r '.ios.heavy3 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_4=$(jq -r '.ios.heavy4 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_5=$(jq -r '.ios.heavy5 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_6=$(jq -r '.ios.heavy6 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_7=$(jq -r '.ios.heavy7 | join("|")' "$GROUPS_JSON")
            IOS_GREP_HEAVY_8=$(jq -r '.ios.heavy8 | join("|")' "$GROUPS_JSON")
            IOS_GREP_LIGHT_A=$(jq -r '.ios.lightA | join("|")' "$GROUPS_JSON")
            IOS_GREP_LIGHT_B=$(jq -r '.ios.lightB | join("|")' "$GROUPS_JSON")

            echo "iOS test split (8 heavy + 2 light = 10 groups):"
            echo "  Heavy 1 (iPhone 16 Pro Max): $IOS_GREP_HEAVY_1"
            echo "  Heavy 2 (iPhone 16 Pro Max): $IOS_GREP_HEAVY_2"
            echo "  Heavy 3 (iPhone 17):         $IOS_GREP_HEAVY_3"
            echo "  Heavy 4 (OcrLighton):        $IOS_GREP_HEAVY_4"
            echo "  Heavy 5 (Reasoning):         $IOS_GREP_HEAVY_5"
            echo "  Heavy 6 (SlidingContext):    $IOS_GREP_HEAVY_6"
            echo "  Heavy 7 (Gemma4):            $IOS_GREP_HEAVY_7"
            echo "  Heavy 8 (OcrPaddle):         $IOS_GREP_HEAVY_8"
            echo "  Light A (iPhone 16 Pro):     $IOS_GREP_LIGHT_A"
            echo "  Light B (iPhone 16 Pro):     $IOS_GREP_LIGHT_B"

            make_split "$IOS_GREP_HEAVY_1" testspec-heavy-1.yml
            make_split "$IOS_GREP_HEAVY_2" testspec-heavy-2.yml
            make_split "$IOS_GREP_HEAVY_3" testspec-heavy-3.yml
            make_split "$IOS_GREP_HEAVY_4" testspec-heavy-4.yml
            make_split "$IOS_GREP_HEAVY_5" testspec-heavy-5.yml
            make_split "$IOS_GREP_HEAVY_6" testspec-heavy-6.yml
            make_split "$IOS_GREP_HEAVY_7" testspec-heavy-7.yml
            make_split "$IOS_GREP_HEAVY_8" testspec-heavy-8.yml
            make_split "$IOS_GREP_LIGHT_A" testspec-light-a.yml
            make_split "$IOS_GREP_LIGHT_B" testspec-light-b.yml

            SPEC_ARN_H1=$(upload_spec testspec-heavy-1.yml "iOS-Heavy1")
            echo "test_spec_arn=$SPEC_ARN_H1" >> $GITHUB_OUTPUT
            SPEC_ARN_H2=$(upload_spec testspec-heavy-2.yml "iOS-Heavy2")
            echo "test_spec_arn_2=$SPEC_ARN_H2" >> $GITHUB_OUTPUT
            SPEC_ARN_H3=$(upload_spec testspec-heavy-3.yml "iOS-Heavy3")
            echo "test_spec_arn_3=$SPEC_ARN_H3" >> $GITHUB_OUTPUT
            SPEC_ARN_H4=$(upload_spec testspec-heavy-4.yml "iOS-Heavy4")
            echo "test_spec_arn_4=$SPEC_ARN_H4" >> $GITHUB_OUTPUT
            SPEC_ARN_H5=$(upload_spec testspec-heavy-5.yml "iOS-Heavy5")
            echo "test_spec_arn_5=$SPEC_ARN_H5" >> $GITHUB_OUTPUT
            SPEC_ARN_H6=$(upload_spec testspec-heavy-6.yml "iOS-Heavy6")
            echo "test_spec_arn_6=$SPEC_ARN_H6" >> $GITHUB_OUTPUT
            SPEC_ARN_LA=$(upload_spec testspec-light-a.yml "iOS-LightA")
            echo "test_spec_arn_7=$SPEC_ARN_LA" >> $GITHUB_OUTPUT
            SPEC_ARN_LB=$(upload_spec testspec-light-b.yml "iOS-LightB")
            echo "test_spec_arn_8=$SPEC_ARN_LB" >> $GITHUB_OUTPUT
            SPEC_ARN_H7=$(upload_spec testspec-heavy-7.yml "iOS-Heavy7")
            echo "test_spec_arn_9=$SPEC_ARN_H7" >> $GITHUB_OUTPUT
            SPEC_ARN_H8=$(upload_spec testspec-heavy-8.yml "iOS-Heavy8")
            echo "test_spec_arn_10=$SPEC_ARN_H8" >> $GITHUB_OUTPUT
          fi

      - name: Schedule Device Farm Test Run
        id: schedule_run
        run: |
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            RUN_NAME="Manual-${{ github.run_number }}-${{ matrix.platform }}"
          else
            RUN_NAME="PR-${{ github.event.pull_request.number || github.run_number }}-${{ matrix.platform }}"
          fi

          PROJECT_ARN="${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}"
          APP_ARN="${{ steps.upload_app.outputs.app_upload_arn }}"
          TEST_PACKAGE_ARN="${{ steps.upload_test_package.outputs.test_package_upload_arn }}"

          schedule_run_with_pool() {
            local pool_arn="$1"
            local name="$2"
            local spec_arn="$3"
            local job_timeout="${4:-60}"
            aws devicefarm schedule-run \
              --project-arn "$PROJECT_ARN" \
              --device-pool-arn "$pool_arn" \
              --app-arn "$APP_ARN" \
              --name "$name" \
              --test "type=APPIUM_NODE,testPackageArn=$TEST_PACKAGE_ARN,testSpecArn=$spec_arn" \
              --execution-configuration jobTimeoutMinutes=$job_timeout \
              --query 'run.arn' --output text
          }

          if [ "${{ matrix.platform }}" == "Android" ]; then
            POOL_ARN="${{ secrets.LLM_ANDROID_DEVICE_POOL_ARN }}"
            TEST_SPEC_ARN_A="${{ steps.upload_test_spec.outputs.test_spec_arn }}"
            TEST_SPEC_ARN_B="${{ steps.upload_test_spec.outputs.test_spec_arn_2 }}"

            echo "🚀 Scheduling 2 Android runs (each runs on all devices in pool)..."
            echo "Device Pool ARN: $POOL_ARN"

            RUN_ARN_1=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-Android-GroupA" "$TEST_SPEC_ARN_A")
            echo "✅ Android Group A scheduled: $RUN_ARN_1"

            RUN_ARN_2=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-Android-GroupB" "$TEST_SPEC_ARN_B" 90)
            echo "✅ Android Group B scheduled: $RUN_ARN_2"

            echo "run_arn_1=$RUN_ARN_1" >> $GITHUB_OUTPUT
            echo "run_arn_2=$RUN_ARN_2" >> $GITHUB_OUTPUT
            echo "run_count=2" >> $GITHUB_OUTPUT
          else
            POOL_ARN="${{ secrets.LLM_IOS_DEVICE_POOL_ARN }}"
            TEST_SPEC_ARN_1="${{ steps.upload_test_spec.outputs.test_spec_arn }}"
            TEST_SPEC_ARN_2="${{ steps.upload_test_spec.outputs.test_spec_arn_2 }}"
            TEST_SPEC_ARN_3="${{ steps.upload_test_spec.outputs.test_spec_arn_3 }}"
            TEST_SPEC_ARN_4="${{ steps.upload_test_spec.outputs.test_spec_arn_4 }}"
            TEST_SPEC_ARN_5="${{ steps.upload_test_spec.outputs.test_spec_arn_5 }}"
            TEST_SPEC_ARN_6="${{ steps.upload_test_spec.outputs.test_spec_arn_6 }}"
            TEST_SPEC_ARN_7="${{ steps.upload_test_spec.outputs.test_spec_arn_7 }}"
            TEST_SPEC_ARN_8="${{ steps.upload_test_spec.outputs.test_spec_arn_8 }}"
            TEST_SPEC_ARN_9="${{ steps.upload_test_spec.outputs.test_spec_arn_9 }}"
            TEST_SPEC_ARN_10="${{ steps.upload_test_spec.outputs.test_spec_arn_10 }}"

            echo "🚀 Scheduling 10 iOS runs (each runs on ALL devices in pool)..."
            echo "Device Pool ARN: $POOL_ARN"

            RUN_ARN_1=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy1-Finetuning" "$TEST_SPEC_ARN_1")
            echo "✅ Heavy1 (Finetuning) scheduled: $RUN_ARN_1"

            RUN_ARN_2=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy2-ToolCalling" "$TEST_SPEC_ARN_2")
            echo "✅ Heavy2 (ToolCalling) scheduled: $RUN_ARN_2"

            RUN_ARN_3=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy3-Image" "$TEST_SPEC_ARN_3")
            echo "✅ Heavy3 (Image) scheduled: $RUN_ARN_3"

            RUN_ARN_4=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy4-OcrLighton" "$TEST_SPEC_ARN_4")
            echo "✅ Heavy4 (OcrLighton) scheduled: $RUN_ARN_4"

            RUN_ARN_5=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy5-Reasoning" "$TEST_SPEC_ARN_5")
            echo "✅ Heavy5 (Reasoning) scheduled: $RUN_ARN_5"

            RUN_ARN_6=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy6-SlidingContext" "$TEST_SPEC_ARN_6")
            echo "✅ Heavy6 (SlidingContext) scheduled: $RUN_ARN_6"

            RUN_ARN_7=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-LightA" "$TEST_SPEC_ARN_7")
            echo "✅ LightA scheduled: $RUN_ARN_7"

            RUN_ARN_8=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-LightB" "$TEST_SPEC_ARN_8")
            echo "✅ LightB scheduled: $RUN_ARN_8"

            RUN_ARN_9=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy7-Gemma4" "$TEST_SPEC_ARN_9")
            echo "✅ Heavy7 (Gemma4) scheduled: $RUN_ARN_9"

            RUN_ARN_10=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy8-OcrPaddle" "$TEST_SPEC_ARN_10")
            echo "✅ Heavy8 (OcrPaddle) scheduled: $RUN_ARN_10"

            echo "run_arn_1=$RUN_ARN_1" >> $GITHUB_OUTPUT
            echo "run_arn_2=$RUN_ARN_2" >> $GITHUB_OUTPUT
            echo "run_arn_3=$RUN_ARN_3" >> $GITHUB_OUTPUT
            echo "run_arn_4=$RUN_ARN_4" >> $GITHUB_OUTPUT
            echo "run_arn_5=$RUN_ARN_5" >> $GITHUB_OUTPUT
            echo "run_arn_6=$RUN_ARN_6" >> $GITHUB_OUTPUT
            echo "run_arn_7=$RUN_ARN_7" >> $GITHUB_OUTPUT
            echo "run_arn_8=$RUN_ARN_8" >> $GITHUB_OUTPUT
            echo "run_arn_9=$RUN_ARN_9" >> $GITHUB_OUTPUT
            echo "run_arn_10=$RUN_ARN_10" >> $GITHUB_OUTPUT
            echo "run_count=10" >> $GITHUB_OUTPUT
          fi

          echo "All runs scheduled."

      - name: Refresh AWS credentials for monitoring
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-west-2
          role-session-name: device-farm-monitor

      - name: Monitor Test Run
        id: monitor_run
        run: |
          aws_retry() {
            local max_attempts=3
            local attempt=1
            local result=""
            while [ $attempt -le $max_attempts ]; do
              result=$("$@" 2>/dev/null) && { echo "$result"; return 0; }
              echo "  ⚠️  AWS API attempt $attempt/$max_attempts failed, retrying in 5s..." >&2
              sleep 5
              attempt=$((attempt + 1))
            done
            echo ""
            return 1
          }

          RUN_ARN_1="${{ steps.schedule_run.outputs.run_arn_1 }}"
          RUN_ARN_2="${{ steps.schedule_run.outputs.run_arn_2 }}"
          RUN_ARN_3="${{ steps.schedule_run.outputs.run_arn_3 }}"
          RUN_ARN_4="${{ steps.schedule_run.outputs.run_arn_4 }}"
          RUN_ARN_5="${{ steps.schedule_run.outputs.run_arn_5 }}"
          RUN_ARN_6="${{ steps.schedule_run.outputs.run_arn_6 }}"
          RUN_ARN_7="${{ steps.schedule_run.outputs.run_arn_7 }}"
          RUN_ARN_8="${{ steps.schedule_run.outputs.run_arn_8 }}"
          RUN_ARN_9="${{ steps.schedule_run.outputs.run_arn_9 }}"
          RUN_ARN_10="${{ steps.schedule_run.outputs.run_arn_10 }}"
          RUN_COUNT="${{ steps.schedule_run.outputs.run_count }}"

          echo "📊 Monitoring $RUN_COUNT Device Farm run(s)..."
          for i in $(seq 1 "$RUN_COUNT"); do
            eval "echo \"  Run $i: \$RUN_ARN_$i\""
          done
          echo ""

          MAX_WAIT_TIME=7200
          ELAPSED=0
          for i in $(seq 1 "$RUN_COUNT"); do eval "DONE_$i=false"; done

          while true; do
            STATUS_LINE="⏳"
            for i in $(seq 1 "$RUN_COUNT"); do
              eval "done_val=\$DONE_$i"
              if [[ "$done_val" != "true" ]]; then
                eval "arn=\$RUN_ARN_$i"
                STATUS_VAL=$(aws_retry aws devicefarm get-run --arn "$arn" --query 'run.status' --output text) || true
                RESULT_VAL=$(aws_retry aws devicefarm get-run --arn "$arn" --query 'run.result' --output text) || true
                eval "STATUS_$i=\"$STATUS_VAL\""
                eval "RESULT_$i=\"$RESULT_VAL\""
                if [[ "$STATUS_VAL" == "COMPLETED" ]]; then eval "DONE_$i=true"; fi
              fi
              eval "STATUS_LINE=\"\$STATUS_LINE Run $i: \$STATUS_$i (\$RESULT_$i) |\""
            done
            echo "$STATUS_LINE ${ELAPSED}s"

            ALL_DONE=true
            for i in $(seq 1 "$RUN_COUNT"); do
              eval "done_val=\$DONE_$i"
              if [[ "$done_val" != "true" ]]; then ALL_DONE=false; fi
            done
            if [[ "$ALL_DONE" == "true" ]]; then
              echo ""
              echo "✅ All runs completed!"
              break
            fi

            if [ $ELAPSED -ge $MAX_WAIT_TIME ]; then
              echo ""
              echo "❌ Timeout: exceeded ${MAX_WAIT_TIME}s"
              exit 1
            fi

            sleep 30
            ELAPSED=$((ELAPSED + 30))
          done

          RUN_ARNS=("$RUN_ARN_1")
          for i in $(seq 2 "$RUN_COUNT"); do
            eval "arn=\$RUN_ARN_$i"
            if [ -n "$arn" ]; then
              RUN_ARNS+=("$arn")
            fi
          done

          DEVICE_COUNT=0
          USER_TEST_COUNT=0
          USER_PASSED=0
          USER_FAILED=0
          FAILED_TEST_DETAILS=()
          ALL_RESULTS_PASSED=true
          TOTAL_TOTAL=0
          TOTAL_PASSED=0
          TOTAL_FAILED=0
          TOTAL_SKIPPED=0

          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "📊 FINAL TEST RESULTS"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""

          RUN_INDEX=0
          for RUN_ARN in "${RUN_ARNS[@]}"; do
            RUN_INDEX=$((RUN_INDEX + 1))
            RUN_DETAILS=$(aws_retry aws devicefarm get-run --arn "$RUN_ARN" --output json) || true
            [ -z "$RUN_DETAILS" ] && RUN_DETAILS='{}'
            RESULT=$(echo "$RUN_DETAILS" | jq -r '.run.result // "UNKNOWN"')
            RUN_NAME_LABEL=$(echo "$RUN_DETAILS" | jq -r '.run.name // "unknown"')
            COUNTERS=$(echo "$RUN_DETAILS" | jq -r '.run.counters // {}')

            if [[ "$RESULT" != "PASSED" && "$RESULT" != "SKIPPED" ]]; then ALL_RESULTS_PASSED=false; fi

            TOTAL_TOTAL=$((TOTAL_TOTAL + $(echo "$COUNTERS" | jq -r '.total // 0')))
            TOTAL_PASSED=$((TOTAL_PASSED + $(echo "$COUNTERS" | jq -r '.passed // 0')))
            TOTAL_FAILED=$((TOTAL_FAILED + $(echo "$COUNTERS" | jq -r '.failed // 0')))
            TOTAL_SKIPPED=$((TOTAL_SKIPPED + $(echo "$COUNTERS" | jq -r '.skipped // 0')))

            PROJECT_ID=$(echo "$RUN_ARN" | sed -n 's/.*:run:\([^/]*\)\/.*/\1/p')
            RUN_ID=$(echo "$RUN_ARN" | sed -n 's/.*:run:[^/]*\/\(.*\)/\1/p')

            if [ "$RUN_COUNT" -ge 2 ]; then
              echo "--- Run $RUN_INDEX: $RUN_NAME_LABEL (Result: $RESULT) ---"
            else
              echo "Result: $RESULT"
            fi

            JOBS=$(aws_retry aws devicefarm list-jobs --arn "$RUN_ARN" --output json) || true
            [ -z "$JOBS" ] && JOBS='{"jobs":[]}'
            for JOB_ARN in $(echo "$JOBS" | jq -r '.jobs[].arn'); do
              DEVICE_COUNT=$((DEVICE_COUNT + 1))
              JOB_DETAILS=$(aws_retry aws devicefarm get-job --arn "$JOB_ARN" --output json) || true
              [ -z "$JOB_DETAILS" ] && JOB_DETAILS='{}'
              DEVICE_NAME=$(echo "$JOB_DETAILS" | jq -r '.job.device.name // "Unknown Device"')
              JOB_RESULT=$(echo "$JOB_DETAILS" | jq -r '.job.result // "UNKNOWN"')
              JOB_ID=$(echo "$JOB_ARN" | sed -n 's/.*:job:[^/]*\/[^/]*\/\(.*\)/\1/p')
              CONSOLE_LINK="https://us-west-2.console.aws.amazon.com/devicefarm/home#/mobile/projects/${PROJECT_ID}/runs/${RUN_ID}/jobs/${JOB_ID}"

              if [ "$JOB_RESULT" = "PASSED" ]; then
                echo "   ✅ $DEVICE_NAME: PASSED"
                USER_PASSED=$((USER_PASSED + 1))
              else
                echo "   ❌ $DEVICE_NAME: $JOB_RESULT"
                USER_FAILED=$((USER_FAILED + 1))
                FAILED_TEST_DETAILS+=("❌ $DEVICE_NAME: $JOB_RESULT")
                FAILED_TEST_DETAILS+=("   📎 View logs: $CONSOLE_LINK")
              fi
              USER_TEST_COUNT=$((USER_TEST_COUNT + 1))
            done

            echo "   🔗 Run link: https://us-west-2.console.aws.amazon.com/devicefarm/home#/mobile/projects/${PROJECT_ID}/runs/${RUN_ID}"
            echo ""
          done

          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "📊 SUMMARY"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""
          echo "Devices tested: $DEVICE_COUNT"
          echo "  ✅ Passed: $USER_PASSED"
          echo "  ❌ Failed: $USER_FAILED"
          echo ""

          if [ ${#FAILED_TEST_DETAILS[@]} -gt 0 ]; then
            echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
            echo "❌ FAILED TESTS"
            echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
            for failed_info in "${FAILED_TEST_DETAILS[@]}"; do
              echo "$failed_info"
            done
            echo ""
          fi

          echo "Device Farm totals: $TOTAL_TOTAL | Passed: $TOTAL_PASSED | Failed: $TOTAL_FAILED | Skipped: $TOTAL_SKIPPED"
          echo ""

          if [ $USER_FAILED -gt 0 ]; then
            echo "test_result=FAILED" >> $GITHUB_OUTPUT
          elif [ $USER_PASSED -gt 0 ]; then
            echo "test_result=PASSED" >> $GITHUB_OUTPUT
          else
            echo "test_result=FAILED" >> $GITHUB_OUTPUT
          fi
          echo "test_total=$TOTAL_TOTAL" >> $GITHUB_OUTPUT
          echo "test_passed=$TOTAL_PASSED" >> $GITHUB_OUTPUT
          echo "test_failed=$TOTAL_FAILED" >> $GITHUB_OUTPUT
          echo "test_skipped=$TOTAL_SKIPPED" >> $GITHUB_OUTPUT
          echo "user_test_count=$USER_TEST_COUNT" >> $GITHUB_OUTPUT
          echo "user_test_passed=$USER_PASSED" >> $GITHUB_OUTPUT
          echo "user_test_failed=$USER_FAILED" >> $GITHUB_OUTPUT

          if [ $USER_FAILED -gt 0 ]; then
            echo ""
            echo "❌ Device Farm tests failed"
            echo "   Devices: $USER_PASSED passed, $USER_FAILED failed (out of $USER_TEST_COUNT)"
            echo "   Totals: $TOTAL_TOTAL | Passed: $TOTAL_PASSED | Failed: $TOTAL_FAILED | Skipped: $TOTAL_SKIPPED"
            exit 1
          elif [ $USER_PASSED -eq 0 ] && [[ "$ALL_RESULTS_PASSED" != "true" ]]; then
            echo ""
            echo "❌ No devices ran tests (all runs skipped or errored)"
            exit 1
          fi

          echo ""
          echo "✅ Device Farm tests passed!"
          echo "   Devices: $USER_PASSED passed (out of $USER_TEST_COUNT)"
          if [ "$TOTAL_SKIPPED" -gt 0 ]; then
            echo "   ⚠️  $TOTAL_SKIPPED test(s) skipped (device unavailable)"
          fi
          echo "   Totals: $TOTAL_TOTAL | Passed: $TOTAL_PASSED | Failed: $TOTAL_FAILED | Skipped: $TOTAL_SKIPPED"

      - name: Refresh AWS credentials for log download
        if: always() && steps.schedule_run.outputs.run_arn_1
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
        with:
          role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
          aws-region: us-west-2
          role-session-name: device-farm-logs

      - name: Download Device Farm Logs
        if: always() && steps.schedule_run.outputs.run_arn_1
        run: |
          RUN_ARN_1="${{ steps.schedule_run.outputs.run_arn_1 }}"
          RUN_ARN_2="${{ steps.schedule_run.outputs.run_arn_2 }}"
          RUN_ARN_3="${{ steps.schedule_run.outputs.run_arn_3 }}"
          RUN_ARN_4="${{ steps.schedule_run.outputs.run_arn_4 }}"
          RUN_ARN_5="${{ steps.schedule_run.outputs.run_arn_5 }}"
          RUN_ARN_6="${{ steps.schedule_run.outputs.run_arn_6 }}"
          RUN_ARN_7="${{ steps.schedule_run.outputs.run_arn_7 }}"
          RUN_ARN_8="${{ steps.schedule_run.outputs.run_arn_8 }}"
          RUN_ARN_9="${{ steps.schedule_run.outputs.run_arn_9 }}"
          RUN_ARN_10="${{ steps.schedule_run.outputs.run_arn_10 }}"
          RUN_COUNT="${{ steps.schedule_run.outputs.run_count }}"
          LOG_DIR="devicefarm-logs/${{ matrix.platform }}"
          PLATFORM="${{ matrix.platform }}"
          mkdir -p "$LOG_DIR"

          RUN_ARNS=("$RUN_ARN_1")
          for i in $(seq 2 "$RUN_COUNT"); do
            eval "arn=\$RUN_ARN_$i"
            if [ -n "$arn" ]; then
              RUN_ARNS+=("$arn")
            fi
          done

          echo ""
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo "📥 DOWNLOADING DEVICE FARM LOGS"
          echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
          echo ""

          for RUN_ARN in "${RUN_ARNS[@]}"; do
            RUN_DETAILS=$(aws devicefarm get-run --arn "$RUN_ARN" --output json 2>/dev/null || echo '{}')
            RUN_LABEL=$(echo "$RUN_DETAILS" | jq -r '.run.name // "unknown"')
            echo ""
            echo "========================================"
            echo "📦 Run: $RUN_LABEL"
            echo "========================================"

            SAFE_RUN=$(echo "$RUN_LABEL" | tr ' /' '__' | tr -cd '[:alnum:]_-')
            JOBS=$(aws devicefarm list-jobs --arn "$RUN_ARN" --output json 2>/dev/null || echo '{"jobs":[]}')

            for JOB_ARN in $(echo "$JOBS" | jq -r '.jobs[].arn'); do
              DEVICE_NAME=$(echo "$JOBS" | jq -r --arg arn "$JOB_ARN" '.jobs[] | select(.arn == $arn) | .device.name // "unknown"')
              JOB_RESULT=$(echo "$JOBS" | jq -r --arg arn "$JOB_ARN" '.jobs[] | select(.arn == $arn) | .result // "UNKNOWN"')
              SAFE_NAME=$(echo "$DEVICE_NAME" | tr ' /' '__' | tr -cd '[:alnum:]_-')

              echo ""
              echo "📱 $DEVICE_NAME ($JOB_RESULT)"

              SUITES=$(aws devicefarm list-suites --arn "$JOB_ARN" --output json 2>/dev/null || echo '{"suites":[]}')

              for SUITE_ARN in $(echo "$SUITES" | jq -r '.suites[].arn'); do
                SUITE_NAME=$(echo "$SUITES" | jq -r --arg arn "$SUITE_ARN" '.suites[] | select(.arn == $arn) | .name // "unknown"')
                SAFE_SUITE=$(echo "$SUITE_NAME" | tr ' /' '__' | tr -cd '[:alnum:]_-')

                ARTIFACTS=$(aws devicefarm list-artifacts --arn "$SUITE_ARN" --type FILE --output json 2>/dev/null || echo '{"artifacts":[]}')
                echo "$ARTIFACTS" | jq -c '.artifacts[]' 2>/dev/null | while read -r ARTIFACT; do
                  ART_NAME=$(echo "$ARTIFACT" | jq -r '.name // "unknown"')
                  ART_URL=$(echo "$ARTIFACT" | jq -r '.url // empty')
                  ART_EXT=$(echo "$ARTIFACT" | jq -r '.extension // "txt"')
                  [ -z "$ART_URL" ] && continue
                  if [ "$PLATFORM" = "Android" ]; then
                    if echo "$ART_NAME" | grep -qiE "^video$" || echo "$ART_EXT" | grep -qiE "^mp4$"; then
                      echo "  Skipped (video): $SUITE_NAME / $ART_NAME"
                      continue
                    fi
                  fi
                  SAFE_ART=$(echo "$ART_NAME" | tr ' /' '__' | tr -cd '[:alnum:]_-')
                  DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_${SAFE_SUITE}_${SAFE_ART}.${ART_EXT}"
                  if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
                    echo "  Downloaded: $SUITE_NAME / $ART_NAME"
                    if echo "$ART_NAME" | grep -qiE "test.spec|testspec"; then
                      echo ""; echo "::group::📋 [$DEVICE_NAME] $SUITE_NAME -- $ART_NAME"
                      cat "$DEST" 2>/dev/null || true; echo "::endgroup::"
                    fi
                  fi
                done

                LOG_ARTIFACTS=$(aws devicefarm list-artifacts --arn "$SUITE_ARN" --type LOG --output json 2>/dev/null || echo '{"artifacts":[]}')
                echo "$LOG_ARTIFACTS" | jq -c '.artifacts[]' 2>/dev/null | while read -r ARTIFACT; do
                  ART_NAME=$(echo "$ARTIFACT" | jq -r '.name // "unknown"')
                  ART_URL=$(echo "$ARTIFACT" | jq -r '.url // empty')
                  ART_EXT=$(echo "$ARTIFACT" | jq -r '.extension // "txt"')
                  [ -z "$ART_URL" ] && continue
                  SAFE_ART=$(echo "$ART_NAME" | tr ' /' '__' | tr -cd '[:alnum:]_-')
                  DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_${SAFE_SUITE}_${SAFE_ART}.${ART_EXT}"
                  if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
                    echo "  Downloaded: $SUITE_NAME / $ART_NAME (LOG)"
                  fi
                done
              done

              JOB_ARTIFACTS=$(aws devicefarm list-artifacts --arn "$JOB_ARN" --type FILE --output json 2>/dev/null || echo '{"artifacts":[]}')
              echo "$JOB_ARTIFACTS" | jq -c '.artifacts[]' 2>/dev/null | while read -r ARTIFACT; do
                ART_NAME=$(echo "$ARTIFACT" | jq -r '.name // "unknown"')
                ART_URL=$(echo "$ARTIFACT" | jq -r '.url // empty')
                ART_EXT=$(echo "$ARTIFACT" | jq -r '.extension // "txt"')
                [ -z "$ART_URL" ] && continue
                if [ "$PLATFORM" = "Android" ]; then
                  if echo "$ART_NAME" | grep -qiE "^video$" || echo "$ART_EXT" | grep -qiE "^mp4$"; then
                    echo "  Skipped (video): job-level / $ART_NAME"
                    continue
                  fi
                fi
                SAFE_ART=$(echo "$ART_NAME" | tr ' /' '__' | tr -cd '[:alnum:]_-')
                DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_job_${SAFE_ART}.${ART_EXT}"
                if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
                  echo "  Downloaded (job-level): $ART_NAME"
                fi
              done
            done
          done

          echo ""
          echo "📦 All downloaded logs:"
          find "$LOG_DIR" -type f -exec ls -lh {} \; 2>/dev/null || echo "  (no logs downloaded)"

      - name: Extract Console Logs
        if: always() && steps.schedule_run.outputs.run_arn_1
        run: |
          LOG_DIR="devicefarm-logs/${{ matrix.platform }}"
          CONSOLE_DIR="console-logs/${{ matrix.platform }}"
          mkdir -p "$CONSOLE_DIR"
          for f in "$LOG_DIR"/*; do
            [ -f "$f" ] || continue
            fname=$(basename "$f")
            if echo "$fname" | grep -qiE "Logcat\."; then
              cp "$f" "$CONSOLE_DIR/"
              echo "  Console log: $fname"
            fi
            if echo "$fname" | grep -qiE "Customer_Artifacts\.zip$"; then
              prefix=$(echo "$fname" | sed 's/_Customer_Artifacts\.zip$//')
              tmpdir=$(mktemp -d)
              if unzip -qo "$f" -d "$tmpdir" 2>/dev/null; then
                bare_log=$(find "$tmpdir" -name "bare_console.log" -type f 2>/dev/null | head -1)
                if [ -n "$bare_log" ]; then
                  cp "$bare_log" "$CONSOLE_DIR/${prefix}_bare_console.log"
                  echo "  Extracted: ${prefix}_bare_console.log"
                fi
                appium_log=$(find "$tmpdir" -name "appium.log" -type f 2>/dev/null | head -1)
                if [ -n "$appium_log" ]; then
                  cp "$appium_log" "$CONSOLE_DIR/${prefix}_appium.log"
                  echo "  Extracted: ${prefix}_appium.log"
                fi
              else
                echo "  WARNING: Failed to unzip $fname"
              fi
              rm -rf "$tmpdir"
            fi
          done
          echo ""
          echo "📋 Console logs extracted:"
          find "$CONSOLE_DIR" -type f -exec ls -lh {} \; 2>/dev/null || echo "  (no console logs found)"

      - name: Upload Console Logs
        if: always() && steps.schedule_run.outputs.run_arn_1
        uses: actions/upload-artifact@v4
        with:
          name: console-logs-${{ matrix.platform }}
          path: console-logs/
          retention-days: 30
          if-no-files-found: ignore

      - name: Upload Full Device Farm Logs
        if: always() && steps.schedule_run.outputs.run_arn_1
        uses: actions/upload-artifact@v4
        with:
          name: devicefarm-logs-${{ matrix.platform }}
          path: devicefarm-logs/
          retention-days: 30
          if-no-files-found: ignore