.github/workflows/integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml

Create new buckets to run tests in independent processes (#1889) #5

Workflow file for this run

.github/workflows/integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml at 9293bb2

	name: Mobile Integration Tests (LLM)
Check failure on line 1 in .github/workflows/integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml View workflow run for this annotation GitHub Actions / .github/workflows/integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml Invalid workflow file `(Line: 914, Col: 14): Exceeded max expression length 21000`

	on:
	workflow_call:
	inputs:
	ref:
	description: "Git ref to checkout"
	type: string
	required: false
	repository:
	description: "Repository to checkout"
	type: string
	required: false
	qvac_perf_runs:
	description: "Override QVAC_PERF_RUNS (number of counted iterations per perf test). Empty = test default."
	type: string
	required: false
	default: ""
	qvac_perf_warmup_runs:
	description: "Override QVAC_PERF_WARMUP_RUNS (number of warmup iterations per perf test). Empty = test default."
	type: string
	required: false
	default: ""
	workflow_dispatch:
	inputs:
	ref:
	description: "Git ref (branch/tag/SHA) to test"
	type: string
	required: false
	default: main
	package:
	description: "Full NPM package spec to test (default: @qvac/llm-llamacpp@latest)"
	type: string
	required: true
	default: "@qvac/llm-llamacpp@latest"
	qvac_perf_runs:
	description: "Override QVAC_PERF_RUNS (number of counted iterations per perf test). Empty = test default."
	type: string
	required: false
	default: ""
	qvac_perf_warmup_runs:
	description: "Override QVAC_PERF_WARMUP_RUNS (number of warmup iterations per perf test). Empty = test default."
	type: string
	required: false
	default: ""

	env:
	NODE_VERSION: "lts/*"
	ADDON_NAME: "@qvac/llm-llamacpp"
	PREBUILD_ARTIFACT_PREFIX: "llama-cpp-" # Prefix for prebuild artifacts (empty string for generic patterns)
	TEST_FRAMEWORK_REF: "main" # Branch/tag of qvac-test-addon-mobile framework
	APP_BUNDLE_ID: "io.tether.test.qvac" # Bundle ID for the test app (same for all addons)
	ADDON_WORKDIR: "addon/packages/qvac-lib-infer-llamacpp-llm"

	jobs:
	build-and-test:
	name: Build ${{ matrix.platform }} and Run E2E Tests
	runs-on: ${{ matrix.runner }}
	environment: release
	timeout-minutes: 120
	continue-on-error: true # Don't block PR merges if tests fail
	permissions:
	contents: read
	packages: read
	pull-requests: write # Allow commenting on PRs
	id-token: write
	strategy:
	fail-fast: false
	matrix:
	include:
	- platform: Android
	os: ubuntu-24.04
	runner: ai-run-linux
	- platform: iOS
	os: macos-14
	runner: macos-14

	steps:
	# Validate input data to prevent "Artifact Poisoning"
	- name: Validate Dispatch Inputs
	if: github.event_name == 'workflow_dispatch' && github.event.inputs.package
	run: \|
	if [[ ! "${{ github.event.inputs.package }}" =~ ^@qvac/ ]]; then
	echo "::error::Invalid package scope. Only @qvac/* is allowed."
	exit 1
	fi

	# Free up disk space on Ubuntu runner to prevent "No space left on device" errors
	- name: Free up disk space
	if: matrix.platform == 'Android'
	run: \|
	echo "Disk space before cleanup:"
	df -h
	# Remove unnecessary software to free up disk space (\|\| true to handle self-hosted runners)
	sudo rm -rf /usr/share/dotnet \|\| true
	sudo rm -rf /opt/ghc \|\| true
	sudo rm -rf /opt/hostedtoolcache/CodeQL \|\| true
	sudo rm -rf /opt/hostedtoolcache/go \|\| true
	sudo rm -rf /opt/hostedtoolcache/Python \|\| true
	sudo rm -rf /opt/hostedtoolcache/Ruby \|\| true
	sudo rm -rf /usr/local/lib/android/sdk/ndk \|\| true
	sudo rm -rf /usr/local/share/boost \|\| true
	sudo rm -rf /usr/share/swift \|\| true
	sudo docker image prune --all --force \|\| true
	# Clean APT cache
	sudo apt-get clean \|\| true
	echo "Disk space after cleanup:"
	df -h

	- name: Checkout addon repository
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
	with:
	repository: ${{ inputs.repository \|\| github.repository }}
	ref: ${{ inputs.ref \|\| github.ref }}
	token: ${{ secrets.PAT_TOKEN }}
	path: addon
	fetch-depth: 0

	- name: Checkout mobile test framework
	uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # 6.0.2
	with:
	repository: tetherto/qvac-test-addon-mobile
	ref: ${{ env.TEST_FRAMEWORK_REF }}
	token: ${{ secrets.PAT_TOKEN }}
	path: test-framework
	fetch-depth: 0

	- name: Setup Node.js
	uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # 6.3.0
	with:
	node-version: ${{ env.NODE_VERSION }}

	- name: Install global dependencies
	run: \|
	echo "Installing global dependencies..."
	npm install -g @expo/cli@latest

	- name: Download Android prebuilds (from artifacts)
	if: matrix.platform == 'Android' && !inputs.package
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
	with:
	path: ${{ env.ADDON_WORKDIR }}/prebuilds
	pattern: ${{ env.PREBUILD_ARTIFACT_PREFIX }}android-*
	merge-multiple: true
	continue-on-error: true

	- name: Download iOS prebuilds (from artifacts)
	if: matrix.platform == 'iOS' && !inputs.package
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
	with:
	path: ${{ env.ADDON_WORKDIR }}/prebuilds
	pattern: ${{ env.PREBUILD_ARTIFACT_PREFIX }}ios-*
	merge-multiple: true
	continue-on-error: true

	- name: Download prebuilds from package
	if: inputs.package
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	PACKAGE_SPEC="${{ inputs.package }}"
	echo "📦 Downloading $PACKAGE_SPEC from npm for manual trigger..."

	PACKAGE_NAME="${PACKAGE_SPEC%@*}"

	if ! npm pack "$PACKAGE_SPEC" --ignore-scripts; then
	echo "ERROR: Failed to download $PACKAGE_SPEC from npm"
	echo "Please check that the package exists at https://www.npmjs.com/package/$PACKAGE_NAME"
	exit 1
	fi

	# Extract the tarball (pattern matches any addon name)
	tar -xzf *.tgz

	# Validate prebuilds directory exists
	if [ ! -d "package/prebuilds" ]; then
	echo "ERROR: No prebuilds directory found in package"
	echo "The downloaded package may not contain prebuilt binaries"
	exit 1
	fi

	# Move prebuilds to expected location
	mv package/prebuilds ./prebuilds

	# Cleanup
	rm -rf package *.tgz

	echo "✅ Prebuilds downloaded from npm:"
	ls -la prebuilds/

	- name: Verify and prepare prebuilds
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	echo "Checking for prebuilds..."
	echo "Current directory: $(pwd)"
	if [ -d "prebuilds" ] && [ "$(ls -A prebuilds)" ]; then
	echo "✅ Prebuilds found from artifacts:"
	ls -la prebuilds/
	else
	echo "❌ ERROR: No prebuilds found!"
	echo " This workflow requires prebuilds to be available."
	echo " Either:"
	echo " 1. Run this workflow after prebuild job completes"
	echo " 2. Or commit prebuilds to the repository"
	exit 1
	fi

	# Copy mobile prebuilds if needed
	if npm run mobile:copy-prebuilds 2>/dev/null; then
	echo "✅ Mobile prebuilds prepared"
	else
	echo "⚠️ mobile:copy-prebuilds script not available or failed"
	fi

	- name: Remove desktop prebuilds to save disk space
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	echo "Removing desktop prebuilds to save disk space (keeping Android + iOS)..."
	echo "Before cleanup:"
	du -sh prebuilds/* 2>/dev/null \|\| true

	# Remove desktop prebuilds only (not needed for mobile tests)
	rm -rf prebuilds/darwin-* prebuilds/win32-* prebuilds/linux-* 2>/dev/null \|\| true

	echo "After cleanup (Android + iOS only):"
	du -sh prebuilds/* 2>/dev/null \|\| true
	df -h

	- name: Verify test files exist
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	echo "Verifying addon has mobile tests..."

	if [ ! -d "test/mobile" ]; then
	echo "❌ ERROR: test/mobile directory not found!"
	echo ""
	echo "This workflow requires the addon to have mobile tests at:"
	echo " test/mobile/"
	echo ""
	echo "Please create this directory with your test files."
	echo "See qvac-test-addon-mobile README for test file format."
	exit 1
	fi

	# Check for .cjs test files
	CJS_COUNT=$(find test/mobile -name "*.cjs" -type f \| wc -l)
	if [ "$CJS_COUNT" -eq 0 ]; then
	echo "❌ ERROR: No .cjs test files found in test/mobile!"
	exit 1
	fi

	echo "✅ Mobile test files found:"
	ls -la test/mobile/*.cjs

	# Check if testAssets exists
	if [ -d "test/mobile/testAssets" ]; then
	echo ""
	echo "✅ Test assets found:"
	ls -lah test/mobile/testAssets/
	else
	echo ""
	echo "ℹ️ No testAssets directory (this is optional)"
	fi

	- name: Install Ninja build tool
	if: matrix.platform == 'iOS'
	run: \|
	echo "📦 Installing Ninja build system..."
	brew install ninja
	ninja --version
	echo "✅ Ninja installed successfully"

	- name: Install addon dependencies
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	echo "Installing addon dependencies..."
	npm install --ignore-scripts

	- name: Validate mobile tests are up-to-date
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: npm run test:mobile:validate

	- name: Pack addon
	working-directory: ${{ env.ADDON_WORKDIR }}
	run: \|
	echo "Packing addon..."
	if npm run build:pack 2>/dev/null; then
	echo "✅ Addon packed using build:pack script"
	else
	echo "📦 Using npm pack directly..."
	mkdir -p dist
	npm pack --pack-destination dist --ignore-scripts
	fi

	# Verify pack file exists
	PACK_FILE=$(ls dist/*.tgz \| head -1)
	if [ -f "$PACK_FILE" ]; then
	SIZE=$(du -h "$PACK_FILE" \| cut -f1)
	echo "✅ Pack file created: $PACK_FILE (Size: $SIZE)"
	else
	echo "❌ Pack file not found in dist/"
	exit 1
	fi

	- name: Setup test framework dependencies
	working-directory: test-framework
	run: \|
	echo "Setting up mobile test framework..."
	npm install --ignore-scripts
	echo "✅ Test framework dependencies installed"

	- name: Build test app with addon
	working-directory: test-framework
	run: \|
	echo "Building test app with addon..."
	echo "This will:"
	echo " 1. Install the addon package"
	echo " 2. Extract test code from addon's test/mobile/ directory"
	echo " 3. Auto-detect and order test files by dependencies"
	echo " 4. Generate backend.cjs with test functions"
	echo " 5. Generate e2e tests for each test function"
	echo " 6. Copy testAssets if available"
	echo " 7. Bundle the app"
	echo ""

	ADDON_PATH="${{ github.workspace }}/${{ env.ADDON_WORKDIR }}"
	npm run build "$ADDON_PATH" "$ADDON_PATH/test/mobile"

	echo ""
	echo "✅ Test app built successfully"

	# Verify critical files were generated
	if [ ! -f "backend/backend.cjs" ]; then
	echo "❌ ERROR: backend/backend.cjs was not generated!"
	exit 1
	fi

	if [ ! -f "e2e/tests/app.test.js" ]; then
	echo "❌ ERROR: e2e/tests/app.test.js was not generated!"
	exit 1
	fi

	if [ ! -f "backend/app.bundle" ]; then
	echo "❌ ERROR: backend/app.bundle was not created!"
	exit 1
	fi

	echo "✅ All required files generated successfully"

	# Show what tests were extracted
	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "EXTRACTED TEST FUNCTIONS:"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	if [ -f "app/testConfig.js" ]; then
	cat app/testConfig.js
	fi
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

	- name: Display build summary
	if: always()
	working-directory: test-framework
	run: \|
	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "📊 BUILD SUMMARY"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo ""
	echo "Platform: ${{ matrix.platform }}"
	if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
	echo "Package: ${{ inputs.package }}"
	else
	echo "Addon: ${{ env.ADDON_NAME }} (from PR artifacts)"
	fi
	echo ""
	echo "Generated Files:"
	echo " backend/backend.cjs: $([ -f backend/backend.cjs ] && echo '✅' \|\| echo '❌')"
	echo " backend/app.bundle: $([ -f backend/app.bundle ] && echo '✅' \|\| echo '❌')"
	echo " app/testConfig.js: $([ -f app/testConfig.js ] && echo '✅' \|\| echo '❌')"
	echo " app/assetManifest.js: $([ -f app/assetManifest.js ] && echo '✅' \|\| echo '❌')"
	echo " e2e/tests/app.test.js: $([ -f e2e/tests/app.test.js ] && echo '✅' \|\| echo '❌')"
	echo ""
	echo "Test Assets:"
	if [ -d "testAssets" ]; then
	ASSET_COUNT=$(find testAssets -type f \| wc -l)
	echo " ✅ $ASSET_COUNT file(s) in testAssets/"
	else
	echo " ℹ️ No testAssets (optional)"
	fi
	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

	# Android-specific steps
	- name: Set up JDK 17
	if: matrix.platform == 'Android'
	uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # 5.2.0
	with:
	java-version: 17
	distribution: temurin

	- name: Setup Android SDK
	if: matrix.platform == 'Android'
	uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # 3.2.2

	- name: Generate Android project
	if: matrix.platform == 'Android'
	working-directory: test-framework
	run: \|
	echo "Generating Android project with Expo..."
	npx expo prebuild --platform android --clean

	- name: Build Android APK
	if: matrix.platform == 'Android'
	id: build_apk
	working-directory: test-framework
	run: \|
	echo "Building Android APK for Device Farm..."
	export JAVA_HOME=$JAVA_HOME_17_X64

	# Bundle JavaScript
	echo "Bundling JavaScript code..."
	npm run bundle

	if [ $? -ne 0 ]; then
	echo "❌ Bundle failed"
	exit 1
	fi

	echo "✅ Bundle completed successfully"

	# Build RELEASE APK (not debug) to ensure JS bundle is included
	# Debug builds skip bundling by default and try to connect to Metro
	# Release builds embed the JS bundle in the APK
	cd android
	echo "Building APK with Gradle (RELEASE with embedded JS bundle)..."
	./gradlew assembleRelease \
	-PreactNativeArchitectures=arm64-v8a \
	--no-daemon \
	--no-build-cache \
	--stacktrace
	cd ..

	# Find the APK (look for release)
	APK_PATH=$(find android/app/build/outputs/apk -name "*.apk" \| grep "release" \| grep -v "unaligned" \| head -1)

	if [ -f "$APK_PATH" ]; then
	# Convert to absolute path
	APK_ABSOLUTE_PATH="${{ github.workspace }}/test-framework/$APK_PATH"
	SIZE=$(du -h "$APK_PATH" \| cut -f1)
	echo "✅ APK built successfully: $APK_PATH (Size: $SIZE)"
	echo "apk_path=$APK_ABSOLUTE_PATH" >> $GITHUB_OUTPUT
	echo "app_type=ANDROID_APP" >> $GITHUB_OUTPUT
	echo "app_name=test-app-${{ matrix.platform }}.apk" >> $GITHUB_OUTPUT

	# Clean up build intermediates to free disk space
	echo "Cleaning up build intermediates..."
	rm -rf android/app/build/intermediates
	rm -rf android/.gradle
	df -h
	else
	echo "❌ APK file not found"
	echo "Searching in android/app/build/outputs/apk:"
	find android/app/build/outputs/apk -type f 2>/dev/null \|\| echo "Directory not found"
	exit 1
	fi

	# iOS-specific steps
	- name: Set up Xcode version
	if: matrix.platform == 'iOS'
	run: \|
	echo "Available Xcode versions:"
	ls /Applications \| grep Xcode \|\| echo "No Xcode apps found"

	echo ""
	echo "Current Xcode (before switch):"
	xcodebuild -version

	# React Native requires Xcode >= 16.1
	# Use Xcode 16.1 (has iOS 18.1 SDK which is stable and pre-installed)
	if [ -d "/Applications/Xcode_16.1.app" ]; then
	echo ""
	echo "✅ Switching to Xcode 16.1..."
	sudo xcode-select -s /Applications/Xcode_16.1.app
	elif [ -d "/Applications/Xcode_16.1.0.app" ]; then
	echo ""
	echo "✅ Switching to Xcode 16.1.0..."
	sudo xcode-select -s /Applications/Xcode_16.1.0.app
	elif [ -d "/Applications/Xcode_16.2.app" ]; then
	echo ""
	echo "⚠️ Using Xcode 16.2 (16.1 not found)..."
	sudo xcode-select -s /Applications/Xcode_16.2.app
	else
	echo ""
	echo "❌ ERROR: No suitable Xcode version found (need >= 16.1)"
	exit 1
	fi

	echo ""
	echo "Current Xcode (after switch):"
	xcodebuild -version

	echo ""
	echo "Available iOS SDKs:"
	xcodebuild -showsdks \| grep -i ios

	- name: Install CocoaPods
	if: matrix.platform == 'iOS'
	run: \|
	sudo gem install cocoapods
	pod --version

	- name: Create Keychain and Import Certificate
	if: matrix.platform == 'iOS'
	env:
	BUILD_CERTIFICATE_BASE64: ${{ secrets.TEST_APP_APPLE_DISTRIBUTION_CERTIFICATE }}
	P12_PASSWORD: ${{ secrets.APPLE_P12_PASSWORD }}
	BUILD_PROVISION_PROFILE_BASE64: ${{ secrets.TEST_APP_APPLE_PROVISIONING_PROFILE }}
	KEYCHAIN_PASSWORD: ${{ secrets.APPLE_KEYCHAIN_PASSWORD }}
	run: \|
	CERTIFICATE_PATH=$RUNNER_TEMP/build_certificate.p12
	PP_PATH=$RUNNER_TEMP/build_pp.mobileprovision
	KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db

	echo -n "$BUILD_CERTIFICATE_BASE64" \| base64 --decode -o $CERTIFICATE_PATH
	echo -n "$BUILD_PROVISION_PROFILE_BASE64" \| base64 --decode -o $PP_PATH

	security create-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
	security set-keychain-settings -lut 21600 $KEYCHAIN_PATH
	security unlock-keychain -p "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH

	security import $CERTIFICATE_PATH -P "$P12_PASSWORD" -A -t cert -f pkcs12 -k $KEYCHAIN_PATH
	security set-key-partition-list -S apple-tool:,apple: -s -k "$KEYCHAIN_PASSWORD" $KEYCHAIN_PATH
	security list-keychain -d user -s $KEYCHAIN_PATH

	# Extract UUID first, then copy with UUID as filename
	PP_UUID=$(/usr/libexec/PlistBuddy -c 'Print :UUID' /dev/stdin <<< $(security cms -D -i $PP_PATH))
	echo "PP_UUID=$PP_UUID" >> $GITHUB_ENV
	echo "Provisioning Profile UUID: $PP_UUID"

	# Copy provisioning profile with UUID as filename
	mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles
	cp $PP_PATH ~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision

	security find-identity -p codesigning -v

	- name: Verify provisioning profile
	if: matrix.platform == 'iOS'
	run: \|
	echo "🔍 Verifying provisioning profile..."
	echo "PP_UUID: $PP_UUID"

	PP_FILE=~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision
	if [ ! -f "$PP_FILE" ]; then
	echo "❌ Provisioning profile file not found at: $PP_FILE"
	ls -la ~/Library/MobileDevice/Provisioning\ Profiles/
	exit 1
	fi

	echo "📋 Provisioning Profile Details:"
	security cms -D -i "$PP_FILE" > /tmp/profile.plist

	PROFILE_NAME=$(/usr/libexec/PlistBuddy -c "Print :Name" /tmp/profile.plist 2>/dev/null \|\| echo "Unknown")
	PROFILE_BUNDLE_ID=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:application-identifier" /tmp/profile.plist 2>/dev/null \|\| echo "Unknown")
	PROFILE_TEAM_ID=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:com.apple.developer.team-identifier" /tmp/profile.plist 2>/dev/null \|\| echo "Unknown")

	# Detect profile type (Development, Ad Hoc, App Store, Enterprise)
	HAS_DEVICES=$(/usr/libexec/PlistBuddy -c "Print :ProvisionedDevices" /tmp/profile.plist 2>/dev/null && echo "yes" \|\| echo "no")
	PROVISIONS_ALL=$(/usr/libexec/PlistBuddy -c "Print :ProvisionsAllDevices" /tmp/profile.plist 2>/dev/null \|\| echo "false")
	HAS_GET_TASK_ALLOW=$(/usr/libexec/PlistBuddy -c "Print :Entitlements:get-task-allow" /tmp/profile.plist 2>/dev/null \|\| echo "false")

	if [[ "$PROVISIONS_ALL" == "true" ]]; then
	PROFILE_TYPE="Enterprise"
	EXPORT_METHOD="enterprise"
	elif [[ "$HAS_DEVICES" == "yes" && "$HAS_GET_TASK_ALLOW" == "true" ]]; then
	PROFILE_TYPE="Development"
	EXPORT_METHOD="development"
	elif [[ "$HAS_DEVICES" == "yes" && "$HAS_GET_TASK_ALLOW" == "false" ]]; then
	PROFILE_TYPE="Ad Hoc"
	EXPORT_METHOD="ad-hoc"
	else
	PROFILE_TYPE="App Store"
	EXPORT_METHOD="app-store"
	fi

	echo " Name: $PROFILE_NAME"
	echo " Type: $PROFILE_TYPE"
	echo " Export Method: $EXPORT_METHOD"
	echo " Application ID: $PROFILE_BUNDLE_ID"
	echo " Team ID: $PROFILE_TEAM_ID"
	echo " Expected Bundle ID: ${{ env.APP_BUNDLE_ID }}"

	# Save export method for next step
	echo "EXPORT_METHOD=$EXPORT_METHOD" >> $GITHUB_ENV

	# Extract just the bundle ID part (remove team prefix)
	BUNDLE_ID_ONLY=$(echo "$PROFILE_BUNDLE_ID" \| sed 's/^[^.]*\.//')

	if [[ "$BUNDLE_ID_ONLY" != "${{ env.APP_BUNDLE_ID }}" ]]; then
	echo ""
	echo "❌ ERROR: Provisioning profile bundle ID mismatch!"
	echo " Profile has: $BUNDLE_ID_ONLY"
	echo " Expected: ${{ env.APP_BUNDLE_ID }}"
	echo ""
	echo "The provisioning profile was created for a different bundle identifier."
	echo "Please create a new provisioning profile for: ${{ env.APP_BUNDLE_ID }}"
	exit 1
	fi

	echo "✅ Provisioning profile matches expected bundle ID"

	- name: Generate iOS project
	if: matrix.platform == 'iOS'
	working-directory: test-framework
	run: \|
	echo "Generating iOS project with Expo..."
	npx expo prebuild --platform ios --clean

	- name: Install iOS dependencies
	if: matrix.platform == 'iOS'
	working-directory: test-framework/ios
	run: \|
	echo "Installing CocoaPods dependencies..."
	pod install --repo-update

	- name: Build and Archive iOS App
	if: matrix.platform == 'iOS'
	id: build_ios
	working-directory: test-framework
	run: \|
	echo "Building iOS app for Device Farm..."

	# Bundle JavaScript first
	echo "Bundling JavaScript code..."
	npm run bundle

	if [ $? -ne 0 ]; then
	echo "❌ Bundle failed"
	exit 1
	fi

	echo "✅ Bundle completed successfully"

	# Get scheme name
	cd ios
	SCHEME_NAME=$(xcodebuild -list \| grep -A 1 "Schemes:" \| grep -v "Schemes:" \| head -1 \| xargs)
	echo "Detected scheme: $SCHEME_NAME"

	# Debug: Check bundle identifier in project
	echo "🔍 Checking project configuration..."
	BUNDLE_ID=$(xcodebuild -showBuildSettings -workspace $SCHEME_NAME.xcworkspace -scheme "$SCHEME_NAME" -configuration Release -destination "generic/platform=iOS" 2>/dev/null \| grep PRODUCT_BUNDLE_IDENTIFIER \| head -1 \| awk '{print $3}')
	echo "Bundle Identifier in project: $BUNDLE_ID"

	if [[ "$BUNDLE_ID" != "${{ env.APP_BUNDLE_ID }}" ]]; then
	echo "⚠️ Warning: Bundle ID mismatch in Xcode project!"
	echo " Expected: ${{ env.APP_BUNDLE_ID }}"
	echo " Found: $BUNDLE_ID"
	fi

	# Debug: Check provisioning profile
	echo "🔍 Provisioning profile UUID: $PP_UUID"
	security cms -D -i ~/Library/MobileDevice/Provisioning\ Profiles/$PP_UUID.mobileprovision \| grep -A 5 "application-identifier\\|Name\\|TeamIdentifier" \| head -20 \|\| echo "Could not read profile details"

	# Archive for iOS device
	xcodebuild -workspace $SCHEME_NAME.xcworkspace \
	-scheme "$SCHEME_NAME" \
	-sdk iphoneos \
	-configuration Release \
	-destination "generic/platform=iOS" \
	-archivePath $RUNNER_TEMP/$SCHEME_NAME.xcarchive \
	CODE_SIGN_STYLE=Manual \
	PROVISIONING_PROFILE_SPECIFIER="$PP_UUID" \
	CODE_SIGN_IDENTITY="Apple Distribution" \
	DEVELOPMENT_TEAM="${{ secrets.APPLE_TEAM_ID }}" \
	clean archive

	- name: Export IPA
	if: matrix.platform == 'iOS'
	id: export_ipa
	working-directory: test-framework/ios
	run: \|
	SCHEME_NAME=$(xcodebuild -list \| grep -A 1 "Schemes:" \| grep -v "Schemes:" \| head -1 \| xargs)

	# Create export options using auto-detected export method
	# The EXPORT_METHOD was determined in the "Verify provisioning profile" step
	echo "📦 Using export method: $EXPORT_METHOD"

	EXPORT_OPTS_PATH=$RUNNER_TEMP/ExportOptions.plist
	cat > $EXPORT_OPTS_PATH << EOF
	<?xml version="1.0" encoding="UTF-8"?>
	<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
	<plist version="1.0">
	<dict>
	<key>method</key>
	<string>$EXPORT_METHOD</string>
	<key>teamID</key>
	<string>${{ secrets.APPLE_TEAM_ID }}</string>
	<key>signingStyle</key>
	<string>manual</string>
	<key>provisioningProfiles</key>
	<dict>
	<key>${{ env.APP_BUNDLE_ID }}</key>
	<string>$PP_UUID</string>
	</dict>
	</dict>
	</plist>
	EOF

	echo "📋 Export options:"
	cat $EXPORT_OPTS_PATH

	xcodebuild -exportArchive \
	-archivePath $RUNNER_TEMP/$SCHEME_NAME.xcarchive \
	-exportOptionsPlist $EXPORT_OPTS_PATH \
	-exportPath $RUNNER_TEMP/build

	IPA_FILE=$(find $RUNNER_TEMP/build -name "*.ipa" \| head -1)
	if [ -f "$IPA_FILE" ]; then
	echo "✅ IPA exported: $IPA_FILE"
	echo "apk_path=$IPA_FILE" >> $GITHUB_OUTPUT
	echo "app_type=IOS_APP" >> $GITHUB_OUTPUT
	echo "app_name=test-app-${{ matrix.platform }}.ipa" >> $GITHUB_OUTPUT
	else
	echo "❌ IPA file not found"
	exit 1
	fi

	- name: Configure AWS credentials via OIDC
	uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
	with:
	role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
	aws-region: us-west-2
	role-session-name: device-farm-upload
	role-duration-seconds: 7200 # 2hrs for device farm tests

	- name: Upload App to Device Farm
	id: upload_app
	run: \|
	if [ "${{ matrix.platform }}" == "Android" ]; then
	APP_PATH="${{ steps.build_apk.outputs.apk_path }}"
	APP_TYPE="${{ steps.build_apk.outputs.app_type }}"
	APP_NAME="${{ steps.build_apk.outputs.app_name }}"
	else
	APP_PATH="${{ steps.export_ipa.outputs.apk_path }}"
	APP_TYPE="${{ steps.export_ipa.outputs.app_type }}"
	APP_NAME="${{ steps.export_ipa.outputs.app_name }}"
	fi

	echo "📤 Uploading app to AWS Device Farm..."
	UPLOAD_RESPONSE=$(aws devicefarm create-upload \
	--project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
	--name "$APP_NAME" \
	--type "$APP_TYPE" \
	--output json)

	if [ $? -ne 0 ]; then
	echo "❌ Error creating upload in Device Farm"
	echo "Response: $UPLOAD_RESPONSE"
	exit 1
	fi

	APP_UPLOAD_URL=$(echo $UPLOAD_RESPONSE \| jq -r '.upload.url')
	APP_UPLOAD_ARN=$(echo $UPLOAD_RESPONSE \| jq -r '.upload.arn')
	echo "app_upload_arn=$APP_UPLOAD_ARN" >> $GITHUB_OUTPUT
	echo "App upload ARN: $APP_UPLOAD_ARN"

	echo "Uploading app file: $APP_PATH"
	curl -T "$APP_PATH" "$APP_UPLOAD_URL"

	if [ $? -ne 0 ]; then
	echo "❌ Error uploading app file using curl"
	exit 1
	fi

	# Wait for processing
	echo "⏳ Waiting for upload to be processed..."
	MAX_ATTEMPTS=30
	ATTEMPT=1
	while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
	STATUS=$(aws devicefarm get-upload --arn "$APP_UPLOAD_ARN" --query "upload.status" --output text)
	echo "Status (attempt $ATTEMPT/$MAX_ATTEMPTS): $STATUS"

	if [ "$STATUS" = "SUCCEEDED" ]; then
	echo "✅ App upload successful"
	break
	fi

	if [ "$STATUS" = "FAILED" ]; then
	echo "❌ Upload failed"
	aws devicefarm get-upload --arn "$APP_UPLOAD_ARN"
	exit 1
	fi

	sleep 10
	ATTEMPT=$((ATTEMPT + 1))
	done

	- name: Verify test package generation
	working-directory: test-framework/e2e
	run: \|
	echo "Verifying e2e test package..."

	if [ ! -f "package.json" ]; then
	echo "❌ ERROR: e2e/package.json not found!"
	exit 1
	fi

	if [ ! -f "tests/app.test.js" ]; then
	echo "❌ ERROR: e2e/tests/app.test.js not found!"
	exit 1
	fi

	echo "✅ E2E test files verified"
	echo ""
	echo "Test package contents:"
	ls -la
	echo ""
	echo "Test files:"
	ls -la tests/

	- name: Package and Upload Test Package
	id: upload_test_package
	working-directory: test-framework
	run: \|
	echo "📦 Packaging e2e tests..."
	cd e2e

	# Install dependencies before packing
	npm install --ignore-scripts

	# Create tarball
	npm pack --ignore-scripts

	# Create zip with test files only (no node_modules - will be installed on Device Farm)
	ZIP_NAME="e2e-tests-${{ matrix.platform }}.zip"
	zip -r "$ZIP_NAME" \
	package.json \
	tests/ \
	*.tgz

	echo "📦 Package contents (excluding node_modules):"
	unzip -l "$ZIP_NAME" \| head -20

	# Verify zip was created
	if [ ! -f "$ZIP_NAME" ]; then
	echo "❌ ERROR: Failed to create test package zip"
	exit 1
	fi

	SIZE=$(du -h "$ZIP_NAME" \| cut -f1)
	echo "✅ Test package created: $ZIP_NAME (Size: $SIZE)"

	mv "$ZIP_NAME" "$GITHUB_WORKSPACE/"

	# Upload test package to AWS Device Farm
	echo "📤 Uploading test package to AWS Device Farm..."
	UPLOAD_RESPONSE=$(aws devicefarm create-upload \
	--project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
	--name "$ZIP_NAME" \
	--type "APPIUM_NODE_TEST_PACKAGE" \
	--output json)

	if [ $? -ne 0 ]; then
	echo "❌ Error creating test package upload in Device Farm"
	echo "Response: $UPLOAD_RESPONSE"
	exit 1
	fi

	TEST_UPLOAD_URL=$(echo $UPLOAD_RESPONSE \| jq -r '.upload.url')
	TEST_UPLOAD_ARN=$(echo $UPLOAD_RESPONSE \| jq -r '.upload.arn')
	echo "test_package_upload_arn=$TEST_UPLOAD_ARN" >> $GITHUB_OUTPUT
	echo "Test package upload ARN: $TEST_UPLOAD_ARN"

	echo "Uploading to: $TEST_UPLOAD_URL"
	curl -T "$GITHUB_WORKSPACE/$ZIP_NAME" "$TEST_UPLOAD_URL"

	if [ $? -ne 0 ]; then
	echo "❌ Error uploading test package using curl"
	exit 1
	fi

	# Wait for processing
	echo "⏳ Waiting for test package to be processed..."
	MAX_ATTEMPTS=30
	ATTEMPT=1
	while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
	STATUS=$(aws devicefarm get-upload --arn "$TEST_UPLOAD_ARN" --query "upload.status" --output text)
	echo "Test package status (attempt $ATTEMPT/$MAX_ATTEMPTS): $STATUS"

	if [ "$STATUS" = "SUCCEEDED" ]; then
	echo "✅ Test package upload successful"
	break
	fi

	if [ "$STATUS" = "FAILED" ]; then
	echo "❌ Test package upload failed"
	aws devicefarm get-upload --arn "$TEST_UPLOAD_ARN"
	exit 1
	fi

	sleep 10
	ATTEMPT=$((ATTEMPT + 1))
	done

	if [ $ATTEMPT -gt $MAX_ATTEMPTS ]; then
	echo "❌ Timeout waiting for test package processing"
	exit 1
	fi

	# NOTE: Everything below remains unchanged from your source workflow.
	# The only monorepo-related change in this entire file is that "addon" operations
	# now target addon/packages/qvac-lib-infer-llamacpp-llm via env.ADDON_WORKDIR.

	- name: Create and Upload Test Spec
	id: upload_test_spec
	run: \|
	echo "📝 Creating test spec for custom environment mode..."
	echo "Platform: ${{ matrix.platform }}"

	# Create platform-specific test spec using printf for precise control
	# NOTE: Both platforms use a 'before' hook in the wdio config to click the button
	# This ensures a single Appium session for reliability (no session handoff issues)
	# The before hook includes crash detection using queryAppState
	if [ "${{ matrix.platform }}" == "Android" ]; then
	PLATFORM="Android"
	AUTOMATION="UiAutomator2"
	HOST_LINE="android_test_host: amazon_linux_2"
	BUNDLE_ID="${{ env.APP_BUNDLE_ID }}"
	# Android wdio config with crash detection and test filter support.
	# On crash, checkAppCrash/crashMonitor setTimeout-exit with a 5s
	# delay so logcat has time to drain bare stdout / native logs that
	# would otherwise be lost when process.exit(1) tears down Appium
	# before Device Farm finalises the artifact bundle.
	# __TEST_FILTER__, __QVAC_PERF_RUNS__, __QVAC_PERF_WARMUP_RUNS__
	# placeholders are replaced per-split by make_split().
	WDIO_CONFIG='exports.config={runner:"local",hostname:"127.0.0.1",port:4723,path:"/wd/hub",specs:[".spec.js",".test.js"],maxInstances:1,bail:0,capabilities:[{platformName:"Android","appium:automationName":"UiAutomator2","appium:appPackage":"'${{ env.APP_BUNDLE_ID }}'","appium:appActivity":"'${{ env.APP_BUNDLE_ID }}'.MainActivity","appium:newCommandTimeout":300,"appium:autoGrantPermissions":true,"appium:autoAcceptAlerts":true,"appium:noReset":true,"appium:dontStopAppOnReset":true,"appium:forceAppLaunch":false}],logLevel:"debug",waitforTimeout:120000,connectionRetryTimeout:30000,connectionRetryCount:3,services:[],framework:"mocha",reporters:["spec"],mochaOpts:{ui:"bdd",timeout:1800000,grep:"__MOCHA_GREP__"},before:async function(capabilities,specs,browser){const BUNDLE_ID="'${{ env.APP_BUNDLE_ID }}'";const TEST_FILTER="__TEST_FILTER__";const QVAC_PERF_RUNS_VALUE="__QVAC_PERF_RUNS__";const QVAC_PERF_WARMUP_RUNS_VALUE="__QVAC_PERF_WARMUP_RUNS__";global.appCrashed=false;global.checkAppCrash=async(stage)=>{try{const state=await browser.queryAppState(BUNDLE_ID);console.log("["+stage+"] App state: "+state+" (4=foreground,3=background,1=not running)");if(state<3){console.error("\\n🛑 APP CRASHED at "+stage+"! State="+state);console.error("Check device logs for BareKit/native errors.\\n");global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);}return state;}catch(e){console.log("["+stage+"] queryAppState error: "+e.message);return-1;}};console.log("Checking initial app state...");await global.checkAppCrash("startup");console.log("Waiting for app to initialize...");await browser.pause(5000);await global.checkAppCrash("after-pause");const initText=await browser.$("android=new UiSelector().textContains(\"INITIALIZED\")");await initText.waitForDisplayed({timeout:60000});await global.checkAppCrash("after-init");if(TEST_FILTER!=="__TEST_FILTER__"){try{const b64=Buffer.from(TEST_FILTER).toString("base64");await browser.pushFile("/data/local/tmp/testFilter.txt",b64);console.log("Pushed test filter: "+TEST_FILTER);}catch(e){console.log("pushFile failed: "+e.message);}}if(QVAC_PERF_RUNS_VALUE.length>0\|\|QVAC_PERF_WARMUP_RUNS_VALUE.length>0){try{var perfCfg="QVAC_PERF_RUNS="+QVAC_PERF_RUNS_VALUE+"\\nQVAC_PERF_WARMUP_RUNS="+QVAC_PERF_WARMUP_RUNS_VALUE+"\\n";var pcb64=Buffer.from(perfCfg).toString("base64");await browser.pushFile("/data/local/tmp/qvacPerfConfig.txt",pcb64);console.log("Pushed perf config: "+perfCfg.replace(/\\n/g," "));}catch(e){console.log("perfConfig pushFile failed: "+e.message);}}console.log("App initialized, clicking Run Automated Tests...");const button=await browser.$("android=new UiSelector().textContains(\"Run Automated Tests\")");await button.waitForDisplayed({timeout:15000});await button.click();console.log("Button clicked!");await browser.pause(5000);await global.checkAppCrash("after-click");global.crashMonitor=setInterval(async()=>{if(global.appCrashed)return;try{const s=await browser.queryAppState(BUNDLE_ID);if(s<3){console.error("\\n🛑 BACKGROUND CRASH DETECTED! App state="+s);global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);}}catch(e){}},15000);},after:async function(){if(global.crashMonitor)clearInterval(global.crashMonitor);},afterTest:async function(test,context,{error}){if(global.appCrashed)return;await global.checkAppCrash("after-test:"+test.title);}};'
	else
	PLATFORM="iOS"
	AUTOMATION="XCUITest"
	# iOS 18+ requires macos_sequoia test host (supports iOS 15-26)
	HOST_LINE="ios_test_host: macos_sequoia"
	BUNDLE_ID="${{ env.APP_BUNDLE_ID }}"
	# iOS wdio config with crash detection (bail:0 = continue on test
	# failures). On crash we (a) pre-register a 5 s setTimeout
	# process.exit BEFORE attempting any flush — this is the hard
	# upper bound on the crash path so a wedged Appium/WDA cannot
	# turn a crash into a long CI timeout — then (b) best-effort
	# attempt an Appium pull_file of bare_console.log via
	# global.flushBareLog so the C++ side log ends up in
	# $DEVICEFARM_LOG_DIR (and therefore in Customer_Artifacts.zip)
	# even when the test crashes before the after-hook would have
	# run. The flush is wrapped in Promise.race against a 3 s
	# timeout so a hung pull_file cannot extend the crash exit
	# past the 5 s budget. The after hook reuses the same helper
	# on the normal completion path.
	# usePrebuiltWDA uses Device Farm's pre-built WebDriverAgent
	# Increased timeout to 30 minutes (1800000ms) for long-running LLM tests
	WDIO_CONFIG='exports.config={runner:"local",hostname:"127.0.0.1",port:4723,path:"/wd/hub",specs:[".spec.js",".test.js"],maxInstances:1,bail:0,capabilities:[{platformName:"iOS","appium:automationName":"XCUITest","appium:bundleId":"'${{ env.APP_BUNDLE_ID }}'","appium:newCommandTimeout":300,"appium:noReset":true,"appium:forceAppLaunch":false,"appium:usePrebuiltWDA":true,"appium:wdaLocalPort":8100,"appium:showIOSLog":true,"appium:realDeviceLogger":"/usr/local/lib/node_modules/appium/node_modules/deviceconsole/deviceconsole"}],logLevel:"debug",waitforTimeout:120000,connectionRetryTimeout:30000,connectionRetryCount:3,services:[],framework:"mocha",reporters:["spec"],mochaOpts:{ui:"bdd",timeout:1800000,grep:"__MOCHA_GREP__"},before:async function(capabilities,specs,browser){const BUNDLE_ID="'${{ env.APP_BUNDLE_ID }}'";const TEST_FILTER="__TEST_FILTER__";const QVAC_PERF_RUNS_VALUE="__QVAC_PERF_RUNS__";const QVAC_PERF_WARMUP_RUNS_VALUE="__QVAC_PERF_WARMUP_RUNS__";global.appCrashed=false;global.flushBareLog=async function(reason){try{var _h=require("http");var lb64=await new Promise(function(ok,fail){var bd=JSON.stringify({path:"@"+BUNDLE_ID+":documents/bare_console.log"});var rq=_h.request({hostname:"127.0.0.1",port:4723,path:"/wd/hub/session/"+browser.sessionId+"/appium/device/pull_file",method:"POST",headers:{"Content-Type":"application/json","Content-Length":Buffer.byteLength(bd)}},function(rs){var d="";rs.on("data",function(c){d+=c;});rs.on("end",function(){try{ok(JSON.parse(d).value);}catch(e){fail(e);}});});rq.on("error",fail);rq.write(bd);rq.end();});var logTxt=Buffer.from(lb64,"base64").toString();var logDir=process.env.DEVICEFARM_LOG_DIR\|\|".";require("fs").writeFileSync(logDir+"/bare_console.log",logTxt);console.log("[bare-log] "+reason+" flush ok ("+logTxt.length+" bytes)");}catch(e){console.log("[bare-log] "+reason+" flush failed: "+e.message);}};global.checkAppCrash=async(stage)=>{try{const state=await browser.queryAppState(BUNDLE_ID);console.log("["+stage+"] App state: "+state+" (4=foreground,3=background,1=not running)");if(state<3){console.error("\\n🛑 APP CRASHED at "+stage+"! State="+state);console.error("Check device logs for BareKit/native errors.\\n");global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);try{await browser.pause(1500);await Promise.race([global.flushBareLog("crash-"+stage),new Promise(function(_,rj){setTimeout(function(){rj(new Error("bare-log flush timed out"));},3000);})]);}catch(_){}}return state;}catch(e){console.log("["+stage+"] queryAppState error: "+e.message);return-1;}};console.log("Checking initial app state...");await global.checkAppCrash("startup");console.log("Waiting for app to initialize...");await browser.pause(5000);await global.checkAppCrash("after-pause");const initText=await browser.$("-ios predicate string:label CONTAINS \"INITIALIZED\"");await initText.waitForDisplayed({timeout:60000});await global.checkAppCrash("after-init");if(TEST_FILTER!=="__TEST_FILTER__"){try{const b64=Buffer.from(TEST_FILTER).toString("base64");await browser.pushFile("@"+BUNDLE_ID+":documents/testFilter.txt",b64);console.log("Pushed test filter: "+TEST_FILTER);}catch(e){console.log("pushFile failed: "+e.message);}}if(QVAC_PERF_RUNS_VALUE.length>0\|\|QVAC_PERF_WARMUP_RUNS_VALUE.length>0){try{var perfCfg="QVAC_PERF_RUNS="+QVAC_PERF_RUNS_VALUE+"\\nQVAC_PERF_WARMUP_RUNS="+QVAC_PERF_WARMUP_RUNS_VALUE+"\\n";var pcb64=Buffer.from(perfCfg).toString("base64");await browser.pushFile("@"+BUNDLE_ID+":documents/qvacPerfConfig.txt",pcb64);console.log("Pushed perf config: "+perfCfg.replace(/\\n/g," "));}catch(e){console.log("perfConfig pushFile failed: "+e.message);}}console.log("App initialized, clicking Run Automated Tests...");const button=await browser.$("-ios predicate string:label CONTAINS \"Run Automated Tests\"");await button.waitForDisplayed({timeout:15000});await button.click();console.log("Button clicked!");await browser.pause(5000);await global.checkAppCrash("after-click");global.crashMonitor=setInterval(async()=>{if(global.appCrashed)return;try{const s=await browser.queryAppState(BUNDLE_ID);if(s<3){console.error("\\n🛑 BACKGROUND CRASH DETECTED! App state="+s);global.appCrashed=true;setTimeout(function(){process.exit(1);},5000);try{await browser.pause(1500);await Promise.race([global.flushBareLog("crash-bg"),new Promise(function(_,rj){setTimeout(function(){rj(new Error("bare-log flush timed out"));},3000);})]);}catch(_){}}}catch(e){}},15000);},after:async function(){if(global.crashMonitor)clearInterval(global.crashMonitor);console.log("[bare-log] Waiting for log flush...");await browser.pause(3000);if(global.flushBareLog)await global.flushBareLog("after");},afterTest:async function(test,context,{error}){if(global.appCrashed)return;await global.checkAppCrash("after-test:"+test.title);}};'
	fi

	WDIO_B64=$(echo "$WDIO_CONFIG" \| base64 \| tr -d '\n')

	generate_spec() {
	local output_file="$1"
	local wdio_b64="$2"
	{
	printf 'version: 0.1\n'
	if [ -n "$HOST_LINE" ]; then
	printf '%s\n' "$HOST_LINE"
	fi
	printf '\nphases:\n install:\n commands:\n'
	printf ' - echo "Setting up Node.js environment..."\n'
	printf ' - export NVM_DIR=$HOME/.nvm\n'
	printf ' - . $NVM_DIR/nvm.sh 2>/dev/null \|\| true\n'
	printf ' - nvm install 18 2>/dev/null \|\| true\n'
	printf ' - nvm use 18 2>/dev/null \|\| true\n'
	printf ' - node --version \|\| echo "Using system node"\n'
	printf '\n pre_test:\n commands:\n'
	printf ' - echo "Setting up test environment..."\n'
	printf ' - cd $DEVICEFARM_TEST_PACKAGE_PATH\n'
	printf ' - ls -la\n'
	printf ' - echo "Installing dependencies (clean install)..."\n'
	printf ' - rm -rf node_modules package-lock.json 2>/dev/null \|\| true\n'
	printf ' - npm install --legacy-peer-deps 2>&1\n'
	printf ' - echo "Verifying wdio installation..."\n'
	printf ' - ls -la node_modules/.bin/ \| grep wdio \|\| echo "wdio not found in .bin"\n'
	printf ' - node node_modules/@wdio/cli/bin/wdio.js --version \|\| echo "wdio version check failed"\n'
	printf ' - echo "Creating wdio config for Device Farm..."\n'
	printf ' - echo "%s" \| base64 -d > tests/wdio.config.devicefarm.js\n' "$wdio_b64"
	printf ' - cat tests/wdio.config.devicefarm.js\n'
	if [ "${{ matrix.platform }}" == "iOS" ]; then
	printf ' - echo "Configuring WebDriverAgent for iOS..."\n'
	printf ' - export DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH=$DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH_V9\n'
	printf ' - echo "WDA Path: $DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH"\n'
	fi
	printf ' - echo "Starting Appium server..."\n'
	printf ' - export APPIUM_BASE_PATH=/wd/hub\n'
	printf ' - \|\n'
	printf ' appium --base-path=$APPIUM_BASE_PATH --log-timestamp \\\n'
	printf ' --log-no-colors --relaxed-security --default-capabilities \\\n'
	printf ' "{\\"appium:deviceName\\": \\"$DEVICEFARM_DEVICE_NAME\\", \\\n'
	printf ' \\"platformName\\": \\"$DEVICEFARM_DEVICE_PLATFORM_NAME\\", \\\n'
	printf ' \\"appium:app\\": \\"$DEVICEFARM_APP_PATH\\", \\\n'
	printf ' \\"appium:udid\\":\\"$DEVICEFARM_DEVICE_UDID\\", \\\n'
	printf ' \\"appium:platformVersion\\": \\"$DEVICEFARM_DEVICE_OS_VERSION\\", \\\n'
	printf ' \\"appium:chromedriverExecutableDir\\": \\"$DEVICEFARM_CHROMEDRIVER_EXECUTABLE_DIR\\", \\\n'
	printf ' \\"appium:wdaLocalPort\\": 8100, \\\n'
	printf ' \\"appium:derivedDataPath\\": \\"$DEVICEFARM_APPIUM_WDA_DERIVED_DATA_PATH\\", \\\n'
	printf ' \\"appium:usePrebuiltWDA\\": true, \\\n'
	printf ' \\"appium:automationName\\": \\"%s\\"}" \\\n' "$AUTOMATION"
	printf ' >> $DEVICEFARM_LOG_DIR/appium.log 2>&1 &\n'
	printf ' - echo "Waiting for Appium to be ready (max 30 seconds)..."\n'
	printf ' - \|\n'
	printf ' appium_initialization_time=0\n'
	printf ' until curl --silent --fail "http://0.0.0.0:4723${APPIUM_BASE_PATH}/status"; do\n'
	printf ' if [[ $appium_initialization_time -gt 30 ]]; then\n'
	printf ' echo "Appium did not start within 30 seconds. Exiting..."\n'
	printf ' cat $DEVICEFARM_LOG_DIR/appium.log\n'
	printf ' exit 1\n'
	printf ' fi\n'
	printf ' appium_initialization_time=$((appium_initialization_time + 1))\n'
	printf ' echo "Waiting for Appium to start on port 4723 (${appium_initialization_time}s/30s)..."\n'
	printf ' sleep 1\n'
	printf ' done\n'
	printf ' - echo "Appium server is ready!"\n'
	printf ' - curl -s http://0.0.0.0:4723${APPIUM_BASE_PATH}/status \|\| echo "Status check failed"\n'
	printf '\n test:\n commands:\n'
	printf ' - echo "Running WebDriverIO tests..."\n'
	printf ' - cd $DEVICEFARM_TEST_PACKAGE_PATH\n'
	printf ' - ps aux \| grep appium \| grep -v grep \|\| echo "Appium process not found"\n'
	printf ' - curl -s http://127.0.0.1:4723/wd/hub/status \|\| echo "Appium status check failed"\n'
	printf ' - node node_modules/@wdio/cli/bin/wdio.js run tests/wdio.config.devicefarm.js\n'
	printf '\n post_test:\n commands:\n'
	printf ' - echo "Test completed"\n'
	if [ "${{ matrix.platform }}" == "iOS" ]; then
	printf ' - echo ""\n'
	printf ' - echo "iOS Device Console Logs"\n'
	printf ' - \|\n'
	printf ' if [ -f "$DEVICEFARM_LOG_DIR/device_console.log" ]; then\n'
	printf ' grep -i "bare\\\|console\\\|model\\\|embedding\\\|test\\\|error" "$DEVICEFARM_LOG_DIR/device_console.log" \|\| echo "No matching logs"\n'
	printf ' else\n'
	printf ' echo "No device_console.log found"\n'
	printf ' fi\n'
	printf ' - ls -lh $DEVICEFARM_LOG_DIR/ \|\| true\n'
	fi
	printf '\nartifacts:\n - $DEVICEFARM_LOG_DIR\n'
	} > "$output_file"
	echo "Generated: $output_file"
	}

	upload_spec() {
	local spec_file="$1"
	local label="$2"
	local RESP=$(aws devicefarm create-upload \
	--project-arn "${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}" \
	--name "$spec_file" \
	--type "APPIUM_NODE_TEST_SPEC" \
	--output json)
	local URL=$(echo $RESP \| jq -r '.upload.url')
	local ARN=$(echo $RESP \| jq -r '.upload.arn')
	curl -T "$spec_file" "$URL"
	local MAX=20; local A=1
	while [ $A -le $MAX ]; do
	local ST=$(aws devicefarm get-upload --arn "$ARN" --query "upload.status" --output text)
	if [ "$ST" = "SUCCEEDED" ]; then echo "$ARN"; return 0; fi
	if [ "$ST" = "FAILED" ]; then echo "$label spec upload failed" >&2; exit 1; fi
	sleep 5; A=$((A + 1))
	done
	echo "$label spec upload timed out after $MAX attempts" >&2; exit 1
	}

	# For each split: inject mocha grep AND replace __TEST_FILTER__ so
	# the before-hook pushes a testFilter.txt the app reads at runtime.
	# This ensures the app only executes matching tests (real splitting).
	# When set, the perf inputs also get substituted so the before-hook
	# pushes a qvacPerfConfig.txt with the iteration overrides.
	QVAC_PERF_RUNS_INPUT="${{ inputs.qvac_perf_runs }}"
	QVAC_PERF_WARMUP_RUNS_INPUT="${{ inputs.qvac_perf_warmup_runs }}"
	make_split() {
	local pattern="$1" output="$2"
	local cfg
	cfg=$(echo "$WDIO_CONFIG" \| sed "s#__MOCHA_GREP__#$pattern#")
	cfg=$(echo "$cfg" \| sed "s#__TEST_FILTER__#$pattern#")
	cfg=$(echo "$cfg" \| sed "s#__QVAC_PERF_RUNS__#$QVAC_PERF_RUNS_INPUT#")
	cfg=$(echo "$cfg" \| sed "s#__QVAC_PERF_WARMUP_RUNS__#$QVAC_PERF_WARMUP_RUNS_INPUT#")
	local b64
	b64=$(echo "$cfg" \| base64 \| tr -d '\n')
	generate_spec "$output" "$b64"
	}

	GROUPS_JSON="${{ github.workspace }}/${{ env.ADDON_WORKDIR }}/test/mobile/test-groups.json"

	if [ "${{ matrix.platform }}" == "Android" ]; then
	ANDROID_GROUP_A=$(jq -r '.android.groupA \| join("\|")' "$GROUPS_JSON")
	ANDROID_GROUP_B=$(jq -r '.android.groupB \| join("\|")' "$GROUPS_JSON")

	echo "Android test split (2 groups):"
	echo " Group A: $ANDROID_GROUP_A"
	echo " Group B: $ANDROID_GROUP_B"

	make_split "$ANDROID_GROUP_A" testspec-android-a.yml
	make_split "$ANDROID_GROUP_B" testspec-android-b.yml

	SPEC_ARN_A=$(upload_spec testspec-android-a.yml "Android-A")
	echo "test_spec_arn=$SPEC_ARN_A" >> $GITHUB_OUTPUT
	SPEC_ARN_B=$(upload_spec testspec-android-b.yml "Android-B")
	echo "test_spec_arn_2=$SPEC_ARN_B" >> $GITHUB_OUTPUT
	else
	IOS_GREP_HEAVY_1=$(jq -r '.ios.heavy1 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_2=$(jq -r '.ios.heavy2 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_3=$(jq -r '.ios.heavy3 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_4=$(jq -r '.ios.heavy4 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_5=$(jq -r '.ios.heavy5 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_6=$(jq -r '.ios.heavy6 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_7=$(jq -r '.ios.heavy7 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_HEAVY_8=$(jq -r '.ios.heavy8 \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_LIGHT_A=$(jq -r '.ios.lightA \| join("\|")' "$GROUPS_JSON")
	IOS_GREP_LIGHT_B=$(jq -r '.ios.lightB \| join("\|")' "$GROUPS_JSON")

	echo "iOS test split (8 heavy + 2 light = 10 groups):"
	echo " Heavy 1 (iPhone 16 Pro Max): $IOS_GREP_HEAVY_1"
	echo " Heavy 2 (iPhone 16 Pro Max): $IOS_GREP_HEAVY_2"
	echo " Heavy 3 (iPhone 17): $IOS_GREP_HEAVY_3"
	echo " Heavy 4 (OcrLighton): $IOS_GREP_HEAVY_4"
	echo " Heavy 5 (Reasoning): $IOS_GREP_HEAVY_5"
	echo " Heavy 6 (SlidingContext): $IOS_GREP_HEAVY_6"
	echo " Heavy 7 (Gemma4): $IOS_GREP_HEAVY_7"
	echo " Heavy 8 (OcrPaddle): $IOS_GREP_HEAVY_8"
	echo " Light A (iPhone 16 Pro): $IOS_GREP_LIGHT_A"
	echo " Light B (iPhone 16 Pro): $IOS_GREP_LIGHT_B"

	make_split "$IOS_GREP_HEAVY_1" testspec-heavy-1.yml
	make_split "$IOS_GREP_HEAVY_2" testspec-heavy-2.yml
	make_split "$IOS_GREP_HEAVY_3" testspec-heavy-3.yml
	make_split "$IOS_GREP_HEAVY_4" testspec-heavy-4.yml
	make_split "$IOS_GREP_HEAVY_5" testspec-heavy-5.yml
	make_split "$IOS_GREP_HEAVY_6" testspec-heavy-6.yml
	make_split "$IOS_GREP_HEAVY_7" testspec-heavy-7.yml
	make_split "$IOS_GREP_HEAVY_8" testspec-heavy-8.yml
	make_split "$IOS_GREP_LIGHT_A" testspec-light-a.yml
	make_split "$IOS_GREP_LIGHT_B" testspec-light-b.yml

	SPEC_ARN_H1=$(upload_spec testspec-heavy-1.yml "iOS-Heavy1")
	echo "test_spec_arn=$SPEC_ARN_H1" >> $GITHUB_OUTPUT
	SPEC_ARN_H2=$(upload_spec testspec-heavy-2.yml "iOS-Heavy2")
	echo "test_spec_arn_2=$SPEC_ARN_H2" >> $GITHUB_OUTPUT
	SPEC_ARN_H3=$(upload_spec testspec-heavy-3.yml "iOS-Heavy3")
	echo "test_spec_arn_3=$SPEC_ARN_H3" >> $GITHUB_OUTPUT
	SPEC_ARN_H4=$(upload_spec testspec-heavy-4.yml "iOS-Heavy4")
	echo "test_spec_arn_4=$SPEC_ARN_H4" >> $GITHUB_OUTPUT
	SPEC_ARN_H5=$(upload_spec testspec-heavy-5.yml "iOS-Heavy5")
	echo "test_spec_arn_5=$SPEC_ARN_H5" >> $GITHUB_OUTPUT
	SPEC_ARN_H6=$(upload_spec testspec-heavy-6.yml "iOS-Heavy6")
	echo "test_spec_arn_6=$SPEC_ARN_H6" >> $GITHUB_OUTPUT
	SPEC_ARN_LA=$(upload_spec testspec-light-a.yml "iOS-LightA")
	echo "test_spec_arn_7=$SPEC_ARN_LA" >> $GITHUB_OUTPUT
	SPEC_ARN_LB=$(upload_spec testspec-light-b.yml "iOS-LightB")
	echo "test_spec_arn_8=$SPEC_ARN_LB" >> $GITHUB_OUTPUT
	SPEC_ARN_H7=$(upload_spec testspec-heavy-7.yml "iOS-Heavy7")
	echo "test_spec_arn_9=$SPEC_ARN_H7" >> $GITHUB_OUTPUT
	SPEC_ARN_H8=$(upload_spec testspec-heavy-8.yml "iOS-Heavy8")
	echo "test_spec_arn_10=$SPEC_ARN_H8" >> $GITHUB_OUTPUT
	fi

	- name: Schedule Device Farm Test Run
	id: schedule_run
	run: \|
	if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
	RUN_NAME="Manual-${{ github.run_number }}-${{ matrix.platform }}"
	else
	RUN_NAME="PR-${{ github.event.pull_request.number \|\| github.run_number }}-${{ matrix.platform }}"
	fi

	PROJECT_ARN="${{ secrets.LLM_AWS_DEVICE_FARM_PROJECT_ARN }}"
	APP_ARN="${{ steps.upload_app.outputs.app_upload_arn }}"
	TEST_PACKAGE_ARN="${{ steps.upload_test_package.outputs.test_package_upload_arn }}"

	schedule_run_with_pool() {
	local pool_arn="$1"
	local name="$2"
	local spec_arn="$3"
	aws devicefarm schedule-run \
	--project-arn "$PROJECT_ARN" \
	--device-pool-arn "$pool_arn" \
	--app-arn "$APP_ARN" \
	--name "$name" \
	--test "type=APPIUM_NODE,testPackageArn=$TEST_PACKAGE_ARN,testSpecArn=$spec_arn" \
	--execution-configuration jobTimeoutMinutes=60 \
	--query 'run.arn' --output text
	}

	if [ "${{ matrix.platform }}" == "Android" ]; then
	POOL_ARN="${{ secrets.LLM_ANDROID_DEVICE_POOL_ARN }}"
	TEST_SPEC_ARN_A="${{ steps.upload_test_spec.outputs.test_spec_arn }}"
	TEST_SPEC_ARN_B="${{ steps.upload_test_spec.outputs.test_spec_arn_2 }}"

	echo "🚀 Scheduling 2 Android runs (each runs on all devices in pool)..."
	echo "Device Pool ARN: $POOL_ARN"

	RUN_ARN_1=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-Android-GroupA" "$TEST_SPEC_ARN_A")
	echo "✅ Android Group A scheduled: $RUN_ARN_1"

	RUN_ARN_2=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-Android-GroupB" "$TEST_SPEC_ARN_B")
	echo "✅ Android Group B scheduled: $RUN_ARN_2"

	echo "run_arn_1=$RUN_ARN_1" >> $GITHUB_OUTPUT
	echo "run_arn_2=$RUN_ARN_2" >> $GITHUB_OUTPUT
	echo "run_count=2" >> $GITHUB_OUTPUT
	else
	POOL_ARN="${{ secrets.LLM_IOS_DEVICE_POOL_ARN }}"
	TEST_SPEC_ARN_1="${{ steps.upload_test_spec.outputs.test_spec_arn }}"
	TEST_SPEC_ARN_2="${{ steps.upload_test_spec.outputs.test_spec_arn_2 }}"
	TEST_SPEC_ARN_3="${{ steps.upload_test_spec.outputs.test_spec_arn_3 }}"
	TEST_SPEC_ARN_4="${{ steps.upload_test_spec.outputs.test_spec_arn_4 }}"
	TEST_SPEC_ARN_5="${{ steps.upload_test_spec.outputs.test_spec_arn_5 }}"
	TEST_SPEC_ARN_6="${{ steps.upload_test_spec.outputs.test_spec_arn_6 }}"
	TEST_SPEC_ARN_7="${{ steps.upload_test_spec.outputs.test_spec_arn_7 }}"
	TEST_SPEC_ARN_8="${{ steps.upload_test_spec.outputs.test_spec_arn_8 }}"
	TEST_SPEC_ARN_9="${{ steps.upload_test_spec.outputs.test_spec_arn_9 }}"
	TEST_SPEC_ARN_10="${{ steps.upload_test_spec.outputs.test_spec_arn_10 }}"

	echo "🚀 Scheduling 10 iOS runs (each runs on ALL devices in pool)..."
	echo "Device Pool ARN: $POOL_ARN"

	RUN_ARN_1=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy1-Finetuning" "$TEST_SPEC_ARN_1")
	echo "✅ Heavy1 (Finetuning) scheduled: $RUN_ARN_1"

	RUN_ARN_2=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy2-ToolCalling" "$TEST_SPEC_ARN_2")
	echo "✅ Heavy2 (ToolCalling) scheduled: $RUN_ARN_2"

	RUN_ARN_3=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy3-Image" "$TEST_SPEC_ARN_3")
	echo "✅ Heavy3 (Image) scheduled: $RUN_ARN_3"

	RUN_ARN_4=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy4-OcrLighton" "$TEST_SPEC_ARN_4")
	echo "✅ Heavy4 (OcrLighton) scheduled: $RUN_ARN_4"

	RUN_ARN_5=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy5-Reasoning" "$TEST_SPEC_ARN_5")
	echo "✅ Heavy5 (Reasoning) scheduled: $RUN_ARN_5"

	RUN_ARN_6=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy6-SlidingContext" "$TEST_SPEC_ARN_6")
	echo "✅ Heavy6 (SlidingContext) scheduled: $RUN_ARN_6"

	RUN_ARN_7=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-LightA" "$TEST_SPEC_ARN_7")
	echo "✅ LightA scheduled: $RUN_ARN_7"

	RUN_ARN_8=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-LightB" "$TEST_SPEC_ARN_8")
	echo "✅ LightB scheduled: $RUN_ARN_8"

	RUN_ARN_9=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy7-Gemma4" "$TEST_SPEC_ARN_9")
	echo "✅ Heavy7 (Gemma4) scheduled: $RUN_ARN_9"

	RUN_ARN_10=$(schedule_run_with_pool "$POOL_ARN" "$RUN_NAME-iOS-Heavy8-OcrPaddle" "$TEST_SPEC_ARN_10")
	echo "✅ Heavy8 (OcrPaddle) scheduled: $RUN_ARN_10"

	echo "run_arn_1=$RUN_ARN_1" >> $GITHUB_OUTPUT
	echo "run_arn_2=$RUN_ARN_2" >> $GITHUB_OUTPUT
	echo "run_arn_3=$RUN_ARN_3" >> $GITHUB_OUTPUT
	echo "run_arn_4=$RUN_ARN_4" >> $GITHUB_OUTPUT
	echo "run_arn_5=$RUN_ARN_5" >> $GITHUB_OUTPUT
	echo "run_arn_6=$RUN_ARN_6" >> $GITHUB_OUTPUT
	echo "run_arn_7=$RUN_ARN_7" >> $GITHUB_OUTPUT
	echo "run_arn_8=$RUN_ARN_8" >> $GITHUB_OUTPUT
	echo "run_arn_9=$RUN_ARN_9" >> $GITHUB_OUTPUT
	echo "run_arn_10=$RUN_ARN_10" >> $GITHUB_OUTPUT
	echo "run_count=10" >> $GITHUB_OUTPUT
	fi

	echo "All runs scheduled."

	- name: Refresh AWS credentials for monitoring
	uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
	with:
	role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
	aws-region: us-west-2
	role-session-name: device-farm-monitor

	- name: Monitor Test Run
	id: monitor_run
	run: \|
	aws_retry() {
	local max_attempts=3
	local attempt=1
	local result=""
	while [ $attempt -le $max_attempts ]; do
	result=$("$@" 2>/dev/null) && { echo "$result"; return 0; }
	echo " ⚠️ AWS API attempt $attempt/$max_attempts failed, retrying in 5s..." >&2
	sleep 5
	attempt=$((attempt + 1))
	done
	echo ""
	return 1
	}

	RUN_ARN_1="${{ steps.schedule_run.outputs.run_arn_1 }}"
	RUN_ARN_2="${{ steps.schedule_run.outputs.run_arn_2 }}"
	RUN_ARN_3="${{ steps.schedule_run.outputs.run_arn_3 }}"
	RUN_ARN_4="${{ steps.schedule_run.outputs.run_arn_4 }}"
	RUN_ARN_5="${{ steps.schedule_run.outputs.run_arn_5 }}"
	RUN_ARN_6="${{ steps.schedule_run.outputs.run_arn_6 }}"
	RUN_ARN_7="${{ steps.schedule_run.outputs.run_arn_7 }}"
	RUN_ARN_8="${{ steps.schedule_run.outputs.run_arn_8 }}"
	RUN_ARN_9="${{ steps.schedule_run.outputs.run_arn_9 }}"
	RUN_ARN_10="${{ steps.schedule_run.outputs.run_arn_10 }}"
	RUN_COUNT="${{ steps.schedule_run.outputs.run_count }}"

	echo "📊 Monitoring $RUN_COUNT Device Farm run(s)..."
	for i in $(seq 1 "$RUN_COUNT"); do
	eval "echo \" Run $i: \$RUN_ARN_$i\""
	done
	echo ""

	MAX_WAIT_TIME=7200
	ELAPSED=0
	for i in $(seq 1 "$RUN_COUNT"); do eval "DONE_$i=false"; done

	while true; do
	STATUS_LINE="⏳"
	for i in $(seq 1 "$RUN_COUNT"); do
	eval "done_val=\$DONE_$i"
	if [[ "$done_val" != "true" ]]; then
	eval "arn=\$RUN_ARN_$i"
	STATUS_VAL=$(aws_retry aws devicefarm get-run --arn "$arn" --query 'run.status' --output text) \|\| true
	RESULT_VAL=$(aws_retry aws devicefarm get-run --arn "$arn" --query 'run.result' --output text) \|\| true
	eval "STATUS_$i=\"$STATUS_VAL\""
	eval "RESULT_$i=\"$RESULT_VAL\""
	if [[ "$STATUS_VAL" == "COMPLETED" ]]; then eval "DONE_$i=true"; fi
	fi
	eval "STATUS_LINE=\"\$STATUS_LINE Run $i: \$STATUS_$i (\$RESULT_$i) \|\""
	done
	echo "$STATUS_LINE ${ELAPSED}s"

	ALL_DONE=true
	for i in $(seq 1 "$RUN_COUNT"); do
	eval "done_val=\$DONE_$i"
	if [[ "$done_val" != "true" ]]; then ALL_DONE=false; fi
	done
	if [[ "$ALL_DONE" == "true" ]]; then
	echo ""
	echo "✅ All runs completed!"
	break
	fi

	if [ $ELAPSED -ge $MAX_WAIT_TIME ]; then
	echo ""
	echo "❌ Timeout: exceeded ${MAX_WAIT_TIME}s"
	exit 1
	fi

	sleep 30
	ELAPSED=$((ELAPSED + 30))
	done

	RUN_ARNS=("$RUN_ARN_1")
	for i in $(seq 2 "$RUN_COUNT"); do
	eval "arn=\$RUN_ARN_$i"
	if [ -n "$arn" ]; then
	RUN_ARNS+=("$arn")
	fi
	done

	DEVICE_COUNT=0
	USER_TEST_COUNT=0
	USER_PASSED=0
	USER_FAILED=0
	FAILED_TEST_DETAILS=()
	ALL_RESULTS_PASSED=true
	TOTAL_TOTAL=0
	TOTAL_PASSED=0
	TOTAL_FAILED=0
	TOTAL_SKIPPED=0

	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "📊 FINAL TEST RESULTS"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo ""

	RUN_INDEX=0
	for RUN_ARN in "${RUN_ARNS[@]}"; do
	RUN_INDEX=$((RUN_INDEX + 1))
	RUN_DETAILS=$(aws_retry aws devicefarm get-run --arn "$RUN_ARN" --output json) \|\| true
	[ -z "$RUN_DETAILS" ] && RUN_DETAILS='{}'
	RESULT=$(echo "$RUN_DETAILS" \| jq -r '.run.result // "UNKNOWN"')
	RUN_NAME_LABEL=$(echo "$RUN_DETAILS" \| jq -r '.run.name // "unknown"')
	COUNTERS=$(echo "$RUN_DETAILS" \| jq -r '.run.counters // {}')

	if [[ "$RESULT" != "PASSED" && "$RESULT" != "SKIPPED" ]]; then ALL_RESULTS_PASSED=false; fi

	TOTAL_TOTAL=$((TOTAL_TOTAL + $(echo "$COUNTERS" \| jq -r '.total // 0')))
	TOTAL_PASSED=$((TOTAL_PASSED + $(echo "$COUNTERS" \| jq -r '.passed // 0')))
	TOTAL_FAILED=$((TOTAL_FAILED + $(echo "$COUNTERS" \| jq -r '.failed // 0')))
	TOTAL_SKIPPED=$((TOTAL_SKIPPED + $(echo "$COUNTERS" \| jq -r '.skipped // 0')))

	PROJECT_ID=$(echo "$RUN_ARN" \| sed -n 's/.:run:$[^/]$\/.*/\1/p')
	RUN_ID=$(echo "$RUN_ARN" \| sed -n 's/.:run:[^/]\/$.*$/\1/p')

	if [ "$RUN_COUNT" -ge 2 ]; then
	echo "--- Run $RUN_INDEX: $RUN_NAME_LABEL (Result: $RESULT) ---"
	else
	echo "Result: $RESULT"
	fi

	JOBS=$(aws_retry aws devicefarm list-jobs --arn "$RUN_ARN" --output json) \|\| true
	[ -z "$JOBS" ] && JOBS='{"jobs":[]}'
	for JOB_ARN in $(echo "$JOBS" \| jq -r '.jobs[].arn'); do
	DEVICE_COUNT=$((DEVICE_COUNT + 1))
	JOB_DETAILS=$(aws_retry aws devicefarm get-job --arn "$JOB_ARN" --output json) \|\| true
	[ -z "$JOB_DETAILS" ] && JOB_DETAILS='{}'
	DEVICE_NAME=$(echo "$JOB_DETAILS" \| jq -r '.job.device.name // "Unknown Device"')
	JOB_RESULT=$(echo "$JOB_DETAILS" \| jq -r '.job.result // "UNKNOWN"')
	JOB_ID=$(echo "$JOB_ARN" \| sed -n 's/.:job:[^/]\/[^/]\/$.$/\1/p')
	CONSOLE_LINK="https://us-west-2.console.aws.amazon.com/devicefarm/home#/mobile/projects/${PROJECT_ID}/runs/${RUN_ID}/jobs/${JOB_ID}"

	if [ "$JOB_RESULT" = "PASSED" ]; then
	echo " ✅ $DEVICE_NAME: PASSED"
	USER_PASSED=$((USER_PASSED + 1))
	else
	echo " ❌ $DEVICE_NAME: $JOB_RESULT"
	USER_FAILED=$((USER_FAILED + 1))
	FAILED_TEST_DETAILS+=("❌ $DEVICE_NAME: $JOB_RESULT")
	FAILED_TEST_DETAILS+=(" 📎 View logs: $CONSOLE_LINK")
	fi
	USER_TEST_COUNT=$((USER_TEST_COUNT + 1))
	done

	echo " 🔗 Run link: https://us-west-2.console.aws.amazon.com/devicefarm/home#/mobile/projects/${PROJECT_ID}/runs/${RUN_ID}"
	echo ""
	done

	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "📊 SUMMARY"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo ""
	echo "Devices tested: $DEVICE_COUNT"
	echo " ✅ Passed: $USER_PASSED"
	echo " ❌ Failed: $USER_FAILED"
	echo ""

	if [ ${#FAILED_TEST_DETAILS[@]} -gt 0 ]; then
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "❌ FAILED TESTS"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	for failed_info in "${FAILED_TEST_DETAILS[@]}"; do
	echo "$failed_info"
	done
	echo ""
	fi

	echo "Device Farm totals: $TOTAL_TOTAL \| Passed: $TOTAL_PASSED \| Failed: $TOTAL_FAILED \| Skipped: $TOTAL_SKIPPED"
	echo ""

	if [ $USER_FAILED -gt 0 ]; then
	echo "test_result=FAILED" >> $GITHUB_OUTPUT
	elif [ $USER_PASSED -gt 0 ]; then
	echo "test_result=PASSED" >> $GITHUB_OUTPUT
	else
	echo "test_result=FAILED" >> $GITHUB_OUTPUT
	fi
	echo "test_total=$TOTAL_TOTAL" >> $GITHUB_OUTPUT
	echo "test_passed=$TOTAL_PASSED" >> $GITHUB_OUTPUT
	echo "test_failed=$TOTAL_FAILED" >> $GITHUB_OUTPUT
	echo "test_skipped=$TOTAL_SKIPPED" >> $GITHUB_OUTPUT
	echo "user_test_count=$USER_TEST_COUNT" >> $GITHUB_OUTPUT
	echo "user_test_passed=$USER_PASSED" >> $GITHUB_OUTPUT
	echo "user_test_failed=$USER_FAILED" >> $GITHUB_OUTPUT

	if [ $USER_FAILED -gt 0 ]; then
	echo ""
	echo "❌ Device Farm tests failed"
	echo " Devices: $USER_PASSED passed, $USER_FAILED failed (out of $USER_TEST_COUNT)"
	echo " Totals: $TOTAL_TOTAL \| Passed: $TOTAL_PASSED \| Failed: $TOTAL_FAILED \| Skipped: $TOTAL_SKIPPED"
	exit 1
	elif [ $USER_PASSED -eq 0 ] && [[ "$ALL_RESULTS_PASSED" != "true" ]]; then
	echo ""
	echo "❌ No devices ran tests (all runs skipped or errored)"
	exit 1
	fi

	echo ""
	echo "✅ Device Farm tests passed!"
	echo " Devices: $USER_PASSED passed (out of $USER_TEST_COUNT)"
	if [ "$TOTAL_SKIPPED" -gt 0 ]; then
	echo " ⚠️ $TOTAL_SKIPPED test(s) skipped (device unavailable)"
	fi
	echo " Totals: $TOTAL_TOTAL \| Passed: $TOTAL_PASSED \| Failed: $TOTAL_FAILED \| Skipped: $TOTAL_SKIPPED"

	- name: Refresh AWS credentials for log download
	if: always() && steps.schedule_run.outputs.run_arn_1
	uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # 6.0.0
	with:
	role-to-assume: ${{ secrets.AWS_OIDC_ROLE_ARN }}
	aws-region: us-west-2
	role-session-name: device-farm-logs

	- name: Download Device Farm Logs
	if: always() && steps.schedule_run.outputs.run_arn_1
	run: \|
	RUN_ARN_1="${{ steps.schedule_run.outputs.run_arn_1 }}"
	RUN_ARN_2="${{ steps.schedule_run.outputs.run_arn_2 }}"
	RUN_ARN_3="${{ steps.schedule_run.outputs.run_arn_3 }}"
	RUN_ARN_4="${{ steps.schedule_run.outputs.run_arn_4 }}"
	RUN_ARN_5="${{ steps.schedule_run.outputs.run_arn_5 }}"
	RUN_ARN_6="${{ steps.schedule_run.outputs.run_arn_6 }}"
	RUN_ARN_7="${{ steps.schedule_run.outputs.run_arn_7 }}"
	RUN_ARN_8="${{ steps.schedule_run.outputs.run_arn_8 }}"
	RUN_ARN_9="${{ steps.schedule_run.outputs.run_arn_9 }}"
	RUN_ARN_10="${{ steps.schedule_run.outputs.run_arn_10 }}"
	RUN_COUNT="${{ steps.schedule_run.outputs.run_count }}"
	LOG_DIR="devicefarm-logs/${{ matrix.platform }}"
	PLATFORM="${{ matrix.platform }}"
	mkdir -p "$LOG_DIR"

	RUN_ARNS=("$RUN_ARN_1")
	for i in $(seq 2 "$RUN_COUNT"); do
	eval "arn=\$RUN_ARN_$i"
	if [ -n "$arn" ]; then
	RUN_ARNS+=("$arn")
	fi
	done

	echo ""
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo "📥 DOWNLOADING DEVICE FARM LOGS"
	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
	echo ""

	for RUN_ARN in "${RUN_ARNS[@]}"; do
	RUN_DETAILS=$(aws devicefarm get-run --arn "$RUN_ARN" --output json 2>/dev/null \|\| echo '{}')
	RUN_LABEL=$(echo "$RUN_DETAILS" \| jq -r '.run.name // "unknown"')
	echo ""
	echo "========================================"
	echo "📦 Run: $RUN_LABEL"
	echo "========================================"

	SAFE_RUN=$(echo "$RUN_LABEL" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')
	JOBS=$(aws devicefarm list-jobs --arn "$RUN_ARN" --output json 2>/dev/null \|\| echo '{"jobs":[]}')

	for JOB_ARN in $(echo "$JOBS" \| jq -r '.jobs[].arn'); do
	DEVICE_NAME=$(echo "$JOBS" \| jq -r --arg arn "$JOB_ARN" '.jobs[] \| select(.arn == $arn) \| .device.name // "unknown"')
	JOB_RESULT=$(echo "$JOBS" \| jq -r --arg arn "$JOB_ARN" '.jobs[] \| select(.arn == $arn) \| .result // "UNKNOWN"')
	SAFE_NAME=$(echo "$DEVICE_NAME" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')

	echo ""
	echo "📱 $DEVICE_NAME ($JOB_RESULT)"

	SUITES=$(aws devicefarm list-suites --arn "$JOB_ARN" --output json 2>/dev/null \|\| echo '{"suites":[]}')

	for SUITE_ARN in $(echo "$SUITES" \| jq -r '.suites[].arn'); do
	SUITE_NAME=$(echo "$SUITES" \| jq -r --arg arn "$SUITE_ARN" '.suites[] \| select(.arn == $arn) \| .name // "unknown"')
	SAFE_SUITE=$(echo "$SUITE_NAME" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')

	ARTIFACTS=$(aws devicefarm list-artifacts --arn "$SUITE_ARN" --type FILE --output json 2>/dev/null \|\| echo '{"artifacts":[]}')
	echo "$ARTIFACTS" \| jq -c '.artifacts[]' 2>/dev/null \| while read -r ARTIFACT; do
	ART_NAME=$(echo "$ARTIFACT" \| jq -r '.name // "unknown"')
	ART_URL=$(echo "$ARTIFACT" \| jq -r '.url // empty')
	ART_EXT=$(echo "$ARTIFACT" \| jq -r '.extension // "txt"')
	[ -z "$ART_URL" ] && continue
	if [ "$PLATFORM" = "Android" ]; then
	if echo "$ART_NAME" \| grep -qiE "^video$" \|\| echo "$ART_EXT" \| grep -qiE "^mp4$"; then
	echo " Skipped (video): $SUITE_NAME / $ART_NAME"
	continue
	fi
	fi
	SAFE_ART=$(echo "$ART_NAME" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')
	DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_${SAFE_SUITE}_${SAFE_ART}.${ART_EXT}"
	if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
	echo " Downloaded: $SUITE_NAME / $ART_NAME"
	if echo "$ART_NAME" \| grep -qiE "test.spec\|testspec"; then
	echo ""; echo "::group::📋 [$DEVICE_NAME] $SUITE_NAME -- $ART_NAME"
	cat "$DEST" 2>/dev/null \|\| true; echo "::endgroup::"
	fi
	fi
	done

	LOG_ARTIFACTS=$(aws devicefarm list-artifacts --arn "$SUITE_ARN" --type LOG --output json 2>/dev/null \|\| echo '{"artifacts":[]}')
	echo "$LOG_ARTIFACTS" \| jq -c '.artifacts[]' 2>/dev/null \| while read -r ARTIFACT; do
	ART_NAME=$(echo "$ARTIFACT" \| jq -r '.name // "unknown"')
	ART_URL=$(echo "$ARTIFACT" \| jq -r '.url // empty')
	ART_EXT=$(echo "$ARTIFACT" \| jq -r '.extension // "txt"')
	[ -z "$ART_URL" ] && continue
	SAFE_ART=$(echo "$ART_NAME" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')
	DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_${SAFE_SUITE}_${SAFE_ART}.${ART_EXT}"
	if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
	echo " Downloaded: $SUITE_NAME / $ART_NAME (LOG)"
	fi
	done
	done

	JOB_ARTIFACTS=$(aws devicefarm list-artifacts --arn "$JOB_ARN" --type FILE --output json 2>/dev/null \|\| echo '{"artifacts":[]}')
	echo "$JOB_ARTIFACTS" \| jq -c '.artifacts[]' 2>/dev/null \| while read -r ARTIFACT; do
	ART_NAME=$(echo "$ARTIFACT" \| jq -r '.name // "unknown"')
	ART_URL=$(echo "$ARTIFACT" \| jq -r '.url // empty')
	ART_EXT=$(echo "$ARTIFACT" \| jq -r '.extension // "txt"')
	[ -z "$ART_URL" ] && continue
	if [ "$PLATFORM" = "Android" ]; then
	if echo "$ART_NAME" \| grep -qiE "^video$" \|\| echo "$ART_EXT" \| grep -qiE "^mp4$"; then
	echo " Skipped (video): job-level / $ART_NAME"
	continue
	fi
	fi
	SAFE_ART=$(echo "$ART_NAME" \| tr ' /' '__' \| tr -cd '[:alnum:]_-')
	DEST="$LOG_DIR/${SAFE_RUN}_${SAFE_NAME}_job_${SAFE_ART}.${ART_EXT}"
	if curl -fsSL --max-time 300 -o "$DEST" "$ART_URL" 2>/dev/null; then
	echo " Downloaded (job-level): $ART_NAME"
	fi
	done
	done
	done

	echo ""
	echo "📦 All downloaded logs:"
	find "$LOG_DIR" -type f -exec ls -lh {} \; 2>/dev/null \|\| echo " (no logs downloaded)"

	- name: Extract Console Logs
	if: always() && steps.schedule_run.outputs.run_arn_1
	run: \|
	LOG_DIR="devicefarm-logs/${{ matrix.platform }}"
	CONSOLE_DIR="console-logs/${{ matrix.platform }}"
	mkdir -p "$CONSOLE_DIR"
	for f in "$LOG_DIR"/*; do
	[ -f "$f" ] \|\| continue
	fname=$(basename "$f")
	if echo "$fname" \| grep -qiE "Logcat\."; then
	cp "$f" "$CONSOLE_DIR/"
	echo " Console log: $fname"
	fi
	if echo "$fname" \| grep -qiE "Customer_Artifacts\.zip$"; then
	prefix=$(echo "$fname" \| sed 's/_Customer_Artifacts\.zip$//')
	tmpdir=$(mktemp -d)
	if unzip -qo "$f" -d "$tmpdir" 2>/dev/null; then
	bare_log=$(find "$tmpdir" -name "bare_console.log" -type f 2>/dev/null \| head -1)
	if [ -n "$bare_log" ]; then
	cp "$bare_log" "$CONSOLE_DIR/${prefix}_bare_console.log"
	echo " Extracted: ${prefix}_bare_console.log"
	fi
	appium_log=$(find "$tmpdir" -name "appium.log" -type f 2>/dev/null \| head -1)
	if [ -n "$appium_log" ]; then
	cp "$appium_log" "$CONSOLE_DIR/${prefix}_appium.log"
	echo " Extracted: ${prefix}_appium.log"
	fi
	else
	echo " WARNING: Failed to unzip $fname"
	fi
	rm -rf "$tmpdir"
	fi
	done
	echo ""
	echo "📋 Console logs extracted:"
	find "$CONSOLE_DIR" -type f -exec ls -lh {} \; 2>/dev/null \|\| echo " (no console logs found)"

	- name: Upload Console Logs
	if: always() && steps.schedule_run.outputs.run_arn_1
	uses: actions/upload-artifact@v4
	with:
	name: console-logs-${{ matrix.platform }}
	path: console-logs/
	retention-days: 30
	if-no-files-found: ignore

	- name: Upload Full Device Farm Logs
	if: always() && steps.schedule_run.outputs.run_arn_1
	uses: actions/upload-artifact@v4
	with:
	name: devicefarm-logs-${{ matrix.platform }}
	path: devicefarm-logs/
	retention-days: 30
	if-no-files-found: ignore

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Create new buckets to run tests in independent processes (#1889) #5

Workflow file

Create new buckets to run tests in independent processes (#1889) #5

Uh oh!

Workflow file for this run

GitHub Actions / .github/workflows/integration-mobile-test-qvac-lib-infer-llamacpp-llm.yml