mobile_job.yml

name: Run mobile tests on devices

on:
  workflow_call:
    inputs:
      job-name:
        description: Name for the job, which is displayed in the GitHub UI
        default: mobile-job
        type: string
      device-type:
        description: The type of device (iOS or Android) to test against
        type: string
      runner:
        description: The runner to run the test on
        type: string
      timeout:
        description: Timeout for the job (in minutes)
        required: false
        default: 60
        type: number
      python-version:
        description: Set the python version used in the job
        required: false
        type: string
        default: '3.11'

      # AWS Device Farm, this can be copied from AWS console and it's default to
      # PyTorch project
      project-arn:
        description: The AWS Device Farm project where the test runs
        default: 'arn:aws:devicefarm:us-west-2:308535385114:project:b531574a-fb82-40ae-b687-8f0b81341ae0'
        type: string
      device-pool-arn:
        description: The device pool associated with the project
        default: 'arn:aws:devicefarm:us-west-2::devicepool:082d10e5-d7d7-48a5-ba5c-b33d66efa1f5'
        type: string
      new-output-format-flag:
        description: experiment flag to enable the new artifact json format
        required: false
        default: false
        type: boolean

      # Pulling test-infra itself for device farm runner script
      test-infra-repository:
        description: Test infra repository to use
        default: 'pytorch/test-infra'
        type: string
      test-infra-ref:
        description: Test infra reference to use
        default: ''
        type: string

      # iOS-specific inputs
      ios-ipa-archive:
        description: |
          Either a link to the iOS app IPA archive to run the tests or an existing ARN
        required: false
        type: string
        default: ''
      ios-xctestrun-zip:
        description: |
          Either a link to the iOS xctestrun zip archive or an existing ARN
        required: false
        type: string
        default: ''

      # Android-specific inputs
      android-app-archive:
        description: |
          Either a link to the Android app APK archive to run or an existing ARN
        required: false
        type: string
        default: ''
      android-test-archive:
        description: |
          Either a link to the Android instrumentation tests APK archive to run or
          an existing ARN
        required: false
        type: string
        default: ''

      # Some share test inputs
      test-spec:
        description: |
          Specify how the test should be run on device. This could either be a link to
          download the spec or an existing ARN if the spec has previously been uploaded
          to AWS
        required: false
        type: string
        default: ''
      # The extra data archive could be large, so it's better to keep them on S3
      extra-data:
        description: |
          Either a link to a zip archive on S3 to be uploaded to the test device or
          an existing ARN, for example, exported models
        required: false
        type: string
        default: ''

    outputs:
      artifacts:
        description: |
          The list of artifacts from AWS in JSON format returned to the caller
        value: ${{ jobs.mobile.outputs.artifacts }}

jobs:
  mobile:
    name: ${{ inputs.job-name }} (${{ inputs.device-type }})
    runs-on: ${{ inputs.runner }}
    timeout-minutes: ${{ inputs.timeout }}
    permissions:
      id-token: write
      contents: read
    outputs:
      artifacts: ${{ inputs.device-type == 'ios' && steps.ios-test.outputs.artifacts || inputs.device-type == 'android' && steps.android-test.outputs.artifacts || '[]' }}
    env:
      CURL_PARAMS: -s --fail
    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Authenticate with AWS
        uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0
        with:
          role-to-assume: arn:aws:iam::308535385114:role/gha_workflow_mobile_job
          # The max duration enforced by the server side
          role-duration-seconds: 18000
          aws-region: us-east-1

      - name: Checkout repository (${{ inputs.test-infra-repository }}@${{ inputs.test-infra-ref }})
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          repository: ${{ inputs.test-infra-repository }}
          ref: ${{ inputs.test-infra-ref }}
          path: test-infra

      - name: Setup miniconda
        uses: ./test-infra/.github/actions/setup-miniconda
        with:
          python-version: ${{ inputs.python-version }}

      - name: Install pip dependencies
        shell: bash
        working-directory: test-infra/tools/device-farm-runner
        run: |
          ${CONDA_RUN} pip install -r requirements.txt

      - name: Verify iOS artifacts
        id: verify-ios-artifacts
        if: ${{ inputs.device-type == 'ios' }}
        shell: bash
        working-directory: test-infra/tools/device-farm-runner
        env:
          IPA_ARCHIVE: ${{ inputs.ios-ipa-archive }}
          XCTESTRUN_ZIP: ${{ inputs.ios-xctestrun-zip }}
        run: |
          set -ex

          if [ -z "${IPA_ARCHIVE}" ] || [ -z "${XCTESTRUN_ZIP}" ]; then
            echo "Missing IPA archive or xctestrun zip"
            exit 1
          fi

          if [[ "${IPA_ARCHIVE}" == http* ]]; then
            IPA_ARCHIVE_OUTPUT="ci.ipa"

            # shellcheck disable=SC2086
            curl ${CURL_PARAMS} "${IPA_ARCHIVE}" -o "${IPA_ARCHIVE_OUTPUT}"
            ls -lah "${IPA_ARCHIVE_OUTPUT}"
          else
            IPA_ARCHIVE_OUTPUT="${IPA_ARCHIVE}"
          fi

          echo "ipa-archive-output=${IPA_ARCHIVE_OUTPUT}" >> "${GITHUB_OUTPUT}"

          if [[ "${XCTESTRUN_ZIP}" == http* ]]; then
            XCTESTRUN_ZIP_OUTPUT="ci.xctestrun.zip"

            # shellcheck disable=SC2086
            curl ${CURL_PARAMS} "${XCTESTRUN_ZIP}" -o "${XCTESTRUN_ZIP_OUTPUT}"
            ls -lah "${XCTESTRUN_ZIP_OUTPUT}"
          else
            XCTESTRUN_ZIP_OUTPUT="${XCTESTRUN_ZIP}"
          fi

          echo "xctestrun-zip-output=${XCTESTRUN_ZIP_OUTPUT}" >> "${GITHUB_OUTPUT}"

      - name: Verify Android artifacts
        id: verify-android-artifacts
        if: ${{ inputs.device-type == 'android' }}
        shell: bash
        working-directory: test-infra/tools/device-farm-runner
        env:
          APP_ARCHIVE: ${{ inputs.android-app-archive }}
          TEST_ARCHIVE: ${{ inputs.android-test-archive }}
        run: |
          set -ex

          if [ -z "${APP_ARCHIVE}" ] || [ -z "${TEST_ARCHIVE}" ]; then
            echo "Missing the app or test archives"
            exit 1
          fi

          if [[ "${APP_ARCHIVE}" == http* ]]; then
            APP_ARCHIVE_OUTPUT="ci.apk"

            # shellcheck disable=SC2086
            curl ${CURL_PARAMS} "${APP_ARCHIVE}" -o "${APP_ARCHIVE_OUTPUT}"
            ls -lah "${APP_ARCHIVE_OUTPUT}"
          else
            APP_ARCHIVE_OUTPUT="${APP_ARCHIVE}"
          fi

          echo "app-archive-output=${APP_ARCHIVE_OUTPUT}" >> "${GITHUB_OUTPUT}"

          if [[ "${TEST_ARCHIVE}" == http* ]]; then
            TEST_ARCHIVE_OUTPUT="ci.test.apk"

            # shellcheck disable=SC2086
            curl ${CURL_PARAMS} "${TEST_ARCHIVE}" -o "${TEST_ARCHIVE_OUTPUT}"
            ls -lah "${TEST_ARCHIVE_OUTPUT}"
          else
            TEST_ARCHIVE_OUTPUT="${TEST_ARCHIVE}"
          fi

          echo "test-archive-output=${TEST_ARCHIVE_OUTPUT}" >> "${GITHUB_OUTPUT}"

      - name: Verify test spec
        id: verify-test-spec
        shell: bash
        working-directory: test-infra/tools/device-farm-runner
        env:
          TEST_SPEC: ${{ inputs.test-spec }}
        run: |
          set -ex

          if [[ "${TEST_SPEC}" == http* ]]; then
            TEST_SPEC_OUTPUT="ci.yml"

            # shellcheck disable=SC2086
            curl ${CURL_PARAMS} "${TEST_SPEC}" -o "${TEST_SPEC_OUTPUT}"
            cat "${TEST_SPEC_OUTPUT}"
          else
            TEST_SPEC_OUTPUT="${TEST_SPEC}"
          fi

          echo "test-spec-output=${TEST_SPEC_OUTPUT}" >> "${GITHUB_OUTPUT}"

      - name: Verify extra data archive
        id: verify-extra-data
        shell: bash
        working-directory: test-infra/tools/device-farm-runner
        env:
          EXTRA_DATA: ${{ inputs.extra-data }}
        run: |
          set -ex

          if [ -n "${EXTRA_DATA}" ]; then
            if [[ "${EXTRA_DATA}" == http* ]]; then
              EXTRA_DATA_OUTPUT="extra-data.zip"

              # shellcheck disable=SC2086
              curl ${CURL_PARAMS} "${EXTRA_DATA}" -o "${EXTRA_DATA_OUTPUT}"
              ls -lah "${EXTRA_DATA_OUTPUT}"
            else
              EXTRA_DATA_OUTPUT="${EXTRA_DATA}"
            fi
          else
            EXTRA_DATA_OUTPUT=""
          fi

          echo "extra-data-output=${EXTRA_DATA_OUTPUT}" >> "${GITHUB_OUTPUT}"

      - name: Get workflow job id
        id: get-job-id
        uses: ./test-infra/.github/actions/get-workflow-job-id
        if: always()
        with:
          working-directory: test-infra
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Slow down the incoming requests to mitigate AWS rate limit
        id: randomize-retry
        shell: bash
        continue-on-error: true
        env:
          MAX_WAIT_TIME_IN_SECONDS: 120
        run: |
          set -ex

          # NB: AWS imposes a rate limit somewhere on the number of requests
          # we can submit to them. Let's just slow down a bit here
          WAIT_TIME_IN_SECONDS=$((RANDOM % MAX_WAIT_TIME_IN_SECONDS))
          echo "WAIT_TIME_IN_SECONDS=${WAIT_TIME_IN_SECONDS}" >> "${GITHUB_ENV}"

          sleep "${WAIT_TIME_IN_SECONDS}"

      - name: Run iOS tests on devices
        id: ios-test
        if: ${{ inputs.device-type == 'ios' }}
        env:
          PROJECT_ARN: ${{ inputs.project-arn }}
          DEVICE_POOL_ARN: ${{ inputs.device-pool-arn }}
          IPA_ARCHIVE: ${{ steps.verify-ios-artifacts.outputs.ipa-archive-output }}
          XCTESTRUN_ZIP: ${{ steps.verify-ios-artifacts.outputs.xctestrun-zip-output }}
          TEST_SPEC: ${{ steps.verify-test-spec.outputs.test-spec-output }}
          EXTRA_DATA: ${{ steps.verify-extra-data.outputs.extra-data-output }}
          # For record keeping
          JOB_NAME: ${{ inputs.job-name }}
          DEVICE_TYPE: ${{ inputs.device-type }}
          RUN_ID: ${{ github.run_id }}
          RUN_ATTEMPT: ${{ github.run_attempt }}
          JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
          GIT_JOB_NAME:  ${{ steps.get-job-id.outputs.job-name }}
          WORKING_DIRECTORY: test-infra/tools/device-farm-runner
          NEW_OUTPUT_FORMAT_FLAG: ${{ inputs.new-output-format-flag }}
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          shell: bash
          timeout_minutes: ${{ inputs.timeout }}
          max_attempts: 3
          retry_wait_seconds: ${{ env.WAIT_TIME_IN_SECONDS || 120 }}
          command: |
            set -ex

            pushd "${WORKING_DIRECTORY}"
            ${CONDA_RUN} python run_on_aws_devicefarm.py \
              --project-arn "${PROJECT_ARN}" \
              --device-pool-arn "${DEVICE_POOL_ARN}" \
              --app "${IPA_ARCHIVE}" \
              --ios-xctestrun "${XCTESTRUN_ZIP}" \
              --extra-data "${EXTRA_DATA}" \
              --test-spec "${TEST_SPEC}" \
              --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
              --workflow-id "${RUN_ID}" \
              --workflow-attempt "${RUN_ATTEMPT}" \
              --output "ios-artifacts-${JOB_ID}.json" \
              --git-job-name "${GIT_JOB_NAME}" \
              --new-json-output-format "${NEW_OUTPUT_FORMAT_FLAG}"
            popd

      - name: Run Android tests on devices
        id: android-test
        if: ${{ inputs.device-type == 'android' }}
        env:
          PROJECT_ARN: ${{ inputs.project-arn }}
          DEVICE_POOL_ARN: ${{ inputs.device-pool-arn }}
          APP_ARCHIVE: ${{ steps.verify-android-artifacts.outputs.app-archive-output }}
          TEST_ARCHIVE: ${{ steps.verify-android-artifacts.outputs.test-archive-output }}
          TEST_SPEC: ${{ steps.verify-test-spec.outputs.test-spec-output }}
          EXTRA_DATA: ${{ steps.verify-extra-data.outputs.extra-data-output }}
          # For record keeping
          JOB_NAME: ${{ inputs.job-name }}
          DEVICE_TYPE: ${{ inputs.device-type }}
          RUN_ID: ${{ github.run_id }}
          RUN_ATTEMPT: ${{ github.run_attempt }}
          JOB_ID: ${{ steps.get-job-id.outputs.job-id }}
          GIT_JOB_NAME:  ${{ steps.get-job-id.outputs.job-name }}
          WORKING_DIRECTORY: test-infra/tools/device-farm-runner
          NEW_OUTPUT_FORMAT_FLAG: ${{ inputs.new-output-format-flag }}
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          shell: bash
          timeout_minutes: ${{ inputs.timeout }}
          max_attempts: 3
          retry_wait_seconds: ${{ env.WAIT_TIME_IN_SECONDS || 120 }}
          command: |
            set -ex

            pushd "${WORKING_DIRECTORY}"
            ${CONDA_RUN} python run_on_aws_devicefarm.py \
              --project-arn "${PROJECT_ARN}" \
              --device-pool-arn "${DEVICE_POOL_ARN}" \
              --app "${APP_ARCHIVE}" \
              --android-instrumentation-test "${TEST_ARCHIVE}" \
              --extra-data "${EXTRA_DATA}" \
              --test-spec "${TEST_SPEC}" \
              --name-prefix "${JOB_NAME}-${DEVICE_TYPE}" \
              --workflow-id "${RUN_ID}" \
              --workflow-attempt "${RUN_ATTEMPT}" \
              --output "android-artifacts-${JOB_ID}.json" \
              --git-job-name "${GIT_JOB_NAME}" \
              --new-json-output-format "${NEW_OUTPUT_FORMAT_FLAG}"
            popd

      - name: Check artifacts if any job fails
        if: failure()
        working-directory: test-infra/tools/device-farm-runner
        shell: bash
        env:
          DEVICE_TYPE: ${{ inputs.device-type }}
          BENCHMARK_OUTPUT: ${{ inputs.device-type }}-artifacts-${{ steps.get-job-id.outputs.job-id }}.json
          GIT_JOB_NAME: ${{ steps.get-job-id.outputs.job-name }}
        run: |
          if [[ ! -f "$BENCHMARK_OUTPUT" ]]; then
            echo "missing artifact json file for ${DEVICE_TYPE} with name ${BENCHMARK_OUTPUT}, generating ... "
            echo "{\"git_job_name\": \"$GIT_JOB_NAME\"}" >> "$BENCHMARK_OUTPUT"
          fi

      - name: Upload artifacts to S3
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: always()
        with:
          retention-days: 14
          s3-bucket: gha-artifacts
          s3-prefix: |
            device_farm/${{ github.run_id }}/${{ github.run_attempt }}/artifacts
          path: |
            test-infra/tools/device-farm-runner/${{ inputs.device-type }}-artifacts-${{ steps.get-job-id.outputs.job-id }}.json