full_stack_tests.yml

name: Full-stack tests
permissions: read-all
on:
  merge_group:
    types: [checks_requested]
  push:
    branches:
      - develop
      - release-*
  pull_request:
    branches:
      - develop
      - release-*

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true
jobs:
  e2e_and_acceptance_coverage:
    name: Verify all e2e/acceptance tests are included
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Check that all e2e and acceptance test files are captured in wdio.conf.js and core/tests/ci-test-suite-configs
        run: python -m scripts.check_tests_are_captured_in_ci
      - name: Report failure if failed on oppia/oppia develop branch
        if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
        uses: ./.github/actions/send-webhook-notification
        with:
          message: "The e2e/acceptance coverage test failed on the upstream develop branch."
          webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
  check_test_suites_to_run:
    name: Compute which tests to run
    runs-on: ubuntu-22.04
    outputs:
      TEST_SUITES_TO_RUN: ${{ steps.compute_test_suites.outputs.TEST_SUITES_TO_RUN }}
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
        with:
          # This fetches all git refs, which is needed in the
          # check_ci_test_suites_to_run script below.
          fetch-depth: 0
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - id: compute_test_suites
        name: Check test suites to run
        env:
          SHOULD_OUTPUT_ALL_TESTS: ${{ github.event_name != 'pull_request' || vars.RUN_SUITES_ON_CHANGED_FILES == 'false' }}
        # Note that the script also writes the output to $GITHUB_OUTPUT.
        run: |
          TEST_SUITES_TO_RUN=$(python -m scripts.check_ci_test_suites_to_run --github_head_ref="HEAD" --github_base_ref="origin/${{ github.event.pull_request.base.ref }}" ${{ env.SHOULD_OUTPUT_ALL_TESTS == 'true' && '--output_all_test_suites' || '' }})
      - name: Upload root files mapping as a GitHub artifact
        uses: actions/upload-artifact@v4
        with:
          name: root-files-mapping
          path: root-files-mapping.json
  build:
    name: Build the app, and store build files as an artifact
    needs: [check_test_suites_to_run]
    runs-on: ubuntu-22.04
    if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.count > 0 ||
            fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.count > 0 ||
            fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.count > 0 }}
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Generate build files
        uses: ./.github/actions/generate-build-files
  e2e_test:
    needs: [check_test_suites_to_run, build]
    runs-on: ubuntu-22.04
    if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.count > 0 }}
    strategy:
      fail-fast: false
      matrix:
        suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).e2e.suites }}
    name: E2E (${{ matrix.suite.name }})
    steps:
      - name: Check for broken suites
        id: check_skip
        # TODO(#22448): Remove once all suites are fixed.
        run: |
          broken_suites=("contributorDashboard" "contributorAdminDashboard" "explorationTranslationTab" "fileUploadFeatures")
          for suite in "${broken_suites[@]}"; do
            if [[ "${{ matrix.suite.name }}" == "$suite" ]]; then
              echo "Suite '${{ matrix.suite.name }}' is marked to be skipped."
              echo "SKIP_SUITE=true" >> $GITHUB_OUTPUT
              break
            fi
          done
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Generate build files
        uses: ./.github/actions/generate-build-files
      - name: Install Chrome
        uses: ./.github/actions/install-chrome
      - name: Install ffmpeg for wdio videos
        run: |
          sudo apt-get update
          sudo apt install ffmpeg
      - name: Generate modified suite name for artifacts
        id: generate_suite_name_for_artifacts
        # Replace slashes in the filename with hyphens.
        run: |
          SUITE_NAME=${{ matrix.suite.name }}
          echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT
      - name: Run E2E Test ${{ matrix.suite.name }}
        # TODO(#22448): Remove once all suites are fixed.
        if: ${{ steps.check_skip.outputs.SKIP_SUITE != 'true' }}
        run: >
          set -o pipefail;
          VIDEO_RECORDING_IS_ENABLED=0
          xvfb-run -a --server-args="-screen 0, 1285x1000x24"
          python -m scripts.run_e2e_tests --skip_install
          --skip_build --suite=${{ matrix.suite.name }} --prod_env --server_log_level=info
          | tee e2e_test_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log
      - name: Upload E2E test logs
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: e2e-test-logs-${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}
          path: e2e_test_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log
      - name: Run E2E Test ${{ matrix.suite.name }} (Attempt 2 with screen recording)
        if: ${{ failure() }}
        run: >
          set -o pipefail;
          VIDEO_RECORDING_IS_ENABLED=1
          xvfb-run -a --server-args="-screen 0, 1285x1000x24"
          python -m scripts.run_e2e_tests --skip_install
          --skip_build --suite=${{ matrix.suite.name }} --prod_env --server_log_level=info
          | tee e2e_test_retry_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log
      - name: Upload E2E retry test logs
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: e2e-test-retry-logs-${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}
          path: e2e_test_retry_${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.log
      - name: Uploading webdriverio-video as Artifacts
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: webdriverio-video-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/webdriverio-video
      - name: Uploading webdriverio screenshots as Artifacts
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: webdriverio-screenshots-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/webdriverio-screenshots
      - name: Uploading webpack bundles as an artifact
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: webpack-bundles-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/oppia/build
      - name: Report failure if failed on oppia/oppia develop branch
        if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
        uses: ./.github/actions/send-webhook-notification
        with:
          message: "An E2E test failed on the upstream develop branch."
          webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
  acceptance_test:
    needs: [check_test_suites_to_run, build]
    runs-on: ubuntu-22.04
    if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.count > 0 }}
    strategy:
      # Fast fail only if it's a merge queue.
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).acceptance.suites }}
    name: Acceptance (${{ matrix.suite.name }})
    steps:
      - name: Check for broken suites
        id: check_skip
        # TODO(#22448): Remove once all suites are fixed.
        run: |
          broken_suites=("exploration-editor/manage-exploration-misconceptions" "exploration-editor/modify-translations-through-modal" "exploration-editor/publish-the-exploration-with-an-interaction" "practice-question-submitter/submit-practice-questions-with-different-interactions-and-difficulties")
          for suite in "${broken_suites[@]}"; do
            if [[ "${{ matrix.suite.name }}" == "$suite" ]]; then
              echo "Suite '${{ matrix.suite.name }}' is marked to be skipped."
              echo "SKIP_SUITE=true" >> $GITHUB_OUTPUT
              break
            fi
          done

      # Checkout repository, merge develop, and generate build files.
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Generate build files
        uses: ./.github/actions/generate-build-files

      - name: Generate modified suite name for artifacts
        id: generate_suite_name_for_artifacts
        # Replace slashes in the filename with underscores.
        run: |
          SUITE_NAME=${{ matrix.suite.name }}
          echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT

      # Desktop Test
      - name: Run Desktop Acceptance Test ${{ matrix.suite.name }}
        id: desktop-test
        # TODO(#22448): Remove once all suites are fixed.
        if: ${{ steps.check_skip.outputs.SKIP_SUITE != 'true' }}
        timeout-minutes: 30
        uses: ./.github/actions/run-a-specific-acceptance-test
        with:
          test-suite: ${{ matrix.suite.name }}
          viewport-type: 'desktop'
          enable-screen-recording: true
          modified-suite-name-for-uploads: ${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}_desktop_original

      - name: Re-run Desktop Acceptance Test to check flakiness for ${{ matrix.suite.name }}
        # We need to pass always() because if not, GitHub will implicitly use success(),
        # which requires all previous jobs/steps to have succeeded. This is not what we want.
        # Reference: https://docs.github.com/en/actions/reference/workflows-and-actions/expressions#status-check-functions
        # Quote: “A default status check of success() is applied unless you *include* one of these functions.”
        if: ${{ always() && steps.desktop-test.outcome == 'failure' }}
        timeout-minutes: 30
        uses: ./.github/actions/run-a-specific-acceptance-test
        with:
          test-suite: ${{ matrix.suite.name }}
          viewport-type: 'desktop'
          enable-screen-recording: false

      # Mobile Test
      - name: Run Mobile Acceptance Test ${{ matrix.suite.name }}
        id: mobile-test
        # We are using always() to prevent the job step from being skipped if desktop test fails.
        # TODO(#22448): Remove steps.check_skip.outputs.SKIP_SUITE != 'true' check,
        # once all suites are fixed. Please don't remove the always() check.
        if: ${{ steps.check_skip.outputs.SKIP_SUITE != 'true' && always() }}
        timeout-minutes: 30
        # TODO(#23106): Currently, video recording failure interfares with
        # acceptance tests, making acceptance test suite fail even if sub-tests
        # passes. Once fixed, enable video recording by updating
        # VIDEO_RECORDING_IS_ENABLED=0 to VIDEO_RECORDING_IS_ENABLED=1 in run block.
        uses: ./.github/actions/run-a-specific-acceptance-test
        with:
          test-suite: ${{ matrix.suite.name }}
          viewport-type: 'mobile'
          enable-screen-recording: true
          modified-suite-name-for-uploads: ${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}_mobile_original

      - name: Re-run Mobile Acceptance Test to check flakiness for ${{ matrix.suite.name }}
        # We need to pass always() because if not, GitHub will implicitly use success(),
        # which requires all previous jobs/steps to have succeeded. This is not what we want.
        # Reference: https://docs.github.com/en/actions/reference/workflows-and-actions/expressions#status-check-functions
        # Quote: “A default status check of success() is applied unless you *include* one of these functions.”
        if: ${{ always() && steps.mobile-test.outcome == 'failure' }}
        timeout-minutes: 30
        uses: ./.github/actions/run-a-specific-acceptance-test
        with:
          test-suite: ${{ matrix.suite.name }}
          viewport-type: 'mobile'
          enable-screen-recording: false

      # Upload other essential files.
      - name: Upload generated test to angular module mapping
        if: ${{ failure() && (steps.desktop-test-attempt-1.outcome == 'failure' || steps.mobile-test-attempt-1.outcome == 'failure') }}
        uses: actions/upload-artifact@v4
        with:
          name: generated-test-to-angular-module-mapping-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/oppia/core/tests/test-modules-mappings/acceptance/${{ steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME }}.txt

      - name: Upload webpack bundles
        if: ${{ failure() && (steps.desktop-test-attempt-1.outcome == 'failure' || steps.mobile-test-attempt-1.outcome == 'failure') }}
        uses: actions/upload-artifact@v4
        with:
          name: webpack-bundles-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/oppia/build

      - name: Report failure if failed on oppia/oppia develop branch
        if: ${{
          failure() &&
          github.event_name == 'push' &&
          github.repository == 'oppia/oppia' &&
          github.ref == 'refs/heads/develop'
          }}
        uses: ./.github/actions/send-webhook-notification
        with:
          message: "An acceptance test failed on the upstream develop branch."
          webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}

  lighthouse_accessibility_test:
    needs: [check_test_suites_to_run]
    if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_accessibility.count > 0 }}
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_accessibility.suites }}
    name: Lighthouse a11y (shard ${{ matrix.suite.name }})
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Install Chrome
        if: startsWith(github.head_ref, 'update-changelog-for-release') == false
        uses: ./.github/actions/install-chrome
      - name: Run Lighthouse accessibility checks shard ${{ matrix.suite.name }}
        if: startsWith(github.head_ref, 'update-changelog-for-release') == false
        run: |
          python -m scripts.run_lighthouse_tests --mode accessibility --pages ${{ join(matrix.suite.pages_to_run, ',') }}
        shell: bash
        timeout-minutes: 45
      - name: Report failure if failed on oppia/oppia develop branch
        if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
        uses: ./.github/actions/send-webhook-notification
        with:
          message: "A Lighthouse test failed on the upstream develop branch."
          webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
  lighthouse_performance_test:
    needs: [check_test_suites_to_run, build]
    runs-on: ubuntu-22.04
    if: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.count > 0 }}
    strategy:
      matrix:
        suite: ${{ fromJSON(needs.check_test_suites_to_run.outputs.TEST_SUITES_TO_RUN).lighthouse_performance.suites }}
    name: Lighthouse perf (shard ${{ matrix.suite.name }})
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Generate build files
        uses: ./.github/actions/generate-build-files
      - name: Install Chrome
        if: startsWith(github.head_ref, 'update-changelog-for-release') == false
        uses: ./.github/actions/install-chrome
      - name: Generate modified suite name for artifacts
        id: generate_suite_name_for_artifacts
        # Replace slashes in the filename with underscores.
        run: |
          SUITE_NAME=${{ matrix.suite.name }}
          echo "MODIFIED_SUITE_NAME=${SUITE_NAME//\//_}" >> $GITHUB_OUTPUT
      - name: Run Lighthouse performance checks (shard ${{ matrix.suite.name }})
        if: startsWith(github.head_ref, 'update-changelog-for-release') == false
        run: python -m scripts.run_lighthouse_tests --mode performance --skip_build --record_screen --pages ${{ join(matrix.suite.pages_to_run, ',') }}
        timeout-minutes: 45
      - name: Uploading puppeteer video as artifact
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: lhci-puppeteer-video-${{steps.generate_suite_name_for_artifacts.outputs.MODIFIED_SUITE_NAME}}
          path: /home/runner/work/oppia/lhci-puppeteer-video/video.mp4
      - name: Report failure if failed on oppia/oppia develop branch
        if: ${{ failure() && github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
        uses: ./.github/actions/send-webhook-notification
        with:
          message: "A Lighthouse performance test failed on the upstream develop branch."
          webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
  check_workflow_statuses:
    # This job is needed because we cannot make each e2e/acceptance/lighthouse
    # task a "required" check via GitHub, since different tasks may execute
    # depending on which files are modified in the PR.
    name: Check that all necessary tests pass
    needs:
      - check_test_suites_to_run
      - e2e_test
      - acceptance_test
      - lighthouse_accessibility_test
      - lighthouse_performance_test
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout repository so that local actions can be used
        uses: actions/checkout@v4
      - name: Merge develop and set up dependencies
        uses: ./.github/actions/merge-develop-and-set-up-dependencies
      - name: Check workflow status
        uses: ./.github/actions/check-workflow-status
        id: check_workflow_status
        with:
          jobs: ${{ toJson(needs) }}
      - name: Fail if workflow status is failure
        if: ${{ steps.check_workflow_status.outputs.WORKFLOW_STATUS == 'failure' }}
        run: |
          echo ""
          echo "NOTE TO DEVELOPERS: This CI check is failing because another e2e/acceptance/lighthouse full-stack test workflow has failed. Please see that other workflow for the relevant logs to investigate. When all full-stack workflows pass, this CI check will automatically pass."
          echo ""
          echo "If all other full-stack workflows have passed and this is the only failure, please file an issue on our issue tracker and link to this log. Thanks!"
          echo ""
          exit 1
        shell: bash
    permissions:
      pull-requests: write
  trigger_build_and_deploy:
    name: Trigger build and deploy
    needs: [check_workflow_statuses]
    runs-on: ubuntu-22.04
    # Only broadcast this for commits to develop branch in the oppia/oppia repo.
    # Further checks to ensure that the relevant commit can be deployed (e.g.
    # the commit is not an older commit than the latest deployed commit) are
    # done in the build_release_candidate script.
    if: ${{ github.event_name == 'push' && github.repository == 'oppia/oppia' && github.ref == 'refs/heads/develop'}}
    steps:
      - name: Set environment variables
        id: set_env
        run: |
          echo "VERSION=auto-$(echo $GITHUB_SHA | cut -c1-7)" >> $GITHUB_ENV
        shell: bash
      - name: Generate token
        id: generate_token
        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92
        with:
          app_id: ${{ secrets.NOTIFICATIONS_TOKEN_GENERATOR_GH_APP_ID }}
          private_key: ${{ secrets.NOTIFICATIONS_TOKEN_GENERATOR_GH_APP_PRIVATE_KEY }}
      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@87c5425cae5ba8b5bc7da27674076c78588babf3
        with:
          token: ${{ steps.generate_token.outputs.token }}
          repository: oppia/release-scripts
          event-type: develop-commit
          client-payload: '{
            "version": "${{ env.VERSION }}",
            "app_name": "oppiatestserver",
            "maintenance_mode": false,
            "sha": "${{ github.sha }}",
            "repo_owner": "${{ github.repository_owner }}"
          }'