stage-build.yml

name: Stage Build

# NOTE! This is the *STAGE* workflow.
# Keep in mind that much of the configuration is repeated in `prod-build.yml`
# and `dev-build.yml`
#
# For a complete picture of all environments, see:
#
#  https://docs.google.com/spreadsheets/d/1VnnEl-iTtKYmlyN02FiEXygxZCgE4o_ZO8wSleebne4/edit?usp=sharing
#

on:
  schedule:
    - cron: "0 0 * * *"

  workflow_dispatch:
    inputs:
      notes:
        description: "Notes"
        required: false
        default: ""

      invalidate:
        description: "Invalidate CDN (use only in exceptional circumstances)"
        type: boolean
        required: false
        default: false

  workflow_call:
    secrets:
      GCP_PROJECT_NAME:
        required: true
      WIP_PROJECT_ID:
        required: true

permissions:
  contents: read
  id-token: write

jobs:
  trigger:
    runs-on: ubuntu-latest

    # When run from `main` branch (schedule or manual), trigger workflow on `next` branch instead.
    if: ${{ github.repository == 'mdn/dex' && github.ref_name == 'main'  }}
    steps:
      - run: gh workflow run "${{ github.workflow }}" --repo "${{ github.repository }}" --ref "next"
        env:
          GH_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }}

  build:
    environment: stage
    runs-on: ubuntu-latest

    # We only ever want to deploy the `next` branch to stage.
    if: ${{ github.repository == 'mdn/dex' && github.ref_name == 'next' }}

    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
          path: mdn/dex
          persist-credentials: false

      - name: Merge main
        working-directory: mdn/dex
        run: |
          git config --global user.email "108879845+mdn-bot@users.noreply.github.com"
          git config --global user.name "mdn-bot"
          git status
          git pull
          git checkout main
          git status
          git checkout -
          git merge main --no-edit || git merge --abort

      - name: Checkout (content)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        with:
          repository: mdn/content
          path: mdn/content
          # Yes, this means fetch EVERY COMMIT EVER.
          # It's probably not sustainable in the far future (e.g. past 2021)
          # but for now it's good enough. We'll need all the history
          # so we can figure out each document's last-modified date.
          fetch-depth: 0
          persist-credentials: false

      - name: Checkout (blog)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD }}
        with:
          repository: mdn/blog
          path: mdn/blog
          lfs: true
          token: ${{ secrets.MDN_STUDIO_PAT }}
          persist-credentials: false

      - name: Checkout (generic-content)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD }}
        with:
          repository: mdn/generic-content
          path: mdn/generic-content
          persist-credentials: false

      - name: Checkout (curriculum)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD }}
        with:
          repository: mdn/curriculum
          path: mdn/curriculum
          persist-credentials: false

      - name: Checkout (translated-content)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        with:
          repository: mdn/translated-content
          path: mdn/translated-content
          # See matching warning for mdn/content checkout step
          fetch-depth: 0
          persist-credentials: false

      - name: Checkout (translated-content-de)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        with:
          repository: mdn/translated-content-de
          path: mdn/translated-content-de
          persist-credentials: false

      - name: Move de into translated-content
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        run: |
          mv mdn/translated-content-de/files/de mdn/translated-content/files/
          rm -rf mdn/translated-content-de

      - name: Clean and commit de
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        working-directory: mdn/translated-content
        run: |
          git add files/de
          git -c user.name='MDN' -c user.email='mdn-dev@mozilla.com' commit -m 'de'

      - name: Checkout (mdn-contributor-spotlight)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD }}
        with:
          repository: mdn/mdn-contributor-spotlight
          path: mdn/mdn-contributor-spotlight
          persist-credentials: false

      - name: Checkout (fred)
        uses: actions/checkout@v5
        if: ${{ ! vars.SKIP_BUILD }}
        with:
          repository: mdn/fred
          path: mdn/fred
          ref: fix-ga
          persist-credentials: false

      - name: Setup Node.js environment
        if: ${{ ! vars.SKIP_BUILD || ! vars.SKIP_FUNCTION }}
        uses: actions/setup-node@v6
        with:
          node-version-file: mdn/fred/.nvmrc
          package-manager-cache: false

      - name: Install all fred packages
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/fred
        env:
          # Use a GITHUB_TOKEN to bypass rate limiting for rari.
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: npm ci

      - name: Install Python
        if: ${{ ! vars.SKIP_BUILD }}
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Python poetry
        if: ${{ ! vars.SKIP_BUILD }}
        uses: snok/install-poetry@v1

      - name: Install deployer
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/dex/deployer
        run: poetry install

      - name: Display Python & Poetry version
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/dex/deployer
        run: |
          python --version
          poetry --version

      - name: Print information about build
        env:
          NOTES: ${{ github.event.inputs.notes }}
        run: |
          echo "notes: $NOTES"

      - name: Print information about CPU
        run: cat /proc/cpuinfo

      - name: Build everything
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/fred
        env:
          # Remember, the mdn/content repo got cloned into `pwd` into a
          # sub-folder called "mdn/content"
          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
          CONTRIBUTOR_SPOTLIGHT_ROOT: ${{ github.workspace }}/mdn/mdn-contributor-spotlight/contributors
          BLOG_ROOT: ${{ github.workspace }}/mdn/blog/content/posts
          CURRICULUM_ROOT: ${{ github.workspace }}/mdn/curriculum
          GENERIC_CONTENT_ROOT: ${{ github.workspace }}/mdn/generic-content/files
          DEX_ROOT: ${{ github.workspace }}/mdn/dex
          BASE_URL: "https://developer.allizom.org"

          # rari
          BUILD_OUT_ROOT: "out"
          LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev
          INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net
          ADDITIONAL_LOCALES_FOR_GENERICS_AND_SPAS: de

          FRED_ROBOTS_GLOBAL_ALLOW: false
          FRED_BCD_BASE_URL: https://bcd.developer.allizom.org

          # The default for this environment variable is geared for writers
          # (aka. local development). Usually defaults are supposed to be for
          # secure production but this is an exception and default
          # is not insecure.
          BUILD_LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev
          BUILD_LEGACY_LIVE_SAMPLES_BASE_URL: https://live.mdnyalp.dev

          # Sign key for code samples
          BUILD_SAMPLE_SIGN_KEY: ${{ secrets.SAMPLE_SIGN_KEY }}

          # This is the Google Analytics measurement ID for:
          # - developer.allizom.org (GA4)
          # Using measurement ids on other domains is okay, as GA will filter these events.
          BUILD_GOOGLE_ANALYTICS_MEASUREMENT_ID: G-ZG5HNVZRY0
          FRED_GA_ENABLED: true
          FRED_GA_MEASUREMENT_ID: G-ZG5HNVZRY0

          # This enables the Plus call-to-action banner and the Plus landing page
          REACT_APP_ENABLE_PLUS: true

          # This adds the ability to sign in (stage only for now)
          REACT_APP_DISABLE_AUTH: false

          # Use the stage version of interactive examples in react app
          REACT_APP_INTERACTIVE_EXAMPLES_BASE_URL: https://interactive-examples.mdn.allizom.net

          # Offline updates
          REACT_APP_UPDATES_BASE_URL: https://updates.developer.allizom.org

          # Firefox Accounts and SubPlat settings
          REACT_APP_FXA_SIGNIN_URL: /users/fxa/login/authenticate/
          REACT_APP_FXA_SETTINGS_URL: https://accounts.stage.mozaws.net/settings/
          REACT_APP_MDN_PLUS_SUBSCRIBE_URL: https://accounts.stage.mozaws.net/subscriptions/products/prod_Jtbg9tyGyLRuB0
          REACT_APP_MDN_PLUS_5M_PLAN: price_1JFoTYKb9q6OnNsLalexa03p
          REACT_APP_MDN_PLUS_5Y_PLAN: price_1JpIPwKb9q6OnNsLJLsIqMp7
          REACT_APP_MDN_PLUS_10M_PLAN: price_1K6X7gKb9q6OnNsLi44HdLcC
          REACT_APP_MDN_PLUS_10Y_PLAN: price_1K6X8VKb9q6OnNsLFlUcEiu4

          # Support for SP3
          REACT_APP_MDN_PLUS_SUBSCRIBE_URL_SP3_BASE: https://payments-next.allizom.org
          REACT_APP_MDN_PLUS_5M_SP3_ID: mdnplus5mstage
          REACT_APP_MDN_PLUS_5Y_SP3_ID: mdnplus5ystage
          REACT_APP_MDN_PLUS_10M_SP3_ID: mdnsupporter10mstage
          REACT_APP_MDN_PLUS_10Y_SP3_ID: mdnsupporter10ystage

          # Surveys.
          REACT_APP_SURVEY_START_JS_PROPOSALS_2025: 0 # stage
          REACT_APP_SURVEY_END_JS_PROPOSALS_2025: 1745971200000 # new Date("2025-04-30Z").getTime()
          REACT_APP_SURVEY_RATE_FROM_JS_PROPOSALS_2025: 0.0
          REACT_APP_SURVEY_RATE_TILL_JS_PROPOSALS_20255: 0.05 # 5%

          # Telemetry.
          REACT_APP_GLEAN_CHANNEL: stage
          REACT_APP_GLEAN_ENABLED: true
          FRED_GLEAN_CHANNEL: stage
          FRED_GLEAN_ENABLED: true

          # Newsletter
          REACT_APP_NEWSLETTER_ENABLED: true

          # Placement
          REACT_APP_PLACEMENT_ENABLED: true

          # Playground
          REACT_APP_PLAYGROUND_BASE_HOST: mdnyalp.dev
          FRED_PLAYGROUND_BASE_HOST: mdnyalp.dev

          # Observatory
          REACT_APP_OBSERVATORY_API_URL: https://observatory-api.mdn.allizom.net
          FRED_OBSERVATORY_API_URL: https://observatory-api.mdn.allizom.net

          # Sentry.
          SENTRY_DSN_BUILD: ${{ secrets.SENTRY_DSN_BUILD }}
          SENTRY_ENVIRONMENT: stage
          SENTRY_RELEASE: ${{ github.sha }}

          # Increase GitHub API rate limit.
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eo pipefail

          # Info about which CONTENT_* environment variables were set and to what.
          echo "CONTENT_ROOT=$CONTENT_ROOT"
          echo "CONTENT_TRANSLATED_ROOT=$CONTENT_TRANSLATED_ROOT"
          echo "BLOG_ROOT=$BLOG_ROOT"

          npm run rari content sync-translated-content
          npm run rari git-history

          npm run rari build -- --all --issues "$BUILD_OUT_ROOT/issues.json" --templ-stats

          # SSR all pages
          npm run build
          node build/ssr.js

          cp -r "$DEX_ROOT/client/public/assets" "$BUILD_OUT_ROOT"
          cp "$DEX_ROOT/assets/nonprod/robots.txt" "$BUILD_OUT_ROOT/robots.txt"

      - name: Install all dex dependencies
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/dex
        run: npm ci
        env:
          # Use a GITHUB_TOKEN to bypass rate limiting for rari.
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Run dex scripts
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/dex
        env:
          FRED_ROOT: ${{ github.workspace }}/mdn/fred
        run: |
          set -eo pipefail

          # Generate whatsdeployed files.
          npm run tool:legacy -- whatsdeployed --output "$FRED_ROOT/out/_whatsdeployed/code.json"
          npm run tool:legacy -- whatsdeployed $CONTENT_ROOT --output "$FRED_ROOT/out/_whatsdeployed/content.json"
          npm run tool:legacy -- whatsdeployed $CONTENT_TRANSLATED_ROOT --output "$FRED_ROOT/out/_whatsdeployed/translated-content.json"

          # Sort DE search index by en-US popularity.
          node scripts/reorder-search-index.mjs "$FRED_ROOT/out/en-us/search-index.json" "$FRED_ROOT/out/de/search-index.json"

      - name: Update search index
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/dex/deployer
        env:
          DEPLOYER_ELASTICSEARCH_URL: ${{ secrets.DEPLOYER_STAGE_ELASTICSEARCH_URL }}
          FRED_ROOT: ${{ github.workspace }}/mdn/fred
        run: poetry run deployer search-index "$FRED_ROOT/out"

      - name: Authenticate with GCP
        if: ${{ ! vars.SKIP_BUILD }}
        uses: google-github-actions/auth@v3
        with:
          token_format: access_token
          service_account: deploy-stage-content@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions

      - name: Setup gcloud
        if: ${{ ! vars.SKIP_BUILD }}
        uses: google-github-actions/setup-gcloud@v3

      - name: Sync build
        if: ${{ ! vars.SKIP_BUILD }}
        working-directory: mdn/fred
        run: |-
          time gsutil -q -m -h "Cache-Control: public, max-age=3600" cp -r out/static gs://${{ vars.GCP_BUCKET_NAME }}/main/
          time gsutil -q -m -h "Cache-Control: public, max-age=3600" rsync -cr gs://${{ vars.GCP_BUCKET_NAME }}/main/static/ gs://${{ vars.GCP_BUCKET_NAME }}/fred/static/
          time gsutil -q -m -h "Cache-Control: public, max-age=3600" rsync -cdrj html,json,txt -y "^static/" out gs://${{ vars.GCP_BUCKET_NAME }}/fred

      - name: Authenticate with GCP
        if: ${{ ! vars.SKIP_FUNCTION }}
        uses: google-github-actions/auth@v3
        with:
          token_format: access_token
          service_account: deploy-stage-nonprod-mdn-ingre@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com
          workload_identity_provider: projects/${{ secrets.WIP_PROJECT_ID }}/locations/global/workloadIdentityPools/github-actions/providers/github-actions

      - name: Setup gcloud
        if: ${{ ! vars.SKIP_FUNCTION }}
        uses: google-github-actions/setup-gcloud@v3
        with:
          install_components: "beta"

      - name: Generate redirects map
        if: ${{ ! vars.SKIP_FUNCTION }}
        working-directory: mdn/dex/cloud-function
        env:
          CONTENT_ROOT: ${{ github.workspace }}/mdn/content/files
          CONTENT_TRANSLATED_ROOT: ${{ github.workspace }}/mdn/translated-content/files
          FRED_ROOT: ${{ github.workspace }}/mdn/fred
        run: |
          mkdir -p ../client/build/
          cp "$FRED_ROOT/out/sitemap.txt" ../client/build/
          npm ci
          npm run build-redirects
          npm run build-canonicals

      - name: Deploy Function
        if: ${{ ! vars.SKIP_FUNCTION }}
        working-directory: mdn/dex
        run: |-
          set -eo pipefail

          for region in europe-west1 us-west1 asia-east1; do
            gcloud beta functions deploy mdn-nonprod-stage-$region \
            --gen2 \
            --runtime=nodejs22 \
            --region=$region \
            --source=cloud-function \
            --trigger-http \
            --allow-unauthenticated \
            --entry-point=mdnHandler \
            --concurrency=100 \
            --min-instances=1 \
            --max-instances=100 \
            --memory=2GB \
            --timeout=120s \
            --run-service-account=run-nonprod-stage-functions@${{ secrets.GCP_PROJECT_NAME }}.iam.gserviceaccount.com \
            --set-env-vars="IGNORED_ROUTES=" \
            --set-env-vars="ORIGIN_MAIN=developer.allizom.org" \
            --set-env-vars="ORIGIN_LIVE_SAMPLES=live.mdnyalp.dev" \
            --set-env-vars="ORIGIN_PLAY=mdnyalp.dev" \
            --set-env-vars="SOURCE_CONTENT=https://storage.googleapis.com/${{ vars.GCP_BUCKET_NAME }}/fred/" \
            --set-env-vars="SOURCE_API=https://api.developer.allizom.org/" \
            --set-env-vars="ORIGIN_TRIAL_TOKEN=AwIRl96ZjeFFEsdas+GXmpHvd3ARcQPYgBIGhQzZtyG9PpESvUi8ea4gpiQq9QsYRsVAWBeeBgBYqp/oLQPna0YAAABfeyJvcmlnaW4iOiJodHRwczovL2RldmVsb3Blci5hbGxpem9tLm9yZyIsImZlYXR1cmUiOiJQcml2YXRlQXR0cmlidXRpb25WMiIsImV4cGlyeSI6MTc0MjA3OTYwMH0=" \
            --set-env-vars="BSA_ENABLED=true" \
            --set-env-vars="SENTRY_DSN=${{ secrets.SENTRY_DSN_CLOUD_FUNCTION }}" \
            --set-env-vars="SENTRY_ENVIRONMENT=stage" \
            --set-env-vars="SENTRY_TRACES_SAMPLE_RATE=${{ vars.SENTRY_TRACES_SAMPLE_RATE }}" \
            --set-env-vars="SENTRY_RELEASE=${{ github.sha }}" \
            --set-secrets="KEVEL_SITE_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-kevel-site-id/versions/latest" \
            --set-secrets="KEVEL_NETWORK_ID=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-kevel-network-id/versions/latest" \
            --set-secrets="SIGN_SECRET=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-sign-secret/versions/latest" \
            --set-secrets="BSA_ZONE_KEYS=projects/${{ secrets.GCP_PROJECT_NAME }}/secrets/stage-bsa-zone-keys/versions/latest" \
            2>&1 | sed "s/^/[$region] /" &
            pids+=($!)
          done

          for pid in "${pids[@]}"; do
            wait $pid
          done

      - name: Update AI Help index with macros
        working-directory: mdn/dex
        run: npm run ai-help-macros -- update-index "$FRED_ROOT/out/en-us/docs"
        env:
          OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
          PG_URI: ${{ secrets.PG_URI }}
          FRED_ROOT: ${{ github.workspace }}/mdn/fred

      - name: Slack Notification
        if: failure()
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_CHANNEL: mdn-notifications
          SLACK_COLOR: ${{ job.status }}
          SLACK_ICON: https://avatars.slack-edge.com/2020-11-17/1513880588420_fedd7f0e9456888e69ff_96.png
          SLACK_TITLE: "Stage"
          SLACK_MESSAGE: "Build failed :collision:"
          SLACK_FOOTER: "Powered by stage-build.yml"
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

      - name: Invalidate CDN
        if: ${{ github.event.inputs.invalidate }}
        run: gcloud compute url-maps invalidate-cdn-cache ${{ secrets.GCP_LOAD_BALANCER_NAME }} --path "/*" --async