flashsales-loadtest-manual.yml

name: Flashsales Perf Test

on:
  workflow_dispatch:
    inputs:
      flashsale_source_ref:
        description: Optional flashsale ref to record with perf results.
        required: false
        type: string
        default: ''
      flashsale_source_sha:
        description: Optional flashsale sha to record with perf results.
        required: false
        type: string
        default: ''
      flashsale_image_tag:
        description: Optional deployed flashsale image tag to record with perf results.
        required: false
        type: string
        default: ''

concurrency:
  group: flashsales-perf-test-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  packages: write

env:
  NAMESPACE: ${{ vars.K8S_NAMESPACE || 'flashsales' }}
  PERF_RESULTS_ROOT: ${{ github.workspace }}/flashsale-perf-results
  TS_CI_TAGS: ${{ vars.TS_CI_TAGS }}
  TS_OAUTH_CLIENT_ID: ${{ secrets.TS_OAUTH_CLIENT_ID }}
  TS_OAUTH_SECRET: ${{ secrets.TS_OAUTH_SECRET }}
  FLASHSALE_RESULTS_PR_TOKEN: ${{ secrets.HOMELAB_CLOUD_REPO_DISPATCH_TOKEN }}

jobs:
  contract:
    name: Resolve Flashsale Quality Contract
    runs-on: ubuntu-latest
    outputs:
      perf_cadence_json: ${{ steps.export.outputs.perf_cadence_json }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install contract parser dependencies
        run: python -m pip install --upgrade pip pyyaml jsonschema

      - name: Export flashsale quality contract
        id: export
        run: |
          python3 .github/scripts/export_flashsale_quality_contract.py \
            application/flashsale/release/flashsale-quality-contract.yaml \
            "$GITHUB_OUTPUT"

  prepare:
    name: Prepare Perf Environment
    runs-on: ${{ fromJson(vars.K8S_RUNNER_LABELS_JSON || '["ubuntu-latest"]') }}
    needs:
      - contract

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Setup Node.js 24
        uses: actions/setup-node@v4
        with:
          node-version: '24'

      - name: Install kubectl
        uses: azure/setup-kubectl@v4

      - name: Install k6
        uses: grafana/setup-k6-action@v1

      - name: Configure private k3s access
        uses: ./.github/actions/setup-private-k3s-access
        with:
          kube_config_data: ${{ secrets.KUBE_CONFIG_DATA }}
          k3s_server_url_tailscale: ${{ vars.K3S_SERVER_URL_TAILSCALE }}
          ts_oauth_client_id: ${{ env.TS_OAUTH_CLIENT_ID }}
          ts_oauth_secret: ${{ env.TS_OAUTH_SECRET }}
          ts_ci_tags: ${{ env.TS_CI_TAGS }}
          verify_sleep_seconds: "5"

      - name: Verify target services
        run: |
          set -euo pipefail
          kubectl -n "$NAMESPACE" get svc flashsales-user-service
          kubectl -n "$NAMESPACE" get svc flashsales-product-service
          kubectl -n "$NAMESPACE" get svc flashsales-order-service

          if kubectl -n "$NAMESPACE" get externalsecret flashsales-postgres-auth >/dev/null 2>&1; then
            kubectl -n "$NAMESPACE" annotate externalsecret flashsales-postgres-auth \
              force-sync="$(date +%s)" \
              --overwrite
          fi

          for attempt in $(seq 1 30); do
            protocol="$(
              kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \
                -o jsonpath='{.data.KAFKA_SECURITY_PROTOCOL}' 2>/dev/null \
                | base64 --decode 2>/dev/null || true
            )"
            kafka_ca_cert="$(
              kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \
                -o jsonpath='{.data.KAFKA_CA_CERT}' 2>/dev/null \
                | base64 --decode 2>/dev/null || true
            )"

            if [ "$protocol" = "SASL_SSL" ] && printf '%s' "$kafka_ca_cert" | grep -q 'BEGIN CERTIFICATE'; then
              echo "Kafka runtime secret has SASL_SSL and KAFKA_CA_CERT"
              break
            fi

            echo "Waiting for Kafka runtime secret to sync SASL_SSL + CA cert (attempt ${attempt}/30, protocol=${protocol:-missing})..."
            sleep 5
          done

          protocol="$(
            kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \
              -o jsonpath='{.data.KAFKA_SECURITY_PROTOCOL}' 2>/dev/null \
              | base64 --decode 2>/dev/null || true
          )"
          kafka_ca_cert="$(
            kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \
              -o jsonpath='{.data.KAFKA_CA_CERT}' 2>/dev/null \
              | base64 --decode 2>/dev/null || true
          )"
          if [ "$protocol" != "SASL_SSL" ] || ! printf '%s' "$kafka_ca_cert" | grep -q 'BEGIN CERTIFICATE'; then
            echo "Kafka runtime secret must contain SASL_SSL and a PEM KAFKA_CA_CERT before perf; protocol=${protocol:-missing}"
            echo "ExternalSecret status:"
            kubectl -n "$NAMESPACE" get externalsecret flashsales-postgres-auth -o yaml || true
            exit 1
          fi

          kubectl -n "$NAMESPACE" rollout restart deploy/flashsales-order-service
          kubectl -n "$NAMESPACE" rollout restart deploy/flashsales-order-worker

          kubectl -n "$NAMESPACE" rollout status deploy/flashsales-user-service --timeout=300s
          kubectl -n "$NAMESPACE" rollout status deploy/flashsales-product-service --timeout=300s
          kubectl -n "$NAMESPACE" rollout status deploy/flashsales-order-service --timeout=300s
          kubectl -n "$NAMESPACE" rollout status deploy/flashsales-order-worker --timeout=300s

          secret_json="$(mktemp)"
          kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth -o json > "$secret_json"

          SECRET_JSON="$secret_json" python3 - <<'PY'
          import base64
          import json
          import os

          with open(os.environ["SECRET_JSON"], encoding="utf-8") as handle:
              secret = json.load(handle)
          data = secret.get("data", {})

          def value(name: str) -> str:
              encoded = data.get(name)
              if not encoded:
                  raise SystemExit(f"Missing required Kafka secret key: {name}")
              return base64.b64decode(encoded).decode("utf-8", errors="replace")

          protocol = value("KAFKA_SECURITY_PROTOCOL")
          if protocol != "SASL_SSL":
              raise SystemExit(
                  f"KAFKA_SECURITY_PROTOCOL must be SASL_SSL before perf; got {protocol!r}"
              )

          for name in (
              "KAFKA_BOOTSTRAP_SERVERS",
              "KAFKA_SERVICE_URI",
              "KAFKA_USERNAME",
              "KAFKA_PASSWORD",
              "KAFKA_CA_CERT",
              "KAFKA_TERMINALIZATION_TOPIC",
              "KAFKA_TERMINALIZATION_RETRY_TOPIC",
              "KAFKA_TERMINALIZATION_DLQ_TOPIC",
              "KAFKA_TERMINALIZATION_CONSUMER_GROUP",
          ):
              current = value(name)
              if not current.strip():
                  raise SystemExit(f"Kafka secret key {name} is empty")
              if "<redacted>" in current:
                  raise SystemExit(
                      f"Kafka secret key {name} still contains an Aiven redacted placeholder"
                  )
              if name == "KAFKA_CA_CERT" and "BEGIN CERTIFICATE" not in current:
                  raise SystemExit("KAFKA_CA_CERT must contain a PEM certificate")

          print("Kafka runtime secret preflight passed")
          PY

  perf-cadence:
    name: Run Flashsale Perf Cadence
    runs-on: ${{ fromJson(vars.K8S_RUNNER_LABELS_JSON || '["ubuntu-latest"]') }}
    needs:
      - contract
      - prepare
    if: ${{ needs.prepare.result == 'success' }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Resolve perf source metadata
        id: source_metadata
        run: |
          set -euo pipefail

          source_ref="${{ github.event.inputs.flashsale_source_ref }}"
          source_sha="${{ github.event.inputs.flashsale_source_sha }}"
          image_tag="${{ github.event.inputs.flashsale_image_tag }}"

          if [ -z "$source_ref" ]; then
            source_ref="$(git -C application/flashsale rev-parse --abbrev-ref HEAD)"
            if [ "$source_ref" = "HEAD" ]; then
              source_ref="${{ github.ref_name }}"
            fi
          fi

          if [ -z "$source_sha" ]; then
            source_sha="$(git -C application/flashsale rev-parse HEAD)"
          fi

          {
            echo "source_ref=$source_ref"
            echo "source_sha=$source_sha"
            echo "image_tag=$image_tag"
          } >> "$GITHUB_OUTPUT"

      - name: Setup Node.js 24
        uses: actions/setup-node@v4
        with:
          node-version: '24'

      - name: Install kubectl
        uses: azure/setup-kubectl@v4

      - name: Install k6
        uses: grafana/setup-k6-action@v1

      - name: Configure private k3s access
        uses: ./.github/actions/setup-private-k3s-access
        with:
          kube_config_data: ${{ secrets.KUBE_CONFIG_DATA }}
          k3s_server_url_tailscale: ${{ vars.K3S_SERVER_URL_TAILSCALE }}
          ts_oauth_client_id: ${{ env.TS_OAUTH_CLIENT_ID }}
          ts_oauth_secret: ${{ env.TS_OAUTH_SECRET }}
          ts_ci_tags: ${{ env.TS_CI_TAGS }}
          verify_sleep_seconds: "5"

      - name: Run flashsale-owned perf cadence
        run: |
          set -euo pipefail
          export KUBECONFIG_PATH="$HOME/.kube/config"
          export USE_K8S_PORT_FORWARD="true"
          python3 .github/scripts/run_flashsale_perf_cadence.py \
            --lanes-json '${{ needs.contract.outputs.perf_cadence_json }}' \
            --results-root "$PERF_RESULTS_ROOT"

      - name: Upload perf results artifact
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: flashsale-perf-results-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ env.PERF_RESULTS_ROOT }}
          if-no-files-found: ignore

      - name: Materialize perf results into flashsale repo
        if: ${{ always() && env.FLASHSALE_RESULTS_PR_TOKEN != '' }}
        id: materialize_results
        run: |
          set -euo pipefail
          if [ ! -f "$PERF_RESULTS_ROOT/manifest.json" ]; then
            echo "No perf results manifest found; skipping flashsale PR creation"
            echo "results_relative_dir=" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          results_relative_dir="$(python3 .github/scripts/materialize_flashsale_perf_results.py \
            --results-dir "$PERF_RESULTS_ROOT" \
            --flashsale-dir application/flashsale \
            --workflow-name "Flashsales Perf Test" \
            --run-id "${{ github.run_id }}" \
            --run-attempt "${{ github.run_attempt }}" \
            --source-ref "${{ steps.source_metadata.outputs.source_ref }}" \
            --source-sha "${{ steps.source_metadata.outputs.source_sha }}" \
            --image-tag "${{ steps.source_metadata.outputs.image_tag }}")"
          echo "results_relative_dir=$results_relative_dir" >> "$GITHUB_OUTPUT"

      - name: Create flashsale perf results PR
        if: ${{ always() && env.FLASHSALE_RESULTS_PR_TOKEN != '' && steps.materialize_results.outputs.results_relative_dir != '' }}
        env:
          GH_TOKEN: ${{ env.FLASHSALE_RESULTS_PR_TOKEN }}
        run: |
          set -euo pipefail
          branch_name="perf-results/run-${{ github.run_id }}-attempt-${{ github.run_attempt }}"
          pr_title="perf: record run ${{ github.run_id }} attempt ${{ github.run_attempt }}"
          pr_body="$(mktemp)"

          cat > "$pr_body" <<EOF
          Record flashsale perf results from GitHub Actions.

          - Workflow: Flashsales Perf Test
          - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          - Attempt: ${{ github.run_attempt }}
          - Flashsale ref: ${{ steps.source_metadata.outputs.source_ref }}
          - Flashsale sha: ${{ steps.source_metadata.outputs.source_sha }}
          - Image tag: ${{ steps.source_metadata.outputs.image_tag }}
          - Results path: \`${{ steps.materialize_results.outputs.results_relative_dir }}\`
          EOF

          git -C application/flashsale config user.name "github-actions[bot]"
          git -C application/flashsale config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git -C application/flashsale remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/jingyi-zhao-01/flashsale.git"
          git -C application/flashsale checkout -B "$branch_name"
          git -C application/flashsale add "${{ steps.materialize_results.outputs.results_relative_dir }}"
          git -C application/flashsale commit -m "$pr_title" || {
            echo "No perf result changes to commit; skipping PR creation"
            exit 0
          }
          git -C application/flashsale push --force-with-lease origin "$branch_name"
          gh pr create \
            --repo jingyi-zhao-01/flashsale \
            --base main \
            --head "$branch_name" \
            --title "$pr_title" \
            --body-file "$pr_body"

  notify-discord:
    name: Notify Discord
    needs:
      - contract
      - prepare
      - perf-cadence
    if: ${{ always() }}
    runs-on: ubuntu-latest
    continue-on-error: true

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        continue-on-error: true

      - name: Send Discord notification
        continue-on-error: true
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ vars.AWS_REGION }}
          DISCORD_WEBHOOK_SSM_NAME: ${{ vars.DISCORD_WEBHOOK_SSM_NAME || '/codex/discord-webhook' }}
          WORKFLOW_NAME: Flashsales Perf Test
          NEEDS_JSON: ${{ toJson(needs) }}
        run: bash .github/scripts/notify-discord.sh