Flashsales Perf Test #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Flashsales Perf Test | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| flashsale_source_ref: | |
| description: Optional flashsale ref to record with perf results. | |
| required: false | |
| type: string | |
| default: '' | |
| flashsale_source_sha: | |
| description: Optional flashsale sha to record with perf results. | |
| required: false | |
| type: string | |
| default: '' | |
| flashsale_image_tag: | |
| description: Optional deployed flashsale image tag to record with perf results. | |
| required: false | |
| type: string | |
| default: '' | |
| concurrency: | |
| group: flashsales-perf-test-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: write | |
| env: | |
| NAMESPACE: ${{ vars.K8S_NAMESPACE || 'flashsales' }} | |
| PERF_RESULTS_ROOT: ${{ github.workspace }}/flashsale-perf-results | |
| TS_CI_TAGS: ${{ vars.TS_CI_TAGS }} | |
| TS_OAUTH_CLIENT_ID: ${{ secrets.TS_OAUTH_CLIENT_ID }} | |
| TS_OAUTH_SECRET: ${{ secrets.TS_OAUTH_SECRET }} | |
| FLASHSALE_RESULTS_PR_TOKEN: ${{ secrets.HOMELAB_CLOUD_REPO_DISPATCH_TOKEN }} | |
| jobs: | |
| contract: | |
| name: Resolve Flashsale Quality Contract | |
| runs-on: ubuntu-latest | |
| outputs: | |
| perf_cadence_json: ${{ steps.export.outputs.perf_cadence_json }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install contract parser dependencies | |
| run: python -m pip install --upgrade pip pyyaml jsonschema | |
| - name: Export flashsale quality contract | |
| id: export | |
| run: | | |
| python3 .github/scripts/export_flashsale_quality_contract.py \ | |
| application/flashsale/release/flashsale-quality-contract.yaml \ | |
| "$GITHUB_OUTPUT" | |
| prepare: | |
| name: Prepare Perf Environment | |
| runs-on: ${{ fromJson(vars.K8S_RUNNER_LABELS_JSON || '["ubuntu-latest"]') }} | |
| needs: | |
| - contract | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Setup Node.js 24 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24' | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v4 | |
| - name: Install k6 | |
| uses: grafana/setup-k6-action@v1 | |
| - name: Configure private k3s access | |
| uses: ./.github/actions/setup-private-k3s-access | |
| with: | |
| kube_config_data: ${{ secrets.KUBE_CONFIG_DATA }} | |
| k3s_server_url_tailscale: ${{ vars.K3S_SERVER_URL_TAILSCALE }} | |
| ts_oauth_client_id: ${{ env.TS_OAUTH_CLIENT_ID }} | |
| ts_oauth_secret: ${{ env.TS_OAUTH_SECRET }} | |
| ts_ci_tags: ${{ env.TS_CI_TAGS }} | |
| verify_sleep_seconds: "5" | |
| - name: Verify target services | |
| run: | | |
| set -euo pipefail | |
| kubectl -n "$NAMESPACE" get svc flashsales-user-service | |
| kubectl -n "$NAMESPACE" get svc flashsales-product-service | |
| kubectl -n "$NAMESPACE" get svc flashsales-order-service | |
| if kubectl -n "$NAMESPACE" get externalsecret flashsales-postgres-auth >/dev/null 2>&1; then | |
| kubectl -n "$NAMESPACE" annotate externalsecret flashsales-postgres-auth \ | |
| force-sync="$(date +%s)" \ | |
| --overwrite | |
| fi | |
| for attempt in $(seq 1 30); do | |
| protocol="$( | |
| kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \ | |
| -o jsonpath='{.data.KAFKA_SECURITY_PROTOCOL}' 2>/dev/null \ | |
| | base64 --decode 2>/dev/null || true | |
| )" | |
| if [ "$protocol" = "SASL_SSL" ]; then | |
| echo "Kafka runtime secret has KAFKA_SECURITY_PROTOCOL=SASL_SSL" | |
| break | |
| fi | |
| echo "Waiting for Kafka runtime secret to sync SASL_SSL (attempt ${attempt}/30, current=${protocol:-missing})..." | |
| sleep 5 | |
| done | |
| protocol="$( | |
| kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth \ | |
| -o jsonpath='{.data.KAFKA_SECURITY_PROTOCOL}' 2>/dev/null \ | |
| | base64 --decode 2>/dev/null || true | |
| )" | |
| if [ "$protocol" != "SASL_SSL" ]; then | |
| echo "KAFKA_SECURITY_PROTOCOL must be SASL_SSL before perf; got ${protocol:-missing}" | |
| echo "ExternalSecret status:" | |
| kubectl -n "$NAMESPACE" get externalsecret flashsales-postgres-auth -o yaml || true | |
| exit 1 | |
| fi | |
| kubectl -n "$NAMESPACE" rollout restart deploy/flashsales-order-service | |
| kubectl -n "$NAMESPACE" rollout restart deploy/flashsales-order-worker | |
| kubectl -n "$NAMESPACE" rollout status deploy/flashsales-user-service --timeout=120s | |
| kubectl -n "$NAMESPACE" rollout status deploy/flashsales-product-service --timeout=120s | |
| kubectl -n "$NAMESPACE" rollout status deploy/flashsales-order-service --timeout=120s | |
| kubectl -n "$NAMESPACE" rollout status deploy/flashsales-order-worker --timeout=120s | |
| secret_json="$(mktemp)" | |
| kubectl -n "$NAMESPACE" get secret flashsales-postgres-auth -o json > "$secret_json" | |
| SECRET_JSON="$secret_json" python3 - <<'PY' | |
| import base64 | |
| import json | |
| import os | |
| with open(os.environ["SECRET_JSON"], encoding="utf-8") as handle: | |
| secret = json.load(handle) | |
| data = secret.get("data", {}) | |
| def value(name: str) -> str: | |
| encoded = data.get(name) | |
| if not encoded: | |
| raise SystemExit(f"Missing required Kafka secret key: {name}") | |
| return base64.b64decode(encoded).decode("utf-8", errors="replace") | |
| protocol = value("KAFKA_SECURITY_PROTOCOL") | |
| if protocol != "SASL_SSL": | |
| raise SystemExit( | |
| f"KAFKA_SECURITY_PROTOCOL must be SASL_SSL before perf; got {protocol!r}" | |
| ) | |
| for name in ( | |
| "KAFKA_BOOTSTRAP_SERVERS", | |
| "KAFKA_SERVICE_URI", | |
| "KAFKA_USERNAME", | |
| "KAFKA_PASSWORD", | |
| "KAFKA_TERMINALIZATION_TOPIC", | |
| "KAFKA_TERMINALIZATION_RETRY_TOPIC", | |
| "KAFKA_TERMINALIZATION_DLQ_TOPIC", | |
| "KAFKA_TERMINALIZATION_CONSUMER_GROUP", | |
| ): | |
| current = value(name) | |
| if not current.strip(): | |
| raise SystemExit(f"Kafka secret key {name} is empty") | |
| if "<redacted>" in current: | |
| raise SystemExit( | |
| f"Kafka secret key {name} still contains an Aiven redacted placeholder" | |
| ) | |
| print("Kafka runtime secret preflight passed") | |
| PY | |
| perf-cadence: | |
| name: Run Flashsale Perf Cadence | |
| runs-on: ${{ fromJson(vars.K8S_RUNNER_LABELS_JSON || '["ubuntu-latest"]') }} | |
| needs: | |
| - contract | |
| - prepare | |
| if: ${{ needs.prepare.result == 'success' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Resolve perf source metadata | |
| id: source_metadata | |
| run: | | |
| set -euo pipefail | |
| source_ref="${{ github.event.inputs.flashsale_source_ref }}" | |
| source_sha="${{ github.event.inputs.flashsale_source_sha }}" | |
| image_tag="${{ github.event.inputs.flashsale_image_tag }}" | |
| if [ -z "$source_ref" ]; then | |
| source_ref="$(git -C application/flashsale rev-parse --abbrev-ref HEAD)" | |
| if [ "$source_ref" = "HEAD" ]; then | |
| source_ref="${{ github.ref_name }}" | |
| fi | |
| fi | |
| if [ -z "$source_sha" ]; then | |
| source_sha="$(git -C application/flashsale rev-parse HEAD)" | |
| fi | |
| { | |
| echo "source_ref=$source_ref" | |
| echo "source_sha=$source_sha" | |
| echo "image_tag=$image_tag" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Setup Node.js 24 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '24' | |
| - name: Install kubectl | |
| uses: azure/setup-kubectl@v4 | |
| - name: Install k6 | |
| uses: grafana/setup-k6-action@v1 | |
| - name: Configure private k3s access | |
| uses: ./.github/actions/setup-private-k3s-access | |
| with: | |
| kube_config_data: ${{ secrets.KUBE_CONFIG_DATA }} | |
| k3s_server_url_tailscale: ${{ vars.K3S_SERVER_URL_TAILSCALE }} | |
| ts_oauth_client_id: ${{ env.TS_OAUTH_CLIENT_ID }} | |
| ts_oauth_secret: ${{ env.TS_OAUTH_SECRET }} | |
| ts_ci_tags: ${{ env.TS_CI_TAGS }} | |
| verify_sleep_seconds: "5" | |
| - name: Run flashsale-owned perf cadence | |
| run: | | |
| set -euo pipefail | |
| export KUBECONFIG_PATH="$HOME/.kube/config" | |
| export USE_K8S_PORT_FORWARD="true" | |
| python3 .github/scripts/run_flashsale_perf_cadence.py \ | |
| --lanes-json '${{ needs.contract.outputs.perf_cadence_json }}' \ | |
| --results-root "$PERF_RESULTS_ROOT" | |
| - name: Upload perf results artifact | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: flashsale-perf-results-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ env.PERF_RESULTS_ROOT }} | |
| if-no-files-found: ignore | |
| - name: Materialize perf results into flashsale repo | |
| if: ${{ always() && env.FLASHSALE_RESULTS_PR_TOKEN != '' }} | |
| id: materialize_results | |
| run: | | |
| set -euo pipefail | |
| if [ ! -f "$PERF_RESULTS_ROOT/manifest.json" ]; then | |
| echo "No perf results manifest found; skipping flashsale PR creation" | |
| echo "results_relative_dir=" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| results_relative_dir="$(python3 .github/scripts/materialize_flashsale_perf_results.py \ | |
| --results-dir "$PERF_RESULTS_ROOT" \ | |
| --flashsale-dir application/flashsale \ | |
| --workflow-name "Flashsales Perf Test" \ | |
| --run-id "${{ github.run_id }}" \ | |
| --run-attempt "${{ github.run_attempt }}" \ | |
| --source-ref "${{ steps.source_metadata.outputs.source_ref }}" \ | |
| --source-sha "${{ steps.source_metadata.outputs.source_sha }}" \ | |
| --image-tag "${{ steps.source_metadata.outputs.image_tag }}")" | |
| echo "results_relative_dir=$results_relative_dir" >> "$GITHUB_OUTPUT" | |
| - name: Create flashsale perf results PR | |
| if: ${{ always() && env.FLASHSALE_RESULTS_PR_TOKEN != '' && steps.materialize_results.outputs.results_relative_dir != '' }} | |
| env: | |
| GH_TOKEN: ${{ env.FLASHSALE_RESULTS_PR_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| branch_name="perf-results/run-${{ github.run_id }}-attempt-${{ github.run_attempt }}" | |
| pr_title="perf: record run ${{ github.run_id }} attempt ${{ github.run_attempt }}" | |
| pr_body="$(mktemp)" | |
| cat > "$pr_body" <<EOF | |
| Record flashsale perf results from GitHub Actions. | |
| - Workflow: Flashsales Perf Test | |
| - Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - Attempt: ${{ github.run_attempt }} | |
| - Flashsale ref: ${{ steps.source_metadata.outputs.source_ref }} | |
| - Flashsale sha: ${{ steps.source_metadata.outputs.source_sha }} | |
| - Image tag: ${{ steps.source_metadata.outputs.image_tag }} | |
| - Results path: \`${{ steps.materialize_results.outputs.results_relative_dir }}\` | |
| EOF | |
| git -C application/flashsale config user.name "github-actions[bot]" | |
| git -C application/flashsale config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git -C application/flashsale remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/jingyi-zhao-01/flashsale.git" | |
| git -C application/flashsale checkout -B "$branch_name" | |
| git -C application/flashsale add "${{ steps.materialize_results.outputs.results_relative_dir }}" | |
| git -C application/flashsale commit -m "$pr_title" || { | |
| echo "No perf result changes to commit; skipping PR creation" | |
| exit 0 | |
| } | |
| git -C application/flashsale push --force-with-lease origin "$branch_name" | |
| gh pr create \ | |
| --repo jingyi-zhao-01/flashsale \ | |
| --base main \ | |
| --head "$branch_name" \ | |
| --title "$pr_title" \ | |
| --body-file "$pr_body" | |
| notify-discord: | |
| name: Notify Discord | |
| needs: | |
| - contract | |
| - prepare | |
| - perf-cadence | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| continue-on-error: true | |
| - name: Send Discord notification | |
| continue-on-error: true | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| DISCORD_WEBHOOK_SSM_NAME: ${{ vars.DISCORD_WEBHOOK_SSM_NAME || '/codex/discord-webhook' }} | |
| WORKFLOW_NAME: Flashsales Perf Test | |
| NEEDS_JSON: ${{ toJson(needs) }} | |
| run: bash .github/scripts/notify-discord.sh |