Synthetic E2E monitor (demo) #157
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Synthetic E2E monitor (demo) | |
| # Periodic happy-path checks through both UIs against demo, so an outage is caught here | |
| # instead of waiting for someone to report it. Runs the committed e2e/ Playwright suite: | |
| # - Advisor: login → chat → assistant responds (exercises advisor → MCP → GraphQL → QP) | |
| # - MDR: Cognito login → Export Playground → run export (exercises MDR + LDE + composite auth) | |
| # A failed run turns the workflow red (the alert). Uses the dedicated e2e accounts; passwords | |
| # are read from SSM at run time via the demo OIDC role (never stored in the repo). | |
| on: | |
| schedule: | |
| - cron: "17 */2 * * *" # every 2 hours (offset off the top of the hour) | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| actions: read # recovery step queries the previous run's conclusion | |
| env: | |
| AWS_REGION: us-east-1 | |
| GHA_ROLE: arn:aws:iam::381492161417:role/lif-github-actions-demo | |
| jobs: | |
| monitor: | |
| name: Happy-path monitor (demo) | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| working-directory: e2e | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node 20 | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| cache: npm | |
| cache-dependency-path: e2e/package-lock.json | |
| - name: Install dependencies + Chromium | |
| run: | | |
| npm ci | |
| npx playwright install --with-deps chromium | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v5 | |
| with: | |
| audience: sts.amazonaws.com | |
| role-to-assume: ${{ env.GHA_ROLE }} | |
| role-session-name: synthetic-e2e | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Load e2e credentials from SSM (masked) | |
| run: | | |
| adv=$(aws ssm get-parameter --name /demo/advisor-api/DemoUserPassword --with-decryption --query Parameter.Value --output text) | |
| mdr=$(aws ssm get-parameter --name /demo/e2e/mdr-playwright-pw --with-decryption --query Parameter.Value --output text) | |
| echo "::add-mask::$adv" | |
| echo "::add-mask::$mdr" | |
| echo "ADVISOR_PASSWORD=$adv" >> "$GITHUB_ENV" | |
| echo "MDR_PASSWORD=$mdr" >> "$GITHUB_ENV" | |
| - name: Advisor happy path | |
| env: | |
| BASE_URL: https://advisor.demo.lif.unicon.net/ | |
| E2E_USERNAME: atsatrian_lifdemo@stateu.edu | |
| E2E_PASSWORD: ${{ env.ADVISOR_PASSWORD }} | |
| run: npx playwright test --grep @chat | |
| - name: MDR happy path | |
| if: always() # run even if the advisor check failed, so we see the status of both | |
| env: | |
| BASE_URL: https://mdr.demo.lif.unicon.net/ | |
| MDR_USERNAME: lif-e2e-test@unicon.net | |
| MDR_PASSWORD: ${{ env.MDR_PASSWORD }} | |
| run: npx playwright test --grep @happy-path | |
| - name: Upload Playwright report on failure | |
| if: failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: playwright-report | |
| path: e2e/playwright-report/ | |
| retention-days: 7 | |
| # Notify the team (email + SMS) when a happy-path check fails — this is the | |
| # "alert" half of #1089. Topic + subscriptions live in repositories.yml; the | |
| # demo OIDC role already assumed above has sns:Publish on the topic. | |
| # | |
| # #1157: alert only on the SECOND consecutive failure. A single failed run is | |
| # usually a blip (a slow dependency, a cold start) and self-clears on the next | |
| # pass. Alerting on every failure meant a flapping check produced a failure mail | |
| # AND a recovery mail per cycle — during the Aug 20-22 export timeouts that was | |
| # roughly two mails per run for days, which trains people to ignore the topic. | |
| # A real sustained outage still pages, one cycle (~2h) later. | |
| - name: Alert on failure (SNS) | |
| if: failure() | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Most-recent COMPLETED run excludes this still-running one, so it's the previous run. | |
| prev=$(gh run list --repo "${GITHUB_REPOSITORY}" --workflow synthetic-e2e.yml \ | |
| --status completed --limit 1 --json conclusion --jq '.[0].conclusion // ""') | |
| echo "previous completed run conclusion: ${prev:-<none>}" | |
| if [ "$prev" != "failure" ]; then | |
| echo "First failure in a row — staying silent; will alert if the next run also fails." | |
| exit 0 | |
| fi | |
| topic=$(aws ssm get-parameter --name /demo/alerts/TopicArn --query Parameter.Value --output text) | |
| run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| aws sns publish --topic-arn "$topic" \ | |
| --subject "LIF demo synthetic monitor FAILED (2+ consecutive)" \ | |
| --message "The demo happy-path monitor has failed at least twice in a row (advisor and/or MDR/LDE). Investigate: ${run_url}" | |
| # Send an "all clear" only when we actually alerted — i.e. the two previous | |
| # completed runs both failed, which is exactly the condition that fired the | |
| # failure mail above. Keeps failure/recovery symmetric so a suppressed blip | |
| # never produces an unexplained "RECOVERED" with no preceding alert (#1103). | |
| - name: Notify on recovery (SNS) | |
| if: success() | |
| working-directory: ${{ github.workspace }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| prev2=$(gh run list --repo "${GITHUB_REPOSITORY}" --workflow synthetic-e2e.yml \ | |
| --status completed --limit 2 --json conclusion --jq '[.[].conclusion] | join(",")') | |
| echo "two previous completed run conclusions: ${prev2:-<none>}" | |
| if [ "$prev2" != "failure,failure" ]; then | |
| echo "No alert was sent for the preceding runs — staying silent." | |
| exit 0 | |
| fi | |
| topic=$(aws ssm get-parameter --name /demo/alerts/TopicArn --query Parameter.Value --output text) | |
| run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| aws sns publish --topic-arn "$topic" \ | |
| --subject "LIF demo synthetic monitor RECOVERED" \ | |
| --message "The demo happy-path monitor is passing again after a sustained failure. ${run_url}" |