Kurtosis Smoke Test #112
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Kurtosis Smoke Test | |
| on: | |
| schedule: | |
| - cron: "0 6 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| charon_image_tag: | |
| description: "Charon Docker image tag to test" | |
| required: false | |
| default: "next" | |
| type: string | |
| branch: | |
| description: "kurtosis-charon branch to use" | |
| required: false | |
| default: "main" | |
| type: string | |
| lifetime_minutes: | |
| description: "Cluster lifetime in minutes (60-300)" | |
| required: false | |
| default: 60 | |
| type: number | |
| concurrency: | |
| group: kurtosis-smoke-test | |
| cancel-in-progress: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| kurtosis-smoke-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - name: Validate secrets | |
| env: | |
| KURTOSIS_AWS_IAM_ROLE: ${{ secrets.KURTOSIS_AWS_IAM_ROLE }} | |
| KURTOSIS_AWS_REGION: ${{ secrets.KURTOSIS_AWS_REGION }} | |
| KURTOSIS_MONITORING_TOKEN: ${{ secrets.KURTOSIS_MONITORING_TOKEN }} | |
| OBOL_GRAFANA_API_TOKEN: ${{ secrets.OBOL_GRAFANA_API_TOKEN }} | |
| SLACK_DEV_PROTOCOL_WEBHOOK: ${{ secrets.SLACK_DEV_PROTOCOL_WEBHOOK }} | |
| run: | | |
| missing=() | |
| [ -z "$KURTOSIS_AWS_IAM_ROLE" ] && missing+=("KURTOSIS_AWS_IAM_ROLE") | |
| [ -z "$KURTOSIS_AWS_REGION" ] && missing+=("KURTOSIS_AWS_REGION") | |
| [ -z "$KURTOSIS_MONITORING_TOKEN" ] && missing+=("KURTOSIS_MONITORING_TOKEN") | |
| [ -z "$OBOL_GRAFANA_API_TOKEN" ] && missing+=("OBOL_GRAFANA_API_TOKEN") | |
| [ -z "$SLACK_DEV_PROTOCOL_WEBHOOK" ] && missing+=("SLACK_DEV_PROTOCOL_WEBHOOK") | |
| if [ ${#missing[@]} -gt 0 ]; then | |
| echo "::error::Missing required secrets: ${missing[*]}" | |
| exit 1 | |
| fi | |
| echo "All required secrets are configured." | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6 | |
| with: | |
| role-to-assume: ${{ secrets.KURTOSIS_AWS_IAM_ROLE }} | |
| role-session-name: github-actions-aws | |
| aws-region: ${{ secrets.KURTOSIS_AWS_REGION }} | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.14" | |
| - name: Clone kurtosis-charon and install dependencies | |
| env: | |
| KURTOSIS_BRANCH: ${{ inputs.branch || 'main' }} | |
| run: | | |
| git clone --depth 1 --branch "$KURTOSIS_BRANCH" \ | |
| https://github.com/ObolNetwork/kurtosis-charon.git \ | |
| /tmp/kurtosis-charon | |
| pip install -r /tmp/kurtosis-charon/kurtosis-aws-runner/requirements.txt | |
| - name: Deploy Kurtosis clusters | |
| id: deploy | |
| env: | |
| CHARON_VERSION: "obolnetwork/charon:${{ inputs.charon_image_tag || 'next' }}" | |
| KURTOSIS_MONITORING_TOKEN: ${{ secrets.KURTOSIS_MONITORING_TOKEN }} | |
| KURTOSIS_BRANCH: ${{ inputs.branch || 'main' }} | |
| KURTOSIS_INSTANCE_TYPE: c6a.4xlarge | |
| LIFETIME_MINUTES: ${{ inputs.lifetime_minutes || '60' }} | |
| run: | | |
| echo "deploy_start=$(date +%s)" >> "$GITHUB_OUTPUT" | |
| cd /tmp/kurtosis-charon/kurtosis-aws-runner | |
| set -o pipefail | |
| echo y | python kurtosis_aws_runner.py \ | |
| --branch "$KURTOSIS_BRANCH" \ | |
| --lifetime "${LIFETIME_MINUTES}m" \ | |
| --monitoring-token "$KURTOSIS_MONITORING_TOKEN" \ | |
| --instance-type "$KURTOSIS_INSTANCE_TYPE" \ | |
| --on-demand 2>&1 | tee /tmp/deploy_output.txt | |
| CLUSTER_COUNT=$(sed -n 's/.*Found \([0-9][0-9]*\) combinations.*/\1/p' /tmp/deploy_output.txt | head -1) | |
| if [ -z "$CLUSTER_COUNT" ] || [ "$CLUSTER_COUNT" -lt 1 ] 2>/dev/null; then | |
| echo "::error::Failed to extract cluster count from deploy output" | |
| exit 1 | |
| fi | |
| echo "cluster_count=$CLUSTER_COUNT" >> "$GITHUB_OUTPUT" | |
| - name: Wait for clusters to finish | |
| env: | |
| LIFETIME_MINUTES: ${{ inputs.lifetime_minutes || '60' }} | |
| run: | | |
| WAIT_SECONDS=$(( (LIFETIME_MINUTES + 5) * 60 )) | |
| echo "Waiting $((WAIT_SECONDS / 60)) minutes (${LIFETIME_MINUTES}m lifetime + 5m buffer)..." | |
| sleep "$WAIT_SECONDS" | |
| - name: Check alerts | |
| id: alerts | |
| env: | |
| OBOL_GRAFANA_API_TOKEN: ${{ secrets.OBOL_GRAFANA_API_TOKEN }} | |
| DEPLOY_START: ${{ steps.deploy.outputs.deploy_start }} | |
| LIFETIME_MINUTES: ${{ inputs.lifetime_minutes || '60' }} | |
| CHARON_IMAGE_TAG: ${{ inputs.charon_image_tag || 'next' }} | |
| KURTOSIS_BRANCH: ${{ inputs.branch || 'main' }} | |
| EXPECTED_CLUSTERS: ${{ steps.deploy.outputs.cluster_count }} | |
| run: | | |
| WARMUP_SECONDS=$((20 * 60)) | |
| ALERT_FROM=$((DEPLOY_START + WARMUP_SECONDS)) | |
| ALERT_TO=$((DEPLOY_START + (LIFETIME_MINUTES - 5) * 60)) | |
| echo "Alert window: $(date -u -d @${ALERT_FROM} +%Y-%m-%dT%H:%M:%SZ) to $(date -u -d @${ALERT_TO} +%Y-%m-%dT%H:%M:%SZ)" | |
| set +e | |
| python scripts/debug/kurtosis_alerts.py \ | |
| --from "$ALERT_FROM" --to "$ALERT_TO" \ | |
| --lookback "$WARMUP_SECONDS" \ | |
| --expected-clusters "$EXPECTED_CLUSTERS" \ | |
| --failed-clusters-out failed_clusters.txt | tee report.txt | |
| EXIT_CODE=${PIPESTATUS[0]} | |
| set -e | |
| echo "" >> report.txt | |
| echo "=== Workflow Metadata ===" >> report.txt | |
| echo "Charon image: obolnetwork/charon:${CHARON_IMAGE_TAG}" >> report.txt | |
| echo "kurtosis-charon branch: ${KURTOSIS_BRANCH}" >> report.txt | |
| echo "Deploy time: $(date -u -d @${DEPLOY_START} +%Y-%m-%dT%H:%M:%SZ)" >> report.txt | |
| echo "Alert window: $(date -u -d @${ALERT_FROM} +%Y-%m-%dT%H:%M:%SZ) to $(date -u -d @${ALERT_TO} +%Y-%m-%dT%H:%M:%SZ)" >> report.txt | |
| echo "Alerts exit code: ${EXIT_CODE}" >> report.txt | |
| echo "alerts_exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" | |
| { | |
| echo "failed_clusters_block<<FAILED_CLUSTERS_EOF" | |
| if [ -s failed_clusters.txt ]; then | |
| echo "*Failed clusters:*" | |
| echo '```' | |
| echo "$(head -c 1500 failed_clusters.txt)" | |
| echo '```' | |
| fi | |
| echo "FAILED_CLUSTERS_EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Upload report | |
| id: upload_report | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: kurtosis-smoke-test-report | |
| path: | | |
| report.txt | |
| failed_clusters.txt | |
| if-no-files-found: ignore | |
| retention-days: 30 | |
| - name: Evaluate results | |
| env: | |
| ALERTS_EXIT_CODE: ${{ steps.alerts.outputs.alerts_exit_code }} | |
| run: | | |
| if [ "$ALERTS_EXIT_CODE" != "0" ]; then | |
| echo "::error::Smoke test failed: firing alerts detected or incomplete metrics coverage. See report artifact for details." | |
| exit 1 | |
| fi | |
| echo "Smoke test passed: no firing alerts, all clusters reporting." | |
| - name: Notify Slack on success | |
| if: success() | |
| uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3 | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_DEV_PROTOCOL_WEBHOOK }} | |
| SLACK_COLOR: good | |
| SLACK_TITLE: "Kurtosis Smoke Test Passed" | |
| SLACK_MESSAGE: | | |
| *Charon image:* `obolnetwork/charon:${{ inputs.charon_image_tag || 'next' }}` | |
| *Branch:* `${{ inputs.branch || 'main' }}` | |
| *Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run> | |
| *Report:* <${{ steps.upload_report.outputs.artifact-url }}|Download report> | |
| SLACK_USERNAME: GitHub Actions | |
| SLACK_ICON: https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png | |
| - name: Notify Slack on failure | |
| if: failure() | |
| uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3 | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_DEV_PROTOCOL_WEBHOOK }} | |
| SLACK_COLOR: danger | |
| SLACK_TITLE: "Kurtosis Smoke Test Failed" | |
| SLACK_MESSAGE: | | |
| *Charon image:* `obolnetwork/charon:${{ inputs.charon_image_tag || 'next' }}` | |
| *Branch:* `${{ inputs.branch || 'main' }}` | |
| ${{ steps.alerts.outputs.failed_clusters_block }} | |
| *Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View workflow run> | |
| *Report:* <${{ steps.upload_report.outputs.artifact-url }}|Download report> | |
| SLACK_USERNAME: GitHub Actions | |
| SLACK_ICON: https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png |