diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89ea884..8a1e4c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,20 +176,17 @@ jobs: yarn workspace batch-poster-monitor build yarn workspace retryable-monitor build - # Run each monitor and extract summary output (from summary header to end) + # Run each monitor with --enableAlerting (NODE_ENV=CI outputs slack messages without posting) echo "=== ASSERTION MONITOR ===" > ../target-output.txt - timeout 180s yarn workspace assertion-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/Assertion Monitor Alert Summary/,$p; /Monitoring complete/p' >> ../target-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace assertion-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../target-output.txt 2>&1 || true echo "" >> ../target-output.txt echo "=== BATCH POSTER MONITOR ===" >> ../target-output.txt - timeout 180s yarn workspace batch-poster-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/Batch poster monitor summary/,$p' >> ../target-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace batch-poster-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../target-output.txt 2>&1 || true echo "" >> ../target-output.txt echo "=== RETRYABLE MONITOR ===" >> ../target-output.txt - timeout 180s yarn workspace retryable-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/No retryables found/p; /retryables requiring action/,$p' >> ../target-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace retryable-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../target-output.txt 2>&1 || true continue-on-error: true - name: Run monitors on PR branch @@ -202,20 +199,17 @@ jobs: yarn workspace batch-poster-monitor build yarn workspace retryable-monitor build - # Run each monitor and extract summary output (from summary header to end) + # Run each monitor with --enableAlerting (NODE_ENV=CI outputs slack messages without posting) echo "=== ASSERTION MONITOR ===" > ../pr-output.txt - timeout 180s yarn workspace assertion-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/Assertion Monitor Alert Summary/,$p; /Monitoring complete/p' >> ../pr-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace assertion-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../pr-output.txt 2>&1 || true echo "" >> ../pr-output.txt echo "=== BATCH POSTER MONITOR ===" >> ../pr-output.txt - timeout 180s yarn workspace batch-poster-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/Batch poster monitor summary/,$p' >> ../pr-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace batch-poster-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../pr-output.txt 2>&1 || true echo "" >> ../pr-output.txt echo "=== RETRYABLE MONITOR ===" >> ../pr-output.txt - timeout 180s yarn workspace retryable-monitor dev --configPath=../../config.ci.json 2>&1 | \ - sed -n '/No retryables found/p; /retryables requiring action/,$p' >> ../pr-output.txt || true + NODE_ENV=CI timeout 300s yarn workspace retryable-monitor dev --configPath=../../config.ci.json --enableAlerting >> ../pr-output.txt 2>&1 || true continue-on-error: true - name: Show captured output @@ -226,6 +220,22 @@ jobs: echo "=== PR BRANCH OUTPUT ===" cat pr-output.txt + - name: Extract Slack messages + run: | + # Extract content between [SLACK_MESSAGE_START] and [SLACK_MESSAGE_END] markers + extract_slack_messages() { + sed -n '/\[SLACK_MESSAGE_START\]/,/\[SLACK_MESSAGE_END\]/p' | grep -v '\[SLACK_MESSAGE_' + } + + cat target-output.txt | extract_slack_messages > target-slack.txt || true + cat pr-output.txt | extract_slack_messages > pr-slack.txt || true + + echo "=== TARGET SLACK MESSAGES ===" + cat target-slack.txt + echo "" + echo "=== PR SLACK MESSAGES ===" + cat pr-slack.txt + - name: Normalize outputs and generate diff run: | # Normalize dynamic values (block numbers, addresses) for comparison @@ -233,11 +243,17 @@ jobs: sed -E 's/[0-9]{7,}/BLOCK_NUM/g; s/[0-9]+n\b/BIGINT/g; s/0x[a-fA-F0-9]{40}/ADDR/g' } + # Normalize full output cat target-output.txt | normalize > target-normalized.txt cat pr-output.txt | normalize > pr-normalized.txt - # Generate unified diff (ignore exit code since diff returns 1 when files differ) + # Normalize slack messages + cat target-slack.txt | normalize > target-slack-normalized.txt + cat pr-slack.txt | normalize > pr-slack-normalized.txt + + # Generate unified diffs (ignore exit code since diff returns 1 when files differ) diff -u target-normalized.txt pr-normalized.txt > output-diff.txt || true + diff -u target-slack-normalized.txt pr-slack-normalized.txt > slack-diff.txt || true - name: Post comment uses: actions/github-script@v7 @@ -245,15 +261,33 @@ jobs: script: | const fs = require('fs'); - const diffOutput = fs.existsSync('output-diff.txt') + const outputDiff = fs.existsSync('output-diff.txt') ? fs.readFileSync('output-diff.txt', 'utf8').trim() : ''; - const hasChanges = diffOutput.length > 0; + const slackDiff = fs.existsSync('slack-diff.txt') + ? fs.readFileSync('slack-diff.txt', 'utf8').trim() + : ''; + + const hasOutputChanges = outputDiff.length > 0; + const hasSlackChanges = slackDiff.length > 0; let body = ''; - if (hasChanges) { - body = '### ⚠️ Monitor Output Changes\n\n```diff\n' + diffOutput + '\n```'; + + if (hasOutputChanges || hasSlackChanges) { + body = '### ⚠️ Monitor Output Changes\n\n'; + + if (hasSlackChanges) { + body += '#### Slack Message Changes\n```diff\n' + slackDiff + '\n```\n\n'; + } else { + body += '#### ✅ Slack Messages: No Changes\n\n'; + } + + if (hasOutputChanges) { + body += '
\nFull Output Changes\n\n```diff\n' + outputDiff + '\n```\n
'; + } else { + body += '#### ✅ Full Output: No Changes'; + } } else { body = '### ✅ Monitor Output: No Changes'; } diff --git a/packages/assertion-monitor/alerts.ts b/packages/assertion-monitor/alerts.ts index 21906cc..d365b38 100644 --- a/packages/assertion-monitor/alerts.ts +++ b/packages/assertion-monitor/alerts.ts @@ -1,17 +1,17 @@ -export const NO_CREATION_EVENTS_ALERT = `No assertion creation events found` +export const NO_CREATION_EVENTS_ALERT = `No assertion creation events found (TEST)` -export const CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT = `Chain activity detected, but no assertions created in the last 4 hours` +export const CHAIN_ACTIVITY_WITHOUT_ASSERTIONS_ALERT = `Chain activity detected, but no assertions created in the last 4 hours (TEST)` -export const NO_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation events found` +export const NO_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation events found (TEST)` -export const CONFIRMATION_DELAY_ALERT = `Confirmation period exceeded` +export const CONFIRMATION_DELAY_ALERT = `Confirmation period exceeded (TEST)` -export const CREATION_EVENT_STUCK_ALERT = `Assertion event stuck in challenge period` +export const CREATION_EVENT_STUCK_ALERT = `Assertion event stuck in challenge period (TEST)` -export const NON_BOLD_NO_RECENT_CREATION_ALERT = `No recent node creation events detected for non-BOLD chain` +export const NON_BOLD_NO_RECENT_CREATION_ALERT = `No recent node creation events detected for non-BOLD chain (TEST)` -export const VALIDATOR_WHITELIST_DISABLED_ALERT = `Validator whitelist disabled - this may indicate security concerns for Classic chains` +export const VALIDATOR_WHITELIST_DISABLED_ALERT = `Validator whitelist disabled - this may indicate security concerns for Classic chains (TEST)` -export const BOLD_LOW_BASE_STAKE_ALERT = `BoLD chain has low base stake (below 1 ETH) which may indicate restricted validation` +export const BOLD_LOW_BASE_STAKE_ALERT = `BoLD chain has low base stake (below 1 ETH) which may indicate restricted validation (TEST)` -export const NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation blocks found but confirmation events detected` +export const NO_CONFIRMATION_BLOCKS_WITH_CONFIRMATION_EVENTS_ALERT = `No assertion confirmation blocks found but confirmation events detected (TEST)` diff --git a/packages/assertion-monitor/index.ts b/packages/assertion-monitor/index.ts index 0f0c93d..558bbf6 100644 --- a/packages/assertion-monitor/index.ts +++ b/packages/assertion-monitor/index.ts @@ -185,7 +185,7 @@ export const main = async () => { } if (alerts.length > 0) { - const alertMessage = `Assertion Monitor Alert Summary:\n\n${alerts.join( + const alertMessage = `Assertion Monitor Alert Summary (TEST CHANGE):\n\n${alerts.join( '\n\n' )}` console.log(alertMessage) diff --git a/packages/utils/createSlackPoster.ts b/packages/utils/createSlackPoster.ts index 8079ef0..3fe7e50 100644 --- a/packages/utils/createSlackPoster.ts +++ b/packages/utils/createSlackPoster.ts @@ -8,6 +8,16 @@ export const createSlackPoster = ({ channelEnvVar: string }) => { return ({ message }: { message: string }) => { + // In CI mode, output the message with markers for extraction (skip actual posting) + if (process.env.NODE_ENV === 'CI') { + if (message !== 'success') { + console.log('[SLACK_MESSAGE_START]') + console.log(message) + console.log('[SLACK_MESSAGE_END]') + } + return + } + const slackToken = process.env[tokenEnvVar] const slackChannel = process.env[channelEnvVar] @@ -15,7 +25,6 @@ export const createSlackPoster = ({ if (!slackChannel) throw new Error('Slack channel is required.') if (process.env.NODE_ENV === 'DEV') return - if (process.env.NODE_ENV === 'CI' && message === 'success') return return postSlackMessage({ slackToken,