wip: dirty test unconditional alert

agronskiy · agronskiy · commit 6953bef554f5 · 2025-12-09T14:02:11.000+01:00
Signed-off-by: Alex Gronskiy &lt;agronskiy@nvidia.com&gt;
diff --git a/.github/actions/slack-alert/action.yml b/.github/actions/slack-alert/action.yml
@@ -0,0 +1,89 @@
+# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+name: "Slack Alert"
+description: "Send alerts to Slack using the alerting service"
+
+inputs:
+  token:
+    description: "Slack alerting token"
+    required: true
+  channel:
+    description: "Slack channel ID"
+    required: true
+    default: "D08N6TYDS79"
+  severity:
+    description: "Alert severity (info, error, silent)"
+    required: true
+  title:
+    description: "Alert title"
+    required: true
+  text:
+    description: "Alert message text"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Send Slack Alert
+      shell: bash
+      env:
+        SLACK_FRONTIER_EVAL_ALERTING_TOKEN: ${{ inputs.token }}
+      run: |
+        set -euo pipefail  # Fail fast on any error
+        
+        # Fail fast if token is missing
+        if [ -z "$SLACK_FRONTIER_EVAL_ALERTING_TOKEN" ]; then
+          echo "Error: SLACK_FRONTIER_EVAL_ALERTING_TOKEN is not set"
+          exit 1
+        fi
+        
+        # Check if alerting endpoint is reachable
+        echo "Checking connectivity to alerting endpoint..."
+        if ! curl -sL --max-time 10 --head https://alerting-frontier-evals.nvidia.com/alert > /dev/null 2>&1; then
+          echo "Warning: Alerting endpoint https://alerting-frontier-evals.nvidia.com/alert is not reachable"
+          echo "Attempting to ping endpoint..."
+          if ! curl -sL --max-time 10 https://alerting-frontier-evals.nvidia.com/alert > /dev/null 2>&1; then
+            echo "Error: Cannot reach alerting endpoint"
+            exit 1
+          fi
+        fi
+        echo "Alerting endpoint is reachable"
+        
+        # Download the alert script first to check for errors
+        ALERT_SCRIPT=$(mktemp)
+        if ! curl -sL -o "$ALERT_SCRIPT" https://gitlab-master.nvidia.com/dl/JoC/competitive_evaluation/nvpark-k8s-infra/-/raw/main/alerting-service-helm/alert.sh; then
+          echo "Error: Failed to download alert script"
+          rm -f "$ALERT_SCRIPT"
+          exit 1
+        fi
+        
+        # Run the alert script and capture exit code
+        set +e  # Temporarily disable exit on error to capture exit code
+        bash "$ALERT_SCRIPT" \
+          --token "$SLACK_FRONTIER_EVAL_ALERTING_TOKEN" \
+          --channel "${{ inputs.channel }}" \
+          --severity "${{ inputs.severity }}" \
+          --title "${{ inputs.title }}" \
+          --text "${{ inputs.text }}"
+        ALERT_EXIT_CODE=$?
+        set -e  # Re-enable exit on error
+        
+        # Clean up
+        rm -f "$ALERT_SCRIPT"
+        
+        # Check if the alert script failed
+        if [ $ALERT_EXIT_CODE -ne 0 ]; then
+          echo "Error: Alert script failed with exit code $ALERT_EXIT_CODE"
+          exit $ALERT_EXIT_CODE
+        fi
diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml
@@ -54,6 +54,22 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Send test alert
+        uses: ./.github/actions/slack-alert
+        with:
+          token: ${{ secrets.SLACK_FRONTIER_EVAL_ALERTING_TOKEN }}
+          channel: "D08N6TYDS79"
+          severity: "error"
+          title: "CI Pipeline Test Alert - ${{ github.ref_name }}"
+          text: |
+            🧪 TEST ALERT: This is a test alert sent at the beginning of CI pipeline
+
+            Workflow: ${{ github.workflow }}
+            Run: ${{ github.run_id }}
+            Branch: ${{ github.ref_name }}
+            Commit: ${{ github.sha }}
+            Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
@@ -200,9 +216,6 @@ jobs:
     runs-on: ubuntu-latest
     permissions: write-all
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
       - name: Get workflow result
         id: result
         shell: bash -x -e -u -o pipefail {0}
@@ -326,26 +339,16 @@ jobs:
       - name: Determine workflow status
         id: status
         shell: bash
-        env:
-          TEST_MODE: ${{ vars.TEST_ALERT_MODE == 'true' }}
         run: |
           set -euo pipefail  # Fail fast
-          
-          # TEST MODE: Force error alert for testing
-          if [ "$TEST_MODE" == "true" ]; then
-            echo "severity=error" >> $GITHUB_OUTPUT
-            echo "status=failure" >> $GITHUB_OUTPUT
-            echo "message=🧪 TEST MODE: Forced error alert for testing purposes" >> $GITHUB_OUTPUT
-            exit 0
-          fi
-          
+
           # Check if main test job succeeded
           MAIN_TEST_RESULT="${{ needs.Nemo_CICD_Test.result }}"
-          
+
           # Coverage jobs might be skipped, so we only care if they failed (not if skipped)
           COVERAGE_RESULT="${{ needs.Coverage.result }}"
           COVERAGE_FAKE_RESULT="${{ needs.Coverage_Fake.result }}"
-          
+
           if [ "$MAIN_TEST_RESULT" == "success" ] && [ "$COVERAGE_RESULT" != "failure" ] && [ "$COVERAGE_FAKE_RESULT" != "failure" ]; then
             echo "severity=info" >> $GITHUB_OUTPUT
             echo "status=success" >> $GITHUB_OUTPUT
@@ -365,10 +368,9 @@ jobs:
           title: "CI Pipeline - ${{ github.ref_name }}"
           text: |
             ${{ steps.status.outputs.message }}
-            
+
             Workflow: ${{ github.workflow }}
             Run: ${{ github.run_id }}
             Commit: ${{ github.sha }}
             Author: ${{ github.actor }}
             Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          test-mode: ${{ vars.TEST_ALERT_MODE == 'true' }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -110,19 +110,9 @@ jobs:
         env:
           IS_SCHEDULED: ${{ github.event_name == 'schedule' }}
           DRY_RUN_INPUT: ${{ inputs.dry-run }}
-          TEST_MODE: ${{ vars.TEST_ALERT_MODE == 'true' }}
         run: |
           set -euo pipefail  # Fail fast
           
-          # TEST MODE: Force error alert for testing
-          if [ "$TEST_MODE" == "true" ]; then
-            echo "severity=error" >> $GITHUB_OUTPUT
-            echo "status=failure" >> $GITHUB_OUTPUT
-            echo "message=🧪 TEST MODE: Forced error alert for testing purposes" >> $GITHUB_OUTPUT
-            echo "release_type=Test" >> $GITHUB_OUTPUT
-            exit 0
-          fi
-          
           RELEASE_RESULT="${{ needs.release.result }}"
           
           # Determine if this is a dry run: scheduled releases are production, manual can be dry-run
@@ -160,7 +150,6 @@ jobs:
             Commit: ${{ github.event_name == 'schedule' && github.sha || inputs.release-ref || github.sha }}
             Trigger: ${{ github.event_name == 'schedule' && 'Scheduled' || 'Manual' }}
             Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          test-mode: ${{ vars.TEST_ALERT_MODE == 'true' }}
 
   release-launcher:
     if: ${{ github.event_name == 'schedule' || inputs.component == 'nemo-evaluator-launcher' }}
@@ -204,19 +193,9 @@ jobs:
         env:
           IS_SCHEDULED: ${{ github.event_name == 'schedule' }}
           DRY_RUN_INPUT: ${{ inputs.dry-run }}
-          TEST_MODE: ${{ vars.TEST_ALERT_MODE == 'true' }}
         run: |
           set -euo pipefail  # Fail fast
           
-          # TEST MODE: Force error alert for testing
-          if [ "$TEST_MODE" == "true" ]; then
-            echo "severity=error" >> $GITHUB_OUTPUT
-            echo "status=failure" >> $GITHUB_OUTPUT
-            echo "message=🧪 TEST MODE: Forced error alert for testing purposes" >> $GITHUB_OUTPUT
-            echo "release_type=Test" >> $GITHUB_OUTPUT
-            exit 0
-          fi
-          
           RELEASE_RESULT="${{ needs.release-launcher.result }}"
           
           # Determine if this is a dry run: scheduled releases are production, manual can be dry-run
@@ -254,4 +233,3 @@ jobs:
             Commit: ${{ github.event_name == 'schedule' && github.sha || inputs.release-ref || github.sha }}
             Trigger: ${{ github.event_name == 'schedule' && 'Scheduled' || 'Manual' }}
             Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
-          test-mode: ${{ vars.TEST_ALERT_MODE == 'true' }}
