Merge pull request #162 from scality/improvement/PTFE-2835-Improve-smart-workflow-triggers-features

rdebay-scality · web-flow · commit 3fc90e56d9a1 · 2025-12-08T15:36:19.000+01:00
Improvement/ptfe 2835 improve smart workflow triggers features
diff --git a/actions-nightly-trigger/README.md b/actions-nightly-trigger/README.md
@@ -52,6 +52,8 @@ jobs:
 | `workflow` | Workflow file to trigger | Yes | `nightly.yaml` |
 | `access_token` | GitHub token with workflow permissions | Yes | - |
 | `run-workflow` | Whether to actually run the workflow or only make decision | No | `true` |
+| `max-start-if-last-failed` | Maximum number of times to restart if last workflow run on last commit failed | No | `0` (disabled) |
+| `commit-check-period` | Time window in days to check for recent commits | No | `1` |
 
 ## Outputs
 
@@ -64,9 +66,52 @@ jobs:
 
 ## Triggering Logic
 
-1. If last commit in last 24 hours → Trigger
-2. If last run was 7, 14, 21... days ago → Trigger (weekly check)
-3. Otherwise → Skip
+The action applies these rules in priority order:
+
+1. **Restart on failure**: If `max-start-if-last-failed` is set and the last workflow run on the last commit failed/timed out/cancelled, and we haven't exceeded the max restart count → Trigger
+2. **Recent commit**: If last commit is within `commit-check-period` days and no successful run exists on that commit → Trigger
+3. **Weekly health check**: If last run was 7, 14, 21... days ago → Trigger (weekly check)
+4. Otherwise → Skip
+
+### Failure Restart Feature
+
+The `max-start-if-last-failed` parameter allows automatic restarts when a workflow fails:
+
+- Counts all workflow runs on the **last commit** of the branch
+- If the most recent run failed/timed out/cancelled, triggers a new run
+- Stops restarting once `max-start-if-last-failed` runs have been attempted
+- If a successful run exists on the last commit, no restart is triggered
+
+Example with failure restart:
+
+```yaml
+- uses: scality/actions/actions-nightly-trigger@main
+  with:
+    branch: "development/4"
+    workflow: "nightly.yaml"
+    access_token: ${{ secrets.GIT_ACCESS_TOKEN }}
+    max-start-if-last-failed: 3  # Restart up to 3 times on failure
+```
+
+### Configurable Commit Check Period
+
+The `commit-check-period` parameter allows you to adjust the time window for checking recent commits:
+
+- Set to `1` (default): Triggers if commit is within last 24 hours
+- Set to `2`: Triggers if commit is within last 48 hours (2 days)
+- Set to `N`: Triggers if commit is within last N days
+
+This is useful when your workflow runs less frequently (e.g., every 2-3 days):
+
+```yaml
+# For a workflow that runs every 2 days
+- uses: scality/actions/actions-nightly-trigger@main
+  with:
+    branch: "development/4"
+    workflow: "nightly.yaml"
+    access_token: ${{ secrets.GIT_ACCESS_TOKEN }}
+    commit-check-period: 2  # Check commits in last 2 days
+```
 
 ## Testing Mode
 
diff --git a/actions-nightly-trigger/action.yaml b/actions-nightly-trigger/action.yaml
@@ -24,6 +24,14 @@ inputs:
     description: 'Whether to actually run the workflow or only make decision (default: true)'
     required: false
     default: 'true'
+  max-start-if-last-failed:
+    description: 'Maximum number of times to retry if last workflow run on last commit failed (default: 0 - disabled)'
+    required: false
+    default: '0'
+  commit-check-period:
+    description: 'Time window in days to check for recent commits (default: 1 day)'
+    required: false
+    default: '1'
 
 outputs:
   trigger:
@@ -56,14 +64,17 @@ runs:
         ref: ${{ steps.set-branch.outputs.branch }}
         fetch-depth: 1
 
-    - name: Get last commit timestamp
+    - name: Get last commit info
       id: check-commits
       shell: bash
       run: |
-        # Get the timestamp of the last commit
+        # Get the timestamp and SHA of the last commit
         last_commit_time=$(git log -1 --format=%ct)
+        last_commit_sha=$(git log -1 --format=%H)
         echo "last_commit_time=$last_commit_time" >> $GITHUB_OUTPUT
+        echo "last_commit_sha=$last_commit_sha" >> $GITHUB_OUTPUT
         echo "📊 Last commit timestamp: $last_commit_time"
+        echo "📊 Last commit SHA: $last_commit_sha"
 
     - name: Check latest workflow run status
       id: check-status
@@ -109,13 +120,50 @@ runs:
 
         echo "🔍 Latest workflow run #$run_id: status=$status, conclusion=$conclusion"
 
+    - name: Count workflow runs on last commit
+      id: check-runs-on-commit
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+        COMMIT_SHA: ${{ steps.check-commits.outputs.last_commit_sha }}
+      run: |
+        # Get all workflow runs for this specific commit SHA
+        runs=$(gh api \
+          -H "Accept: application/vnd.github+json" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          "/repos/${{ github.repository }}/actions/workflows/${{ inputs.workflow }}/runs?head_sha=${COMMIT_SHA}" \
+          --jq '.workflow_runs')
+
+        # Count total runs
+        run_count=$(echo "$runs" | jq 'length')
+        echo "run_count=$run_count" >> $GITHUB_OUTPUT
+        echo "🔢 Total runs on commit ${COMMIT_SHA:0:8}: $run_count"
+
+        # Get the most recent run's conclusion
+        if [ "$run_count" -gt 0 ]; then
+          last_run_conclusion=$(echo "$runs" | jq -r '.[0].conclusion')
+          last_run_status=$(echo "$runs" | jq -r '.[0].status')
+          echo "last_run_on_commit_conclusion=$last_run_conclusion" >> $GITHUB_OUTPUT
+          echo "last_run_on_commit_status=$last_run_status" >> $GITHUB_OUTPUT
+          echo "📊 Last run on this commit: status=$last_run_status, conclusion=$last_run_conclusion"
+        else
+          echo "last_run_on_commit_conclusion=not_found" >> $GITHUB_OUTPUT
+          echo "last_run_on_commit_status=not_found" >> $GITHUB_OUTPUT
+          echo "📊 No runs found on this commit"
+        fi
+
     - name: Decide whether to trigger workflow
       id: decide
       shell: bash
       env:
         LAST_COMMIT_TIME: ${{ steps.check-commits.outputs.last_commit_time }}
         STATUS: ${{ steps.check-status.outputs.status }}
         DAYS_SINCE: ${{ steps.check-status.outputs.days_since_last_run }}
+        RUN_COUNT: ${{ steps.check-runs-on-commit.outputs.run_count }}
+        LAST_RUN_CONCLUSION: ${{ steps.check-runs-on-commit.outputs.last_run_on_commit_conclusion }}
+        LAST_RUN_STATUS: ${{ steps.check-runs-on-commit.outputs.last_run_on_commit_status }}
+        MAX_RETRIES: ${{ inputs.max-start-if-last-failed }}
+        COMMIT_CHECK_PERIOD: ${{ inputs.commit-check-period }}
       run: |
         # Run the decision script
         ${{ github.action_path }}/decide-trigger.sh
@@ -129,6 +177,7 @@ runs:
 
     - name: Trigger workflow
       if: steps.decide.outputs.trigger == 'true' && inputs.run-workflow == 'true'
+      id: trigger
       shell: bash
       env:
         GITHUB_TOKEN: ${{ inputs.access_token }}
@@ -138,18 +187,51 @@ runs:
         echo "📋 Reason: ${{ steps.decide.outputs.reason }}"
         gh workflow run "${{ inputs.workflow }}" --ref="${BRANCH}"
 
+        # Wait for the workflow run to be created
+        echo "⏳ Waiting for workflow run to be created..."
+        sleep 3
+
+        # Get the latest workflow run ID
+        new_run_id=$(gh run list --workflow="${{ inputs.workflow }}" --branch="${BRANCH}" --limit=1 --json databaseId --jq '.[0].databaseId')
+
+        if [ -n "${new_run_id}" ]; then
+          echo "triggered_run_id=${new_run_id}" >> $GITHUB_OUTPUT
+          echo "✅ Triggered run ID: ${new_run_id}"
+        fi
+
     - name: Log decision
       shell: bash
       env:
         BRANCH: ${{ steps.set-branch.outputs.branch }}
+        WORKFLOW_URL: https://github.com/${{ github.repository }}/actions/workflows/${{ inputs.workflow }}
+        TRIGGERED_RUN_ID: ${{ steps.trigger.outputs.triggered_run_id }}
       run: |
-        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-        echo "📊 Smart Nightly Trigger Decision Summary"
-        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-        echo "Branch: ${BRANCH}"
-        echo "Trigger: ${{ steps.decide.outputs.trigger }}"
-        echo "Reason: ${{ steps.decide.outputs.reason }}"
-        echo "Last commit time: ${{ steps.check-commits.outputs.last_commit_time }}"
-        echo "Days since last run: ${{ steps.check-status.outputs.days_since_last_run }}"
-        echo "Last status: ${{ steps.check-status.outputs.conclusion }}"
-        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        # Build summary using heredoc
+        SUMMARY=$(cat <<EOF
+        ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+        📊 Smart Nightly Trigger Decision Summary
+        ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+        Workflow: ${{ inputs.workflow }}
+        Workflow URL: ${WORKFLOW_URL}
+        Branch: ${BRANCH}
+        Trigger: ${{ steps.decide.outputs.trigger }}
+        Reason: ${{ steps.decide.outputs.reason }}
+        EOF
+        )
+
+        if [ -n "${TRIGGERED_RUN_ID}" ]; then
+          SUMMARY="${SUMMARY}
+        Triggered run: https://github.com/${{ github.repository }}/actions/runs/${TRIGGERED_RUN_ID}"
+        fi
+
+        SUMMARY="${SUMMARY}
+        Last commit time: ${{ steps.check-commits.outputs.last_commit_time }}
+        Days since last run: ${{ steps.check-status.outputs.days_since_last_run }}
+        Last status: ${{ steps.check-status.outputs.conclusion }}
+        ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+        # Display to console
+        echo "${SUMMARY}"
+
+        # Write to GITHUB_STEP_SUMMARY
+        echo "${SUMMARY}" >> $GITHUB_STEP_SUMMARY
diff --git a/actions-nightly-trigger/decide-trigger.sh b/actions-nightly-trigger/decide-trigger.sh
@@ -8,29 +8,66 @@ set -e
 LAST_COMMIT_TIME="${LAST_COMMIT_TIME:-0}"
 STATUS="${STATUS:-not_found}"
 DAYS_SINCE="${DAYS_SINCE:-999}"
+RUN_COUNT="${RUN_COUNT:-0}"
+LAST_RUN_CONCLUSION="${LAST_RUN_CONCLUSION:-not_found}"
+LAST_RUN_STATUS="${LAST_RUN_STATUS:-not_found}"
+MAX_RETRIES="${MAX_RETRIES:-0}"
+COMMIT_CHECK_PERIOD="${COMMIT_CHECK_PERIOD:-1}"
 
 trigger="false"
 reason="No trigger conditions met"
 
-# Calculate if last commit is within 24 hours
+# Calculate if last commit is within the check period
 current_time=$(date +%s)
 time_diff=$((current_time - LAST_COMMIT_TIME))
 hours_ago=$((time_diff / 3600))
+days_ago=$((time_diff / 86400))
+check_period_seconds=$((COMMIT_CHECK_PERIOD * 86400))
 
-# Priority 1: Trigger if last commit is within 24 hours
-if [ "$LAST_COMMIT_TIME" -gt 0 ] && [ $time_diff -le 86400 ]; then
-    trigger="true"
-    reason="Last commit was ${hours_ago}h ago (within 24h)"
-    echo "✨ $reason"
+# Priority 1: Restart if last run on last commit failed and under max restarts
+if [ "$MAX_RETRIES" -gt 0 ]; then
+    # Check if last run on commit failed
+    if [ "$LAST_RUN_CONCLUSION" = "failure" ] || [ "$LAST_RUN_CONCLUSION" = "timed_out" ] || [ "$LAST_RUN_CONCLUSION" = "cancelled" ]; then
+        # Check if we haven't exceeded max restarts
+        if [ "$RUN_COUNT" -lt "$MAX_RETRIES" ]; then
+            trigger="true"
+            reason="Last run failed (${LAST_RUN_CONCLUSION}), restarting ($RUN_COUNT/$MAX_RETRIES)"
+            echo "🔄 $reason"
+        else
+            echo "⏭️ Max restarts reached ($RUN_COUNT/$MAX_RETRIES), last run: ${LAST_RUN_CONCLUSION}"
+        fi
+    elif [ "$LAST_RUN_CONCLUSION" = "success" ]; then
+        echo "✅ Last run on commit succeeded, no restart needed"
+    fi
+fi
+
+# Priority 2: Trigger if last commit is within check period (and not already succeeded)
+if [ "$trigger" = "false" ] && [ "$LAST_COMMIT_TIME" -gt 0 ] && [ $time_diff -le $check_period_seconds ]; then
+    # Only trigger if no successful run exists on this commit
+    if [ "$LAST_RUN_CONCLUSION" != "success" ]; then
+        trigger="true"
+        if [ "$COMMIT_CHECK_PERIOD" -eq 1 ]; then
+            reason="Last commit was ${hours_ago}h ago (within ${COMMIT_CHECK_PERIOD} day)"
+        else
+            reason="Last commit was ${days_ago}d ago (within ${COMMIT_CHECK_PERIOD} days)"
+        fi
+        echo "✨ $reason"
+    else
+        if [ "$COMMIT_CHECK_PERIOD" -eq 1 ]; then
+            echo "✅ Last commit was ${hours_ago}h ago but already has successful run"
+        else
+            echo "✅ Last commit was ${days_ago}d ago but already has successful run"
+        fi
+    fi
 
-# Priority 2: Weekly health check (modulo 7 ensures only once per week)
-elif [ "$STATUS" != "not_found" ] && [ "$DAYS_SINCE" -gt 0 ] && [ $(($DAYS_SINCE % 7)) -eq 0 ]; then
+# Priority 3: Weekly health check (modulo 7 ensures only once per week)
+elif [ "$trigger" = "false" ] && [ "$STATUS" != "not_found" ] && [ "$DAYS_SINCE" -gt 0 ] && [ $(($DAYS_SINCE % 7)) -eq 0 ]; then
     trigger="true"
     reason="Weekly health check ($DAYS_SINCE days since last run)"
     echo "🏥 $reason"
 
 # Default: Skip
-else
+elif [ "$trigger" = "false" ]; then
     echo "⏭️ Skipping nightly - $reason"
     echo "   - Last commit: ${hours_ago}h ago"
     echo "   - Days since last run: $DAYS_SINCE"
diff --git a/actions-nightly-trigger/test-decide-trigger.sh b/actions-nightly-trigger/test-decide-trigger.sh
@@ -113,6 +113,88 @@ export STATUS="not_found"
 export DAYS_SINCE=999
 run_test "Status not found, old commit" 0 "false"
 
+# Test 8: Restart on failure - first failure (1 run, max 3)
+export LAST_COMMIT_TIME=$((current_time - 172800))  # 48 hours ago (old commit)
+export STATUS="completed"
+export DAYS_SINCE=1
+export RUN_COUNT=1
+export LAST_RUN_CONCLUSION="failure"
+export LAST_RUN_STATUS="completed"
+export MAX_RETRIES=3
+run_test "Restart on failure - first failure (1/3)" 0 "true"
+
+# Test 9: Restart on failure - max restarts reached
+export RUN_COUNT=3
+export LAST_RUN_CONCLUSION="failure"
+export MAX_RETRIES=3
+run_test "Max restarts reached (3/3)" 0 "false"
+
+# Test 10: No restart - last run succeeded
+export RUN_COUNT=2
+export LAST_RUN_CONCLUSION="success"
+export MAX_RETRIES=3
+run_test "Last run succeeded, no restart needed" 0 "false"
+
+# Test 11: Restart on timeout
+export RUN_COUNT=1
+export LAST_RUN_CONCLUSION="timed_out"
+export MAX_RETRIES=3
+run_test "Restart on timeout (1/3)" 0 "true"
+
+# Test 12: Restart on cancelled
+export RUN_COUNT=2
+export LAST_RUN_CONCLUSION="cancelled"
+export MAX_RETRIES=3
+run_test "Restart on cancelled (2/3)" 0 "true"
+
+# Test 13: Max retries disabled (0)
+export RUN_COUNT=1
+export LAST_RUN_CONCLUSION="failure"
+export MAX_RETRIES=0
+run_test "Max retries disabled (MAX_RETRIES=0)" 0 "false"
+
+# Test 14: Recent commit with successful run - should not trigger
+export LAST_COMMIT_TIME=$((current_time - 18000))  # 5 hours ago
+export RUN_COUNT=1
+export LAST_RUN_CONCLUSION="success"
+export MAX_RETRIES=3
+run_test "Recent commit with successful run" 0 "false"
+
+# Test 15: Weekly health check takes priority even when max restarts exceeded
+export LAST_COMMIT_TIME=$((current_time - 172800))  # 48 hours ago (old commit)
+export STATUS="completed"
+export DAYS_SINCE=7
+export RUN_COUNT=3
+export LAST_RUN_CONCLUSION="failure"
+export MAX_RETRIES=3
+export COMMIT_CHECK_PERIOD=1
+run_test "Weekly health check with max restarts reached (7 days)" 0 "true"
+
+# Test 16: Commit within 2 day check period (30 hours ago)
+export LAST_COMMIT_TIME=$((current_time - 108000))  # 30 hours ago
+export STATUS="not_found"
+export DAYS_SINCE=999
+export RUN_COUNT=0
+export LAST_RUN_CONCLUSION="not_found"
+export MAX_RETRIES=0
+export COMMIT_CHECK_PERIOD=2
+run_test "Commit 30h ago within 2 day period" 0 "true"
+
+# Test 17: Commit outside 1 day check period (30 hours ago)
+export LAST_COMMIT_TIME=$((current_time - 108000))  # 30 hours ago
+export COMMIT_CHECK_PERIOD=1
+run_test "Commit 30h ago outside 1 day period" 0 "false"
+
+# Test 18: Commit within 3 day check period (50 hours ago)
+export LAST_COMMIT_TIME=$((current_time - 180000))  # 50 hours ago
+export COMMIT_CHECK_PERIOD=3
+run_test "Commit 50h ago within 3 day period" 0 "true"
+
+# Test 19: Commit outside 2 day check period (50 hours ago)
+export LAST_COMMIT_TIME=$((current_time - 180000))  # 50 hours ago
+export COMMIT_CHECK_PERIOD=2
+run_test "Commit 50h ago outside 2 day period" 0 "false"
+
 # Summary
 echo ""
 echo "═══════════════════════════════════════════"
