deploy_delay_notifications.yml

name: Deploy Delay Notifications

on:
  schedule:
    - cron: "*/10 * * * *" # Runs every 10 minutes
permissions:
  id-token: write
  contents: read
jobs:
  check-deployment:
    runs-on: ubuntu-latest

    outputs:
      dev_summary: ${{ steps.check-dev-status.outputs.dev_summary }}
      staging_summary: ${{ steps.check-staging-status.outputs.staging_summary }}

    steps:
      - name: Get latest commit SHA and time from master branch
        id: git-info
        run: |
          latest_commit_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
            "https://api.github.com/repos/${{ github.repository }}/commits/master")
          latest_sha=$(echo "${latest_commit_info}" | jq -r '.sha')
          echo "latest_sha=${latest_sha}" >> $GITHUB_ENV
          echo "latest_sha: ${latest_sha}"
          commit_time=$(echo "${latest_commit_info}" | jq -r '.commit.committer.date')
          echo "commit_time=${commit_time}" >> $GITHUB_ENV
          echo "commit_time: ${commit_time}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Get deployed SHA for development
        id: dev-deploy-sha
        run: |
          deployed_sha=$(curl -s https://dev-api.va.gov/v0/status | jq -r .git_revision)
          echo "dev_deployed_sha=${deployed_sha}" >> $GITHUB_ENV

      - name: Get deployed SHA for staging
        id: staging-deploy-sha
        run: |
          deployed_sha=$(curl -s https://staging-api.va.gov/v0/status | jq -r .git_revision)
          echo "staging_deployed_sha=${deployed_sha}" >> $GITHUB_ENV

      - name: Check deployment status for development
        if: ${{ env.latest_sha != '' && env.dev_deployed_sha != '' }}
        id: check-dev-status
        run: |
          latest_sha=${{ env.latest_sha }}
          commit_time=${{ env.commit_time }}
          deployed_sha=${{ env.dev_deployed_sha }}
          info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to development"
          action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-dev|ArgoCD dev>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"


          if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
            echo "${info_message} has been deployed."
            echo "dev_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
          elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
            echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
            echo "dev_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
          elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
            echo "${info_message} has been delayed for more than 30 minutes."
            echo "Current commit on development is ${deployed_sha:0:8}."
            echo "dev_summary=${info_message} has been delayed for more than 30 minutes. Current commit on development is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
            exit 1
          else
            echo "Awaiting deployment of ${info_message}."
            echo "dev_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
          fi

      - name: Check deployment status for staging
        if: ${{ always() && env.latest_sha != '' && env.staging_deployed_sha != '' }}
        id: check-staging-status
        run: |
          latest_sha=${{ env.latest_sha }}
          commit_time=${{ env.commit_time }}
          deployed_sha=${{ env.staging_deployed_sha }}
          info_message="Latest commit (${latest_sha:0:8}, ${commit_time}) to staging"
          action_items="\n- <https://argocd.vfs.va.gov/applications/vets-api-staging|ArgoCD staging>\n- <https://github.com/department-of-veterans-affairs/vets-api/actions/workflows/build.yml?query=branch%3Amaster|Build, Push, & Deploy> GitHub Action\n- <https://www.va.gov/atlas/apps/vets-api/deploy_status|Deploy dashboard>\n- <https://github.com/department-of-veterans-affairs/vets-api/commits/master/|Latest commits>"

          if [ "${latest_sha:0:8}" == "${deployed_sha:0:8}" ]; then
            echo "${info_message} has been deployed."
            echo "staging_summary=${info_message} has been deployed." >> $GITHUB_OUTPUT
          elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '45 minutes ago' +%s)" ]; then
            echo "${info_message} has been delayed for more than 45 minutes. Skipping notification."
            echo "staging_summary=${info_message} has been delayed for more than 45 minutes. Skipping notification." >> $GITHUB_OUTPUT
          elif [ "$(date -d "${commit_time}" +%s)" -lt "$(date -d '30 minutes ago' +%s)" ]; then
            echo "${info_message} has been delayed for more than 30 minutes."
            echo "Current commit on staging is ${deployed_sha:0:8}."
            echo "staging_summary=${info_message} has been delayed for more than 30 minutes. Current commit on staging is ${deployed_sha:0:8}.\n\nCheck the following list of items for errors: ${action_items}" >> $GITHUB_OUTPUT
            exit 1
          else
            echo "Awaiting deployment of ${info_message}."
            echo "staging_summary=Awaiting deployment of ${info_message}." >> $GITHUB_OUTPUT
          fi

  notify-on-failure:
    runs-on: ubuntu-latest
    needs: [check-deployment]
    env:
      dev_summary: ${{ needs.check-deployment.outputs.dev_summary }}
      staging_summary: ${{ needs.check-deployment.outputs.staging_summary }}
    if: ${{ failure() }}
    steps:
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6
        with:
          role-to-assume: ${{ vars.AWS_ASSUME_ROLE }}
          aws-region: "us-gov-west-1"

      - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb # latest
        with:
          ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN
          env_variable_name: VA_VSP_BOT_GITHUB_TOKEN

      - name: Checkout VSP actions
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: department-of-veterans-affairs/vsp-github-actions
          ref: refs/heads/main
          token: ${{ env.VA_VSP_BOT_GITHUB_TOKEN }}
          persist-credentials: false
          path: ./.github/actions/vsp-github-actions

      - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb # latest
        with:
          ssm_parameter: /devops/github_actions_slack_socket_token
          env_variable_name: SLACK_APP_TOKEN

      - uses: department-of-veterans-affairs/action-inject-ssm-secrets@d8e6de3bde4dd728c9d732baef58b3c854b8c4bb # latest
        with:
          ssm_parameter: /devops/github_actions_slack_bot_user_token
          env_variable_name: SLACK_BOT_TOKEN

      - name: Notify for deployment failure
        if: ${{ env.dev_summary != '' || env.staging_summary != '' }}
        uses: ./.github/actions/vsp-github-actions/slack-socket
        with:
          slack_app_token: ${{ env.SLACK_APP_TOKEN }}
          slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
          message: "Vets API Deployment Delay:"
          blocks: |
            [
              { "type": "divider" },
              {
                "type": "section",
                "text": {
                  "type": "mrkdwn",
                  "text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>\n\n *Development Summary:*\n${{ env.dev_summary }}\n\n *Staging Summary:*\n${{ env.staging_summary }}"
                }
              },
              { "type": "divider" }
            ]
          channel_id: "C0APA867M0V"

      - name: Notify for other failure
        if: ${{ env.dev_summary == '' && env.staging_summary == '' }}
        uses: ./.github/actions/vsp-github-actions/slack-socket
        with:
          slack_app_token: ${{ env.SLACK_APP_TOKEN }}
          slack_bot_token: ${{ env.SLACK_BOT_TOKEN }}
          message: "Vets API Deployment Delay:"
          blocks: |
            [
              { "type": "divider" },
              {
                "type": "section",
                "text": {
                  "type": "mrkdwn",
                  "text": ":scared_and_sweating_smiley: GitHub Action Runner Workflow failed! :scared_and_sweating_smiley:\n\n Unknown error occured. See logs:\n <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }} Run #${{ github.run_number }}>"
                }
              },
              { "type": "divider" }
            ]
          channel_id: "C0APA867M0V"