Add token circuit breaker notification (issue #18886)

When a Token::Workflow fails to deliver to the SCM with authorization
errors ('Unauthorized request' or 'Request forbidden'), the token gets
disabled. This change adds:

- Event::TokenDisabled: New event triggered when token is disabled
- Notification system: Sends notifications to token executor and members
- Default subscriptions: Enabled by default for instant_email and web channels
- Tests: Added comprehensive test coverage for the new functionality

The notification helps users immediately understand why their token was
disabled and take corrective action.

Fixes #18886

Author: geetxnshgoyal

src/api/app/models/event/base.rb

@@ -22,7 +22,7 @@ def notification_events
          Event::CommentForRequest, Event::CommentForReport,
          Event::RelationshipCreate, Event::RelationshipDelete,
          Event::Report, Event::Decision, Event::AppealCreated,
-         Event::WorkflowRunFail,
+         Event::WorkflowRunFail, Event::TokenDisabled,
          Event::AddedUserToGroup, Event::RemovedUserFromGroup,
          Event::Assignment]
       end

src/api/app/models/event/token_disabled.rb

@@ -0,0 +1,55 @@
+module Event
+  class TokenDisabled < Base
+    self.description = 'SCM/CI Token disabled due to authorization failure'
+    payload_keys :id, :token_id, :scm_vendor, :summary, :token_description
+
+    receiver_roles :token_executor, :token_member
+    delegate :members, to: :token, prefix: true
+
+    self.notification_explanation = 'Receive notifications when an SCM/CI integration token is disabled due to authorization problems.'
+
+    # Example of subject:
+    #   GitHub workflow token disabled
+    def subject
+      vendor = payload['scm_vendor']&.capitalize
+      "#{vendor} workflow token disabled"
+    end
+
+    def token_executors
+      [token&.executor].compact
+    end
+
+    def parameters_for_notification
+      super.merge(notifiable_type: 'Token::Workflow', notifiable_id: payload['token_id'], type: 'NotificationToken')
+    end
+
+    def event_object
+      Token.find(payload['token_id'])
+    end
+
+    private
+
+    def token
+      Token.find_by(id: payload['token_id'], type: 'Token::Workflow')
+    end
+  end
+end
+
+# == Schema Information
+#
+# Table name: events
+#
+#  id          :bigint           not null, primary key
+#  eventtype   :string(255)      not null, indexed
+#  mails_sent  :boolean          default(FALSE), indexed
+#  payload     :text(16777215)
+#  undone_jobs :integer          default(0)
+#  created_at  :datetime         indexed
+#  updated_at  :datetime
+#
+# Indexes
+#
+#  index_events_on_created_at  (created_at)
+#  index_events_on_eventtype   (eventtype)
+#  index_events_on_mails_sent  (mails_sent)
+#

src/api/app/models/workflow_run.rb

@@ -81,7 +81,16 @@ def update_as_failed(message)
     # "Failed to report back to GitHub: Unauthorized request. Please check your credentials again."
     # "Failed to report back to GitHub: Request is forbidden."
 
-    token.update(enabled: false) if message.include?('Unauthorized request') || /Request (is )?forbidden/.match?(message)
+    return unless message.include?('Unauthorized request') || /Request (is )?forbidden/.match?(message)
+
+    token.update(enabled: false)
+    # Create event notification for token being disabled
+    Event::TokenDisabled.create(
+      token_id: token.id,
+      scm_vendor: scm_vendor,
+      summary: message,
+      token_description: token.description
+    )
   end
 
   # Stores debug info to help figure out what went wrong when trying to save a Status in the SCM.

src/api/db/data/20251207091632_create_default_token_disabled_subscriptions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class CreateDefaultTokenDisabledSubscriptions < ActiveRecord::Migration[7.2]
+  def up; end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

src/api/spec/models/event/token_disabled_spec.rb

@@ -0,0 +1,58 @@
+RSpec.describe Event::TokenDisabled do
+  describe '#token_executors' do
+    subject { event.token_executors }
+
+    let(:token) { create(:workflow_token) }
+    let(:event) do
+      Event::TokenDisabled.create(
+        token_id: token.id,
+        scm_vendor: 'github',
+        summary: 'Failed to report back to GitHub: Unauthorized request.',
+        token_description: 'My workflow token'
+      )
+    end
+
+    it { expect(subject).to contain_exactly(token.executor) }
+
+    context 'when the token does not exist' do
+      before do
+        event
+        token.destroy
+      end
+
+      it { expect(subject).to be_empty }
+    end
+  end
+
+  describe '#subject' do
+    subject { event.subject }
+
+    let(:token) { create(:workflow_token) }
+
+    context 'with GitHub vendor' do
+      let(:event) do
+        Event::TokenDisabled.create(
+          token_id: token.id,
+          scm_vendor: 'github',
+          summary: 'Failed to report back to GitHub: Unauthorized request.',
+          token_description: 'My workflow token'
+        )
+      end
+
+      it { expect(subject).to eq('Github workflow token disabled') }
+    end
+
+    context 'with GitLab vendor' do
+      let(:event) do
+        Event::TokenDisabled.create(
+          token_id: token.id,
+          scm_vendor: 'gitlab',
+          summary: 'Failed to report back to GitLab: Request forbidden.',
+          token_description: 'My workflow token'
+        )
+      end
+
+      it { expect(subject).to eq('Gitlab workflow token disabled') }
+    end
+  end
+end

src/api/spec/models/workflow_run_spec.rb

@@ -98,6 +98,35 @@
       it 'disables the token of the token workflow' do
         expect { subject }.to change { workflow_run.token.reload.enabled }.from(true).to(false)
       end
+
+      it 'creates a TokenDisabled event' do
+        expect { subject }.to change(Event::TokenDisabled, :count).by(1)
+      end
+
+      it 'stores token_id in the TokenDisabled event payload' do
+        subject
+        event = Event::TokenDisabled.last
+        expect(event.payload['token_id']).to eq(workflow_run.token.id)
+      end
+
+      it 'stores scm_vendor and summary in the TokenDisabled event payload' do
+        subject
+        event = Event::TokenDisabled.last
+        expect(event.payload['scm_vendor']).to eq(workflow_run.scm_vendor)
+        expect(event.payload['summary']).to include('Request is forbidden')
+      end
+    end
+
+    context 'when the SCM responds with an unauthorized message' do
+      subject { workflow_run.save_scm_report_failure('Failed to report back to GitLab: Unauthorized request. Please check your credentials again.', { api_endpoint: 'https://gitlab.com/api/v4' }) }
+
+      it 'disables the token of the token workflow' do
+        expect { subject }.to change { workflow_run.token.reload.enabled }.from(true).to(false)
+      end
+
+      it 'creates a TokenDisabled event' do
+        expect { subject }.to change(Event::TokenDisabled, :count).by(1)
+      end
     end
   end