feat(nimbus): Handle emojis added reaction (#14927)

### Because

In production, the enrollment ending notification was failing with a
Slack API error already has that reaction.

The root cause is that `send_threaded_success_message()` is being called
multiple times for the same enrollment ending request - either due to
Kinto syncs running multiple times, task retries, or periodic checks.
Each time it's called, it attempts to add the white_check_mark reaction
to the original message. On subsequent calls, the reaction already
exists, causing the API to reject the request with `already_reacted`.

Previously, any Slack API error (including `already_reacted`) was
treated as a fatal failure, logging the error and returning False. This
caused the entire notification flow to fail, even though the actual
desired state (the message exists and the reaction is on it) was already
achieved.

### This commit

Adds graceful error handling for the `already_reacted` Slack error. When
`reactions_add` fails with `already_reacted`, the error is caught and
ignored since the desired end state is already met - the reaction exists
on the message. Other Slack API errors are still re-raised and properly
logged as failures.

Additionally, this commit adds the underlying support for enrollment
ending and experiment ending request notifications with dedicated alert
types (`END_ENROLLMENT_REQUEST` and `END_EXPERIMENT_REQUEST`),
consolidates duplicate Slack notification logic, and adds comprehensive
tests to verify the fix works correctly.

Fixes #14921

Author: yashikakhurana

experimenter/experimenter/slack/constants.py

@@ -49,6 +49,10 @@ class EmojiReaction:
         CANCEL = "x"
         APPROVE = "eyes"
 
+    # Slack API error codes
+    class ErrorCode:
+        ALREADY_REACTED = "already_reacted"
+
     # Slack message templates
     SLACK_DM_PREFIX = "🔔 Join {channel} to get slack notifications: {message}"
     SLACK_DM_CHANNEL_LINK_SUFFIX = "\n\n🔗 View in channel: {channel_message_link}"

experimenter/experimenter/slack/notification.py

@@ -193,11 +193,18 @@ def send_threaded_success_message(
         channel_id = post_response["channel"]
 
         # Add reaction emoji to original message
-        client.reactions_add(
-            channel=channel_id,
-            name="white_check_mark",
-            timestamp=thread_ts,
-        )
+        try:
+            client.reactions_add(
+                channel=channel_id,
+                name="white_check_mark",
+                timestamp=thread_ts,
+            )
+        except SlackApiError as emoji_error:
+            if (
+                emoji_error.response.get("error")
+                != SlackConstants.ErrorCode.ALREADY_REACTED
+            ):
+                raise
 
         logger.info(success_log_message(experiment.slug))
         return True

experimenter/experimenter/slack/tests/test_notification.py

@@ -608,6 +608,59 @@ def test_send_experiment_launch_success_message_slack_error(self, mock_webclient
         self.assertFalse(result)
         mock_client.chat_postMessage.assert_called_once()
 
+    @override_settings(
+        SLACK_AUTH_TOKEN="test-token",
+        SLACK_NIMBUS_CHANNEL="C123456",
+    )
+    @patch("experimenter.slack.notification.WebClient")
+    def test_send_experiment_launch_success_message_reaction_already_exists(
+        self, mock_webclient
+    ):
+        mock_client = Mock()
+        mock_webclient.return_value = mock_client
+        mock_client.chat_postMessage.return_value = {"ok": True, "channel": "C123456"}
+        # Simulate reaction already exists error
+        mock_client.reactions_add.side_effect = SlackApiError(
+            message="Slack error",
+            response={"ok": False, "error": SlackConstants.ErrorCode.ALREADY_REACTED},
+        )
+
+        thread_ts = "1234567890.123456"
+        result = send_threaded_success_message(
+            experiment_id=self.experiment.id,
+            thread_ts=thread_ts,
+            message_template=SlackConstants.SLACK_LAUNCH_SUCCESS_MESSAGE,
+            log_operation=SlackConstants.SLACK_OPERATION_LAUNCH_SUCCESS,
+            success_log_message=lambda slug: f"Sent launch success message for {slug}",
+        )
+
+        self.assertTrue(result)
+        mock_client.chat_postMessage.assert_called_once()
+        mock_client.reactions_add.assert_called_once()
+
+    @override_settings(
+        SLACK_AUTH_TOKEN="test-token",
+        SLACK_NIMBUS_CHANNEL="C123456",
+    )
+    @patch("experimenter.slack.notification.WebClient")
+    def test_send_experiment_launch_success_message_reaction_error(self, mock_webclient):
+        mock_client = Mock()
+        mock_webclient.return_value = mock_client
+        mock_client.chat_postMessage.return_value = {"ok": True, "channel": "C123456"}
+        mock_client.reactions_add.side_effect = SlackApiError(
+            message="Slack error", response={"ok": False, "error": "channel_not_found"}
+        )
+
+        thread_ts = "1234567890.123456"
+        result = send_threaded_success_message(
+            experiment_id=self.experiment.id,
+            thread_ts=thread_ts,
+            message_template=SlackConstants.SLACK_LAUNCH_SUCCESS_MESSAGE,
+            log_operation=SlackConstants.SLACK_OPERATION_LAUNCH_SUCCESS,
+            success_log_message=lambda slug: f"Sent launch success message for {slug}",
+        )
+        self.assertFalse(result)
+
     @override_settings(SLACK_AUTH_TOKEN="test-token")
     @patch("experimenter.slack.notification.WebClient")
     def test_send_experiment_launch_success_message_not_found(self, mock_webclient):