Notify users on GitHub when comment contains id/label missmatch

Signed-off-by: Jakub Stejskal <[email protected]>

Author: Frawless

.gitignore

@@ -9,3 +9,4 @@ secrets
 *.egg-info
 .mypy_cache/
 /build/
+node_modules

packit_service/worker/checker/abstract.py

@@ -29,10 +29,28 @@ def __init__(
         self.job_config = job_config
         self.data = EventData.from_event_dict(event)
         self.task_name = task_name
+        self._mismatch_data: Optional[dict] = None
 
     @abstractmethod
     def pre_check(self) -> bool: ...
 
+    def get_failure_message(self) -> Optional[dict]:
+        """
+        Get the failure message/mismatch data if the check failed.
+        This is used to aggregate failure messages into a single comment.
+
+        Returns:
+            Failure message/mismatch data if check failed, None otherwise.
+            Dict with structured data:
+            {
+                "type": str, # type of the matcher failure
+                "job_value": str | list[str],  # job's identifier or labels
+                "comment_value": str | list[str],  # what was specified in command
+                "targets": list[str],  # all targets for this job
+            }
+        """
+        return self._mismatch_data
+
 
 class ActorChecker(Checker):
     @property

packit_service/worker/checker/testing_farm.py

@@ -154,6 +154,8 @@ class IsIdentifierFromCommentMatching(Checker, GetTestingFarmJobHelperMixin):
     otherwise only jobs with the same identifier.
     """
 
+    DESCRIPTION: str = "**Identifiers don't match** - test run will be skipped"
+
     def pre_check(self) -> bool:
         if (
             not self.testing_farm_job_helper.comment_arguments.labels
@@ -163,7 +165,17 @@ def pre_check(self) -> bool:
             logger.info(
                 f"Using the default identifier for test command: {default_identifier}",
             )
-            return self.job_config.identifier == default_identifier
+            if self.job_config.identifier != default_identifier:
+                # Create structured mismatch data for aggregation
+                self._mismatch_data = {
+                    "type": "identifier_default",
+                    "job_value": self.job_config.identifier,
+                    "comment_value": default_identifier,
+                    "targets": list(self.job_config.targets),
+                }
+
+                return False
+            return True
 
         if (
             not self.testing_farm_job_helper.comment_arguments.identifier
@@ -177,6 +189,15 @@ def pre_check(self) -> bool:
             f"(job:{self.job_config.identifier} "
             f"!= comment:${self.testing_farm_job_helper.comment_arguments.identifier})",
         )
+
+        # Create structured mismatch data for aggregation
+        self._mismatch_data = {
+            "type": "identifier_explicit",
+            "job_value": self.job_config.identifier,
+            "comment_value": self.testing_farm_job_helper.comment_arguments.identifier,
+            "targets": list(self.job_config.targets),
+        }
+
         return False
 
 
@@ -187,17 +208,39 @@ class IsLabelFromCommentMatching(Checker, GetTestingFarmJobHelperMixin):
     otherwise only jobs with the same label.
     """
 
+    DESCRIPTION: str = "**Labels don't match** - test run will be skipped"
+
     def pre_check(self) -> bool:
         if (
             not self.testing_farm_job_helper.comment_arguments.labels
             and not self.testing_farm_job_helper.comment_arguments.identifier
             and (default_labels := self.job_config.test_command.default_labels)
         ):
             logger.info(f"Using the default labels for test command: {default_labels}")
+
             if not self.job_config.labels:
+                # Create structured mismatch data for aggregation
+                self._mismatch_data = {
+                    "type": "labels_default",
+                    "job_value": self.job_config.labels or [],
+                    "comment_value": list(default_labels),
+                    "targets": list(self.job_config.targets),
+                }
+
                 return False
 
-            return any(x in default_labels for x in self.job_config.labels)
+            if not any(x in default_labels for x in self.job_config.labels):
+                # Create structured mismatch data for aggregation
+                self._mismatch_data = {
+                    "type": "labels_default",
+                    "job_value": list(self.job_config.labels),
+                    "comment_value": list(default_labels),
+                    "targets": list(self.job_config.targets),
+                }
+
+                return False
+
+            return True
 
         if not self.testing_farm_job_helper.comment_arguments.labels or (
             self.job_config.labels
@@ -213,4 +256,13 @@ def pre_check(self) -> bool:
             f"(job:{self.job_config.labels} "
             f"!= comment:${self.testing_farm_job_helper.comment_arguments.labels})",
         )
+
+        # Create structured mismatch data for aggregation
+        self._mismatch_data = {
+            "type": "labels_explicit",
+            "job_value": list(self.job_config.labels) if self.job_config.labels else [],
+            "comment_value": list(self.testing_farm_job_helper.comment_arguments.labels),
+            "targets": list(self.job_config.targets),
+        }
+
         return False

packit_service/worker/handlers/abstract.py

@@ -336,12 +336,18 @@ def pre_check(
         package_config: PackageConfig,
         job_config: JobConfig,
         event: dict,
-    ) -> bool:
+    ) -> tuple[bool, list[dict]]:
         """
-        Returns
-            bool: False if we have to skip the job execution.
+        Run pre-checks for the handler.
+
+        Returns:
+            tuple[bool, list[dict]]: (checks_pass, failure_messages)
+                - checks_pass: False if we have to skip the job execution
+                - failure_messages: List of failure messages from failed checkers
         """
         checks_pass = True
+        failure_messages = []
+
         for checker_cls in cls.get_checkers():
             task_name = getattr(cls, "task_name", None)
             checker = checker_cls(
@@ -350,9 +356,14 @@ def pre_check(
                 event=event,
                 task_name=task_name.value if task_name else None,
             )
-            checks_pass = checks_pass and checker.pre_check()
+            passed = checker.pre_check()
+            checks_pass = checks_pass and passed
+
+            # Collect failure messages to be aggregated at event level
+            if not passed and (failure_msg := checker.get_failure_message()):
+                failure_messages.append(failure_msg)
 
-        return checks_pass
+        return checks_pass, failure_messages
 
     @staticmethod
     def get_handler_specific_task_accepted_message(