Add Triage agent testsuite

Containerfile.c10s

@@ -38,6 +38,7 @@ RUN dnf -y install --allowerasing \
       sed \
       gawk \
       rsync \
+      python3-tabulate \
     && dnf clean all
 
 RUN pip3 install --no-cache-dir \
@@ -46,7 +47,9 @@ RUN pip3 install --no-cache-dir \
       openinference-instrumentation-beeai \
       arize-phoenix-otel \
       redis \
-      specfile
+      specfile \
+      pytest \
+      pytest-asyncio
 
 # Create user
 RUN useradd -m -G wheel beeai

Makefile

@@ -27,8 +27,12 @@ run-triage-agent-standalone:
 		-e MOCK_JIRA=$(MOCK_JIRA) \
 		triage-agent
 
-
-
+.PHONY: run-triage-agent-e2e-tests
+run-triage-agent-e2e-tests:
+	$(COMPOSE_AGENTS) run --rm \
+		-e MOCK_JIRA="true" \
+		-e DRY_RUN=$(DRY_RUN) \
+		triage-agent-e2e-tests
 
 .PHONY: run-rebase-agent-c9s-standalone
 run-rebase-agent-c9s-standalone:

README-agents.md

@@ -19,6 +19,23 @@ Three agents process tasks through Redis queues:
 - **Modify JIRA issues** (add comments, update fields, apply labels)
 - **Create GitLab merge requests** and push commits
 
+## Jira mocking
+
+If you clone testing Jira files from
+`[email protected]:jotnar-project/testing-jiras.git`
+you can use them to work with instead of real Jira server.
+
+Example:
+
+`make run-triage-agent-standalone JIRA_ISSUE=RHEL-15216 MOCK_JIRA=true`
+
+If used together with `DRY_RUN`, the agents won't edit the Jira files,
+otherwise they will.
+
+Example:
+
+`make run-triage-agent-standalone JIRA_ISSUE=RHEL-15216 DRY_RUN=true MOCK_JIRA=true`
+
 ## Setup
 
 ### Required API Tokens & Authentication

agents/backport_agent.py

@@ -37,6 +37,8 @@
     LogInputSchema,
     LogOutputSchema,
     Task,
+    BackportData,
+    ErrorData,
 )
 from common.utils import redis_client, fix_await
 from constants import I_AM_JOTNAR, CAREFULLY_REVIEW_CHANGES
@@ -68,7 +70,6 @@
     GitPatchCreationTool,
     GitPreparePackageSources,
 )
-from triage_agent import BackportData, ErrorData
 from utils import (
     check_subprocess,
     get_agent_execution_config,

agents/metrics_middleware.py

@@ -0,0 +1,44 @@
+from datetime import datetime
+
+from beeai_framework.context import (
+    RunContextStartEvent,
+    RunContextFinishEvent,
+    RunMiddlewareProtocol,
+    RunContext
+)
+from beeai_framework.emitter import EmitterOptions, EventMeta
+from beeai_framework.emitter.utils import create_internal_event_matcher
+
+
+class MetricsMiddleware(RunMiddlewareProtocol):
+    def __init__(self) -> None:
+        self.start_time: datetime | None = None
+        self.end_time: datetime | None = None
+        self.tool_calls: int = 0
+
+    def bind(self, ctx: RunContext) -> None:
+        ctx.emitter.on(
+            create_internal_event_matcher("start", ctx.instance),
+            self._on_run_context_start,
+            EmitterOptions(is_blocking=True, priority=1),
+        )
+        ctx.emitter.on(
+            create_internal_event_matcher("finish", ctx.instance),
+            self._on_run_context_finish,
+            EmitterOptions(is_blocking=True, priority=1),
+        )
+
+    async def _on_run_context_start(self, event: RunContextStartEvent, meta: EventMeta) -> None:
+        self.start_time = datetime.now()
+
+    async def _on_run_context_finish(self, event: RunContextFinishEvent, meta: EventMeta) -> None:
+        self.end_time = datetime.now()
+
+    @property
+    def duration(self) -> float:
+        if self.start_time and self.end_time:
+            return (self.end_time - self.start_time).total_seconds()
+        return 0
+
+    def get_metrics(self) -> dict[str, float]:
+        return {"duration": self.duration}

agents/tasks.py

@@ -9,9 +9,9 @@
 
 from common.models import LogOutputSchema, CachedMRMetadata
 from common.utils import is_cs_branch
-from constants import BRANCH_PREFIX, JIRA_COMMENT_TEMPLATE
-from utils import check_subprocess, run_subprocess, run_tool, mcp_tools
-from tools.specfile import UpdateReleaseTool
+from agents.constants import BRANCH_PREFIX, JIRA_COMMENT_TEMPLATE
+from agents.utils import check_subprocess, run_subprocess, run_tool, mcp_tools
+from agents.tools.specfile import UpdateReleaseTool
 
 logger = logging.getLogger(__name__)
 

agents/tests/e2e/conftest.py

@@ -0,0 +1,16 @@
+from typing import Generator
+
+
+import pytest
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_terminal_summary(
+    terminalreporter: pytest.TerminalReporter, exitstatus, config: pytest.Config
+) -> Generator:
+    yield
+    metrics = config.stash.get("metrics", None)
+
+    if metrics:
+        terminalreporter.write_sep("=", "Metrics")
+        terminalreporter.write_line(metrics, flush=True)

agents/tests/e2e/test_triage.py

@@ -0,0 +1,131 @@
+from tabulate import tabulate
+import pytest
+import os
+
+from agents.triage_agent import run_workflow, TriageState, create_triage_agent
+from agents.metrics_middleware import MetricsMiddleware
+from agents.observability import setup_observability
+from common.models import TriageOutputSchema, Resolution, BackportData
+
+
+class TriageAgentTestCase:
+    def __init__(self, input: str, expected_output: TriageOutputSchema):
+        self.input: str = input
+        self.expected_output: TriageOutputSchema = expected_output
+        self.metrics: dict = None
+
+    async def run(self) -> TriageState:
+        metrics_middleware = MetricsMiddleware()
+
+        def testing_factory(gateway_tools):
+            triage_agent = create_triage_agent(gateway_tools)
+            triage_agent.middlewares.append(metrics_middleware)
+            return triage_agent
+
+        finished_state = await run_workflow(self.input, False, testing_factory)
+        self.metrics = metrics_middleware.get_metrics()
+        return finished_state
+
+
+test_cases = [
+    TriageAgentTestCase(
+        input="RHEL-15216",
+        expected_output=TriageOutputSchema(
+            resolution=Resolution.BACKPORT,
+            data=BackportData(
+                package="dnsmasq",
+                patch_urls=[
+                    "http://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=patch;h=dd33e98da09c487a58b6cb6693b8628c0b234a3b"
+                ],
+                justification="not-implemented",
+                jira_issue="RHEL-15216",
+                cve_id=None,
+                fix_version="rhel-8.10",
+            ),
+        ),
+    ),
+    TriageAgentTestCase(
+        input="RHEL-112546",
+        expected_output=TriageOutputSchema(
+            resolution=Resolution.BACKPORT,
+            data=BackportData(
+                package="libtiff",
+                patch_urls=[
+                    "https://gitlab.com/libtiff/libtiff/-/commit/d1c0719e004fbb223c571d286c73911569d4dbb6.patch"
+                ],
+                justification="not-implemented",
+                jira_issue="RHEL-112546",
+                cve_id="CVE-2025-9900",
+                fix_version="rhel-9.6.z",
+            ),
+        ),
+    ),
+    TriageAgentTestCase(
+        input="RHEL-61943",
+        expected_output=TriageOutputSchema(
+            resolution=Resolution.BACKPORT,
+            data=BackportData(
+                package="dnsmasq",
+                patch_urls=[
+                    "http://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=patch;h=eb1fe15ca80b6bc43cd6bfdf309ec6c590aff811"
+                ],
+                justification="not-implemented",
+                jira_issue="RHEL-61943",
+                cve_id=None,
+                fix_version="rhel-8.10.z",
+            ),
+        ),
+    ),
+    TriageAgentTestCase(
+        input="RHEL-29712",
+        expected_output=TriageOutputSchema(
+            resolution=Resolution.BACKPORT,
+            data=BackportData(
+                package="bind",
+                patch_urls=[
+                    "https://gitlab.isc.org/isc-projects/bind9/-/commit/7e2f50c36958f8c98d54e6d131f088a4837ce269"
+                ],
+                justification="not-implemented",
+                jira_issue="RHEL-29712",
+                cve_id=None,
+                fix_version="rhel-8.10.z",
+            ),
+        ),
+    ),
+]
+
+
+@pytest.fixture(scope="session", autouse=True)
+def observability_fixture():
+    return setup_observability(os.environ["COLLECTOR_ENDPOINT"])
+
+
+@pytest.fixture(scope="session", autouse=True)
+def mydata(request):
+    yield
+    collected_metrics = []
+    for test_case in test_cases:
+        if test_case.metrics is None:
+            continue
+        collected_metrics.append([test_case.input] + list(test_case.metrics.values()))
+    request.config.stash["metrics"] = tabulate(collected_metrics, ["Issue", "Time"])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "test_case",
+    test_cases,
+)
+async def test_triage_agent(test_case: TriageAgentTestCase):
+    def verify_result(
+        real_output: TriageOutputSchema, expected_output: TriageOutputSchema
+    ):
+        assert real_output.resolution == expected_output.resolution
+        assert real_output.data.package == expected_output.data.package
+        assert real_output.data.patch_urls == expected_output.data.patch_urls
+        assert real_output.data.jira_issue == expected_output.data.jira_issue
+        assert real_output.data.cve_id == expected_output.data.cve_id
+        assert real_output.data.fix_version == expected_output.data.fix_version
+
+    finished_state = await test_case.run()
+    verify_result(finished_state.triage_result, test_case.expected_output)

agents/tools/commands.py

@@ -8,7 +8,7 @@
 from beeai_framework.emitter import Emitter
 from beeai_framework.tools import JSONToolOutput, Tool, ToolError, ToolRunOptions
 
-from utils import run_subprocess
+from agents.utils import run_subprocess
 
 TIMEOUT = 10 * 60  # seconds
 ELLIPSIZED_LINES = 200

agents/tools/specfile.py

@@ -15,7 +15,7 @@
 
 from common.constants import BREWHUB_URL
 from common.validators import NonEmptyString
-from utils import get_absolute_path
+from agents.utils import get_absolute_path
 
 
 class GetPackageInfoToolInput(BaseModel):