Merge pull request #44 from buildkite/pie-3500-add-execution-level-tagging-for-pytest-test-collector

Add execution level tagging

Author: nprizal

src/buildkite_test_collector/collector/payload.py

@@ -1,6 +1,6 @@
 """Buildkite Test Analytics payload"""
 
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, replace, field
 from typing import Dict, Tuple, Optional, Union, Literal
 from datetime import timedelta
 from uuid import UUID
@@ -107,12 +107,15 @@ def as_json(self, started_at: Instant) -> JsonDict:
 @dataclass(frozen=True)
 class TestData:
     """An individual test execution"""
+    # 8 attributes for this class seems reasonable
+    # pylint: disable=too-many-instance-attributes
     id: UUID
     scope: str
     name: str
     history: TestHistory
     location: Optional[str] = None
     file_name: Optional[str] = None
+    tags: Dict[str,str] = field(default_factory=dict)
     result: Union[TestResultPassed, TestResultFailed,
                   TestResultSkipped, None] = None
 
@@ -133,6 +136,13 @@ def start(cls, id: UUID,
             history=TestHistory(start_at=Instant.now())
         )
 
+    def tag_execution(self, key: str, val: str) -> 'TestData':
+        """Set tag to test execution"""
+        if not isinstance(key, str) or not isinstance(val, str):
+            raise TypeError("Expected string for key and value")
+
+        self.tags[key] = val
+
     def finish(self) -> 'TestData':
         """Set the end_at and duration on this test"""
         if self.is_finished():
@@ -175,6 +185,9 @@ def as_json(self, started_at: Instant) -> JsonDict:
             "history": self.history.as_json(started_at)
         }
 
+        if len(self.tags) > 0:
+            attrs["tags"] = self.tags
+
         if isinstance(self.result, TestResultPassed):
             attrs["result"] = "passed"
 

src/buildkite_test_collector/pytest_plugin/__init__.py

@@ -28,10 +28,13 @@ def pytest_configure(config):
     env = detect_env()
     debug = environ.get("BUILDKITE_ANALYTICS_DEBUG_ENABLED")
 
+    config.addinivalue_line("markers", "execution_tag(key, value): add tag to test execution for Buildkite Test Collector. Both key and value must be a string.")
+
     if env:
         plugin = BuildkitePlugin(Payload.init(env))
         setattr(config, '_buildkite', plugin)
         config.pluginmanager.register(plugin)
+
     elif debug:
         warning("Unable to detect CI environment.  No test analytics will be sent.")
 

src/buildkite_test_collector/pytest_plugin/buildkite_plugin.py

@@ -29,9 +29,18 @@ def pytest_runtest_logstart(self, nodeid, location):
         )
         self.in_flight[nodeid] = test_data
 
+    def pytest_runtest_teardown(self, item):
+        """pytest_runtest_hook hook callback to collect execution_tag"""
+        test_data = self.in_flight.get(item.nodeid)
+
+        if test_data:
+            tags = item.iter_markers("execution_tag")
+            for tag in tags:
+                test_data.tag_execution(tag.args[0], tag.args[1])
+
     def pytest_runtest_logreport(self, report):
         """pytest_runtest_logreport hook callback"""
-        if report.when != 'call':
+        if report.when != 'teardown':
             return
 
         nodeid = report.nodeid

tests/buildkite_test_collector/collector/test_payload.py

@@ -168,3 +168,22 @@ def test_test_data_as_json_when_skipped(skipped_test):
     json = skipped_test.as_json(Instant.now())
 
     assert json["result"] == "skipped"
+
+class TestTestDataTagExecution:
+    def test_test_data_tag_execution(self, successful_test):
+        successful_test.tag_execution("owner", "test-engine")
+        successful_test.tag_execution("python.version", "3.12.3")
+
+        expected_tags = {"owner": "test-engine", "python.version": "3.12.3"}
+
+        assert successful_test.tags == expected_tags
+
+        json = successful_test.as_json(Instant.now())
+        assert json["tags"] == {"owner": "test-engine", "python.version": "3.12.3"}
+
+    def test_test_data_tag_execution_non_string(self, successful_test):
+        with pytest.raises(TypeError):
+            successful_test.tag_execution("feature", True)
+
+        with pytest.raises(TypeError):
+            successful_test.tag_execution(777, "lucky")