Skip to content

Commit 078f0d3

Browse files
committed
Merge branch 'mr/pmderodat/notify-test-results' into 'master'
Introduce a notification system (`--notify-events` testsuite argument) Closes #34 See merge request it/e3-testsuite!53
2 parents ac0d4ca + 61bd81c commit 078f0d3

11 files changed

Lines changed: 622 additions & 6 deletions

File tree

.flake8

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[flake8]
22
exclude = .git,__pycache__,build,dist,.tox
3-
ignore = A003, C901, E203, E266, E501, W503,D100,D101,D102,D102,D103,D104,D105,D106,D107,D203,D403,D213,B028,B906,B907,C420,E704
3+
ignore = A003, C901, E203, E266, E501, W503,D100,D101,D102,D102,D103,D104,D105,D106,D107,D202,D203,D403,D213,B028,B906,B907,C420,E704
44
# line length is intentionally set to 80 here because black uses Bugbear
55
# See https://github.com/psf/black/blob/master/README.md#line-length for more details
66
max-line-length = 80

NEWS.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
27.3 (Not released yet)
22
=================
33

4+
* Introduce a notification system (`--notify-events` testsuite argument).
45
* Add a `--list-json=FILE` CLI argument to dump the list of tests in a
56
testsuite to a file in JSON format.
67

src/e3/testsuite/__init__.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
from e3.main import Main
3535
from e3.os.process import quote_arg
3636
from e3.testsuite._helpers import deprecated
37+
import e3.testsuite.event_notifications as event_notifications
3738
from e3.testsuite.report.gaia import (
3839
GAIAResultFiles,
3940
dump_gaia_report,
@@ -357,6 +358,21 @@ def testsuite_main(self, args: Optional[List[str]] = None) -> int:
357358
" the environement that existed when this testsuite was run"
358359
" to produce a given testsuite report.",
359360
)
361+
output_group.add_argument(
362+
"--notify-events",
363+
help="If provided, run the given command each time a event that is"
364+
" tracked by the testsuite happens. See the documentation for the"
365+
" ``e3.testsuite.event_notifications`` module for more information"
366+
" about events and notification commands. Base arguments for"
367+
" notification commands are specified using the POSIX shell"
368+
" syntax. If the commands starts with `python:`, then the expected"
369+
" format is `python:MODULE:CALLABLE`. The Python module MODULE is"
370+
" imported, and its CALLABLE` attribute is fetched. It is called"
371+
" with one positional argument: the testsuite instance, and must"
372+
" return another callable that will be invoked for each"
373+
" notification event, with one positional argument: the"
374+
" corresponding TestNotification instance.",
375+
)
360376

361377
exec_group = parser.add_argument_group(
362378
title="execution control arguments"
@@ -533,6 +549,12 @@ def testsuite_main(self, args: Optional[List[str]] = None) -> int:
533549
self.env.working_dir = self.working_dir
534550
self.env.options = self.main.args
535551

552+
try:
553+
self.event_notifier = event_notifications.EventNotifier(
554+
self, self.main.args.notify_events
555+
)
556+
except event_notifications.InvalidNotifyCommand:
557+
return 1
536558
self.running_status = RunningStatus(
537559
os.path.join(self.output_dir, "status"),
538560
self.main.args.status_update_interval,
@@ -826,6 +848,8 @@ def add_test(self, dag: DAG, parsed_test: ParsedTest) -> None:
826848
message=str(e),
827849
tb=traceback.format_exc(),
828850
)
851+
else:
852+
self.event_notifier.notify_test_queue(instance.test_name)
829853

830854
def dump_testsuite_result(self) -> None:
831855
"""Log a summary of test results.
@@ -879,6 +903,12 @@ def collect_result(self, fragment: TestFragment) -> None:
879903
while fragment.result_queue:
880904
self.add_result(fragment.result_queue.pop())
881905

906+
# Send a notification if this was the last fragment to run for this
907+
# test driver. This is done here rather than in test fragment's
908+
# collect_result so that test result events are notified before test
909+
# end events.
910+
fragment.maybe_notify_ended()
911+
882912
# Now that this fragment is completed, make sure to remove all
883913
# references to its test drivers so that it can be garbage collected.
884914
# This is necessary to keep memory consumption under control for big
@@ -966,6 +996,12 @@ def format_log(log: Log) -> str:
966996
log_line += format_log(full_result.log)
967997
logger.info(log_line)
968998

999+
self.event_notifier.notify_test_result(
1000+
item.test_name,
1001+
item.result,
1002+
os.path.join(self.report_index.results_dir, item.filename),
1003+
)
1004+
9691005
def add_test_error(
9701006
self, test_name: str, message: str, tb: Optional[str] = None
9711007
) -> None:
@@ -986,6 +1022,7 @@ def add_test_error(
9861022

9871023
self.add_result(
9881024
ResultQueueItem(
1025+
test_name,
9891026
result.summary,
9901027
result.save(self.output_dir),
9911028
traceback.format_stack(),
@@ -1081,6 +1118,7 @@ def filter_key(k: str) -> str:
10811118
{filter_key(k): self.return_values[k] for k in predecessors},
10821119
notify_end,
10831120
self.running_status,
1121+
self.event_notifier,
10841122
)
10851123

10861124
def collect_result(job: Job) -> bool:
@@ -1126,6 +1164,7 @@ def job_factory(
11261164
data.name,
11271165
slot,
11281166
self.running_status,
1167+
self.event_notifier,
11291168
self.env,
11301169
)
11311170

src/e3/testsuite/driver/__init__.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ class ResultQueueItem:
3030
needed for the various stages of this pipeline.
3131
"""
3232

33+
test_name: str
34+
"""Name of the test that created this test result."""
35+
3336
result: TestResultSummary
3437
"""Summary for this test result."""
3538

@@ -99,6 +102,18 @@ def __init__(self, env: e3.env.Env, test_env: Dict[str, Any]) -> None:
99102
testsuite report.
100103
"""
101104

105+
self.execution_started: bool = False
106+
"""
107+
Whether the execution of at least one fragment for this test driver has
108+
started.
109+
"""
110+
111+
self.pending_fragments: set[str] = set()
112+
"""
113+
Set of UIDs for fragments whose execution has not yet completed for
114+
this test driver.
115+
"""
116+
102117
def push_result(self, result: Optional[TestResult] = None) -> None:
103118
"""Push a result to the testsuite.
104119
@@ -117,6 +132,7 @@ def push_result(self, result: Optional[TestResult] = None) -> None:
117132

118133
self.result_queue.append(
119134
ResultQueueItem(
135+
self.test_name,
120136
result.summary,
121137
result.save(self.env.output_dir),
122138
traceback.format_stack(),
@@ -165,6 +181,7 @@ class called ``name``.
165181
callback_by_name=callback_by_name,
166182
)
167183

184+
self.pending_fragments.add(fragment.uid)
168185
dag.update_vertex(
169186
vertex_id=fragment.uid,
170187
data=fragment,
Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
from __future__ import annotations
2+
3+
"""
4+
External notifications for events that occur during testsuite execution.
5+
6+
Each time a tracked event occurs during the testsuite execution (see the
7+
``TestNotification`` subclasses below for the kinds of events that are
8+
tracked), a user-provided command is executed with arguments that give details
9+
about the event that occurred.
10+
"""
11+
12+
import abc
13+
import dataclasses
14+
import importlib
15+
import logging
16+
import re
17+
import shlex
18+
import subprocess
19+
from typing import Callable, TYPE_CHECKING
20+
21+
from e3.testsuite.result import TestResultSummary
22+
23+
if TYPE_CHECKING:
24+
from e3.testsuite import TestsuiteCore
25+
26+
27+
logger = logging.getLogger("testsuite")
28+
29+
30+
@dataclasses.dataclass(frozen=True)
31+
class TestNotification:
32+
"""Notification for a given test."""
33+
34+
test_name: str
35+
"""
36+
Name of the test that this notification refers to.
37+
"""
38+
39+
@abc.abstractmethod
40+
def to_args(self) -> list[str]:
41+
raise NotImplementedError
42+
43+
44+
@dataclasses.dataclass(frozen=True)
45+
class TestQueueNotification(TestNotification):
46+
"""Notification to signal that a test was queued."""
47+
48+
def to_args(self) -> list[str]:
49+
return ["--queue", self.test_name]
50+
51+
52+
@dataclasses.dataclass(frozen=True)
53+
class TestStartNotification(TestNotification):
54+
"""Notification to signal that the execution of a test has just started."""
55+
56+
def to_args(self) -> list[str]:
57+
return ["--start", self.test_name]
58+
59+
60+
@dataclasses.dataclass(frozen=True)
61+
class TestResultNotification(TestNotification):
62+
"""Notification to signal the addition of a new test result."""
63+
64+
result: TestResultSummary
65+
"""Summary for the added test result."""
66+
67+
yaml_result_filename: str
68+
"""Absolute filename for the YAML file that stores the test result.
69+
70+
This YAML file must be loaded through the ``TestResult.load`` static
71+
method.
72+
"""
73+
74+
def to_args(self) -> list[str]:
75+
return ["--result", self.test_name, self.yaml_result_filename]
76+
77+
78+
@dataclasses.dataclass(frozen=True)
79+
class TestEndNotification(TestNotification):
80+
"""Notification to signal that the execution of a test has just ended."""
81+
82+
def to_args(self) -> list[str]:
83+
return ["--end", self.test_name]
84+
85+
86+
class InvalidNotifyCommand(Exception):
87+
pass
88+
89+
90+
class EventNotifier:
91+
"""Abstraction to send notifications."""
92+
93+
python_cmd_re = re.compile(
94+
r"(?P<module>[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*)"
95+
r":"
96+
r"(?P<callback>[a-zA-Z_][a-zA-Z0-9_]*)"
97+
)
98+
99+
def __init__(self, testsuite: TestsuiteCore, notify_cmd: str | None):
100+
"""Initialize an EventNotifier.
101+
102+
:param testsuite: Testsuite instance for which this event notifier is
103+
created.
104+
:param notify_cmd: Base command line arguments for the notification
105+
command. Notification specifics are added as extra arguments for
106+
each notification that is sent. If ``None``, notifications are
107+
discarded.
108+
109+
This raises an `InvalidNotifyCommand` if `notify_cmd` cannot be
110+
decoded/loaded.
111+
"""
112+
self.notify_callback: Callable[[TestNotification], None] | None = None
113+
self.notify_cmd: list[str] | None = None
114+
self._parse_notify_cmd(testsuite, notify_cmd)
115+
116+
def _parse_notify_cmd(
117+
self,
118+
testsuite: TestsuiteCore,
119+
cmd: str | None,
120+
) -> None:
121+
"""Decode the given notify command."""
122+
if cmd is None:
123+
return
124+
125+
python_prefix = "python:"
126+
if cmd.startswith(python_prefix):
127+
m = self.python_cmd_re.match(cmd[len(python_prefix) :])
128+
if not m:
129+
logger.exception(f"Wrong format: {cmd!r}")
130+
raise InvalidNotifyCommand()
131+
132+
module_name = m.group("module")
133+
callback_name = m.group("callback")
134+
135+
try:
136+
module = importlib.import_module(module_name)
137+
except ImportError as exc:
138+
logger.exception(f"Cannot load module {module_name!r}")
139+
raise InvalidNotifyCommand from exc
140+
141+
try:
142+
callback = getattr(module, callback_name)
143+
except AttributeError as exc:
144+
logger.exception(f"Cannot load callback {callback_name!r}")
145+
raise InvalidNotifyCommand from exc
146+
147+
try:
148+
self.notify_callback = callback(testsuite)
149+
except Exception as exc:
150+
logger.exception("Cannot create notification callback")
151+
raise InvalidNotifyCommand from exc
152+
return
153+
154+
self.notify_cmd = shlex.split(cmd)
155+
156+
def notify(self, notification: TestNotification) -> None:
157+
"""Send a given notification."""
158+
if self.notify_callback is not None:
159+
try:
160+
self.notify_callback(notification)
161+
except Exception:
162+
logger.exception(
163+
"Error while executing the event notification callback"
164+
)
165+
elif self.notify_cmd is not None:
166+
try:
167+
subprocess.check_call(
168+
self.notify_cmd + notification.to_args(),
169+
stdin=subprocess.DEVNULL,
170+
)
171+
except (OSError, subprocess.SubprocessError):
172+
logger.exception(
173+
"Error while running the event notification command"
174+
)
175+
176+
def notify_test_queue(self, test_name: str) -> None:
177+
self.notify(TestQueueNotification(test_name))
178+
179+
def notify_test_start(self, test_name: str) -> None:
180+
self.notify(TestStartNotification(test_name))
181+
182+
def notify_test_result(
183+
self,
184+
test_name: str,
185+
result: TestResultSummary,
186+
yaml_result_filename: str,
187+
) -> None:
188+
self.notify(
189+
TestResultNotification(test_name, result, yaml_result_filename)
190+
)
191+
192+
def notify_test_end(self, test_name: str) -> None:
193+
self.notify(TestEndNotification(test_name))

0 commit comments

Comments
 (0)