[Tests] Ensure that all subprocesses are terminated on cleanup (project-chip#42380)

* Ensure that all subprocesses are terminated on cleanup

This reduces the chance of leaving orphan processes on exit.

Signed-off-by: Marek Pikuła <[email protected]>

* Fix LinuxNamespacedExecutor.run() override

Signed-off-by: Marek Pikuła <[email protected]>

* Fix process termination order

Signed-off-by: Marek Pikuła <[email protected]>

* Reduce indentation levels in terminate

Signed-off-by: Marek Pikuła <[email protected]>

* Remove unused import

Signed-off-by: Marek Pikuła <[email protected]>

* Rename CLEANUP_TIMEOUT to CLEANUP_TIMEOUT_S

Signed-off-by: Marek Pikuła <[email protected]>

* Increase exit timeout to 5s

Signed-off-by: Marek Pikuła <[email protected]>

* Handle possible race condition in process termination flow

Signed-off-by: Marek Pikuła <[email protected]>

---------

Signed-off-by: Marek Pikuła <[email protected]>

Author: MarekPikula

scripts/tests/chiptest/linux.py

@@ -198,11 +198,12 @@ def terminate(self):
 
 class LinuxNamespacedExecutor(Executor):
     def __init__(self, ns: IsolatedNetworkNamespace):
+        super().__init__()
         self.ns = ns
 
     def run(self, subproc: SubprocessInfo, stdin=None, stdout=None, stderr=None):
         wrapped = subproc.wrap_with("ip", "netns", "exec", self.ns.netns_for_subprocess(subproc))
-        return subprocess.Popen(wrapped.to_cmd(), stdin=stdin, stdout=stdout, stderr=stderr)
+        return super().run(wrapped, stdin=stdin, stdout=stdout, stderr=stderr)
 
 
 class DBusTestSystemBus(subprocess.Popen):

scripts/tests/chiptest/runner.py

@@ -22,6 +22,7 @@
 import subprocess
 import threading
 import typing
+from contextlib import suppress
 from dataclasses import dataclass
 from typing import Literal
 
@@ -148,8 +149,50 @@ def to_cmd(self) -> typing.List[str]:
 
 
 class Executor:
+    CLEANUP_TIMEOUT_S = 5
+
+    def __init__(self) -> None:
+        self._processes: queue.Queue[subprocess.Popen[bytes]] = queue.Queue()
+
     def run(self, subproc: SubprocessInfo, stdin=None, stdout=None, stderr=None):
-        return subprocess.Popen(subproc.to_cmd(), stdin=stdin, stdout=stdout, stderr=stderr)
+        self._processes.put(process := subprocess.Popen(subproc.to_cmd(), stdin=stdin, stdout=stdout, stderr=stderr))
+        return process
+
+    def terminate(self) -> None:
+        while True:
+            # Get process from the queue.
+            try:
+                process = self._processes.get_nowait()
+            except queue.Empty:
+                break
+
+            # Check if process already exited.
+            if process.poll() is not None:
+                continue
+            cmd = str(process.args)
+
+            # SIGTERM
+            log.debug('Terminating leftover process "%s"', cmd)
+            try:
+                process.terminate()
+            except OSError:
+                # Can occur in case of race condition when process exits between poll and terminate.
+                continue
+            with suppress(subprocess.TimeoutExpired):
+                process.wait(self.CLEANUP_TIMEOUT_S)
+                continue
+
+            # SIGKILL
+            log.warning('Failed to terminate the process "%s". Killing instead', cmd)
+            try:
+                process.kill()
+            except OSError:
+                continue
+            with suppress(subprocess.TimeoutExpired):
+                process.wait(self.CLEANUP_TIMEOUT_S)
+                continue
+
+            log.error('Failed to kill process "%s". It may become a zombie', cmd)
 
 
 class Runner:

scripts/tests/run_test_suite.py

@@ -438,6 +438,7 @@ def build_app(arg_value, kind: str, key: str):
 
     def cleanup():
         apps_register.uninit()
+        executor.terminate()
         if sys.platform == 'linux':
             if ble_wifi:
                 wifi.terminate()