Repository navigation
Expand file tree
/
Copy pathsession_process.py
More file actions
228 lines (204 loc) · 8.44 KB
/
Copy pathsession_process.py
File metadata and controls
228 lines (204 loc) · 8.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
#!/usr/bin/env python3
# Copyright (c) 2026 Relax Authors. All Rights Reserved.
"""Linux per-session process ownership, independent of cgroup version.
The supervisor stays alive until its worker, named instance and all adopted
descendants have settled. Only ECHILD is a positive child-release proof.
"""
from __future__ import annotations
import argparse
import ctypes
import json
import os
import signal
import subprocess
import sys
import time
from pathlib import Path
def enable_subreaper() -> None:
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER failed")
def reap_exited_children(worker_pid: int) -> None:
"""Reap daemonized exits while the worker is still verifying instance stop.
Apptainer treats a zombie instance PID as still present. Waiting until the
worker exits would make its cleanup falsely fail. Never steal the Popen
worker's status and never terminate live children here.
"""
for child in Path(f"/proc/self/task/{os.getpid()}/children").read_text().split():
pid = int(child)
if pid == worker_pid:
continue
try:
os.waitpid(pid, os.WNOHANG)
except ChildProcessError:
pass
def reap_children(deadline: float) -> bool:
"""Terminate only our direct, adopted children and wait for ECHILD."""
while time.monotonic() < deadline:
try:
pid, _ = os.waitpid(-1, os.WNOHANG)
except ChildProcessError:
return True
if pid:
continue
# WNOHANG=0 means at least one child is still alive, not clean.
children = Path(f"/proc/self/task/{os.getpid()}/children").read_text().split()
for child in children:
try:
# Parentage is the authority; PID reuse cannot adopt an unrelated process.
status = Path(f"/proc/{child}/status").read_text().splitlines()
ppid = next(int(line.split()[1]) for line in status if line.startswith("PPid:"))
if ppid == os.getpid():
os.kill(int(child), signal.SIGKILL)
except OSError:
pass
time.sleep(0.02)
return False
def instance_absent(spec: dict, deadline: float, *, stop: bool) -> bool:
executable, global_args = spec["apptainer_args"]
name = spec["instance_name"]
def command(argv: list[str]) -> subprocess.CompletedProcess:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError("Instance cleanup deadline elapsed")
return subprocess.run(
argv,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=min(remaining, 12.0),
check=False,
)
if stop:
command([executable, *global_args, "instance", "stop", "--timeout", "10", name])
result = command([executable, *global_args, "instance", "list"])
if result.returncode:
raise RuntimeError("Cannot verify the session's Apptainer instance")
return not any(line.split() and line.split()[0] == name for line in result.stdout.splitlines()[1:])
def supervise(
spec_path: Path, report_path: Path, *, worker_command: list[str] | None = None, cancel_fd: int | None = None
) -> None:
"""Own the cancellation pipe until supervision and cleanup have settled."""
try:
if cancel_fd is not None:
os.set_blocking(cancel_fd, False)
_supervise(spec_path, report_path, worker_command=worker_command, cancel_fd=cancel_fd)
finally:
if cancel_fd is not None:
os.close(cancel_fd)
def _supervise(spec_path: Path, report_path: Path, *, worker_command: list[str] | None, cancel_fd: int | None) -> None:
cancelled_at: float | None = None
parent_pid = os.getppid()
def cancel(_signum, _frame):
nonlocal cancelled_at
if cancelled_at is None:
cancelled_at = time.monotonic()
signal.signal(signal.SIGTERM, cancel)
signal.signal(signal.SIGINT, cancel)
enable_subreaper()
spec = json.loads(spec_path.read_text())
parent_pid = spec.get("server_pid", parent_pid)
def check_cancel() -> None:
if os.getppid() != parent_pid:
cancel(0, None)
if cancel_fd is not None:
try:
if not os.read(cancel_fd, 1):
cancel(0, None)
except BlockingIOError:
pass
check_cancel()
result_path = report_path.with_suffix(".worker.json")
command = worker_command or [
sys.executable,
str(Path(__file__).with_name("agent_server.py")),
"--session-worker",
str(spec_path),
str(result_path),
]
worker = None if cancelled_at is not None else subprocess.Popen(command, start_new_session=True)
sent_term = False
supervisor_error = ""
try:
while worker is not None and worker.poll() is None:
reap_exited_children(worker.pid)
check_cancel()
if cancelled_at is not None:
try:
if not sent_term:
os.killpg(worker.pid, signal.SIGTERM)
sent_term = True
if time.monotonic() - cancelled_at >= 12.0:
os.killpg(worker.pid, signal.SIGKILL)
except ProcessLookupError:
pass
time.sleep(0.02)
except BaseException as exc:
supervisor_error = f"{type(exc).__name__}: {exc}"
finally:
if worker is not None:
if worker.poll() is None:
try:
os.killpg(worker.pid, signal.SIGKILL)
except ProcessLookupError:
pass
worker.wait()
deadline = (cancelled_at + 35.0) if cancelled_at is not None else time.monotonic() + 25.0
error = supervisor_error
absent = False
try:
absent = instance_absent(spec, deadline, stop=worker is not None)
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
children_reaped = reap_children(deadline)
if children_reaped:
try:
absent = instance_absent(spec, deadline, stop=False)
except Exception as exc:
absent = False
error = f"{type(exc).__name__}: {exc}"
# The verification subprocess was also wait()ed, but verify the invariant again.
children_reaped = reap_children(deadline)
result = {"info": {}, "error": "Session cancelled before worker launch"}
if worker is not None:
result["error"] = f"Session worker exited with code {worker.returncode} without a valid result"
try:
candidate = json.loads(result_path.read_text())
if not isinstance(candidate, dict) or not isinstance(candidate.get("info"), dict):
raise ValueError("Worker result has invalid info")
if not isinstance(candidate.get("error"), str):
raise ValueError("Worker result has invalid error")
result = candidate
except (OSError, ValueError) as exc:
result["error"] += f": {type(exc).__name__}: {exc}"
if not result["error"]:
if worker.returncode != 0:
result["error"] = f"Session worker exited with code {worker.returncode}"
elif cancelled_at is not None:
result["error"] = "Session supervisor cancelled the worker"
elif supervisor_error:
result["error"] = supervisor_error
report = {
**result,
"nonce": spec["release_nonce"],
"client_session_id": spec["record"]["session_id"],
"resources_released": absent and children_reaped,
"children_reaped": children_reaped,
"instance_absent": absent,
"cleanup_error": error,
}
temporary = report_path.with_suffix(".tmp")
try:
temporary.write_text(json.dumps(report))
temporary.replace(report_path)
finally:
# A deadline or failed result write is not permission to orphan descendants.
while not children_reaped:
children_reaped = reap_children(time.monotonic() + 5.0)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("spec", type=Path)
parser.add_argument("report", type=Path)
parser.add_argument("--cancel-fd", type=int)
arguments = parser.parse_args()
supervise(arguments.spec, arguments.report, cancel_fd=arguments.cancel_fd)