Skip to content

Commit d76a4bb

Browse files
glehmannglehmann
authored
Merge pull request #636 from xcp-ng/gln/installer-fail-early-lkvk
lib/installer: fail early when installation fails
2 parents e5ef628 + deaf242 commit d76a4bb

1 file changed

Lines changed: 57 additions & 49 deletions

File tree

‎lib/installer.py‎

Lines changed: 57 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77
from lib.commands import ssh
88
from lib.common import wait_for
99

10-
from typing import Any, Self
10+
from typing import Any, Callable, Self
11+
12+
class InstallationFailed(Exception):
13+
pass
1114

1215
class AnswerFile:
1316
def __init__(self, kind: str, /):
@@ -84,76 +87,81 @@ def _defn_to_xml_et(defn: dict[str, Any], *, parent: ET.Element | None = None) -
8487
def poweroff(ip: str) -> None:
8588
ssh(ip, "nohup sh -c 'sleep 2 && poweroff' >/dev/null 2>&1 &")
8689

90+
def wait_for_install_failure_or(ip: str, cmd: Callable[[], bool], msg: str | None = None, timeout_secs=2 * 60) -> None:
91+
def inner():
92+
# Scans the logs for a failure entry formatted as: INFO [<timestamp>] INSTALL FAILED.
93+
# If found, it returns that line and the rest of the file (to capture stack traces/errors).
94+
# If not found, it returns an empty string.
95+
failed = ssh(ip, r"sed -En '/INFO[[:space:]]+\[[-0-9 :]+\] INSTALL FAILED\./,$p' /tmp/install-log")
96+
if failed:
97+
raise InstallationFailed(failed)
98+
return cmd()
99+
return wait_for(inner, msg, timeout_secs=timeout_secs)
100+
87101
def monitor_install(*, ip: str) -> None:
88102
# wait for "yum install" phase to finish
89-
wait_for(lambda: ssh(ip, "grep 'DISPATCH: NEW PHASE: Completing installation' /tmp/install-log",
90-
check=False, simple_output=False,
91-
).returncode == 0,
92-
"Wait for rpm installation to succeed",
93-
timeout_secs=40 * 60) # FIXME too big
103+
wait_for_install_failure_or(
104+
ip, lambda: ssh(ip, "grep 'DISPATCH: NEW PHASE: Completing installation' /tmp/install-log",
105+
check=False, simple_output=False).returncode == 0,
106+
"Wait for rpm installation to succeed", timeout_secs=40 * 60) # FIXME too big
94107

95108
# wait for install to finish
96-
wait_for(lambda: ssh(ip, "grep 'The installation completed successfully' /tmp/install-log",
97-
check=False, simple_output=False,
98-
).returncode == 0,
99-
"Wait for system installation to succeed",
100-
timeout_secs=40 * 60) # FIXME too big
109+
wait_for_install_failure_or(
110+
ip, lambda: ssh(ip, "grep 'The installation completed successfully' /tmp/install-log",
111+
check=False, simple_output=False).returncode == 0,
112+
"Wait for system installation to succeed", timeout_secs=40 * 60) # FIXME too big
113+
114+
wait_for_install_failure_or(
115+
ip, lambda: ssh(ip, "ps a|grep '[0-9]. python /opt/xensource/installer/init'",
116+
check=False, simple_output=False).returncode == 1,
117+
"Wait for installer to terminate")
101118

102-
wait_for(lambda: ssh(ip, "ps a|grep '[0-9]. python /opt/xensource/installer/init'",
103-
check=False, simple_output=False,
104-
).returncode == 1,
105-
"Wait for installer to terminate")
106119

107120
def monitor_upgrade(*, ip: str) -> None:
108121
# wait for "yum install" phase to start
109-
wait_for(lambda: ssh(ip, "grep 'DISPATCH: NEW PHASE: Reading package information' /tmp/install-log",
110-
check=False, simple_output=False,
111-
).returncode == 0,
112-
"Wait for upgrade preparations to finish",
113-
timeout_secs=40 * 60) # FIXME too big
122+
wait_for_install_failure_or(
123+
ip, lambda: ssh(ip,
124+
"grep 'DISPATCH: NEW PHASE: Reading package information' /tmp/install-log",
125+
check=False, simple_output=False).returncode == 0,
126+
"Wait for upgrade preparations to finish", timeout_secs=40 * 60) # FIXME too big
114127

115128
# wait for "yum install" phase to finish
116-
wait_for(lambda: ssh(ip, "grep 'DISPATCH: NEW PHASE: Completing installation' /tmp/install-log",
117-
check=False, simple_output=False,
118-
).returncode == 0,
119-
"Wait for rpm installation to succeed",
120-
timeout_secs=40 * 60) # FIXME too big
129+
wait_for_install_failure_or(
130+
ip, lambda: ssh(ip, "grep 'DISPATCH: NEW PHASE: Completing installation' /tmp/install-log",
131+
check=False, simple_output=False).returncode == 0,
132+
"Wait for rpm installation to succeed", timeout_secs=40 * 60) # FIXME too big
121133

122134
# wait for install to finish
123-
wait_for(lambda: ssh(ip, "grep 'The installation completed successfully' /tmp/install-log",
124-
check=False, simple_output=False,
125-
).returncode == 0,
126-
"Wait for system installation to succeed",
127-
timeout_secs=40 * 60) # FIXME too big
135+
wait_for_install_failure_or(
136+
ip, lambda: ssh(ip, "grep 'The installation completed successfully' /tmp/install-log",
137+
check=False, simple_output=False).returncode == 0,
138+
"Wait for system installation to succeed", timeout_secs=40 * 60) # FIXME too big
128139

129-
wait_for(lambda: ssh(ip, "ps a|grep '[0-9]. python /opt/xensource/installer/init'",
130-
check=False, simple_output=False,
131-
).returncode == 1,
132-
"Wait for installer to terminate")
140+
wait_for_install_failure_or(
141+
ip, lambda: ssh(ip, "ps a|grep '[0-9]. python /opt/xensource/installer/init'",
142+
check=False, simple_output=False).returncode == 1,
143+
"Wait for installer to terminate")
133144

134145
def monitor_restore(*, ip: str) -> None:
135146
# wait for "yum install" phase to start
136-
wait_for(lambda: ssh(ip, "grep 'Restoring backup' /tmp/install-log",
137-
check=False, simple_output=False,
138-
).returncode == 0,
139-
"Wait for data restoration to start",
140-
timeout_secs=40 * 60) # FIXME too big
147+
wait_for_install_failure_or(
148+
ip, lambda: ssh(ip, "grep 'Restoring backup' /tmp/install-log",
149+
check=False, simple_output=False).returncode == 0,
150+
"Wait for data restoration to start", timeout_secs=40 * 60) # FIXME too big
141151

142152
# wait for "yum install" phase to finish
143-
wait_for(lambda: ssh(ip, "grep 'Data restoration complete. About to re-install bootloader.' /tmp/install-log",
144-
check=False, simple_output=False,
145-
).returncode == 0,
146-
"Wait for data restoration to complete",
147-
timeout_secs=40 * 60) # FIXME too big
153+
wait_for_install_failure_or(
154+
ip, lambda: ssh(ip, "grep 'Data restoration complete. About to re-install bootloader.' /tmp/install-log",
155+
check=False, simple_output=False).returncode == 0,
156+
"Wait for data restoration to complete", timeout_secs=40 * 60) # FIXME too big
148157

149158
# The installer will not terminate in restore mode, it
150159
# requires human interaction and does not even log it, so
151160
# wait for last known action log (tested with 8.3b2)
152-
wait_for(lambda: ssh(ip, "grep 'ran .*swaplabel.*rc 0' /tmp/install-log",
153-
check=False, simple_output=False,
154-
).returncode == 0,
155-
"Wait for installer to hopefully finish",
156-
timeout_secs=40 * 60) # FIXME too big
161+
wait_for_install_failure_or(
162+
ip, lambda: ssh(ip, "grep 'ran .*swaplabel.*rc 0' /tmp/install-log",
163+
check=False, simple_output=False).returncode == 0,
164+
"Wait for installer to hopefully finish", timeout_secs=40 * 60) # FIXME too big
157165

158166
# "wait a bit to be extra sure". Yuck.
159167
time.sleep(30)

0 commit comments

Comments
 (0)