Skip to content

Commit 63b7605

Browse files
committed
WIP script automatic update for CI pools
Signed-off-by: Gael Duperrey <gduperrey@vates.tech>
1 parent 999f19e commit 63b7605

File tree

2 files changed

+144
-1
lines changed

2 files changed

+144
-1
lines changed

lib/host.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -558,10 +558,14 @@ def reboot(self, verify=False):
558558
raise
559559
if verify:
560560
wait_for_not(self.is_enabled, "Wait for host down")
561-
wait_for(lambda: not os.system(f"ping -c1 {self.hostname_or_ip} > /dev/null 2>&1"),
561+
wait_for(lambda: not os.system(f"ping -c1 -W2 {self.hostname_or_ip} > /dev/null 2>&1"),
562562
"Wait for host up", timeout_secs=10 * 60, retry_delay_secs=10)
563563
wait_for(lambda: not os.system(f"nc -zw5 {self.hostname_or_ip} 22"),
564564
"Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5)
565+
# wait_for(lambda: commands.local_cmd(["ping", "-c1", "-W2", self.hostname_or_ip], check=False).returncode == 0,
566+
# "Wait for host up", timeout_secs=10 * 60, retry_delay_secs=10)
567+
# wait_for(lambda: commands.local_cmd(["nc", "-zw5", self.hostname_or_ip, "22"], check=False).returncode == 0,
568+
# "Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5)
565569
wait_for(self.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60)
566570

567571
def management_network(self) -> str:

scripts/update_xcnpg.py

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import logging
5+
import os
6+
import sys
7+
import threading
8+
9+
sys.path.append(f"{os.path.abspath(os.path.dirname(__file__))}/..") # noqa
10+
from lib.commands import ssh, SSHCommandFailed
11+
from lib.common import is_uuid
12+
from lib.pool import Pool
13+
14+
logging.basicConfig()
15+
logging.getLogger().setLevel(logging.DEBUG)
16+
logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO)
17+
18+
def is_ip_active(ip):
19+
return not os.system(f"ping -c 3 -W 10 {ip} > /dev/null 2>&1")
20+
21+
def is_ssh_up(ip):
22+
try:
23+
ssh(ip, ['true'], options=['-o "ConnectTimeout 10"'])
24+
return True
25+
except SSHCommandFailed:
26+
# probably not up yet
27+
return False
28+
29+
def is_new_host_ready(ip_address):
30+
try:
31+
output = ssh(ip_address, ['xe', 'host-list', 'enabled=true', '--minimal'])
32+
return is_uuid(output)
33+
except Exception:
34+
return False
35+
36+
def install_updates_reboot(host):
37+
try:
38+
logging.info('Running updates on the host:' + str(host))
39+
# host.install_updates()
40+
logging.info('Reboot the host:' + str(host))
41+
host.reboot(verify=True)
42+
# host.reboot()
43+
# wait_for(lambda: not os.system(f"nc -zw5 {host} 22"),
44+
# "Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5)
45+
return(True)
46+
except Exception:
47+
return False
48+
49+
def main():
50+
parser = argparse.ArgumentParser()
51+
parser.add_argument(
52+
"host",
53+
help="hostname or IP address of the host which is the primary of the pool which will be updated"
54+
)
55+
args = parser.parse_args()
56+
57+
# Verify the host's IP address
58+
logging.info('Testing IP: ' + args.host)
59+
# TODO : How to manage and validate IPv6?
60+
61+
# verify that the host is available (ssh) and that it is indeed the master of a pool
62+
# TODO: Exit the script if there's a problem, or confirm that it's working correctly.
63+
64+
# TODO: replace with an existing function in host or pool
65+
if not is_ssh_up(args.host):
66+
raise Exception(f"Could not SSH into host `{args.host}`")
67+
68+
try:
69+
pool = Pool(args.host) # will fail if host is not XCP-ng or XAPI doesn't respond yet
70+
except Exception as e:
71+
raise Exception(f"Host `{args.host}` isn't ready or isn't an XCP-ng host")
72+
73+
host = pool.master
74+
logging.info('IP of the master is: %s', args.host)
75+
assert host.is_enabled()
76+
# TODO: que faire si pas ok ?
77+
78+
# We build a list of secondaries, if they exist because we will run updates and reboots in parallel on these,
79+
# once the primary one is done and available.
80+
slaves_list = pool.hosts[1:]
81+
logging.info('Here is the slaves list: %s', slaves_list)
82+
83+
# TODO: Adding a feature/setting to add/enable repos if requested?
84+
85+
# We check if there are any updates to run. If not, we exit saying so.
86+
# TODO Add yum clean metadata - create a function ?
87+
update_to_do = host.has_updates()
88+
if not update_to_do:
89+
logging.info('No updates to do.')
90+
return
91+
print(update_to_do)
92+
logging.info('DO I NEED TO MAKE UPDATES: ' + str(update_to_do))
93+
94+
# There are updates to be done, we do them on the main one, then we reboot it.
95+
mupdates = install_updates_reboot(host)
96+
if not mupdates:
97+
logging.info('Errors durring the update process on the master: ' + str(host))
98+
return
99+
100+
logging.info('How many secondaries? ' + str(len(slaves_list)))
101+
102+
# Management of secondary servers, if any.
103+
if len(slaves_list) == 0 :
104+
logging.info('No other servers to process in the pool.')
105+
return
106+
107+
if len(slaves_list) == 1 :
108+
print(slaves_list[0])
109+
logging.info('ONE SERVER TO DO')
110+
# logging.info(slaves_list[0].install_updates())
111+
# slaves_list[0].reboot()
112+
install_updates_reboot(host)
113+
elif len(slaves_list) > 1 :
114+
logging.info('MULTIPLE SERVERS TO DO')
115+
# Several secondary processes. We're using multithreading for updates and reboots, all at the same time.
116+
threads = []
117+
for secondary in slaves_list:
118+
# Add function in the thread below where the desired secondary is passed.
119+
# t = threading.Thread(target=print(secondary))
120+
t = threading.Thread(target=install_updates_reboot, args=(secondary,))
121+
threads.append(t)
122+
123+
for t in threads:
124+
t.start()
125+
126+
for t in threads:
127+
t.join()
128+
129+
130+
# TODO: when everyone is up to date, say it
131+
132+
# TODO:
133+
# For the VMs, we want to take snapshots at the end.
134+
# How do we know where the VM is? Pass the host as a parameter? So add two parameters, one of which is optional for the VMs?
135+
# We need to know if the VM and if the VM will take a snapshot. But we need to know the VM's UUID and the host's IP for it to take the snapshot.
136+
# Local file for testing, read the netbox (to be updated?), read the GRIST table?
137+
138+
if __name__ == '__main__':
139+
main()

0 commit comments

Comments
 (0)