diff --git a/.cirrus.yml b/.cirrus.yml index b0c0baf9..630f7150 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,12 +3,13 @@ low_scale_task: compute_engine_instance: matrix: - image_project: fedora-cloud - image: family/fedora-cloud-38 + image: family/fedora-cloud-43-x86-64 - image_project: ubuntu-os-cloud image: family/ubuntu-2404-lts-amd64 platform: linux - memory: 8G + memory: 16G disk: 40 + cpu: 4 env: DEPENDENCIES: git ansible podman diff --git a/Dockerfile b/Dockerfile index 5f16e5f0..e63052b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM ovn/ovn-multi-node ARG SSH_KEY +COPY utils/helpers.sh /scripts/helpers.sh COPY ovn-tester /ovn-tester RUN mkdir -p /root/.ssh/ @@ -9,8 +10,10 @@ COPY $SSH_KEY /root/.ssh/ COPY ovn-fake-multinode-utils/process-monitor.py /tmp/ +RUN /bin/bash -c ". /scripts/helpers.sh; install_latest_python" + # This variable is needed on systems where global python's # environment is marked as "Externally managed" (PEP 668) to allow pip # installation of "global" packages. ENV PIP_BREAK_SYSTEM_PACKAGES=1 -RUN pip3 install -r /ovn-tester/requirements.txt +RUN python3 -m pip install -r /ovn-tester/requirements.txt diff --git a/do.sh b/do.sh index 5650de71..55a36aff 100755 --- a/do.sh +++ b/do.sh @@ -30,70 +30,7 @@ ovn_tester=${topdir}/ovn-tester EXTRA_OPTIMIZE=${EXTRA_OPTIMIZE:-no} USE_OVSDB_ETCD=${USE_OVSDB_ETCD:-no} -# We want values from both the `ID` and `ID_LIKE` fields to ensure successful -# categorization. The shell will happily accept both spaces and newlines as -# separators: -# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 -DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"') - -DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"') - -function is_rpm_based() { - for id in $DISTRO_IDS; do - case $id in - centos* | rhel* | fedora*) - true - return - ;; - esac - done - false -} - -function is_rhel() { - for id in $DISTRO_IDS; do - case $id in - centos* | rhel*) - true - return - ;; - esac - done - false -} - -function is_fedora() { - for id in $DISTRO_IDS; do - case $id in - fedora*) - true - return - ;; - esac - done - false -} - -function is_deb_based() { - for id in $DISTRO_IDS; do - case $id in - debian* | ubuntu*) - true - return - ;; - esac - done - false -} - -function die() { - echo $1 - exit 1 -} - -function die_distro() { - die "Unable to determine distro type, rpm- and deb-based are supported." -} +source $topdir/utils/helpers.sh function generate() { # Make sure rundir exists. @@ -107,9 +44,7 @@ function generate() { function install_deps_local_rpm() { echo "-- Installing local dependencies" - yum install redhat-lsb-core datamash \ - python3-netaddr python3 python3-devel \ - podman \ + yum install datamash python3-netaddr python3 python3-devel podman \ --skip-broken -y } diff --git a/ovn-fake-multinode-utils/generate-hosts.py b/ovn-fake-multinode-utils/generate-hosts.py index 7a320767..69bf5d16 100755 --- a/ovn-fake-multinode-utils/generate-hosts.py +++ b/ovn-fake-multinode-utils/generate-hosts.py @@ -19,22 +19,26 @@ def usage(name): ) -def generate_node_string(host: str, **kwargs) -> None: +def generate_node_string( + host: str, internal_iface: str | None, **kwargs +) -> None: + if internal_iface is not None: + kwargs['internal_iface'] = internal_iface args = ' '.join(f"{key}={value}" for key, value in kwargs.items()) print(f"{host} {args}") -def generate_node(config: Dict, internal_iface: str, **kwargs) -> None: +def generate_node(config: Dict, internal_iface: str | None, **kwargs) -> None: host: str = config['name'] internal_iface = config.get('internal-iface', internal_iface) generate_node_string( host, - internal_iface=internal_iface, + internal_iface, **kwargs, ) -def generate_tester(config: Dict, internal_iface: str) -> None: +def generate_tester(config: Dict, internal_iface: str | None) -> None: ssh_key = config["ssh_key"] ssh_key = Path(ssh_key).resolve() generate_node( @@ -45,7 +49,7 @@ def generate_tester(config: Dict, internal_iface: str) -> None: ) -def generate_nodes(nodes_config: Dict, internal_iface: str, **kwargs): +def generate_nodes(nodes_config: Dict, internal_iface: str | None, **kwargs): for node_config in nodes_config: host, node_config = helpers.get_node_config(node_config) iface = node_config.get('internal-iface', internal_iface) @@ -62,7 +66,7 @@ def generate(input_file: str, target: str, repo: str, branch: str) -> None: user = config.get('user', 'root') prefix = config.get('prefix', 'ovn-scale') tester_config = config['tester-node'] - internal_iface = config['internal-iface'] + internal_iface = config.get('internal-iface') print('[tester_hosts]') generate_tester(tester_config, internal_iface) diff --git a/ovn-tester/ovn_sandbox.py b/ovn-tester/ovn_sandbox.py index 5103d54b..937946cf 100644 --- a/ovn-tester/ovn_sandbox.py +++ b/ovn-tester/ovn_sandbox.py @@ -4,6 +4,7 @@ from io import StringIO from ovn_exceptions import SSHError +from typing import List log = logging.getLogger(__name__) @@ -79,6 +80,15 @@ def ensure_channel(self) -> None: # Checking + consuming all the unwanted output from the shell. self.run(cmd="echo Hello", stdout=stdout, raise_on_error=True) + # Splits 'out' by universal newline characters with the addition that it + # considers the terminal String Terminator character '\x1b\' as a newline. + @staticmethod + def split_channel_output(out: str) -> List[str]: + lines = [] + for line in out.splitlines(): + lines += line.split('\x1b\\') + return lines + def run( self, cmd: str = "", @@ -109,7 +119,7 @@ def run( while '++++end' not in out.splitlines(): out = out + self.channel.recv(10240).decode() except (paramiko.buffered_pipe.PipeTimeout, socket.timeout): - if '++++start' not in out.splitlines(): + if '++++start' not in self.split_channel_output(out): out = '++++start\n' + out out = out + '\n42\n++++end' timed_out = True @@ -120,7 +130,7 @@ def run( pass # Splitting and removing all lines with terminal control chars. - out = out.splitlines() + out = self.split_channel_output(out) start = out.index('++++start') + 1 end = out.index('++++end') - 1 exit_status = int(out[end]) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 47e7ccd7..2eafc7b4 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -64,12 +64,8 @@ class CentralNode(Node): def __init__(self, phys_node, container: str, mgmt_ip: str, protocol: str): super().__init__(phys_node, container, mgmt_ip, protocol) - def start( - self, cluster_cfg: ClusterConfig, update_election_timeout: bool = False - ): + def start(self, cluster_cfg: ClusterConfig): log.info('Configuring central node') - if cluster_cfg.clustered_db and update_election_timeout: - self.set_raft_election_timeout(cluster_cfg.raft_election_to) self.enable_trim_on_compaction() self.set_northd_threads(cluster_cfg.northd_threads) if cluster_cfg.log_txns_db: @@ -83,20 +79,18 @@ def set_northd_threads(self, n_threads: int): f'{n_threads}' ) - def set_raft_election_timeout(self, timeout_s: int): - for timeout in range(1000, (timeout_s + 1) * 1000, 1000): - log.info(f'Setting RAFT election timeout to {timeout}ms') - self.run( - cmd=f'ovs-appctl -t ' - f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer ' - f'OVN_Northbound {timeout}' - ) - self.run( - cmd=f'ovs-appctl -t ' - f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer ' - f'OVN_Southbound {timeout}' - ) - time.sleep(1) + def set_raft_election_timeout(self, timeout_ms: int): + log.info(f'Setting RAFT election timeout to {timeout_ms}ms') + self.run( + cmd=f'ovs-appctl -t ' + f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer ' + f'OVN_Northbound {timeout_ms}' + ) + self.run( + cmd=f'ovs-appctl -t ' + f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer ' + f'OVN_Southbound {timeout_ms}' + ) def enable_trim_on_compaction(self): log.info('Setting DB trim-on-compaction') @@ -346,12 +340,22 @@ def add_workers(self, worker_nodes): def prepare_test(self): self.start() + def set_raft_election_timeout(self): + if not self.cluster_cfg.clustered_db: + return + + log.info('Setting raft cluster election timeout') + for timeout_ms in range( + 1000, (self.cluster_cfg.raft_election_to * 1000), 1000 + ): + for c in self.central_nodes: + c.set_raft_election_timeout(timeout_ms) + time.sleep(1) + def start(self): + self.set_raft_election_timeout() for c in self.central_nodes: - c.start( - self.cluster_cfg, - update_election_timeout=(c is self.central_nodes[0]), - ) + c.start(self.cluster_cfg) nb_conn = self.get_nb_connection_string() inactivity_probe = self.cluster_cfg.db_inactivity_probe // 1000 self.nbctl = ovn_utils.OvnNbctl( diff --git a/physical-deployments/ci.yml b/physical-deployments/ci.yml index 34bf2beb..34cab7b6 100644 --- a/physical-deployments/ci.yml +++ b/physical-deployments/ci.yml @@ -1,5 +1,3 @@ -internal-iface: lo - central-nodes: - diff --git a/utils/helpers.sh b/utils/helpers.sh new file mode 100644 index 00000000..edfda42a --- /dev/null +++ b/utils/helpers.sh @@ -0,0 +1,84 @@ +# We want values from both the `ID` and `ID_LIKE` fields to ensure successful +# categorization. The shell will happily accept both spaces and newlines as +# separators: +# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 +DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"') + +DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"') + +function is_rpm_based() { + for id in $DISTRO_IDS; do + case $id in + centos* | rhel* | fedora*) + true + return + ;; + esac + done + false +} + +function is_rhel() { + for id in $DISTRO_IDS; do + case $id in + centos* | rhel*) + true + return + ;; + esac + done + false +} + +function is_fedora() { + for id in $DISTRO_IDS; do + case $id in + fedora*) + true + return + ;; + esac + done + false +} + +function is_deb_based() { + for id in $DISTRO_IDS; do + case $id in + debian* | ubuntu*) + true + return + ;; + esac + done + false +} + +# Installs the most recent available python3 and pip version for the +# current distro. +function install_latest_python() { + if is_rhel + then + py_pkg=$(dnf list available 'python3.[0-9][0-9]' -q 2> /dev/null | + grep -o '^python3.[0-9][0-9]' | sort -V | tail -n 1) && + dnf install -y ${py_pkg}-pip && + alternatives --install /usr/bin/python3 python3 /usr/bin/$py_pkg 100 + elif is_fedora + then + dnf install -y python3-pip + elif is_deb_based + then + apt install -y python3-pip + else + die_distro + fi +} + +function die() { + echo $1 + exit 1 +} + +function die_distro() { + die "Unable to determine distro type, rpm- and deb-based are supported." +}