From e9391e52a8e2d4fdf72e7430b0aad9c84fa14962 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Mon, 12 Jan 2026 12:31:06 +0100 Subject: [PATCH 1/7] ovn_workload: Try setting raft timeout on all members. The command only works through the leader and the first node might not be the elected leader. In such cases we get: ovs-appctl: /run/ovn/ovnsb_db.ctl: server returned an error ovn_workload |INFO| Setting RAFT election timeout to 11000ms ovn_sandbox |INFO| Logging command: ssh ovn-central-az0-1 "ovs-appctl -t /run/ovn/ovnnb_db.ctl cluster/change-election-timer OVN_Northbound 11000" ovn_sandbox |INFO| Result: ['election timer must be changed through leader.', 'ovs-appctl: /run/ovn/ovnnb_db.ctl: server returned an error'], Exit status: 2 ovn_sandbox |INFO| --- election timer must be changed through leader. Signed-off-by: Dumitru Ceara --- ovn-tester/ovn_workload.py | 50 ++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/ovn-tester/ovn_workload.py b/ovn-tester/ovn_workload.py index 47e7ccd7..2eafc7b4 100644 --- a/ovn-tester/ovn_workload.py +++ b/ovn-tester/ovn_workload.py @@ -64,12 +64,8 @@ class CentralNode(Node): def __init__(self, phys_node, container: str, mgmt_ip: str, protocol: str): super().__init__(phys_node, container, mgmt_ip, protocol) - def start( - self, cluster_cfg: ClusterConfig, update_election_timeout: bool = False - ): + def start(self, cluster_cfg: ClusterConfig): log.info('Configuring central node') - if cluster_cfg.clustered_db and update_election_timeout: - self.set_raft_election_timeout(cluster_cfg.raft_election_to) self.enable_trim_on_compaction() self.set_northd_threads(cluster_cfg.northd_threads) if cluster_cfg.log_txns_db: @@ -83,20 +79,18 @@ def set_northd_threads(self, n_threads: int): f'{n_threads}' ) - def set_raft_election_timeout(self, timeout_s: int): - for timeout in range(1000, (timeout_s + 1) * 1000, 1000): - log.info(f'Setting RAFT election timeout to {timeout}ms') - self.run( - cmd=f'ovs-appctl -t ' - f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer ' - f'OVN_Northbound {timeout}' - ) - self.run( - cmd=f'ovs-appctl -t ' - f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer ' - f'OVN_Southbound {timeout}' - ) - time.sleep(1) + def set_raft_election_timeout(self, timeout_ms: int): + log.info(f'Setting RAFT election timeout to {timeout_ms}ms') + self.run( + cmd=f'ovs-appctl -t ' + f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer ' + f'OVN_Northbound {timeout_ms}' + ) + self.run( + cmd=f'ovs-appctl -t ' + f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer ' + f'OVN_Southbound {timeout_ms}' + ) def enable_trim_on_compaction(self): log.info('Setting DB trim-on-compaction') @@ -346,12 +340,22 @@ def add_workers(self, worker_nodes): def prepare_test(self): self.start() + def set_raft_election_timeout(self): + if not self.cluster_cfg.clustered_db: + return + + log.info('Setting raft cluster election timeout') + for timeout_ms in range( + 1000, (self.cluster_cfg.raft_election_to * 1000), 1000 + ): + for c in self.central_nodes: + c.set_raft_election_timeout(timeout_ms) + time.sleep(1) + def start(self): + self.set_raft_election_timeout() for c in self.central_nodes: - c.start( - self.cluster_cfg, - update_election_timeout=(c is self.central_nodes[0]), - ) + c.start(self.cluster_cfg) nb_conn = self.get_nb_connection_string() inactivity_probe = self.cluster_cfg.db_inactivity_probe // 1000 self.nbctl = ovn_utils.OvnNbctl( From 3fc21c158cf594a28b7e56fa6bea5243586e8946 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 8 Jan 2026 14:47:41 +0100 Subject: [PATCH 2/7] do.sh: Don't install redhat-lsb-core. It's not really needed (and not available on newer Fedora). Signed-off-by: Dumitru Ceara --- do.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/do.sh b/do.sh index 5650de71..f4c6bb60 100755 --- a/do.sh +++ b/do.sh @@ -107,9 +107,7 @@ function generate() { function install_deps_local_rpm() { echo "-- Installing local dependencies" - yum install redhat-lsb-core datamash \ - python3-netaddr python3 python3-devel \ - podman \ + yum install datamash python3-netaddr python3 python3-devel podman \ --skip-broken -y } From b3d9b7537ed898f292b0e9ed95dece8a1b076bf6 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 8 Jan 2026 13:22:50 +0100 Subject: [PATCH 3/7] Dockerfile: Use the latest available python version. Otherwise other packages we need might fail to install. E.g., ovsdbapp requires Python >= 3.10: ERROR: Package 'ovsdbapp' requires a different Python: 3.9.25 not in '>=3.10' Error: building at STEP "RUN pip3 install -r /ovn-tester/requirements.txt": while running runtime: exit status 1 For RHEL we query the available packages and choose the most recent python3 version and the corresponding python3.x-pip package. For Fedora and Deb-based distros we use the python3 version pulled in by the python3-pip package. Suggested-by: Ilya Maximets Signed-off-by: Dumitru Ceara --- Dockerfile | 5 ++- do.sh | 65 +------------------------------------ utils/helpers.sh | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 65 deletions(-) create mode 100644 utils/helpers.sh diff --git a/Dockerfile b/Dockerfile index 5f16e5f0..e63052b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM ovn/ovn-multi-node ARG SSH_KEY +COPY utils/helpers.sh /scripts/helpers.sh COPY ovn-tester /ovn-tester RUN mkdir -p /root/.ssh/ @@ -9,8 +10,10 @@ COPY $SSH_KEY /root/.ssh/ COPY ovn-fake-multinode-utils/process-monitor.py /tmp/ +RUN /bin/bash -c ". /scripts/helpers.sh; install_latest_python" + # This variable is needed on systems where global python's # environment is marked as "Externally managed" (PEP 668) to allow pip # installation of "global" packages. ENV PIP_BREAK_SYSTEM_PACKAGES=1 -RUN pip3 install -r /ovn-tester/requirements.txt +RUN python3 -m pip install -r /ovn-tester/requirements.txt diff --git a/do.sh b/do.sh index f4c6bb60..55a36aff 100755 --- a/do.sh +++ b/do.sh @@ -30,70 +30,7 @@ ovn_tester=${topdir}/ovn-tester EXTRA_OPTIMIZE=${EXTRA_OPTIMIZE:-no} USE_OVSDB_ETCD=${USE_OVSDB_ETCD:-no} -# We want values from both the `ID` and `ID_LIKE` fields to ensure successful -# categorization. The shell will happily accept both spaces and newlines as -# separators: -# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 -DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"') - -DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"') - -function is_rpm_based() { - for id in $DISTRO_IDS; do - case $id in - centos* | rhel* | fedora*) - true - return - ;; - esac - done - false -} - -function is_rhel() { - for id in $DISTRO_IDS; do - case $id in - centos* | rhel*) - true - return - ;; - esac - done - false -} - -function is_fedora() { - for id in $DISTRO_IDS; do - case $id in - fedora*) - true - return - ;; - esac - done - false -} - -function is_deb_based() { - for id in $DISTRO_IDS; do - case $id in - debian* | ubuntu*) - true - return - ;; - esac - done - false -} - -function die() { - echo $1 - exit 1 -} - -function die_distro() { - die "Unable to determine distro type, rpm- and deb-based are supported." -} +source $topdir/utils/helpers.sh function generate() { # Make sure rundir exists. diff --git a/utils/helpers.sh b/utils/helpers.sh new file mode 100644 index 00000000..edfda42a --- /dev/null +++ b/utils/helpers.sh @@ -0,0 +1,84 @@ +# We want values from both the `ID` and `ID_LIKE` fields to ensure successful +# categorization. The shell will happily accept both spaces and newlines as +# separators: +# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05 +DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"') + +DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"') + +function is_rpm_based() { + for id in $DISTRO_IDS; do + case $id in + centos* | rhel* | fedora*) + true + return + ;; + esac + done + false +} + +function is_rhel() { + for id in $DISTRO_IDS; do + case $id in + centos* | rhel*) + true + return + ;; + esac + done + false +} + +function is_fedora() { + for id in $DISTRO_IDS; do + case $id in + fedora*) + true + return + ;; + esac + done + false +} + +function is_deb_based() { + for id in $DISTRO_IDS; do + case $id in + debian* | ubuntu*) + true + return + ;; + esac + done + false +} + +# Installs the most recent available python3 and pip version for the +# current distro. +function install_latest_python() { + if is_rhel + then + py_pkg=$(dnf list available 'python3.[0-9][0-9]' -q 2> /dev/null | + grep -o '^python3.[0-9][0-9]' | sort -V | tail -n 1) && + dnf install -y ${py_pkg}-pip && + alternatives --install /usr/bin/python3 python3 /usr/bin/$py_pkg 100 + elif is_fedora + then + dnf install -y python3-pip + elif is_deb_based + then + apt install -y python3-pip + else + die_distro + fi +} + +function die() { + echo $1 + exit 1 +} + +function die_distro() { + die "Unable to determine distro type, rpm- and deb-based are supported." +} From 2a2b73937b08869c72f874b78703c3d9983943fb Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 8 Jan 2026 15:14:10 +0100 Subject: [PATCH 4/7] generate-hosts: Support optional internal-iface. On single node deployments, e.g., in CI, we don't need an internal-iface. Signed-off-by: Dumitru Ceara --- ovn-fake-multinode-utils/generate-hosts.py | 16 ++++++++++------ physical-deployments/ci.yml | 2 -- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ovn-fake-multinode-utils/generate-hosts.py b/ovn-fake-multinode-utils/generate-hosts.py index 7a320767..69bf5d16 100755 --- a/ovn-fake-multinode-utils/generate-hosts.py +++ b/ovn-fake-multinode-utils/generate-hosts.py @@ -19,22 +19,26 @@ def usage(name): ) -def generate_node_string(host: str, **kwargs) -> None: +def generate_node_string( + host: str, internal_iface: str | None, **kwargs +) -> None: + if internal_iface is not None: + kwargs['internal_iface'] = internal_iface args = ' '.join(f"{key}={value}" for key, value in kwargs.items()) print(f"{host} {args}") -def generate_node(config: Dict, internal_iface: str, **kwargs) -> None: +def generate_node(config: Dict, internal_iface: str | None, **kwargs) -> None: host: str = config['name'] internal_iface = config.get('internal-iface', internal_iface) generate_node_string( host, - internal_iface=internal_iface, + internal_iface, **kwargs, ) -def generate_tester(config: Dict, internal_iface: str) -> None: +def generate_tester(config: Dict, internal_iface: str | None) -> None: ssh_key = config["ssh_key"] ssh_key = Path(ssh_key).resolve() generate_node( @@ -45,7 +49,7 @@ def generate_tester(config: Dict, internal_iface: str) -> None: ) -def generate_nodes(nodes_config: Dict, internal_iface: str, **kwargs): +def generate_nodes(nodes_config: Dict, internal_iface: str | None, **kwargs): for node_config in nodes_config: host, node_config = helpers.get_node_config(node_config) iface = node_config.get('internal-iface', internal_iface) @@ -62,7 +66,7 @@ def generate(input_file: str, target: str, repo: str, branch: str) -> None: user = config.get('user', 'root') prefix = config.get('prefix', 'ovn-scale') tester_config = config['tester-node'] - internal_iface = config['internal-iface'] + internal_iface = config.get('internal-iface') print('[tester_hosts]') generate_tester(tester_config, internal_iface) diff --git a/physical-deployments/ci.yml b/physical-deployments/ci.yml index 34bf2beb..34cab7b6 100644 --- a/physical-deployments/ci.yml +++ b/physical-deployments/ci.yml @@ -1,5 +1,3 @@ -internal-iface: lo - central-nodes: - From 3aff56c7385353263c83838845bc01f2a6d34a53 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Tue, 13 Jan 2026 10:37:42 +0100 Subject: [PATCH 5/7] ovn_sandbox: Handle String Terminator characters. The Sandbox class uses an interactive shell to maintain a channel to each of the containers (fake nodes) it needs to runs commands in. As this is a real shell emulator the Sandbox implementation must also handle (ignore mostly) the vt100 control/escape symbols. The implementation uses special markers to determine where the beginning of the command output starts and where it ends. However, the implementation assumed that the first line of the output was always separated from the previous (command) line by an universal newline character (e.g., '\n'). Which meant it was enough to call splitlines() on the output we got from the channel and then search for the start marker as one of the split lines. Some shell implementations however may occasionally use the String Terminator character 'ESC \', i.e. '\x1b\\', to end the command part of the output, _without_ adding an explicit newline. This caused issues as ovn-heater failed to determine where the actual output started in those cases and was failing with: File "/ovn-tester/ovn_sandbox.py", line 92, in run self.ensure_channel() ~~~~~~~~~~~~~~~~~~~^^ File "/ovn-tester/ovn_sandbox.py", line 80, in ensure_channel self.run(cmd="echo Hello", stdout=stdout, raise_on_error=True) ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/ovn-tester/ovn_sandbox.py", line 128, in run start = out.index('++++start') + 1 ~~~~~~~~~^^^^^^^^^^^^^ While the output itself did actually include the '++++start' marker, e.g.: '\x1b]3008;start=...user=;hostname=..;pid=...;cwd=/\x1b\\++++start' In the output above '\x1b]' ('ESC ]') is the Operating System Command control character and is terminated by '\x1b\\' ('ESC \') the String Terminator control character. In order to handle such cases we now explicitly handle String Terminator as an additional delimiter on top of the universal newline characters considered by Python's splitlines(). Signed-off-by: Dumitru Ceara --- ovn-tester/ovn_sandbox.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ovn-tester/ovn_sandbox.py b/ovn-tester/ovn_sandbox.py index 5103d54b..937946cf 100644 --- a/ovn-tester/ovn_sandbox.py +++ b/ovn-tester/ovn_sandbox.py @@ -4,6 +4,7 @@ from io import StringIO from ovn_exceptions import SSHError +from typing import List log = logging.getLogger(__name__) @@ -79,6 +80,15 @@ def ensure_channel(self) -> None: # Checking + consuming all the unwanted output from the shell. self.run(cmd="echo Hello", stdout=stdout, raise_on_error=True) + # Splits 'out' by universal newline characters with the addition that it + # considers the terminal String Terminator character '\x1b\' as a newline. + @staticmethod + def split_channel_output(out: str) -> List[str]: + lines = [] + for line in out.splitlines(): + lines += line.split('\x1b\\') + return lines + def run( self, cmd: str = "", @@ -109,7 +119,7 @@ def run( while '++++end' not in out.splitlines(): out = out + self.channel.recv(10240).decode() except (paramiko.buffered_pipe.PipeTimeout, socket.timeout): - if '++++start' not in out.splitlines(): + if '++++start' not in self.split_channel_output(out): out = '++++start\n' + out out = out + '\n42\n++++end' timed_out = True @@ -120,7 +130,7 @@ def run( pass # Splitting and removing all lines with terminal control chars. - out = out.splitlines() + out = self.split_channel_output(out) start = out.index('++++start') + 1 end = out.index('++++end') - 1 exit_status = int(out[end]) From 80dc2b4af8298a746e1647a7be9f789c4ccfa356 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Mon, 12 Jan 2026 15:04:15 +0100 Subject: [PATCH 6/7] cirrus: Use 16GB ram VMs in CI. The Ubuntu runs were running out of memory (due to non-OVN components). We should probably investigate why those components are installed and remove them if possible. However, using 16GB of RAM for VMs in CI sounds like a good idea nevertheless. Signed-off-by: Dumitru Ceara --- .cirrus.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index b0c0baf9..36615627 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -7,8 +7,9 @@ low_scale_task: - image_project: ubuntu-os-cloud image: family/ubuntu-2404-lts-amd64 platform: linux - memory: 8G + memory: 16G disk: 40 + cpu: 4 env: DEPENDENCIES: git ansible podman From 1abecd8d6bedafa4a6def1844644c4e541dc5ee9 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Thu, 8 Jan 2026 13:58:38 +0100 Subject: [PATCH 7/7] cirrus: Update Fedora image to 43. Signed-off-by: Dumitru Ceara --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 36615627..630f7150 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,7 +3,7 @@ low_scale_task: compute_engine_instance: matrix: - image_project: fedora-cloud - image: family/fedora-cloud-38 + image: family/fedora-cloud-43-x86-64 - image_project: ubuntu-os-cloud image: family/ubuntu-2404-lts-amd64 platform: linux