Fix ovn-tester python version and issues uncovered in CI.

.cirrus.yml

@@ -3,12 +3,13 @@ low_scale_task:
   compute_engine_instance:
     matrix:
       - image_project: fedora-cloud
-        image: family/fedora-cloud-38
+        image: family/fedora-cloud-43-x86-64
       - image_project: ubuntu-os-cloud
         image: family/ubuntu-2404-lts-amd64
     platform: linux
-    memory: 8G
+    memory: 16G
     disk: 40
+    cpu: 4
 
   env:
     DEPENDENCIES: git ansible podman

Dockerfile

@@ -2,15 +2,18 @@ FROM ovn/ovn-multi-node
 
 ARG SSH_KEY
 
+COPY utils/helpers.sh /scripts/helpers.sh
 COPY ovn-tester /ovn-tester
 
 RUN mkdir -p /root/.ssh/
 COPY $SSH_KEY /root/.ssh/
 
 COPY ovn-fake-multinode-utils/process-monitor.py /tmp/
 
+RUN /bin/bash -c ". /scripts/helpers.sh; install_latest_python"
+
 # This variable is needed on systems where global python's
 # environment is marked as "Externally managed" (PEP 668) to allow pip
 # installation of "global" packages.
 ENV PIP_BREAK_SYSTEM_PACKAGES=1
-RUN pip3 install -r /ovn-tester/requirements.txt
+RUN python3 -m pip install -r /ovn-tester/requirements.txt

do.sh

@@ -30,70 +30,7 @@ ovn_tester=${topdir}/ovn-tester
 EXTRA_OPTIMIZE=${EXTRA_OPTIMIZE:-no}
 USE_OVSDB_ETCD=${USE_OVSDB_ETCD:-no}
 
-# We want values from both the `ID` and `ID_LIKE` fields to ensure successful
-# categorization.  The shell will happily accept both spaces and newlines as
-# separators:
-# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05
-DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"')
-
-DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"')
-
-function is_rpm_based() {
-    for id in $DISTRO_IDS; do
-        case $id in
-        centos* | rhel* | fedora*)
-            true
-            return
-        ;;
-        esac
-    done
-    false
-}
-
-function is_rhel() {
-    for id in $DISTRO_IDS; do
-        case $id in
-        centos* | rhel*)
-            true
-            return
-        ;;
-        esac
-    done
-    false
-}
-
-function is_fedora() {
-    for id in $DISTRO_IDS; do
-        case $id in
-        fedora*)
-            true
-            return
-        ;;
-        esac
-    done
-    false
-}
-
-function is_deb_based() {
-    for id in $DISTRO_IDS; do
-        case $id in
-        debian* | ubuntu*)
-            true
-            return
-            ;;
-        esac
-    done
-    false
-}
-
-function die() {
-    echo $1
-    exit 1
-}
-
-function die_distro() {
-    die "Unable to determine distro type, rpm- and deb-based are supported."
-}
+source $topdir/utils/helpers.sh
 
 function generate() {
     # Make sure rundir exists.
@@ -107,9 +44,7 @@ function generate() {
 
 function install_deps_local_rpm() {
     echo "-- Installing local dependencies"
-    yum install redhat-lsb-core datamash \
-        python3-netaddr python3 python3-devel \
-        podman \
+    yum install datamash python3-netaddr python3 python3-devel podman \
         --skip-broken -y
 }
 

ovn-fake-multinode-utils/generate-hosts.py

@@ -19,22 +19,26 @@ def usage(name):
     )
 
 
-def generate_node_string(host: str, **kwargs) -> None:
+def generate_node_string(
+    host: str, internal_iface: str | None, **kwargs
+) -> None:
+    if internal_iface is not None:
+        kwargs['internal_iface'] = internal_iface
     args = ' '.join(f"{key}={value}" for key, value in kwargs.items())
     print(f"{host} {args}")
 
 
-def generate_node(config: Dict, internal_iface: str, **kwargs) -> None:
+def generate_node(config: Dict, internal_iface: str | None, **kwargs) -> None:
     host: str = config['name']
     internal_iface = config.get('internal-iface', internal_iface)
     generate_node_string(
         host,
-        internal_iface=internal_iface,
+        internal_iface,
         **kwargs,
     )
 
 
-def generate_tester(config: Dict, internal_iface: str) -> None:
+def generate_tester(config: Dict, internal_iface: str | None) -> None:
     ssh_key = config["ssh_key"]
     ssh_key = Path(ssh_key).resolve()
     generate_node(
@@ -45,7 +49,7 @@ def generate_tester(config: Dict, internal_iface: str) -> None:
     )
 
 
-def generate_nodes(nodes_config: Dict, internal_iface: str, **kwargs):
+def generate_nodes(nodes_config: Dict, internal_iface: str | None, **kwargs):
     for node_config in nodes_config:
         host, node_config = helpers.get_node_config(node_config)
         iface = node_config.get('internal-iface', internal_iface)
@@ -62,7 +66,7 @@ def generate(input_file: str, target: str, repo: str, branch: str) -> None:
         user = config.get('user', 'root')
         prefix = config.get('prefix', 'ovn-scale')
         tester_config = config['tester-node']
-        internal_iface = config['internal-iface']
+        internal_iface = config.get('internal-iface')
 
         print('[tester_hosts]')
         generate_tester(tester_config, internal_iface)

ovn-tester/ovn_sandbox.py

@@ -4,6 +4,7 @@
 
 from io import StringIO
 from ovn_exceptions import SSHError
+from typing import List
 
 log = logging.getLogger(__name__)
 
@@ -79,6 +80,15 @@ def ensure_channel(self) -> None:
         # Checking + consuming all the unwanted output from the shell.
         self.run(cmd="echo Hello", stdout=stdout, raise_on_error=True)
 
+    # Splits 'out' by universal newline characters with the addition that it
+    # considers the terminal String Terminator character '\x1b\' as a newline.
+    @staticmethod
+    def split_channel_output(out: str) -> List[str]:
+        lines = []
+        for line in out.splitlines():
+            lines += line.split('\x1b\\')
+        return lines
+
     def run(
         self,
         cmd: str = "",
@@ -109,7 +119,7 @@ def run(
             while '++++end' not in out.splitlines():
                 out = out + self.channel.recv(10240).decode()
         except (paramiko.buffered_pipe.PipeTimeout, socket.timeout):
-            if '++++start' not in out.splitlines():
+            if '++++start' not in self.split_channel_output(out):
                 out = '++++start\n' + out
             out = out + '\n42\n++++end'
             timed_out = True
@@ -120,7 +130,7 @@ def run(
             pass
 
         # Splitting and removing all lines with terminal control chars.
-        out = out.splitlines()
+        out = self.split_channel_output(out)
         start = out.index('++++start') + 1
         end = out.index('++++end') - 1
         exit_status = int(out[end])

ovn-tester/ovn_workload.py

@@ -64,12 +64,8 @@ class CentralNode(Node):
     def __init__(self, phys_node, container: str, mgmt_ip: str, protocol: str):
         super().__init__(phys_node, container, mgmt_ip, protocol)
 
-    def start(
-        self, cluster_cfg: ClusterConfig, update_election_timeout: bool = False
-    ):
+    def start(self, cluster_cfg: ClusterConfig):
         log.info('Configuring central node')
-        if cluster_cfg.clustered_db and update_election_timeout:
-            self.set_raft_election_timeout(cluster_cfg.raft_election_to)
         self.enable_trim_on_compaction()
         self.set_northd_threads(cluster_cfg.northd_threads)
         if cluster_cfg.log_txns_db:
@@ -83,20 +79,18 @@ def set_northd_threads(self, n_threads: int):
             f'{n_threads}'
         )
 
-    def set_raft_election_timeout(self, timeout_s: int):
-        for timeout in range(1000, (timeout_s + 1) * 1000, 1000):
-            log.info(f'Setting RAFT election timeout to {timeout}ms')
-            self.run(
-                cmd=f'ovs-appctl -t '
-                f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer '
-                f'OVN_Northbound {timeout}'
-            )
-            self.run(
-                cmd=f'ovs-appctl -t '
-                f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer '
-                f'OVN_Southbound {timeout}'
-            )
-            time.sleep(1)
+    def set_raft_election_timeout(self, timeout_ms: int):
+        log.info(f'Setting RAFT election timeout to {timeout_ms}ms')
+        self.run(
+            cmd=f'ovs-appctl -t '
+            f'/run/ovn/ovnnb_db.ctl cluster/change-election-timer '
+            f'OVN_Northbound {timeout_ms}'
+        )
+        self.run(
+            cmd=f'ovs-appctl -t '
+            f'/run/ovn/ovnsb_db.ctl cluster/change-election-timer '
+            f'OVN_Southbound {timeout_ms}'
+        )
 
     def enable_trim_on_compaction(self):
         log.info('Setting DB trim-on-compaction')
@@ -346,12 +340,22 @@ def add_workers(self, worker_nodes):
     def prepare_test(self):
         self.start()
 
+    def set_raft_election_timeout(self):
+        if not self.cluster_cfg.clustered_db:
+            return
+
+        log.info('Setting raft cluster election timeout')
+        for timeout_ms in range(
+            1000, (self.cluster_cfg.raft_election_to * 1000), 1000
+        ):
+            for c in self.central_nodes:
+                c.set_raft_election_timeout(timeout_ms)
+            time.sleep(1)
+
     def start(self):
+        self.set_raft_election_timeout()
         for c in self.central_nodes:
-            c.start(
-                self.cluster_cfg,
-                update_election_timeout=(c is self.central_nodes[0]),
-            )
+            c.start(self.cluster_cfg)
         nb_conn = self.get_nb_connection_string()
         inactivity_probe = self.cluster_cfg.db_inactivity_probe // 1000
         self.nbctl = ovn_utils.OvnNbctl(

physical-deployments/ci.yml

@@ -1,5 +1,3 @@
-internal-iface: lo
-
 central-nodes:
   - <host>
 

utils/helpers.sh

@@ -0,0 +1,84 @@
+# We want values from both the `ID` and `ID_LIKE` fields to ensure successful
+# categorization.  The shell will happily accept both spaces and newlines as
+# separators:
+# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_05
+DISTRO_IDS=$(awk -F= '/^ID/{print$2}' /etc/os-release | tr -d '"')
+
+DISTRO_VERSION_ID=$(awk -F= '/^VERSION_ID/{print$2}' /etc/os-release | tr -d '"')
+
+function is_rpm_based() {
+    for id in $DISTRO_IDS; do
+        case $id in
+        centos* | rhel* | fedora*)
+            true
+            return
+        ;;
+        esac
+    done
+    false
+}
+
+function is_rhel() {
+    for id in $DISTRO_IDS; do
+        case $id in
+        centos* | rhel*)
+            true
+            return
+        ;;
+        esac
+    done
+    false
+}
+
+function is_fedora() {
+    for id in $DISTRO_IDS; do
+        case $id in
+        fedora*)
+            true
+            return
+        ;;
+        esac
+    done
+    false
+}
+
+function is_deb_based() {
+    for id in $DISTRO_IDS; do
+        case $id in
+        debian* | ubuntu*)
+            true
+            return
+            ;;
+        esac
+    done
+    false
+}
+
+# Installs the most recent available python3 and pip version for the
+# current distro.
+function install_latest_python() {
+    if is_rhel
+    then
+        py_pkg=$(dnf list available 'python3.[0-9][0-9]' -q 2> /dev/null |
+                 grep -o '^python3.[0-9][0-9]' | sort -V | tail -n 1) &&
+        dnf install -y ${py_pkg}-pip &&
+        alternatives --install /usr/bin/python3 python3 /usr/bin/$py_pkg 100
+    elif is_fedora
+    then
+        dnf install -y python3-pip
+    elif is_deb_based
+    then
+        apt install -y python3-pip
+    else
+        die_distro
+    fi
+}
+
+function die() {
+    echo $1
+    exit 1
+}
+
+function die_distro() {
+    die "Unable to determine distro type, rpm- and deb-based are supported."
+}