ci: Run system tests in upgrade scenario.

When ovn is upgraded, ovn-controller is updated first on the compute
nodes. Then ovn-northd and DB are upgraded.
This patch tests whether the intermediate state (i.e. with
ovn-controller being upgraded) works properly, running system tests
from the base line (i.e. before the upgrade).

Flow tables might change between releases.
Hence this patch must take that into account by updating the (old)
system tests with any updated table numbers.
In some cases, (new) ovn-controller might change flows in existing
tables, causing some 'upgrade' tests to fail.
Such tests can be skipped using the TAG_TEST_NOT_UPGRADABLE tag.

This patch upgrades the ci to run automatically some upgrade tests
weekly, on schedule. It also provides a shell script to run those tests
locally.

This patch depends on patch [1] on branch-25.09.

[1] "tests: Add new TAG_TEST_NOT_UPGRADABLE to some tests."

Reported-at: https://issues.redhat.com/browse/FDP-1240
Assisted-by: claude, with model: Claude Sonnet 4.5
Signed-off-by: Xavier Simonart <xsimonar@redhat.com>
Acked-by: Mark Michelson <mmichels@redhat.com>
Signed-off-by: Ales Musil <amusil@redhat.com>

Author: simonartxavier

.ci/ci.sh

@@ -54,6 +54,9 @@ function archive_logs() {
         cp -r $CONTAINER_WORKDIR/tests/system-*-testsuite.* \
         $log_dir || true \
         && \
+        cp -r $CONTAINER_WORKDIR/tests/upgrade-testsuite.* \
+        $log_dir || true \
+        && \
         chmod -R +r $log_dir \
         &&
         tar -czvf $CONTAINER_WORKSPACE/logs.tgz $log_dir
@@ -102,7 +105,7 @@ function run_tests() {
         ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
         TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
         RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \
-        ./.ci/linux-build.sh
+        BASE_VERSION=$BASE_VERSION ./.ci/linux-build.sh
     "
 }
 

.ci/linux-build.sh

@@ -1,7 +1,12 @@
 #!/bin/bash
 
 set -o errexit
-set -x
+
+# Enable debug output for CI, optional for local
+NO_DEBUG=${NO_DEBUG:-0}
+if [ "$NO_DEBUG" = "0" ]; then
+    set -x
+fi
 
 ARCH=${ARCH:-"x86_64"}
 USE_SPARSE=${USE_SPARSE:-"yes"}
@@ -181,17 +186,23 @@ function run_system_tests()
 
     if ! sudo timeout -k 5m -v $TIMEOUT make $JOBS $type \
         TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \
-        SKIP_UNSTABLE=$SKIP_UNSTABLE; then
-        # $log_file is necessary for debugging.
-        cat tests/$log_file
+        SKIP_UNSTABLE=$SKIP_UNSTABLE UPGRADE_TEST=$UPGRADE_TEST \
+        BASE_VERSION=$BASE_VERSION; then
+        # Suppress output locally when NO_DEBUG not 0.
+        if [ "$NO_DEBUG" = "0" ]; then
+            cat tests/$log_file
+        fi
         return 1
     fi
 }
 
 function execute_system_tests()
 {
-    configure_ovn $OPTS
-    make $JOBS || { cat config.log; exit 1; }
+    # Upgrade tests build separately
+    if [ "$UPGRADE_TEST" != "yes" ]; then
+        configure_ovn $OPTS
+        make $JOBS || { cat config.log; exit 1; }
+    fi
 
     local stable_rc=0
     local unstable_rc=0
@@ -201,8 +212,12 @@ function execute_system_tests()
     fi
 
     if [ "$UNSTABLE" ]; then
-        if ! SKIP_UNSTABLE=no TEST_RANGE="-k unstable" RECHECK=yes \
-                run_system_tests $@; then
+        if [[ "$TEST_RANGE" == *"-d"* ]]; then
+            TEST_RANGE="-k unstable -d"
+        else
+            TEST_RANGE="-k unstable"
+        fi
+        if ! SKIP_UNSTABLE=no RECHECK=yes run_system_tests $@; then
             unstable_rc=1
         fi
     fi
@@ -238,6 +253,10 @@ if [ "$TESTSUITE" ]; then
         sudo bash -c "echo 2048 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
         execute_system_tests "check-system-dpdk" "system-dpdk-testsuite.log"
         ;;
+
+        "upgrade-test")
+        execute_system_tests "check-upgrade" "system-kmod-testsuite.log"
+        ;;
     esac
 else
     configure_ovn $OPTS

.ci/ovn_upgrade_test.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+import atexit
+import os
+import signal
+import sys
+from pathlib import Path
+
+
+from ovn_upgrade_utils import (
+    log,
+    chdir,
+    run_command,
+    run_shell_command,
+    ovn_upgrade_save_current_binaries,
+    ovn_upgrade_extract_info,
+    run_upgrade_workflow,
+    remove_upgrade_test_directory,
+    UpgradeConfig
+)
+
+DEFAULT_BASE_BRANCH = 'branch-24.03'
+
+
+def run_tests(config):
+    log(f"Running system tests in upgrade scenario with flags "
+        f"{config.env.flags}")
+
+    # Tests are run from the base-branch folder (when upgrading ovn-controller
+    # and not yet northd, new features do not work. Hence we cannot use new
+    # system-tests. We use the latest .ci/linux-build.sh i.e. from
+    # ovn_root_dir.
+    with chdir(config.path.base_dir):
+        no_debug = "0" if config.is_ci else "1"
+
+        cmd = f"""CC={config.env.cc} TESTSUITE=system-test UPGRADE_TEST=yes
+              TEST_RANGE="{config.env.flags}" UNSTABLE={config.env.unstable}
+              NO_DEBUG={no_debug}
+              . {config.path.ovn_root_dir}/.ci/linux-build.sh"""
+
+        return run_shell_command(cmd)
+
+
+def main():
+    test_success = False
+
+    def cleanup():
+        flags = os.environ.get('TESTSUITEFLAGS', '')
+        if '-d' in flags or '--debug' in flags or not test_success:
+            log(f"Keeping {config.path.upgrade_dir} for debugging")
+        else:
+            remove_upgrade_test_directory(config)
+
+    atexit.register(cleanup)
+    signal.signal(signal.SIGINT, lambda s, f: sys.exit(1))
+    signal.signal(signal.SIGTERM, lambda s, f: sys.exit(1))
+
+    config = UpgradeConfig.get(Path.cwd(), DEFAULT_BASE_BRANCH)
+
+    log("=" * 70)
+    log(f"OVN Upgrade Test - Base: {config.base_version}, "
+        f"Flags: {config.env.flags}")
+    log("=" * 70)
+
+    if run_command("sudo -v").returncode:
+        log("sudo access required")
+        return 1
+
+    if not remove_upgrade_test_directory(config):
+        return 1
+
+    config.path.upgrade_dir.mkdir(parents=True, exist_ok=True)
+    config.path.base_dir.mkdir(parents=True, exist_ok=True)
+    config.path.binaries_dir.mkdir(parents=True, exist_ok=True)
+
+    if not ovn_upgrade_save_current_binaries(config):
+        return 1
+
+    if not ovn_upgrade_extract_info(config):
+        return 1
+
+    if not run_upgrade_workflow(config):
+        if config.is_ci:
+            print(config.file.git_log.read_text(encoding='utf-8'))
+        else:
+            log(f"Check: {config.file.git_log}")
+        return 1
+
+    test_success = run_tests(config)
+
+    log("=" * 70)
+    if test_success:
+        log("UPGRADE TESTS PASSED")
+    else:
+        log("UPGRADE TESTS FAILED")
+        log(f"Check: {config.file.test_log}")
+    log("=" * 70)
+
+    return 0 if test_success else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())