MINIFICPP-2422 Remove Ubuntu 20.04 support

- Update CI scripts to use Ubuntu 24.04 workers instead of Ubuntu 20.04 - Update python dependencies for docker tests Closes #1928 Signed-off-by: Marton Szasz <[email protected]>
apache · Feb 18, 2025 · 7b6b5de · 7b6b5de
1 parent f04f222
commit 7b6b5de
Show file tree

Hide file tree

Showing 13 changed files with 50 additions and 55 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -475,7 +475,7 @@ jobs:
           path: build/bin
   rocky:
     name: "rocky"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 180
     steps:
       - id: checkout
@@ -589,7 +589,7 @@ jobs:
   docker_tests:
     name: "Docker integration tests"
     needs: docker_build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 180
     steps:
       - id: checkout

diff --git a/.github/workflows/clear-actions-cache.yml b/.github/workflows/clear-actions-cache.yml
@@ -6,9 +6,9 @@ on:
       - completed
   workflow_dispatch:
 jobs:
-  ubuntu_20_04:
-    name: "ubuntu-20.04"
-    runs-on: ubuntu-20.04
+  ubuntu_24_04:
+    name: "ubuntu-24.04"
+    runs-on: ubuntu-24.04
     steps:
       - id: checkout
         uses: actions/checkout@v4

diff --git a/aptitude.sh b/aptitude.sh
@@ -23,10 +23,6 @@ verify_enable_platform(){
 add_os_flags() {
     CC=${CC:-gcc}
     CXX=${CXX:-g++}
-    if [[ "$OS" = Ubuntu* && "$OS_MAJOR" -lt 22 ]]; then
-        CC=${CC:-gcc-11}
-        CXX=${CXX:-g++-11}
-    fi
     export CC
     export CXX
     CMAKE_BUILD_COMMAND="${CMAKE_BUILD_COMMAND} -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX"
@@ -41,10 +37,6 @@ bootstrap_cmake(){
 }
 bootstrap_compiler() {
     compiler_pkgs="gcc g++"
-    if [[ "$OS" = Ubuntu* && "$OS_MAJOR" -lt 22 ]]; then
-        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-        compiler_pkgs="gcc-11 g++-11"
-    fi
     # shellcheck disable=SC2086
     sudo apt-get -y install $compiler_pkgs
 }

diff --git a/bootstrap/package_manager.py b/bootstrap/package_manager.py
@@ -127,12 +127,6 @@ def _get_installed_packages(self) -> Set[str]:
         return set(lines)
 
     def install_compiler(self) -> str:
-        if distro.id() == "ubuntu" and int(distro.major_version()) < 22:
-            self.install({"compiler_prereq": {"apt-transport-https", "ca-certificates", "software-properties-common"}})
-            _run_command_with_confirm("sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test",
-                                      no_confirm=self.no_confirm)
-            self.install({"compiler": {"build-essential", "g++-11"}})
-            return "-DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11"
         self.install({"compiler": {"g++"}})
         return ""
 

diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh
@@ -43,7 +43,7 @@ function usage {
   echo "-p, --prefix          Additional prefix added to the image tag"
   echo "-u, --uid             User id to be used in the Docker image (default: 1000)"
   echo "-g, --gid             Group id to be used in the Docker image (default: 1000)"
-  echo "-d, --distro-name     Linux distribution build to be used for alternative builds (bionic|focal|fedora|centos)"
+  echo "-d, --distro-name     Linux distribution build to be used for alternative builds (centos|rockylinux)"
   echo "-l  --dump-location   Path where to the output dump to be put"
   echo "-c  --cmake-param     CMake parameter passed in PARAM=value format"
   echo "-o  --options         Minifi options string"

diff --git a/docker/requirements.txt b/docker/requirements.txt
@@ -1,15 +1,14 @@
 pyjks==20.0.0
-shortuuid==1.0.11
-behavex==2.0.1
-docker==5.0.0
-kafka-python-ng==2.2.3
+shortuuid==1.0.13
+behavex==4.0.10
+docker==7.1.0
 confluent-kafka==2.8.0
-PyYAML==6.0.1
-m2crypto==0.43.0
-watchdog==2.1.2
-pyopenssl==23.0.0
-azure-storage-blob==12.13.0
-prometheus-api-client==0.5.0
+PyYAML==6.0.2
+m2crypto==0.41.0
+watchdog==6.0.0
+pyopenssl==25.0.0
+azure-storage-blob==12.24.1
+prometheus-api-client==0.5.5
 humanfriendly==10.0
 requests<2.29  # https://github.com/docker/docker-py/issues/3113
-couchbase==4.3.2
+couchbase==4.3.5
diff --git a/docker/test/integration/cluster/checkers/ElasticSearchChecker.py b/docker/test/integration/cluster/checkers/ElasticSearchChecker.py
@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import json
+from utils import retry_check
 
 
 class ElasticSearchChecker:
     def __init__(self, container_communicator):
         self.container_communicator = container_communicator
 
+    @retry_check()
     def is_elasticsearch_empty(self, container_name):
         (code, output) = self.container_communicator.execute_command(container_name, ["curl", "-s", "-u", "elastic:password", "-k", "-XGET", "https://localhost:9200/_search"])
         return code == 0 and '"hits":[]' in output

diff --git a/docker/test/integration/cluster/containers/AzureStorageServerContainer.py b/docker/test/integration/cluster/containers/AzureStorageServerContainer.py
@@ -31,7 +31,7 @@ def deploy(self):
 
         logging.info('Creating and running azure storage server docker container...')
         self.client.containers.run(
-            "mcr.microsoft.com/azure-storage/azurite:3.23.0",
+            "mcr.microsoft.com/azure-storage/azurite:3.33.0",
             detach=True,
             name=self.name,
             network=self.network.name,

diff --git a/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py b/docker/test/integration/cluster/containers/MinifiC2ServerContainer.py
@@ -20,6 +20,8 @@
 
 from .Container import Container
 from OpenSSL import crypto
+from cryptography.hazmat.primitives.serialization import pkcs12, BestAvailableEncryption, load_pem_private_key
+from cryptography import x509
 from ssl_utils.SSL_cert_utils import make_server_cert
 
 
@@ -37,10 +39,17 @@ def __init__(self, feature_context, name, vols, network, image_store, command=No
             os.chmod(self.server_keystore_file.name, 0o644)
 
             self.server_truststore_file = tempfile.NamedTemporaryFile(delete=False)
-            pkcs12 = crypto.PKCS12()
-            pkcs12.set_privatekey(feature_context.root_ca_key)
-            pkcs12.set_certificate(feature_context.root_ca_cert)
-            pkcs12_data = pkcs12.export(passphrase="abcdefgh")
+            private_key_pem = crypto.dump_privatekey(crypto.FILETYPE_PEM, feature_context.root_ca_key)
+            private_key = load_pem_private_key(private_key_pem, password=None)
+            certificate_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, feature_context.root_ca_cert)
+            certificate = x509.load_pem_x509_certificate(certificate_pem)
+            pkcs12_data = pkcs12.serialize_key_and_certificates(
+                name=None,
+                key=private_key,
+                cert=certificate,
+                cas=None,
+                encryption_algorithm=BestAvailableEncryption(b'abcdefgh')
+            )
             self.server_truststore_file.write(pkcs12_data)
             self.server_truststore_file.close()
             os.chmod(self.server_truststore_file.name, 0o644)

diff --git a/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py b/docker/test/integration/cluster/containers/SyslogTcpClientContainer.py
@@ -29,7 +29,7 @@ def deploy(self):
 
         logging.info('Creating and running a Syslog tcp client docker container...')
         self.client.containers.run(
-            "ubuntu:20.04",
+            "ubuntu:24.04",
             detach=True,
             name=self.name,
             network=self.network.name,

diff --git a/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py b/docker/test/integration/cluster/containers/SyslogUdpClientContainer.py
@@ -29,7 +29,7 @@ def deploy(self):
 
         logging.info('Creating and running a Syslog udp client docker container...')
         self.client.containers.run(
-            "ubuntu:20.04",
+            "ubuntu:24.04",
             detach=True,
             name=self.name,
             network=self.network.name,

diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py
@@ -39,7 +39,6 @@
 import humanfriendly
 import OpenSSL.crypto
 
-from kafka import KafkaProducer
 from confluent_kafka.admin import AdminClient, NewTopic
 from confluent_kafka import Producer
 import socket
@@ -803,9 +802,9 @@ def step_impl(context, content, topic_name, semicolon_separated_headers):
     for header in semicolon_separated_headers.split(";"):
         kv = header.split(":")
         headers.append((kv[0].strip(), kv[1].strip().encode("utf-8")))
-    producer = KafkaProducer(bootstrap_servers='localhost:29092')
-    future = producer.send(topic_name, content.encode("utf-8"), headers=headers)
-    assert future.get(timeout=60)
+    producer = Producer({"bootstrap.servers": "localhost:29092"})
+    producer.produce(topic_name, content.encode("utf-8"), headers=headers)
+    producer.flush(10)
 
 
 @when("the Kafka consumer is registered in kafka broker")

diff --git a/github_scripts/github_actions_cache_cleanup_tests.py b/github_scripts/github_actions_cache_cleanup_tests.py
@@ -32,11 +32,11 @@ def create_mock_github_request_sender(self):
                 },
                 {
                     "id": 11111,
-                    "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                    "key": "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
                 },
                 {
                     "id": 11112,
-                    "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
+                    "key": "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
                 },
                 {
                     "id": 12345,
@@ -52,11 +52,11 @@ def create_mock_github_request_sender(self):
                 },
                 {
                     "id": 44444,
-                    "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
+                    "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
                 },
                 {
                     "id": 55555,
-                    "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
+                    "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
                 }
             ]
         }
@@ -77,11 +77,11 @@ def create_empty_open_pr_mock_github_request_sender(self):
                 },
                 {
                     "id": 11111,
-                    "key": "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                    "key": "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
                 },
                 {
                     "id": 11112,
-                    "key": "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
+                    "key": "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
                 },
                 {
                     "id": 12345,
@@ -97,11 +97,11 @@ def create_empty_open_pr_mock_github_request_sender(self):
                 },
                 {
                     "id": 44444,
-                    "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
+                    "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456",
                 },
                 {
                     "id": 55555,
-                    "key": "ubuntu-20.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
+                    "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567",
                 }
             ]
         }
@@ -143,20 +143,20 @@ def test_cache_cleanup(self):
         self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]),
                          {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
                           "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
-                          "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
-                          "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
+                          "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                          "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
 
     def test_cache_cleanup_with_zero_open_prs(self):
         cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")
         cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender()
         cleaner.remove_obsolete_cache_entries()
         self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]),
                          {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123",
-                          "ubuntu-20.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
-                          "ubuntu-20.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
+                          "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4",
+                          "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487",
                           "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536",
                           "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456",
-                          "ubuntu-20.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
+                          "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"})
 
     def test_cache_cleanup_with_zero_action_caches(self):
         cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp")