[container_benchmark] add container image pre pull management

Signed-off-by: Jan Rodák <[email protected]>

Author: Honny1

projects/container_bench/testing/config.yaml

@@ -100,6 +100,13 @@ prepare:
       - glibc-static
       - golang
 
+  container_images:
+    pull_images: true
+    dir: "{@remote_host.base_work_dir}/images"
+    images:
+      - "quay.io/podman/hello:latest"
+      - "registry.fedoraproject.org/fedora:latest"
+
   podman:
     # Custom binary or repo version must be disabled for matbenchmarking
     custom_binary:
@@ -149,6 +156,7 @@ cleanup:
     podman: true
     exec_time: true
     venv: true
+    container_images: false
 
   podman_machine:
     delete: true

projects/container_bench/testing/config_manager.py

@@ -57,13 +57,17 @@ class ConfigKeys:
     PREPARE_DNF_INSTALL_DEPENDENCIES = "prepare.dnf.install_dependencies"
     PREPARE_DNF_ENABLE_DOCKER_REPO = "prepare.dnf.enable_docker_repo"
     PREPARE_DNF_DEPENDENCIES = "prepare.dnf.dependencies"
+    PREPARE_CONTAINER_IMAGES_PULL_IMAGES = "prepare.container_images.pull_images"
+    PREPARE_CONTAINER_IMAGES_IMAGES = "prepare.container_images.images"
+    PREPARE_CONTAINER_IMAGES_IMAGES_DIR = "prepare.container_images.dir"
 
     # Additional cleanup configuration
     CLEANUP_FILES_EXEC_TIME = "cleanup.files.exec_time"
     CLEANUP_FILES_VENV = "cleanup.files.venv"
     CLEANUP_PODMAN_MACHINE_DELETE = "cleanup.podman_machine.delete"
     CLEANUP_DOCKER_SERVICE_STOP = "cleanup.docker_service.stop"
     CLEANUP_DOCKER_DESKTOP_STOP = "cleanup.docker_desktop.stop"
+    CLEANUP_CONTAINER_IMAGES = "cleanup.files.container_images"
 
     # Additional remote host configuration
     REMOTE_HOST_DOCKER_ENABLED = "remote_host.docker.enabled"
@@ -193,6 +197,17 @@ def get_dnf_config():
             'dependencies': config.project.get_config(ConfigKeys.PREPARE_DNF_DEPENDENCIES, print=False),
         }
 
+    @staticmethod
+    def get_container_images_config():
+        return {
+            'pull_images': config.project.get_config(
+                ConfigKeys.PREPARE_CONTAINER_IMAGES_PULL_IMAGES, print=False),
+            'images': config.project.get_config(
+                ConfigKeys.PREPARE_CONTAINER_IMAGES_IMAGES, print=False),
+            'dir': config.project.get_config(
+                ConfigKeys.PREPARE_CONTAINER_IMAGES_IMAGES_DIR, print=False),
+        }
+
     @staticmethod
     def get_extended_cleanup_config():
         return {
@@ -205,6 +220,8 @@ def get_extended_cleanup_config():
                 ConfigKeys.CLEANUP_DOCKER_SERVICE_STOP, print=False),
             'docker_desktop_stop': config.project.get_config(
                 ConfigKeys.CLEANUP_DOCKER_DESKTOP_STOP, print=False),
+            'container_images': config.project.get_config(
+                ConfigKeys.CLEANUP_CONTAINER_IMAGES, print=False),
         }
 
     @staticmethod

projects/container_bench/testing/container_engine.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import remote_access
 import json
 import logging
@@ -75,6 +76,32 @@ def get_command(self):
 
         return cmd
 
+    def store_container_images_as_tar(self):
+        container_images_config = ConfigManager.get_container_images_config()
+        pull_images = container_images_config['pull_images']
+        images = container_images_config['images']
+        images_dir = container_images_config['dir']
+        dest = Path(images_dir)
+
+        if not pull_images:
+            logging.info("Skipping pulling container images as per configuration.")
+            return
+
+        if not remote_access.exists(dest):
+            logging.info(f"Creating images directory at {dest} ...")
+            remote_access.create_remote_directory(dest)
+
+        for image in images:
+            logging.info(f"Pulling container image: {image} ...")
+            image_filename = image.replace("/", "_").replace(":", "_").replace(".", "_") + ".tar"
+            if remote_access.exists(dest / image_filename):
+                continue
+            cmd = f"{self.get_command()} pull {image}"
+            remote_access.run_with_ansible_ssh_conf(self.base_work_dir, cmd)
+            cmd = f"{self.get_command()} save -o {dest / image_filename} {image}"
+            remote_access.run_with_ansible_ssh_conf(self.base_work_dir, cmd)
+        self.rm_images(images)
+
     def is_rootful(self):
         if ConfigManager.is_linux():
             return self.podman_config['linux_rootful']
@@ -100,10 +127,10 @@ def cleanup(self):
 
         return ret.returncode == 0
 
-    def rm_image(self, image):
+    def rm_images(self, images):
         ret = remote_access.run_with_ansible_ssh_conf(
             self.base_work_dir,
-            f"{self.get_command()} image rm {image}",
+            f"{self.get_command()} image rm {' '.join(images)}",
             check=False,
             capture_stdout=True,
             capture_stderr=True,

projects/container_bench/testing/prepare.py

@@ -49,6 +49,10 @@ def cleanup():
         logging.info("Cleaning up Podman files")
         utils.cleanup_podman_files(remote_access.prepare())
 
+    if cleanup_config['container_images']:
+        logging.info("Cleaning up container images")
+        utils.cleanup_container_images(remote_access.prepare())
+
     cleanup_docker_platform()
     return 0
 
@@ -119,6 +123,7 @@ def prepare_docker_platform():
         docker_desktop.start()
 
     docker = ContainerEngine("docker")
+    docker.store_container_images_as_tar()
     docker.cleanup()
 
 
@@ -162,6 +167,7 @@ def prepare_podman_platform():
 
     logging.info("cleaning up podman")
     podman = ContainerEngine("podman")
+    podman.store_container_images_as_tar()
     podman.cleanup()
 
     return 0

projects/container_bench/testing/remote_access.py

@@ -302,3 +302,24 @@ def exists(path):
         return exists_windows(path)
     else:
         return exists_unix(path)
+
+
+def create_remote_directory(path):
+    if config.project.get_config("remote_host.run_locally", print=False):
+        path.mkdir(parents=True, exist_ok=True)
+        return
+
+    base_work_dir = prepare()
+
+    if ConfigManager.is_windows():
+        run_with_ansible_ssh_conf_windows(
+            base_work_dir,
+            f"New-Item -ItemType Directory -Path '{path}' -Force",
+            check=True,
+        )
+    else:
+        run_with_ansible_ssh_conf_unix(
+            base_work_dir,
+            f"mkdir -p {shlex.quote(str(path))}",
+            check=True,
+        )

projects/container_bench/testing/utils.py

@@ -96,6 +96,16 @@ def cleanup_podman_files(base_work_dir):
         prepare.remove_remote_file(base_work_dir, dest, recursive=True)
 
 
+def cleanup_container_images(base_work_dir):
+    container_images_config = ConfigManager.get_container_images_config()
+    images_dir = container_images_config['dir']
+    dest = Path(images_dir)
+
+    if remote_access.exists(dest):
+        logging.info(f"Removing container images directory {dest} ...")
+        prepare.remove_remote_file(base_work_dir, dest, recursive=True)
+
+
 def parse_platform(platform_str):
     p = SimpleNamespace()
     p.container_engine = platform_str