diff --git a/projects/container_bench/testing/config.yaml b/projects/container_bench/testing/config.yaml index 37c203b372..0a4d76e2cb 100644 --- a/projects/container_bench/testing/config.yaml +++ b/projects/container_bench/testing/config.yaml @@ -100,6 +100,13 @@ prepare: - glibc-static - golang + container_images: + pull_images: true + dir: "{@remote_host.base_work_dir}/images" + images: + - "quay.io/podman/hello:latest" + - "registry.fedoraproject.org/fedora:latest" + podman: # Custom binary or repo version must be disabled for matbenchmarking custom_binary: @@ -149,6 +156,7 @@ cleanup: podman: true exec_time: true venv: true + container_images: false podman_machine: delete: true @@ -201,7 +209,6 @@ test: # - custom runtime: # Linux only - crun - - krun - runc capture_metrics: enabled: true diff --git a/projects/container_bench/testing/config_manager.py b/projects/container_bench/testing/config_manager.py index 5562ccce6a..5e2e18c590 100644 --- a/projects/container_bench/testing/config_manager.py +++ b/projects/container_bench/testing/config_manager.py @@ -57,6 +57,9 @@ class ConfigKeys: PREPARE_DNF_INSTALL_DEPENDENCIES = "prepare.dnf.install_dependencies" PREPARE_DNF_ENABLE_DOCKER_REPO = "prepare.dnf.enable_docker_repo" PREPARE_DNF_DEPENDENCIES = "prepare.dnf.dependencies" + PREPARE_CONTAINER_IMAGES_PULL_IMAGES = "prepare.container_images.pull_images" + PREPARE_CONTAINER_IMAGES_IMAGES = "prepare.container_images.images" + PREPARE_CONTAINER_IMAGES_IMAGES_DIR = "prepare.container_images.dir" # Additional cleanup configuration CLEANUP_FILES_EXEC_TIME = "cleanup.files.exec_time" @@ -64,6 +67,7 @@ class ConfigKeys: CLEANUP_PODMAN_MACHINE_DELETE = "cleanup.podman_machine.delete" CLEANUP_DOCKER_SERVICE_STOP = "cleanup.docker_service.stop" CLEANUP_DOCKER_DESKTOP_STOP = "cleanup.docker_desktop.stop" + CLEANUP_CONTAINER_IMAGES = "cleanup.files.container_images" # Additional remote host configuration REMOTE_HOST_DOCKER_ENABLED = "remote_host.docker.enabled" @@ -193,6 +197,17 @@ def get_dnf_config(): 'dependencies': config.project.get_config(ConfigKeys.PREPARE_DNF_DEPENDENCIES, print=False), } + @staticmethod + def get_container_images_config(): + return { + 'pull_images': config.project.get_config( + ConfigKeys.PREPARE_CONTAINER_IMAGES_PULL_IMAGES, print=False), + 'images': config.project.get_config( + ConfigKeys.PREPARE_CONTAINER_IMAGES_IMAGES, print=False), + 'dir': config.project.get_config( + ConfigKeys.PREPARE_CONTAINER_IMAGES_IMAGES_DIR, print=False), + } + @staticmethod def get_extended_cleanup_config(): return { @@ -205,6 +220,8 @@ def get_extended_cleanup_config(): ConfigKeys.CLEANUP_DOCKER_SERVICE_STOP, print=False), 'docker_desktop_stop': config.project.get_config( ConfigKeys.CLEANUP_DOCKER_DESKTOP_STOP, print=False), + 'container_images': config.project.get_config( + ConfigKeys.CLEANUP_CONTAINER_IMAGES, print=False), } @staticmethod diff --git a/projects/container_bench/testing/container_engine.py b/projects/container_bench/testing/container_engine.py index 18283ddf75..f3c742a554 100644 --- a/projects/container_bench/testing/container_engine.py +++ b/projects/container_bench/testing/container_engine.py @@ -1,3 +1,4 @@ +from pathlib import Path import remote_access import json import logging @@ -75,6 +76,32 @@ def get_command(self): return cmd + def store_container_images_as_tar(self): + container_images_config = ConfigManager.get_container_images_config() + pull_images = container_images_config['pull_images'] + images = container_images_config['images'] + images_dir = container_images_config['dir'] + dest = Path(images_dir) + + if not pull_images: + logging.info("Skipping pulling container images as per configuration.") + return + + if not remote_access.exists(dest): + logging.info(f"Creating images directory at {dest} ...") + remote_access.create_remote_directory(dest) + + for image in images: + logging.info(f"Pulling container image: {image} ...") + image_filename = image.replace("/", "_").replace(":", "_").replace(".", "_") + ".tar" + if remote_access.exists(dest / image_filename): + continue + cmd = f"{self.get_command()} pull {image}" + remote_access.run_with_ansible_ssh_conf(self.base_work_dir, cmd) + cmd = f"{self.get_command()} save -o {dest / image_filename} {image}" + remote_access.run_with_ansible_ssh_conf(self.base_work_dir, cmd) + self.rm_images(images) + def is_rootful(self): if ConfigManager.is_linux(): return self.podman_config['linux_rootful'] @@ -100,10 +127,10 @@ def cleanup(self): return ret.returncode == 0 - def rm_image(self, image): + def rm_images(self, images): ret = remote_access.run_with_ansible_ssh_conf( self.base_work_dir, - f"{self.get_command()} image rm {image}", + f"{self.get_command()} image rm {' '.join(images)}", check=False, capture_stdout=True, capture_stderr=True, diff --git a/projects/container_bench/testing/platform_builders.py b/projects/container_bench/testing/platform_builders.py index a9250c84fa..d5608a3319 100644 --- a/projects/container_bench/testing/platform_builders.py +++ b/projects/container_bench/testing/platform_builders.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from config_manager import ConfigManager +import shlex class PlatformCommandBuilder(ABC): @@ -11,6 +12,38 @@ def build_env_command(self, env_dict): def build_service_start_script(self, service_name, start_command, binary_path): pass + @abstractmethod + def build_chdir_command(self, chdir): + pass + + @abstractmethod + def build_rm_command(self, file_path, recursive=False): + pass + + @abstractmethod + def build_mkdir_command(self, path): + pass + + @abstractmethod + def build_exists_command(self, path): + pass + + @abstractmethod + def get_shell_command(self): + pass + + @abstractmethod + def build_entrypoint_script(self, env_cmd, chdir_cmd, cmd, verbose): + pass + + @abstractmethod + def check_exists_result(self, ret): + pass + + +def escape_powershell_single_quote(value): + return str(value).replace("'", "''") + class WindowsCommandBuilder(PlatformCommandBuilder): def build_env_command(self, env_dict): @@ -21,8 +54,7 @@ def build_env_command(self, env_dict): for k, v in env_dict.items(): if v is None or v == "": continue - env_commands.append(f"$env:{k}='{v}'") - + env_commands.append(f"$env:{escape_powershell_single_quote(k)}='{escape_powershell_single_quote(v)}'") return "; ".join(env_commands) + ";" def build_service_start_script(self, service_name, start_command, binary_path): @@ -66,18 +98,92 @@ def build_service_start_script(self, service_name, start_command, binary_path): Remove-Item "$env:USERPROFILE\\start_{service_name}.ps1" -Force -ErrorAction SilentlyContinue """ + def build_chdir_command(self, chdir): + if chdir is None: + return "Set-Location $env:USERPROFILE" + return f"Set-Location '{escape_powershell_single_quote(chdir)}'" + + def build_rm_command(self, file_path, recursive=False): + flags = "-Force -ErrorAction SilentlyContinue" + if recursive: + flags += " -Recurse" + return f"Remove-Item '{escape_powershell_single_quote(str(file_path))}' {flags}" + + def build_mkdir_command(self, path): + return f"New-Item -ItemType Directory -Path '{escape_powershell_single_quote(str(path))}' -Force" + + def build_exists_command(self, path): + return f"Test-Path '{escape_powershell_single_quote(str(path))}'" + + def get_shell_command(self): + return "powershell.exe -Command -" + + def build_entrypoint_script(self, env_cmd, chdir_cmd, cmd, verbose): + env_section = f"{env_cmd}\n" if env_cmd else "" + script = f""" +$ErrorActionPreference = "Stop" + +{env_section}{chdir_cmd} + +{cmd} + """ + if verbose: + script = f"$VerbosePreference = 'Continue'\n{script}" + return script + + def check_exists_result(self, ret): + return ret.stdout and ret.stdout.strip().lower() == "true" + class UnixCommandBuilder(PlatformCommandBuilder): def build_env_command(self, env_dict): if not env_dict: return "" - env_values = " ".join(f"'{k}={v}'" for k, v in env_dict.items() if v is not None and v != "") - return f"env {env_values}" + env_values = " ".join(f"{k}={shlex.quote(str(v))}" for k, v in env_dict.items() if v is not None and v != "") + return f"export {env_values}\n" def build_service_start_script(self, service_name, start_command, binary_path) -> str: return start_command + def build_chdir_command(self, chdir): + if chdir is None: + return "cd $HOME" + return f"cd '{shlex.quote(str(chdir))}'" + + def build_rm_command(self, file_path, recursive=False): + flag = "-rf" if recursive else "-f" + return f"rm {flag} {shlex.quote(str(file_path))}" + + def build_mkdir_command(self, path): + return f"mkdir -p {shlex.quote(str(path))}" + + def build_exists_command(self, path): + return f"test -e {shlex.quote(str(path))}" + + def get_shell_command(self): + return "bash" + + def build_entrypoint_script(self, env_cmd, chdir_cmd, cmd, verbose): + script = f""" +set -o pipefail +set -o errexit +set -o nounset +set -o errtrace + +{env_cmd} + +{chdir_cmd} + +{cmd} + """ + if verbose: + script = f"set -x\n{script}" + return script + + def check_exists_result(self, ret): + return ret.returncode == 0 + class PlatformFactory: @staticmethod diff --git a/projects/container_bench/testing/prepare.py b/projects/container_bench/testing/prepare.py index ae87e48ed8..2d78ded893 100644 --- a/projects/container_bench/testing/prepare.py +++ b/projects/container_bench/testing/prepare.py @@ -1,5 +1,4 @@ import logging -import shlex from container_engine import PodmanMachine, ContainerEngine, DockerDesktopMachine from config_manager import ConfigManager @@ -8,19 +7,6 @@ import utils -def remove_remote_file(base_work_dir, file_path, recursive=False): - if ConfigManager.is_windows(): - flags = "-Force -ErrorAction SilentlyContinue" - if recursive: - flags += " -Recurse" - cmd = f"Remove-Item '{file_path}' {flags}" - else: - flag = "-rf" if recursive else "-f" - cmd = f"rm {flag} {shlex.quote(str(file_path))}" - - remote_access.run_with_ansible_ssh_conf(base_work_dir, cmd) - - def cleanup(): cleanup_config = ConfigManager.get_extended_cleanup_config() @@ -30,7 +16,7 @@ def cleanup(): exec_time_script = utils.get_benchmark_script_path(base_work_dir) if remote_access.exists(exec_time_script): logging.info(f"Removing {exec_time_script} ...") - remove_remote_file(base_work_dir, exec_time_script) + remote_access.remove_remote_file(base_work_dir, exec_time_script) if cleanup_config['files_venv']: logging.info("Cleaning up virtual environment") @@ -38,7 +24,7 @@ def cleanup(): venv_path = utils.get_benchmark_script_path(base_work_dir).parent / ".venv" if remote_access.exists(venv_path): logging.info(f"Removing {venv_path} ...") - remove_remote_file(base_work_dir, venv_path, recursive=True) + remote_access.remove_remote_file(base_work_dir, venv_path, recursive=True) try: cleanup_podman_platform() @@ -49,6 +35,10 @@ def cleanup(): logging.info("Cleaning up Podman files") utils.cleanup_podman_files(remote_access.prepare()) + if cleanup_config['container_images']: + logging.info("Cleaning up container images") + utils.cleanup_container_images(remote_access.prepare()) + cleanup_docker_platform() return 0 @@ -119,6 +109,7 @@ def prepare_docker_platform(): docker_desktop.start() docker = ContainerEngine("docker") + docker.store_container_images_as_tar() docker.cleanup() @@ -162,6 +153,7 @@ def prepare_podman_platform(): logging.info("cleaning up podman") podman = ContainerEngine("podman") + podman.store_container_images_as_tar() podman.cleanup() return 0 diff --git a/projects/container_bench/testing/remote_access.py b/projects/container_bench/testing/remote_access.py index e4b8587816..d2a2881b99 100644 --- a/projects/container_bench/testing/remote_access.py +++ b/projects/container_bench/testing/remote_access.py @@ -3,11 +3,12 @@ import logging import utils import yaml -import shlex +import shutil from projects.core.library import config, run from constants import CONTAINER_BENCH_SECRET_PATH from config_manager import ConfigManager +from platform_builders import PlatformFactory def prepare(): @@ -82,7 +83,7 @@ def prepare(): return base_work_dir -def run_with_ansible_ssh_conf_windows( +def run_with_ansible_ssh_conf( base_work_dir, cmd, extra_env=None, check=True, @@ -102,7 +103,8 @@ def run_with_ansible_ssh_conf_windows( ) if config.project.get_config("remote_host.run_locally", print=False): - logging.info(f"Running on the local Windows host: {cmd}") + platform = "Windows" if ConfigManager.is_windows() else "Unix" + logging.info(f"Running on the local {platform} host: {cmd}") return run.run(cmd, **run_kwargs) with open(os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS"]) as f: @@ -117,188 +119,72 @@ def run_with_ansible_ssh_conf_windows( with open(os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_ENV"]) as f: ansible_extra_env = yaml.safe_load(f) - def escape_powershell_single_quote(value): - """Escape single quotes in PowerShell by replacing ' with ''""" - return str(value).replace("'", "''") - + builder = PlatformFactory.create_command_builder() env_vars = ansible_extra_env | extra_env - if env_vars: - env_cmd = "\n".join( - f"$env:{escape_powershell_single_quote(k)}='{escape_powershell_single_quote(v)}'" - for k, v in env_vars.items() - ) - else: - env_cmd = "" - - chdir_cmd = f"Set-Location '{escape_powershell_single_quote(chdir)}'" if chdir else "Set-Location $env:USERPROFILE" - - tmp_file_path, tmp_file = utils.get_tmp_fd() - - env_section = f"{env_cmd}\n" if env_cmd else "" - - entrypoint_script = f""" -$ErrorActionPreference = "Stop" - -{env_section}{chdir_cmd} - -{cmd} - """ - - if config.project.get_config("remote_host.verbose_ssh_commands", print=False): - entrypoint_script = f"$VerbosePreference = 'Continue'\n{entrypoint_script}" - logging.info(f"Running on the remote Windows host: {chdir_cmd}; {cmd}") + env_cmd = builder.build_env_command(env_vars) + chdir_cmd = builder.build_chdir_command(chdir) + verbose = config.project.get_config("remote_host.verbose_ssh_commands", print=False) + entrypoint_script = builder.build_entrypoint_script(env_cmd, chdir_cmd, cmd, verbose) - with open(tmp_file_path, "w") as f: - print(entrypoint_script, file=f) - if print_cmd: - print(entrypoint_script) - - proc = run.run(f"ssh {ssh_flags} -i {private_key_path} {user}@{host} -p {port} -- " - "powershell.exe -Command -", - **run_kwargs, stdin_file=tmp_file) - - return proc - - -def run_with_ansible_ssh_conf_unix( - base_work_dir, cmd, - extra_env=None, - check=True, - capture_stdout=False, - capture_stderr=False, - chdir=None, - print_cmd=False, -): - """Linux/Unix-specific SSH execution function using bash.""" - if extra_env is None: - extra_env = {} - - run_kwargs = dict( - log_command=False, - check=check, - capture_stdout=capture_stdout, - capture_stderr=capture_stderr, - ) - - if config.project.get_config("remote_host.run_locally", print=False): - logging.info(f"Running on the local Unix host: {cmd}") - return run.run(cmd, **run_kwargs) - - with open(os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_VARS"]) as f: - ansible_ssh_config = yaml.safe_load(f) - - ssh_flags = ansible_ssh_config["ansible_ssh_common_args"] - host = os.environ["TOPSAIL_REMOTE_HOSTNAME"] - port = ansible_ssh_config["ansible_port"] - user = ansible_ssh_config["ansible_ssh_user"] - private_key_path = ansible_ssh_config["ansible_ssh_private_key_file"] - - with open(os.environ["TOPSAIL_ANSIBLE_PLAYBOOK_EXTRA_ENV"]) as f: - ansible_extra_env = yaml.safe_load(f) - - export_cmd = "\n".join(f"export {k}='{v}'" for k, v in (ansible_extra_env | extra_env).items()) - - chdir_cmd = f"cd '{chdir}'" if chdir else "cd $HOME" + logging.info(f"Running on the remote host: {cmd}") tmp_file_path, tmp_file = utils.get_tmp_fd() - entrypoint_script = f""" -set -o pipefail -set -o errexit -set -o nounset -set -o errtrace - -{export_cmd} - -{chdir_cmd} - -exec {cmd} - """ - - if config.project.get_config("remote_host.verbose_ssh_commands", print=False): - entrypoint_script = f"set -x\n{entrypoint_script}" - - logging.info(f"Running on the remote Unix host: {chdir_cmd}; {cmd}") - with open(tmp_file_path, "w") as f: print(entrypoint_script, file=f) if print_cmd: print(entrypoint_script) - proc = run.run(f"ssh {ssh_flags} -i {private_key_path} {user}@{host} -p {port} -- " - "bash", + shell_cmd = builder.get_shell_command() + proc = run.run(f"ssh {ssh_flags} -i {private_key_path} {user}@{host} -p {port} -- {shell_cmd}", **run_kwargs, stdin_file=tmp_file) return proc -def run_with_ansible_ssh_conf( - base_work_dir, cmd, - extra_env=None, - check=True, - capture_stdout=False, - capture_stderr=False, - chdir=None, - print_cmd=False, -): - if ConfigManager.is_windows(): - return run_with_ansible_ssh_conf_windows( - base_work_dir, cmd, - extra_env=extra_env, - check=check, - capture_stdout=capture_stdout, - capture_stderr=capture_stderr, - chdir=chdir, - print_cmd=print_cmd, - ) - else: - return run_with_ansible_ssh_conf_unix( - base_work_dir, cmd, - extra_env=extra_env, - check=check, - capture_stdout=capture_stdout, - capture_stderr=capture_stderr, - chdir=chdir, - print_cmd=print_cmd, - ) - - -def exists_windows(path): +def exists(path): if config.project.get_config("remote_host.run_locally", print=False): return path.exists() base_work_dir = prepare() + builder = PlatformFactory.create_command_builder() - ret = run_with_ansible_ssh_conf_windows( + ret = run_with_ansible_ssh_conf( base_work_dir, - f"Test-Path '{path}'", + builder.build_exists_command(path), capture_stdout=True, check=False, ) - # PowerShell Test-Path returns "True" or "False" as text - return ret.stdout and ret.stdout.strip().lower() == "true" + return builder.check_exists_result(ret) -def exists_unix(path): +def create_remote_directory(path): if config.project.get_config("remote_host.run_locally", print=False): - return path.exists() + path.mkdir(parents=True, exist_ok=True) + return base_work_dir = prepare() + builder = PlatformFactory.create_command_builder() - ret = run_with_ansible_ssh_conf_unix( + run_with_ansible_ssh_conf( base_work_dir, - f"test -e {shlex.quote(str(path))}", - capture_stdout=True, - check=False, + builder.build_mkdir_command(path), + check=True, ) - return ret.returncode == 0 - -def exists(path): - if ConfigManager.is_windows(): - return exists_windows(path) - else: - return exists_unix(path) +def remove_remote_file(base_work_dir, file_path, recursive=False): + if config.project.get_config("remote_host.run_locally", print=False): + path = pathlib.Path(file_path) + if path.exists(): + if recursive and path.is_dir(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + return + + builder = PlatformFactory.create_command_builder() + cmd = builder.build_rm_command(file_path, recursive) + run_with_ansible_ssh_conf(base_work_dir, cmd) diff --git a/projects/container_bench/testing/test_container_bench.py b/projects/container_bench/testing/test_container_bench.py index c660fae081..4aefb90d14 100644 --- a/projects/container_bench/testing/test_container_bench.py +++ b/projects/container_bench/testing/test_container_bench.py @@ -85,6 +85,12 @@ def _separate_benchmark_values_by_platform(benchmark_values): platform_config["test.platform"] = platform expe_to_run[f"container_bench_{platform}"] = platform_config + test_config = ConfigManager.get_test_config() + for to_skip in (test_config['platforms_to_skip'] or []): + key = f"container_bench_{to_skip}" + if key in expe_to_run: + logging.info(f"Skipping {to_skip} test as per test.platforms_to_skip.") + expe_to_run.pop(key) return expe_to_run diff --git a/projects/container_bench/testing/utils.py b/projects/container_bench/testing/utils.py index 587729b58d..5f93f77126 100644 --- a/projects/container_bench/testing/utils.py +++ b/projects/container_bench/testing/utils.py @@ -87,13 +87,25 @@ def cleanup_podman_files(base_work_dir): if remote_access.exists(dest): logging.info(f"Removing {dest} ...") - prepare.remove_remote_file(base_work_dir, dest, recursive=True) + remote_access.remove_remote_file(base_work_dir, dest, recursive=True) dest = base_work_dir / "podman-custom" if remote_access.exists(dest): logging.info(f"Removing {dest} ...") - prepare.remove_remote_file(base_work_dir, dest, recursive=True) + remote_access.remove_remote_file(base_work_dir, dest, recursive=True) + + +def cleanup_container_images(base_work_dir): + container_images_config = ConfigManager.get_container_images_config() + if "dir" not in container_images_config: + return + images_dir = container_images_config['dir'] + dest = Path(images_dir) + + if remote_access.exists(dest): + logging.info(f"Removing container images directory {dest} ...") + remote_access.remove_remote_file(base_work_dir, dest, recursive=True) def parse_platform(platform_str): diff --git a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/main.yaml b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/main.yaml index 0909812b6e..5f1f668b02 100644 --- a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/main.yaml +++ b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/main.yaml @@ -1,6 +1,5 @@ --- -# Define common command template -- name: Set common command +- name: Set info command set_fact: container_info_command: "{{ container_bench_capture_container_engine_info_binary_path }} {{ container_bench_capture_container_engine_info_additional_args }} info --format json" diff --git a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/unix.yml b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/unix.yml index 2edc337976..ced5ad7fc0 100644 --- a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/unix.yml +++ b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/unix.yml @@ -1,12 +1,7 @@ --- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' +- name: Include shared artifacts directory creation + include_tasks: "../../shared_tasks/create_artifacts_dir.yml" -# Capture container engine information - name: Capture the container engine information become: "{{ container_bench_capture_container_engine_info_rootfull }}" shell: | diff --git a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/windows.yml b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/windows.yml index bbdd621f7e..4343fede57 100644 --- a/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/windows.yml +++ b/projects/container_bench/toolbox/container_bench_capture_container_engine_info/tasks/windows.yml @@ -1,11 +1,7 @@ --- -# Create artifacts directory -- name: Create the artifacts directory - ansible.windows.win_file: - path: "{{ artifact_extra_logs_dir }}\\artifacts" - state: directory +- name: Include shared artifacts directory creation + include_tasks: "../../shared_tasks/create_artifacts_dir.yml" -# Capture container engine information - name: Capture the container engine information ansible.windows.win_shell: | $output = {{ container_info_command }} diff --git a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/darwin.yml b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/darwin.yml index 45053a46b3..1d7017b752 100644 --- a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/darwin.yml +++ b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/darwin.yml @@ -1,11 +1,6 @@ --- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' +- name: Include shared artifacts directory creation + include_tasks: "../../shared_tasks/create_artifacts_dir.yml" -# Capture system information - macOS/Darwin - name: Capture the system versions (macOS only) shell: system_profiler SPSoftwareDataType SPHardwareDataType > "{{ artifact_extra_logs_dir }}/artifacts/system_info.txt" diff --git a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/linux.yml b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/linux.yml index 3dbae51a42..b513e85254 100644 --- a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/linux.yml +++ b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/linux.yml @@ -1,12 +1,7 @@ --- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' +- name: Include shared artifacts directory creation + include_tasks: "../../shared_tasks/create_artifacts_dir.yml" -# Capture system information - Linux - name: Capture the system information shell: | set -o pipefail diff --git a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/windows.yml b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/windows.yml index 5514e8025d..4652494756 100644 --- a/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/windows.yml +++ b/projects/container_bench/toolbox/container_bench_capture_system_state/tasks/windows.yml @@ -1,11 +1,7 @@ --- -# Create artifacts directory -- name: Create the artifacts directory - ansible.windows.win_file: - path: "{{ artifact_extra_logs_dir }}\\artifacts" - state: directory +- name: Include shared artifacts directory creation + include_tasks: "../../shared_tasks/create_artifacts_dir.yml" -# Capture system information - Windows - name: Capture the system information ansible.windows.win_shell: | # Create output file and add Software section header diff --git a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/main.yml b/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/main.yml index badd1b3a16..0a7374ae01 100644 --- a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/main.yml +++ b/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/main.yml @@ -1,22 +1,32 @@ --- -# Define common command templates (shared across all platforms) -- name: Set common commands +- name: Validate required variables are defined + assert: + that: + - container_bench_exec_benchmark_exec_time_path is defined + - container_bench_exec_benchmark_binary_path is defined + - container_bench_exec_benchmark_rootfull is defined + - container_bench_exec_benchmark_additional_args is defined + when: false # This task never runs but validates variable definitions + +- include_tasks: "../../shared_tasks/map_variables.yml" + vars: + role_name: "container_bench_exec_benchmark" + +- name: Set common commands (shared across all platforms) set_fact: + image_to_pull: "registry.fedoraproject.org/fedora:latest" prepare_commands: | - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} system prune -a -f - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} pull registry.fedoraproject.org/fedora:latest - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} run --name benchmark_exec -d registry.fedoraproject.org/fedora:latest sleep infinity + {{ binary_path }} {{ additional_args }} run --name benchmark_exec -d registry.fedoraproject.org/fedora:latest sleep infinity cleanup_commands: | - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} stop benchmark_exec - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} rm -f benchmark_exec - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} system prune -a -f + {{ binary_path }} {{ additional_args }} stop benchmark_exec + {{ binary_path }} {{ additional_args }} rm -f benchmark_exec benchmark_command: | - {{ container_bench_exec_benchmark_binary_path }} {{ container_bench_exec_benchmark_additional_args }} exec benchmark_exec true + {{ binary_path }} {{ additional_args }} exec benchmark_exec true - name: Include Unix/Linux tasks - include_tasks: unix.yml + include_tasks: "../../shared_tasks/unix_default_benchmark_body.yml" when: ansible_os_family != 'Windows' - name: Include Windows tasks - include_tasks: windows.yml + include_tasks: ../../shared_tasks/windows_default_benchmark_body.yml when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/unix.yml b/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/unix.yml deleted file mode 100644 index 768264670a..0000000000 --- a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/unix.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' - -# Prepare container environment -- name: Prepare container environment - become: "{{ container_bench_exec_benchmark_rootfull }}" - shell: "{{ prepare_commands }}" - -# Run benchmark commands -- name: Run benchmark commands - become: "{{ container_bench_exec_benchmark_rootfull }}" - shell: | - {{ container_bench_exec_benchmark_exec_time_path | dirname }}/.venv/bin/python \ - {{ container_bench_exec_benchmark_exec_time_path }} \ - --output "{{ artifact_extra_logs_dir }}/artifacts/output.log" \ - --metrics-log-file "{{ artifact_extra_logs_dir }}/artifacts/metrics.json" \ - {{ benchmark_command }} - -# Clean up container environment -- name: Clean up container environment - become: "{{ container_bench_exec_benchmark_rootfull }}" - shell: "{{ cleanup_commands }}" diff --git a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/windows.yml b/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/windows.yml deleted file mode 100644 index 26e0ec4d6b..0000000000 --- a/projects/container_bench/toolbox/container_bench_exec_benchmark/tasks/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - ansible.windows.win_file: - path: "{{ artifact_extra_logs_dir }}\\artifacts" - state: directory - -# Prepare container environment -- name: Prepare container environment - ansible.windows.win_shell: "{{ prepare_commands }}" - -# Run benchmark commands -- name: Run benchmark commands - ansible.windows.win_shell: | - & "{{ container_bench_exec_benchmark_exec_time_path | dirname }}\.venv\Scripts\python.exe" ` - "{{ container_bench_exec_benchmark_exec_time_path }}" ` - --output "{{ artifact_extra_logs_dir }}\artifacts\output.log" ` - --metrics-log-file "{{ artifact_extra_logs_dir }}\artifacts\metrics.json" ` - {{ benchmark_command }} - -# Clean up container environment -- name: Clean up container environment - ansible.windows.win_shell: "{{ cleanup_commands }}" diff --git a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/main.yml b/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/main.yml index 9880a53f15..f089c029a6 100644 --- a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/main.yml +++ b/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/main.yml @@ -1,19 +1,29 @@ --- -# Define common command templates (shared across all platforms) -- name: Set common commands +- name: Validate required variables are defined + assert: + that: + - container_bench_helloworld_benchmark_exec_time_path is defined + - container_bench_helloworld_benchmark_binary_path is defined + - container_bench_helloworld_benchmark_rootfull is defined + - container_bench_helloworld_benchmark_additional_args is defined + when: false # This task never runs but validates variable definitions + +- include_tasks: "../../shared_tasks/map_variables.yml" + vars: + role_name: "container_bench_helloworld_benchmark" + +- name: Set common commands (shared across all platforms) set_fact: - prepare_commands: | - {{ container_bench_helloworld_benchmark_binary_path }} {{ container_bench_helloworld_benchmark_additional_args }} system prune -a -f - {{ container_bench_helloworld_benchmark_binary_path }} {{ container_bench_helloworld_benchmark_additional_args }} pull quay.io/podman/hello - cleanup_commands: | - {{ container_bench_helloworld_benchmark_binary_path }} {{ container_bench_helloworld_benchmark_additional_args }} system prune -a -f + image_to_pull: "quay.io/podman/hello" + prepare_commands: "" # Image is pulled using shared pull_image task below + cleanup_commands: "" # Cleanup handled by shared tasks benchmark_command: | - {{ container_bench_helloworld_benchmark_binary_path }} {{ container_bench_helloworld_benchmark_additional_args }} run --rm quay.io/podman/hello + {{ binary_path }} {{ additional_args }} run --rm quay.io/podman/hello - name: Include Unix/Linux tasks - include_tasks: unix.yml + include_tasks: "../../shared_tasks/unix_default_benchmark_body.yml" when: ansible_os_family != 'Windows' - name: Include Windows tasks - include_tasks: windows.yml + include_tasks: "../../shared_tasks/windows_default_benchmark_body.yml" when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/unix.yml b/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/unix.yml deleted file mode 100644 index 04628b7671..0000000000 --- a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/unix.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' - -# Prepare container environment -- name: Prepare container environment - become: "{{ container_bench_helloworld_benchmark_rootfull }}" - shell: "{{ prepare_commands }}" - -# Run benchmark commands -- name: Run benchmark commands - become: "{{ container_bench_helloworld_benchmark_rootfull }}" - shell: | - {{ container_bench_helloworld_benchmark_exec_time_path | dirname }}/.venv/bin/python \ - {{ container_bench_helloworld_benchmark_exec_time_path }} \ - --output "{{ artifact_extra_logs_dir }}/artifacts/output.log" \ - --metrics-log-file "{{ artifact_extra_logs_dir }}/artifacts/metrics.json" \ - {{ benchmark_command }} - -# Clean up container environment -- name: Clean up container environment - become: "{{ container_bench_helloworld_benchmark_rootfull }}" - shell: "{{ cleanup_commands }}" diff --git a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/windows.yml b/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/windows.yml deleted file mode 100644 index 6098ce0756..0000000000 --- a/projects/container_bench/toolbox/container_bench_helloworld_benchmark/tasks/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - ansible.windows.win_file: - path: "{{ artifact_extra_logs_dir }}\\artifacts" - state: directory - -# Prepare container environment -- name: Prepare container environment - ansible.windows.win_shell: "{{ prepare_commands }}" - -# Run benchmark commands -- name: Run benchmark commands - ansible.windows.win_shell: | - & "{{ container_bench_helloworld_benchmark_exec_time_path | dirname }}\.venv\Scripts\python.exe" ` - "{{ container_bench_helloworld_benchmark_exec_time_path }}" ` - --output "{{ artifact_extra_logs_dir }}\artifacts\output.log" ` - --metrics-log-file "{{ artifact_extra_logs_dir }}\artifacts\metrics.json" ` - {{ benchmark_command }} - -# Clean up container environment -- name: Clean up container environment - ansible.windows.win_shell: "{{ cleanup_commands }}" diff --git a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/main.yaml b/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/main.yaml index 3925bffe51..8048b53c33 100644 --- a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/main.yaml +++ b/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/main.yaml @@ -1,25 +1,77 @@ --- -# Define common command templates (shared across all platforms) -- name: Set common commands +- name: Validate required variables are defined + assert: + that: + - container_bench_image_build_large_build_context_benchmark_exec_time_path is defined + - container_bench_image_build_large_build_context_benchmark_binary_path is defined + - container_bench_image_build_large_build_context_benchmark_rootfull is defined + - container_bench_image_build_large_build_context_benchmark_additional_args is defined + when: false # This task never runs but validates variable definitions + +- include_tasks: "../../shared_tasks/map_variables.yml" + vars: + role_name: "container_bench_image_build_large_build_context_benchmark" + +- name: Set common variables set_fact: - initial_prune_command: "{{ container_bench_image_build_large_build_context_benchmark_binary_path }} {{ container_bench_image_build_large_build_context_benchmark_additional_args }} system prune -a -f" - cleanup_commands: | - {{ container_bench_image_build_large_build_context_benchmark_binary_path }} {{ container_bench_image_build_large_build_context_benchmark_additional_args }} rmi -f large-build-context-benchmark - {{ container_bench_image_build_large_build_context_benchmark_binary_path }} {{ container_bench_image_build_large_build_context_benchmark_additional_args }} system prune -a -f dockerfile_content: | FROM scratch COPY random-1 random-1 COPY random-10 random-10 build_context_path: "{{ '$env:USERPROFILE\\large-build-context' if ansible_os_family == 'Windows' else '${HOME}/large-build-context' }}" -- name: Set platform-specific paths +- name: Set benchmark command + set_fact: + benchmark_command: "{{ binary_path }} {{ additional_args }} build -t large-build-context-benchmark {{ build_context_path }}" + +- name: Set Unix/Linux cleanup commands + set_fact: + cleanup_commands: | + {{ binary_path }} {{ additional_args }} rmi -f large-build-context-benchmark + rm -rf {{ build_context_path }} + when: ansible_os_family != 'Windows' + +- name: Set Windows cleanup commands set_fact: - benchmark_command: "{{ container_bench_image_build_large_build_context_benchmark_binary_path }} {{ container_bench_image_build_large_build_context_benchmark_additional_args }} build -t large-build-context-benchmark {{ build_context_path }}" + cleanup_commands: | + {{ binary_path }} {{ additional_args }} rmi -f large-build-context-benchmark + $buildContextDir = "{{ build_context_path }}" + if (Test-Path $buildContextDir) { Remove-Item -Recurse -Force $buildContextDir } + when: ansible_os_family == 'Windows' + +- name: Set Unix/Linux prepare commands + set_fact: + prepare_commands: | + mkdir {{ build_context_path }} + cd {{ build_context_path }} + cat <<'EOF' > Dockerfile + {{ dockerfile_content }} + EOF + {% for i in range(1, 11) %} + dd if=/dev/urandom of=random-{{ i }} bs=1G count=1 + {% endfor %} + when: ansible_os_family != 'Windows' + +- name: Set Windows prepare commands + set_fact: + prepare_commands: | + $buildContextDir = "{{ build_context_path }}" + if (Test-Path $buildContextDir) { Remove-Item -Recurse -Force $buildContextDir } + New-Item -ItemType Directory -Force -Path $buildContextDir + $dockerfileContent = @' + {{ dockerfile_content }} + '@ + Set-Content -Path "$buildContextDir\Dockerfile" -Value $dockerfileContent + for ($i = 1; $i -le 10; $i++) { + $randomFile = "$buildContextDir\random-$i" + fsutil file createnew $randomFile 1073741824 + } + when: ansible_os_family == 'Windows' - name: Include Unix/Linux tasks - include_tasks: unix.yml + include_tasks: "../../shared_tasks/unix_default_benchmark_body.yml" when: ansible_os_family != 'Windows' - name: Include Windows tasks - include_tasks: windows.yml + include_tasks: "../../shared_tasks/windows_default_benchmark_body.yml" when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/unix.yml b/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/unix.yml deleted file mode 100644 index 03a0f33b82..0000000000 --- a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/unix.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - file: - path: "{{ artifact_extra_logs_dir }}/artifacts" - state: directory - mode: '0755' - -# Prepare build context -- name: Prepare - become: "{{ container_bench_image_build_large_build_context_benchmark_rootfull }}" - shell: | - {{ initial_prune_command }} - mkdir ${HOME}/large-build-context - cd ${HOME}/large-build-context - cat <<'EOF' > Dockerfile - {{ dockerfile_content }} - EOF - for i in $(seq 1 10); do - dd if=/dev/urandom of=random-$i bs=1G count=1 - done - ls - -# Run benchmark commands -- name: Run benchmark commands - become: "{{ container_bench_image_build_large_build_context_benchmark_rootfull }}" - shell: | - {{ container_bench_image_build_large_build_context_benchmark_exec_time_path | dirname }}/.venv/bin/python \ - {{ container_bench_image_build_large_build_context_benchmark_exec_time_path }} \ - --output "{{ artifact_extra_logs_dir }}/artifacts/output.log" \ - --metrics-log-file "{{ artifact_extra_logs_dir }}/artifacts/metrics.json" \ - {{ benchmark_command }} - -- name: Clean up - become: "{{ container_bench_image_build_large_build_context_benchmark_rootfull }}" - shell: | - {{ cleanup_commands }} - rm -rf {{ build_context_path }} diff --git a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/windows.yml b/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/windows.yml deleted file mode 100644 index abc6bedbbf..0000000000 --- a/projects/container_bench/toolbox/container_bench_image_build_large_build_context_benchmark/tasks/windows.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -# Create artifacts directory -- name: Create the artifacts directory - ansible.windows.win_file: - path: "{{ artifact_extra_logs_dir }}\\artifacts" - state: directory - -- name: Prepare - ansible.windows.win_shell: | - {{ initial_prune_command }} - - # Create large-build-context directory - $buildContextDir = "$env:USERPROFILE\large-build-context" - if (Test-Path $buildContextDir) { Remove-Item -Recurse -Force $buildContextDir } - New-Item -ItemType Directory -Force -Path $buildContextDir - - # Create Dockerfile - $dockerfileContentBase64 = "{{ dockerfile_content | b64encode }}" - $dockerfileContent = [System.Text.Encoding]::UTF8.GetString( - [System.Convert]::FromBase64String($dockerfileContentBase64) - ) - Set-Content -Path "$buildContextDir\Dockerfile" -Value $dockerfileContent - - # Generate random files (1GB each) - for ($i = 1; $i -le 10; $i++) { - $randomFile = "$buildContextDir\random-$i" - fsutil file createnew $randomFile 1073741824 - } - - # List files - Get-ChildItem $buildContextDir - -- name: Run benchmark commands - ansible.windows.win_shell: | - & "{{ container_bench_image_build_large_build_context_benchmark_exec_time_path | dirname }}\.venv\Scripts\python.exe" ` - "{{ container_bench_image_build_large_build_context_benchmark_exec_time_path }}" ` - --output "{{ artifact_extra_logs_dir }}\artifacts\output.log" ` - --metrics-log-file "{{ artifact_extra_logs_dir }}\artifacts\metrics.json" ` - {{ benchmark_command }} - -- name: Clean up - ansible.windows.win_shell: | - {{ cleanup_commands }} - $buildContextDir = "{{ build_context_path | replace('\\', '\\\\') }}" - if (Test-Path $buildContextDir) { Remove-Item -Recurse -Force $buildContextDir } diff --git a/projects/container_bench/toolbox/shared_tasks/cleanup_container_resources.yml b/projects/container_bench/toolbox/shared_tasks/cleanup_container_resources.yml new file mode 100644 index 0000000000..2b1117db42 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/cleanup_container_resources.yml @@ -0,0 +1,96 @@ +--- +# Comprehensive cleanup of container resources (containers, images, volumes, networks) +# Usage: +# - include_tasks: "../../shared_tasks/cleanup_container_resources.yml" +# Requires: binary_path variable (additional_args, rootfull optional) + +- name: Validate required cleanup variables + fail: + msg: "Required variable 'binary_path' is not defined" + when: binary_path is not defined + +- name: Stop all running containers (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} stop $({{ binary_path }} {{ additional_args | default('') }} ps -q) 2>/dev/null || true" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: Stop all running containers (Windows) + ansible.windows.win_shell: | + $ids = & {{ binary_path }} {{ additional_args | default('') }} ps -q 2>$null + if ($ids) { & {{ binary_path }} {{ additional_args | default('') }} stop $ids 2>$null } + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' + +- name: Remove all containers (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} rm -f $({{ binary_path }} {{ additional_args | default('') }} ps -aq) 2>/dev/null || true" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: Remove all containers (Windows) + ansible.windows.win_shell: | + $ids = & {{ binary_path }} {{ additional_args | default('') }} ps -aq 2>$null + if ($ids) { & {{ binary_path }} {{ additional_args | default('') }} rm -f $ids 2>$null } + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' + +- name: Remove all images (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} rmi -f $({{ binary_path }} {{ additional_args | default('') }} images -q) 2>/dev/null || true" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: Remove all images (Windows) + ansible.windows.win_shell: | + $ids = & {{ binary_path }} {{ additional_args | default('') }} images -q 2>$null + if ($ids) { & {{ binary_path }} {{ additional_args | default('') }} rmi -f $ids 2>$null } + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' + +- name: Remove all volumes (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} volume rm -f $({{ binary_path }} {{ additional_args | default('') }} volume ls -q) 2>/dev/null || true" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: Remove all volumes (Windows) + ansible.windows.win_shell: | + $ids = & {{ binary_path }} {{ additional_args | default('') }} volume ls -q 2>$null + if ($ids) { & {{ binary_path }} {{ additional_args | default('') }} volume rm -f $ids 2>$null } + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' + +- name: System prune (remove unused containers, networks, images) (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} system prune -a -f" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: System prune (remove unused containers, networks, images) (Windows) + ansible.windows.win_shell: "{{ binary_path }} {{ additional_args | default('') }} system prune -a -f" + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' + +- name: System prune with volumes (remove unused volumes as well) (Unix/Linux) + become: "{{ rootfull | default(false) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} system prune --volumes -a -f" + ignore_errors: true + changed_when: false + when: ansible_os_family != 'Windows' + +- name: System prune with volumes (remove unused volumes as well) (Windows) + ansible.windows.win_shell: "{{ binary_path }} {{ additional_args | default('') }} system prune --volumes -a -f" + ignore_errors: true + changed_when: false + when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/shared_tasks/create_artifacts_dir.yml b/projects/container_bench/toolbox/shared_tasks/create_artifacts_dir.yml new file mode 100644 index 0000000000..042e906d95 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/create_artifacts_dir.yml @@ -0,0 +1,17 @@ +--- +# Shared tasks for creating artifacts directory across platforms +# Usage: +# - include_tasks: "../../shared_tasks/create_artifacts_dir.yml" + +- name: Create artifacts directory (Unix/Linux/macOS) + file: + path: "{{ artifact_extra_logs_dir }}/artifacts" + state: directory + mode: '0755' + when: ansible_os_family != 'Windows' + +- name: Create artifacts directory (Windows) + ansible.windows.win_file: + path: "{{ artifact_extra_logs_dir }}\\artifacts" + state: directory + when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/shared_tasks/map_variables.yml b/projects/container_bench/toolbox/shared_tasks/map_variables.yml new file mode 100644 index 0000000000..6b96d6cfb2 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/map_variables.yml @@ -0,0 +1,13 @@ +--- +# Shared task for mapping long variable names to short aliases +# Usage: +# - include_tasks: "../../shared_tasks/map_variables.yml" +# vars: +# role_name: "container_bench_exec_benchmark" + +- name: Map long variable names to short aliases for readability + set_fact: + binary_path: "{{ vars[role_name + '_binary_path'] }}" + additional_args: "{{ vars[role_name + '_additional_args'] }}" + exec_time_path: "{{ vars[role_name + '_exec_time_path'] }}" + rootfull: "{{ vars[role_name + '_rootfull'] }}" diff --git a/projects/container_bench/toolbox/shared_tasks/pull_image.yml b/projects/container_bench/toolbox/shared_tasks/pull_image.yml new file mode 100644 index 0000000000..0dc5ecb089 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/pull_image.yml @@ -0,0 +1,86 @@ +--- +# Shared task for optimized image pulling/loading +# Usage: +# - include_tasks: "../../shared_tasks/pull_image.yml" +# vars: +# image_name: "quay.io/example/image:tag" +# Requires: binary_path, additional_args, image_name variables +# Note: Image names are sanitized by replacing all special characters with '_' + +- name: Validate required variables for image pulling + fail: + msg: "Required variable '{{ item }}' is not defined" + when: vars[item] is not defined + loop: + - binary_path + - image_name + - environment + +- name: Set path to directory with images tarfiles + set_fact: + images_path: "{{ vars['environment'][0]['HOME'] + '\\images' if ansible_os_family == 'Windows' else vars['environment'][0]['HOME'] + '/images' }}" + +- name: Ensure images directory exists (Unix/Linux/macOS) + file: + path: "{{ images_path }}" + state: directory + mode: '0755' + when: ansible_os_family != 'Windows' + +- name: Ensure images directory exists (Windows) + ansible.windows.win_file: + path: "{{ images_path }}" + state: directory + when: ansible_os_family == 'Windows' + +- name: Set image tarfile path (Unix/Linux/macOS) + set_fact: + image_tarfile: "{{ images_path }}/{{ image_name | regex_replace('[^a-zA-Z0-9_-]', '_') }}.tar" + when: ansible_os_family != 'Windows' + +- name: Set image tarfile path (Windows) + set_fact: + image_tarfile: "{{ images_path }}\\{{ image_name | regex_replace('[^a-zA-Z0-9_-]', '_') }}.tar" + when: ansible_os_family == 'Windows' + +- name: Check if image tarfile exists (Unix/Linux/macOS) + stat: + path: "{{ image_tarfile }}" + register: tarfile_stat_unix + when: ansible_os_family != 'Windows' + +- name: Check if image tarfile exists (Windows) + ansible.windows.win_stat: + path: "{{ image_tarfile }}" + register: tarfile_stat_windows + when: ansible_os_family == 'Windows' + +- name: Set unified tarfile status (Unix) + set_fact: + tarfile_exists: "{{ tarfile_stat_unix.stat.exists | default(false) }}" + when: ansible_os_family != 'Windows' + +- name: Set unified tarfile status (Windows) + set_fact: + tarfile_exists: "{{ tarfile_stat_windows.stat.exists | default(false) }}" + when: ansible_os_family == 'Windows' + +- name: Load image from local tarfile (Unix/Linux/macOS) + become: "{{ rootfull | default(omit) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} load -i {{ image_tarfile }}" + when: ansible_os_family != 'Windows' and tarfile_exists + + +- name: Load image from local tarfile (Windows) + ansible.windows.win_shell: "{{ binary_path }} {{ additional_args | default('') }} load -i {{ image_tarfile }}" + when: ansible_os_family == 'Windows' and tarfile_exists + +- name: Pull image from registry (Unix/Linux/macOS) + become: "{{ rootfull | default(omit) }}" + shell: "{{ binary_path }} {{ additional_args | default('') }} pull {{ image_name }}" + when: ansible_os_family != 'Windows' and not tarfile_exists + +- name: Pull image from registry (Windows) + ansible.windows.win_shell: "{{ binary_path }} {{ additional_args | default('') }} pull {{ image_name }}" + when: ansible_os_family == 'Windows' and not tarfile_exists + diff --git a/projects/container_bench/toolbox/shared_tasks/run_benchmark.yml b/projects/container_bench/toolbox/shared_tasks/run_benchmark.yml new file mode 100644 index 0000000000..518fdc5570 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/run_benchmark.yml @@ -0,0 +1,33 @@ +--- +# Shared task for running benchmarked commands with timing and metrics collection +# Usage: +# - include_tasks: "../../shared_tasks/run_benchmark.yml" +# Requires: exec_time_path, artifact_extra_logs_dir, benchmark_command variables + +- name: Validate required benchmark variables + fail: + msg: "Required variable '{{ item }}' is not defined" + when: vars[item] is not defined + loop: + - exec_time_path + - artifact_extra_logs_dir + - benchmark_command + +- name: Run benchmark commands (Unix/Linux) + become: "{{ rootfull | default(omit) }}" + shell: | + {{ exec_time_path | dirname }}/.venv/bin/python \ + {{ exec_time_path }} \ + --output "{{ artifact_extra_logs_dir }}/artifacts/output.log" \ + --metrics-log-file "{{ artifact_extra_logs_dir }}/artifacts/metrics.json" \ + {{ benchmark_command }} + when: ansible_os_family != 'Windows' + +- name: Run benchmark commands (Windows) + ansible.windows.win_shell: | + & "{{ exec_time_path | dirname }}\.venv\Scripts\python.exe" ` + "{{ exec_time_path }}" ` + --output "{{ artifact_extra_logs_dir }}\artifacts\output.log" ` + --metrics-log-file "{{ artifact_extra_logs_dir }}\artifacts\metrics.json" ` + {{ benchmark_command }} + when: ansible_os_family == 'Windows' diff --git a/projects/container_bench/toolbox/shared_tasks/unix_default_benchmark_body.yml b/projects/container_bench/toolbox/shared_tasks/unix_default_benchmark_body.yml new file mode 100644 index 0000000000..b6ca0811b0 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/unix_default_benchmark_body.yml @@ -0,0 +1,42 @@ +--- +# Default benchmark body for Unix/Linux/Darwin platforms +# Usage: +# - include_tasks: "../../shared_tasks/unix_default_benchmark_body.yml" +# Requires: image_to_pull, prepare_commands, benchmark_command, cleanup_commands variables + +- name: Validate required variables + fail: + msg: "Required variable '{{ item }}' is not defined" + when: vars[item] is not defined + loop: + - prepare_commands + - benchmark_command + - cleanup_commands + +- name: Create benchmark artifacts directory + include_tasks: "create_artifacts_dir.yml" + +- name: Perform full container resource cleanup before benchmark + include_tasks: "cleanup_container_resources.yml" + +- name: Pull required container image (with local tarfile optimization) + include_tasks: "pull_image.yml" + vars: + image_name: "{{ image_to_pull }}" + when: image_to_pull is defined and image_to_pull | length > 0 + +- name: Execute benchmark preparation commands + become: "{{ rootfull | default(false) }}" + shell: "{{ prepare_commands }}" + when: prepare_commands | length > 0 + +- name: Execute benchmark measurement + include_tasks: "run_benchmark.yml" + +- name: Execute benchmark cleanup commands + become: "{{ rootfull | default(false) }}" + shell: "{{ cleanup_commands }}" + when: cleanup_commands | length > 0 + +- name: Perform full container resource cleanup after benchmark + include_tasks: "cleanup_container_resources.yml" diff --git a/projects/container_bench/toolbox/shared_tasks/windows_default_benchmark_body.yml b/projects/container_bench/toolbox/shared_tasks/windows_default_benchmark_body.yml new file mode 100644 index 0000000000..23a0a7c628 --- /dev/null +++ b/projects/container_bench/toolbox/shared_tasks/windows_default_benchmark_body.yml @@ -0,0 +1,39 @@ +--- +# Default benchmark body for Windows platforms +# Usage: include_tasks: "../../shared_tasks/windows_default_benchmark_body.yml" +# Requires: image_to_pull, prepare_commands, benchmark_command, cleanup_commands variables + +- name: Validate required variables + fail: + msg: "Required variable '{{ item }}' is not defined" + when: vars[item] is not defined + loop: + - prepare_commands + - benchmark_command + - cleanup_commands + +- name: Create benchmark artifacts directory + include_tasks: "create_artifacts_dir.yml" + +- name: Perform full container resource cleanup before benchmark + include_tasks: "cleanup_container_resources.yml" + +- name: Pull required container image (with local tarfile optimization) + include_tasks: "pull_image.yml" + vars: + image_name: "{{ image_to_pull }}" + when: image_to_pull is defined and image_to_pull | length > 0 + +- name: Execute benchmark preparation commands + ansible.windows.win_shell: "{{ prepare_commands }}" + when: prepare_commands | length > 0 + +- name: Execute benchmark measurement + include_tasks: "run_benchmark.yml" + +- name: Execute benchmark cleanup commands + ansible.windows.win_shell: "{{ cleanup_commands }}" + when: cleanup_commands | length > 0 + +- name: Perform full container resource cleanup after benchmark + include_tasks: "cleanup_container_resources.yml" diff --git a/projects/container_bench/visualizations/benchmark/plotting/comparison_report.py b/projects/container_bench/visualizations/benchmark/plotting/comparison_report.py index cdd6f6c50b..6ee426f73f 100644 --- a/projects/container_bench/visualizations/benchmark/plotting/comparison_report.py +++ b/projects/container_bench/visualizations/benchmark/plotting/comparison_report.py @@ -9,11 +9,6 @@ get_all_configuration_info, generate_display_config_label ) -from .utils.metrics import ( - METRIC_DISPLAY_CONFIG, - calculate_config_metrics, - calculate_usage_deltas -) from .utils.shared import ( MIN_PLOT_BENCHMARK_TIME, @@ -24,12 +19,10 @@ create_table_header, create_config_cell, create_execution_time_content, - create_metric_value_cell, create_delta_content, create_code_cell, format_benchmark_title, has_long_running_benchmarks, - create_usage_table_headers, create_summary_info_card, create_host_info_card, create_engine_info_card, @@ -37,7 +30,11 @@ def create_performance_delta_row(sorted_configurations, fastest_time): - valid_times = [c.get("exec_time", 0) for c in sorted_configurations if c.get("exec_time", 0) > 0] + valid_times = [ + c.get("execution_time_95th_percentile", 0) + for c in sorted_configurations + if c.get("execution_time_95th_percentile", 0) > 0 + ] if len(valid_times) <= 1: return None @@ -97,14 +94,26 @@ def create_shared_info_section(shared_info): if summary_card: cards.append(summary_card) - host_card = create_host_info_card(shared_info.get("system"), SYSTEM_INFO_FIELD_MAPPINGS) + host_card = create_host_info_card( + shared_info.get("system"), + title="Host System Information", + field_mappings=SYSTEM_INFO_FIELD_MAPPINGS + ) if host_card: cards.append(host_card) + # Filter out Client_version for Linux systems + system_info = shared_info.get("system", {}) + is_linux = "linux" in system_info.get("OS_version", "").lower() + filtered_engine_mappings = ENGINE_INFO_FIELD_MAPPINGS.copy() + if is_linux: + filtered_engine_mappings = {k: v for k, v in ENGINE_INFO_FIELD_MAPPINGS.items() if k != "Client_version"} + engine_card = create_engine_info_card( shared_info.get("engine"), - shared_info.get("container_engine_provider"), - ENGINE_INFO_FIELD_MAPPINGS + title="Container Engine Information", + provider_info=shared_info.get("container_engine_provider") if not is_linux else None, + field_mappings=filtered_engine_mappings ) if engine_card: cards.append(engine_card) @@ -119,29 +128,35 @@ def create_performance_comparison_table(configurations): if not configurations: return None - sorted_configurations = sorted(configurations, key=lambda config: config.get("exec_time", 0)) + sorted_configurations = sorted(configurations, key=lambda config: config.get("execution_time_95th_percentile", 0)) rows = [] - headers = ["Configuration", "Execution Time", "Command", "Timestamp"] + headers = ["Configuration", "Execution Time (95th Percentile)", "Command", "Timestamp"] header_row = create_table_header(headers) rows.append(header_row) - valid_times = [c.get("exec_time", 0) for c in sorted_configurations if c.get("exec_time", 0) > 0] + valid_times = [ + c.get("execution_time_95th_percentile", 0) + for c in sorted_configurations + if c.get("execution_time_95th_percentile", 0) > 0 + ] fastest_time = min(valid_times) if valid_times else 0 for config in sorted_configurations: display_config_label = generate_display_config_label(config, sorted_configurations) - exec_time = config.get("exec_time", 0) - runs = config.get("runs", 1) + exec_time_95th_percentile = config.get("execution_time_95th_percentile", 0) + jitter = config.get("jitter", 0) + jitter = jitter if jitter is not None else 0 + command = config.get("command", "N/A") timestamp = config.get("timestamp", "N/A") - is_fastest = (exec_time == fastest_time and + is_fastest = (exec_time_95th_percentile == fastest_time and len(sorted_configurations) > 1 and - exec_time > 0) + exec_time_95th_percentile > 0) - time_content = create_execution_time_content(exec_time, runs, is_fastest) + time_content = create_execution_time_content(exec_time_95th_percentile, jitter, is_fastest) row_cells = [ create_config_cell(display_config_label), @@ -165,95 +180,6 @@ def create_performance_comparison_table(configurations): return html.Table(rows, style=css.STYLE_COMPARISON_TABLE) -def create_average_usage_comparison_table(configurations): - if not configurations: - return None - - has_metrics = any(config.get('exec_time', 0) > MIN_PLOT_BENCHMARK_TIME for config in configurations) - if not has_metrics: - return None - - sorted_configurations = sorted(configurations, key=lambda config: config.get("exec_time", 0)) - - config_metrics = {} - for config in sorted_configurations: - config_label = config.get("config_label", "Unknown") - settings = config.get("settings", {}) - config_metrics[config_label] = calculate_config_metrics(settings) - - if not any(config_metrics.values()): - return None - - rows = [] - - headers = create_usage_table_headers(METRIC_DISPLAY_CONFIG) - header_row = create_table_header(headers) - rows.append(header_row) - - metric_ranges = {} - for metric_type in METRIC_DISPLAY_CONFIG.keys(): - values = [] - for config in sorted_configurations: - config_label = config.get("config_label", "Unknown") - metrics = config_metrics.get(config_label, {}) - value = metrics.get(metric_type) - if value is not None: - values.append(value) - - if values: - min_val = min(values) - max_val = max(values) - metric_ranges[metric_type] = { - 'min': min_val, - 'max': max_val, - 'has_multiple': len(values) > 1 and min_val != max_val # Check if values are actually different - } - - for config in sorted_configurations: - config_label = config.get("config_label", "Unknown") - display_config_label = generate_display_config_label(config, sorted_configurations) - metrics = config_metrics.get(config_label, {}) - - row_cells = [create_config_cell(display_config_label)] - - for metric_type, (label, unit) in METRIC_DISPLAY_CONFIG.items(): - value = metrics.get(metric_type) - if value is not None: - # Determine if this value is min or max for highlighting - metric_range = metric_ranges.get(metric_type, {}) - is_min = metric_range.get('has_multiple', False) and value == metric_range.get('min') - is_max = metric_range.get('has_multiple', False) and value == metric_range.get('max') - is_single = len(sorted_configurations) == 1 - - metric_cell = create_metric_value_cell(value, unit, is_min, is_max, is_single) - row_cells.append(metric_cell) - else: - row_cells.append(html.Td("N/A", style=css.STYLE_TABLE_CELL)) - - rows.append(html.Tr(row_cells)) - - if len(sorted_configurations) > 1: - usage_deltas = calculate_usage_deltas(sorted_configurations) - if usage_deltas: - delta_cells = [create_config_cell("Performance Delta")] - - for metric_type, (label, unit) in METRIC_DISPLAY_CONFIG.items(): - if metric_type in usage_deltas: - delta_info = usage_deltas[metric_type] - delta_content = create_delta_content( - delta_info['delta'], - delta_info['percentage'], - unit - ) - delta_cells.append(html.Td(delta_content, style=css.STYLE_TABLE_CELL)) - else: - delta_cells.append(html.Td("-", style=css.STYLE_TABLE_CELL)) - - rows.append(html.Tr(delta_cells)) - - return html.Table(rows, style=css.STYLE_COMPARISON_TABLE) - - def create_differences_comparison_table(configurations): if not configurations or len(configurations) < 2: return html.Div("No differences found between configurations") @@ -267,13 +193,6 @@ def create_differences_comparison_table(configurations): perf_table ])) - usage_table = create_average_usage_comparison_table(configurations) - if usage_table: - tables.append(html.Div([ - html.H4("Average System Usage", style=css.STYLE_H4), - usage_table - ])) - if tables: return html.Div([table for table in tables if table], style={'margin-bottom': '1rem'}) else: @@ -321,7 +240,7 @@ def create_plots_section(configurations, args): plot_sections = [] for config in configurations: - if config.get('exec_time', 0) > MIN_PLOT_BENCHMARK_TIME: + if config.get('execution_time_95th_percentile', 0) > MIN_PLOT_BENCHMARK_TIME: settings = config.get("settings", {}) plot_cards = [ diff --git a/projects/container_bench/visualizations/benchmark/plotting/metrics.py b/projects/container_bench/visualizations/benchmark/plotting/metrics.py index b4e7568d83..4f5aac6ca5 100644 --- a/projects/container_bench/visualizations/benchmark/plotting/metrics.py +++ b/projects/container_bench/visualizations/benchmark/plotting/metrics.py @@ -1,9 +1,11 @@ +import types import pandas as pd import plotly.graph_objects as go import matrix_benchmarking.plotting.table_stats as table_stats import matrix_benchmarking.common as common -from .utils.metrics import BYTES_TO_MEGABYTES + +BYTES_TO_MEGABYTES = 1024 * 1024 METRIC_TYPES = { "cpu": { @@ -28,12 +30,35 @@ } } +COLOR_GREEN = "rgba(44,160,44,1)" +COLOR_GREEN_FILL = "rgba(44,160,44,0.14)" +COLOR_RED = "rgba(214,39,40,1)" +COLOR_RED_FILL = "rgba(214,39,40,0.14)" +COLOR_BLUE = "rgba(31,119,180,1)" +COLOR_JITTER_FILL_DEFAULT = "rgba(100,100,200,0.12)" +DEFAULT_LINE_COLOR = "rgba(50,50,50,1)" +JITTER_LINE_TRANSPARENT = "rgba(255,255,255,0)" + def register(): for metric_type in METRIC_TYPES.keys(): MetricUsage(metric_type) +def _to_mb_s(item, interval): + if isinstance(item, types.SimpleNamespace): + percentile_50th = (item.percentile_50th / BYTES_TO_MEGABYTES) / interval + percentile_75th = (item.percentile_75th / BYTES_TO_MEGABYTES) / interval + percentile_25th = (item.percentile_25th / BYTES_TO_MEGABYTES) / interval + return types.SimpleNamespace( + percentile_50th=percentile_50th, + percentile_75th=percentile_75th, + percentile_25th=percentile_25th + ) + val = (item / BYTES_TO_MEGABYTES) / interval + return types.SimpleNamespace(percentile_50th=val, percentile_75th=val, percentile_25th=val) + + def _process_network_data(main_field, entry_name, interval): data = [] if not hasattr(main_field, "network") or not main_field.network: @@ -45,8 +70,8 @@ def _process_network_data(main_field, entry_name, interval): if not send_data or not recv_data: return data - mb_s_sent = [(item / BYTES_TO_MEGABYTES) / interval for item in send_data] - mb_s_recv = [(item / BYTES_TO_MEGABYTES) / interval for item in recv_data] + mb_s_sent = [_to_mb_s(item, interval) for item in send_data] + mb_s_recv = [_to_mb_s(item, interval) for item in recv_data] time_points = [i * interval for i in range(len(send_data))] for send, recv, timestamp in zip(mb_s_sent, mb_s_recv, time_points): @@ -70,8 +95,8 @@ def _process_disk_data(main_field, entry_name, interval): if not read_data or not write_data: return data - read_mb_s = [(item / BYTES_TO_MEGABYTES) / interval for item in read_data] - write_mb_s = [(item / BYTES_TO_MEGABYTES) / interval for item in write_data] + read_mb_s = [_to_mb_s(item, interval) for item in read_data] + write_mb_s = [_to_mb_s(item, interval) for item in write_data] time_points = [i * interval for i in range(len(read_data))] for read, write, timestamp in zip(read_mb_s, write_mb_s, time_points): @@ -146,30 +171,104 @@ def do_hover(self, meta_value, variables, figure, data, click_info): return "nothing" def _create_dual_metric_traces(self, df, metric_keys): - traces = [] + traces_by_key = {} + fill_colors = { + "send": COLOR_GREEN_FILL, + "recv": COLOR_RED_FILL, + "read": COLOR_GREEN_FILL, + "write": COLOR_RED_FILL, + } + line_colors = { + "send": COLOR_GREEN, + "recv": COLOR_RED, + "read": COLOR_GREEN, + "write": COLOR_RED, + } for key in metric_keys: + traces_for_key = [] + if df[key].apply(lambda v: isinstance(v, types.SimpleNamespace)).all(): + values = df[key].apply(lambda v: v.percentile_50th if isinstance(v, types.SimpleNamespace) else v) + upper = df[key].apply(lambda v: v.percentile_75th if isinstance(v, types.SimpleNamespace) else 0) + lower = df[key].apply(lambda v: v.percentile_25th if isinstance(v, types.SimpleNamespace) else 0) + traces_for_key.append( + go.Scatter( + x=list(df["ts"]) + list(df["ts"][::-1]), + y=list(upper) + list(lower[::-1]), + fill="toself", + fillcolor=fill_colors.get(key, COLOR_JITTER_FILL_DEFAULT), + line=dict(color=JITTER_LINE_TRANSPARENT), + hoverinfo="skip", + name=f"Range (P25 - P75) ({key.capitalize()})", + showlegend=True, + legendgroup=key, + ) + ) + traces_for_key.append( + go.Scatter( + x=df["ts"], + y=values, + mode="lines", + name=f"{key.capitalize()} ({self.metric_config['unit']}) (Median)", + legendgroup=key, + line=dict(color=line_colors.get(key, DEFAULT_LINE_COLOR), width=2, shape="linear"), + hoverinfo="x+y+name", + ) + ) + else: + traces_for_key.append( + go.Scatter( + x=df["ts"], + y=df[key], + mode="lines", + name=f"{key.capitalize()} ({self.metric_config['unit']})", + legendgroup=key, + line=dict(color=line_colors.get(key, DEFAULT_LINE_COLOR), width=2, shape="linear"), + hoverinfo="x+y+name", + ) + ) + traces_by_key[key] = traces_for_key + return traces_by_key + + def _create_single_metric_trace(self, df): + traces = [] + if df["usage"].apply(lambda v: isinstance(v, types.SimpleNamespace)).all(): + values = df["usage"].apply(lambda v: v.percentile_50th if isinstance(v, types.SimpleNamespace) else v) + upper = df["usage"].apply(lambda v: v.percentile_75th if isinstance(v, types.SimpleNamespace) else 0) + lower = df["usage"].apply(lambda v: v.percentile_25th if isinstance(v, types.SimpleNamespace) else 0) + traces.append( + go.Scatter( + x=list(df["ts"]) + list(df["ts"][::-1]), + y=list(upper) + list(lower[::-1]), + fill="toself", + fillcolor=COLOR_JITTER_FILL_DEFAULT, + line=dict(color=JITTER_LINE_TRANSPARENT), + hoverinfo="skip", + showlegend=True, + name="Range (P25 - P75)", + ) + ) traces.append( go.Scatter( x=df["ts"], - y=df[key], + y=values, mode="lines", - name=f"{key.capitalize()} ({self.metric_config['unit']})", - legendgroup=key, - line=dict(shape="linear"), + name=f"{self.metric_config['name']} ({self.metric_config['unit']}) (Median)", + line=dict(color=COLOR_BLUE, width=2, shape="linear"), hoverinfo="x+y+name", ) ) - return traces - - def _create_single_metric_trace(self, df): - return go.Scatter( - x=df["ts"], - y=df["usage"], - mode="lines", - name=f"{self.metric_config['name']} ({self.metric_config['unit']})", - line=dict(shape="linear"), - hoverinfo="x+y+name", - ) + return traces + else: + return [ + go.Scatter( + x=df["ts"], + y=df["usage"], + mode="lines", + name=f"{self.metric_config['name']} ({self.metric_config['unit']})", + line=dict(shape="linear"), + hoverinfo="x+y+name", + ) + ] def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): current_settings = cfg.get("current_settings", False) @@ -182,16 +281,22 @@ def do_plot(self, ordered_vars, settings, setting_lists, variables, cfg): df = df.sort_values(by=["ts", "name"], ascending=False) fig = go.Figure() - # Create traces based on metric type if self.key in ["network", "disk"]: metric_keys = ["send", "recv"] if self.key == "network" else ["read", "write"] - traces = self._create_dual_metric_traces(df, metric_keys) - for trace in traces: - fig.add_trace(trace) + for _, group in df.groupby("name"): + group = group.sort_values(by=["ts"]) # ensure increasing time + traces_by_key = self._create_dual_metric_traces(group, metric_keys) + for key in metric_keys: + for trace in traces_by_key.get(key, []): + fig.add_trace(trace) fig.update_traces(marker=dict(size=4)) fig.update_layout(legend_title_text="Type") else: # cpu or power or memory - fig.add_trace(self._create_single_metric_trace(df)) + for _, group in df.groupby("name"): + group = group.sort_values(by=["ts"]) + traces = self._create_single_metric_trace(group) + for trace in traces: + fig.add_trace(trace) fig.update_yaxes(title=self.metric_config["y_title"]) fig.update_xaxes(title="Time (s)") diff --git a/projects/container_bench/visualizations/benchmark/plotting/report.py b/projects/container_bench/visualizations/benchmark/plotting/report.py index 9dfce13ebd..2f74ec416d 100644 --- a/projects/container_bench/visualizations/benchmark/plotting/report.py +++ b/projects/container_bench/visualizations/benchmark/plotting/report.py @@ -47,19 +47,17 @@ def generate_host_items(system_info): def generate_engine_items(container_engine_info, system_info, provider_info): is_linux = detect_linux_system(system_info) + # Filter out Client_version for Linux systems + filtered_engine_mappings = ENGINE_INFO_FIELD_MAPPINGS.copy() + if is_linux: + filtered_engine_mappings = {k: v for k, v in ENGINE_INFO_FIELD_MAPPINGS.items() if k != "Client_version"} + engine_items = create_engine_info_items( container_engine_info, provider_info if not is_linux else None, - ENGINE_INFO_FIELD_MAPPINGS + filtered_engine_mappings ) - if not is_linux: - client_version_display = ENGINE_INFO_FIELD_MAPPINGS["Client_version"] - client_version_value = container_engine_info.get('Client_version', 'N/A') - - if len(engine_items) > 1: - engine_items.insert(1, (client_version_display, client_version_value, False, False)) - formatted_items = [] for i, (name, value, is_last, highlight) in enumerate(engine_items): field_key = None @@ -108,7 +106,7 @@ def generate_one_benchmark_report(report_components, settings, benchmark, args): ]) body = [info_section] - if info.get('exec_time', 0) > MIN_PLOT_BENCHMARK_TIME: # Only show plots for longer benchmarks + if info.get('execution_time_95th_percentile', 0) > MIN_PLOT_BENCHMARK_TIME: # Only show plots for longer benchmarks plot_names = [ "System CPU Usage", "System Memory Usage", diff --git a/projects/container_bench/visualizations/benchmark/plotting/utils/config.py b/projects/container_bench/visualizations/benchmark/plotting/utils/config.py index e43c1503fb..8e463087b9 100644 --- a/projects/container_bench/visualizations/benchmark/plotting/utils/config.py +++ b/projects/container_bench/visualizations/benchmark/plotting/utils/config.py @@ -8,9 +8,10 @@ detect_linux_system, detect_windows_system ) + CONFIGURATION_EXCLUDED_KEYS = { "container_engine", "benchmark", "benchmark_runs", "stats", - "test_mac_ai", "platform", "repo_version", + "test_mac_ai", "platform", "repo_version", "test.podman.machine_provider", "test.podman.repo_version", "test.docker.repo_version" } @@ -169,7 +170,8 @@ def GetInfo(settings): continue data.update({ - "exec_time": metrics.execution_time, + "execution_time_95th_percentile": metrics.execution_time_95th_percentile, + "jitter": metrics.execution_time_jitter, "command": metrics.command, "timestamp": metrics.timestamp, "runs": entry.settings.__dict__.get("benchmark_runs", 1) diff --git a/projects/container_bench/visualizations/benchmark/plotting/utils/metrics.py b/projects/container_bench/visualizations/benchmark/plotting/utils/metrics.py deleted file mode 100644 index 9e036f973a..0000000000 --- a/projects/container_bench/visualizations/benchmark/plotting/utils/metrics.py +++ /dev/null @@ -1,145 +0,0 @@ -import matrix_benchmarking.common as common - -BYTES_TO_MEGABYTES = 1024 * 1024 - -SUPPORTED_METRIC_TYPES = [ - 'cpu', 'memory', 'network_send', 'network_recv', 'disk_read', 'disk_write' -] - -METRIC_DISPLAY_CONFIG = { - 'cpu': ('Average CPU Usage', '%'), - 'memory': ('Average Memory Usage', '%'), - 'network_send': ('Average Network Send', 'MB/s'), - 'network_recv': ('Average Network Recv', 'MB/s'), - 'disk_read': ('Average Disk Read', 'MB/s'), - 'disk_write': ('Average Disk Write', 'MB/s') -} - -METRIC_CALCULATION_CONFIG = { - 'cpu': {'type': 'simple', 'attribute': 'cpu'}, - 'memory': {'type': 'simple', 'attribute': 'memory'}, - 'network_send': {'type': 'network', 'direction': 'send'}, - 'network_recv': {'type': 'network', 'direction': 'recv'}, - 'disk_read': {'type': 'disk', 'operation': 'read'}, - 'disk_write': {'type': 'disk', 'operation': 'write'} -} - - -def compute_metric_average(metrics, metric_type, interval=1): - if metric_type not in METRIC_CALCULATION_CONFIG: - return None - - config = METRIC_CALCULATION_CONFIG[metric_type] - calc_type = config['type'] - - if calc_type == 'simple': - return _calculate_simple_average(metrics, config['attribute']) - elif calc_type == 'network': - return _calculate_throughput_average(metrics, calc_type, config['direction'], interval) - elif calc_type == 'disk': - return _calculate_throughput_average(metrics, calc_type, config['operation'], interval) - return None - - -def _calculate_simple_average(metrics, attribute_name): - data = getattr(metrics, attribute_name, None) - if not data or not isinstance(data, (list, tuple)): - return None - - try: - return sum(data) / len(data) - except (TypeError, ZeroDivisionError): - return None - - -def _calculate_throughput_average(metrics, container_attr, data_key, interval): - container = getattr(metrics, container_attr, None) - if not container or not isinstance(container, dict): - return None - - data = container.get(data_key, []) - if not data or not isinstance(data, (list, tuple)): - return None - - try: - mb_per_second = [(item / BYTES_TO_MEGABYTES) / interval for item in data] - return sum(mb_per_second) / len(mb_per_second) - except (TypeError, ZeroDivisionError, ValueError): - return None - - -def calculate_config_metrics(settings): - entries = common.Matrix.filter_records(settings) - if not entries: - return {} - - config_averages = {} - - for entry in entries: - metrics = entry.results.__dict__.get("metrics") - if not metrics: - continue - - interval = getattr(metrics, 'interval', 1) - - for metric_type in SUPPORTED_METRIC_TYPES: - avg_value = compute_metric_average(metrics, metric_type, interval) - if avg_value is not None: - config_averages[metric_type] = avg_value - - return config_averages - - -def _extract_metric_values(usage_averages, metric): - metric_values = [] - metric_labels = [] - - for config_label, averages in usage_averages.items(): - if metric in averages: - metric_values.append(averages[metric]) - metric_labels.append(config_label) - - return metric_values, metric_labels - - -def _calculate_metric_delta(metric_values, metric_labels): - if len(metric_values) < 2: - return None - - min_val = min(metric_values) - max_val = max(metric_values) - delta = max_val - min_val - - min_idx = metric_values.index(min_val) - max_idx = metric_values.index(max_val) - - return { - 'delta': delta, - 'min_value': min_val, - 'max_value': max_val, - 'min_config': metric_labels[min_idx], - 'max_config': metric_labels[max_idx], - 'percentage': (delta / min_val * 100) if min_val > 0 else 0 - } - - -def calculate_usage_deltas(configurations): - if len(configurations) < 2: - return {} - - usage_averages = {} - for config in configurations: - config_label = config.get("config_label", "Unknown") - settings = config.get("settings", {}) - config_averages = calculate_config_metrics(settings) - usage_averages[config_label] = config_averages - - deltas = {} - for metric in SUPPORTED_METRIC_TYPES: - metric_values, metric_labels = _extract_metric_values(usage_averages, metric) - delta_stats = _calculate_metric_delta(metric_values, metric_labels) - - if delta_stats: - deltas[metric] = delta_stats - - return deltas diff --git a/projects/container_bench/visualizations/benchmark/plotting/utils/shared.py b/projects/container_bench/visualizations/benchmark/plotting/utils/shared.py index feb14482c1..70fd54b77a 100644 --- a/projects/container_bench/visualizations/benchmark/plotting/utils/shared.py +++ b/projects/container_bench/visualizations/benchmark/plotting/utils/shared.py @@ -52,22 +52,6 @@ 'word-break': 'break-all' } -TABLE_CONFIG = { - 'performance': { - 'headers': ["Configuration", "Execution Time", "Command", "Timestamp"], - 'title': "Performance Metrics" - }, - 'system_usage': { - 'title': "Average System Usage" - }, - 'system_differences': { - 'title': "Host System Differences" - }, - 'engine_differences': { - 'title': "Container Engine Differences" - } -} - def format_field_value(field_key, value): if field_key == "Host_memory" and value != "N/A": @@ -100,18 +84,18 @@ def create_na_cell(): return html.Td("N/A", style=css.STYLE_TABLE_CELL) -def create_execution_time_content(exec_time, runs, is_fastest=False, fastest_style=FASTEST_INDICATOR): - if not exec_time: +def create_execution_time_content(exec_time_95th_percentile, jitter, is_fastest=False, fastest_style=FASTEST_INDICATOR): + if not exec_time_95th_percentile: return "N/A" content = [ html.Span( - f"{units.format_duration(exec_time)}", + f"{units.format_duration(exec_time_95th_percentile)}", style=css.STYLE_INFO_VALUE_HIGHLIGHT ), html.Br(), html.Small( - f"(Average of {runs} runs)", + f"(Jitter +- {units.format_duration(jitter)})", style=css.STYLE_SMALL_TEXT ) ] @@ -125,31 +109,6 @@ def create_execution_time_content(exec_time, runs, is_fastest=False, fastest_sty return content -def create_metric_value_cell(value, unit, is_min=False, is_max=False, is_single=False): - if value is None: - return create_na_cell() - - formatted_value = f"{value:.2f} {unit}" - - if is_single or (not is_min and not is_max): - return create_standard_cell(formatted_value) - - if is_min: - content = [ - html.Span(formatted_value, style={'font-weight': 'bold'}), - html.Br(), - html.Small(LOWEST_INDICATOR, style=HIGHLIGHT_STYLE) - ] - return create_standard_cell(content, is_highlighted=True) - else: - content = [ - html.Span(formatted_value, style={'font-weight': 'bold'}), - html.Br(), - html.Small(HIGHEST_INDICATOR, style={'color': DANGER_COLOR, 'font-weight': 'bold'}) - ] - return create_standard_cell(content, is_highlighted=True) - - def create_delta_content(delta_value, delta_percentage, unit="", is_time=False): if is_time: delta_text = f"{DELTA_TIME_LABEL} {units.format_duration(delta_value)}" @@ -172,19 +131,19 @@ def create_summary_items(info, include_exec_time=True): if info.get('timestamp') and info['timestamp'] != 'N/A': summary_items.append(("Timestamp", info['timestamp'], False, False)) - if include_exec_time and info.get('exec_time'): + if include_exec_time and info.get('execution_time_95th_percentile'): exec_time_content = [ html.Span( - f"{units.format_duration(info['exec_time'])}", + f"{units.format_duration(info['execution_time_95th_percentile'])}", style=css.STYLE_INFO_VALUE_HIGHLIGHT ), html.Br(), html.Small( - f"(Average of {info.get('runs', 1)} runs)", + f"(Jitter +- {units.format_duration(info.get('jitter', 0))})", style=css.STYLE_SMALL_TEXT ) ] - summary_items.append(("Execution Time", exec_time_content, False, True)) + summary_items.append(("Execution Time (95th Percentile)", exec_time_content, False, True)) if summary_items: last_item = summary_items[-1] @@ -194,27 +153,32 @@ def create_summary_items(info, include_exec_time=True): def create_summary_info_card(info, title="Benchmark Summary", include_exec_time=True): + if not info: + return None + summary_items = create_summary_items(info, include_exec_time) if summary_items: return html_elements.info_card(title, summary_items) return None -def create_host_info_card(system_info, field_mappings, title="Host System Information"): +def create_host_info_card(system_info, title="Host System Information", field_mappings=None): if not system_info: return None - host_items = create_host_info_items(system_info, field_mappings) + mappings = field_mappings or SYSTEM_INFO_FIELD_MAPPINGS + host_items = create_host_info_items(system_info, mappings) if host_items: return html_elements.info_card(title, host_items) return None -def create_engine_info_card(engine_info, provider_info, field_mappings, title="Container Engine Information"): +def create_engine_info_card(engine_info, title="Container Engine Information", provider_info=None, field_mappings=None): if not engine_info: return None - engine_items = create_engine_info_items(engine_info, provider_info, field_mappings) + mappings = field_mappings or ENGINE_INFO_FIELD_MAPPINGS + engine_items = create_engine_info_items(engine_info, provider_info, mappings) if engine_items: return html_elements.info_card(title, engine_items) return None @@ -225,7 +189,7 @@ def format_benchmark_title(benchmark): def has_long_running_benchmarks(configurations): - return any(config.get('exec_time', 0) > MIN_PLOT_BENCHMARK_TIME for config in configurations) + return any(config.get('execution_time_95th_percentile', 0) > MIN_PLOT_BENCHMARK_TIME for config in configurations) def create_code_cell(command): @@ -265,13 +229,6 @@ def create_engine_info_items(engine_info, provider_info, field_mappings): return finalize_info_items(engine_items) -def create_usage_table_headers(metric_display_config): - headers = ["Configuration"] - for _, (label, unit) in metric_display_config.items(): - headers.append(f"{label} ({unit})") - return headers - - def detect_linux_system(system_info): return "linux" in system_info.get("OS_version", "").lower() diff --git a/projects/container_bench/visualizations/benchmark/store/parsers.py b/projects/container_bench/visualizations/benchmark/store/parsers.py index 819c581869..36c1711c0f 100644 --- a/projects/container_bench/visualizations/benchmark/store/parsers.py +++ b/projects/container_bench/visualizations/benchmark/store/parsers.py @@ -1,3 +1,4 @@ +import math import types import yaml import json @@ -86,17 +87,26 @@ def _parse_metrics(dirname): if not execution_times: return None - metric.cpu = [sum(cpu) / len(cpu) for cpu in zip(*cpu_usages)] - metric.execution_time = sum(execution_times) / len(execution_times) - metric.memory = [sum(memory) / len(memory) for memory in zip(*memory_usages)] - - network_send_avg = [sum(send) / len(send) for send in zip(*network_send_usages)] - network_recv_avg = [sum(recv) / len(recv) for recv in zip(*network_recv_usages)] - metric.network = dict(send=network_send_avg, recv=network_recv_avg) - - disk_read_avg = [sum(read) / len(read) for read in zip(*disk_read_usages)] - disk_write_avg = [sum(write) / len(write) for write in zip(*disk_write_usages)] - metric.disk = dict(read=disk_read_avg, write=disk_write_avg) + metric.cpu = [_calculate_usage_metric(cpu) for cpu in zip(*cpu_usages)] + metric.execution_time_95th_percentile = _calculate_percentile(execution_times, 95) + metric.execution_time_jitter = _calculate_jitter(execution_times) + metric.memory = [_calculate_usage_metric(memory) for memory in zip(*memory_usages)] + + network_send = [ + _calculate_usage_metric(send) for send in zip(*network_send_usages) + ] + network_recv = [ + _calculate_usage_metric(recv) for recv in zip(*network_recv_usages) + ] + metric.network = dict(send=network_send, recv=network_recv) + + disk_read = [ + _calculate_usage_metric(read) for read in zip(*disk_read_usages) + ] + disk_write = [ + _calculate_usage_metric(write) for write in zip(*disk_write_usages) + ] + metric.disk = dict(read=disk_read, write=disk_write) metric.interval = interval metric.command = command @@ -131,3 +141,41 @@ def _parse_container_engine_info(dirname): ) as f: container_engine_info = json.load(f) return container_engine_info + + +def _calculate_usage_metric(data): + if data is None or len(data) == 0: + return None + return types.SimpleNamespace( + percentile_75th=_calculate_percentile(data, 75), + percentile_50th=_calculate_percentile(data, 50), + percentile_25th=_calculate_percentile(data, 25), + ) + + +def _calculate_percentile(data, percentile=95): + if data is None or len(data) == 0: + return None + + sorted_values = sorted(data) + n = len(sorted_values) + + rank = (percentile / 100) * (n - 1) + + if rank.is_integer(): + return sorted_values[int(rank)] + lower_index = math.floor(rank) + upper_index = math.ceil(rank) + fraction = rank - lower_index + + lower_value = sorted_values[lower_index] + upper_value = sorted_values[upper_index] + + return lower_value + fraction * (upper_value - lower_value) + + +def _calculate_jitter(data): + if data is None or len(data) < 2: + return None + differences = [abs(data[i] - data[i-1]) for i in range(1, len(data))] + return sum(differences) / len(differences) diff --git a/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py b/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py index 50fcc43fb1..cd89d36bc4 100644 --- a/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py +++ b/projects/matrix_benchmarking/visualizations/helpers/plotting/units.py @@ -10,6 +10,8 @@ def human_readable_size(value): Returns: str: The human-readable string representation of the byte value. """ + if value is None: + return "N/A" suffix = [" kB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB", " RB", " QB"] base = 1000 @@ -32,7 +34,8 @@ def format_duration(seconds): Returns: str: The human-readable string representation of the duration. """ - + if seconds is None: + return "N/A" if seconds < 1e-6: return f"{seconds * 1e9:.2f} ns" if seconds < 1e-3: