Skip to content

Fix and enhance benchmark test scripts. #676

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 19, 2025
Merged

Fix and enhance benchmark test scripts. #676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions docs/testing/automated_benchmark_test.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ If the two Servers, that host VMs in tests, run Gardenlinux, and they require ex
sudo nft add chain inet filter input '{ policy accept; }'
```

Additionally, if the used hypervisors are running Gardenlinux, it is needed to remount `/tmp` to allow execute binary files being uploaded to it, due to the strict security policy. Simply execute `sudo mount -o remount,exec /tmp`.


## Enable QEMU's default networking
To ssh into VMs, QEMU's default networking needs to be activated and configured to support IP address assignment via DHCP. Enter the libvirt's default network editing mode by running `sudo virsh net-edit default`, copy the configureation and restart libvirt service by running `sudo systemctl restart libvirtd`.
Expand Down Expand Up @@ -79,6 +77,7 @@ The most commonly used commands to run the provision script are as follows.

2. `./provision.py --clean-up`. It is expected that the provisioned VMs are destroyed and undefined.

It is possible to login into the provisioned VMs with the user name `root` and password `test123`, using the libvirt's console.

# Execution of test script
This test suite is invoked by executing the script `runtest.py` under the repository `/test/benchmark_test`. In oder to run dpservice either natively or via container, please make sure that a valid dp_service.conf file is created under `/tmp`.
Expand All @@ -91,7 +90,7 @@ The testing script assumes that dpservice-cli exists under '/tmp' on hypervisors
This script accepts several parameters, which are explained as follows.
1. `--mode`. This option specifies which operation mode of dpservice needs to be tested. Select from 'offload', 'non-offload' and 'both'. It must be specified.

2. `--stage`. This option specifies which testing stage needs to be used. Choose its value from 'dev' and 'cicd'. The stage of 'dev' is intended for carrying out tests during the development. If this option is set to 'dev', a docker image will be generated from the local repository of dpservice, and this image will be transferred to the hypervisors and executed. For example, a command like `./runtest.py --mode non-offloading --stage deploy -v` will achieve this purpose.
2. `--stage`. This option specifies which testing stage needs to be used. Choose its value from 'dev' and 'cicd'. The stage of 'dev' is intended for carrying out tests during the development. If this option is set to 'dev', a docker image will be generated from the local repository of dpservice, and this image will be transferred to the hypervisors and executed. For example, a command like `./runtest.py --mode non-offloading --stage dev -v` will achieve this purpose.
Alternatively, if this option is set as 'cicd', the above described docker image generating process will not happen. Instead, a docker image specified by the option "--docker-image" will be used on hypervisors. This is a required option.

3. `--docker-image`. This option specifies the container image to be deployed to hypervisors. It is optional but required for the 'cicd' stage.
Expand Down
17 changes: 0 additions & 17 deletions test/benchmark_test/benchmark_test_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ def init_dpservice(
):
try:
for hypervisor_info in env_config["hypervisors"]:
check_dpservice_cli(hypervisor_info["machine_name"], build_path)

if stage == "dev":
upload_and_import_docker_image(
hypervisor_info["machine_name"],
Expand All @@ -62,7 +60,6 @@ def init_dpservice(
remote_machine_op_dpservice_start(
hypervisor_info["machine_name"],
offload=is_offload,
is_docker=True,
docker_image_url=docker_image,
)
machine_name = hypervisor_info["machine_name"]
Expand Down Expand Up @@ -158,20 +155,6 @@ def tear_down_lb(lb_config):
remote_machine_op_dpservice_delete_lb(node, lb_config.get_id())


def check_dpservice_cli(machine_name, build_dir):
downloaded_cli_path = f"{build_dir}/cli/dpservice-cli/dpservice-cli"
target_cli_path = "/tmp/dpservice-cli"
try:
if not remote_machine_op_file_exists(machine_name, target_cli_path):
test_logger.info(f"Uploading dpservice-cli to {machine_name}")
remote_machine_op_upload(
machine_name, downloaded_cli_path, target_cli_path)
remote_machine_op_make_runnable(machine_name, target_cli_path)
test_logger.info(f"Uploaded dpservice-cli to {machine_name}")
except Exception as e:
test_logger.error(f"Failed to prepare dpservice cli on hypervisors due to {e}")


def prepare_test_environment(
is_offload, stage, docker_image_url, reboot_vm, config, build_path
):
Expand Down
2 changes: 1 addition & 1 deletion test/benchmark_test/config_templates/provision_tmpl.ign
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"ignition":{"version":"3.2.0"},"passwd":{"users":[{"name":"root","passwordHash":"PLEASE ASK MAINTAINER FOR A PROPER HASHEDPASS","sshAuthorizedKeys":["{{ pub_rsa_key }}"],"shell":"/bin/bash"}]},"systemd":{"units":[{"contents":"[Unit]\nDescription=Allow SSH Root Login\nAfter=network.target\n\n[Service]\nType=oneshot\nExecStart=/bin/bash -c \"sed -i 's/^#\\?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config\"\nExecStartPost=/bin/systemctl restart sshd\n\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"allowrootssh.service"},{"contents":"[Unit]\nDescription=Setup iperf3 on Debian\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/bin/bash -c 'echo \"deb http://deb.debian.org/debian/ bookworm main\" \u003e /etc/apt/sources.list.d/bookworm.list'\nExecStart=/usr/bin/apt-get update\nExecStart=/usr/bin/apt-get install -y iperf3 nftables\nExecStart=/bin/rm /etc/apt/sources.list.d/bookworm.list\nExecStart=/usr/bin/apt-get update\nRemainAfterExit=true\n\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"setup-iperf3.service"}]}}
{"ignition":{"version":"3.2.0"},"passwd":{"users":[{"name":"root","passwordHash":"$6$YsWW3XO9clG5EKHV$SWHTsFWkhblZVKD4hdj5eSi91mMEbXKwT.U69krkVLfukoO6rHog/0k0P8fjyNPbz3i8w21CtrOwvazQB1OD71","sshAuthorizedKeys":["{{ pub_rsa_key }}"],"shell":"/bin/bash"}]},"systemd":{"units":[{"contents":"[Unit]\nDescription=Allow SSH Root Login\nAfter=network.target\n\n[Service]\nType=oneshot\nExecStart=/bin/bash -c \"sed -i 's/^#\\?PermitRootLogin .*/PermitRootLogin yes/' /etc/ssh/sshd_config\"\nExecStartPost=/bin/systemctl restart sshd\n\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"allowrootssh.service"},{"contents":"[Unit]\nDescription=Use google DNS\nAfter=network.target\n\n[Service]\nType=oneshot\nExecStart=/bin/bash -c \"sed -i 's/^#DNS=.*/DNS=8.8.8.8/' /etc/systemd/resolved.conf\"\nExecStartPost=/bin/systemctl restart systemd-resolved\n\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"usenormaldns.service"},{"contents":"[Unit]\nDescription=Setup iperf3 on Debian\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/bin/bash -c 'echo \"deb http://deb.debian.org/debian/ bookworm main\" \u003e /etc/apt/sources.list.d/bookworm.list'\nExecStart=/usr/bin/apt-get update\nExecStart=/usr/bin/apt-get install -y iperf3\nExecStart=/bin/rm /etc/apt/sources.list.d/bookworm.list\nExecStart=/usr/bin/apt-get update\nRemainAfterExit=true\n\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"setup-iperf3.service"}]}}
68 changes: 27 additions & 41 deletions test/benchmark_test/remote_machine_operations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

from remote_machine_management import get_remote_machine, add_vm_config_info, add_vm_nat_config

def generate_dpservice_cli_base_cmd(machine_name):
dpservice_container_name = f"dpservice_{machine_name}"
return f"docker exec {dpservice_container_name} dpservice-cli "

def remote_machine_op_upload(machine_name, local_path, remote_path):
machine = None
Expand Down Expand Up @@ -75,9 +78,9 @@ def remote_machine_op_docker_load_image(machine_name, image_file):
machine.logger.error(f"Failed to load docker image {image_file}: {e}")


def remote_machine_op_dpservice_start(machine_name, offload, is_docker, docker_image_url='', path_to_bin="/tmp/dpservice-bin"):
def remote_machine_op_dpservice_start(machine_name, offload, docker_image_url='', path_to_bin="/tmp/dpservice-bin"):

if is_docker and docker_image_url == '':
if docker_image_url == '':
raise ValueError(
f"docker image url is required if dpservice is running in docker env")
machine = None
Expand All @@ -92,33 +95,16 @@ def remote_machine_op_dpservice_start(machine_name, offload, is_docker, docker_i
flag += '--enable-ipv6-overlay '
parameters = f"-l 0,1 {flag}"

if is_docker:
docker_run_command = f"docker run"
docker_run_parameters = f"-d --privileged --network host --name {docker_container_name} " \
f"--mount type=bind,source=/dev/hugepages,target=/dev/hugepages " \
f"--mount type=bind,source=/tmp,target=/tmp {docker_image_url} {parameters} "
docker_run_task = [
{"command": docker_run_command, "parameters": docker_run_parameters,
"background": False, "sudo": True}
]
machine.exec_task(docker_run_task)
else:
# Check if dpservice is already running
check_command = f"pgrep -f {path_to_bin}"
running_pid = machine.ssh_manager.run_command(check_command)
if running_pid:
# If running, kill the existing process
kill_command = f"kill {running_pid.strip()}"
machine.ssh_manager.run_command(kill_command, sudo=True)
time.sleep(1)
machine.logger.info(
f"Existing dpservice process killed: PID {running_pid.strip()}")

dpservice_task = [
{"command": path_to_bin, "parameters": parameters,
"background": True, "sudo": True, "cmd_output_name": "dpservice"},
]
machine.exec_task(dpservice_task)
docker_run_command = f"docker run"
docker_run_parameters = f"-d --privileged --network host --name {docker_container_name} " \
f"--mount type=bind,source=/dev/hugepages,target=/dev/hugepages " \
f"--mount type=bind,source=/tmp,target=/tmp {docker_image_url} {parameters} "
docker_run_task = [
{"command": docker_run_command, "parameters": docker_run_parameters,
"background": False, "sudo": True}
]
machine.exec_task(docker_run_task)

except Exception as e:
if machine:
machine.logger.error(f"Failed to start dpservice on hypervisor: {e}")
Expand Down Expand Up @@ -152,7 +138,7 @@ def remote_machine_op_dpservice_init(machine_name):
raise NotImplementedError(
f"Cannot configure dpservice on a vm machine {machine_name}")
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": "init", "delay": 5},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": "init", "delay": 5},
]
machine.exec_task(cli_task)
time.sleep(1)
Expand All @@ -170,7 +156,7 @@ def remote_machine_op_dpservice_create_interface(machine_name, if_id, vni, ipv4,
f"Cannot configure dpservice on a vm machine {machine_name}")
parameters = f"create interface --id={if_id} --vni={vni} --ipv4={ipv4} --ipv6={ipv6} --device={pci_dev}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -187,7 +173,7 @@ def remote_machine_op_dpservice_create_nat(machine_name, if_id, ip, ports):
f"Cannot configure dpservice and create nat on a vm machine {machine_name}")
parameters = f"create nat --interface-id={if_id} --nat-ip={ip} --minport={ports[0]} --maxport={ports[1]}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -204,7 +190,7 @@ def remote_machine_op_dpservice_delete_nat(machine_name, if_id):
f"Cannot configure dpservice and delete nat on a vm machine {machine_name}")
parameters = f"delete nat --interface-id={if_id} "
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -221,7 +207,7 @@ def remote_machine_op_dpservice_create_lb(machine_name, lb_name, lb_vni, lb_ip,
f"Cannot configure dpservice and create lb on a vm machine {machine_name}")
parameters = f"create loadbalancer --id={lb_name} --vni={lb_vni} --vip={lb_ip} --lbports={lb_ports}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -238,7 +224,7 @@ def remote_machine_op_dpservice_delete_lb(machine_name, lb_name):
f"Cannot configure dpservice and delete lb on a vm machine {machine_name}")
parameters = f"delete loadbalancer --id={lb_name}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -255,7 +241,7 @@ def remote_machine_op_dpservice_create_lbpfx(machine_name, prefix, if_id):
f"Cannot configure dpservice and create lbpfx on a vm machine {machine_name}")
parameters = f"create lbprefix --interface-id={if_id} --prefix={prefix}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -272,7 +258,7 @@ def remote_machine_op_dpservice_delete_lbpfx(machine_name, prefix, if_id):
f"Cannot configure dpservice and delete lbpfx on a vm machine {machine_name}")
parameters = f"delete lbprefix --interface-id={if_id} --prefix={prefix}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -289,7 +275,7 @@ def remote_machine_op_dpservice_create_lbtarget(machine_name, target_ip, lb_name
f"Cannot configure dpservice and create lbtarget on a vm machine {machine_name}")
parameters = f"create lbtarget --target-ip={target_ip} --lb-id={lb_name}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -306,7 +292,7 @@ def remote_machine_op_dpservice_delete_lbtarget(machine_name, target_ip, lb_name
f"Cannot configure dpservice and delete lbtarget on a vm machine {machine_name}")
parameters = f"delete lbtarget --target-ip={target_ip} --lb-id={lb_name}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
return machine.exec_task(cli_task)
except Exception as e:
Expand Down Expand Up @@ -342,7 +328,7 @@ def remote_machine_op_dpservice_create_route(machine_name, prefix, nxt_hop_vni,
f"Cannot configure dpservice on a vm machine {machine_name}")
parameters = f"create route --prefix={prefix} --next-hop-vni={nxt_hop_vni} --next-hop-ip={nxt_hop_underly_ip} --vni={vni}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
machine.exec_task(cli_task)
except Exception as e:
Expand All @@ -359,7 +345,7 @@ def remote_machine_op_dpservice_delete_route(machine_name, prefix, vni):
f"Cannot configure dpservice on a vm machine {machine_name}")
parameters = f"delete route --prefix={prefix} --vni={vni}"
cli_task = [
{"command": "/tmp/dpservice-cli", "parameters": parameters},
{"command": generate_dpservice_cli_base_cmd(machine_name), "sudo": True, "parameters": parameters},
]
machine.exec_task(cli_task)
except Exception as e:
Expand Down
Loading