diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 20bfec1..b2c7f5e 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -131,8 +131,8 @@ jobs: --commits "${{ github.event.inputs.commit_range || 'HEAD^!' }}" \ --push - - name: Cleanup on failure - if: failure() + - name: Cleanup servers + if: always() env: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} run: | diff --git a/csp_benchmarks/hetzner/cli.py b/csp_benchmarks/hetzner/cli.py index 9915aa9..c639515 100644 --- a/csp_benchmarks/hetzner/cli.py +++ b/csp_benchmarks/hetzner/cli.py @@ -45,6 +45,7 @@ def run_benchmarks(args: argparse.Namespace) -> int: ) manager = HetznerServerManager(token=token, config=server_config) + server = None try: # Check if server already exists @@ -77,9 +78,12 @@ def run_benchmarks(args: argparse.Namespace) -> int: return 1 finally: - if not args.keep_server and server: + if not args.keep_server and server is not None: logger.info("Cleaning up server...") - manager.delete_server(server) + try: + manager.delete_server(server) + except Exception as cleanup_error: + logger.error(f"Failed to cleanup server: {cleanup_error}") def cleanup_servers(args: argparse.Namespace) -> int: diff --git a/csp_benchmarks/hetzner/runner.py b/csp_benchmarks/hetzner/runner.py index 1007522..289337c 100644 --- a/csp_benchmarks/hetzner/runner.py +++ b/csp_benchmarks/hetzner/runner.py @@ -136,10 +136,34 @@ def _scp_from_server(self, remote_path: str, local_path: str) -> None: subprocess.run(scp_args, check=True) + def _wait_for_ssh(self, timeout: int = 120, interval: int = 10) -> None: + """Wait for SSH to become available on the server.""" + import time + + logger.info("Waiting for SSH to become available...") + start_time = time.time() + + while time.time() - start_time < timeout: + try: + result = self._run_ssh_command("echo 'SSH ready'", check=False) + if result.returncode == 0: + logger.info("SSH connection established") + return + except Exception: + pass + + logger.debug(f"SSH not ready, retrying in {interval}s...") + time.sleep(interval) + + raise TimeoutError(f"SSH not available after {timeout} seconds") + def _setup_environment(self) -> None: """Set up the benchmark environment on the remote server.""" logger.info("Setting up benchmark environment...") + # Wait for SSH to become available + self._wait_for_ssh() + # Determine machine name based on server type server_type = self.server.server_type.name machine_name = f"hetzner-{server_type}" diff --git a/csp_benchmarks/hetzner/server.py b/csp_benchmarks/hetzner/server.py index 736cf2e..ae9ed8c 100644 --- a/csp_benchmarks/hetzner/server.py +++ b/csp_benchmarks/hetzner/server.py @@ -88,11 +88,8 @@ def create_server(self, wait_for_ready: bool = True) -> BoundServer: ) server = response.server - root_password = response.root_password logger.info(f"Server created: {server.name} (ID: {server.id})") - if root_password: - logger.info(f"Root password: {root_password}") if wait_for_ready: self._wait_for_server_ready(server)