Merge pull request #8 from Point72/tkp/initbm

Miscellaneous fixes and cleanup for benchmark servers

Author: timkpaine

.github/workflows/benchmarks.yaml

@@ -131,8 +131,8 @@ jobs:
             --commits "${{ github.event.inputs.commit_range || 'HEAD^!' }}" \
             --push
 
-      - name: Cleanup on failure
-        if: failure()
+      - name: Cleanup servers
+        if: always()
         env:
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
         run: |

csp_benchmarks/hetzner/cli.py

@@ -45,6 +45,7 @@ def run_benchmarks(args: argparse.Namespace) -> int:
     )
 
     manager = HetznerServerManager(token=token, config=server_config)
+    server = None
 
     try:
         # Check if server already exists
@@ -77,9 +78,12 @@ def run_benchmarks(args: argparse.Namespace) -> int:
         return 1
 
     finally:
-        if not args.keep_server and server:
+        if not args.keep_server and server is not None:
             logger.info("Cleaning up server...")
-            manager.delete_server(server)
+            try:
+                manager.delete_server(server)
+            except Exception as cleanup_error:
+                logger.error(f"Failed to cleanup server: {cleanup_error}")
 
 
 def cleanup_servers(args: argparse.Namespace) -> int:

csp_benchmarks/hetzner/runner.py

@@ -136,10 +136,34 @@ def _scp_from_server(self, remote_path: str, local_path: str) -> None:
 
         subprocess.run(scp_args, check=True)
 
+    def _wait_for_ssh(self, timeout: int = 120, interval: int = 10) -> None:
+        """Wait for SSH to become available on the server."""
+        import time
+
+        logger.info("Waiting for SSH to become available...")
+        start_time = time.time()
+
+        while time.time() - start_time < timeout:
+            try:
+                result = self._run_ssh_command("echo 'SSH ready'", check=False)
+                if result.returncode == 0:
+                    logger.info("SSH connection established")
+                    return
+            except Exception:
+                pass
+
+            logger.debug(f"SSH not ready, retrying in {interval}s...")
+            time.sleep(interval)
+
+        raise TimeoutError(f"SSH not available after {timeout} seconds")
+
     def _setup_environment(self) -> None:
         """Set up the benchmark environment on the remote server."""
         logger.info("Setting up benchmark environment...")
 
+        # Wait for SSH to become available
+        self._wait_for_ssh()
+
         # Determine machine name based on server type
         server_type = self.server.server_type.name
         machine_name = f"hetzner-{server_type}"

csp_benchmarks/hetzner/server.py

@@ -88,11 +88,8 @@ def create_server(self, wait_for_ready: bool = True) -> BoundServer:
         )
 
         server = response.server
-        root_password = response.root_password
 
         logger.info(f"Server created: {server.name} (ID: {server.id})")
-        if root_password:
-            logger.info(f"Root password: {root_password}")
 
         if wait_for_ready:
             self._wait_for_server_ready(server)