Fix: Fixes devpi test flakiness with healthcheck and longer log check (#2200)

* Fix: Fixes devpi test flakiness with healthcheck and longer log check

* Increase sleep

* Adding debugging

* Add a wait that checks if the server is ready

Author: cjames23

tests/conftest.py

@@ -295,17 +295,33 @@ def devpi(tmp_path_factory, worker_id):
     with FileLock(lock_file):
         if not any(devpi_started_sessions.iterdir()):
             with EnvVars(env_vars):
-                subprocess.check_call(["docker", "compose", "-f", compose_file, "up", "--build", "-d"])
-
-            for _ in range(60):
+                result = subprocess.run(
+                    ["docker", "compose", "-f", compose_file, "up", "--build", "-d", "--wait"],
+                    check=False,
+                    capture_output=True,
+                )
+                if result.returncode != 0:
+                    # Debugging info for if devpi fails to start
+                    logs = subprocess.run(["docker", "logs", "hatch-devpi"], check=False, capture_output=True)
+                    pytest.fail(
+                        f"Failed to start devpi container, see logs:\n{logs.stdout.decode()}\n{logs.stderr.decode()}"
+                    )
+
+            for _ in range(120):
                 output = subprocess.check_output(["docker", "logs", "hatch-devpi"]).decode("utf-8")
                 if f"Serving index {dp.user}/{dp.index_name}" in output:
-                    time.sleep(5)
+                    time.sleep(15)
                     break
 
                 time.sleep(1)
             else:  # no cov
-                pass
+                # Add logging here too for timeout case
+                import warnings
+
+                logs = subprocess.run(["docker", "logs", "hatch-devpi"], check=False, capture_output=True)
+                warnings.warn(
+                    f"devpi container logs (timeout):\n{logs.stdout.decode()}\n{logs.stderr.decode()}", stacklevel=1
+                )
 
         (devpi_started_sessions / worker_id).touch()
 

tests/index/server/devpi/entrypoint.sh

@@ -10,7 +10,17 @@ echo "==:> Starting server"
 devpi-server --host 0.0.0.0 --port 3141 &
 
 echo "==:> Waiting on server"
-sleep 5
+for i in $(seq 1 30); do
+    if devpi use http://localhost:3141 2>/dev/null; then
+        break
+    fi
+    if [ "$i" -eq 30 ]; then
+        echo "Timed out waiting for devpi-server"
+        exit 1
+    fi
+    sleep 1
+done
+
 
 echo "==:> Setting up index"
 devpi use http://localhost:3141

tests/index/server/docker-compose.yaml

@@ -9,6 +9,12 @@ services:
     - DEVPI_INDEX_NAME
     - DEVPI_USERNAME
     - DEVPI_PASSWORD
+    healthcheck:
+      test: [ "CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:3141')" ]
+      interval: 2s
+      timeout: 5s
+      retries: 30
+      start_period: 10s
 
   nginx:
     container_name: hatch-nginx
@@ -19,4 +25,5 @@ services:
     volumes:
     - ./nginx:/etc/nginx
     depends_on:
-    - devpi
+      devpi:
+        condition: service_healthy