fix: refuse to start a second lemond when the port is already in use (#2255)

.github/actions/install-lemonade-deb/action.yml

@@ -16,6 +16,10 @@ inputs:
     description: 'Whether to download the artifact (set to false if already downloaded)'
     required: false
     default: 'true'
+  port:
+    description: "Port to start lemond on. Use 'auto' to pick a free port (needed on shared self-hosted runners where the default port may be taken)."
+    required: false
+    default: '13305'
 
 outputs:
   bin-path:
@@ -125,26 +129,73 @@ runs:
         chmod 700 "$RUNNER_TEMP/xdg-runtime"
         echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg-runtime" >> "$GITHUB_ENV"
         export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"
-        "$SERVER_BIN" "$CACHE_DIR" > "$LOG_FILE" 2>&1 &
-        SERVER_PID=$!
-
-        echo "Waiting for server to be ready (PID $SERVER_PID)..."
-        for i in $(seq 1 30); do
-          if curl -sf http://127.0.0.1:13305/live > /dev/null 2>&1; then
-            echo "Server is running and healthy"
-            exit 0
+
+        # With port=auto, start on a job-unique free port instead of the default
+        # 13305. Non-containerized self-hosted jobs share the host network, so a
+        # concurrent job may already hold 13305 — and lemond now refuses to bind
+        # an in-use port. The resolved port is exported so the test harness
+        # (LEMONADE_TEST_PORT) and the lemonade CLI (LEMONADE_PORT) target it.
+        REQUESTED_PORT="${{ inputs.port }}"
+
+        # Print a free TCP port (loopback). Prefer an OS-assigned ephemeral port
+        # via python3; fall back to a random high port checked with ss.
+        pick_free_port() {
+          if command -v python3 >/dev/null 2>&1; then
+            python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' && return 0
           fi
-          # Check if process is still alive
-          if ! kill -0 $SERVER_PID 2>/dev/null; then
-            echo "ERROR: Server process (PID $SERVER_PID) exited prematurely"
-            echo "=== Server log ==="
-            cat "$LOG_FILE" 2>/dev/null || echo "(no log file)"
-            exit 1
+          for _ in $(seq 1 50); do
+            local p=$(( (RANDOM % 20000) + 20000 ))
+            if ! ss -ltn 2>/dev/null | grep -q ":$p "; then
+              echo "$p"; return 0
+            fi
+          done
+          return 1
+        }
+
+        # Start lemond; with auto, retry on a fresh port if the chosen one got
+        # taken in the race between picking and binding.
+        started=0
+        for start_attempt in $(seq 1 5); do
+          if [ "$REQUESTED_PORT" = "auto" ]; then
+            PORT=$(pick_free_port) || { echo "ERROR: could not find a free port"; exit 1; }
+          else
+            PORT="$REQUESTED_PORT"
           fi
-          echo "Waiting for server... ($i/30)"
-          sleep 2
+          echo "Attempt $start_attempt: starting lemond on 127.0.0.1:$PORT..."
+          "$SERVER_BIN" "$CACHE_DIR" --port "$PORT" > "$LOG_FILE" 2>&1 &
+          SERVER_PID=$!
+
+          for i in $(seq 1 30); do
+            if curl -sf "http://127.0.0.1:$PORT/live" > /dev/null 2>&1; then
+              echo "Server is running and healthy on port $PORT (PID $SERVER_PID)"
+              echo "LEMONADE_TEST_PORT=$PORT" >> "$GITHUB_ENV"
+              echo "LEMONADE_PORT=$PORT" >> "$GITHUB_ENV"
+              started=1
+              break
+            fi
+            if ! kill -0 $SERVER_PID 2>/dev/null; then
+              echo "Server (PID $SERVER_PID) exited prematurely on port $PORT"
+              echo "=== Server log ==="
+              cat "$LOG_FILE" 2>/dev/null || echo "(no log file)"
+              break
+            fi
+            echo "Waiting for server... ($i/30)"
+            sleep 2
+          done
+
+          [ "$started" = "1" ] && break
+          # Only an auto port is worth retrying (a fixed port won't change).
+          if [ "$REQUESTED_PORT" != "auto" ]; then
+            break
+          fi
+          echo "Start attempt $start_attempt failed; retrying on a new port..."
+          kill -9 $SERVER_PID 2>/dev/null || true
+          sleep 1
         done
-        echo "ERROR: Server did not start within 60 seconds"
-        echo "=== Server log (last 50 lines) ==="
-        tail -50 "$LOG_FILE" 2>/dev/null || echo "(no log file)"
-        exit 1
+
+        if [ "$started" != "1" ]; then
+          echo "ERROR: server failed to start after multiple attempts"
+          echo "=== Server log (last 50 lines) ==="
+          tail -50 "$LOG_FILE" 2>/dev/null || echo "(no log file)"
+          exit 1
+        fi

.github/workflows/cpp_server_build_test_release.yml

@@ -1409,6 +1409,10 @@ jobs:
         uses: ./.github/actions/install-lemonade-deb
         with:
           version: ${{ env.LEMONADE_VERSION }}
+          # These jobs run directly on shared self-hosted runners (no container
+          # network isolation), so the default port can be held by a concurrent
+          # job. Use a job-unique free port to avoid the collision.
+          port: auto
 
       - name: Setup Python and virtual environment
         uses: ./.github/actions/setup-venv

src/cpp/include/lemon/server.h

@@ -51,6 +51,10 @@ class Server {
     // Get server status
     bool is_running() const;
 
+    // True if run() aborted startup (e.g. the port was already in use), so
+    // main() can report failure and exit non-zero.
+    bool startup_failed() const;
+
 private:
     std::string resolve_host_to_ip(int ai_family, const std::string& host);
     void setup_routes(httplib::Server &web_server);
@@ -264,6 +268,7 @@ class Server {
     std::map<std::string, std::shared_ptr<DownloadJob>> download_jobs_;
 
     bool running_;
+    bool startup_failed_ = false;
     std::atomic<bool> shutdown_requested_{false};
     std::atomic<bool> rebind_requested_{false};
     std::atomic<bool> metrics_access_logged_{false};

src/cpp/server/main.cpp

@@ -142,6 +142,14 @@ int main(int argc, char** argv) {
         server.run();
         g_server_instance = nullptr;
 
+        // Startup aborted (e.g. port already in use): exit non-zero now and skip
+        // destructors, whose teardown logging would bury the error message.
+        if (server.startup_failed()) {
+            std::cout.flush();
+            std::cerr.flush();
+            std::_Exit(1);
+        }
+
         return 0;
 
     } catch (const std::exception& e) {

src/cpp/server/server.cpp

@@ -18,6 +18,7 @@
 #include "lemon/system_info.h"
 #include "lemon/version.h"
 #include <cctype>
+#include <cstdint>
 #include <cstdlib>
 #include <iostream>
 #include <iomanip>
@@ -41,6 +42,8 @@
 #else
     #include <sys/types.h>
     #include <sys/socket.h>
+    #include <netinet/in.h>  // sockaddr_in / sockaddr_in6
+    #include <arpa/inet.h>   // inet_pton, htons
     #include <netdb.h>  // Crucial for getaddrinfo and addrinfo struct
     #include <unistd.h>
 #endif
@@ -305,6 +308,11 @@ void Server::setup_http_servers() {
     http_front_ = std::make_unique<UpgradableFrontServer>(http_server_.get(), upgrade_handler);
     http_front_v6_ = std::make_unique<UpgradableFrontServer>(http_server_v6_.get(), upgrade_handler);
 
+    // Keep cpp-httplib's default socket options here. httplib binds IPv6 with
+    // IPV6_V6ONLY=0, so "::" overlaps the IPv4 wildcard "0.0.0.0" and only the
+    // default SO_REUSEPORT lets the two coexist. Duplicate detection is done by
+    // port_is_available() in run(), not by making these listeners exclusive.
+
     // CRITICAL: Enable multi-threading so the server can handle concurrent requests
     // Without this, the server is single-threaded and blocks on long operations
 
@@ -1173,6 +1181,52 @@ void Server::setup_http_logger(httplib::Server &web_server) {
     });
 }
 
+// Probe whether host_ip:port can be bound (i.e. the port is free). Uses an
+// exclusive socket (SO_EXCLUSIVEADDRUSE on Windows; SO_REUSEADDR but NOT
+// SO_REUSEPORT on POSIX) so an actively-listening duplicate is detected, while a
+// socket left in TIME_WAIT by a just-exited server is still bindable.
+static bool port_is_available(int family, const std::string& host_ip, int port) {
+    socket_t sock = socket(family, SOCK_STREAM, IPPROTO_TCP);
+    if (sock == INVALID_SOCKET) {
+        return true;  // Can't probe; let the real bind path report any error.
+    }
+    auto close_sock = [&]() {
+#ifdef _WIN32
+        closesocket(sock);
+#else
+        close(sock);
+#endif
+    };
+
+    int opt = 1;
+#ifdef _WIN32
+    setsockopt(sock, SOL_SOCKET, SO_EXCLUSIVEADDRUSE,
+               reinterpret_cast<const char*>(&opt), sizeof(opt));
+#else
+    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt));
+#endif
+
+    sockaddr_storage ss{};
+    socklen_t len;
+    if (family == AF_INET) {
+        auto* a = reinterpret_cast<sockaddr_in*>(&ss);
+        a->sin_family = AF_INET;
+        a->sin_port = htons(static_cast<uint16_t>(port));
+        if (inet_pton(AF_INET, host_ip.c_str(), &a->sin_addr) != 1) { close_sock(); return true; }
+        len = sizeof(sockaddr_in);
+    } else {
+        auto* a = reinterpret_cast<sockaddr_in6*>(&ss);
+        a->sin6_family = AF_INET6;
+        a->sin6_port = htons(static_cast<uint16_t>(port));
+        if (inet_pton(AF_INET6, host_ip.c_str(), &a->sin6_addr) != 1) { close_sock(); return true; }
+        len = sizeof(sockaddr_in6);
+    }
+
+    bool available = bind(sock, reinterpret_cast<sockaddr*>(&ss), len) == 0;
+    close_sock();
+    return available;
+}
+
 void Server::run() {
     std::string host = config_->host();
     LOG(INFO, "Server") << "Starting HTTP server on " << host << ":" << port_ << std::endl;
@@ -1188,6 +1242,26 @@ void Server::run() {
                                  "Cannot start server.");
     }
 
+    // Fail fast if the port is already taken (usually another lemond). Detecting
+    // it here keeps the error from being buried under later startup logs.
+    {
+        std::string in_use_ip;
+        if (!ipv4.empty() && !port_is_available(AF_INET, ipv4, port_)) {
+            in_use_ip = ipv4;
+        } else if (!ipv6.empty() && !port_is_available(AF_INET6, ipv6, port_)) {
+            in_use_ip = ipv6;
+        }
+        if (!in_use_ip.empty()) {
+            std::string msg = "Port " + std::to_string(port_) + " on " + in_use_ip +
+                " is already in use. Another Lemonade server (lemond) is likely "
+                "already running on this port. This instance will now exit.";
+            std::cerr << "[Server] ERROR: " << msg << std::endl;  // terminal visibility
+            LOG(ERROR, "Server") << msg << std::endl;
+            startup_failed_ = true;
+            return;
+        }
+    }
+
     // Operators binding beyond loopback should secure the server with an API
     // key, since every endpoint is reachable from other machines once the host
     // is non-loopback. The regular API routes (/api, /v0, /v1) are gated by
@@ -1393,6 +1467,10 @@ bool Server::is_running() const {
     return running_;
 }
 
+bool Server::startup_failed() const {
+    return startup_failed_;
+}
+
 void Server::stop() {
     if (running_) {
         LOG(INFO, "Server") << "Stopping HTTP server..." << std::endl;