diff --git a/README.md b/README.md index 21c1423..018a36f 100644 --- a/README.md +++ b/README.md @@ -194,11 +194,13 @@ These are the labels scanned: - `mc-router.port`: This value must be set to the port the Minecraft server is listening on. The default value is 25565. - `mc-router.default`: Set this to a truthy value to make this server the default backend. Please note that `mc-router.host` is still required to be set. - `mc-router.network`: Specify the network you are using for the router if multiple are present in the container/service. You can either use the network ID, it's full name or an alias. -- `mc-router.auto-scale-up`: Per-container override to enable/disable auto scale up for Docker. When true (or left unspecified and the global `-auto-scale-up` flag is enabled), mc-router will start or unpause this container when a client connects to the declared hostname(s). -- `mc-router.auto-scale-down`: Per-container override to enable/disable auto scale down for Docker. When true (or left unspecified and the global `-auto-scale-down` flag is enabled), mc-router will stop this container after it has been idle for the configured `-auto-scale-down-after` duration. -- `mc-router.auto-scale-asleep-motd`: Per-container override for MOTD to show when container is scaled to zero. If empty or not set the host will -appear unresponsive. -- `mc-router.auto-scale-loading-motd`: Per-container override for MOTD to show while the container is waking and not yet reachable. If empty or not set, the global `-auto-scale-loading-motd` value is used. +- `mc-router.auto-scale-up`: Per-container/service override to enable/disable auto scale up for Docker/Swarm. When true (or left unspecified and the global `-auto-scale-up` flag is enabled), mc-router will start the container or scale up the Swarm service when a client connects. +- `mc-router.auto-scale-down`: Per-container/service override to enable/disable auto scale down for Docker/Swarm. When true (or left unspecified and the global `-auto-scale-down` flag is enabled), mc-router will stop the container or scale down the Swarm service to 0 after it has been idle. +- `mc-router.auto-scale-asleep-motd`: Per-container/service override for MOTD to show when scaled to zero. If empty or not set the host will appear unresponsive. +- `mc-router.auto-scale-loading-motd`: Per-container/service override for MOTD to show while waking up. Supports replacing the `{duration}` token with the remaining Swarm restart delay if the task is waiting to retry. If empty or not set, the global `-auto-scale-loading-motd` value is used. +- `mc-router.auto-scale-wait-timeout`: Configure the maximum duration the router waits for the container or Swarm task to become reachable after scaling up (e.g. `"5m"` or `"300s"`). Defaults to 60s. Note: Since the Minecraft Java client has a strict connection timeout of 30 seconds, configuring this value above 30s is not recommended for player join connections. +- `mc-router.auto-scale-restart-delay-motd`: MOTD to show while the service is in a temporary restart delay (e.g. `"Server failed to start. Retrying in {duration}."`). Supports the `{duration}` countdown token, which dynamically updates. +- `mc-router.auto-scale-failed-motd`: MOTD to show if the container/service fails to start permanently or Swarm exhausts its restart policy (e.g. `"Server crashed and stopped retrying."`). Does not support the countdown token. #### Docker Auto Scale Up/Down @@ -224,7 +226,8 @@ Behavior: - While that wake-up is in progress and status pings are received, mc-router can return a loading MOTD (per-container override or `-auto-scale-loading-motd`). - When no clients remain connected and the idle timer elapses (`-auto-scale-down-after`), mc-router gracefully stops the container. -Note: Docker Swarm deployments can use auto scaling via the [Webhook Auto Scale](#webhook-auto-scale) integration. Native Swarm service scaling via `-auto-scale-up`/`-auto-scale-down` is not supported. +> [!NOTE] +> Native Swarm service scaling via `-auto-scale-up`/`-auto-scale-down` is supported in both VIP (Virtual IP) and DNSRR (DNS Round-Robin) modes. Bypassing the VIP (by routing directly to the task container IP) is automatically enabled when `replicas == 1` to prevent VIP routing delay. Note that DNSRR mode has not been actively tested. #### Example Docker deployment @@ -539,7 +542,7 @@ To override the MOTD shown when the server is scaled down or scaling up, you can - `mc-router.itzg.me/autoScaleLoadingMOTD` You can also customize how long the router will wait for a scaling backend to become reachable (default: 60s): -- `mc-router.itzg.me/autoScaleWaitTimeout` (e.g. `2m`, `30s`) +- `mc-router.itzg.me/autoScaleWaitTimeout` (e.g. `2m`, `30s`). Note: Since the Minecraft Java client has a strict connection timeout of 30 seconds, configuring this value above 30s is not recommended for player join connections. Example server with custom MOTD and timeout: ```yaml diff --git a/docs/docker-swarm-states.md b/docs/docker-swarm-states.md new file mode 100644 index 0000000..8a73aec --- /dev/null +++ b/docs/docker-swarm-states.md @@ -0,0 +1,55 @@ +# Docker Swarm Task States for Autoscaling + +When building plugins or routers (like `mc-router`) that autoscale and monitor Docker Swarm services, interpreting task scheduling states is historically challenging due to sparse Docker documentation. + +This document defines how to map Swarm's **Actual Task Status State** (`task.Status.State`) and **Desired Task State** (`task.DesiredState`) to logical route status classifications. + +--- + +## Task State Matrix + +| State | Status State (`task.Status.State`) | Desired State (`task.DesiredState`) | Description | +| :--- | :--- | :--- | :--- | +| **Running** | `running` | `running` | The container is alive, healthy, and fully running. Connections can be routed directly. | +| **Restart Delay** | `ready` | `ready` | The service task crashed and Swarm is holding the next task retry until the restart delay timer expires. | +| **Starting / Waking** | `new`, `pending`, `assigned`, `preparing`, `ready`, `starting` | `running` | Swarm has scaled the replicas to `> 0` and is actively preparing or starting the container. | +| **Stopping** | `running` | `shutdown` or `remove` | Swarm has commanded the task to shut down or be removed, but the container is still cleaning up. | +| **Permanently Failed** | `failed`, `shutdown`, `rejected` | `shutdown` | Swarm has commanded a shutdown on the last failed container and is no longer attempting to schedule a new task. | + +--- + +## State Classifications & Logic + +### 1. Sleeping State +* **Condition**: `replicas == 0` +* **Interpretation**: The service is explicitly scaled down to zero replicas. +* **Router Action**: Keep the backend route empty (`""`) and display the configured asleep MOTD to prompt a wake-up on join. + +### 2. Running (Healthy) State +* **Condition**: At least one task has `Status.State == running` and `DesiredState == running`. +* **Interpretation**: The server is fully online and ready for traffic. +* **Router Action**: Resolve the route to the service's Virtual IP (VIP) or DNS Round Robin (DNSRR) name. + +### 3. Restart Delay State +* **Condition**: `replicas > 0` AND at least one task has `Status.State == ready` and `DesiredState == ready`. +* **Interpretation**: Swarm has scheduled a retry but is waiting for the configured restart policy delay to expire before spawning the container. +* **Router Action**: Keep the backend route empty (`""`) to route connection attempts to the waker, and display the restart delay countdown MOTD. + +### 4. Permanently Failed State +* **Condition**: `replicas > 0` AND the most recently updated task in history has `DesiredState == shutdown` AND no active tasks exist. +* **Interpretation**: Swarm has exhausted its restart attempts and stopped trying. +* **Router Action**: Keep the backend route empty (`""`) and display the custom permanently failed MOTD. + +### 5. Starting / Waking State +* **Condition**: `replicas > 0` AND the desired state of the task is `running` but the actual status is not yet `running` (and not delayed or failed). +* **Interpretation**: The container is actively being assigned, prepared, or started by Swarm. +* **Router Action**: Keep the backend route empty (`""`) to preserve the waker, and display the loading/waking MOTD. + +--- + +## Polling Implementation Details + +> [!NOTE] +> **Why Polling is Used Instead of API Events:** +> Although `mc-router` subscribes to the Docker Event Stream, the Docker Engine API does **not** broadcast task-level scheduling transitions (such as `task.Status.State` moving from `preparing` to `ready` or `running`) over the event system. +> To reliably capture these low-level task state changes and resolve the direct task IP without transient routing failures, `mc-router` utilizes a background polling ticker that queries the Swarm Service and Task API every 5 seconds. diff --git a/server/docker.go b/server/docker.go index e70197a..e552344 100644 --- a/server/docker.go +++ b/server/docker.go @@ -30,6 +30,10 @@ const ( DockerRouterLabelAutoScaleDown = "mc-router.auto-scale-down" DockerRouterLabelAutoScaleAsleepMOTD = "mc-router.auto-scale-asleep-motd" DockerRouterLabelAutoScaleLoadingMOTD = "mc-router.auto-scale-loading-motd" + DockerRouterLabelAutoScaleWaitTimeout = "mc-router.auto-scale-wait-timeout" + DockerRouterLabelAutoScaleFailedMOTD = "mc-router.auto-scale-failed-motd" + DockerRouterLabelAutoScaleRestartDelayMOTD = "mc-router.auto-scale-restart-delay-motd" + DockerRouterEventTypeTask = "task" ) type dockerWatcherConfig struct { diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 124b4a3..e5aface 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -33,35 +33,265 @@ func NewDockerSwarmWatcher(socket string, timeout time.Duration, autoScaleUp boo } } +type routableSwarmService struct { + externalServiceName string + containerEndpoint string + serviceID string + serviceName string + networkID string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time + statusState string +} + type dockerSwarmWatcherImpl struct { sync.RWMutex config dockerWatcherConfig client *client.Client - serviceMap map[string]*routableService + serviceMap map[string]*routableSwarmService monitorLock sync.Mutex routes IRoutes } -func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableService) WakerFunc { - if !w.config.autoScaleUp { +func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFunc { + if rs == nil || !rs.autoScaleUp { return nil } return func(ctx context.Context) (string, error) { - logrus.Fatal("Auto scale up is not yet supported for docker swarm") - return "", nil + serviceID := rs.serviceID + if serviceID == "" { + return "", fmt.Errorf("missing service id for wake") + } + + service, _, err := w.client.ServiceInspectWithRaw(ctx, serviceID, dockertypes.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + if service.Spec.Mode.Replicated == nil { + return "", fmt.Errorf("service %s is not replicated and cannot be scaled", serviceID) + } + + var delay time.Duration + if service.Spec.TaskTemplate.RestartPolicy != nil { + if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { + delay = *service.Spec.TaskTemplate.RestartPolicy.Delay + } + } + + waitTimeout := rs.autoScaleWaitTimeout + if waitTimeout == 0 { + waitTimeout = 60 * time.Second + } + + if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil && *service.Spec.Mode.Replicated.Replicas == 0 { + logrus.WithFields(logrus.Fields{ + "serviceID": serviceID, + "serviceName": rs.serviceName, + }).Debug("Scaling up Swarm service to 1 replica") + one := uint64(1) + service.Spec.Mode.Replicated.Replicas = &one + + _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) + if err != nil { + return "", err + } + } + + // Wait until a task is running and has an IP address + var taskIP string + deadline := time.Now().Add(waitTimeout) + for { + tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + Filters: filters.NewArgs(filters.Arg("service", serviceID)), + }) + if err == nil && len(tasks) > 0 { + var hasActiveTask bool + var hasReadyTask bool + var readyTaskTimestamp time.Time + var latestTask swarm.Task + + for _, task := range tasks { + state := task.Status.State + desiredState := task.DesiredState + + // Track the most recently created task to inspect its DesiredState. + if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { + latestTask = task + } + + // The task is considered fully Running only if both actual and desired states are running. + if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { + for _, attachment := range task.NetworksAttachments { + matchesNetwork := rs.networkID != "" && attachment.Network.ID == rs.networkID + isIngress := attachment.Network.Spec.Name == "ingress" + + if (matchesNetwork || (rs.networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { + parts := strings.Split(attachment.Addresses[0], "/") + if ip := net.ParseIP(parts[0]); ip != nil { + taskIP = parts[0] + break + } + } + } + } + + // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. + // Find the latest ready task's status timestamp to measure the restart delay start. + if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { + hasReadyTask = true + if task.Status.Timestamp.After(readyTaskTimestamp) { + readyTaskTimestamp = task.Status.Timestamp + } + } + + // Track active task states to see if Swarm is actively attempting to schedule/start a task. + if state == swarm.TaskStateNew || + state == swarm.TaskStatePending || + state == swarm.TaskStateAssigned || + state == swarm.TaskStateAccepted || + state == swarm.TaskStatePreparing || + state == swarm.TaskStateReady || + state == swarm.TaskStateStarting || + state == swarm.TaskStateRunning || + desiredState == swarm.TaskStateReady || + desiredState == swarm.TaskStateRunning { + hasActiveTask = true + } + } + + if taskIP != "" { + break + } + + // Check if Swarm gave up or is in restart delay + swarmGaveUp := false + var remainingDelay time.Duration + + if hasReadyTask && delay > 0 { + // Waker is waiting for a restart delay to expire. Dynamically extend the deadline + // so we do not timeout the connection while Swarm holds the start attempt. + if !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { + timeSinceReady := time.Since(readyTaskTimestamp) + remainingDelay = delay - timeSinceReady + newDeadline := readyTaskTimestamp.Add(delay).Add(waitTimeout) + if newDeadline.After(deadline) { + deadline = newDeadline + logrus.WithFields(logrus.Fields{ + "service": serviceID, + "remaining": remainingDelay, + "extendedWait": time.Until(deadline), + }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") + } + } + } else if !hasActiveTask && latestTask.DesiredState == swarm.TaskStateShutdown && len(tasks) > 0 { + // Mentality: If the latest task has DesiredState == Shutdown and there are no active tasks, + // Swarm has given up retrying. + swarmGaveUp = true + } + + if swarmGaveUp { + return "", fmt.Errorf("Swarm has stopped attempting to start service %s: all tasks have terminated", serviceID) + } + } + if taskIP != "" { + break + } + if ctx.Err() != nil { + return "", ctx.Err() + } + if time.Now().After(deadline) { + return "", fmt.Errorf("timeout waiting for running task for service %s", serviceID) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + _, portStr, err := net.SplitHostPort(rs.containerEndpoint) + if err != nil { + portStr = "25565" + } + endpoint := net.JoinHostPort(taskIP, portStr) + + // Wait for the task endpoint to be reachable + for { + conn, err := net.DialTimeout("tcp", endpoint, 1*time.Second) + if err == nil { + _ = conn.Close() + break + } + if ctx.Err() != nil { + return endpoint, ctx.Err() + } + if time.Now().After(deadline) { + return endpoint, fmt.Errorf("timeout waiting for Swarm service task to become reachable at %s", endpoint) + } + select { + case <-ctx.Done(): + return endpoint, ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + return endpoint, nil } } -func (w *dockerSwarmWatcherImpl) makeSleeperFunc(_ *routableService) SleeperFunc { - if !w.config.autoScaleDown { +func (w *dockerSwarmWatcherImpl) makeSleeperFunc(rs *routableSwarmService) SleeperFunc { + if rs == nil || !rs.autoScaleDown { return nil } return func(ctx context.Context) error { - logrus.Fatal("Auto scale down is not yet supported for docker swarm") + serviceID := rs.serviceID + if serviceID == "" { + return fmt.Errorf("missing service id for sleep") + } + + service, _, err := w.client.ServiceInspectWithRaw(ctx, serviceID, dockertypes.ServiceInspectOptions{}) + if err != nil { + return err + } + + if service.Spec.Mode.Replicated == nil { + return fmt.Errorf("service %s is not replicated and cannot be scaled", serviceID) + } + + replicas := service.Spec.Mode.Replicated.Replicas + if replicas != nil && *replicas > 0 { + logrus.WithFields(logrus.Fields{ + "serviceID": serviceID, + "serviceName": rs.serviceName, + }).Debug("Scaling down Swarm service to 0 replicas") + zero := uint64(0) + service.Spec.Mode.Replicated.Replicas = &zero + + _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) + if err != nil { + return err + } + } + return nil } } +func (w *dockerSwarmWatcherImpl) makeServiceLifecycleFuncs(rs *routableSwarmService) (WakerFunc, SleeperFunc) { + var wakerFunc WakerFunc + if rs.statusState == "sleeping" || rs.statusState == "waking" || rs.statusState == "running" { + wakerFunc = w.makeWakerFunc(rs) + } + return wakerFunc, w.makeSleeperFunc(rs) +} + func (w *dockerSwarmWatcherImpl) Start(ctx context.Context) error { var err error @@ -79,7 +309,7 @@ func (w *dockerSwarmWatcherImpl) Start(ctx context.Context) error { return err } - w.serviceMap = map[string]*routableService{} + w.serviceMap = map[string]*routableSwarmService{} logrus.Trace("Performing initial listing of Docker swarm services") if err := w.reconcileServices(ctx); err != nil { @@ -104,26 +334,60 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { visited := map[string]struct{}{} for _, rs := range services { + // If this is a newly discovered service, set up wakers/sleepers and create the route mapping. if oldRs, ok := w.serviceMap[rs.externalServiceName]; !ok { w.serviceMap[rs.externalServiceName] = rs - logrus.WithField("routableService", rs).Debug("ADD") - wakerFunc := w.makeWakerFunc(rs) - sleeperFunc := w.makeSleeperFunc(rs) + ipDetail := "" + if rs.statusState == "running" && rs.containerEndpoint != "" { + ipDetail = fmt.Sprintf(" (Endpoint: %s)", rs.containerEndpoint) + } + logrus.WithFields(logrus.Fields{ + "service": rs.serviceName, + "hosts": rs.externalServiceName, + }).Infof("Swarm service state: %s%s", rs.statusState, ipDetail) + + wakerFunc, sleeperFunc := w.makeServiceLifecycleFuncs(rs) if rs.externalServiceName != "" { - w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } else { - w.routes.SetDefaultRoute(rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) + } + w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) + // If the service is already tracked, check if any metadata, endpoint, MOTDs, or deadline + // changed. If so, recreate wakers/sleepers and update the route table. + } else if oldRs.containerEndpoint != rs.containerEndpoint || + oldRs.serviceID != rs.serviceID || + oldRs.networkID != rs.networkID || + oldRs.autoScaleUp != rs.autoScaleUp || + oldRs.autoScaleDown != rs.autoScaleDown || + oldRs.autoScaleAsleepMOTD != rs.autoScaleAsleepMOTD || + oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD || + oldRs.autoScaleWaitTimeout != rs.autoScaleWaitTimeout || + oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD || + oldRs.autoScaleRestartDelayMOTD != rs.autoScaleRestartDelayMOTD || + oldRs.countdownDeadline != rs.countdownDeadline || + oldRs.statusState != rs.statusState { + + if oldRs.statusState != rs.statusState { + ipDetail := "" + if rs.statusState == "running" && rs.containerEndpoint != "" { + ipDetail = fmt.Sprintf(" (Endpoint: %s)", rs.containerEndpoint) + } + logrus.WithFields(logrus.Fields{ + "service": rs.serviceName, + "hosts": rs.externalServiceName, + }).Infof("Swarm service state transition: %s -> %s%s", oldRs.statusState, rs.statusState, ipDetail) } - } else if oldRs.containerEndpoint != rs.containerEndpoint { + w.serviceMap[rs.externalServiceName] = rs - wakerFunc := w.makeWakerFunc(rs) - sleeperFunc := w.makeSleeperFunc(rs) + wakerFunc, sleeperFunc := w.makeServiceLifecycleFuncs(rs) if rs.externalServiceName != "" { w.routes.DeleteMapping(rs.externalServiceName) - w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } else { - w.routes.SetDefaultRoute(rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } + w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) logrus.WithFields(logrus.Fields{"old": oldRs, "new": rs}).Debug("UPDATE") } visited[rs.externalServiceName] = struct{}{} @@ -154,9 +418,6 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { eventFilters := filters.NewArgs( filters.Arg("type", string(events.ServiceEventType)), - filters.Arg("event", string(events.ActionCreate)), - filters.Arg("event", string(events.ActionUpdate)), - filters.Arg("event", string(events.ActionRemove)), ) eventCh, errCh := w.client.Events(ctx, events.ListOptions{Filters: eventFilters}) @@ -167,11 +428,18 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { backoff = time.Second } + ticker := time.NewTicker(5 * time.Second) + loop: for { select { case <-ctx.Done(): + ticker.Stop() return + case <-ticker.C: + if err := w.reconcileServices(ctx); err != nil { + logrus.WithError(err).Error("Docker Swarm reconciliation failed") + } case ev, ok := <-eventCh: if !ok { break loop @@ -185,12 +453,14 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { break loop } if ctx.Err() != nil { + ticker.Stop() return } logrus.WithError(err).Warn("Docker Swarm event stream error, reconnecting") break loop } } + ticker.Stop() select { case <-ctx.Done(): @@ -206,7 +476,7 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { } } -func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableService, error) { +func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableSwarmService, error) { services, err := w.client.ServiceList(ctx, dockertypes.ServiceListOptions{}) if err != nil { return nil, err @@ -236,30 +506,61 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS networkMap[network.ID] = &networkToAdd } - var result []*routableService + var result []*routableSwarmService for _, service := range services { - if service.Spec.EndpointSpec == nil || service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP { + if service.Spec.EndpointSpec == nil || + (service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP && + service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeDNSRR) { continue } - if len(service.Endpoint.VirtualIPs) == 0 { + if service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeVIP && len(service.Endpoint.VirtualIPs) == 0 { continue } - data, ok := w.parseServiceData(&service, networkMap) + data, ok := w.evaluateSwarmService(ctx, &service, networkMap) if !ok { continue } + endpoint := "" + if data.ip != "" { + endpoint = fmt.Sprintf("%s:%d", data.ip, data.port) + } + for _, host := range data.hosts { - result = append(result, &routableService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), - externalServiceName: host, + result = append(result, &routableSwarmService{ + containerEndpoint: endpoint, + externalServiceName: host, + serviceID: data.serviceID, + serviceName: data.serviceName, + networkID: data.networkID, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, + autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, + countdownDeadline: data.countdownDeadline, + statusState: data.statusState, }) } if data.def != nil && *data.def { - result = append(result, &routableService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), - externalServiceName: "", + result = append(result, &routableSwarmService{ + containerEndpoint: endpoint, + externalServiceName: "", + serviceID: data.serviceID, + serviceName: data.serviceName, + networkID: data.networkID, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, + autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, + countdownDeadline: data.countdownDeadline, + statusState: data.statusState, }) } } @@ -289,33 +590,96 @@ func dockerCheckNetworkName(id string, name string, networkMap map[string]*netwo } type parsedDockerServiceData struct { - hosts []string - port uint64 - def *bool - network *string - ip string + hosts []string + port uint64 + def *bool + network *string + networkID string + ip string + serviceID string + serviceName string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time + isDNSRR bool + statusState string } -func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases +func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { + data.autoScaleUp = w.config.autoScaleUp + data.autoScaleDown = w.config.autoScaleDown + data.autoScaleWaitTimeout = 60 * time.Second + data.serviceID = service.ID + data.serviceName = service.Spec.Name + + if !w.parseServiceLabels(service, &data) { + return + } + + // probably not minecraft related + if len(data.hosts) == 0 { + return + } + + isVIP := service.Spec.EndpointSpec != nil && service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeVIP + isDNSRR := service.Spec.EndpointSpec != nil && service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeDNSRR + data.isDNSRR = isDNSRR + + if !isVIP && !isDNSRR { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with unsupported endpoint resolution mode") + return + } + + if isVIP && len(service.Endpoint.VirtualIPs) == 0 { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, no VirtualIPs found") + return + } + + if data.port == 0 { + data.port = 25565 + } + + replicas := uint64(0) + if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil { + replicas = *service.Spec.Mode.Replicated.Replicas } + networkAliases := getNetworkAliases(service) + data.networkID = resolveTargetNetwork(service, data.network, networkMap, networkAliases) + + tasksSummary := w.queryServiceTasks(ctx, service, replicas, data.networkID) + + if !classifyServiceState(&data, service, replicas, isVIP, isDNSRR, tasksSummary, networkMap, networkAliases) { + return + } + + ok = true + return +} + +func (w *dockerSwarmWatcherImpl) parseServiceLabels(service *swarm.Service, data *parsedDockerServiceData) bool { for key, value := range service.Spec.Labels { - if key == DockerRouterLabelHost { + switch key { + case DockerRouterLabelHost: if data.hosts != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) - return + return false } data.hosts = SplitExternalHosts(value) - } - if key == DockerRouterLabelPort { + + case DockerRouterLabelPort: if data.port != 0 { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) - return + return false } var err error data.port, err = strconv.ParseUint(value, 10, 32) @@ -323,71 +687,278 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). WithError(err). Warnf("ignoring service with invalid %s", DockerRouterLabelPort) - return + return false } - } - if key == DockerRouterLabelDefault { + + case DockerRouterLabelDefault: if data.def != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) - return + return false } data.def = new(bool) lowerValue := strings.TrimSpace(strings.ToLower(value)) *data.def = lowerValue != "" && lowerValue != "0" && lowerValue != "false" && lowerValue != "no" - } - if key == DockerRouterLabelNetwork { + + case DockerRouterLabelNetwork: if data.network != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) - return + return false } data.network = new(string) *data.network = value + + case DockerRouterLabelAutoScaleUp: + autoScaleUp, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleUp) + return false + } + data.autoScaleUp = autoScaleUp + + case DockerRouterLabelAutoScaleDown: + autoScaleDown, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleDown) + return false + } + data.autoScaleDown = autoScaleDown + + case DockerRouterLabelAutoScaleAsleepMOTD: + data.autoScaleAsleepMOTD = value + + case DockerRouterLabelAutoScaleLoadingMOTD: + data.autoScaleLoadingMOTD = value + + case DockerRouterLabelAutoScaleWaitTimeout: + dur, err := time.ParseDuration(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleWaitTimeout) + return false + } + data.autoScaleWaitTimeout = dur + + case DockerRouterLabelAutoScaleFailedMOTD: + data.autoScaleFailedMOTD = value + + case DockerRouterLabelAutoScaleRestartDelayMOTD: + data.autoScaleRestartDelayMOTD = value } } + return true +} - // probably not minecraft related - if len(data.hosts) == 0 { - return +func getNetworkAliases(service *swarm.Service) map[string][]string { + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases } + return networkAliases +} - if len(service.Endpoint.VirtualIPs) == 0 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, no VirtualIPs found") - return +func resolveTargetNetwork(service *swarm.Service, labelNetwork *string, networkMap map[string]*network.Inspect, networkAliases map[string][]string) string { + if labelNetwork != nil { + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if ok, _ := dockerCheckNetworkName(netSpec.Target, *labelNetwork, networkMap, networkAliases); ok { + return netSpec.Target + } + } + } else { + // Default: Find the first non-ingress network in the task template + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if network := networkMap[netSpec.Target]; network != nil { + if network.Name != "ingress" { + return netSpec.Target + } + } + } + // Fallback to first network if all are ingress or not found in networkMap + if len(service.Spec.TaskTemplate.Networks) > 0 { + return service.Spec.TaskTemplate.Networks[0].Target + } } + return "" +} - if data.port == 0 { - data.port = 25565 +type serviceTasksSummary struct { + hasRunningTask bool + runningTaskIP string + hasReadyTask bool + readyTaskTimestamp time.Time + hasActiveTask bool + latestTask swarm.Task + delay time.Duration + tasks []swarm.Task +} + +func (w *dockerSwarmWatcherImpl) queryServiceTasks(ctx context.Context, service *swarm.Service, replicas uint64, networkID string) serviceTasksSummary { + var summary serviceTasksSummary + if replicas == 0 { + return summary } - vipIndex := -1 - if data.network != nil { - for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { - vipIndex = i - break - } else if err != nil { - // we intentionally ignore name check errors - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) + tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + Filters: filters.NewArgs(filters.Arg("service", service.ID)), + }) + if err != nil || len(tasks) == 0 { + return summary + } + + summary.tasks = tasks + + if service.Spec.TaskTemplate.RestartPolicy != nil { + if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { + summary.delay = *service.Spec.TaskTemplate.RestartPolicy.Delay + } + } + + for _, task := range tasks { + state := task.Status.State + desiredState := task.DesiredState + + // Track the most recently created task to inspect its DesiredState. + if summary.latestTask.CreatedAt.IsZero() || task.CreatedAt.After(summary.latestTask.CreatedAt) { + summary.latestTask = task + } + + // The task is considered fully Running only if both actual and desired states are running. + if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { + summary.hasRunningTask = true + // Extract direct task IP to bypass VIP propagation lag when replicas == 1 + for _, attachment := range task.NetworksAttachments { + matchesNetwork := networkID != "" && attachment.Network.ID == networkID + isIngress := attachment.Network.Spec.Name == "ingress" + + if (matchesNetwork || (networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { + parts := strings.Split(attachment.Addresses[0], "/") + if ip := net.ParseIP(parts[0]); ip != nil { + summary.runningTaskIP = parts[0] + break + } + } } } - if vipIndex == -1 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, network %s not found", *data.network) - return + + // Swarm task desired state 'ready' marks a task held during a restart delay. + // Find the latest ready task's status timestamp to measure the restart delay start. + if desiredState == swarm.TaskStateReady { + summary.hasReadyTask = true + if task.Status.Timestamp.After(summary.readyTaskTimestamp) { + summary.readyTaskTimestamp = task.Status.Timestamp + } + } + + // Track active task states to see if Swarm is actively attempting to schedule/start a task. + if state == swarm.TaskStateNew || + state == swarm.TaskStatePending || + state == swarm.TaskStateAssigned || + state == swarm.TaskStateAccepted || + state == swarm.TaskStatePreparing || + state == swarm.TaskStateReady || + state == swarm.TaskStateStarting || + state == swarm.TaskStateRunning || + desiredState == swarm.TaskStateReady || + desiredState == swarm.TaskStateRunning { + summary.hasActiveTask = true + } + } + + return summary +} + +func classifyServiceState(data *parsedDockerServiceData, service *swarm.Service, replicas uint64, isVIP bool, isDNSRR bool, tasksSummary serviceTasksSummary, networkMap map[string]*network.Inspect, networkAliases map[string][]string) bool { + if replicas == 0 { + // 1. Sleeping State: Service is scaled to zero. + data.statusState = "sleeping" + data.ip = "" + // data.autoScaleAsleepMOTD is already populated by parsing labels. + data.countdownDeadline = time.Time{} + } else if tasksSummary.hasRunningTask { + // 2. Running (Healthy) State: Service has a fully running task. + data.statusState = "running" + if replicas == 1 && tasksSummary.runningTaskIP != "" { + data.ip = tasksSummary.runningTaskIP + } else if isVIP { + vipIndex := -1 + if data.networkID != "" { + for i, vip := range service.Endpoint.VirtualIPs { + if vip.NetworkID == data.networkID { + vipIndex = i + break + } + } + } + if vipIndex == -1 { + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + } else { + vipIndex = 0 + } + } + if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + data.networkID = virtualIP.NetworkID + } else { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, unable to find match in VirtualIPs") + return false + } + } else if isDNSRR { + data.ip = service.Spec.Name } } else { - // if network isn't specified assume it's the first one - vipIndex = 0 + // Non-running states: keep data.ip empty to ensure the waker captures client connections + data.ip = "" + + // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. + if tasksSummary.hasReadyTask { + data.statusState = "restart_delay" + if data.autoScaleRestartDelayMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD + } else if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." + } + data.countdownDeadline = tasksSummary.readyTaskTimestamp.Add(tasksSummary.delay) + } else if !tasksSummary.hasActiveTask && tasksSummary.latestTask.DesiredState == swarm.TaskStateShutdown && len(tasksSummary.tasks) > 0 { + // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task + // and is no longer attempting to restart the service. + data.statusState = "failed" + if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." + } + data.countdownDeadline = time.Time{} + } else { + // 5. Starting / Waking State: Swarm has scaled the service up and the task is starting up (transited past ready), + // or it is the very first start. We display the loading MOTD on pings. + data.statusState = "waking" + if data.autoScaleLoadingMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleLoadingMOTD + } + data.countdownDeadline = time.Time{} + } } - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() - ok = true - return + return true } diff --git a/server/routes.go b/server/routes.go index 67bb8ee..07d4734 100644 --- a/server/routes.go +++ b/server/routes.go @@ -5,6 +5,7 @@ import ( "regexp" "strings" "sync" + "time" "github.com/sirupsen/logrus" ) @@ -70,6 +71,7 @@ type IRoutes interface { GetDefaultRoute() (string, string, WakerFunc, SleeperFunc) GetAsleepMOTD(serverAddress string) string GetLoadingMOTD(serverAddress string) string + SetCountdownDeadline(serverAddress string, deadline time.Time) SimplifySRV(srvEnabled bool) // BulkRegister registers a set of static mappings, attaching the scaler's waker/sleeper pair. nil-safe: a nil scaler registers without autoscaling. // Reset must be called separately and previous to this if you want to clear existing mappings. @@ -88,12 +90,13 @@ func NewRoutes(ctx context.Context) IRoutes { } type mapping struct { - backend string - waker WakerFunc - sleeper SleeperFunc - asleepMOTD string - loadingMOTD string - scalingTarget string // The endpoint to scale (may differ from backend when using proxy) + backend string + waker WakerFunc + sleeper SleeperFunc + asleepMOTD string + loadingMOTD string + scalingTarget string // The endpoint to scale (may differ from backend when using proxy) + countdownDeadline time.Time } type routesImpl struct { @@ -172,16 +175,33 @@ func (r *routesImpl) GetDefaultRoute() (string, string, WakerFunc, SleeperFunc) return r.defaultRoute.backend, r.defaultRoute.scalingTarget, r.defaultRoute.waker, r.defaultRoute.sleeper } +func formatMOTD(motd string, deadline time.Time) string { + if !strings.Contains(motd, "{duration}") { + return motd + } + if deadline.IsZero() { + return strings.ReplaceAll(motd, "{duration}", "now") + } + now := time.Now() + if now.Before(deadline) { + remaining := deadline.Sub(now) + durationStr := remaining.Round(time.Second).String() + return strings.ReplaceAll(motd, "{duration}", durationStr) + } + return strings.ReplaceAll(motd, "{duration}", "now") +} + func (r *routesImpl) GetAsleepMOTD(serverAddress string) string { r.RLock() defer r.RUnlock() if serverAddress == "" { - return r.defaultRoute.asleepMOTD + return formatMOTD(r.defaultRoute.asleepMOTD, r.defaultRoute.countdownDeadline) } + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { - return m.asleepMOTD + return formatMOTD(m.asleepMOTD, m.countdownDeadline) } return "" } @@ -191,15 +211,32 @@ func (r *routesImpl) GetLoadingMOTD(serverAddress string) string { defer r.RUnlock() if serverAddress == "" { - return r.defaultRoute.loadingMOTD + return formatMOTD(r.defaultRoute.loadingMOTD, r.defaultRoute.countdownDeadline) } + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { - return m.loadingMOTD + return formatMOTD(m.loadingMOTD, m.countdownDeadline) } return "" } +func (r *routesImpl) SetCountdownDeadline(serverAddress string, deadline time.Time) { + r.Lock() + defer r.Unlock() + + if serverAddress == "" { + r.defaultRoute.countdownDeadline = deadline + return + } + + serverAddress = strings.ToLower(serverAddress) + if m, ok := r.mappings[serverAddress]; ok { + m.countdownDeadline = deadline + r.mappings[serverAddress] = m + } +} + func (r *routesImpl) SimplifySRV(srvEnabled bool) { r.simplifySRV = srvEnabled } @@ -208,6 +245,7 @@ func (r *routesImpl) HasRoute(serverAddress string) bool { r.RLock() defer r.RUnlock() + serverAddress = strings.ToLower(serverAddress) _, exists := r.mappings[serverAddress] return exists } @@ -289,6 +327,7 @@ func (r *routesImpl) DeleteMapping(serverAddress string) bool { defer r.Unlock() logrus.WithField("serverAddress", serverAddress).Info("Deleting route") + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { r.downScaler.Cancel(m.scalingTarget) delete(r.mappings, serverAddress) diff --git a/server/server.go b/server/server.go index 7437423..7bdae8d 100644 --- a/server/server.go +++ b/server/server.go @@ -53,7 +53,7 @@ func NewServer(ctx context.Context, config *Config) (*Server, error) { routes := NewRoutes(ctx) webhookScalerConfigured := config.AutoScale.Webhook.Url != "" - downScalerEnabled := (config.AutoScale.Down && (config.InKubeCluster || config.KubeConfig != "" || config.InDocker)) || webhookScalerConfigured + downScalerEnabled := (config.AutoScale.Down && (config.InKubeCluster || config.KubeConfig != "" || config.InDocker || config.InDockerSwarm)) || webhookScalerConfigured downScalerDelay := config.AutoScale.DownAfter // Only one instance should be created // TODO why create it if not enabled? nil checks needed if optional