From cf0eae746eca7feae1ae322c81562c5e4a2a37a2 Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:22:18 -0500 Subject: [PATCH 01/38] feat(server): enable downscaler for docker swarm watcher --- server/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index 7437423..7bdae8d 100644 --- a/server/server.go +++ b/server/server.go @@ -53,7 +53,7 @@ func NewServer(ctx context.Context, config *Config) (*Server, error) { routes := NewRoutes(ctx) webhookScalerConfigured := config.AutoScale.Webhook.Url != "" - downScalerEnabled := (config.AutoScale.Down && (config.InKubeCluster || config.KubeConfig != "" || config.InDocker)) || webhookScalerConfigured + downScalerEnabled := (config.AutoScale.Down && (config.InKubeCluster || config.KubeConfig != "" || config.InDocker || config.InDockerSwarm)) || webhookScalerConfigured downScalerDelay := config.AutoScale.DownAfter // Only one instance should be created // TODO why create it if not enabled? nil checks needed if optional From d00f6d17e07cd6328882347eaf713714ead1ef03 Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:23:42 -0500 Subject: [PATCH 02/38] refactor(swarm): introduce routableSwarmService struct --- server/docker_swarm.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 124b4a3..ab4ec5d 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -33,16 +33,27 @@ func NewDockerSwarmWatcher(socket string, timeout time.Duration, autoScaleUp boo } } +type routableSwarmService struct { + externalServiceName string + containerEndpoint string + serviceID string + serviceName string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string +} + type dockerSwarmWatcherImpl struct { sync.RWMutex config dockerWatcherConfig client *client.Client - serviceMap map[string]*routableService + serviceMap map[string]*routableSwarmService monitorLock sync.Mutex routes IRoutes } -func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableService) WakerFunc { +func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableSwarmService) WakerFunc { if !w.config.autoScaleUp { return nil } @@ -52,7 +63,7 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableService) WakerFunc { } } -func (w *dockerSwarmWatcherImpl) makeSleeperFunc(_ *routableService) SleeperFunc { +func (w *dockerSwarmWatcherImpl) makeSleeperFunc(_ *routableSwarmService) SleeperFunc { if !w.config.autoScaleDown { return nil } @@ -79,7 +90,7 @@ func (w *dockerSwarmWatcherImpl) Start(ctx context.Context) error { return err } - w.serviceMap = map[string]*routableService{} + w.serviceMap = map[string]*routableSwarmService{} logrus.Trace("Performing initial listing of Docker swarm services") if err := w.reconcileServices(ctx); err != nil { @@ -206,7 +217,7 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { } } -func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableService, error) { +func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableSwarmService, error) { services, err := w.client.ServiceList(ctx, dockertypes.ServiceListOptions{}) if err != nil { return nil, err @@ -236,7 +247,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS networkMap[network.ID] = &networkToAdd } - var result []*routableService + var result []*routableSwarmService for _, service := range services { if service.Spec.EndpointSpec == nil || service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP { continue @@ -251,13 +262,13 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS } for _, host := range data.hosts { - result = append(result, &routableService{ + result = append(result, &routableSwarmService{ containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), externalServiceName: host, }) } if data.def != nil && *data.def { - result = append(result, &routableService{ + result = append(result, &routableSwarmService{ containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), externalServiceName: "", }) From 2913289005854bfd1e769e1b27b0371755bba73a Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:25:33 -0500 Subject: [PATCH 03/38] feat(swarm): parse autoscale and MOTD labels for services --- server/docker_swarm.go | 124 ++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 31 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index ab4ec5d..9f12b18 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -249,10 +249,12 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS var result []*routableSwarmService for _, service := range services { - if service.Spec.EndpointSpec == nil || service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP { + if service.Spec.EndpointSpec == nil || + (service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeVIP && + service.Spec.EndpointSpec.Mode != swarmtypes.ResolutionModeDNSRR) { continue } - if len(service.Endpoint.VirtualIPs) == 0 { + if service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeVIP && len(service.Endpoint.VirtualIPs) == 0 { continue } @@ -300,18 +302,25 @@ func dockerCheckNetworkName(id string, name string, networkMap map[string]*netwo } type parsedDockerServiceData struct { - hosts []string - port uint64 - def *bool - network *string - ip string + hosts []string + port uint64 + def *bool + network *string + ip string + serviceID string + serviceName string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string + isDNSRR bool } func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases - } + data.autoScaleUp = w.config.autoScaleUp + data.autoScaleDown = w.config.autoScaleDown + data.serviceID = service.ID + data.serviceName = service.Spec.Name for key, value := range service.Spec.Labels { if key == DockerRouterLabelHost { @@ -357,6 +366,32 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ data.network = new(string) *data.network = value } + if key == DockerRouterLabelAutoScaleUp { + autoScaleUp, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleUp) + return + } + data.autoScaleUp = autoScaleUp + } + if key == DockerRouterLabelAutoScaleDown { + autoScaleDown, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleDown) + return + } + data.autoScaleDown = autoScaleDown + } + if key == DockerRouterLabelAutoScaleAsleepMOTD { + data.autoScaleAsleepMOTD = value + } + if key == DockerRouterLabelAutoScaleLoadingMOTD { + data.autoScaleLoadingMOTD = value + } } // probably not minecraft related @@ -364,7 +399,17 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ return } - if len(service.Endpoint.VirtualIPs) == 0 { + isVIP := service.Spec.EndpointSpec != nil && service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeVIP + isDNSRR := service.Spec.EndpointSpec != nil && service.Spec.EndpointSpec.Mode == swarmtypes.ResolutionModeDNSRR + data.isDNSRR = isDNSRR + + if !isVIP && !isDNSRR { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with unsupported endpoint resolution mode") + return + } + + if isVIP && len(service.Endpoint.VirtualIPs) == 0 { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service, no VirtualIPs found") return @@ -374,31 +419,48 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ data.port = 25565 } - vipIndex := -1 - if data.network != nil { - for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { - vipIndex = i - break - } else if err != nil { + replicas := uint64(0) + if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil { + replicas = *service.Spec.Mode.Replicated.Replicas + } + + if replicas == 0 { + data.ip = "" + } else if isVIP { + vipIndex := -1 + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases + } + + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { // we intentionally ignore name check errors + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + if vipIndex == -1 { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) + Warnf("ignoring service, network %s not found", *data.network) + return } - } - if vipIndex == -1 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, network %s not found", *data.network) - return - } - } else { + } else { // if network isn't specified assume it's the first one - vipIndex = 0 + vipIndex = 0 + } + + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + } else if isDNSRR { + data.ip = service.Spec.Name } - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() ok = true return } From 8551a0a5557df135fd89b253d15247f096dd7866 Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:26:20 -0500 Subject: [PATCH 04/38] feat(swarm): implement service scale-down sleeper function --- server/docker_swarm.go | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 9f12b18..cc45265 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -63,12 +63,40 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableSwarmService) WakerFun } } -func (w *dockerSwarmWatcherImpl) makeSleeperFunc(_ *routableSwarmService) SleeperFunc { - if !w.config.autoScaleDown { +func (w *dockerSwarmWatcherImpl) makeSleeperFunc(rs *routableSwarmService) SleeperFunc { + if rs == nil || !rs.autoScaleDown { return nil } return func(ctx context.Context) error { - logrus.Fatal("Auto scale down is not yet supported for docker swarm") + serviceID := rs.serviceID + if serviceID == "" { + return fmt.Errorf("missing service id for sleep") + } + + service, _, err := w.client.ServiceInspectWithRaw(ctx, serviceID, dockertypes.ServiceInspectOptions{}) + if err != nil { + return err + } + + if service.Spec.Mode.Replicated == nil { + return fmt.Errorf("service %s is not replicated and cannot be scaled", serviceID) + } + + replicas := service.Spec.Mode.Replicated.Replicas + if replicas != nil && *replicas > 0 { + logrus.WithFields(logrus.Fields{ + "serviceID": serviceID, + "serviceName": rs.serviceName, + }).Debug("Scaling down Swarm service to 0 replicas") + zero := uint64(0) + service.Spec.Mode.Replicated.Replicas = &zero + + _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) + if err != nil { + return err + } + } + return nil } } From 72929ac511398bc7e87f9e97ec0de79293e3094c Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:27:14 -0500 Subject: [PATCH 05/38] feat(swarm): implement service scale-up waker function with task IP resolution --- server/docker_swarm.go | 106 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index cc45265..bedd808 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -53,13 +53,111 @@ type dockerSwarmWatcherImpl struct { routes IRoutes } -func (w *dockerSwarmWatcherImpl) makeWakerFunc(_ *routableSwarmService) WakerFunc { - if !w.config.autoScaleUp { +func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFunc { + if rs == nil || !rs.autoScaleUp { return nil } return func(ctx context.Context) (string, error) { - logrus.Fatal("Auto scale up is not yet supported for docker swarm") - return "", nil + serviceID := rs.serviceID + if serviceID == "" { + return "", fmt.Errorf("missing service id for wake") + } + + service, _, err := w.client.ServiceInspectWithRaw(ctx, serviceID, dockertypes.ServiceInspectOptions{}) + if err != nil { + return "", err + } + + if service.Spec.Mode.Replicated == nil { + return "", fmt.Errorf("service %s is not replicated and cannot be scaled", serviceID) + } + + replicas := service.Spec.Mode.Replicated.Replicas + if replicas == nil || *replicas == 0 { + logrus.WithFields(logrus.Fields{ + "serviceID": serviceID, + "serviceName": rs.serviceName, + }).Debug("Scaling up Swarm service to 1 replica") + one := uint64(1) + service.Spec.Mode.Replicated.Replicas = &one + + _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) + if err != nil { + return "", err + } + } + + // Wait until a task is running and has an IP address + var taskIP string + deadline := time.Now().Add(60 * time.Second) + for { + tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + Filters: filters.NewArgs( + filters.Arg("service", serviceID), + filters.Arg("desired-state", "running"), + ), + }) + if err == nil && len(tasks) > 0 { + for _, task := range tasks { + if task.Status.State == swarm.TaskStateRunning { + for _, attachment := range task.NetworksAttachments { + if len(attachment.Addresses) > 0 { + parts := strings.Split(attachment.Addresses[0], "/") + if ip := net.ParseIP(parts[0]); ip != nil { + taskIP = parts[0] + break + } + } + } + } + if taskIP != "" { + break + } + } + } + if taskIP != "" { + break + } + if ctx.Err() != nil { + return "", ctx.Err() + } + if time.Now().After(deadline) { + return "", fmt.Errorf("timeout waiting for running task for service %s", serviceID) + } + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + _, portStr, err := net.SplitHostPort(rs.containerEndpoint) + if err != nil { + portStr = "25565" + } + endpoint := net.JoinHostPort(taskIP, portStr) + + // Wait for the task endpoint to be reachable + for { + conn, err := net.DialTimeout("tcp", endpoint, 1*time.Second) + if err == nil { + _ = conn.Close() + break + } + if ctx.Err() != nil { + return endpoint, ctx.Err() + } + if time.Now().After(deadline) { + return endpoint, fmt.Errorf("timeout waiting for Swarm service task to become reachable at %s", endpoint) + } + select { + case <-ctx.Done(): + return endpoint, ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } + + return endpoint, nil } } From 859a7682724d8e1c2bb8a23c5a7cd8e849fc2e8f Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 18:29:39 -0500 Subject: [PATCH 06/38] feat(swarm): map wakers, sleepers, and custom MOTDs to route table --- server/docker_swarm.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index bedd808..e677870 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -247,19 +247,25 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { wakerFunc := w.makeWakerFunc(rs) sleeperFunc := w.makeSleeperFunc(rs) if rs.externalServiceName != "" { - w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } else { - w.routes.SetDefaultRoute(rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } - } else if oldRs.containerEndpoint != rs.containerEndpoint { + } else if oldRs.containerEndpoint != rs.containerEndpoint || + oldRs.serviceID != rs.serviceID || + oldRs.autoScaleUp != rs.autoScaleUp || + oldRs.autoScaleDown != rs.autoScaleDown || + oldRs.autoScaleAsleepMOTD != rs.autoScaleAsleepMOTD || + oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD { + w.serviceMap[rs.externalServiceName] = rs wakerFunc := w.makeWakerFunc(rs) sleeperFunc := w.makeSleeperFunc(rs) if rs.externalServiceName != "" { w.routes.DeleteMapping(rs.externalServiceName) - w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } else { - w.routes.SetDefaultRoute(rs.containerEndpoint, "", wakerFunc, sleeperFunc, "", "") + w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } logrus.WithFields(logrus.Fields{"old": oldRs, "new": rs}).Debug("UPDATE") } @@ -393,12 +399,24 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS result = append(result, &routableSwarmService{ containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), externalServiceName: host, + serviceID: data.serviceID, + serviceName: data.serviceName, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, }) } if data.def != nil && *data.def { result = append(result, &routableSwarmService{ containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), externalServiceName: "", + serviceID: data.serviceID, + serviceName: data.serviceName, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, }) } } From cf90b66db8aa04fbf77ec0cdba5ffc071dd0decd Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 6 Jul 2026 20:11:13 -0500 Subject: [PATCH 07/38] fix(swarm): resolve correct task IP by matching container network ID --- server/docker_swarm.go | 87 +++++++++++++++++++++++++++++++----------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index e677870..ad1f40a 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -38,6 +38,7 @@ type routableSwarmService struct { containerEndpoint string serviceID string serviceName string + networkID string autoScaleUp bool autoScaleDown bool autoScaleAsleepMOTD string @@ -101,7 +102,10 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu for _, task := range tasks { if task.Status.State == swarm.TaskStateRunning { for _, attachment := range task.NetworksAttachments { - if len(attachment.Addresses) > 0 { + matchesNetwork := rs.networkID != "" && attachment.Network.ID == rs.networkID + isIngress := attachment.Network.Spec.Name == "ingress" + + if (matchesNetwork || (rs.networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { parts := strings.Split(attachment.Addresses[0], "/") if ip := net.ParseIP(parts[0]); ip != nil { taskIP = parts[0] @@ -253,6 +257,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { } } else if oldRs.containerEndpoint != rs.containerEndpoint || oldRs.serviceID != rs.serviceID || + oldRs.networkID != rs.networkID || oldRs.autoScaleUp != rs.autoScaleUp || oldRs.autoScaleDown != rs.autoScaleDown || oldRs.autoScaleAsleepMOTD != rs.autoScaleAsleepMOTD || @@ -401,6 +406,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS externalServiceName: host, serviceID: data.serviceID, serviceName: data.serviceName, + networkID: data.networkID, autoScaleUp: data.autoScaleUp, autoScaleDown: data.autoScaleDown, autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, @@ -413,6 +419,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS externalServiceName: "", serviceID: data.serviceID, serviceName: data.serviceName, + networkID: data.networkID, autoScaleUp: data.autoScaleUp, autoScaleDown: data.autoScaleDown, autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, @@ -450,6 +457,7 @@ type parsedDockerServiceData struct { port uint64 def *bool network *string + networkID string ip string serviceID string serviceName string @@ -568,39 +576,74 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ replicas = *service.Spec.Mode.Replicated.Replicas } + // Resolve target networkID based on label or task template networks + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases + } + + if data.network != nil { + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if ok, _ := dockerCheckNetworkName(netSpec.Target, *data.network, networkMap, networkAliases); ok { + data.networkID = netSpec.Target + break + } + } + } else { + // Default: Find the first non-ingress network in the task template + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if network := networkMap[netSpec.Target]; network != nil { + if network.Name != "ingress" { + data.networkID = netSpec.Target + break + } + } + } + // Fallback to first network if all are ingress or not found in networkMap + if data.networkID == "" && len(service.Spec.TaskTemplate.Networks) > 0 { + data.networkID = service.Spec.TaskTemplate.Networks[0].Target + } + } + if replicas == 0 { data.ip = "" } else if isVIP { vipIndex := -1 - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases - } - - if data.network != nil { + if data.networkID != "" { for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + if vip.NetworkID == data.networkID { vipIndex = i break - } else if err != nil { - // we intentionally ignore name check errors - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) } } - if vipIndex == -1 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, network %s not found", *data.network) - return + } + if vipIndex == -1 { + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { + // we intentionally ignore name check errors + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + } else { + // if network isn't specified assume it's the first one + vipIndex = 0 } + } + if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + data.networkID = virtualIP.NetworkID } else { - // if network isn't specified assume it's the first one - vipIndex = 0 + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, unable to find match in VirtualIPs") + return } - - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() } else if isDNSRR { data.ip = service.Spec.Name } From 8bd5d5c32c98cee73f17ebaf26ab0f9fdc10572d Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 11:02:57 -0500 Subject: [PATCH 08/38] fix(swarm): register empty backend endpoint when replicas are 0 to allow status interception --- server/docker_swarm.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index ad1f40a..8bc0aca 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -400,9 +400,14 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS continue } + endpoint := "" + if data.ip != "" { + endpoint = fmt.Sprintf("%s:%d", data.ip, data.port) + } + for _, host := range data.hosts { result = append(result, &routableSwarmService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + containerEndpoint: endpoint, externalServiceName: host, serviceID: data.serviceID, serviceName: data.serviceName, @@ -415,7 +420,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS } if data.def != nil && *data.def { result = append(result, &routableSwarmService{ - containerEndpoint: fmt.Sprintf("%s:%d", data.ip, data.port), + containerEndpoint: endpoint, externalServiceName: "", serviceID: data.serviceID, serviceName: data.serviceName, From cc2d803a9ce78f57cff85a233b6f67da2ee55e62 Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 16:34:19 -0500 Subject: [PATCH 09/38] feat(swarm): parse timeout and failed MOTD labels --- server/docker.go | 2 ++ server/docker_swarm.go | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/server/docker.go b/server/docker.go index e70197a..2c9c3cc 100644 --- a/server/docker.go +++ b/server/docker.go @@ -30,6 +30,8 @@ const ( DockerRouterLabelAutoScaleDown = "mc-router.auto-scale-down" DockerRouterLabelAutoScaleAsleepMOTD = "mc-router.auto-scale-asleep-motd" DockerRouterLabelAutoScaleLoadingMOTD = "mc-router.auto-scale-loading-motd" + DockerRouterLabelAutoScaleWaitTimeout = "mc-router.auto-scale-wait-timeout" + DockerRouterLabelAutoScaleFailedMOTD = "mc-router.auto-scale-failed-motd" ) type dockerWatcherConfig struct { diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 8bc0aca..56da034 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -43,6 +43,8 @@ type routableSwarmService struct { autoScaleDown bool autoScaleAsleepMOTD string autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string } type dockerSwarmWatcherImpl struct { @@ -470,12 +472,15 @@ type parsedDockerServiceData struct { autoScaleDown bool autoScaleAsleepMOTD string autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string isDNSRR bool } func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { data.autoScaleUp = w.config.autoScaleUp data.autoScaleDown = w.config.autoScaleDown + data.autoScaleWaitTimeout = 60 * time.Second data.serviceID = service.ID data.serviceName = service.Spec.Name @@ -549,6 +554,19 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ if key == DockerRouterLabelAutoScaleLoadingMOTD { data.autoScaleLoadingMOTD = value } + if key == DockerRouterLabelAutoScaleWaitTimeout { + dur, err := time.ParseDuration(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleWaitTimeout) + return + } + data.autoScaleWaitTimeout = dur + } + if key == DockerRouterLabelAutoScaleFailedMOTD { + data.autoScaleFailedMOTD = value + } } // probably not minecraft related From 6f3ab8960130c4170b06155d9d8b8e95e58c803d Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 16:36:34 -0500 Subject: [PATCH 10/38] feat(swarm): detect restart delays and crash loops during service parsing --- server/docker_swarm.go | 94 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 4 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 56da034..22c557c 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -263,7 +263,9 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { oldRs.autoScaleUp != rs.autoScaleUp || oldRs.autoScaleDown != rs.autoScaleDown || oldRs.autoScaleAsleepMOTD != rs.autoScaleAsleepMOTD || - oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD { + oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD || + oldRs.autoScaleWaitTimeout != rs.autoScaleWaitTimeout || + oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD { w.serviceMap[rs.externalServiceName] = rs wakerFunc := w.makeWakerFunc(rs) @@ -397,7 +399,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS continue } - data, ok := w.parseServiceData(&service, networkMap) + data, ok := w.parseServiceData(ctx, &service, networkMap) if !ok { continue } @@ -418,6 +420,8 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleDown: data.autoScaleDown, autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, }) } if data.def != nil && *data.def { @@ -431,6 +435,8 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleDown: data.autoScaleDown, autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, }) } } @@ -477,7 +483,7 @@ type parsedDockerServiceData struct { isDNSRR bool } -func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { +func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { data.autoScaleUp = w.config.autoScaleUp data.autoScaleDown = w.config.autoScaleDown data.autoScaleWaitTimeout = 60 * time.Second @@ -628,8 +634,88 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(service *swarm.Service, networ } } - if replicas == 0 { + var hasActiveTask bool + var terminalFailureCount int + var lastFailedTime time.Time + var delay time.Duration + var maxAttempts uint64 + + if replicas > 0 { + tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + Filters: filters.NewArgs(filters.Arg("service", service.ID)), + }) + if err == nil && len(tasks) > 0 { + if service.Spec.TaskTemplate.RestartPolicy != nil { + if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { + delay = *service.Spec.TaskTemplate.RestartPolicy.Delay + } + if service.Spec.TaskTemplate.RestartPolicy.MaxAttempts != nil { + maxAttempts = *service.Spec.TaskTemplate.RestartPolicy.MaxAttempts + } + } + + for _, task := range tasks { + state := task.Status.State + if state == swarm.TaskStateNew || + state == swarm.TaskStatePending || + state == swarm.TaskStateAssigned || + state == swarm.TaskStateAccepted || + state == swarm.TaskStatePreparing || + state == swarm.TaskStateStarting || + state == swarm.TaskStateRunning { + hasActiveTask = true + } else if state == swarm.TaskStateFailed || state == swarm.TaskStateShutdown || state == swarm.TaskStateRejected { + terminalFailureCount++ + if task.Status.Timestamp.After(lastFailedTime) { + lastFailedTime = task.Status.Timestamp + } + } + } + } + } + + swarmGaveUp := false + inRestartDelay := false + var remainingDelay time.Duration + + if replicas > 0 && !hasActiveTask { + if delay > 0 && !lastFailedTime.IsZero() { + timeSinceFailed := time.Since(lastFailedTime) + if timeSinceFailed < delay { + inRestartDelay = true + remainingDelay = delay - timeSinceFailed + } else { + swarmGaveUp = true + } + } else if maxAttempts > 0 && uint64(terminalFailureCount) >= maxAttempts { + swarmGaveUp = true + } else { + // Fallback: if no active tasks and no delay or max attempts, assume Swarm gave up + swarmGaveUp = true + } + } + + if replicas == 0 || swarmGaveUp || inRestartDelay { data.ip = "" + + // Format dynamic countdown or failed message + if inRestartDelay { + durationStr := remainingDelay.Round(time.Second).String() + if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleFailedMOTD, "{duration}", durationStr) + } else { + data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleAsleepMOTD, "{duration}", durationStr) + } + if data.autoScaleLoadingMOTD != "" { + data.autoScaleLoadingMOTD = strings.ReplaceAll(data.autoScaleLoadingMOTD, "{duration}", durationStr) + } + } else if swarmGaveUp { + if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleFailedMOTD, "{duration}", "failed") + } else { + data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleAsleepMOTD, "{duration}", "failed") + } + } } else if isVIP { vipIndex := -1 if data.networkID != "" { From db8cfcd6f0c15556a30a1e58c4219409d1229ff5 Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 16:43:36 -0500 Subject: [PATCH 11/38] feat(swarm): extend waker timeout on restart delay and fail fast on crash loop --- server/docker_swarm.go | 83 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 22c557c..bfe63e3 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -75,6 +75,22 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu return "", fmt.Errorf("service %s is not replicated and cannot be scaled", serviceID) } + var delay time.Duration + var maxAttempts uint64 + if service.Spec.TaskTemplate.RestartPolicy != nil { + if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { + delay = *service.Spec.TaskTemplate.RestartPolicy.Delay + } + if service.Spec.TaskTemplate.RestartPolicy.MaxAttempts != nil { + maxAttempts = *service.Spec.TaskTemplate.RestartPolicy.MaxAttempts + } + } + + waitTimeout := rs.autoScaleWaitTimeout + if waitTimeout == 0 { + waitTimeout = 60 * time.Second + } + replicas := service.Spec.Mode.Replicated.Replicas if replicas == nil || *replicas == 0 { logrus.WithFields(logrus.Fields{ @@ -92,17 +108,19 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu // Wait until a task is running and has an IP address var taskIP string - deadline := time.Now().Add(60 * time.Second) + deadline := time.Now().Add(waitTimeout) for { tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ - Filters: filters.NewArgs( - filters.Arg("service", serviceID), - filters.Arg("desired-state", "running"), - ), + Filters: filters.NewArgs(filters.Arg("service", serviceID)), }) if err == nil && len(tasks) > 0 { + var hasActiveTask bool + var terminalFailureCount int + var lastFailedTime time.Time + for _, task := range tasks { - if task.Status.State == swarm.TaskStateRunning { + state := task.Status.State + if state == swarm.TaskStateRunning { for _, attachment := range task.NetworksAttachments { matchesNetwork := rs.networkID != "" && attachment.Network.ID == rs.networkID isIngress := attachment.Network.Spec.Name == "ingress" @@ -116,8 +134,57 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu } } } - if taskIP != "" { - break + + if state == swarm.TaskStateNew || + state == swarm.TaskStatePending || + state == swarm.TaskStateAssigned || + state == swarm.TaskStateAccepted || + state == swarm.TaskStatePreparing || + state == swarm.TaskStateStarting || + state == swarm.TaskStateRunning { + hasActiveTask = true + } else if state == swarm.TaskStateFailed || state == swarm.TaskStateShutdown || state == swarm.TaskStateRejected { + terminalFailureCount++ + if task.Status.Timestamp.After(lastFailedTime) { + lastFailedTime = task.Status.Timestamp + } + } + } + + if taskIP != "" { + break + } + + // Check if Swarm gave up or is in restart delay + if !hasActiveTask { + swarmGaveUp := false + var remainingDelay time.Duration + + if delay > 0 && !lastFailedTime.IsZero() { + timeSinceFailed := time.Since(lastFailedTime) + if timeSinceFailed < delay { + remainingDelay = delay - timeSinceFailed + newDeadline := lastFailedTime.Add(delay).Add(waitTimeout) + if newDeadline.After(deadline) { + deadline = newDeadline + logrus.WithFields(logrus.Fields{ + "service": serviceID, + "remaining": remainingDelay, + "extendedWait": time.Until(deadline), + }).Info("Swarm task entered restart delay. Dynamically extending waker deadline.") + } + } else { + swarmGaveUp = true + } + } else if maxAttempts > 0 && uint64(terminalFailureCount) >= maxAttempts { + swarmGaveUp = true + } else { + // Fallback: if no active tasks and no restart delay/max attempts configured, assume Swarm gave up + swarmGaveUp = true + } + + if swarmGaveUp { + return "", fmt.Errorf("Swarm has stopped attempting to start service %s: all tasks have terminated", serviceID) } } } From 00d68559d6bb42fde442fc4bc6fa3e845b334eb7 Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 16:49:27 -0500 Subject: [PATCH 12/38] docs(readme): document Swarm wait-timeout and failed-motd labels --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 21c1423..eb81807 100644 --- a/README.md +++ b/README.md @@ -194,11 +194,12 @@ These are the labels scanned: - `mc-router.port`: This value must be set to the port the Minecraft server is listening on. The default value is 25565. - `mc-router.default`: Set this to a truthy value to make this server the default backend. Please note that `mc-router.host` is still required to be set. - `mc-router.network`: Specify the network you are using for the router if multiple are present in the container/service. You can either use the network ID, it's full name or an alias. -- `mc-router.auto-scale-up`: Per-container override to enable/disable auto scale up for Docker. When true (or left unspecified and the global `-auto-scale-up` flag is enabled), mc-router will start or unpause this container when a client connects to the declared hostname(s). -- `mc-router.auto-scale-down`: Per-container override to enable/disable auto scale down for Docker. When true (or left unspecified and the global `-auto-scale-down` flag is enabled), mc-router will stop this container after it has been idle for the configured `-auto-scale-down-after` duration. -- `mc-router.auto-scale-asleep-motd`: Per-container override for MOTD to show when container is scaled to zero. If empty or not set the host will -appear unresponsive. -- `mc-router.auto-scale-loading-motd`: Per-container override for MOTD to show while the container is waking and not yet reachable. If empty or not set, the global `-auto-scale-loading-motd` value is used. +- `mc-router.auto-scale-up`: Per-container/service override to enable/disable auto scale up for Docker/Swarm. When true (or left unspecified and the global `-auto-scale-up` flag is enabled), mc-router will start the container or scale up the Swarm service when a client connects. +- `mc-router.auto-scale-down`: Per-container/service override to enable/disable auto scale down for Docker/Swarm. When true (or left unspecified and the global `-auto-scale-down` flag is enabled), mc-router will stop the container or scale down the Swarm service to 0 after it has been idle. +- `mc-router.auto-scale-asleep-motd`: Per-container/service override for MOTD to show when scaled to zero. If empty or not set the host will appear unresponsive. +- `mc-router.auto-scale-loading-motd`: Per-container/service override for MOTD to show while waking up. Supports replacing the `{duration}` token with the remaining Swarm restart delay if the task is waiting to retry. If empty or not set, the global `-auto-scale-loading-motd` value is used. +- `mc-router.auto-scale-wait-timeout`: Configure the maximum duration the router waits for the container or Swarm task to become reachable after scaling up (e.g. `"5m"` or `"300s"`). Defaults to 60s. +- `mc-router.auto-scale-failed-motd`: MOTD to show if the container/service fails to start or Swarm exhausts its restart policy (e.g. `"Server failed to start. Retrying in {duration}."`). Supports the `{duration}` countdown token, which dynamically updates during Swarm restart delays. #### Docker Auto Scale Up/Down From 3eb7e385443e5f705ffff2677e8104761b2f42fd Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 17:32:30 -0500 Subject: [PATCH 13/38] fix(swarm): resolve loading MOTD bypassing and stuck restart delay countdowns --- server/docker_swarm.go | 57 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index bfe63e3..b9297d3 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -140,6 +140,7 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu state == swarm.TaskStateAssigned || state == swarm.TaskStateAccepted || state == swarm.TaskStatePreparing || + state == swarm.TaskStateReady || state == swarm.TaskStateStarting || state == swarm.TaskStateRunning { hasActiveTask = true @@ -156,10 +157,12 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu } // Check if Swarm gave up or is in restart delay - if !hasActiveTask { - swarmGaveUp := false - var remainingDelay time.Duration + swarmGaveUp := false + var remainingDelay time.Duration + if !hasActiveTask && len(tasks) > 0 { + swarmGaveUp = true + } else if hasActiveTask && taskIP == "" { if delay > 0 && !lastFailedTime.IsZero() { timeSinceFailed := time.Since(lastFailedTime) if timeSinceFailed < delay { @@ -171,21 +174,14 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu "service": serviceID, "remaining": remainingDelay, "extendedWait": time.Until(deadline), - }).Info("Swarm task entered restart delay. Dynamically extending waker deadline.") + }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") } - } else { - swarmGaveUp = true } - } else if maxAttempts > 0 && uint64(terminalFailureCount) >= maxAttempts { - swarmGaveUp = true - } else { - // Fallback: if no active tasks and no restart delay/max attempts configured, assume Swarm gave up - swarmGaveUp = true } + } - if swarmGaveUp { - return "", fmt.Errorf("Swarm has stopped attempting to start service %s: all tasks have terminated", serviceID) - } + if swarmGaveUp { + return "", fmt.Errorf("Swarm has stopped attempting to start service %s: all tasks have terminated", serviceID) } } if taskIP != "" { @@ -701,14 +697,17 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } } + var hasRunningTask bool var hasActiveTask bool var terminalFailureCount int var lastFailedTime time.Time var delay time.Duration var maxAttempts uint64 + var tasks []swarm.Task + var err error if replicas > 0 { - tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + tasks, err = w.client.TaskList(ctx, dockertypes.TaskListOptions{ Filters: filters.NewArgs(filters.Arg("service", service.ID)), }) if err == nil && len(tasks) > 0 { @@ -723,11 +722,16 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * for _, task := range tasks { state := task.Status.State + if state == swarm.TaskStateRunning { + hasRunningTask = true + } + if state == swarm.TaskStateNew || state == swarm.TaskStatePending || state == swarm.TaskStateAssigned || state == swarm.TaskStateAccepted || state == swarm.TaskStatePreparing || + state == swarm.TaskStateReady || state == swarm.TaskStateStarting || state == swarm.TaskStateRunning { hasActiveTask = true @@ -745,20 +749,17 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * inRestartDelay := false var remainingDelay time.Duration - if replicas > 0 && !hasActiveTask { - if delay > 0 && !lastFailedTime.IsZero() { - timeSinceFailed := time.Since(lastFailedTime) - if timeSinceFailed < delay { - inRestartDelay = true - remainingDelay = delay - timeSinceFailed - } else { - swarmGaveUp = true - } - } else if maxAttempts > 0 && uint64(terminalFailureCount) >= maxAttempts { - swarmGaveUp = true - } else { - // Fallback: if no active tasks and no delay or max attempts, assume Swarm gave up + if replicas > 0 && !hasRunningTask { + if !hasActiveTask && len(tasks) > 0 { swarmGaveUp = true + } else if hasActiveTask { + if delay > 0 && !lastFailedTime.IsZero() { + timeSinceFailed := time.Since(lastFailedTime) + if timeSinceFailed < delay { + inRestartDelay = true + remainingDelay = delay - timeSinceFailed + } + } } } From 3821c565ddc4959c61a23598deb91ca14ceba288 Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 17:34:05 -0500 Subject: [PATCH 14/38] refactor(swarm): remove unused maxAttempts variable from waker and parser --- server/docker_swarm.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index b9297d3..c813d9d 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -76,14 +76,10 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu } var delay time.Duration - var maxAttempts uint64 if service.Spec.TaskTemplate.RestartPolicy != nil { if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { delay = *service.Spec.TaskTemplate.RestartPolicy.Delay } - if service.Spec.TaskTemplate.RestartPolicy.MaxAttempts != nil { - maxAttempts = *service.Spec.TaskTemplate.RestartPolicy.MaxAttempts - } } waitTimeout := rs.autoScaleWaitTimeout @@ -702,7 +698,6 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * var terminalFailureCount int var lastFailedTime time.Time var delay time.Duration - var maxAttempts uint64 var tasks []swarm.Task var err error @@ -715,9 +710,6 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { delay = *service.Spec.TaskTemplate.RestartPolicy.Delay } - if service.Spec.TaskTemplate.RestartPolicy.MaxAttempts != nil { - maxAttempts = *service.Spec.TaskTemplate.RestartPolicy.MaxAttempts - } } for _, task := range tasks { From c184593f6584ddf5560b42c96d671be367001779 Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 17:59:17 -0500 Subject: [PATCH 15/38] feat(swarm): resolve dynamic MOTD countdowns on-the-fly and listen to TaskEventType events --- server/docker.go | 1 + server/docker_swarm.go | 11 ++++++++- server/routes.go | 54 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/server/docker.go b/server/docker.go index 2c9c3cc..53e5e25 100644 --- a/server/docker.go +++ b/server/docker.go @@ -32,6 +32,7 @@ const ( DockerRouterLabelAutoScaleLoadingMOTD = "mc-router.auto-scale-loading-motd" DockerRouterLabelAutoScaleWaitTimeout = "mc-router.auto-scale-wait-timeout" DockerRouterLabelAutoScaleFailedMOTD = "mc-router.auto-scale-failed-motd" + DockerRouterEventTypeTask = "task" ) type dockerWatcherConfig struct { diff --git a/server/docker_swarm.go b/server/docker_swarm.go index c813d9d..7c5b79e 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -45,6 +45,7 @@ type routableSwarmService struct { autoScaleLoadingMOTD string autoScaleWaitTimeout time.Duration autoScaleFailedMOTD string + countdownDeadline time.Time } type dockerSwarmWatcherImpl struct { @@ -316,6 +317,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { } else { w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } + w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) } else if oldRs.containerEndpoint != rs.containerEndpoint || oldRs.serviceID != rs.serviceID || oldRs.networkID != rs.networkID || @@ -324,7 +326,8 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { oldRs.autoScaleAsleepMOTD != rs.autoScaleAsleepMOTD || oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD || oldRs.autoScaleWaitTimeout != rs.autoScaleWaitTimeout || - oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD { + oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD || + oldRs.countdownDeadline != rs.countdownDeadline { w.serviceMap[rs.externalServiceName] = rs wakerFunc := w.makeWakerFunc(rs) @@ -335,6 +338,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { } else { w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } + w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) logrus.WithFields(logrus.Fields{"old": oldRs, "new": rs}).Debug("UPDATE") } visited[rs.externalServiceName] = struct{}{} @@ -368,6 +372,7 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { filters.Arg("event", string(events.ActionCreate)), filters.Arg("event", string(events.ActionUpdate)), filters.Arg("event", string(events.ActionRemove)), + filters.Arg("type", DockerRouterEventTypeTask), ) eventCh, errCh := w.client.Events(ctx, events.ListOptions{Filters: eventFilters}) @@ -481,6 +486,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, autoScaleWaitTimeout: data.autoScaleWaitTimeout, autoScaleFailedMOTD: data.autoScaleFailedMOTD, + countdownDeadline: data.countdownDeadline, }) } if data.def != nil && *data.def { @@ -496,6 +502,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, autoScaleWaitTimeout: data.autoScaleWaitTimeout, autoScaleFailedMOTD: data.autoScaleFailedMOTD, + countdownDeadline: data.countdownDeadline, }) } } @@ -539,6 +546,7 @@ type parsedDockerServiceData struct { autoScaleLoadingMOTD string autoScaleWaitTimeout time.Duration autoScaleFailedMOTD string + countdownDeadline time.Time isDNSRR bool } @@ -750,6 +758,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if timeSinceFailed < delay { inRestartDelay = true remainingDelay = delay - timeSinceFailed + data.countdownDeadline = lastFailedTime.Add(delay) } } } diff --git a/server/routes.go b/server/routes.go index 67bb8ee..80ccf9d 100644 --- a/server/routes.go +++ b/server/routes.go @@ -5,6 +5,7 @@ import ( "regexp" "strings" "sync" + "time" "github.com/sirupsen/logrus" ) @@ -70,6 +71,7 @@ type IRoutes interface { GetDefaultRoute() (string, string, WakerFunc, SleeperFunc) GetAsleepMOTD(serverAddress string) string GetLoadingMOTD(serverAddress string) string + SetCountdownDeadline(serverAddress string, deadline time.Time) SimplifySRV(srvEnabled bool) // BulkRegister registers a set of static mappings, attaching the scaler's waker/sleeper pair. nil-safe: a nil scaler registers without autoscaling. // Reset must be called separately and previous to this if you want to clear existing mappings. @@ -88,12 +90,13 @@ func NewRoutes(ctx context.Context) IRoutes { } type mapping struct { - backend string - waker WakerFunc - sleeper SleeperFunc - asleepMOTD string - loadingMOTD string - scalingTarget string // The endpoint to scale (may differ from backend when using proxy) + backend string + waker WakerFunc + sleeper SleeperFunc + asleepMOTD string + loadingMOTD string + scalingTarget string // The endpoint to scale (may differ from backend when using proxy) + countdownDeadline time.Time } type routesImpl struct { @@ -172,16 +175,32 @@ func (r *routesImpl) GetDefaultRoute() (string, string, WakerFunc, SleeperFunc) return r.defaultRoute.backend, r.defaultRoute.scalingTarget, r.defaultRoute.waker, r.defaultRoute.sleeper } +func formatMOTD(motd string, deadline time.Time) string { + if !strings.Contains(motd, "{duration}") { + return motd + } + if deadline.IsZero() { + return strings.ReplaceAll(motd, "{duration}", "failed") + } + now := time.Now() + if now.Before(deadline) { + remaining := deadline.Sub(now) + durationStr := remaining.Round(time.Second).String() + return strings.ReplaceAll(motd, "{duration}", durationStr) + } + return strings.ReplaceAll(motd, "{duration}", "failed") +} + func (r *routesImpl) GetAsleepMOTD(serverAddress string) string { r.RLock() defer r.RUnlock() if serverAddress == "" { - return r.defaultRoute.asleepMOTD + return formatMOTD(r.defaultRoute.asleepMOTD, r.defaultRoute.countdownDeadline) } if m, ok := r.mappings[serverAddress]; ok { - return m.asleepMOTD + return formatMOTD(m.asleepMOTD, m.countdownDeadline) } return "" } @@ -191,15 +210,30 @@ func (r *routesImpl) GetLoadingMOTD(serverAddress string) string { defer r.RUnlock() if serverAddress == "" { - return r.defaultRoute.loadingMOTD + return formatMOTD(r.defaultRoute.loadingMOTD, r.defaultRoute.countdownDeadline) } if m, ok := r.mappings[serverAddress]; ok { - return m.loadingMOTD + return formatMOTD(m.loadingMOTD, m.countdownDeadline) } return "" } +func (r *routesImpl) SetCountdownDeadline(serverAddress string, deadline time.Time) { + r.Lock() + defer r.Unlock() + + if serverAddress == "" { + r.defaultRoute.countdownDeadline = deadline + return + } + + if m, ok := r.mappings[serverAddress]; ok { + m.countdownDeadline = deadline + r.mappings[serverAddress] = m + } +} + func (r *routesImpl) SimplifySRV(srvEnabled bool) { r.simplifySRV = srvEnabled } From 753456796033217f5b42ab6753c5380b62f5e9fb Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 18:26:36 -0500 Subject: [PATCH 16/38] fix(swarm): remove event action filters and preserve failed MOTD template for dynamic formatting --- server/docker_swarm.go | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 7c5b79e..fd5a379 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -369,9 +369,6 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { eventFilters := filters.NewArgs( filters.Arg("type", string(events.ServiceEventType)), - filters.Arg("event", string(events.ActionCreate)), - filters.Arg("event", string(events.ActionUpdate)), - filters.Arg("event", string(events.ActionRemove)), filters.Arg("type", DockerRouterEventTypeTask), ) @@ -747,7 +744,6 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * swarmGaveUp := false inRestartDelay := false - var remainingDelay time.Duration if replicas > 0 && !hasRunningTask { if !hasActiveTask && len(tasks) > 0 { @@ -757,7 +753,6 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * timeSinceFailed := time.Since(lastFailedTime) if timeSinceFailed < delay { inRestartDelay = true - remainingDelay = delay - timeSinceFailed data.countdownDeadline = lastFailedTime.Add(delay) } } @@ -767,22 +762,9 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if replicas == 0 || swarmGaveUp || inRestartDelay { data.ip = "" - // Format dynamic countdown or failed message - if inRestartDelay { - durationStr := remainingDelay.Round(time.Second).String() + if inRestartDelay || swarmGaveUp { if data.autoScaleFailedMOTD != "" { - data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleFailedMOTD, "{duration}", durationStr) - } else { - data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleAsleepMOTD, "{duration}", durationStr) - } - if data.autoScaleLoadingMOTD != "" { - data.autoScaleLoadingMOTD = strings.ReplaceAll(data.autoScaleLoadingMOTD, "{duration}", durationStr) - } - } else if swarmGaveUp { - if data.autoScaleFailedMOTD != "" { - data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleFailedMOTD, "{duration}", "failed") - } else { - data.autoScaleAsleepMOTD = strings.ReplaceAll(data.autoScaleAsleepMOTD, "{duration}", "failed") + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD } } } else if isVIP { From a64b9331cd59e8fd83373a4788d582aeaffea15b Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 18:46:57 -0500 Subject: [PATCH 17/38] feat(swarm): separate restart delay motd from permanent failed motd --- README.md | 3 +- server/docker.go | 1 + server/docker_swarm.go | 74 +++++++++++++++++++++++++----------------- server/routes.go | 4 +-- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index eb81807..37d32d4 100644 --- a/README.md +++ b/README.md @@ -199,7 +199,8 @@ These are the labels scanned: - `mc-router.auto-scale-asleep-motd`: Per-container/service override for MOTD to show when scaled to zero. If empty or not set the host will appear unresponsive. - `mc-router.auto-scale-loading-motd`: Per-container/service override for MOTD to show while waking up. Supports replacing the `{duration}` token with the remaining Swarm restart delay if the task is waiting to retry. If empty or not set, the global `-auto-scale-loading-motd` value is used. - `mc-router.auto-scale-wait-timeout`: Configure the maximum duration the router waits for the container or Swarm task to become reachable after scaling up (e.g. `"5m"` or `"300s"`). Defaults to 60s. -- `mc-router.auto-scale-failed-motd`: MOTD to show if the container/service fails to start or Swarm exhausts its restart policy (e.g. `"Server failed to start. Retrying in {duration}."`). Supports the `{duration}` countdown token, which dynamically updates during Swarm restart delays. +- `mc-router.auto-scale-restart-delay-motd`: MOTD to show while the service is in a temporary restart delay (e.g. `"Server failed to start. Retrying in {duration}."`). Supports the `{duration}` countdown token, which dynamically updates. +- `mc-router.auto-scale-failed-motd`: MOTD to show if the container/service fails to start permanently or Swarm exhausts its restart policy (e.g. `"Server crashed and stopped retrying."`). Does not support the countdown token. #### Docker Auto Scale Up/Down diff --git a/server/docker.go b/server/docker.go index 53e5e25..e552344 100644 --- a/server/docker.go +++ b/server/docker.go @@ -32,6 +32,7 @@ const ( DockerRouterLabelAutoScaleLoadingMOTD = "mc-router.auto-scale-loading-motd" DockerRouterLabelAutoScaleWaitTimeout = "mc-router.auto-scale-wait-timeout" DockerRouterLabelAutoScaleFailedMOTD = "mc-router.auto-scale-failed-motd" + DockerRouterLabelAutoScaleRestartDelayMOTD = "mc-router.auto-scale-restart-delay-motd" DockerRouterEventTypeTask = "task" ) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index fd5a379..4a8d216 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -44,8 +44,9 @@ type routableSwarmService struct { autoScaleAsleepMOTD string autoScaleLoadingMOTD string autoScaleWaitTimeout time.Duration - autoScaleFailedMOTD string - countdownDeadline time.Time + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time } type dockerSwarmWatcherImpl struct { @@ -327,6 +328,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { oldRs.autoScaleLoadingMOTD != rs.autoScaleLoadingMOTD || oldRs.autoScaleWaitTimeout != rs.autoScaleWaitTimeout || oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD || + oldRs.autoScaleRestartDelayMOTD != rs.autoScaleRestartDelayMOTD || oldRs.countdownDeadline != rs.countdownDeadline { w.serviceMap[rs.externalServiceName] = rs @@ -472,34 +474,36 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS for _, host := range data.hosts { result = append(result, &routableSwarmService{ - containerEndpoint: endpoint, - externalServiceName: host, - serviceID: data.serviceID, - serviceName: data.serviceName, - networkID: data.networkID, - autoScaleUp: data.autoScaleUp, - autoScaleDown: data.autoScaleDown, - autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, - autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, - autoScaleWaitTimeout: data.autoScaleWaitTimeout, - autoScaleFailedMOTD: data.autoScaleFailedMOTD, - countdownDeadline: data.countdownDeadline, + containerEndpoint: endpoint, + externalServiceName: host, + serviceID: data.serviceID, + serviceName: data.serviceName, + networkID: data.networkID, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, + autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, + countdownDeadline: data.countdownDeadline, }) } if data.def != nil && *data.def { result = append(result, &routableSwarmService{ - containerEndpoint: endpoint, - externalServiceName: "", - serviceID: data.serviceID, - serviceName: data.serviceName, - networkID: data.networkID, - autoScaleUp: data.autoScaleUp, - autoScaleDown: data.autoScaleDown, - autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, - autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, - autoScaleWaitTimeout: data.autoScaleWaitTimeout, - autoScaleFailedMOTD: data.autoScaleFailedMOTD, - countdownDeadline: data.countdownDeadline, + containerEndpoint: endpoint, + externalServiceName: "", + serviceID: data.serviceID, + serviceName: data.serviceName, + networkID: data.networkID, + autoScaleUp: data.autoScaleUp, + autoScaleDown: data.autoScaleDown, + autoScaleAsleepMOTD: data.autoScaleAsleepMOTD, + autoScaleLoadingMOTD: data.autoScaleLoadingMOTD, + autoScaleWaitTimeout: data.autoScaleWaitTimeout, + autoScaleFailedMOTD: data.autoScaleFailedMOTD, + autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, + countdownDeadline: data.countdownDeadline, }) } } @@ -542,9 +546,10 @@ type parsedDockerServiceData struct { autoScaleAsleepMOTD string autoScaleLoadingMOTD string autoScaleWaitTimeout time.Duration - autoScaleFailedMOTD string - countdownDeadline time.Time - isDNSRR bool + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time + isDNSRR bool } func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { @@ -637,6 +642,9 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if key == DockerRouterLabelAutoScaleFailedMOTD { data.autoScaleFailedMOTD = value } + if key == DockerRouterLabelAutoScaleRestartDelayMOTD { + data.autoScaleRestartDelayMOTD = value + } } // probably not minecraft related @@ -762,7 +770,13 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if replicas == 0 || swarmGaveUp || inRestartDelay { data.ip = "" - if inRestartDelay || swarmGaveUp { + if inRestartDelay { + if data.autoScaleRestartDelayMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD + } else if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } + } else if swarmGaveUp { if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD } diff --git a/server/routes.go b/server/routes.go index 80ccf9d..3c57333 100644 --- a/server/routes.go +++ b/server/routes.go @@ -180,7 +180,7 @@ func formatMOTD(motd string, deadline time.Time) string { return motd } if deadline.IsZero() { - return strings.ReplaceAll(motd, "{duration}", "failed") + return strings.ReplaceAll(motd, "{duration}", "never") } now := time.Now() if now.Before(deadline) { @@ -188,7 +188,7 @@ func formatMOTD(motd string, deadline time.Time) string { durationStr := remaining.Round(time.Second).String() return strings.ReplaceAll(motd, "{duration}", durationStr) } - return strings.ReplaceAll(motd, "{duration}", "failed") + return strings.ReplaceAll(motd, "{duration}", "never") } func (r *routesImpl) GetAsleepMOTD(serverAddress string) string { From 9a8a1e9f69fab1d818aab6d8c7409e8b6554a4aa Mon Sep 17 00:00:00 2001 From: Bionix Date: Tue, 7 Jul 2026 19:09:58 -0500 Subject: [PATCH 18/38] fix(routes): enforce case insensitivity in all mapping lookups and deletions --- server/routes.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/routes.go b/server/routes.go index 3c57333..ed4af65 100644 --- a/server/routes.go +++ b/server/routes.go @@ -199,6 +199,7 @@ func (r *routesImpl) GetAsleepMOTD(serverAddress string) string { return formatMOTD(r.defaultRoute.asleepMOTD, r.defaultRoute.countdownDeadline) } + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { return formatMOTD(m.asleepMOTD, m.countdownDeadline) } @@ -213,6 +214,7 @@ func (r *routesImpl) GetLoadingMOTD(serverAddress string) string { return formatMOTD(r.defaultRoute.loadingMOTD, r.defaultRoute.countdownDeadline) } + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { return formatMOTD(m.loadingMOTD, m.countdownDeadline) } @@ -228,6 +230,7 @@ func (r *routesImpl) SetCountdownDeadline(serverAddress string, deadline time.Ti return } + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { m.countdownDeadline = deadline r.mappings[serverAddress] = m @@ -242,6 +245,7 @@ func (r *routesImpl) HasRoute(serverAddress string) bool { r.RLock() defer r.RUnlock() + serverAddress = strings.ToLower(serverAddress) _, exists := r.mappings[serverAddress] return exists } @@ -323,6 +327,7 @@ func (r *routesImpl) DeleteMapping(serverAddress string) bool { defer r.Unlock() logrus.WithField("serverAddress", serverAddress).Info("Deleting route") + serverAddress = strings.ToLower(serverAddress) if m, ok := r.mappings[serverAddress]; ok { r.downScaler.Cancel(m.scalingTarget) delete(r.mappings, serverAddress) From 1f296797719e0a4bc611b62ebec61452a8a65de1 Mon Sep 17 00:00:00 2001 From: Bionix Date: Wed, 8 Jul 2026 11:42:02 -0500 Subject: [PATCH 19/38] fix(swarm): simplify state machine using undocumented task StateReady --- server/docker_swarm.go | 187 ++++++++++++++++++++++------------------- server/routes.go | 4 +- 2 files changed, 103 insertions(+), 88 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 4a8d216..4d2955d 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -113,8 +113,8 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu }) if err == nil && len(tasks) > 0 { var hasActiveTask bool - var terminalFailureCount int - var lastFailedTime time.Time + var hasReadyTask bool + var readyTaskTimestamp time.Time for _, task := range tasks { state := task.Status.State @@ -133,6 +133,16 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu } } + // Swarm task state 'ready' is undocumented but marks a task held during a restart delay. + // Find the latest ready task's status timestamp to measure the restart delay start. + if state == swarm.TaskStateReady { + hasReadyTask = true + if task.Status.Timestamp.After(readyTaskTimestamp) { + readyTaskTimestamp = task.Status.Timestamp + } + } + + // Track active task states to see if Swarm is actively attempting to schedule/start a task. if state == swarm.TaskStateNew || state == swarm.TaskStatePending || state == swarm.TaskStateAssigned || @@ -142,11 +152,6 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu state == swarm.TaskStateStarting || state == swarm.TaskStateRunning { hasActiveTask = true - } else if state == swarm.TaskStateFailed || state == swarm.TaskStateShutdown || state == swarm.TaskStateRejected { - terminalFailureCount++ - if task.Status.Timestamp.After(lastFailedTime) { - lastFailedTime = task.Status.Timestamp - } } } @@ -158,24 +163,23 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu swarmGaveUp := false var remainingDelay time.Duration - if !hasActiveTask && len(tasks) > 0 { - swarmGaveUp = true - } else if hasActiveTask && taskIP == "" { - if delay > 0 && !lastFailedTime.IsZero() { - timeSinceFailed := time.Since(lastFailedTime) - if timeSinceFailed < delay { - remainingDelay = delay - timeSinceFailed - newDeadline := lastFailedTime.Add(delay).Add(waitTimeout) - if newDeadline.After(deadline) { - deadline = newDeadline - logrus.WithFields(logrus.Fields{ - "service": serviceID, - "remaining": remainingDelay, - "extendedWait": time.Until(deadline), - }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") - } - } + if hasReadyTask && delay > 0 && !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { + // Waker is waiting for a restart delay to expire. Dynamically extend the deadline + // so we do not timeout the connection while Swarm holds the start attempt. + timeSinceReady := time.Since(readyTaskTimestamp) + remainingDelay = delay - timeSinceReady + newDeadline := readyTaskTimestamp.Add(delay).Add(waitTimeout) + if newDeadline.After(deadline) { + deadline = newDeadline + logrus.WithFields(logrus.Fields{ + "service": serviceID, + "remaining": remainingDelay, + "extendedWait": time.Until(deadline), + }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") } + } else if !hasActiveTask && len(tasks) > 0 { + // Mentality: If all tasks are completed/failed and there are no active tasks being scheduled, Swarm gave up. + swarmGaveUp = true } if swarmGaveUp { @@ -707,9 +711,9 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } var hasRunningTask bool + var hasReadyTask bool + var readyTaskTimestamp time.Time var hasActiveTask bool - var terminalFailureCount int - var lastFailedTime time.Time var delay time.Duration var tasks []swarm.Task @@ -731,6 +735,16 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * hasRunningTask = true } + // Swarm task state 'ready' is undocumented but marks a task held during a restart delay. + // Find the latest ready task's status timestamp to measure the restart delay start. + if state == swarm.TaskStateReady { + hasReadyTask = true + if task.Status.Timestamp.After(readyTaskTimestamp) { + readyTaskTimestamp = task.Status.Timestamp + } + } + + // Track active task states to see if Swarm is actively attempting to schedule/start a task. if state == swarm.TaskStateNew || state == swarm.TaskStatePending || state == swarm.TaskStateAssigned || @@ -740,86 +754,87 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * state == swarm.TaskStateStarting || state == swarm.TaskStateRunning { hasActiveTask = true - } else if state == swarm.TaskStateFailed || state == swarm.TaskStateShutdown || state == swarm.TaskStateRejected { - terminalFailureCount++ - if task.Status.Timestamp.After(lastFailedTime) { - lastFailedTime = task.Status.Timestamp - } } } } } - swarmGaveUp := false - inRestartDelay := false - - if replicas > 0 && !hasRunningTask { - if !hasActiveTask && len(tasks) > 0 { - swarmGaveUp = true - } else if hasActiveTask { - if delay > 0 && !lastFailedTime.IsZero() { - timeSinceFailed := time.Since(lastFailedTime) - if timeSinceFailed < delay { - inRestartDelay = true - data.countdownDeadline = lastFailedTime.Add(delay) + // State machine classification: + // To prevent premature VIP/DNS resolution routing client connections away from the waker, + // we keep data.ip = "" in all non-running states. This forces client connections to use + // the waker, keeps the loading MOTD active, and avoids dialing starting/unreachable containers. + + if replicas == 0 { + // 1. Sleeping State: Service is scaled to zero. + data.ip = "" + // data.autoScaleAsleepMOTD is already populated by parsing labels. + data.countdownDeadline = time.Time{} + } else if hasRunningTask { + // 2. Running (Healthy) State: Service has a fully running task. + if isVIP { + vipIndex := -1 + if data.networkID != "" { + for i, vip := range service.Endpoint.VirtualIPs { + if vip.NetworkID == data.networkID { + vipIndex = i + break + } } } + if vipIndex == -1 { + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + } else { + vipIndex = 0 + } + } + if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + data.networkID = virtualIP.NetworkID + } else { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, unable to find match in VirtualIPs") + return + } + } else if isDNSRR { + data.ip = service.Spec.Name } - } - - if replicas == 0 || swarmGaveUp || inRestartDelay { + } else { + // Non-running states: keep data.ip empty to ensure the waker captures client connections data.ip = "" - if inRestartDelay { + // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. + if hasReadyTask && delay > 0 && !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { if data.autoScaleRestartDelayMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD } else if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD } - } else if swarmGaveUp { + data.countdownDeadline = readyTaskTimestamp.Add(delay) + } else if !hasActiveTask && len(tasks) > 0 { + // 4. Permanently Failed State: Swarm has terminated all tasks in history and gave up (no active task scheduled). if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD } - } - } else if isVIP { - vipIndex := -1 - if data.networkID != "" { - for i, vip := range service.Endpoint.VirtualIPs { - if vip.NetworkID == data.networkID { - vipIndex = i - break - } - } - } - if vipIndex == -1 { - if data.network != nil { - for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { - vipIndex = i - break - } else if err != nil { - // we intentionally ignore name check errors - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) - } - } - } else { - // if network isn't specified assume it's the first one - vipIndex = 0 - } - } - if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() - data.networkID = virtualIP.NetworkID + data.countdownDeadline = time.Time{} } else { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, unable to find match in VirtualIPs") - return + // 5. Starting / Waking State: Swarm has scaled the service up and the task is starting up (transited past ready), + // or it is the very first start. We display the loading MOTD on pings. + if data.autoScaleLoadingMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleLoadingMOTD + } + data.countdownDeadline = time.Time{} } - } else if isDNSRR { - data.ip = service.Spec.Name } ok = true diff --git a/server/routes.go b/server/routes.go index ed4af65..07d4734 100644 --- a/server/routes.go +++ b/server/routes.go @@ -180,7 +180,7 @@ func formatMOTD(motd string, deadline time.Time) string { return motd } if deadline.IsZero() { - return strings.ReplaceAll(motd, "{duration}", "never") + return strings.ReplaceAll(motd, "{duration}", "now") } now := time.Now() if now.Before(deadline) { @@ -188,7 +188,7 @@ func formatMOTD(motd string, deadline time.Time) string { durationStr := remaining.Round(time.Second).String() return strings.ReplaceAll(motd, "{duration}", durationStr) } - return strings.ReplaceAll(motd, "{duration}", "never") + return strings.ReplaceAll(motd, "{duration}", "now") } func (r *routesImpl) GetAsleepMOTD(serverAddress string) string { From 76075438d4fbe95347b6cd1312dc63ddcfa19c93 Mon Sep 17 00:00:00 2001 From: Bionix Date: Wed, 8 Jul 2026 12:32:50 -0500 Subject: [PATCH 20/38] docs: document Docker Swarm task states mapping logic for autoscaling --- docs/docker-swarm-states.md | 46 +++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/docker-swarm-states.md diff --git a/docs/docker-swarm-states.md b/docs/docker-swarm-states.md new file mode 100644 index 0000000..553817d --- /dev/null +++ b/docs/docker-swarm-states.md @@ -0,0 +1,46 @@ +# Docker Swarm Task States for Autoscaling + +When building plugins or routers (like `mc-router`) that autoscale and monitor Docker Swarm services, interpreting task scheduling states is historically challenging due to sparse Docker documentation. + +This document defines how to map Swarm's **Actual Task Status State** (`task.Status.State`) and **Desired Task State** (`task.DesiredState`) to logical route status classifications. + +--- + +## Task State Matrix + +| State | Status State (`task.Status.State`) | Desired State (`task.DesiredState`) | Description | +| :--- | :--- | :--- | :--- | +| **Running** | `running` | `running` | The container is alive, healthy, and fully running. Connections can be routed directly. | +| **Restart Delay** | `ready` | `ready` | The service task crashed and Swarm is holding the next task retry until the restart delay timer expires. | +| **Starting / Waking** | `new`, `pending`, `assigned`, `preparing`, `ready`, `starting` | `running` | Swarm has scaled the replicas to `> 0` and is actively preparing or starting the container. | +| **Stopping** | `running` | `shutdown` or `remove` | Swarm has commanded the task to shut down or be removed, but the container is still cleaning up. | +| **Permanently Failed** | `failed`, `shutdown`, `rejected` | `shutdown` | Swarm has commanded a shutdown on the last failed container and is no longer attempting to schedule a new task. | + +--- + +## State Classifications & Logic + +### 1. Sleeping State +* **Condition**: `replicas == 0` +* **Interpretation**: The service is explicitly scaled down to zero replicas. +* **Router Action**: Keep the backend route empty (`""`) and display the configured asleep MOTD to prompt a wake-up on join. + +### 2. Running (Healthy) State +* **Condition**: At least one task has `Status.State == running` and `DesiredState == running`. +* **Interpretation**: The server is fully online and ready for traffic. +* **Router Action**: Resolve the route to the service's Virtual IP (VIP) or DNS Round Robin (DNSRR) name. + +### 3. Restart Delay State +* **Condition**: `replicas > 0` AND at least one task has `Status.State == ready` and `DesiredState == ready`. +* **Interpretation**: Swarm has scheduled a retry but is waiting for the configured restart policy delay to expire before spawning the container. +* **Router Action**: Keep the backend route empty (`""`) to route connection attempts to the waker, and display the restart delay countdown MOTD. + +### 4. Permanently Failed State +* **Condition**: `replicas > 0` AND the most recently updated task in history has `DesiredState == shutdown` AND no active tasks exist. +* **Interpretation**: Swarm has exhausted its restart attempts and stopped trying. +* **Router Action**: Keep the backend route empty (`""`) and display the custom permanently failed MOTD. + +### 5. Starting / Waking State +* **Condition**: `replicas > 0` AND the desired state of the task is `running` but the actual status is not yet `running` (and not delayed or failed). +* **Interpretation**: The container is actively being assigned, prepared, or started by Swarm. +* **Router Action**: Keep the backend route empty (`""`) to preserve the waker, and display the loading/waking MOTD. From ac8671a5729957b3783891ac45e44d2c062d7001 Mon Sep 17 00:00:00 2001 From: Bionix Date: Wed, 8 Jul 2026 12:51:33 -0500 Subject: [PATCH 21/38] fix(swarm): simplify state machine using actual and desired task states - Refactor Swarm task parsing to inspect both task.Status.State and task.DesiredState. - Treat a task as fully running only if both actual and desired states are "running". - Trust the "ready" state (actual or desired) directly to identify tasks in a restart delay. - Track the most recently created task to detect when Swarm has given up (latestTask.DesiredState == "shutdown"). - Add a default fallback permanently failed MOTD ("Server failed to start.") when no custom MOTD is configured. --- server/docker_swarm.go | 79 +++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 4d2955d..264b581 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -115,10 +115,19 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu var hasActiveTask bool var hasReadyTask bool var readyTaskTimestamp time.Time + var latestTask swarm.Task for _, task := range tasks { state := task.Status.State - if state == swarm.TaskStateRunning { + desiredState := task.DesiredState + + // Track the most recently created task to inspect its DesiredState. + if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { + latestTask = task + } + + // The task is considered fully Running only if both actual and desired states are running. + if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { for _, attachment := range task.NetworksAttachments { matchesNetwork := rs.networkID != "" && attachment.Network.ID == rs.networkID isIngress := attachment.Network.Spec.Name == "ingress" @@ -133,9 +142,9 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu } } - // Swarm task state 'ready' is undocumented but marks a task held during a restart delay. + // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. // Find the latest ready task's status timestamp to measure the restart delay start. - if state == swarm.TaskStateReady { + if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { hasReadyTask = true if task.Status.Timestamp.After(readyTaskTimestamp) { readyTaskTimestamp = task.Status.Timestamp @@ -150,7 +159,9 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu state == swarm.TaskStatePreparing || state == swarm.TaskStateReady || state == swarm.TaskStateStarting || - state == swarm.TaskStateRunning { + state == swarm.TaskStateRunning || + desiredState == swarm.TaskStateReady || + desiredState == swarm.TaskStateRunning { hasActiveTask = true } } @@ -163,22 +174,25 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu swarmGaveUp := false var remainingDelay time.Duration - if hasReadyTask && delay > 0 && !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { + if hasReadyTask && delay > 0 { // Waker is waiting for a restart delay to expire. Dynamically extend the deadline // so we do not timeout the connection while Swarm holds the start attempt. - timeSinceReady := time.Since(readyTaskTimestamp) - remainingDelay = delay - timeSinceReady - newDeadline := readyTaskTimestamp.Add(delay).Add(waitTimeout) - if newDeadline.After(deadline) { - deadline = newDeadline - logrus.WithFields(logrus.Fields{ - "service": serviceID, - "remaining": remainingDelay, - "extendedWait": time.Until(deadline), - }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") + if !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { + timeSinceReady := time.Since(readyTaskTimestamp) + remainingDelay = delay - timeSinceReady + newDeadline := readyTaskTimestamp.Add(delay).Add(waitTimeout) + if newDeadline.After(deadline) { + deadline = newDeadline + logrus.WithFields(logrus.Fields{ + "service": serviceID, + "remaining": remainingDelay, + "extendedWait": time.Until(deadline), + }).Info("Swarm task is in restart delay. Dynamically extending waker deadline.") + } } - } else if !hasActiveTask && len(tasks) > 0 { - // Mentality: If all tasks are completed/failed and there are no active tasks being scheduled, Swarm gave up. + } else if !hasActiveTask && latestTask.DesiredState == swarm.TaskStateShutdown && len(tasks) > 0 { + // Mentality: If the latest task has DesiredState == Shutdown and there are no active tasks, + // Swarm has given up retrying. swarmGaveUp = true } @@ -714,6 +728,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * var hasReadyTask bool var readyTaskTimestamp time.Time var hasActiveTask bool + var latestTask swarm.Task var delay time.Duration var tasks []swarm.Task @@ -731,13 +746,22 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * for _, task := range tasks { state := task.Status.State - if state == swarm.TaskStateRunning { + desiredState := task.DesiredState + + // Track the most recently created task to inspect its DesiredState. + if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { + latestTask = task + } + + // The task is considered fully Running only if both actual and desired states are running. + // If DesiredState is Shutdown or Remove, it is stopping. + if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { hasRunningTask = true } - // Swarm task state 'ready' is undocumented but marks a task held during a restart delay. + // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. // Find the latest ready task's status timestamp to measure the restart delay start. - if state == swarm.TaskStateReady { + if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { hasReadyTask = true if task.Status.Timestamp.After(readyTaskTimestamp) { readyTaskTimestamp = task.Status.Timestamp @@ -752,7 +776,9 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * state == swarm.TaskStatePreparing || state == swarm.TaskStateReady || state == swarm.TaskStateStarting || - state == swarm.TaskStateRunning { + state == swarm.TaskStateRunning || + desiredState == swarm.TaskStateReady || + desiredState == swarm.TaskStateRunning { hasActiveTask = true } } @@ -814,17 +840,22 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * data.ip = "" // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. - if hasReadyTask && delay > 0 && !readyTaskTimestamp.IsZero() && time.Since(readyTaskTimestamp) < delay { + if hasReadyTask && delay > 0 { if data.autoScaleRestartDelayMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD } else if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." } data.countdownDeadline = readyTaskTimestamp.Add(delay) - } else if !hasActiveTask && len(tasks) > 0 { - // 4. Permanently Failed State: Swarm has terminated all tasks in history and gave up (no active task scheduled). + } else if !hasActiveTask && latestTask.DesiredState == swarm.TaskStateShutdown && len(tasks) > 0 { + // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task + // and is no longer attempting to restart the service. if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." } data.countdownDeadline = time.Time{} } else { From 1f57e333610706f9651ee745978acdc7f7aecf49 Mon Sep 17 00:00:00 2001 From: Bionix Date: Wed, 8 Jul 2026 20:40:14 -0500 Subject: [PATCH 22/38] fix(swarm): resolve scale-up race conditions and VIP routing propagation latency --- server/docker_swarm.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 264b581..34a9170 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -89,6 +89,7 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu waitTimeout = 60 * time.Second } + var filterTime time.Time replicas := service.Spec.Mode.Replicated.Replicas if replicas == nil || *replicas == 0 { logrus.WithFields(logrus.Fields{ @@ -98,10 +99,13 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu one := uint64(1) service.Spec.Mode.Replicated.Replicas = &one + filterTime = time.Now() _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) if err != nil { return "", err } + } else { + filterTime = service.Meta.UpdatedAt } // Wait until a task is running and has an IP address @@ -118,6 +122,11 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu var latestTask swarm.Task for _, task := range tasks { + // Ignore tasks created before the current scale-up/deployment cycle to avoid stale history. + if !filterTime.IsZero() && task.CreatedAt.Before(filterTime) { + continue + } + state := task.Status.State desiredState := task.DesiredState @@ -725,6 +734,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } var hasRunningTask bool + var runningTaskIP string var hasReadyTask bool var readyTaskTimestamp time.Time var hasActiveTask bool @@ -748,6 +758,16 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * state := task.Status.State desiredState := task.DesiredState + isTerminal := state == swarm.TaskStateFailed || + state == swarm.TaskStateShutdown || + state == swarm.TaskStateRejected || + state == swarm.TaskStateComplete + + // Ignore terminal tasks from previous service deployments/runs + if isTerminal && !service.Meta.UpdatedAt.IsZero() && task.CreatedAt.Before(service.Meta.UpdatedAt) { + continue + } + // Track the most recently created task to inspect its DesiredState. if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { latestTask = task @@ -757,6 +777,19 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * // If DesiredState is Shutdown or Remove, it is stopping. if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { hasRunningTask = true + // Extract direct task IP to bypass VIP propagation lag when replicas == 1 + for _, attachment := range task.NetworksAttachments { + matchesNetwork := data.networkID != "" && attachment.Network.ID == data.networkID + isIngress := attachment.Network.Spec.Name == "ingress" + + if (matchesNetwork || (data.networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { + parts := strings.Split(attachment.Addresses[0], "/") + if ip := net.ParseIP(parts[0]); ip != nil { + runningTaskIP = parts[0] + break + } + } + } } // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. @@ -797,7 +830,9 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * data.countdownDeadline = time.Time{} } else if hasRunningTask { // 2. Running (Healthy) State: Service has a fully running task. - if isVIP { + if replicas == 1 && runningTaskIP != "" { + data.ip = runningTaskIP + } else if isVIP { vipIndex := -1 if data.networkID != "" { for i, vip := range service.Endpoint.VirtualIPs { From c631f9d0e0f36d2cf99016abfb9e208c62ef9f7d Mon Sep 17 00:00:00 2001 From: Bionix Date: Thu, 9 Jul 2026 10:13:25 -0500 Subject: [PATCH 23/38] fix(swarm): document reconciliation block and add settling delay to task events --- server/docker_swarm.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 34a9170..0441851 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -335,6 +335,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { visited := map[string]struct{}{} for _, rs := range services { + // If this is a newly discovered service, set up wakers/sleepers and create the route mapping. if oldRs, ok := w.serviceMap[rs.externalServiceName]; !ok { w.serviceMap[rs.externalServiceName] = rs logrus.WithField("routableService", rs).Debug("ADD") @@ -346,6 +347,8 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) + // If the service is already tracked, check if any metadata, endpoint, MOTDs, or deadline + // changed. If so, recreate wakers/sleepers and update the route table. } else if oldRs.containerEndpoint != rs.containerEndpoint || oldRs.serviceID != rs.serviceID || oldRs.networkID != rs.networkID || @@ -419,6 +422,11 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { break loop } logrus.WithFields(logrus.Fields{"type": ev.Type, "action": ev.Action, "id": ev.Actor.ID}).Trace("Docker Swarm event") + // Swarm task state updates can have a slight API database propagation lag. + // Introduce a small settling delay to ensure task queries reflect the updated state. + if ev.Type == DockerRouterEventTypeTask { + time.Sleep(200 * time.Millisecond) + } if err := w.reconcileServices(ctx); err != nil { logrus.WithError(err).Error("Docker Swarm reconciliation failed") } From d8055581acef38c467a8a9f689e26dd55b764032 Mon Sep 17 00:00:00 2001 From: Bionix Date: Thu, 9 Jul 2026 11:51:58 -0500 Subject: [PATCH 24/38] fix(swarm): replace task events with 5-second background polling ticker --- server/docker_swarm.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 0441851..85d8b8a 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -401,7 +401,6 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { eventFilters := filters.NewArgs( filters.Arg("type", string(events.ServiceEventType)), - filters.Arg("type", DockerRouterEventTypeTask), ) eventCh, errCh := w.client.Events(ctx, events.ListOptions{Filters: eventFilters}) @@ -412,21 +411,23 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { backoff = time.Second } + ticker := time.NewTicker(5 * time.Second) + loop: for { select { case <-ctx.Done(): + ticker.Stop() return + case <-ticker.C: + if err := w.reconcileServices(ctx); err != nil { + logrus.WithError(err).Error("Docker Swarm reconciliation failed") + } case ev, ok := <-eventCh: if !ok { break loop } logrus.WithFields(logrus.Fields{"type": ev.Type, "action": ev.Action, "id": ev.Actor.ID}).Trace("Docker Swarm event") - // Swarm task state updates can have a slight API database propagation lag. - // Introduce a small settling delay to ensure task queries reflect the updated state. - if ev.Type == DockerRouterEventTypeTask { - time.Sleep(200 * time.Millisecond) - } if err := w.reconcileServices(ctx); err != nil { logrus.WithError(err).Error("Docker Swarm reconciliation failed") } @@ -435,12 +436,14 @@ func (w *dockerSwarmWatcherImpl) streamEvents(ctx context.Context) { break loop } if ctx.Err() != nil { + ticker.Stop() return } logrus.WithError(err).Warn("Docker Swarm event stream error, reconnecting") break loop } } + ticker.Stop() select { case <-ctx.Done(): From 690c462c6dc80f92d2c533e8a31f2a696a9c1714 Mon Sep 17 00:00:00 2001 From: Bionix Date: Thu, 9 Jul 2026 12:24:35 -0500 Subject: [PATCH 25/38] docs: update README to clarify support for native Swarm scaling and VIP bypass behavior --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 37d32d4..ea3ac19 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,8 @@ Behavior: - While that wake-up is in progress and status pings are received, mc-router can return a loading MOTD (per-container override or `-auto-scale-loading-motd`). - When no clients remain connected and the idle timer elapses (`-auto-scale-down-after`), mc-router gracefully stops the container. -Note: Docker Swarm deployments can use auto scaling via the [Webhook Auto Scale](#webhook-auto-scale) integration. Native Swarm service scaling via `-auto-scale-up`/`-auto-scale-down` is not supported. +> [!NOTE] +> Native Swarm service scaling via `-auto-scale-up`/`-auto-scale-down` is supported in both VIP (Virtual IP) and DNSRR (DNS Round-Robin) modes. Bypassing the VIP (by routing directly to the task container IP) is automatically enabled when `replicas == 1` to prevent VIP routing delay. Note that DNSRR mode has not been actively tested. #### Example Docker deployment From 38e5d68bf266faa9f43addb4fdec4a10387834f5 Mon Sep 17 00:00:00 2001 From: Bionix Date: Fri, 10 Jul 2026 21:45:21 -0500 Subject: [PATCH 26/38] fix(swarm): remove fragile timestamp filters to make waker and watcher stateless --- server/docker_swarm.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 85d8b8a..a3b0f55 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -89,7 +89,6 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu waitTimeout = 60 * time.Second } - var filterTime time.Time replicas := service.Spec.Mode.Replicated.Replicas if replicas == nil || *replicas == 0 { logrus.WithFields(logrus.Fields{ @@ -99,13 +98,10 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu one := uint64(1) service.Spec.Mode.Replicated.Replicas = &one - filterTime = time.Now() _, err = w.client.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, dockertypes.ServiceUpdateOptions{}) if err != nil { return "", err } - } else { - filterTime = service.Meta.UpdatedAt } // Wait until a task is running and has an IP address @@ -122,11 +118,6 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu var latestTask swarm.Task for _, task := range tasks { - // Ignore tasks created before the current scale-up/deployment cycle to avoid stale history. - if !filterTime.IsZero() && task.CreatedAt.Before(filterTime) { - continue - } - state := task.Status.State desiredState := task.DesiredState @@ -769,16 +760,6 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * state := task.Status.State desiredState := task.DesiredState - isTerminal := state == swarm.TaskStateFailed || - state == swarm.TaskStateShutdown || - state == swarm.TaskStateRejected || - state == swarm.TaskStateComplete - - // Ignore terminal tasks from previous service deployments/runs - if isTerminal && !service.Meta.UpdatedAt.IsZero() && task.CreatedAt.Before(service.Meta.UpdatedAt) { - continue - } - // Track the most recently created task to inspect its DesiredState. if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { latestTask = task From c9dc320d0b02ee2243e64bb1810761bdbd9cf33d Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 11 Jul 2026 12:47:51 -0500 Subject: [PATCH 27/38] fix(swarm): add state transition logging to reconcileServices --- server/docker_swarm.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index a3b0f55..c3e4bba 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -47,6 +47,7 @@ type routableSwarmService struct { autoScaleFailedMOTD string autoScaleRestartDelayMOTD string countdownDeadline time.Time + statusState string } type dockerSwarmWatcherImpl struct { @@ -329,7 +330,15 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { // If this is a newly discovered service, set up wakers/sleepers and create the route mapping. if oldRs, ok := w.serviceMap[rs.externalServiceName]; !ok { w.serviceMap[rs.externalServiceName] = rs - logrus.WithField("routableService", rs).Debug("ADD") + ipDetail := "" + if rs.statusState == "running" && rs.containerEndpoint != "" { + ipDetail = fmt.Sprintf(" (Endpoint: %s)", rs.containerEndpoint) + } + logrus.WithFields(logrus.Fields{ + "service": rs.serviceName, + "hosts": rs.externalServiceName, + }).Infof("Swarm service state: %s%s", rs.statusState, ipDetail) + wakerFunc := w.makeWakerFunc(rs) sleeperFunc := w.makeSleeperFunc(rs) if rs.externalServiceName != "" { @@ -350,7 +359,19 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { oldRs.autoScaleWaitTimeout != rs.autoScaleWaitTimeout || oldRs.autoScaleFailedMOTD != rs.autoScaleFailedMOTD || oldRs.autoScaleRestartDelayMOTD != rs.autoScaleRestartDelayMOTD || - oldRs.countdownDeadline != rs.countdownDeadline { + oldRs.countdownDeadline != rs.countdownDeadline || + oldRs.statusState != rs.statusState { + + if oldRs.statusState != rs.statusState { + ipDetail := "" + if rs.statusState == "running" && rs.containerEndpoint != "" { + ipDetail = fmt.Sprintf(" (Endpoint: %s)", rs.containerEndpoint) + } + logrus.WithFields(logrus.Fields{ + "service": rs.serviceName, + "hosts": rs.externalServiceName, + }).Infof("Swarm service state transition: %s -> %s%s", oldRs.statusState, rs.statusState, ipDetail) + } w.serviceMap[rs.externalServiceName] = rs wakerFunc := w.makeWakerFunc(rs) @@ -516,6 +537,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleFailedMOTD: data.autoScaleFailedMOTD, autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, countdownDeadline: data.countdownDeadline, + statusState: data.statusState, }) } if data.def != nil && *data.def { @@ -533,6 +555,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS autoScaleFailedMOTD: data.autoScaleFailedMOTD, autoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD, countdownDeadline: data.countdownDeadline, + statusState: data.statusState, }) } } @@ -579,6 +602,7 @@ type parsedDockerServiceData struct { autoScaleRestartDelayMOTD string countdownDeadline time.Time isDNSRR bool + statusState string } func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { @@ -817,11 +841,13 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * if replicas == 0 { // 1. Sleeping State: Service is scaled to zero. + data.statusState = "sleeping" data.ip = "" // data.autoScaleAsleepMOTD is already populated by parsing labels. data.countdownDeadline = time.Time{} } else if hasRunningTask { // 2. Running (Healthy) State: Service has a fully running task. + data.statusState = "running" if replicas == 1 && runningTaskIP != "" { data.ip = runningTaskIP } else if isVIP { @@ -868,6 +894,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. if hasReadyTask && delay > 0 { + data.statusState = "restart_delay" if data.autoScaleRestartDelayMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD } else if data.autoScaleFailedMOTD != "" { @@ -879,6 +906,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } else if !hasActiveTask && latestTask.DesiredState == swarm.TaskStateShutdown && len(tasks) > 0 { // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task // and is no longer attempting to restart the service. + data.statusState = "failed" if data.autoScaleFailedMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD } else { @@ -888,6 +916,7 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } else { // 5. Starting / Waking State: Swarm has scaled the service up and the task is starting up (transited past ready), // or it is the very first start. We display the loading MOTD on pings. + data.statusState = "waking" if data.autoScaleLoadingMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleLoadingMOTD } From 7a2f9ec0e4bb554dfd34e587c20f46d641ec116f Mon Sep 17 00:00:00 2001 From: Bionix Date: Sun, 12 Jul 2026 15:03:45 -0500 Subject: [PATCH 28/38] docs(swarm): document wait timeout client limit and api polling details --- README.md | 4 ++-- docs/docker-swarm-states.md | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ea3ac19..018a36f 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ These are the labels scanned: - `mc-router.auto-scale-down`: Per-container/service override to enable/disable auto scale down for Docker/Swarm. When true (or left unspecified and the global `-auto-scale-down` flag is enabled), mc-router will stop the container or scale down the Swarm service to 0 after it has been idle. - `mc-router.auto-scale-asleep-motd`: Per-container/service override for MOTD to show when scaled to zero. If empty or not set the host will appear unresponsive. - `mc-router.auto-scale-loading-motd`: Per-container/service override for MOTD to show while waking up. Supports replacing the `{duration}` token with the remaining Swarm restart delay if the task is waiting to retry. If empty or not set, the global `-auto-scale-loading-motd` value is used. -- `mc-router.auto-scale-wait-timeout`: Configure the maximum duration the router waits for the container or Swarm task to become reachable after scaling up (e.g. `"5m"` or `"300s"`). Defaults to 60s. +- `mc-router.auto-scale-wait-timeout`: Configure the maximum duration the router waits for the container or Swarm task to become reachable after scaling up (e.g. `"5m"` or `"300s"`). Defaults to 60s. Note: Since the Minecraft Java client has a strict connection timeout of 30 seconds, configuring this value above 30s is not recommended for player join connections. - `mc-router.auto-scale-restart-delay-motd`: MOTD to show while the service is in a temporary restart delay (e.g. `"Server failed to start. Retrying in {duration}."`). Supports the `{duration}` countdown token, which dynamically updates. - `mc-router.auto-scale-failed-motd`: MOTD to show if the container/service fails to start permanently or Swarm exhausts its restart policy (e.g. `"Server crashed and stopped retrying."`). Does not support the countdown token. @@ -542,7 +542,7 @@ To override the MOTD shown when the server is scaled down or scaling up, you can - `mc-router.itzg.me/autoScaleLoadingMOTD` You can also customize how long the router will wait for a scaling backend to become reachable (default: 60s): -- `mc-router.itzg.me/autoScaleWaitTimeout` (e.g. `2m`, `30s`) +- `mc-router.itzg.me/autoScaleWaitTimeout` (e.g. `2m`, `30s`). Note: Since the Minecraft Java client has a strict connection timeout of 30 seconds, configuring this value above 30s is not recommended for player join connections. Example server with custom MOTD and timeout: ```yaml diff --git a/docs/docker-swarm-states.md b/docs/docker-swarm-states.md index 553817d..8a73aec 100644 --- a/docs/docker-swarm-states.md +++ b/docs/docker-swarm-states.md @@ -44,3 +44,12 @@ This document defines how to map Swarm's **Actual Task Status State** (`task.Sta * **Condition**: `replicas > 0` AND the desired state of the task is `running` but the actual status is not yet `running` (and not delayed or failed). * **Interpretation**: The container is actively being assigned, prepared, or started by Swarm. * **Router Action**: Keep the backend route empty (`""`) to preserve the waker, and display the loading/waking MOTD. + +--- + +## Polling Implementation Details + +> [!NOTE] +> **Why Polling is Used Instead of API Events:** +> Although `mc-router` subscribes to the Docker Event Stream, the Docker Engine API does **not** broadcast task-level scheduling transitions (such as `task.Status.State` moving from `preparing` to `ready` or `running`) over the event system. +> To reliably capture these low-level task state changes and resolve the direct task IP without transient routing failures, `mc-router` utilizes a background polling ticker that queries the Swarm Service and Task API every 5 seconds. From b0ef162319d42009a9137384ca3722615590a90d Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 15:10:13 -0500 Subject: [PATCH 29/38] refractor: convert to switch statement for label parsing --- server/docker_swarm.go | 43 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index c3e4bba..e755c7c 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -613,15 +613,16 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * data.serviceName = service.Spec.Name for key, value := range service.Spec.Labels { - if key == DockerRouterLabelHost { + switch key { + case DockerRouterLabelHost: if data.hosts != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) return } data.hosts = SplitExternalHosts(value) - } - if key == DockerRouterLabelPort { + + case DockerRouterLabelPort: if data.port != 0 { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) @@ -635,8 +636,8 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * Warnf("ignoring service with invalid %s", DockerRouterLabelPort) return } - } - if key == DockerRouterLabelDefault { + + case DockerRouterLabelDefault: if data.def != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) @@ -646,8 +647,8 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * lowerValue := strings.TrimSpace(strings.ToLower(value)) *data.def = lowerValue != "" && lowerValue != "0" && lowerValue != "false" && lowerValue != "no" - } - if key == DockerRouterLabelNetwork { + + case DockerRouterLabelNetwork: if data.network != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) @@ -655,8 +656,8 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * } data.network = new(string) *data.network = value - } - if key == DockerRouterLabelAutoScaleUp { + + case DockerRouterLabelAutoScaleUp: autoScaleUp, err := strconv.ParseBool(strings.TrimSpace(value)) if err != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). @@ -665,8 +666,8 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * return } data.autoScaleUp = autoScaleUp - } - if key == DockerRouterLabelAutoScaleDown { + + case DockerRouterLabelAutoScaleDown: autoScaleDown, err := strconv.ParseBool(strings.TrimSpace(value)) if err != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). @@ -675,14 +676,14 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * return } data.autoScaleDown = autoScaleDown - } - if key == DockerRouterLabelAutoScaleAsleepMOTD { + + case DockerRouterLabelAutoScaleAsleepMOTD: data.autoScaleAsleepMOTD = value - } - if key == DockerRouterLabelAutoScaleLoadingMOTD { + + case DockerRouterLabelAutoScaleLoadingMOTD: data.autoScaleLoadingMOTD = value - } - if key == DockerRouterLabelAutoScaleWaitTimeout { + + case DockerRouterLabelAutoScaleWaitTimeout: dur, err := time.ParseDuration(strings.TrimSpace(value)) if err != nil { logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). @@ -691,11 +692,11 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * return } data.autoScaleWaitTimeout = dur - } - if key == DockerRouterLabelAutoScaleFailedMOTD { + + case DockerRouterLabelAutoScaleFailedMOTD: data.autoScaleFailedMOTD = value - } - if key == DockerRouterLabelAutoScaleRestartDelayMOTD { + + case DockerRouterLabelAutoScaleRestartDelayMOTD: data.autoScaleRestartDelayMOTD = value } } From 2e8dd41457baebfe40a1afd4e5ab819f384c14c1 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 19:37:51 -0500 Subject: [PATCH 30/38] fix: update service update to only scale up when replicas is set to 0. --- server/docker_swarm.go | 184 +++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 89 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index e755c7c..bee30f3 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -90,8 +90,7 @@ func (w *dockerSwarmWatcherImpl) makeWakerFunc(rs *routableSwarmService) WakerFu waitTimeout = 60 * time.Second } - replicas := service.Spec.Mode.Replicated.Replicas - if replicas == nil || *replicas == 0 { + if service.Spec.Mode.Replicated != nil && service.Spec.Mode.Replicated.Replicas != nil && *service.Spec.Mode.Replicated.Replicas == 0 { logrus.WithFields(logrus.Fields{ "serviceID": serviceID, "serviceName": rs.serviceName, @@ -612,93 +611,8 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * data.serviceID = service.ID data.serviceName = service.Spec.Name - for key, value := range service.Spec.Labels { - switch key { - case DockerRouterLabelHost: - if data.hosts != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) - return - } - data.hosts = SplitExternalHosts(value) - - case DockerRouterLabelPort: - if data.port != 0 { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) - return - } - var err error - data.port, err = strconv.ParseUint(value, 10, 32) - if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - WithError(err). - Warnf("ignoring service with invalid %s", DockerRouterLabelPort) - return - } - - case DockerRouterLabelDefault: - if data.def != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) - return - } - data.def = new(bool) - - lowerValue := strings.TrimSpace(strings.ToLower(value)) - *data.def = lowerValue != "" && lowerValue != "0" && lowerValue != "false" && lowerValue != "no" - - case DockerRouterLabelNetwork: - if data.network != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) - return - } - data.network = new(string) - *data.network = value - - case DockerRouterLabelAutoScaleUp: - autoScaleUp, err := strconv.ParseBool(strings.TrimSpace(value)) - if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - WithError(err). - Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleUp) - return - } - data.autoScaleUp = autoScaleUp - - case DockerRouterLabelAutoScaleDown: - autoScaleDown, err := strconv.ParseBool(strings.TrimSpace(value)) - if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - WithError(err). - Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleDown) - return - } - data.autoScaleDown = autoScaleDown - - case DockerRouterLabelAutoScaleAsleepMOTD: - data.autoScaleAsleepMOTD = value - - case DockerRouterLabelAutoScaleLoadingMOTD: - data.autoScaleLoadingMOTD = value - - case DockerRouterLabelAutoScaleWaitTimeout: - dur, err := time.ParseDuration(strings.TrimSpace(value)) - if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - WithError(err). - Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleWaitTimeout) - return - } - data.autoScaleWaitTimeout = dur - - case DockerRouterLabelAutoScaleFailedMOTD: - data.autoScaleFailedMOTD = value - - case DockerRouterLabelAutoScaleRestartDelayMOTD: - data.autoScaleRestartDelayMOTD = value - } + if !w.parseServiceLabels(service, &data) { + return } // probably not minecraft related @@ -928,3 +842,95 @@ func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service * ok = true return } + +func (w *dockerSwarmWatcherImpl) parseServiceLabels(service *swarm.Service, data *parsedDockerServiceData) bool { + for key, value := range service.Spec.Labels { + switch key { + case DockerRouterLabelHost: + if data.hosts != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelHost) + return false + } + data.hosts = SplitExternalHosts(value) + + case DockerRouterLabelPort: + if data.port != 0 { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelPort) + return false + } + var err error + data.port, err = strconv.ParseUint(value, 10, 32) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid %s", DockerRouterLabelPort) + return false + } + + case DockerRouterLabelDefault: + if data.def != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelDefault) + return false + } + data.def = new(bool) + + lowerValue := strings.TrimSpace(strings.ToLower(value)) + *data.def = lowerValue != "" && lowerValue != "0" && lowerValue != "false" && lowerValue != "no" + + case DockerRouterLabelNetwork: + if data.network != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service with duplicate %s", DockerRouterLabelNetwork) + return false + } + data.network = new(string) + *data.network = value + + case DockerRouterLabelAutoScaleUp: + autoScaleUp, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleUp) + return false + } + data.autoScaleUp = autoScaleUp + + case DockerRouterLabelAutoScaleDown: + autoScaleDown, err := strconv.ParseBool(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleDown) + return false + } + data.autoScaleDown = autoScaleDown + + case DockerRouterLabelAutoScaleAsleepMOTD: + data.autoScaleAsleepMOTD = value + + case DockerRouterLabelAutoScaleLoadingMOTD: + data.autoScaleLoadingMOTD = value + + case DockerRouterLabelAutoScaleWaitTimeout: + dur, err := time.ParseDuration(strings.TrimSpace(value)) + if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + WithError(err). + Warnf("ignoring service with invalid value for %s", DockerRouterLabelAutoScaleWaitTimeout) + return false + } + data.autoScaleWaitTimeout = dur + + case DockerRouterLabelAutoScaleFailedMOTD: + data.autoScaleFailedMOTD = value + + case DockerRouterLabelAutoScaleRestartDelayMOTD: + data.autoScaleRestartDelayMOTD = value + } + } + return true +} From f7f0a040c3c30be223ec15ccddfcf14789283c94 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 19:56:15 -0500 Subject: [PATCH 31/38] refractor: update parseServiceData symbol name to accurately represent function --- server/docker_swarm.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index bee30f3..ea9c2ba 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -511,7 +511,7 @@ func (w *dockerSwarmWatcherImpl) listServices(ctx context.Context) ([]*routableS continue } - data, ok := w.parseServiceData(ctx, &service, networkMap) + data, ok := w.evaluateSwarmService(ctx, &service, networkMap) if !ok { continue } @@ -604,7 +604,7 @@ type parsedDockerServiceData struct { statusState string } -func (w *dockerSwarmWatcherImpl) parseServiceData(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { +func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { data.autoScaleUp = w.config.autoScaleUp data.autoScaleDown = w.config.autoScaleDown data.autoScaleWaitTimeout = 60 * time.Second From b07ccc694f4c51d2d732a2e325a790c34107dc51 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 19:56:29 -0500 Subject: [PATCH 32/38] chore: clean whitespace --- server/docker_swarm.go | 68 +++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index ea9c2ba..da11980 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -34,20 +34,20 @@ func NewDockerSwarmWatcher(socket string, timeout time.Duration, autoScaleUp boo } type routableSwarmService struct { - externalServiceName string - containerEndpoint string - serviceID string - serviceName string - networkID string - autoScaleUp bool - autoScaleDown bool - autoScaleAsleepMOTD string - autoScaleLoadingMOTD string - autoScaleWaitTimeout time.Duration - autoScaleFailedMOTD string - autoScaleRestartDelayMOTD string - countdownDeadline time.Time - statusState string + externalServiceName string + containerEndpoint string + serviceID string + serviceName string + networkID string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time + statusState string } type dockerSwarmWatcherImpl struct { @@ -346,8 +346,8 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { w.routes.SetDefaultRoute(rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } w.routes.SetCountdownDeadline(rs.externalServiceName, rs.countdownDeadline) - // If the service is already tracked, check if any metadata, endpoint, MOTDs, or deadline - // changed. If so, recreate wakers/sleepers and update the route table. + // If the service is already tracked, check if any metadata, endpoint, MOTDs, or deadline + // changed. If so, recreate wakers/sleepers and update the route table. } else if oldRs.containerEndpoint != rs.containerEndpoint || oldRs.serviceID != rs.serviceID || oldRs.networkID != rs.networkID || @@ -584,24 +584,24 @@ func dockerCheckNetworkName(id string, name string, networkMap map[string]*netwo } type parsedDockerServiceData struct { - hosts []string - port uint64 - def *bool - network *string - networkID string - ip string - serviceID string - serviceName string - autoScaleUp bool - autoScaleDown bool - autoScaleAsleepMOTD string - autoScaleLoadingMOTD string - autoScaleWaitTimeout time.Duration - autoScaleFailedMOTD string - autoScaleRestartDelayMOTD string - countdownDeadline time.Time - isDNSRR bool - statusState string + hosts []string + port uint64 + def *bool + network *string + networkID string + ip string + serviceID string + serviceName string + autoScaleUp bool + autoScaleDown bool + autoScaleAsleepMOTD string + autoScaleLoadingMOTD string + autoScaleWaitTimeout time.Duration + autoScaleFailedMOTD string + autoScaleRestartDelayMOTD string + countdownDeadline time.Time + isDNSRR bool + statusState string } func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, service *swarm.Service, networkMap map[string]*network.Inspect) (data parsedDockerServiceData, ok bool) { From 89e327aa7c00499eb03b0fe1c36c52c311fe2e96 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 20:04:33 -0500 Subject: [PATCH 33/38] refactor: extract resolveTargetNetwork helper from evaluateSwarmService --- server/docker_swarm.go | 62 +++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index da11980..821e385 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -645,34 +645,7 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi replicas = *service.Spec.Mode.Replicated.Replicas } - // Resolve target networkID based on label or task template networks - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases - } - - if data.network != nil { - for _, netSpec := range service.Spec.TaskTemplate.Networks { - if ok, _ := dockerCheckNetworkName(netSpec.Target, *data.network, networkMap, networkAliases); ok { - data.networkID = netSpec.Target - break - } - } - } else { - // Default: Find the first non-ingress network in the task template - for _, netSpec := range service.Spec.TaskTemplate.Networks { - if network := networkMap[netSpec.Target]; network != nil { - if network.Name != "ingress" { - data.networkID = netSpec.Target - break - } - } - } - // Fallback to first network if all are ingress or not found in networkMap - if data.networkID == "" && len(service.Spec.TaskTemplate.Networks) > 0 { - data.networkID = service.Spec.TaskTemplate.Networks[0].Target - } - } + data.networkID = resolveTargetNetwork(service, data.network, networkMap) var hasRunningTask bool var runningTaskIP string @@ -777,6 +750,10 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi } if vipIndex == -1 { if data.network != nil { + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases + } for i, vip := range service.Endpoint.VirtualIPs { if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { vipIndex = i @@ -934,3 +911,32 @@ func (w *dockerSwarmWatcherImpl) parseServiceLabels(service *swarm.Service, data } return true } + +func resolveTargetNetwork(service *swarm.Service, labelNetwork *string, networkMap map[string]*network.Inspect) string { + networkAliases := map[string][]string{} + for _, network := range service.Spec.TaskTemplate.Networks { + networkAliases[network.Target] = network.Aliases + } + + if labelNetwork != nil { + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if ok, _ := dockerCheckNetworkName(netSpec.Target, *labelNetwork, networkMap, networkAliases); ok { + return netSpec.Target + } + } + } else { + // Default: Find the first non-ingress network in the task template + for _, netSpec := range service.Spec.TaskTemplate.Networks { + if network := networkMap[netSpec.Target]; network != nil { + if network.Name != "ingress" { + return netSpec.Target + } + } + } + // Fallback to first network if all are ingress or not found in networkMap + if len(service.Spec.TaskTemplate.Networks) > 0 { + return service.Spec.TaskTemplate.Networks[0].Target + } + } + return "" +} From 1d3a37e67e0d6bc03123a233ae6f1cf4bce95345 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 20:07:13 -0500 Subject: [PATCH 34/38] fix: build networkAliases once via getNetworkAliases helper --- server/docker_swarm.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 821e385..de41de8 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -645,7 +645,8 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi replicas = *service.Spec.Mode.Replicated.Replicas } - data.networkID = resolveTargetNetwork(service, data.network, networkMap) + networkAliases := getNetworkAliases(service) + data.networkID = resolveTargetNetwork(service, data.network, networkMap, networkAliases) var hasRunningTask bool var runningTaskIP string @@ -750,10 +751,6 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi } if vipIndex == -1 { if data.network != nil { - networkAliases := map[string][]string{} - for _, network := range service.Spec.TaskTemplate.Networks { - networkAliases[network.Target] = network.Aliases - } for i, vip := range service.Endpoint.VirtualIPs { if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { vipIndex = i @@ -912,12 +909,15 @@ func (w *dockerSwarmWatcherImpl) parseServiceLabels(service *swarm.Service, data return true } -func resolveTargetNetwork(service *swarm.Service, labelNetwork *string, networkMap map[string]*network.Inspect) string { +func getNetworkAliases(service *swarm.Service) map[string][]string { networkAliases := map[string][]string{} for _, network := range service.Spec.TaskTemplate.Networks { networkAliases[network.Target] = network.Aliases } + return networkAliases +} +func resolveTargetNetwork(service *swarm.Service, labelNetwork *string, networkMap map[string]*network.Inspect, networkAliases map[string][]string) string { if labelNetwork != nil { for _, netSpec := range service.Spec.TaskTemplate.Networks { if ok, _ := dockerCheckNetworkName(netSpec.Target, *labelNetwork, networkMap, networkAliases); ok { From 1e3715458f1b370d05c8562f11d3142e1373da33 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 20:10:14 -0500 Subject: [PATCH 35/38] refactor: extract queryServiceTasks helper from evaluateSwarmService --- server/docker_swarm.go | 173 ++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 80 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index de41de8..7c77000 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -648,80 +648,7 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi networkAliases := getNetworkAliases(service) data.networkID = resolveTargetNetwork(service, data.network, networkMap, networkAliases) - var hasRunningTask bool - var runningTaskIP string - var hasReadyTask bool - var readyTaskTimestamp time.Time - var hasActiveTask bool - var latestTask swarm.Task - var delay time.Duration - - var tasks []swarm.Task - var err error - if replicas > 0 { - tasks, err = w.client.TaskList(ctx, dockertypes.TaskListOptions{ - Filters: filters.NewArgs(filters.Arg("service", service.ID)), - }) - if err == nil && len(tasks) > 0 { - if service.Spec.TaskTemplate.RestartPolicy != nil { - if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { - delay = *service.Spec.TaskTemplate.RestartPolicy.Delay - } - } - - for _, task := range tasks { - state := task.Status.State - desiredState := task.DesiredState - - // Track the most recently created task to inspect its DesiredState. - if latestTask.CreatedAt.IsZero() || task.CreatedAt.After(latestTask.CreatedAt) { - latestTask = task - } - - // The task is considered fully Running only if both actual and desired states are running. - // If DesiredState is Shutdown or Remove, it is stopping. - if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { - hasRunningTask = true - // Extract direct task IP to bypass VIP propagation lag when replicas == 1 - for _, attachment := range task.NetworksAttachments { - matchesNetwork := data.networkID != "" && attachment.Network.ID == data.networkID - isIngress := attachment.Network.Spec.Name == "ingress" - - if (matchesNetwork || (data.networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { - parts := strings.Split(attachment.Addresses[0], "/") - if ip := net.ParseIP(parts[0]); ip != nil { - runningTaskIP = parts[0] - break - } - } - } - } - - // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. - // Find the latest ready task's status timestamp to measure the restart delay start. - if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { - hasReadyTask = true - if task.Status.Timestamp.After(readyTaskTimestamp) { - readyTaskTimestamp = task.Status.Timestamp - } - } - - // Track active task states to see if Swarm is actively attempting to schedule/start a task. - if state == swarm.TaskStateNew || - state == swarm.TaskStatePending || - state == swarm.TaskStateAssigned || - state == swarm.TaskStateAccepted || - state == swarm.TaskStatePreparing || - state == swarm.TaskStateReady || - state == swarm.TaskStateStarting || - state == swarm.TaskStateRunning || - desiredState == swarm.TaskStateReady || - desiredState == swarm.TaskStateRunning { - hasActiveTask = true - } - } - } - } + tasksSummary := w.queryServiceTasks(ctx, service, replicas, data.networkID) // State machine classification: // To prevent premature VIP/DNS resolution routing client connections away from the waker, @@ -734,11 +661,11 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi data.ip = "" // data.autoScaleAsleepMOTD is already populated by parsing labels. data.countdownDeadline = time.Time{} - } else if hasRunningTask { + } else if tasksSummary.hasRunningTask { // 2. Running (Healthy) State: Service has a fully running task. data.statusState = "running" - if replicas == 1 && runningTaskIP != "" { - data.ip = runningTaskIP + if replicas == 1 && tasksSummary.runningTaskIP != "" { + data.ip = tasksSummary.runningTaskIP } else if isVIP { vipIndex := -1 if data.networkID != "" { @@ -782,7 +709,7 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi data.ip = "" // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. - if hasReadyTask && delay > 0 { + if tasksSummary.hasReadyTask && tasksSummary.delay > 0 { data.statusState = "restart_delay" if data.autoScaleRestartDelayMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD @@ -791,8 +718,8 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi } else { data.autoScaleAsleepMOTD = "Server failed to start." } - data.countdownDeadline = readyTaskTimestamp.Add(delay) - } else if !hasActiveTask && latestTask.DesiredState == swarm.TaskStateShutdown && len(tasks) > 0 { + data.countdownDeadline = tasksSummary.readyTaskTimestamp.Add(tasksSummary.delay) + } else if !tasksSummary.hasActiveTask && tasksSummary.latestTask.DesiredState == swarm.TaskStateShutdown && len(tasksSummary.tasks) > 0 { // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task // and is no longer attempting to restart the service. data.statusState = "failed" @@ -940,3 +867,89 @@ func resolveTargetNetwork(service *swarm.Service, labelNetwork *string, networkM } return "" } + +type serviceTasksSummary struct { + hasRunningTask bool + runningTaskIP string + hasReadyTask bool + readyTaskTimestamp time.Time + hasActiveTask bool + latestTask swarm.Task + delay time.Duration + tasks []swarm.Task +} + +func (w *dockerSwarmWatcherImpl) queryServiceTasks(ctx context.Context, service *swarm.Service, replicas uint64, networkID string) serviceTasksSummary { + var summary serviceTasksSummary + if replicas == 0 { + return summary + } + + tasks, err := w.client.TaskList(ctx, dockertypes.TaskListOptions{ + Filters: filters.NewArgs(filters.Arg("service", service.ID)), + }) + if err != nil || len(tasks) == 0 { + return summary + } + + summary.tasks = tasks + + if service.Spec.TaskTemplate.RestartPolicy != nil { + if service.Spec.TaskTemplate.RestartPolicy.Delay != nil { + summary.delay = *service.Spec.TaskTemplate.RestartPolicy.Delay + } + } + + for _, task := range tasks { + state := task.Status.State + desiredState := task.DesiredState + + // Track the most recently created task to inspect its DesiredState. + if summary.latestTask.CreatedAt.IsZero() || task.CreatedAt.After(summary.latestTask.CreatedAt) { + summary.latestTask = task + } + + // The task is considered fully Running only if both actual and desired states are running. + if state == swarm.TaskStateRunning && desiredState == swarm.TaskStateRunning { + summary.hasRunningTask = true + // Extract direct task IP to bypass VIP propagation lag when replicas == 1 + for _, attachment := range task.NetworksAttachments { + matchesNetwork := networkID != "" && attachment.Network.ID == networkID + isIngress := attachment.Network.Spec.Name == "ingress" + + if (matchesNetwork || (networkID == "" && !isIngress)) && len(attachment.Addresses) > 0 { + parts := strings.Split(attachment.Addresses[0], "/") + if ip := net.ParseIP(parts[0]); ip != nil { + summary.runningTaskIP = parts[0] + break + } + } + } + } + + // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. + // Find the latest ready task's status timestamp to measure the restart delay start. + if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { + summary.hasReadyTask = true + if task.Status.Timestamp.After(summary.readyTaskTimestamp) { + summary.readyTaskTimestamp = task.Status.Timestamp + } + } + + // Track active task states to see if Swarm is actively attempting to schedule/start a task. + if state == swarm.TaskStateNew || + state == swarm.TaskStatePending || + state == swarm.TaskStateAssigned || + state == swarm.TaskStateAccepted || + state == swarm.TaskStatePreparing || + state == swarm.TaskStateReady || + state == swarm.TaskStateStarting || + state == swarm.TaskStateRunning || + desiredState == swarm.TaskStateReady || + desiredState == swarm.TaskStateRunning { + summary.hasActiveTask = true + } + } + + return summary +} From 496b60e5317ea5eea1b7bac0fbb99bc1f3093346 Mon Sep 17 00:00:00 2001 From: Bionix Date: Sat, 18 Jul 2026 20:12:28 -0500 Subject: [PATCH 36/38] refactor: extract classifyServiceState helper from evaluateSwarmService --- server/docker_swarm.go | 179 +++++++++++++++++++++-------------------- 1 file changed, 91 insertions(+), 88 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 7c77000..8d95979 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -650,94 +650,8 @@ func (w *dockerSwarmWatcherImpl) evaluateSwarmService(ctx context.Context, servi tasksSummary := w.queryServiceTasks(ctx, service, replicas, data.networkID) - // State machine classification: - // To prevent premature VIP/DNS resolution routing client connections away from the waker, - // we keep data.ip = "" in all non-running states. This forces client connections to use - // the waker, keeps the loading MOTD active, and avoids dialing starting/unreachable containers. - - if replicas == 0 { - // 1. Sleeping State: Service is scaled to zero. - data.statusState = "sleeping" - data.ip = "" - // data.autoScaleAsleepMOTD is already populated by parsing labels. - data.countdownDeadline = time.Time{} - } else if tasksSummary.hasRunningTask { - // 2. Running (Healthy) State: Service has a fully running task. - data.statusState = "running" - if replicas == 1 && tasksSummary.runningTaskIP != "" { - data.ip = tasksSummary.runningTaskIP - } else if isVIP { - vipIndex := -1 - if data.networkID != "" { - for i, vip := range service.Endpoint.VirtualIPs { - if vip.NetworkID == data.networkID { - vipIndex = i - break - } - } - } - if vipIndex == -1 { - if data.network != nil { - for i, vip := range service.Endpoint.VirtualIPs { - if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { - vipIndex = i - break - } else if err != nil { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Debugf("%v", err) - } - } - } else { - vipIndex = 0 - } - } - if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { - virtualIP := service.Endpoint.VirtualIPs[vipIndex] - ip, _, _ := net.ParseCIDR(virtualIP.Addr) - data.ip = ip.String() - data.networkID = virtualIP.NetworkID - } else { - logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). - Warnf("ignoring service, unable to find match in VirtualIPs") - return - } - } else if isDNSRR { - data.ip = service.Spec.Name - } - } else { - // Non-running states: keep data.ip empty to ensure the waker captures client connections - data.ip = "" - - // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. - if tasksSummary.hasReadyTask && tasksSummary.delay > 0 { - data.statusState = "restart_delay" - if data.autoScaleRestartDelayMOTD != "" { - data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD - } else if data.autoScaleFailedMOTD != "" { - data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD - } else { - data.autoScaleAsleepMOTD = "Server failed to start." - } - data.countdownDeadline = tasksSummary.readyTaskTimestamp.Add(tasksSummary.delay) - } else if !tasksSummary.hasActiveTask && tasksSummary.latestTask.DesiredState == swarm.TaskStateShutdown && len(tasksSummary.tasks) > 0 { - // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task - // and is no longer attempting to restart the service. - data.statusState = "failed" - if data.autoScaleFailedMOTD != "" { - data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD - } else { - data.autoScaleAsleepMOTD = "Server failed to start." - } - data.countdownDeadline = time.Time{} - } else { - // 5. Starting / Waking State: Swarm has scaled the service up and the task is starting up (transited past ready), - // or it is the very first start. We display the loading MOTD on pings. - data.statusState = "waking" - if data.autoScaleLoadingMOTD != "" { - data.autoScaleAsleepMOTD = data.autoScaleLoadingMOTD - } - data.countdownDeadline = time.Time{} - } + if !classifyServiceState(&data, service, replicas, isVIP, isDNSRR, tasksSummary, networkMap, networkAliases) { + return } ok = true @@ -953,3 +867,92 @@ func (w *dockerSwarmWatcherImpl) queryServiceTasks(ctx context.Context, service return summary } + +func classifyServiceState(data *parsedDockerServiceData, service *swarm.Service, replicas uint64, isVIP bool, isDNSRR bool, tasksSummary serviceTasksSummary, networkMap map[string]*network.Inspect, networkAliases map[string][]string) bool { + if replicas == 0 { + // 1. Sleeping State: Service is scaled to zero. + data.statusState = "sleeping" + data.ip = "" + // data.autoScaleAsleepMOTD is already populated by parsing labels. + data.countdownDeadline = time.Time{} + } else if tasksSummary.hasRunningTask { + // 2. Running (Healthy) State: Service has a fully running task. + data.statusState = "running" + if replicas == 1 && tasksSummary.runningTaskIP != "" { + data.ip = tasksSummary.runningTaskIP + } else if isVIP { + vipIndex := -1 + if data.networkID != "" { + for i, vip := range service.Endpoint.VirtualIPs { + if vip.NetworkID == data.networkID { + vipIndex = i + break + } + } + } + if vipIndex == -1 { + if data.network != nil { + for i, vip := range service.Endpoint.VirtualIPs { + if ok, err := dockerCheckNetworkName(vip.NetworkID, *data.network, networkMap, networkAliases); ok { + vipIndex = i + break + } else if err != nil { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Debugf("%v", err) + } + } + } else { + vipIndex = 0 + } + } + if vipIndex != -1 && vipIndex < len(service.Endpoint.VirtualIPs) { + virtualIP := service.Endpoint.VirtualIPs[vipIndex] + ip, _, _ := net.ParseCIDR(virtualIP.Addr) + data.ip = ip.String() + data.networkID = virtualIP.NetworkID + } else { + logrus.WithFields(logrus.Fields{"serviceId": service.ID, "serviceName": service.Spec.Name}). + Warnf("ignoring service, unable to find match in VirtualIPs") + return false + } + } else if isDNSRR { + data.ip = service.Spec.Name + } + } else { + // Non-running states: keep data.ip empty to ensure the waker captures client connections + data.ip = "" + + // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. + if tasksSummary.hasReadyTask && tasksSummary.delay > 0 { + data.statusState = "restart_delay" + if data.autoScaleRestartDelayMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD + } else if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." + } + data.countdownDeadline = tasksSummary.readyTaskTimestamp.Add(tasksSummary.delay) + } else if !tasksSummary.hasActiveTask && tasksSummary.latestTask.DesiredState == swarm.TaskStateShutdown && len(tasksSummary.tasks) > 0 { + // 4. Permanently Failed State: Swarm has commanded a shutdown on the failed task + // and is no longer attempting to restart the service. + data.statusState = "failed" + if data.autoScaleFailedMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleFailedMOTD + } else { + data.autoScaleAsleepMOTD = "Server failed to start." + } + data.countdownDeadline = time.Time{} + } else { + // 5. Starting / Waking State: Swarm has scaled the service up and the task is starting up (transited past ready), + // or it is the very first start. We display the loading MOTD on pings. + data.statusState = "waking" + if data.autoScaleLoadingMOTD != "" { + data.autoScaleAsleepMOTD = data.autoScaleLoadingMOTD + } + data.countdownDeadline = time.Time{} + } + } + + return true +} From ba992c49b3922c54dc3ce45d31ded032eb503774 Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 20 Jul 2026 14:51:05 -0500 Subject: [PATCH 37/38] fix: restart delay and permafailture motd not showing --- server/docker_swarm.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 8d95979..12f47b6 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -284,6 +284,14 @@ func (w *dockerSwarmWatcherImpl) makeSleeperFunc(rs *routableSwarmService) Sleep } } +func (w *dockerSwarmWatcherImpl) makeServiceLifecycleFuncs(rs *routableSwarmService) (WakerFunc, SleeperFunc) { + var wakerFunc WakerFunc + if rs.statusState == "sleeping" || rs.statusState == "waking" { + wakerFunc = w.makeWakerFunc(rs) + } + return wakerFunc, w.makeSleeperFunc(rs) +} + func (w *dockerSwarmWatcherImpl) Start(ctx context.Context) error { var err error @@ -338,8 +346,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { "hosts": rs.externalServiceName, }).Infof("Swarm service state: %s%s", rs.statusState, ipDetail) - wakerFunc := w.makeWakerFunc(rs) - sleeperFunc := w.makeSleeperFunc(rs) + wakerFunc, sleeperFunc := w.makeServiceLifecycleFuncs(rs) if rs.externalServiceName != "" { w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) } else { @@ -373,8 +380,7 @@ func (w *dockerSwarmWatcherImpl) reconcileServices(ctx context.Context) error { } w.serviceMap[rs.externalServiceName] = rs - wakerFunc := w.makeWakerFunc(rs) - sleeperFunc := w.makeSleeperFunc(rs) + wakerFunc, sleeperFunc := w.makeServiceLifecycleFuncs(rs) if rs.externalServiceName != "" { w.routes.DeleteMapping(rs.externalServiceName) w.routes.CreateMapping(rs.externalServiceName, rs.containerEndpoint, rs.serviceID, wakerFunc, sleeperFunc, rs.autoScaleAsleepMOTD, rs.autoScaleLoadingMOTD) @@ -841,9 +847,9 @@ func (w *dockerSwarmWatcherImpl) queryServiceTasks(ctx context.Context, service } } - // Swarm task state 'ready' (actual or desired) marks a task held during a restart delay. + // Swarm task desired state 'ready' marks a task held during a restart delay. // Find the latest ready task's status timestamp to measure the restart delay start. - if state == swarm.TaskStateReady || desiredState == swarm.TaskStateReady { + if desiredState == swarm.TaskStateReady { summary.hasReadyTask = true if task.Status.Timestamp.After(summary.readyTaskTimestamp) { summary.readyTaskTimestamp = task.Status.Timestamp @@ -923,7 +929,7 @@ func classifyServiceState(data *parsedDockerServiceData, service *swarm.Service, data.ip = "" // 3. Restart Delay State: Swarm scheduler is delaying task retry, keeping it in the 'ready' state. - if tasksSummary.hasReadyTask && tasksSummary.delay > 0 { + if tasksSummary.hasReadyTask { data.statusState = "restart_delay" if data.autoScaleRestartDelayMOTD != "" { data.autoScaleAsleepMOTD = data.autoScaleRestartDelayMOTD From 3d9e1071b78ed6d1cf93008ecee843ed32e15ed9 Mon Sep 17 00:00:00 2001 From: Bionix Date: Mon, 20 Jul 2026 19:08:01 -0500 Subject: [PATCH 38/38] fix: changed statement that might have prevented the scaledown timer from being stopped while server was in play (active players) --- server/docker_swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/docker_swarm.go b/server/docker_swarm.go index 12f47b6..e5aface 100644 --- a/server/docker_swarm.go +++ b/server/docker_swarm.go @@ -286,7 +286,7 @@ func (w *dockerSwarmWatcherImpl) makeSleeperFunc(rs *routableSwarmService) Sleep func (w *dockerSwarmWatcherImpl) makeServiceLifecycleFuncs(rs *routableSwarmService) (WakerFunc, SleeperFunc) { var wakerFunc WakerFunc - if rs.statusState == "sleeping" || rs.statusState == "waking" { + if rs.statusState == "sleeping" || rs.statusState == "waking" || rs.statusState == "running" { wakerFunc = w.makeWakerFunc(rs) } return wakerFunc, w.makeSleeperFunc(rs)