perf: reduce K8s API calls during DNS server startup

charts/multicluster-ingress/templates/crds/gslbs_multicluster_cyclops_io.yaml

@@ -47,13 +47,11 @@ spec:
                 type: array
               priority:
                 description: Priority to assign this GSLB object. Highest priority is chosen first.
-                exclusiveMinimum: false
                 format: int32
                 minimum: 0.0
                 type: integer
               weight:
                 description: Weight to assign this GSLB object when doing round robin load balancing type. Defaults to 50. The calculation to determine the final weighting of all objects is (weight / sum of all weights) * 100.
-                exclusiveMinimum: false
                 format: int32
                 minimum: 0.0
                 type: integer

charts/multicluster-ingress/templates/crds/gslbs_multicluster_veccsolutions_io.yaml

@@ -47,13 +47,11 @@ spec:
             type: array
           priority:
             description: Priority to assign this GSLB object. Highest priority is chosen first.
-            exclusiveMinimum: false
             format: int32
             minimum: 0.0
             type: integer
           weight:
             description: Weight to assign this GSLB object when doing round robin load balancing type. Defaults to 50. The calculation to determine the final weighting of all objects is (weight / sum of all weights) * 100.
-            exclusiveMinimum: false
             format: int32
             minimum: 0.0
             type: integer

src/Cyclops.MultiCluster.Tests/Controllers/K8sHostnameCacheControllerTests.cs

@@ -62,7 +62,7 @@ public async Task ReconcileAsync_ReturnsSuccess()
         public async Task DeletedAsync_UsesHostnameFromEntity()
         {
             string? capturedHostname = null;
-            _queue.OnHostChangedAsync = (hostname) =>
+            _queue.OnHostChangedAsync = (hostname, _) =>
             {
                 capturedHostname = hostname;
                 return Task.CompletedTask;
@@ -87,7 +87,7 @@ public async Task DeletedAsync_UsesHostnameFromEntity()
         public async Task DeletedAsync_FallsBackToLabel_WhenHostnameNull()
         {
             string? capturedHostname = null;
-            _queue.OnHostChangedAsync = (hostname) =>
+            _queue.OnHostChangedAsync = (hostname, _) =>
             {
                 capturedHostname = hostname;
                 return Task.CompletedTask;
@@ -113,7 +113,7 @@ public async Task DeletedAsync_FallsBackToLabel_WhenHostnameNull()
         public async Task ReconcileAsync_UsesHostnameFromEntity()
         {
             string? capturedHostname = null;
-            _queue.OnHostChangedAsync = (hostname) =>
+            _queue.OnHostChangedAsync = (hostname, _) =>
             {
                 capturedHostname = hostname;
                 return Task.CompletedTask;

src/Cyclops.MultiCluster.Tests/Services/Default/KubernetesQueueTests.cs

@@ -18,7 +18,7 @@ public async Task OnHostChangedAsync_CanBeSet()
         {
             var queue = new KubernetesQueue();
             string? capturedValue = null;
-            queue.OnHostChangedAsync = (value) =>
+            queue.OnHostChangedAsync = (value, _) =>
             {
                 capturedValue = value;
                 return Task.CompletedTask;
@@ -42,7 +42,7 @@ public async Task OnHostChangedAsync_CanHandleNull()
         {
             var queue = new KubernetesQueue();
             string? capturedValue = "initial";
-            queue.OnHostChangedAsync = (value) =>
+            queue.OnHostChangedAsync = (value, _) =>
             {
                 capturedValue = value;
                 return Task.CompletedTask;

src/Cyclops.MultiCluster/Controllers/HealthzController.cs

@@ -2,7 +2,6 @@
 using KubeOps.KubernetesClient;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using Cyclops.MultiCluster.Services;
 
 namespace Cyclops.MultiCluster.Controllers
 {
@@ -15,19 +14,15 @@ namespace Cyclops.MultiCluster.Controllers
     public class HealthzController : Controller
     {
         private readonly IKubernetesClient _kubernetesClient;
-        private readonly IDateTimeProvider _dateTimeProvider;
-        private static DateTime _lastUp;
         private static bool _ready;
 
         /// <summary>
         /// Health related operations happen here
         /// </summary>
         /// <param name="kubernetesClient"></param>
-        /// <param name="dateTimeProvider"></param>
-        public HealthzController(IKubernetesClient kubernetesClient, IDateTimeProvider dateTimeProvider)
+        public HealthzController(IKubernetesClient kubernetesClient)
         {
             _kubernetesClient = kubernetesClient;
-            _dateTimeProvider = dateTimeProvider;
         }
 
         /// <summary>
@@ -60,32 +55,17 @@ public async Task<IActionResult> ReadyAsync()
         }
 
         /// <summary>
-        /// Check if the pod is alive
+        /// Check if the pod is alive.
+        /// This intentionally does NOT call the Kubernetes API. Liveness probes
+        /// should only verify the process is responsive. The previous implementation
+        /// called GetAsync<V1Namespace>; which competed with reconciler traffic and
+        /// timed out under load, causing unnecessary pod restarts across all components.
         /// </summary>
         /// <returns></returns>
         [HttpGet("Liveness")]
-        public async Task<IActionResult> LivenessAsync()
+        public IActionResult Liveness()
         {
-            if (_lastUp < _dateTimeProvider.UtcNow.AddSeconds(-10))
-            {
-                V1Namespace? ns = null;
-                try
-                {
-                    ns = await _kubernetesClient.GetAsync<V1Namespace>(await _kubernetesClient.GetCurrentNamespaceAsync());
-                    if (ns == null)
-                    {
-                        return StatusCode(500, "Namespace result was null");
-                    }
-
-                    _lastUp = _dateTimeProvider.UtcNow;
-                    return Ok(new { Status = "Kubernetes API returned something", Namespace = ns });
-                }
-                catch (Exception ex)
-                {
-                    return StatusCode(500, ex);
-                }
-            }
-            return Ok("Cached response is good.");
+            return Ok("Alive");
         }
     }
 }

src/Cyclops.MultiCluster/Controllers/K8sHostnameCacheController.cs

@@ -2,6 +2,7 @@
 using KubeOps.Abstractions.Rbac;
 using KubeOps.Abstractions.Reconciliation;
 using KubeOps.Abstractions.Reconciliation.Controller;
+using Cyclops.MultiCluster.Models.Core;
 using Cyclops.MultiCluster.Models.K8sEntities;
 using Cyclops.MultiCluster.Services.Default;
 
@@ -53,7 +54,12 @@ public Task<ReconciliationResult<V1HostnameCache>> ReconcileAsync(V1HostnameCach
             var hostname = entity.Hostname ?? entity.GetLabel("hostname");
             using var _scope = _logger.BeginScope(new {@object = "hostnamecache", state="deleted", @namespace = entity.Namespace(), cluster = entity.Name(), hostname });
             _logger.LogInformation("Reconciling hostname cache");
-            _queue.OnHostChangedAsync(hostname);
+            var hostInfo = new Models.Core.Host
+            {
+                Hostname = hostname,
+                HostIPs = entity.Addresses.Select(x => x.ToCore()).ToArray()
+            };
+            _queue.OnHostChangedAsync(hostname, hostInfo);
             _logger.LogInformation("Hostname cache reconciled");
             return Task.FromResult(ReconciliationResult<V1HostnameCache>.Success(entity));
         }

src/Cyclops.MultiCluster/Services/Default/DefaultCache.cs

@@ -24,6 +24,9 @@ public async Task<int> GetEndpointsCountAsync(string ns, string name)
     public Task<Models.Core.Host?> GetHostInformationAsync(string hostname)
         => _kubernetesCache.GetHostInformationAsync(hostname);
 
+    public Task<Dictionary<string, Models.Core.Host>> GetAllHostInformationAsync()
+        => _kubernetesCache.GetAllHostInformationAsync();
+
     public Task<string[]> GetHostnamesAsync()
         => _kubernetesCache.GetHostnamesAsync();
 

src/Cyclops.MultiCluster/Services/Default/DefaultDnsResolver.cs

@@ -74,7 +74,7 @@ public async Task InitializeAsync()
         }
 
         [Transaction]
-        private async Task RefreshHostInformationAsync(string? hostname)
+        private async Task RefreshHostInformationAsync(string? hostname, Models.Core.Host? preloadedHostInformation = null)
         {
             using var scope = _logger.BeginScope(new { hostname });
             _logger.LogInformation("Hostname updated, refreshing state.");
@@ -85,7 +85,7 @@ private async Task RefreshHostInformationAsync(string? hostname)
                 return;
             }
 
-            var hostInformation = await _cache.GetHostInformationAsync(hostname);
+            var hostInformation = preloadedHostInformation ?? await _cache.GetHostInformationAsync(hostname);
             if (hostInformation?.HostIPs == null)
             {
                 //no host information
@@ -322,21 +322,18 @@ private async Task ResyncTimerAsync()
         [Trace]
         private async Task ResyncAsync()
         {
-            var hostnames = await _cache.GetHostnamesAsync();
+            var allHosts = await _cache.GetAllHostInformationAsync();
 
-            if (hostnames != null)
+            foreach (var (hostname, hostInfo) in allHosts)
             {
-                foreach (var hostname in hostnames)
-                {
-                    await RefreshHostInformationAsync(hostname);
-                }
+                await RefreshHostInformationAsync(hostname, hostInfo);
+            }
 
-                foreach (var hostname in _hosts.Keys.ToArray())
+            foreach (var hostname in _hosts.Keys.ToArray())
+            {
+                if (!allHosts.ContainsKey(hostname))
                 {
-                    if (!hostnames.Contains(hostname))
-                    {
-                        _hosts.Remove(hostname, out var _);
-                    }
+                    _hosts.Remove(hostname, out var _);
                 }
             }
         }

src/Cyclops.MultiCluster/Services/Default/KubernetesApiCache.cs

@@ -82,6 +82,33 @@ public async Task<string[]> GetClusterIdentifiersAsync()
             return null;
         }
 
+        public async Task<Dictionary<string, Models.Core.Host>> GetAllHostInformationAsync()
+        {
+            _logger.LogTrace("Getting all host information in bulk");
+            var allCaches = await _kubernetesClient.ListAsync<V1HostnameCache>(_options.Value.Namespace);
+            var result = new Dictionary<string, Models.Core.Host>();
+
+            foreach (var cache in allCaches)
+            {
+                var hostname = cache.Hostname ?? cache.GetLabel("hostname");
+                if (hostname != null)
+                {
+                    result[hostname] = new Models.Core.Host
+                    {
+                        HostIPs = cache.Addresses.Select(x => x.ToCore()).ToArray(),
+                        Hostname = hostname
+                    };
+                }
+                else
+                {
+                    _logger.LogDebug("Skipping hostname cache entry {name} with no hostname", cache.Metadata?.Name);
+                }
+            }
+
+            _logger.LogTrace("Got {count} hosts in bulk", result.Count);
+            return result;
+        }
+
         public async Task<string[]> GetHostnamesAsync()
         {
             _logger.LogTrace("Getting hostnames");

src/Cyclops.MultiCluster/Services/Default/KubernetesQueue.cs

@@ -1,9 +1,10 @@
-
+using Cyclops.MultiCluster.Models.Core;
+
 namespace Cyclops.MultiCluster.Services.Default
 {
     public class KubernetesQueue : IQueue
     {
-        public OnHostChangedAsyncDelegate OnHostChangedAsync { get; set; } = _ => Task.CompletedTask;
+        public OnHostChangedAsyncDelegate OnHostChangedAsync { get; set; } = (_, _) => Task.CompletedTask;
 
         public Task PublishHostChangedAsync(string hostname)
         {

src/Cyclops.MultiCluster/Services/ICache.cs

@@ -2,23 +2,14 @@
 
 namespace Cyclops.MultiCluster.Services
 {
-    public interface ICache
+    public interface ICache : IKubernetesCache
     {
-        Task<string[]> GetClusterIdentifiersAsync();
         Task<int> GetEndpointsCountAsync(string ns, string name);
-        Task<Models.Core.Host?> GetHostInformationAsync(string hostname);
-        Task<string[]> GetHostnamesAsync();
-        Task<Models.Core.Host[]?> GetHostsAsync(string clusterIdentifier);
-        Task<DateTime?> GetClusterHeartbeatTimeAsync(string clusterIdentifier);
         Task<bool> IsServiceMonitoredAsync(string ns, string name);
-        Task SetClusterCacheAsync(string identifier, Models.Core.Host[] hosts);
-        Task SetClusterHeartbeatAsync(string clusterIdentifier, DateTime heartbeat);
         Task SetEndpointsCountAsync(string ns, string name, int count);
-        Task SynchronizeCachesAsync();
         Task TrackServiceAsync(string ns, string name);
         Task UntrackAllServicesAsync();
         Task SetResourceVersionAsync(string uniqueIdentifier, string version);
         Task<string> GetLastResourceVersionAsync(string uniqueIdentifier);
-        Task RemoveClusterCacheAsync(string clusterIdentifier);
     }
 }

src/Cyclops.MultiCluster/Services/IKubernetesCache.cs

@@ -6,6 +6,7 @@ public interface IKubernetesCache
     {
         Task<string[]> GetClusterIdentifiersAsync();
         Task<Models.Core.Host?> GetHostInformationAsync(string hostname);
+        Task<Dictionary<string, Models.Core.Host>> GetAllHostInformationAsync();
         Task<string[]> GetHostnamesAsync();
         Task<Models.Core.Host[]?> GetHostsAsync(string clusterIdentifier);
         Task<DateTime?> GetClusterHeartbeatTimeAsync(string clusterIdentifier);

src/Cyclops.MultiCluster/Services/IQueue.cs

@@ -1,6 +1,6 @@
 namespace Cyclops.MultiCluster.Services
 {
-    public delegate Task OnHostChangedAsyncDelegate(string? value);
+    public delegate Task OnHostChangedAsyncDelegate(string? value, Models.Core.Host? hostInformation = null);
 
     public interface IQueue
     {