DC-264 - Fix AppConfig startup race condition with Agent sidecar (#151)

* Fix AppConfig startup race condition with Agent sidecar

* Add log message when initial AppConfig load starts

Author: spokenbird

src/SEBT.Portal.Infrastructure/Configuration/AppConfigAgentConfigurationProvider.cs

@@ -11,6 +11,8 @@ namespace SEBT.Portal.Infrastructure.Configuration;
 public sealed class AppConfigAgentConfigurationProvider : ConfigurationProvider, IDisposable
 {
     private const int LockReleaseTimeout = 3_000;
+    private const int InitialLoadMaxRetries = 10;
+    private static readonly TimeSpan InitialLoadRetryDelay = TimeSpan.FromSeconds(1);
 
     private readonly HttpClient _httpClient;
     private readonly AppConfigAgentProfile _profile;
@@ -21,6 +23,7 @@ public sealed class AppConfigAgentConfigurationProvider : ConfigurationProvider,
     private Timer? _reloadTimer;
     private int _isLoading; // 0 = not loading, 1 = loading
     private volatile bool _disposed;
+    private bool _initialLoadCompleted;
 
     public AppConfigAgentConfigurationProvider(
         HttpClient httpClient,
@@ -45,7 +48,19 @@ public override void Load()
 
         try
         {
-            LoadAsync().GetAwaiter().GetResult();
+            if (!_initialLoadCompleted)
+            {
+                _logger?.LogInformation(
+                    "Initial load starting for profile {ProfileId} (will retry up to {MaxRetries} times if agent is not ready)",
+                    _profile.ProfileId,
+                    InitialLoadMaxRetries);
+                LoadWithRetryAsync().GetAwaiter().GetResult();
+                _initialLoadCompleted = true;
+            }
+            else
+            {
+                LoadAsync().GetAwaiter().GetResult();
+            }
 
             if (_profile.ReloadAfterSeconds.HasValue)
             {
@@ -60,6 +75,45 @@ public override void Load()
         }
     }
 
+    /// <summary>
+    /// Attempts to load configuration with retries. Used only for the initial load
+    /// to handle the race condition where the AppConfig Agent sidecar may not be
+    /// ready when the API container starts.
+    /// </summary>
+    private async Task LoadWithRetryAsync()
+    {
+        for (var attempt = 1; attempt <= InitialLoadMaxRetries; attempt++)
+        {
+            try
+            {
+                await LoadAsync().ConfigureAwait(false);
+
+                // If Data has items, the load succeeded.
+                if (Data.Count > 0)
+                    return;
+            }
+            catch
+            {
+                // LoadAsync handles its own exceptions — this is a safety net.
+            }
+
+            if (attempt < InitialLoadMaxRetries)
+            {
+                _logger?.LogInformation(
+                    "AppConfig Agent not ready (attempt {Attempt}/{MaxRetries}). Retrying in {Delay}s...",
+                    attempt,
+                    InitialLoadMaxRetries,
+                    InitialLoadRetryDelay.TotalSeconds);
+                await Task.Delay(InitialLoadRetryDelay).ConfigureAwait(false);
+            }
+        }
+
+        _logger?.LogError(
+            "AppConfig Agent not available after {MaxRetries} attempts for profile {ProfileId}. Starting without AppConfig values.",
+            InitialLoadMaxRetries,
+            _profile.ProfileId);
+    }
+
     private void OnReloadTimerFired()
     {
         if (!_disposed)

test/SEBT.Portal.Tests/Unit/Configuration/AppConfigAgentConfigurationProviderTests.cs

@@ -927,6 +927,163 @@ public void AppConfigAgentConfigurationSource_Build_WithoutHttpClient_ShouldCrea
         Assert.Null(exception);
     }
 
+    [Fact]
+    public void Load_InitialLoad_ShouldRetryOnConnectionRefused_ThenSucceed()
+    {
+        // Arrange — simulate AppConfig Agent sidecar not ready for the first 3 attempts
+        var profile = new AppConfigAgentProfile
+        {
+            BaseUrl = "http://localhost:2772",
+            ApplicationId = "test-app",
+            EnvironmentId = "test-env",
+            ProfileId = "test-profile",
+            IsFeatureFlag = false,
+            ReloadAfterSeconds = null // Disable reload timer for test isolation
+        };
+
+        var configJson = """{"Cbms": {"UseMockResponses": true}}""";
+        var handler = new FailThenSucceedHandler(
+            failCount: 3,
+            successResponse: new HttpResponseMessage(HttpStatusCode.OK)
+            {
+                Content = new StringContent(configJson, Encoding.UTF8, "application/json")
+            });
+
+        using var httpClient = new HttpClient(handler);
+        var provider = new AppConfigAgentConfigurationProvider(httpClient, profile, _logger, ownsHttpClient: false);
+
+        // Act
+        provider.Load();
+
+        // Assert — config should be loaded after retries
+        Assert.True(provider.TryGet("Cbms:UseMockResponses", out var value));
+        Assert.Equal("true", value);
+        Assert.Equal(4, handler.CallCount); // 3 failures + 1 success
+
+        provider.Dispose();
+    }
+
+    [Fact]
+    public void Load_InitialLoad_ShouldGiveUpAfterMaxRetries()
+    {
+        // Arrange — agent never becomes available
+        var profile = new AppConfigAgentProfile
+        {
+            BaseUrl = "http://localhost:2772",
+            ApplicationId = "test-app",
+            EnvironmentId = "test-env",
+            ProfileId = "test-profile",
+            IsFeatureFlag = false,
+            ReloadAfterSeconds = null
+        };
+
+        var handler = new FailThenSucceedHandler(
+            failCount: 100, // More than max retries
+            successResponse: new HttpResponseMessage(HttpStatusCode.OK));
+
+        using var httpClient = new HttpClient(handler);
+        var provider = new AppConfigAgentConfigurationProvider(httpClient, profile, _logger, ownsHttpClient: false);
+
+        // Act — should not throw
+        var exception = Record.Exception(() => provider.Load());
+
+        // Assert
+        Assert.Null(exception);
+        Assert.False(provider.TryGet("any-key", out _)); // No config loaded
+        Assert.Equal(10, handler.CallCount); // Should have tried exactly 10 times (max retries)
+
+        provider.Dispose();
+    }
+
+    [Fact]
+    public void Load_SubsequentReloads_ShouldNotRetryOnFailure()
+    {
+        // Arrange — first load succeeds, second load (simulating a reload) should not retry
+        var profile = new AppConfigAgentProfile
+        {
+            BaseUrl = "http://localhost:2772",
+            ApplicationId = "test-app",
+            EnvironmentId = "test-env",
+            ProfileId = "test-profile",
+            IsFeatureFlag = false,
+            ReloadAfterSeconds = null
+        };
+
+        var configJson = """{"Key1": "value1"}""";
+        // First call succeeds, then all subsequent calls fail
+        var handler = new FailThenSucceedHandler(
+            failCount: 0,
+            successResponse: new HttpResponseMessage(HttpStatusCode.OK)
+            {
+                Content = new StringContent(configJson, Encoding.UTF8, "application/json")
+            },
+            failAfterSuccess: true);
+
+        using var httpClient = new HttpClient(handler);
+        var provider = new AppConfigAgentConfigurationProvider(httpClient, profile, _logger, ownsHttpClient: false);
+
+        // Act — initial load succeeds
+        provider.Load();
+        Assert.True(provider.TryGet("Key1", out var value));
+        Assert.Equal("value1", value);
+        Assert.Equal(1, handler.CallCount);
+
+        // Act — subsequent load (reload) fails, should only try once (no retry)
+        provider.Load();
+
+        // Assert — only 2 total calls (1 initial + 1 reload), no retries on reload
+        Assert.Equal(2, handler.CallCount);
+
+        provider.Dispose();
+    }
+
+    /// <summary>
+    /// Test handler that throws HttpRequestException for the first N requests,
+    /// then returns a success response. Simulates the AppConfig Agent sidecar
+    /// startup race condition.
+    /// </summary>
+    private sealed class FailThenSucceedHandler : HttpMessageHandler
+    {
+        private readonly int _failCount;
+        private readonly HttpResponseMessage _successResponse;
+        private readonly bool _failAfterSuccess;
+        private int _callCount;
+
+        public int CallCount => _callCount;
+
+        public FailThenSucceedHandler(
+            int failCount,
+            HttpResponseMessage successResponse,
+            bool failAfterSuccess = false)
+        {
+            _failCount = failCount;
+            _successResponse = successResponse;
+            _failAfterSuccess = failAfterSuccess;
+        }
+
+        protected override Task<HttpResponseMessage> SendAsync(
+            HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            var currentCall = Interlocked.Increment(ref _callCount);
+
+            if (currentCall <= _failCount)
+            {
+                throw new HttpRequestException(
+                    "Connection refused (localhost:2772)",
+                    new System.Net.Sockets.SocketException(111));
+            }
+
+            if (_failAfterSuccess && currentCall > _failCount + 1)
+            {
+                throw new HttpRequestException(
+                    "Connection refused (localhost:2772)",
+                    new System.Net.Sockets.SocketException(111));
+            }
+
+            return Task.FromResult(_successResponse);
+        }
+    }
+
     public void Dispose()
     {
         _httpClient?.Dispose();