NCBC-4201: Fix Stellar HTTP/2 retries and strict timeout enforcement

Motivation
==========
Stellar operations lacked proper timeout enforcement and often suffered
unbounded retry loops. Additionally, transient HTTP/2 transport errors
(like connection resets) surfaced as fatal internal errors instead of
being safely retried.

Modifications
============
- Enabled multiple HTTP/2 connections in StellarCluster to recover
  cleanly from dead connections.
- Enforced strict timeouts via dynamically shrinking `RemainingTimeout`
  in GrpcCallOptions, replacing broken retry-limit checks.
- Delegated timeout enforcement entirely to `DeadlineExceeded`.
- Mapped transient `HttpRequestException` and `IOException` transport
  errors to `ServiceNotAvailable` to enable graceful retries.

Result
======
Transport resets now retry safely with backoff, matching other SDKs.
Initial requests and all retries strictly enforce their timeouts.

Change-Id: Id86e824769965f3985b677e236d57897e62f2f33
Reviewed-on: https://review.couchbase.org/c/couchbase-net-client/+/244798
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Jeffry Morris <jeffrymorris@gmail.com>

Author: davidkelly

src/Couchbase/Stellar/Core/Retry/StellarRequest.cs

@@ -13,12 +13,40 @@ namespace Couchbase.Stellar.Core.Retry;
  */
 public class StellarRequest : IRequest
 {
+    private readonly TimeProvider _timeProvider;
+
+    public StellarRequest() : this(TimeProvider.System) { }
+
+    /// <summary>
+    /// Creates a StellarRequest with a custom TimeProvider for deterministic testing.
+    /// </summary>
+    internal StellarRequest(TimeProvider timeProvider)
+    {
+        _timeProvider = timeProvider;
+        CreatedAt = _timeProvider.GetUtcNow().UtcDateTime;
+    }
+
     public uint Attempts { get; set; }
     public bool Idempotent { get; set; }
     public List<RetryReason> RetryReasons { get; set; } = new();
     public IRetryStrategy RetryStrategy { get; set; } = new BestEffortRetryStrategy();
     public TimeSpan Timeout { get; set; }
     public TimeSpan Elapsed { get; }
+
+    /// <summary>
+    /// The time this request was created. Used to compute remaining timeout.
+    /// </summary>
+    public DateTime CreatedAt { get; }
+
+    /// <summary>
+    /// Returns the remaining time before this request should time out,
+    /// or null if no timeout was set (Timeout is zero).
+    /// Used to set shrinking gRPC deadlines on each retry attempt.
+    /// </summary>
+    public TimeSpan? RemainingTimeout =>
+        Timeout > TimeSpan.Zero
+            ? Timeout - (_timeProvider.GetUtcNow().UtcDateTime - CreatedAt)
+            : null;
     public CancellationToken Token { get; set; }
     public string? ClientContextId { get; set; }
     public string? Statement { get; set; }
@@ -40,5 +68,4 @@ public void LogOrphaned()
     }
 
     public GenericErrorContext Context { get; set; } = new();
-    public int MaxRetryAttempts { get; set; } = 10;
 }

src/Couchbase/Stellar/Core/Retry/StellarRetryHandler.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Couchbase.Core;
 using Couchbase.Core.Exceptions;
@@ -19,9 +20,21 @@ namespace Couchbase.Stellar.Core.Retry;
 
 internal class StellarRetryHandler : IRetryOrchestrator
 {
+    private readonly TimeProvider _timeProvider;
+
+    public StellarRetryHandler() : this(TimeProvider.System) { }
+
+    /// <summary>
+    /// Creates a StellarRetryHandler with a custom TimeProvider for deterministic testing.
+    /// </summary>
+    internal StellarRetryHandler(TimeProvider timeProvider)
+    {
+        _timeProvider = timeProvider;
+    }
+
     public async Task<T> RetryAsync<T>(Func<Task<T>> send, IRequest request) where T : IServiceResult
     {
-        var backoff = ControlledBackoff.Create();
+        var backoff = ControlledBackoff.Create(_timeProvider);
         var context = new GenericErrorContext();
 
         while (true)
@@ -48,6 +61,15 @@ public async Task<T> RetryAsync<T>(Func<Task<T>> send, IRequest request) where T
                     await backoff.Delay(request).ConfigureAwait(false);
                 }
             }
+            catch (Exception e) when (IsTransientTransportException(e))
+            {
+                // HTTP/2 transport failures (e.g. connection reset, broken pipe) can
+                // surface as HttpRequestException or IOException rather than RpcException.
+                // Treat these like Unavailable and retry.
+                context.RetryReasons.Add(RetryReason.ServiceNotAvailable);
+                request.Attempts++;
+                await backoff.Delay(request).ConfigureAwait(false);
+            }
         }
     }
 
@@ -205,6 +227,15 @@ private void HandleException(RpcException protoException, IRequest request, Gene
                 if (request.Idempotent) throw new UnambiguousTimeoutException(detail);
                 throw new AmbiguousTimeoutException();
             case StatusCode.Internal:
+                if (IsTransientGrpcTransportError(protoException))
+                {
+                    // HTTP/2 connection failures surface as StatusCode.Internal in .NET's
+                    // gRPC library. These are transient transport issues, not server errors.
+                    // Retry them like Unavailable, matching Java SDK behavior.
+                    context.RetryReasons.Add(RetryReason.ServiceNotAvailable);
+                    request.Attempts++;
+                    break;
+                }
                 throw new InternalServerFailureException(detail);
             case StatusCode.Unavailable:
                 context.RetryReasons.Add(RetryReason.ServiceNotAvailable);
@@ -253,4 +284,32 @@ public Task<ResponseStatus> RetryAsync(BucketBase bucket, IOperation operation,
     {
         throw new NotImplementedException();
     }
+
+    /// <summary>
+    /// Determines whether a gRPC Internal status error is actually a transient transport
+    /// failure (e.g. HTTP/2 connection reset) rather than a genuine server-side error.
+    /// In production, Grpc.Net.Client wraps transport exceptions (HttpRequestException,
+    /// IOException) as the InnerException of the RpcException.
+    /// </summary>
+    private static bool IsTransientGrpcTransportError(RpcException ex)
+    {
+        return ex.InnerException is not null && IsTransientTransportException(ex.InnerException);
+    }
+
+    /// <summary>
+    /// Determines whether an exception (or any exception in its InnerException chain)
+    /// represents a transient transport failure that should be retried.
+    /// </summary>
+    private static bool IsTransientTransportException(Exception? e)
+    {
+        while (e is not null)
+        {
+            if (e is HttpRequestException or System.IO.IOException)
+            {
+                return true;
+            }
+            e = e.InnerException;
+        }
+        return false;
+    }
 }