NCBC-4128: Clear dead nodes after select bucket failure

Motivation
----------
When attempting to reuses a globally bootstrapped node and select a
bucket so it becomes a bucket-specific node, if the select bucket call
fails the node may have all connections dead. Because the select bucket
call goes through a special path to all connections in the pool, it can
then continue to fail because the dead connection is still there and
will fail on the next attempt.

Modifications
-------------
- Don't attempt to select bucket on dead connections in the connection
  pool
- Throw an error if there are no healthy connections in the pool when
  select bucket is attempted
- On a general exception calling select bucket, discard the node so the
  next attempt will use a fresh connection pool

Results
-------
Greater resiliency in cases where the cluster isn't fully accessible
during bootstrap and the first call to BucketAsync fails. This prevents
the Cluster from being in a bad state until the application is recycled.

Change-Id: I2db1e2f6653ef604f6d3e03405e99a1264ae486d
Reviewed-on: https://review.couchbase.org/c/couchbase-net-client/+/239143
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Jeffry Morris <jeffrymorris@gmail.com>

Author: brantburnett

src/Couchbase/Core/ClusterContext.cs

@@ -19,7 +19,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.ExceptionServices;
-using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -629,6 +628,19 @@ public async Task<BucketBase> CreateAndBootStrapBucketAsync(string name, HostEnd
                 config = await _httpClusterMap.GetClusterMapAsync(
                     name, node.EndPoint, CancellationToken.None).ConfigureAwait(false);
             }
+            catch
+            {
+                // The ClusterNode may be in a bad state, such as all connections being unhealthy, so we'll
+                // never be able to reuse this node for a later SelectBucketAsync call. Clean up so the
+                // next attempt can succeed with a fresh node.
+                if (!newNode)
+                {
+                    RemoveNode(node);
+                }
+
+                node.Dispose();
+                throw;
+            }
 
             //Determine the bucket type to create based off the bucket capabilities
             var type = config.BucketCapabilities.Contains("cccp") ? BucketType.Couchbase : BucketType.Memcached;

src/Couchbase/Core/IO/Connections/ConnectionPoolBase.cs

@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Couchbase.Core.Exceptions.KeyValue;
 using Couchbase.Core.IO.Operations;
 using Couchbase.Utils;
 using Microsoft.Extensions.Logging;
@@ -119,6 +120,7 @@ public virtual async Task SelectBucketAsync(string name, CancellationToken cance
             await using ((await FreezePoolAsync(cancellationToken).ConfigureAwait(false)).ConfigureAwait(false))
             {
                 var tasks = GetConnections()
+                    .Where(p => !p.IsDead)
                     .Select(connection => Task.Run(async () =>
                     {
                         try
@@ -136,6 +138,12 @@ await _connectionInitializer.SelectBucketAsync(connection, name, cancellationTok
                     }, cancellationToken))
                     .ToList();
 
+                if (tasks.Count == 0)
+                {
+                    // There are no healthy connections, so we cannot select the bucket
+                    throw new DocumentNotFoundException($"Failed to select bucket {BucketName}.");
+                }
+
                 await Task.WhenAll(tasks).ConfigureAwait(false);
 
                 BucketName = name;