test(multidb): e2e failover tests via per-member RESP proxies

.github/workflows/test-e2e.yml

@@ -60,3 +60,60 @@ jobs:
           echo "=== proxy-fault-injector logs ==="
           docker logs proxy-fault-injector 2>&1 | tail -100
 
+  test-multidb-e2e:
+    name: MultiDB E2E Tests (Per-Member Proxies)
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version:
+          - stable
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+
+      - name: Set up Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v7
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Start Docker services for MultiDB E2E tests
+        run: docker compose --profile multidb up -d
+
+      - name: Wait for services to be ready
+        run: |
+          echo "Waiting for Redis to be ready..."
+          timeout 30 bash -c 'until docker exec redis-standalone redis-cli ping 2>/dev/null; do sleep 1; done'
+          echo "Waiting for the per-member proxies to be ready..."
+          # 127.0.0.1, not localhost: the e2e harness dials 127.0.0.1 because
+          # IPv6-first hosts resolve localhost to ::1 and can miss Docker's
+          # IPv4-published ports — the readiness probe must match.
+          for port in 18120 18121 18122; do
+            timeout 30 bash -c "until curl -s http://127.0.0.1:$port/stats > /dev/null; do sleep 1; done"
+          done
+          echo "All services are ready!"
+
+      - name: Run MultiDB E2E tests
+        env:
+          E2E_MULTIDB_TESTS: "true"
+        run: |
+          go test -v -race -count=1 -timeout 15m ./multidb/e2e/...
+        continue-on-error: false
+
+      # Logs BEFORE the compose down below: `down` removes the containers,
+      # which would leave the diagnostics step with nothing to read.
+      - name: Show Docker logs on failure
+        if: failure()
+        run: |
+          echo "=== Redis logs ==="
+          docker logs redis-standalone 2>&1 | tail -100
+          for c in cae-proxy-db0 cae-proxy-db1 cae-proxy-db2; do
+            echo "=== $c logs ==="
+            docker logs "$c" 2>&1 | tail -100
+          done
+
+      - name: Stop Docker services
+        if: always()
+        run: docker compose --profile multidb down
+

Makefile

@@ -122,6 +122,17 @@ test.e2e.docker:
 	$(MAKE) docker.e2e.stop
 	@echo "Docker E2E tests completed!"
 
+test.multidb.e2e:
+	@echo "Running MultiDB e2e tests (per-member proxies)..."
+	docker compose --profile multidb up -d || (docker compose --profile multidb down && exit 1)
+	@echo "Waiting for the per-member proxies to be ready..."
+	@for port in 18120 18121 18122; do \
+		timeout 30 bash -c "until curl -s http://127.0.0.1:$$port/stats > /dev/null; do sleep 1; done" || { docker compose --profile multidb down; exit 1; }; \
+	done
+	@E2E_MULTIDB_TESTS=true go test -v -race -count=1 -timeout 15m ./multidb/e2e/... || (docker compose --profile multidb down && exit 1)
+	docker compose --profile multidb down
+	@echo "MultiDB e2e tests completed!"
+
 test.e2e.logic:
 	@echo "Running E2E logic tests (no proxy required)..."
 	@E2E_SCENARIO_TESTS=true \
@@ -130,7 +141,7 @@ test.e2e.logic:
 		go test -v -run "TestCreateTestFaultInjectorLogic|TestFaultInjectorClientCreation" ./maintnotifications/e2e/
 	@echo "Logic tests completed!"
 
-.PHONY: all test test.ci test.ci.skip-vectorsets test.autopipeline-subjects bench fmt test.e2e test.e2e.logic docker.e2e.start docker.e2e.stop
+.PHONY: all test test.ci test.ci.skip-vectorsets test.autopipeline-subjects bench fmt test.e2e test.e2e.logic test.multidb.e2e docker.e2e.start docker.e2e.stop
 
 build:
 	export RE_CLUSTER=$(RE_CLUSTER) && \

docker-compose.yml

@@ -26,6 +26,7 @@ services:
       - all-stack
       - all
       - e2e
+      - multidb
 
   osscluster:
     image: *default-image
@@ -44,6 +45,64 @@ services:
       - all-stack
       - all
 
+  # Per-member RESP proxies for MultiDB (Active-Active) e2e tests. Each proxy
+  # is one "member database" endpoint; all front the same target Redis so data
+  # written through one member is visible through the others (a converged
+  # CRDB). Faults are injected at the container level (docker stop/pause).
+  cae-proxy-db0:
+    image: redislabs/client-resp-proxy:latest
+    container_name: cae-proxy-db0
+    environment:
+      - TARGET_HOST=redis
+      - TARGET_PORT=6379
+      - LISTEN_PORT=17100
+      - LISTEN_HOST=0.0.0.0
+      - API_PORT=3000
+    ports:
+      - "17100:17100"
+      - "18120:3000"
+    depends_on:
+      - redis
+    profiles:
+      - multidb
+      - all
+
+  cae-proxy-db1:
+    image: redislabs/client-resp-proxy:latest
+    container_name: cae-proxy-db1
+    environment:
+      - TARGET_HOST=redis
+      - TARGET_PORT=6379
+      - LISTEN_PORT=17101
+      - LISTEN_HOST=0.0.0.0
+      - API_PORT=3000
+    ports:
+      - "17101:17101"
+      - "18121:3000"
+    depends_on:
+      - redis
+    profiles:
+      - multidb
+      - all
+
+  cae-proxy-db2:
+    image: redislabs/client-resp-proxy:latest
+    container_name: cae-proxy-db2
+    environment:
+      - TARGET_HOST=redis
+      - TARGET_PORT=6379
+      - LISTEN_PORT=17102
+      - LISTEN_HOST=0.0.0.0
+      - API_PORT=3000
+    ports:
+      - "17102:17102"
+      - "18122:3000"
+    depends_on:
+      - redis
+    profiles:
+      - multidb
+      - all
+
   cae-resp-proxy:
     image: redislabs/client-resp-proxy:latest
     container_name: cae-resp-proxy

multidb/e2e/harness_test.go

@@ -0,0 +1,191 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// memberProxy is one MultiDB member endpoint: a cae-resp-proxy container
+// fronting the shared target Redis.
+type memberProxy struct {
+	Container string
+	Addr      string
+}
+
+// proxyFarm drives the per-member proxy containers with docker CLI faults.
+type proxyFarm struct {
+	t       *testing.T
+	members []memberProxy
+}
+
+func newProxyFarm(t *testing.T) *proxyFarm {
+	t.Helper()
+	f := &proxyFarm{
+		t: t,
+		members: []memberProxy{
+			// 127.0.0.1, not localhost: docker publishes on IPv4, and hosts
+			// that resolve localhost to ::1 first would dial the wrong stack.
+			{Container: "cae-proxy-db0", Addr: "127.0.0.1:17100"},
+			{Container: "cae-proxy-db1", Addr: "127.0.0.1:17101"},
+			{Container: "cae-proxy-db2", Addr: "127.0.0.1:17102"},
+		},
+	}
+	// Whatever a test did, the next one starts from "everything running".
+	t.Cleanup(f.RestoreAll)
+	f.RestoreAll()
+	return f
+}
+
+func (f *proxyFarm) docker(args ...string) error {
+	// Bounded: a stuck docker daemon must fail the scenario, not hang the
+	// whole suite until the package timeout.
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	out, err := exec.CommandContext(ctx, "docker", args...).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("docker %v: %v: %s", args, err, out)
+	}
+	return nil
+}
+
+func (f *proxyFarm) Stop(i int) {
+	f.t.Helper()
+	if err := f.docker("stop", "-t", "0", f.members[i].Container); err != nil {
+		f.t.Fatalf("stop member %d: %v", i, err)
+	}
+}
+
+func (f *proxyFarm) Start(i int) {
+	f.t.Helper()
+	if err := f.docker("start", f.members[i].Container); err != nil {
+		f.t.Fatalf("start member %d: %v", i, err)
+	}
+	f.awaitListening(i, 30*time.Second)
+}
+
+func (f *proxyFarm) Pause(i int) {
+	f.t.Helper()
+	if err := f.docker("pause", f.members[i].Container); err != nil {
+		f.t.Fatalf("pause member %d: %v", i, err)
+	}
+}
+
+// RestoreAll brings every member back to a running, listening state.
+func (f *proxyFarm) RestoreAll() {
+	for i, m := range f.members {
+		// Only "is not paused" is benign for unpause (start below is a no-op
+		// when already running). A missing container means the compose
+		// profile is not up: fail fast instead of a 30s dial timeout per
+		// member. Any other unpause failure could leave the member frozen —
+		// a paused container still accepts TCP dials, so awaitListening
+		// would not catch it.
+		if err := f.docker("unpause", m.Container); err != nil {
+			switch {
+			case isMissingContainer(err):
+				f.t.Fatalf("proxy container %s does not exist — start the stack with `docker compose --profile multidb up -d`: %v", m.Container, err)
+			case !isNotPaused(err):
+				f.t.Fatalf("unpause %s: %v", m.Container, err)
+			}
+		}
+		if err := f.docker("start", m.Container); err != nil {
+			f.t.Fatalf("start %s: %v", m.Container, err)
+		}
+		f.awaitListening(i, 30*time.Second)
+	}
+}
+
+func isMissingContainer(err error) bool {
+	return strings.Contains(err.Error(), "No such container")
+}
+
+func isNotPaused(err error) bool {
+	return strings.Contains(err.Error(), "is not paused")
+}
+
+func (f *proxyFarm) awaitListening(i int, timeout time.Duration) {
+	f.t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", f.members[i].Addr, 250*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	f.t.Fatalf("member %d (%s) never came back up", i, f.members[i].Addr)
+}
+
+// fast timings so scenarios complete in seconds while staying CI-jitter safe.
+func fastMultiDBOptions(f *proxyFarm) *redis.MultiDBOptions {
+	return &redis.MultiDBOptions{
+		Clients: []redis.MultiDBClientConfig{
+			{Options: memberOptions(f, 0), Weight: 3},
+			{Options: memberOptions(f, 1), Weight: 2},
+			{Options: memberOptions(f, 2), Weight: 1},
+		},
+		// Every proxy must be genuinely healthy at startup: with the default
+		// majority policy a mis-wired member could slip through and scenarios
+		// that stop member 0 would silently test the wrong topology.
+		InitialDBState:      redis.InitialDBStateAllAvailable,
+		HealthCheckInterval: 500 * time.Millisecond,
+		HealthCheckTimeout:  250 * time.Millisecond,
+		CircuitBreakerConfig: &redis.MultiDBCircuitBreakerConfig{
+			FailureThreshold: 3,
+			SuccessThreshold: 1,
+			GracePeriod:      2 * time.Second,
+		},
+		CommandRetries:       3,
+		AutoFallbackInterval: 3 * time.Second,
+		MaxFailoverAttempts:  4,
+		FailoverAttemptDelay: 500 * time.Millisecond,
+	}
+}
+
+func memberOptions(f *proxyFarm, i int) *redis.Options {
+	return &redis.Options{
+		Addr:         f.members[i].Addr,
+		DialTimeout:  500 * time.Millisecond,
+		ReadTimeout:  time.Second,
+		WriteTimeout: time.Second,
+		// Fail fast inside a single command attempt so MultiDB's own retry
+		// and failover logic drives recovery, not the per-client retries.
+		MaxRetries: -1,
+		// Let the probe context cut socket waits short: without this a
+		// paused (hung) proxy stalls health checks for the full read
+		// timeout instead of the intended HealthCheckTimeout.
+		ContextTimeoutEnabled: true,
+	}
+}
+
+func newE2EClient(t *testing.T, opts *redis.MultiDBOptions) *redis.MultiDBClient {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	mdb, err := redis.NewMultiDBClient(ctx, opts)
+	if err != nil {
+		t.Fatalf("NewMultiDBClient: %v", err)
+	}
+	t.Cleanup(func() { _ = mdb.Close() })
+	return mdb
+}
+
+// eventually polls cond until it is true or the timeout elapses.
+func eventually(t *testing.T, timeout time.Duration, what string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}

multidb/e2e/main_test.go

@@ -0,0 +1,25 @@
+// Package e2e contains end-to-end tests for redis.MultiDBClient driven by
+// per-member RESP proxies (docker-compose profile "multidb"). Member outages
+// are injected at the container level: docker stop (connection reset /
+// refused) and docker pause (hung connections / timeouts).
+//
+// Run via `make test.multidb.e2e`, or manually:
+//
+//	docker compose --profile multidb up -d
+//	E2E_MULTIDB_TESTS=true go test -race ./multidb/e2e/...
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("E2E_MULTIDB_TESTS") != "true" {
+		// Silent gated skip: the suite only runs when explicitly requested
+		// (make test.multidb.e2e), and direct logging is against repository
+		// conventions.
+		os.Exit(0)
+	}
+	os.Exit(m.Run())
+}