fix: Dolt test suite stability — eliminate container crashes and cascading failures (hq-33riy)

DROP DATABASE during test cleanup crashed the Dolt container, causing 189/255
tests to fail with "connection refused". Fixed by:

- Skip DROP DATABASE in test cleanup (container terminates orphans)
- Add concurrency semaphore (size 2) acquired after t.Parallel() with
  t.Cleanup release to prevent deadlocks
- Bypass circuit breaker in BEADS_TEST_MODE (prevents cascade trips)
- Unset BEADS_TEST_MODE in circuit breaker unit tests for real behavior
- Limit concurrent test stores to MaxOpenConns=2 with MaxIdleConns=0
- Fix cross-type DepBlocks tests (epic→task disallowed by GH#1495)

Result: 255/255 pass in ~27s (was 66/255 with 300s timeout).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: steveyegge

internal/storage/dolt/circuit.go

@@ -73,6 +73,13 @@ func newCircuitBreaker(port int) *circuitBreaker {
 // If the probe succeeds, the breaker resets to closed immediately. This
 // avoids the half-open→open re-trip race that can leave the breaker stuck.
 func (cb *circuitBreaker) Allow() bool {
+	// In test mode, bypass the circuit breaker entirely. Tests manage their
+	// own server lifecycle via testcontainers, and the file-based breaker
+	// state persists across test runs causing cascading false-positive trips.
+	if os.Getenv("BEADS_TEST_MODE") == "1" {
+		return true
+	}
+
 	cb.mu.Lock()
 	defer cb.mu.Unlock()
 

internal/storage/dolt/circuit_test.go

@@ -17,6 +17,7 @@ func TestCircuitBreaker_InitiallyAllows(t *testing.T) {
 }
 
 func TestCircuitBreaker_TripsAfterThreshold(t *testing.T) {
+	t.Setenv("BEADS_TEST_MODE", "") // need real breaker behavior
 	cb := newTestCircuitBreaker(t)
 
 	// Record failures up to threshold
@@ -74,6 +75,7 @@ func TestCircuitBreaker_SuccessResets(t *testing.T) {
 }
 
 func TestCircuitBreaker_ActiveProbeAfterCooldown_NoServer(t *testing.T) {
+	t.Setenv("BEADS_TEST_MODE", "")
 	cb := newTestCircuitBreaker(t)
 
 	// Trip the breaker
@@ -101,6 +103,7 @@ func TestCircuitBreaker_ActiveProbeAfterCooldown_NoServer(t *testing.T) {
 }
 
 func TestCircuitBreaker_ActiveProbeAfterCooldown_ServerUp(t *testing.T) {
+	t.Setenv("BEADS_TEST_MODE", "")
 	// Start a TCP listener to simulate a healthy server
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
@@ -133,6 +136,7 @@ func TestCircuitBreaker_ActiveProbeAfterCooldown_ServerUp(t *testing.T) {
 }
 
 func TestCircuitBreaker_LegacyHalfOpenState(t *testing.T) {
+	t.Setenv("BEADS_TEST_MODE", "")
 	// If a state file has half-open from an older version, the breaker
 	// should handle it gracefully via active probe.
 	ln, err := net.Listen("tcp", "127.0.0.1:0")
@@ -182,6 +186,7 @@ func TestCircuitBreaker_Reset(t *testing.T) {
 }
 
 func TestCircuitBreaker_SharedState(t *testing.T) {
+	t.Setenv("BEADS_TEST_MODE", "")
 	// Two breakers for the same port should share state via the file
 	dir := t.TempDir()
 	path := filepath.Join(dir, "circuit.json")

internal/storage/dolt/create_guard_test.go

@@ -95,10 +95,10 @@ func createTestDatabase(t *testing.T, port int, dbName string) {
 }
 
 func dropTestDatabase(t *testing.T, port int, dbName string) {
-	t.Helper()
-	db := rawTestConn(t, port)
-	defer db.Close()
-	db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName)) //nolint:errcheck
+	// No-op: rapid DROP DATABASE crashes the Dolt test container.
+	// Orphan databases are cleaned up when the container terminates.
+	_ = port
+	_ = dbName
 }
 
 func containsAny(s string, substrs ...string) bool {
@@ -112,11 +112,14 @@ func containsAny(s string, substrs ...string) bool {
 }
 
 // skipIfNoServer skips the test if the shared test Dolt server is not running.
+// Acquires a semaphore slot (released via t.Cleanup) to limit container load.
 func skipIfNoServer(t *testing.T) {
 	t.Helper()
 	if testServerPort == 0 {
 		t.Skip("no test Dolt server running")
 	}
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 }
 
 // --- Guard tests ---

internal/storage/dolt/cross_project_test.go

@@ -33,6 +33,8 @@ import (
 func setupTwoProjectStores(t *testing.T, prefixA, prefixB string) (storeA, storeB *DoltStore, cleanup func()) {
 	t.Helper()
 	skipIfNoDolt(t)
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 
 	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
 	defer cancel()
@@ -96,10 +98,8 @@ func setupTwoProjectStores(t *testing.T, prefixA, prefixB string) (storeA, store
 	}
 
 	cleanup = func() {
-		dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer dropCancel()
-		_, _ = storeA.db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbNameA))
-		_, _ = storeB.db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbNameB))
+		// Skip DROP DATABASE — rapid CREATE/DROP cycles crash the Dolt container.
+		// Orphan databases are cleaned up when the container terminates.
 		storeA.Close()
 		storeB.Close()
 		os.RemoveAll(tmpDirA)
@@ -201,6 +201,8 @@ func TestCrossProject_ReadIsolation_DifferentPrefixes(t *testing.T) {
 
 func TestCrossProject_PortCollision_SameDatabase(t *testing.T) {
 	skipIfNoDolt(t)
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 
 	ctx, cancel := concurrentTestContext(t)
 	defer cancel()
@@ -261,9 +263,7 @@ func TestCrossProject_PortCollision_SameDatabase(t *testing.T) {
 	}
 
 	defer func() {
-		dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer dropCancel()
-		_, _ = storeA.db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sharedDB))
+		// Skip DROP DATABASE — rapid CREATE/DROP cycles crash the Dolt container.
 		storeA.Close()
 		storeB.Close()
 		os.RemoveAll(tmpDirA)

internal/storage/dolt/dependencies_extended_test.go

@@ -287,7 +287,7 @@ func TestGetDependencyTree(t *testing.T) {
 	defer cancel()
 
 	// Create a simple tree: root -> child1, root -> child2
-	root := &types.Issue{ID: "tree-root", Title: "Root", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeEpic}
+	root := &types.Issue{ID: "tree-root", Title: "Root", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask}
 	child1 := &types.Issue{ID: "tree-child1", Title: "Child 1", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
 	child2 := &types.Issue{ID: "tree-child2", Title: "Child 2", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}
 

internal/storage/dolt/dolt_test.go

@@ -25,6 +25,21 @@ import (
 // driver's async operations rather than with the DoltStore implementation.
 const testTimeout = 30 * time.Second
 
+// testSem limits concurrent database-touching tests to avoid overwhelming the
+// shared Dolt testcontainer. Without this, 200+ parallel tests cause a
+// connection storm that crashes the container. Size 6 is conservative —
+// the container handles ~10 concurrent connections but we leave headroom.
+var testSem = make(chan struct{}, 2)
+
+// acquireTestSlot blocks until a semaphore slot is available.
+// Must be called AFTER t.Parallel() to avoid deadlocks (t.Parallel suspends
+// the goroutine, and if it holds a semaphore slot while suspended, other
+// tests can't proceed).
+func acquireTestSlot() { testSem <- struct{}{} }
+
+// releaseTestSlot returns a semaphore slot.
+func releaseTestSlot() { <-testSem }
+
 // testContext returns a context with timeout for test operations
 func testContext(t *testing.T) (context.Context, context.CancelFunc) {
 	t.Helper()
@@ -68,6 +83,11 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) {
 	skipIfNoDolt(t)
 	t.Parallel()
 
+	// Acquire AFTER t.Parallel() to avoid deadlock — t.Parallel() suspends
+	// the goroutine until the parent test finishes its sequential phase.
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot) // guaranteed even on panic/Fatalf
+
 	if testSharedDB == "" {
 		t.Fatal("testSharedDB not set — TestMain did not initialize shared database")
 	}
@@ -119,9 +139,12 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) {
 // setupConcurrentTestStore creates a test store with its own database for
 // concurrent tests that need multiple connections. Branch-per-test isolation
 // requires MaxOpenConns=1, which prevents concurrent transactions.
+// MaxOpenConns is limited to 3 to avoid overwhelming the test container.
 func setupConcurrentTestStore(t *testing.T) (*DoltStore, func()) {
 	t.Helper()
 	skipIfNoDolt(t)
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 
 	ctx, cancel := testContext(t)
 	defer cancel()
@@ -138,6 +161,7 @@ func setupConcurrentTestStore(t *testing.T) (*DoltStore, func()) {
 		CommitterName:   "test",
 		CommitterEmail:  "test@example.com",
 		Database:        dbName,
+		MaxOpenConns:    2,    // limit to avoid overwhelming the test container
 		CreateIfMissing: true, // tests create fresh databases
 	}
 
@@ -146,6 +170,9 @@ func setupConcurrentTestStore(t *testing.T) (*DoltStore, func()) {
 		os.RemoveAll(tmpDir)
 		t.Fatalf("failed to create Dolt store: %v", err)
 	}
+	// Close idle connections immediately to reduce container pressure.
+	store.db.SetMaxIdleConns(0)
+	store.db.SetConnMaxIdleTime(time.Second)
 
 	if err := store.SetConfig(ctx, "issue_prefix", "test"); err != nil {
 		store.Close()
@@ -154,9 +181,8 @@ func setupConcurrentTestStore(t *testing.T) (*DoltStore, func()) {
 	}
 
 	cleanup := func() {
-		dropCtx, dropCancel := context.WithTimeout(context.Background(), 5*time.Second)
-		defer dropCancel()
-		_, _ = store.db.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName))
+		// Skip DROP DATABASE — rapid CREATE/DROP cycles crash the Dolt container.
+		// Orphan databases are cleaned up when the container is terminated.
 		store.Close()
 		os.RemoveAll(tmpDir)
 	}

internal/storage/dolt/git_remote_test.go

@@ -622,6 +622,8 @@ func setupEmbeddedGitRemote(t *testing.T) (*DoltStore, *gitRemoteSetup, func())
 	t.Helper()
 	skipIfNoDolt(t)
 	skipIfNoGit(t)
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 
 	baseDir, err := os.MkdirTemp("", "embedded-git-remote-test-*")
 	if err != nil {

internal/storage/dolt/queries_test.go

@@ -616,7 +616,7 @@ func TestGetBlockedIssues_IncludesChildrenOfBlockedParents(t *testing.T) {
 		Title:     "Prerequisite",
 		Status:    types.StatusOpen,
 		Priority:  1,
-		IssueType: types.TypeTask,
+		IssueType: types.TypeEpic, // must match blocked type for DepBlocks (GH#1495)
 	}
 	epic := &types.Issue{
 		ID:        "bi-epic",

internal/storage/dolt/store_unit_test.go

@@ -19,6 +19,8 @@ func newTestDoltDB(t *testing.T) (*sql.DB, func()) {
 	if testServerPort == 0 {
 		t.Skip("Test Dolt server not running, skipping test")
 	}
+	acquireTestSlot()
+	t.Cleanup(releaseTestSlot)
 
 	dbName := uniqueTestDBName(t)
 
@@ -41,11 +43,8 @@ func newTestDoltDB(t *testing.T) (*sql.DB, func()) {
 
 	return db, func() {
 		db.Close()
-		cleanup, cErr := sql.Open("mysql", adminDSN)
-		if cErr == nil {
-			cleanup.Exec("DROP DATABASE IF EXISTS `" + dbName + "`")
-			cleanup.Close()
-		}
+		// Skip DROP DATABASE — rapid CREATE/DROP cycles crash the Dolt container.
+		// Orphan databases are cleaned up when the container terminates.
 	}
 }