fix(test): keep a slow template checkpoint from poisoning the Windows suite (#951)

The shared test-DB templates from #943 checkpoint their WAL under a 1-second context deadline inside a `sync.Once` that caches the error. On a cold windows-latest runner, one checkpoint that misses the deadline permanently poisons the template, and every subsequent test in that binary fails with the same cached `checkpointing db template: wal checkpoint truncate: context deadline exceeded`. That is what happened on #948's CI run — a frontend-only diff — where the `cmd/agentsview` and `internal/db` binaries each hit the deadline once and 599 tests failed in cascade, while the identical SHA passed on main's push run.

Two changes, by blast radius:

- The two base templates (`internal/dbtest/dbtest.go`, `internal/db/db_test.go`) get a best-effort checkpoint with a generous once-per-binary deadline: the copy step already carries the `-wal`/`-shm` files along, so an uncheckpointed template is still a consistent database, and losing the checkpoint only costs compactness. Independently, a failed template build now degrades to creating the per-test database from scratch (the pre-#943 path) instead of failing the test — the template is a setup-cost optimization and should never be a single point of failure for the whole binary.
- The five seeded fixture builders (store contract, chunked analytics, daily usage, server analytics, cmd stats golden) keep their fatal checkpoint semantics — their consumers' copy semantics vary — but move from the 1s deadline to the same 30s one, which is effectively free since each runs once.

Two regression tests cover the new fallback, including cleanup of a partial template copy left behind by a mid-copy failure and the existing-file short-circuit.

Not addressed here: the unrelated `TestEnsureBackgroundServeLaunchLoser…` "server did not become ready within 2s" readiness flake that hit #949's run.

Where to look: `buildTestDBTemplate` in `internal/dbtest/dbtest.go` and `copyTestDBTemplate`/`openTestDBWithTemplate` in `internal/db/db_test.go`.


Co-authored-by: Matthew Jacobs <mjacobs@users.noreply.github.com>

Author: mjacobs

internal/db/db_test.go

@@ -351,8 +351,23 @@ func TestMain(m *testing.M) {
 }
 
 func openCopiedTestDB(path string) (*DB, error) {
-	if err := copyTestDBTemplate(path); err != nil {
-		return nil, err
+	return openTestDBWithTemplate(path, copyTestDBTemplate)
+}
+
+func openTestDBWithTemplate(
+	path string, copyTemplate func(string) error,
+) (*DB, error) {
+	if err := copyTemplate(path); err != nil {
+		// The shared template is only a setup-cost optimization.
+		// Never let a template failure poison every test in the
+		// binary; build this database from scratch instead.
+		fmt.Fprintf(os.Stderr,
+			"db test: template unavailable, creating %s from scratch: %v\n",
+			path, err)
+		for _, suffix := range []string{"", "-wal", "-shm"} {
+			_ = os.Remove(path + suffix)
+		}
+		return Open(path)
 	}
 	return OpenPreparedTestDB(path)
 }
@@ -380,13 +395,19 @@ func copyTestDBTemplate(dst string) error {
 			)
 			return
 		}
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		// Checkpointing keeps the copied template compact, but it
+		// is best-effort: the copy below carries the -wal/-shm
+		// files along, so a checkpoint that cannot finish in time
+		// (slow Windows CI runners) must not fail the build. The
+		// timeout is generous because this runs once per binary.
+		ctx, cancel := context.WithTimeout(
+			context.Background(), 30*time.Second,
+		)
 		defer cancel()
 		if err := template.CheckpointWALTruncate(ctx); err != nil {
-			testDBTemplateErr = fmt.Errorf(
-				"checkpointing db template: %w",
-				err,
-			)
+			fmt.Fprintf(os.Stderr,
+				"db test: template wal checkpoint failed, copying wal as-is: %v\n",
+				err)
 		}
 		if err := template.Close(); err != nil && testDBTemplateErr == nil {
 			testDBTemplateErr = fmt.Errorf(

internal/db/store_contract_test.go

@@ -105,7 +105,7 @@ func storeContractSQLiteTemplate(t *testing.T) (string, storeContractFixture) {
 			return
 		}
 		storeContractSQLiteTemplateFixture = seedStoreContractSQLite(t, d)
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		defer cancel()
 		storeContractSQLiteTemplateErr = d.CheckpointWALTruncate(ctx)
 		if closeErr := d.Close(); storeContractSQLiteTemplateErr == nil {

internal/db/template_fallback_test.go

@@ -0,0 +1,31 @@
+package db
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOpenTestDBWithTemplateFallsBackWhenTemplateUnavailable(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "test.db")
+
+	// Simulate a poisoned template build that also left a partial
+	// copy behind: the fallback must discard it and open a fresh,
+	// fully migrated database instead of failing the test.
+	d, err := openTestDBWithTemplate(path, func(dst string) error {
+		require.NoError(t, os.WriteFile(dst, []byte("garbage"), 0o600),
+			"writing partial template copy")
+		return errors.New("template poisoned")
+	})
+	require.NoError(t, err, "fallback open")
+	t.Cleanup(func() { require.NoError(t, d.Close()) })
+
+	var sessions int
+	require.NoError(t,
+		d.getReader().QueryRow("SELECT COUNT(*) FROM sessions").Scan(&sessions),
+		"querying sessions on fallback db")
+	require.Zero(t, sessions, "fresh fallback db should be empty")
+}

internal/dbtest/dbtest.go

@@ -94,6 +94,13 @@ func OpenTestDBAt(t *testing.T, path string) *db.DB {
 // does not already exist. Existing files are left intact so callers can reopen
 // and add more fixture rows without losing earlier writes.
 func EnsureTestDBAt(t *testing.T, path string) {
+	t.Helper()
+	ensureTestDBAtWith(t, path, copyTestDBTemplate)
+}
+
+func ensureTestDBAtWith(
+	t *testing.T, path string, copyTemplate func(string) error,
+) {
 	t.Helper()
 	_, err := os.Stat(path)
 	if err == nil {
@@ -102,8 +109,22 @@ func EnsureTestDBAt(t *testing.T, path string) {
 	if !errors.Is(err, os.ErrNotExist) {
 		t.Fatalf("checking test db %s: %v", path, err)
 	}
-	if err := copyTestDBTemplate(path); err != nil {
-		t.Fatalf("copying test db template: %v", err)
+	if err := copyTemplate(path); err != nil {
+		// The shared template is only a setup-cost optimization.
+		// Never let a template failure poison every test in the
+		// binary; build this database from scratch instead.
+		t.Logf("test db template unavailable, creating %s from scratch: %v",
+			path, err)
+		for _, suffix := range []string{"", "-wal", "-shm"} {
+			_ = os.Remove(path + suffix)
+		}
+		d, openErr := db.Open(path)
+		if openErr != nil {
+			t.Fatalf("creating test db from scratch: %v", openErr)
+		}
+		if closeErr := d.Close(); closeErr != nil {
+			t.Fatalf("closing scratch test db: %v", closeErr)
+		}
 	}
 }
 
@@ -147,18 +168,19 @@ func buildTestDBTemplate() (map[string][]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("opening db template: %w", err)
 	}
-	// The deadline only guards against a hung checkpoint: flushing
-	// the schema-creation WAL can take well over a second on slow
-	// Windows CI disks under parallel package load.
+	// Checkpointing keeps the copied template compact, but it is best-effort:
+	// the copy below carries the -wal/-shm files along, so a checkpoint that
+	// cannot finish in time must not fail the build. The generous deadline only
+	// guards against a hung checkpoint on slow Windows CI disks.
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
-	checkpointErr := template.CheckpointWALTruncate(ctx)
-	closeErr := template.Close()
-	if checkpointErr != nil {
-		return nil, fmt.Errorf("checkpointing db template: %w", checkpointErr)
+	if err := template.CheckpointWALTruncate(ctx); err != nil {
+		fmt.Fprintf(os.Stderr,
+			"dbtest: template wal checkpoint failed, copying wal as-is: %v\n",
+			err)
 	}
-	if closeErr != nil {
-		return nil, fmt.Errorf("closing db template: %w", closeErr)
+	if err := template.Close(); err != nil {
+		return nil, fmt.Errorf("closing db template: %w", err)
 	}
 
 	files := make(map[string][]byte, 3)

internal/dbtest/dbtest_fallback_test.go

@@ -0,0 +1,50 @@
+package dbtest
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"go.kenn.io/agentsview/internal/db"
+)
+
+func TestEnsureTestDBAtFallsBackWhenTemplateUnavailable(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	// Simulate a poisoned template build that also left a partial
+	// copy behind: the fallback must discard it and still produce a
+	// usable current-schema database.
+	ensureTestDBAtWith(t, path, func(dst string) error {
+		require.NoError(t, os.WriteFile(dst, []byte("garbage"), 0o600),
+			"writing partial template copy")
+		return errors.New("template poisoned")
+	})
+
+	d, err := db.Open(path)
+	require.NoError(t, err, "opening fallback test db")
+	require.NoError(t, d.Close(), "closing fallback test db")
+}
+
+func TestEnsureTestDBAtLeavesExistingFileIntact(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "test.db")
+
+	EnsureTestDBAt(t, path)
+	info, err := os.Stat(path)
+	require.NoError(t, err, "statting created test db")
+
+	// A second call must not rebuild or replace the existing file,
+	// and must not consult the template at all.
+	ensureTestDBAtWith(t, path, func(string) error {
+		t.Fatal("copyTemplate must not run for an existing file")
+		return nil
+	})
+	again, err := os.Stat(path)
+	require.NoError(t, err, "statting test db after second ensure")
+	require.Equal(t, info.ModTime(), again.ModTime(),
+		"existing test db was modified")
+}

internal/server/analytics_test.go

@@ -259,7 +259,7 @@ func buildAnalyticsDBFixture(
 		)
 	}
 	stats := seed(t, &testEnv{db: database})
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 	checkpointErr := database.CheckpointWALTruncate(ctx)
 	closeErr := database.Close()