[ENG-3089] Relax bbolt cache durability to fix goroutine pileup (#4221)

## Summary

- Skip `fdatasync` on every bbolt commit in the sdpcache layer by
setting `NoSync: true` and `NoFreelistSync: true`
- Eliminates the single-writer bottleneck that cascaded into 36K stuck
goroutines under load (bbolt serializes write transactions behind
fdatasync, ~930 pool workers were queuing on the write lock)
- Safe because sdpcache is a pure cache -- crash durability provides no
value

## Linear Ticket

- **Ticket**:
[ENG-3089](https://linear.app/overmind/issue/ENG-3089/relax-bbolt-cache-durability-to-fix-goroutine-pileup-under-load)
— Relax bbolt cache durability to fix goroutine pileup under load
- **Purpose**: Remove the fdatasync bottleneck that causes cascading
goroutine pileups when sources are under heavy query load
- **Priority**: Urgent

## Changes

Single file change in `go/sdpcache/bolt_cache.go`:

1. Added a package-level `cacheOpenOptions` variable with `NoSync:
true`, `NoFreelistSync: true`, and the existing `Timeout: 5s`
2. Replaced all 7 inline `&bbolt.Options{Timeout: 5 * time.Second}` with
`cacheOpenOptions`

All sdpcache tests pass. No API or behavioral change -- only the fsync
guarantee is relaxed, which is irrelevant for a cache.

Made with [Cursor](https://cursor.com)

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> **Medium Risk**
> Disables bbolt fsync and freelist syncing for the cache DB, which can
improve throughput but increases the chance of cache corruption/loss
after crashes or power failures.
>
> **Overview**
> Introduces a shared `cacheOpenOptions` for all `bbolt.Open` calls in
`sdpcache` that sets `NoSync` and `NoFreelistSync` (keeping the existing
5s `Timeout`) to avoid per-commit `fdatasync` overhead.
>
> All cache DB open/reopen paths (startup, deletion recovery, and
compaction temp DB creation/reopen) are updated to use these relaxed
durability settings.
>
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
60297db5a8b4e78b25a685c30ad8242f84ed17b1. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

GitOrigin-RevId: 4de3a30393489e86e14f68b16c8bbd81b8f1f3da

Author: DavidS-ovm

go/sdpcache/bolt_cache.go

@@ -31,6 +31,16 @@ var (
 	deletedBytesKey  = []byte("deletedBytes")
 )
 
+// cacheOpenOptions are the bbolt options used for every Open call in this
+// package. Since this is a cache layer, crash durability is unnecessary:
+//   - NoSync skips fdatasync per commit, removing the single-writer bottleneck.
+//   - NoFreelistSync skips persisting the freelist, reducing write amplification.
+var cacheOpenOptions = &bbolt.Options{
+	Timeout:        5 * time.Second,
+	NoSync:         true,
+	NoFreelistSync: true,
+}
+
 // DefaultCompactThreshold is the default threshold for triggering compaction (100MB)
 const DefaultCompactThreshold = 100 * 1024 * 1024
 
@@ -234,9 +244,7 @@ func NewBoltCache(path string, opts ...BoltCacheOption) (*BoltCache, error) {
 	}
 
 	// bbolt.Open will open an existing file if present, or create a new one
-	db, err := bbolt.Open(path, 0o600, &bbolt.Options{
-		Timeout: 5 * time.Second,
-	})
+	db, err := bbolt.Open(path, 0o600, cacheOpenOptions)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open bolt database: %w", err)
 	}
@@ -457,7 +465,7 @@ func (c *BoltCache) deleteCacheFileLocked(ctx context.Context, span trace.Span)
 	c.resetDeletedBytes()
 
 	// Reopen the database
-	db, err := bbolt.Open(c.path, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+	db, err := bbolt.Open(c.path, 0o600, cacheOpenOptions)
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, "failed to reopen database")
@@ -1341,13 +1349,13 @@ func (c *BoltCache) compact(ctx context.Context) error {
 	}
 
 	// Open the destination database
-	dstDB, err := bbolt.Open(tempPath, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+	dstDB, err := bbolt.Open(tempPath, 0o600, cacheOpenOptions)
 	if err != nil {
 		if isDiskFullError(err) {
 			// Attempt cleanup first - use locked version since we already hold the lock
 			c.purgeLocked(ctx, time.Now())
 			// Try again
-			dstDB, err = bbolt.Open(tempPath, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+			dstDB, err = bbolt.Open(tempPath, 0o600, cacheOpenOptions)
 			if err != nil {
 				return handleDiskFull(err, "temp database creation")
 			}
@@ -1364,7 +1372,7 @@ func (c *BoltCache) compact(ctx context.Context) error {
 			// Attempt cleanup first - use locked version since we already hold the lock
 			c.purgeLocked(ctx, time.Now())
 			// Try compaction again
-			dstDB2, retryErr := bbolt.Open(tempPath, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+			dstDB2, retryErr := bbolt.Open(tempPath, 0o600, cacheOpenOptions)
 			if retryErr != nil {
 				return handleDiskFull(retryErr, "temp database creation after cleanup")
 			}
@@ -1395,12 +1403,12 @@ func (c *BoltCache) compact(ctx context.Context) error {
 	// Replace the old file with the compacted one
 	if err := os.Rename(tempPath, c.path); err != nil {
 		// Try to reopen the original database
-		c.db, _ = bbolt.Open(c.path, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+		c.db, _ = bbolt.Open(c.path, 0o600, cacheOpenOptions)
 		return handleDiskFull(err, "rename")
 	}
 
 	// Reopen the database
-	db, err := bbolt.Open(c.path, 0o600, &bbolt.Options{Timeout: 5 * time.Second})
+	db, err := bbolt.Open(c.path, 0o600, cacheOpenOptions)
 	if err != nil {
 		span.RecordError(err)
 		span.SetStatus(codes.Error, "failed to reopen database")