graph: data race found in `HasChannelEdge`

[yyforyongyu]

From this build, there's an unknown race found in TestStressTestChannelGraphAPI.

Some analysis,

• the chanScheduler holds the s.cacheMu here:

  lnd/graph/db/sql_store.go

  Line 223 in 4b1c499

  db, &s.cacheMu, opts.BatchCommitInterval,

• the batch execute will hold the lock before OnCommit:

  lnd/batch/scheduler.go

  Lines 104 to 107 in 4b1c499

  	if s.locker != nil {
  	s.locker.Lock()
  	defer s.locker.Unlock()
  	}

So it's weird that we have a race, unless there's some unit test specific setups. Also realized the test fail locally, so added a CI job to check for race #10415.

fatal error: concurrent map read and map write

goroutine 10170 [running]:
internal/runtime/maps.fatal({0xf46fcc?, 0xec6bc0?})
	/opt/hostedtoolcache/go/1.25.3/x64/src/runtime/panic.go:1046 +0x18
github.com/lightningnetwork/lnd/graph/db.(*rejectCache).insert(...)
	/home/runner/work/lnd/lnd/graph/db/reject_cache.go:77
github.com/lightningnetwork/lnd/graph/db.(*SQLStore).HasChannelEdge(0xc00d83fab0, 0x5efb766356b8)
	/home/runner/work/lnd/lnd/graph/db/sql_store.go:2227 +0x752
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func8()
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3004 +0x62
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func14(0xc00517e8c0)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3116 +0x56
testing.tRunner(0xc00517e8c0, 0xc00e4fce70)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 1 [chan receive]:
testing.tRunner.func1()
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1891 +0x451
testing.tRunner(0xc00019e8c0, 0xc000273c48)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1940 +0x123
testing.runTests(0xc0000125b8, {0x17fa140, 0x30, 0x30}, {0x6?, 0x0?, 0x1806bc0?})
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2475 +0x4b4
testing.(*M).Run(0xc000233400)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2337 +0x63a
github.com/lightningnetwork/lnd/kvdb.RunTests(0x1828c60?)
	/home/runner/go/pkg/mod/github.com/lightningnetwork/lnd/kvdb@v1.4.16/test_utils.go:23 +0x13
github.com/lightningnetwork/lnd/graph/db.TestMain(...)
	/home/runner/work/lnd/lnd/graph/db/setup_test.go:10
main.main()
	_testmain.go:149 +0xa6

goroutine 35 [chan receive]:
testing.tRunner.func1()
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1891 +0x451
testing.tRunner(0xc0003f68c0, 0x1030ad0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1940 +0x123
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 36 [chan receive]:
testing.tRunner.func1()
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1891 +0x451
testing.tRunner(0xc0003f6c40, 0x10309f0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1940 +0x123
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 8833 [select]:
database/sql.(*DB).connectionOpener(0xc00cea9c70, {0x112dcb0, 0xc00d88c500})
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1261 +0x87
created by database/sql.OpenDB in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:841 +0x130

goroutine 52 [runnable]:
github.com/lightningnetwork/lnd/batch.(*TimeScheduler[...]).Execute(0x112ff60, {0x112dcb0, 0xc00040e0f0}, 0xc0178f63c0)
	/home/runner/work/lnd/lnd/batch/scheduler.go:75 +0x3fc
github.com/lightningnetwork/lnd/graph/db.(*SQLStore).UpdateEdgePolicy(0xc0001a5b90, {0x112dcb0, 0xc00040e0f0}, 0xc0178e66e0, {0x0, 0x0, 0x0})
	/home/runner/work/lnd/lnd/graph/db/sql_store.go:808 +0x2db
github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).UpdateEdgePolicy(0xc0004a7400, {0x112dcb0?, 0xc00040e0f0?}, 0xc0178e66e0, {0x0?, 0x0?, 0x0?})
	/home/runner/work/lnd/lnd/graph/db/graph.go:584 +0x56
github.com/lightningnetwork/lnd/graph/db.fillTestGraph({0x113aa50, 0xc000410380}, 0xc0004a7400, 0x64, 0x4)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:1669 +0xafa
github.com/lightningnetwork/lnd/graph/db.TestGraphLoading(0xc000410380)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:4691 +0x10b
testing.tRunner(0xc000410380, 0x1030a30)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 1
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 4135 [select]:
database/sql.(*DB).connectionOpener(0xc00251aa90, {0x112dcb0, 0xc000535ea0})
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1261 +0x87
created by database/sql.OpenDB in goroutine 52
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:841 +0x130

goroutine 9951 [select]:
github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).handleTopologySubscriptions(0xc00e27a9c0)
	/home/runner/work/lnd/lnd/graph/db/graph.go:111 +0x112
created by github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).Start in goroutine 35
	/home/runner/work/lnd/lnd/graph/db/graph.go:82 +0x1a5

goroutine 10171 [sync.RWMutex.RLock]:
sync.runtime_SemacquireRWMutexR(0x2?, 0x0?, 0xc012e3fd01?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/runtime/sema.go:100 +0x25
sync.(*RWMutex).RLock(...)
	/opt/hostedtoolcache/go/1.25.3/x64/src/sync/rwmutex.go:74
github.com/lightningnetwork/lnd/graph/db.(*SQLStore).HasChannelEdge(0xc00d83fab0, 0x1ff6c239c5a0)
	/home/runner/work/lnd/lnd/graph/db/sql_store.go:2139 +0xe5
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func8()
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3004 +0x62
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func14(0xc00517ea80)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3116 +0x56
testing.tRunner(0xc00517ea80, 0xc00e4fced0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 4799 [runnable]:
github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).handleTopologySubscriptions(0xc0004a7400)
	/home/runner/work/lnd/lnd/graph/db/graph.go:111 +0x112
created by github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).Start in goroutine 52
	/home/runner/work/lnd/lnd/graph/db/graph.go:82 +0x1a5

goroutine 12016 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc005c688c0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestFilterChannelRange.func2(0xc005c688c0)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3309 +0x4a
testing.tRunner(0xc005c688c0, 0xc00e2409c0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 10169 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc00517e700)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func14(0xc00517e700)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3105 +0x45
testing.tRunner(0xc00517e700, 0xc00e4fce10)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 12083 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc005c68e00)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestFilterChannelRange.func2(0xc005c68e00)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3309 +0x4a
testing.tRunner(0xc005c68e00, 0xc00e240ae0)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 11313 [select]:
github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).handleTopologySubscriptions(0xc00e36f180)
	/home/runner/work/lnd/lnd/graph/db/graph.go:111 +0x112
created by github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).Start in goroutine 36
	/home/runner/work/lnd/lnd/graph/db/graph.go:82 +0x1a5

goroutine 12082 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc005c68c40)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestFilterChannelRange.func2(0xc005c68c40)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3309 +0x4a
testing.tRunner(0xc005c68c40, 0xc00e240a80)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 4239 [select]:
database/sql.(*DB).connectionCleaner(0xc00251aa90, 0xc0052bcc80?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1104 +0x9c
created by database/sql.(*DB).startCleanerLocked in goroutine 52
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1091 +0x105

goroutine 10210 [select]:
database/sql.(*DB).connectionOpener(0xc00e4fe750, {0x112dcb0, 0xc00e4fb720})
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1261 +0x87
created by database/sql.OpenDB in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:841 +0x130

goroutine 12015 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc005c68700)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestFilterChannelRange.func2(0xc005c68700)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3309 +0x4a
testing.tRunner(0xc005c68700, 0xc00e240960)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 12017 [chan receive]:
testing.(*testState).waitParallel(0xc000189220)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:2116 +0xaa
testing.(*T).Parallel(0xc005c68a80)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1709 +0x245
github.com/lightningnetwork/lnd/graph/db.TestFilterChannelRange.func2(0xc005c68a80)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3309 +0x4a
testing.tRunner(0xc005c68a80, 0xc00e240a20)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465

goroutine 10334 [select]:
database/sql.(*DB).connectionCleaner(0xc00e4fe750, 0x0?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1104 +0x9c
created by database/sql.(*DB).startCleanerLocked in goroutine 36
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1091 +0x105

goroutine 9017 [select]:
database/sql.(*DB).connectionCleaner(0xc00cea9c70, 0xc00d7e8320?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1104 +0x9c
created by database/sql.(*DB).startCleanerLocked in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/database/sql/sql.go:1091 +0x105

goroutine 10172 [sync.Mutex.Lock]:
internal/sync.runtime_SemacquireMutex(0x0?, 0x0?, 0xc01783be00?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/runtime/sema.go:95 +0x25
internal/sync.(*Mutex).lockSlow(0xc00d83fac8)
	/opt/hostedtoolcache/go/1.25.3/x64/src/internal/sync/mutex.go:149 +0x15d
internal/sync.(*Mutex).Lock(...)
	/opt/hostedtoolcache/go/1.25.3/x64/src/internal/sync/mutex.go:70
sync.(*Mutex).Lock(...)
	/opt/hostedtoolcache/go/1.25.3/x64/src/sync/mutex.go:46
sync.(*RWMutex).Lock(0x1828c60?)
	/opt/hostedtoolcache/go/1.25.3/x64/src/sync/rwmutex.go:150 +0x32
github.com/lightningnetwork/lnd/graph/db.(*SQLStore).PruneGraph(0xc00d83fab0, {0xc017574dc0, 0x5, 0x8}, 0xc000137c60, 0x64)
	/home/runner/work/lnd/lnd/graph/db/sql_store.go:2554 +0x93
github.com/lightningnetwork/lnd/graph/db.(*ChannelGraph).PruneGraph(0xc00e27a9c0, {0xc017574dc0?, 0xc0177b5ec8?, 0x6ccce5?}, 0xdc8f20?, 0x64)
	/home/runner/work/lnd/lnd/graph/db/graph.go:445 +0x46
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func9()
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3024 +0xef
github.com/lightningnetwork/lnd/graph/db.TestStressTestChannelGraphAPI.func14(0xc00517ec40)
	/home/runner/work/lnd/lnd/graph/db/graph_test.go:3116 +0x56
testing.tRunner(0xc00517ec40, 0xc00e4fcf30)
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1934 +0xea
created by testing.(*T).Run in goroutine 35
	/opt/hostedtoolcache/go/1.25.3/x64/src/testing/testing.go:1997 +0x465
FAIL	github.com/lightningnetwork/lnd/graph/db	2.301s
FAIL