turn off badger's prefetch and tuning for memory usage (#168)

Author: zhongzc

database/document/document.go

@@ -18,12 +18,17 @@ var documentDB *genji.DB
 var closeCh chan struct{}
 
 func Init(cfg *config.Config) {
+	badger.DefaultIteratorOptions.PrefetchValues = false
+
 	dataPath := path.Join(cfg.Storage.Path, "docdb")
 	l, _ := initLogger(cfg)
 	opts := badger.DefaultOptions(dataPath).
 		WithZSTDCompressionLevel(3).
 		WithBlockSize(8 * 1024).
 		WithValueThreshold(128 * 1024).
+		WithValueLogFileSize(64 * 1024 * 1024).
+		WithBlockCacheSize(16 * 1024 * 1024).
+		WithMemTableSize(16 * 1024 * 1024).
 		WithLogger(l)
 
 	engine, err := badgerengine.NewEngine(opts)

utils/misc.go

@@ -9,8 +9,9 @@ import (
 
 // GoWithRecovery wraps goroutine startup call with force recovery.
 // it will dump current goroutine stack into log if catch any recover result.
-//   exec:      execute logic function.
-//   recoverFn: handler will be called after recover and before dump stack, passing `nil` means noop.
+//
+//	exec:      execute logic function.
+//	recoverFn: handler will be called after recover and before dump stack, passing `nil` means noop.
 func GoWithRecovery(exec func(), recoverFn func(r interface{})) {
 	defer func() {
 		r := recover()

utils/testutil/util.go

@@ -25,10 +25,15 @@ import (
 )
 
 func NewGenjiDB(t *testing.T, storagePath string) *genji.DB {
+	badger.DefaultIteratorOptions.PrefetchValues = false
+
 	opts := badger.DefaultOptions(storagePath).
 		WithZSTDCompressionLevel(3).
 		WithBlockSize(8 * 1024).
-		WithValueThreshold(128 * 1024)
+		WithValueThreshold(128 * 1024).
+		WithValueLogFileSize(64 * 1024 * 1024).
+		WithBlockCacheSize(16 * 1024 * 1024).
+		WithMemTableSize(16 * 1024 * 1024)
 
 	engine, err := badgerengine.NewEngine(opts)
 	require.NoError(t, err)
@@ -38,10 +43,15 @@ func NewGenjiDB(t *testing.T, storagePath string) *genji.DB {
 }
 
 func NewBadgerDB(t *testing.T, storagePath string) *badger.DB {
+	badger.DefaultIteratorOptions.PrefetchValues = false
+
 	opts := badger.DefaultOptions(storagePath).
 		WithZSTDCompressionLevel(3).
 		WithBlockSize(8 * 1024).
-		WithValueThreshold(128 * 1024)
+		WithValueThreshold(128 * 1024).
+		WithValueLogFileSize(64 * 1024 * 1024).
+		WithBlockCacheSize(16 * 1024 * 1024).
+		WithMemTableSize(16 * 1024 * 1024)
 
 	engine, err := badgerengine.NewEngine(opts)
 	require.NoError(t, err)