feat(database): badger plugin value threshold

database/plugin/blob/badger/database.go

@@ -156,6 +156,7 @@ type BlobStoreBadger struct {
 	indexCacheSize   uint64
 	valueLogFileSize int64
 	memTableSize     int64
+	valueThreshold   int64
 	gcEnabled        bool
 }
 
@@ -168,6 +169,7 @@ func New(opts ...BlobStoreBadgerOptionFunc) (*BlobStoreBadger, error) {
 		indexCacheSize:   DefaultIndexCacheSize,
 		valueLogFileSize: int64(DefaultValueLogFileSize),
 		memTableSize:     int64(DefaultMemTableSize),
+		valueThreshold:   int64(DefaultValueThreshold),
 	}
 	for _, opt := range opts {
 		opt(db)
@@ -182,7 +184,8 @@ func New(opts ...BlobStoreBadgerOptionFunc) (*BlobStoreBadger, error) {
 			WithLogger(NewBadgerLogger(db.logger)).
 			// The default INFO logging is a bit verbose
 			WithLoggingLevel(badger.WARNING).
-			WithInMemory(true)
+			WithInMemory(true).
+			WithValueThreshold(db.valueThreshold)
 		blobDb, err = badger.Open(badgerOpts)
 		if err != nil {
 			return nil, err
@@ -209,6 +212,7 @@ func New(opts ...BlobStoreBadgerOptionFunc) (*BlobStoreBadger, error) {
 			WithIndexCacheSize(int64(db.indexCacheSize)). //nolint:gosec // indexCacheSize is controlled and reasonable
 			WithValueLogFileSize(db.valueLogFileSize).
 			WithMemTableSize(db.memTableSize).
+			WithValueThreshold(db.valueThreshold).
 			WithCompression(options.Snappy)
 		blobDb, err = badger.Open(badgerOpts)
 		if err != nil {

database/plugin/blob/badger/options.go

@@ -79,3 +79,10 @@ func WithMemTableSize(size int64) BlobStoreBadgerOptionFunc {
 		b.memTableSize = size
 	}
 }
+
+// WithValueThreshold specifies the value threshold for keeping values in LSM tree
+func WithValueThreshold(threshold int64) BlobStoreBadgerOptionFunc {
+	return func(b *BlobStoreBadger) {
+		b.valueThreshold = threshold
+	}
+}

database/plugin/blob/badger/plugin.go

@@ -25,7 +25,8 @@ const (
 	DefaultBlockCacheSize   = 1610612736 // 1.5GB (increased from 768MB)
 	DefaultIndexCacheSize   = 536870912  // 512MB (increased from 256MB)
 	DefaultValueLogFileSize = 1073741824 // 1GB (keep large values on disk, write amplification reduction)
-	DefaultMemTableSize     = 67108864   // 64MB (conservative default)
+	DefaultMemTableSize     = 134217728  // 128MB (increased from 64MB for better write buffering)
+	DefaultValueThreshold   = 1048576    // 1MB (keep small values in LSM, large blobs in value log)
 )
 
 var (
@@ -35,6 +36,7 @@ var (
 		indexCacheSize   uint64
 		valueLogFileSize uint64
 		memTableSize     uint64
+		valueThreshold   uint64
 		gcEnabled        bool
 	}
 	cmdlineOptionsMutex sync.RWMutex
@@ -48,6 +50,7 @@ func initCmdlineOptions() {
 	cmdlineOptions.indexCacheSize = DefaultIndexCacheSize
 	cmdlineOptions.valueLogFileSize = DefaultValueLogFileSize
 	cmdlineOptions.memTableSize = DefaultMemTableSize
+	cmdlineOptions.valueThreshold = DefaultValueThreshold
 	cmdlineOptions.gcEnabled = true
 	cmdlineOptions.dataDir = ".dingo"
 }
@@ -104,6 +107,13 @@ func init() {
 					DefaultValue: uint64(DefaultMemTableSize),
 					Dest:         &(cmdlineOptions.memTableSize),
 				},
+				{
+					Name:         "value-threshold",
+					Type:         plugin.PluginOptionTypeUint,
+					Description:  "Badger value threshold for LSM vs value log",
+					DefaultValue: uint64(DefaultValueThreshold),
+					Dest:         &(cmdlineOptions.valueThreshold),
+				},
 			},
 		},
 	)
@@ -120,6 +130,10 @@ func NewFromCmdlineOptions() plugin.Plugin {
 	if memTableSize > 1<<63-1 {
 		memTableSize = 1<<63 - 1 // Cap at max int64
 	}
+	valueThreshold := cmdlineOptions.valueThreshold
+	if valueThreshold > 1<<63-1 {
+		valueThreshold = 1<<63 - 1 // Cap at max int64
+	}
 	// #nosec G115
 	opts := []BlobStoreBadgerOptionFunc{
 		WithDataDir(cmdlineOptions.dataDir),
@@ -131,6 +145,9 @@ func NewFromCmdlineOptions() plugin.Plugin {
 		WithMemTableSize(
 			int64(memTableSize),
 		),
+		WithValueThreshold(
+			int64(valueThreshold),
+		),
 		WithGc(cmdlineOptions.gcEnabled),
 	}
 	cmdlineOptionsMutex.RUnlock()