Merge pull request #2 from informalsystems/jasmina/polygon/compaction_0.38

feat: Pruning and compaction port to 0.38

Author: jmalicevic

config/config.go

@@ -1151,6 +1151,22 @@ type StorageConfig struct {
 	DiscardABCIResponses bool `mapstructure:"discard_abci_responses"`
 	// Configuration related to storage pruning.
 	Pruning *PruningConfig `mapstructure:"pruning"`
+
+	// Compaction on pruning - enable or disable in-process compaction.
+	// If the DB backend supports it, this will force the DB to compact
+	// the database levels and save on storage space. Setting this to true
+	// is most beneficial when used in combination with pruning as it will
+	// phyisically delete the entries marked for deletion.
+	// false by default (forcing compaction is disabled).
+	Compact bool `mapstructure:"compact"`
+	// Compaction interval - number of blocks to try explicit compaction on.
+	// This parameter should be tuned depending on the number of items
+	// you expect to delete between two calls to forced compaction.
+	// If your retain height is 1 block, it is too much of an overhead
+	// to try compaction every block. But it should also not be a very
+	// large multiple of your retain height as it might occur bigger overheads.
+	// 1000 by default.
+	CompactionInterval int64 `mapstructure:"compaction_interval"`
 }
 
 // DefaultStorageConfig returns the default configuration options relating to
@@ -1159,6 +1175,8 @@ func DefaultStorageConfig() *StorageConfig {
 	return &StorageConfig{
 		DiscardABCIResponses: false,
 		Pruning:              DefaultPruningConfig(),
+		Compact:              false,
+		CompactionInterval:   1000,
 	}
 }
 

config/toml.go

@@ -521,6 +521,18 @@ peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"
 # reindex events in the command-line tool.
 discard_abci_responses = {{ .Storage.DiscardABCIResponses}}
 
+# If set to true, CometBFT will force compaction to happen for databases that support this feature.
+# and save on storage space. Setting this to true is most benefits when used in combination
+# with pruning as it will phyisically delete the entries marked for deletion.
+# false by default (forcing compaction is disabled).
+compact = false
+# To avoid forcing compaction every time, this parameter instructs CometBFT to wait 
+# the given amount of blocks to be pruned before triggering compaction.
+# It should be tuned depending on the number of items. If your retain height is 1 block,
+# it is too much of an overhead to try compaction every block. But it should also not be a very
+# large multiple of your retain height as it might occur bigger overheads.
+compaction_interval = 1000
+
 [storage.pruning]
 
 # The time period between automated background pruning operations.

node/setup.go

@@ -110,7 +110,7 @@ func initDBs(config *cfg.Config, dbProvider cfg.DBProvider) (blockStore *store.B
 	if err != nil {
 		return
 	}
-	blockStore = store.NewBlockStore(blockStoreDB)
+	blockStore = store.NewBlockStore(blockStoreDB, store.WithCompaction(config.Storage.Compact, config.Storage.CompactionInterval))
 
 	stateDB, err = dbProvider(&cfg.DBContext{ID: "state", Config: config})
 	if err != nil {

state/export_test.go

@@ -41,7 +41,7 @@ func ValidateValidatorUpdates(abciUpdates []abci.ValidatorUpdate, params types.V
 // SaveValidatorsInfo is an alias for the private saveValidatorsInfo method in
 // store.go, exported exclusively and explicitly for testing.
 func SaveValidatorsInfo(db dbm.DB, height, lastHeightChanged int64, valSet *types.ValidatorSet) error {
-	stateStore := dbStore{db, StoreOptions{DiscardABCIResponses: false}}
+	stateStore := dbStore{db, StoreOptions{DiscardABCIResponses: false}, StoreStateKeeper{}}
 	batch := stateStore.db.NewBatch()
 	err := stateStore.saveValidatorsInfo(height, lastHeightChanged, valSet, batch)
 	if err != nil {

state/mocks/store.go

@@ -37,6 +37,11 @@ type Pruner struct {
 	observer PrunerObserver
 
 	metrics *Metrics
+
+	// Preserve the number of state entries pruned.
+	// Used to calculated correctly when to trigger compactions
+	// TODO This is unused and should be removed from V1 and main as well.
+	prunedStates uint64
 }
 
 type prunerConfig struct {
@@ -303,10 +308,16 @@ func (p *Pruner) pruneABCIResToRetainHeight(lastRetainHeight int64) int64 {
 		return lastRetainHeight
 	}
 
+	// If the block retain height is 0, pruning of the block and state stores might be disabled
+	// This should not prevent Comet from pruning ABCI results if needed.
+	// We could by default always compact when pruning the responses, but in case the state store
+	// is being compacted we introduce an overhead that might cause performance penalties.
+	forceCompact := p.findMinBlockRetainHeight() == 0
+
 	// newRetainHeight is the height just after that which we have successfully
 	// pruned. In case of an error it will be 0, but then it will also be
 	// ignored.
-	numPruned, newRetainHeight, err := p.stateStore.PruneABCIResponses(targetRetainHeight)
+	numPruned, newRetainHeight, err := p.stateStore.PruneABCIResponses(targetRetainHeight, forceCompact)
 	if err != nil {
 		p.logger.Error("Failed to prune ABCI responses", "err", err, "targetRetainHeight", targetRetainHeight)
 		return lastRetainHeight
@@ -357,8 +368,13 @@ func (p *Pruner) pruneBlocksToHeight(height int64) (uint64, int64, error) {
 	if err != nil {
 		return 0, 0, ErrFailedToPruneBlocks{Height: height, Err: err}
 	}
-	if err := p.stateStore.PruneStates(base, height, evRetainHeight); err != nil {
-		return 0, 0, ErrFailedToPruneStates{Height: height, Err: err}
+
+	if pruned > 0 {
+
+		_, err = p.stateStore.PruneStates(base, height, evRetainHeight)
+		if err != nil {
+			return 0, 0, ErrFailedToPruneStates{Height: height, Err: err}
+		}
 	}
 	return pruned, evRetainHeight, err
 }