Skip to content

feat: optimize pruning performance and improve configuration #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dntjd1097 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

feat: optimize pruning performance and improve configuration #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3a02421
feat: optimize pruning performance and improve logging
dntjd1097 Nov 17, 2025
9ccf7af
fix: correct IAVL fast node flag logic
dntjd1097 Nov 17, 2025
4e5fb38
feat: optimize LevelDB options and improve documentation
dntjd1097 Nov 17, 2025
28af845
Revert "feat: optimize LevelDB options and improve documentation"
dntjd1097 Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 57 additions & 36 deletions cmd/pruner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"fmt"
"path/filepath"

"github.com/cometbft/cometbft/libs/log"
"github.com/cometbft/cometbft/state"
"github.com/cosmos/cosmos-sdk/types"
authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
authzkeeper "github.com/cosmos/cosmos-sdk/x/authz/keeper"
Expand All @@ -18,7 +18,6 @@ import (
ibchost "github.com/cosmos/ibc-go/v7/modules/core/exported"

db "github.com/cometbft/cometbft-db"
"github.com/cometbft/cometbft/state"
tmstore "github.com/cometbft/cometbft/store"
storetypes "github.com/cosmos/cosmos-sdk/store/types"
consensusparamtypes "github.com/cosmos/cosmos-sdk/x/consensus/types"
Expand Down Expand Up @@ -49,9 +48,13 @@ func pruneCmd() *cobra.Command {
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {

logger.Info("Starting pruning...")

ctx := cmd.Context()
errs, _ := errgroup.WithContext(ctx)
var err error

// Tendermint pruning (blockstore.db, state.db)
if tendermint {
errs.Go(func() error {
if err = pruneTMData(args[0]); err != nil {
Expand Down Expand Up @@ -94,7 +97,7 @@ func pruneAppState(home string) error {
}

//TODO: need to get all versions in the store, setting randomly is too slow
fmt.Println("pruning application state")
logger.Info("pruning application state")

// only mount keys from core sdk
// todo allow for other keys to be mounted
Expand Down Expand Up @@ -135,7 +138,17 @@ func pruneAppState(home string) error {
}

// TODO: cleanup app state
appStore := rootmulti.NewStore(appDB, log.NewNopLogger())
appStore := rootmulti.NewStore(appDB, logger)

// Configure IAVL fast node
// Default (false): fast node enabled for queries
// With flag (true): fast node disabled for faster pruning
appStore.SetIAVLDisableFastNode(disableFastNode)
if disableFastNode {
logger.Info("IAVL fast node disabled (faster pruning mode)")
} else {
logger.Info("IAVL fast node enabled (default mode)")
}

for _, value := range keys {
appStore.MountStoreWithDB(value, storetypes.StoreTypeIAVL, nil)
Expand All @@ -152,30 +165,34 @@ func pruneAppState(home string) error {
return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight)
}

var pruningHeights []int64
for height := int64(1); height < latestHeight; height++ {
if height < latestHeight-int64(versions) {
pruningHeights = append(pruningHeights, height)
}
}

//pruningHeight := []int64{latestHeight - int64(versions)}

if len(pruningHeights) == 0 {
fmt.Println("no heights to prune")
// var pruningHeights []int64
// for height := int64(1); height < latestHeight; height++ {
// if height < latestHeight-int64(versions) {
// pruningHeights = append(pruningHeights, height)
// }
// }

// Prune the last X versions
// This is the most efficient way to prune the application state
// as it only needs to delete the last X versions
pruneHeight := latestHeight - int64(versions)
if pruneHeight <= 0 {
logger.Error("no heights to prune")
return nil
}
pruningHeights := []int64{pruneHeight}
//pruningHeight := []int64{latestHeight - int64(versions)}

if err = appStore.PruneStores(false, pruningHeights); err != nil {
return err
}
fmt.Println("pruning application state complete")
logger.Info("pruning application state complete")

fmt.Println("compacting application state")
logger.Info("compacting application state")
if err := appDB.Compact(nil, nil); err != nil {
return err
}
fmt.Println("compacting application state complete")
logger.Info("compacting application state complete")

//create a new app store
return nil
Expand All @@ -197,52 +214,56 @@ func pruneTMData(home string) error {
}
blockStore := tmstore.NewBlockStore(blockStoreDB)

// Get StateStore
stateDB, err := db.NewGoLevelDBWithOpts("state", dbDir, &o)
if err != nil {
return err
}

stateStore := state.NewStore(stateDB, state.StoreOptions{
DiscardABCIResponses: true,
})

base := blockStore.Base()

pruneHeight := blockStore.Height() - int64(blocks)

// Check if there's anything to prune
if pruneHeight <= base {
logger.Error("no blocks to prune", "base", base, "target", pruneHeight)
return nil
} // Get StateStore

errs, _ := errgroup.WithContext(context.Background())
errs.Go(func() error {
fmt.Println("pruning block store")
logger.Info("pruning block store")
// prune block store
blocks, err = blockStore.PruneBlocks(pruneHeight)
if err != nil {
return err
}
fmt.Println("pruning block store complete")
logger.Info("pruning block store complete")

fmt.Println("compacting block store")
logger.Info("compacting block store")
if err := blockStoreDB.Compact(nil, nil); err != nil {
return err
}
fmt.Println("compacting block store complete")
logger.Info("compacting block store complete")

return nil
})

fmt.Println("pruning state store")
logger.Info("pruning state store")
stateDB, err := db.NewGoLevelDBWithOpts("state", dbDir, &o)
if err != nil {
return err
}

stateStore := state.NewStore(stateDB, state.StoreOptions{
DiscardABCIResponses: true,
})
// prune state store
err = stateStore.PruneStates(base, pruneHeight)
if err != nil {
return err
}
fmt.Println("pruning state store complete")
logger.Info("pruning state store complete")

fmt.Println("compacting state store")
logger.Info("compacting state store")
if err := stateDB.Compact(nil, nil); err != nil {
return err
}
fmt.Println("compacting state store complete")
logger.Info("compacting state store complete")

return nil
}
Expand Down
43 changes: 34 additions & 9 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,24 @@ package cmd
import (
"os"

"github.com/cometbft/cometbft/libs/log"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)

var (
dataDir string
backend string
app string
cosmosSdk bool
tendermint bool
blocks uint64
versions uint64
appName = "cosmprund"
dataDir string
backend string
app string
cosmosSdk bool
tendermint bool
blocks uint64
versions uint64
debug bool
disableFastNode bool

appName = "cosmprund"
logger log.Logger
)

// NewRootCmd returns the root command for relayer.
Expand All @@ -31,7 +36,15 @@ func NewRootCmd() *cobra.Command {
// if err := initConfig(rootCmd); err != nil {
// return err
// }

logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
// Set log level based on debug flag
if debug {
// Show all logs including Debug level
logger = log.NewFilter(logger, log.AllowDebug())
} else {
// Only show Info level and above (hides Debug logs like loadVersion commitID)
logger = log.NewFilter(logger, log.AllowInfo())
}
return nil
}

Expand Down Expand Up @@ -71,6 +84,18 @@ func NewRootCmd() *cobra.Command {
panic(err)
}

// --debug flag
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug logging (shows debug logs)")
if err := viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug")); err != nil {
panic(err)
}

// --disable-fast-node flag
rootCmd.PersistentFlags().BoolVar(&disableFastNode, "disable-fast-node", true, "disable IAVL fast node for faster pruning (default: false, fast node enabled)")
if err := viper.BindPFlag("disable-fast-node", rootCmd.PersistentFlags().Lookup("disable-fast-node")); err != nil {
panic(err)
}

rootCmd.AddCommand(
pruneCmd(),
)
Expand Down
Loading