Skip to content
Draft
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
1cf5e6c
feat: add tracing
aaronc Oct 29, 2025
a207f94
work on otel tracer impl
aaronc Oct 29, 2025
ee0bfb3
add basic baseapp tracing
aaronc Oct 29, 2025
d5f5ea4
latest WIP
aaronc Oct 30, 2025
47f83e8
add trace exporter setup
aaronc Oct 30, 2025
7fafce3
fixes
aaronc Oct 30, 2025
bdba035
simapp setup, make tracers wrap loggers
aaronc Oct 30, 2025
25e3135
add test setup
aaronc Oct 30, 2025
5c7e464
fix shutdown order
aaronc Oct 30, 2025
d71f7c1
block trace nesting
aaronc Oct 30, 2025
56b215a
update metrics config and instrumentation
aaronc Oct 30, 2025
f9ce55c
start adding otel metric config
aaronc Oct 31, 2025
3fff00f
migrate to pure otel setup
aaronc Oct 31, 2025
5077567
fixes
aaronc Oct 31, 2025
31536b6
add basic metrics
aaronc Oct 31, 2025
c922688
add telemetry shutdown hook
aaronc Oct 31, 2025
ed891cc
docs, cleanup
aaronc Oct 31, 2025
f685bd4
WIP on removing go-metrics
aaronc Oct 31, 2025
42da2f7
Merge branch 'main' of github.com:cosmos/cosmos-sdk into aaronc/traci…
aaronc Oct 31, 2025
699f5d3
setup sim test flag
aaronc Oct 31, 2025
5df2460
integrate slog logging
aaronc Oct 31, 2025
1c84edb
update to use official env var
aaronc Oct 31, 2025
46e4bcb
add README.md
aaronc Nov 3, 2025
f0c3955
delete spaces
aaronc Nov 3, 2025
7dfb754
setup TestingMain
aaronc Nov 3, 2025
1ce344b
update suggested config in README.md
aaronc Nov 3, 2025
edbae92
add otel custom config options
aaronc Nov 3, 2025
0f8085a
add otel custom config options
aaronc Nov 3, 2025
03b6069
add more instrumentation
aaronc Nov 3, 2025
c4dbd07
remove pretty print
aaronc Nov 4, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 36 additions & 5 deletions baseapp/abci.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ const (
)

func (app *BaseApp) InitChain(req *abci.RequestInitChain) (*abci.ResponseInitChain, error) {
span := app.tracer.StartSpan("InitChain")
defer span.End()

if req.ChainId != app.chainID {
return nil, fmt.Errorf("invalid chain-id on InitChain; expected: %s, got: %s", app.chainID, req.ChainId)
}
Expand Down Expand Up @@ -153,6 +156,9 @@ func (app *BaseApp) Info(_ *abci.RequestInfo) (*abci.ResponseInfo, error) {
// Query implements the ABCI interface. It delegates to CommitMultiStore if it
// implements Queryable.
func (app *BaseApp) Query(_ context.Context, req *abci.RequestQuery) (resp *abci.ResponseQuery, err error) {
span := app.tracer.StartSpan("Query")
defer span.End()

// add panic recovery for all queries
//
// Ref: https://github.com/cosmos/cosmos-sdk/pull/8039
Expand Down Expand Up @@ -342,6 +348,9 @@ func (app *BaseApp) ApplySnapshotChunk(req *abci.RequestApplySnapshotChunk) (*ab
// will contain relevant error information. Regardless of tx execution outcome,
// the ResponseCheckTx will contain the relevant gas execution context.
func (app *BaseApp) CheckTx(req *abci.RequestCheckTx) (*abci.ResponseCheckTx, error) {
span := app.tracer.StartSpan("CheckTx")
defer span.End()

var mode sdk.ExecMode

switch req.Type {
Expand All @@ -356,7 +365,7 @@ func (app *BaseApp) CheckTx(req *abci.RequestCheckTx) (*abci.ResponseCheckTx, er
}

if app.abciHandlers.CheckTxHandler == nil {
gasInfo, result, anteEvents, err := app.RunTx(mode, req.Tx, nil, -1, nil, nil)
gasInfo, result, anteEvents, err := app.RunTx(mode, req.Tx, nil, -1, nil, nil, span)
if err != nil {
return sdkerrors.ResponseCheckTxWithEvents(err, gasInfo.GasWanted, gasInfo.GasUsed, anteEvents, app.trace), nil
}
Expand All @@ -372,7 +381,7 @@ func (app *BaseApp) CheckTx(req *abci.RequestCheckTx) (*abci.ResponseCheckTx, er

// Create wrapper to avoid users overriding the execution mode
runTx := func(txBytes []byte, tx sdk.Tx) (gInfo sdk.GasInfo, result *sdk.Result, anteEvents []abci.Event, err error) {
return app.RunTx(mode, txBytes, tx, -1, nil, nil)
return app.RunTx(mode, txBytes, tx, -1, nil, nil, span)
}

return app.abciHandlers.CheckTxHandler(runTx, req)
Expand All @@ -392,6 +401,9 @@ func (app *BaseApp) CheckTx(req *abci.RequestCheckTx) (*abci.ResponseCheckTx, er
// Ref: https://github.com/cosmos/cosmos-sdk/blob/main/docs/architecture/adr-060-abci-1.0.md
// Ref: https://github.com/cometbft/cometbft/blob/main/spec/abci/abci%2B%2B_basic_concepts.md
func (app *BaseApp) PrepareProposal(req *abci.RequestPrepareProposal) (resp *abci.ResponsePrepareProposal, err error) {
span := app.tracer.StartSpan("PrepareProposal")
defer span.End()

if app.abciHandlers.PrepareProposalHandler == nil {
return nil, errors.New("PrepareProposal handler not set")
}
Expand Down Expand Up @@ -479,6 +491,9 @@ func (app *BaseApp) PrepareProposal(req *abci.RequestPrepareProposal) (resp *abc
// Ref: https://github.com/cosmos/cosmos-sdk/blob/main/docs/architecture/adr-060-abci-1.0.md
// Ref: https://github.com/cometbft/cometbft/blob/main/spec/abci/abci%2B%2B_basic_concepts.md
func (app *BaseApp) ProcessProposal(req *abci.RequestProcessProposal) (resp *abci.ResponseProcessProposal, err error) {
span := app.tracer.StartSpan("ProcessProposal")
defer span.End()

if app.abciHandlers.ProcessProposalHandler == nil {
return nil, errors.New("ProcessProposal handler not set")
}
Expand Down Expand Up @@ -576,6 +591,9 @@ func (app *BaseApp) ProcessProposal(req *abci.RequestProcessProposal) (resp *abc
// height and are committed in the subsequent height, i.e. H+2. An error is
// returned if vote extensions are not enabled or if extendVote fails or panics.
func (app *BaseApp) ExtendVote(_ context.Context, req *abci.RequestExtendVote) (resp *abci.ResponseExtendVote, err error) {
span := app.tracer.StartSpan("ExtendVote")
defer span.End()

// Always reset state given that ExtendVote and VerifyVoteExtension can timeout
// and be called again in a subsequent round.
var ctx sdk.Context
Expand Down Expand Up @@ -649,6 +667,9 @@ func (app *BaseApp) ExtendVote(_ context.Context, req *abci.RequestExtendVote) (
// phase. The response MUST be deterministic. An error is returned if vote
// extensions are not enabled or if verifyVoteExt fails or panics.
func (app *BaseApp) VerifyVoteExtension(req *abci.RequestVerifyVoteExtension) (resp *abci.ResponseVerifyVoteExtension, err error) {
span := app.tracer.StartSpan("VerifyVoteExtension")
defer span.End()

if app.abciHandlers.VerifyVoteExtensionHandler == nil {
return nil, errors.New("application VerifyVoteExtension handler not set")
}
Expand Down Expand Up @@ -717,6 +738,9 @@ func (app *BaseApp) VerifyVoteExtension(req *abci.RequestVerifyVoteExtension) (r
// only used to handle early cancellation, for anything related to state app.stateManager.GetState(execModeFinalize).Context()
// must be used.
func (app *BaseApp) internalFinalizeBlock(ctx context.Context, req *abci.RequestFinalizeBlock) (*abci.ResponseFinalizeBlock, error) {
span := app.tracer.StartSpan("internalFinalizeBlock")
defer span.End()

var events []abci.Event

if err := app.checkHalt(req.Height, req.Time); err != nil {
Expand Down Expand Up @@ -782,14 +806,14 @@ func (app *BaseApp) internalFinalizeBlock(ctx context.Context, req *abci.Request
WithHeaderHash(req.Hash))
}

preblockEvents, err := app.preBlock(req)
preblockEvents, err := app.preBlock(span, req)
if err != nil {
return nil, err
}

events = append(events, preblockEvents...)

beginBlock, err := app.beginBlock(req)
beginBlock, err := app.beginBlock(span, req)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -846,7 +870,7 @@ func (app *BaseApp) internalFinalizeBlock(ctx context.Context, req *abci.Request
WithBlockGasUsed(blockGasUsed).
WithBlockGasWanted(blockGasWanted),
)
endBlock, err := app.endBlock(finalizeState.Context())
endBlock, err := app.endBlock(span, finalizeState.Context())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -874,6 +898,7 @@ func (app *BaseApp) executeTxsWithExecutor(ctx context.Context, ms storetypes.Mu
if app.txRunner == nil {
app.txRunner = txnrunner.NewDefaultRunner(
app.txDecoder,
app.tracer,
)
}

Expand All @@ -891,6 +916,9 @@ func (app *BaseApp) executeTxsWithExecutor(ctx context.Context, ms storetypes.Mu
// extensions into the proposal, which should not themselves be executed in cases
// where they adhere to the sdk.Tx interface.
func (app *BaseApp) FinalizeBlock(req *abci.RequestFinalizeBlock) (res *abci.ResponseFinalizeBlock, err error) {
span := app.tracer.StartSpan("FinalizeBlock")
defer span.End()

defer func() {
if res == nil {
return
Expand Down Expand Up @@ -958,6 +986,9 @@ func (app *BaseApp) checkHalt(height int64, time time.Time) error {
// against that height and gracefully halt if it matches the latest committed
// height.
func (app *BaseApp) Commit() (*abci.ResponseCommit, error) {
span := app.tracer.StartSpan("Commit")
defer span.End()

finalizeState := app.stateManager.GetState(execModeFinalize)
header := finalizeState.Context().BlockHeader()
retainHeight := app.GetBlockRetentionHeight(header.Height)
Expand Down
49 changes: 40 additions & 9 deletions baseapp/baseapp.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@ import (
protov2 "google.golang.org/protobuf/proto"

errorsmod "cosmossdk.io/errors"
"cosmossdk.io/log"
"cosmossdk.io/store"
storemetrics "cosmossdk.io/store/metrics"
"cosmossdk.io/store/snapshots"
storetypes "cosmossdk.io/store/types"

"cosmossdk.io/log"

"github.com/cosmos/cosmos-sdk/baseapp/config"
"github.com/cosmos/cosmos-sdk/baseapp/oe"
"github.com/cosmos/cosmos-sdk/baseapp/state"
Expand Down Expand Up @@ -64,6 +65,7 @@ type BaseApp struct {
// initialized on creation
mu sync.Mutex // mu protects the fields below.
logger log.Logger
tracer log.Tracer
name string // application name from abci.BlockInfo
db dbm.DB // common DB backend
cms storetypes.CommitMultiStore // Main (uncached) state
Expand Down Expand Up @@ -186,6 +188,13 @@ func NewBaseApp(
gasConfig: config.GasConfig{QueryGasLimit: math.MaxUint64},
}

// initialize tracer
tracer, ok := app.logger.(log.Tracer)
if !ok {
tracer = log.NewNopTracer()
}
app.tracer = tracer

for _, option := range options {
option(app)
}
Expand Down Expand Up @@ -651,7 +660,10 @@ func (app *BaseApp) cacheTxContext(ctx sdk.Context, txBytes []byte) (sdk.Context
return ctx.WithMultiStore(msCache), msCache
}

func (app *BaseApp) preBlock(req *abci.RequestFinalizeBlock) ([]abci.Event, error) {
func (app *BaseApp) preBlock(tracer log.Tracer, req *abci.RequestFinalizeBlock) ([]abci.Event, error) {
span := tracer.StartSpan("preBlock")
defer span.End()

var events []abci.Event
if app.abciHandlers.PreBlocker != nil {
finalizeState := app.stateManager.GetState(execModeFinalize)
Expand All @@ -674,7 +686,7 @@ func (app *BaseApp) preBlock(req *abci.RequestFinalizeBlock) ([]abci.Event, erro
return events, nil
}

func (app *BaseApp) beginBlock(_ *abci.RequestFinalizeBlock) (sdk.BeginBlock, error) {
func (app *BaseApp) beginBlock(tracer log.Tracer, _ *abci.RequestFinalizeBlock) (sdk.BeginBlock, error) {
var (
resp sdk.BeginBlock
err error
Expand All @@ -700,7 +712,7 @@ func (app *BaseApp) beginBlock(_ *abci.RequestFinalizeBlock) (sdk.BeginBlock, er
return resp, nil
}

func (app *BaseApp) deliverTx(tx []byte, txMultiStore storetypes.MultiStore, txIndex int, incarnationCache map[string]any) *abci.ExecTxResult {
func (app *BaseApp) deliverTx(tx []byte, txMultiStore storetypes.MultiStore, txIndex int, incarnationCache map[string]any, tracer log.Tracer) *abci.ExecTxResult {
gInfo := sdk.GasInfo{}
resultStr := "successful"

Expand All @@ -713,7 +725,7 @@ func (app *BaseApp) deliverTx(tx []byte, txMultiStore storetypes.MultiStore, txI
telemetry.SetGauge(float32(gInfo.GasWanted), "tx", "gas", "wanted")
}()

gInfo, result, anteEvents, err := app.RunTx(execModeFinalize, tx, nil, txIndex, txMultiStore, incarnationCache)
gInfo, result, anteEvents, err := app.RunTx(execModeFinalize, tx, nil, txIndex, txMultiStore, incarnationCache, tracer)
if err != nil {
resultStr = "failed"
resp = sdkerrors.ResponseExecTxResultWithEvents(
Expand All @@ -739,7 +751,10 @@ func (app *BaseApp) deliverTx(tx []byte, txMultiStore storetypes.MultiStore, txI

// endBlock is an application-defined function that is called after transactions
// have been processed in FinalizeBlock.
func (app *BaseApp) endBlock(_ context.Context) (sdk.EndBlock, error) {
func (app *BaseApp) endBlock(tracer log.Tracer, _ context.Context) (sdk.EndBlock, error) {
span := tracer.StartSpan("endBlock")
defer span.End()

var endblock sdk.EndBlock

if app.abciHandlers.EndBlocker != nil {
Expand Down Expand Up @@ -772,7 +787,10 @@ func (app *BaseApp) endBlock(_ context.Context) (sdk.EndBlock, error) {
// and execute successfully. An error is returned otherwise.
// both txbytes and the decoded tx are passed to runTx to avoid the state machine encoding the tx and decoding the transaction twice
// passing the decoded tx to runTX is optional, it will be decoded if the tx is nil
func (app *BaseApp) RunTx(mode sdk.ExecMode, txBytes []byte, tx sdk.Tx, txIndex int, txMultiStore storetypes.MultiStore, incarnationCache map[string]any) (gInfo sdk.GasInfo, result *sdk.Result, anteEvents []abci.Event, err error) {
func (app *BaseApp) RunTx(mode sdk.ExecMode, txBytes []byte, tx sdk.Tx, txIndex int, txMultiStore storetypes.MultiStore, incarnationCache map[string]any, tracer log.Tracer) (gInfo sdk.GasInfo, result *sdk.Result, anteEvents []abci.Event, err error) {
span := tracer.StartSpan("RunTx", "txBytes", txBytes, "txIndex", txIndex, "mode", mode)
defer span.End()

// NOTE: GasWanted should be returned by the AnteHandler. GasUsed is
// determined by the GasMeter. We need access to the context to get the gas
// meter, so we initialize upfront.
Expand Down Expand Up @@ -861,7 +879,9 @@ func (app *BaseApp) RunTx(mode sdk.ExecMode, txBytes []byte, tx sdk.Tx, txIndex
// performance benefits, but it'll be more difficult to get right.
anteCtx, msCache = app.cacheTxContext(ctx, txBytes)
anteCtx = anteCtx.WithEventManager(sdk.NewEventManager())
anteSpan := tracer.StartSpan("anteHandler")
newCtx, err := app.anteHandler(anteCtx, tx, mode == execModeSimulate)
anteSpan.End()

if !newCtx.IsZero() {
// At this point, newCtx.MultiStore() is a store branch, or something else
Expand Down Expand Up @@ -910,6 +930,7 @@ func (app *BaseApp) RunTx(mode sdk.ExecMode, txBytes []byte, tx sdk.Tx, txIndex
// in case message processing fails. At this point, the MultiStore
// is a branch of a branch.
runMsgCtx, msCache := app.cacheTxContext(ctx, txBytes)
runMsgCtx = runMsgCtx.WithLogger(tracer) // attach the tracer to the context as the logger

// Attempt to execute all messages and only update state if all messages pass
// and we're in DeliverTx. Note, runMsgs will never return a reference to a
Expand Down Expand Up @@ -968,6 +989,14 @@ func (app *BaseApp) RunTx(mode sdk.ExecMode, txBytes []byte, tx sdk.Tx, txIndex
// Handler does not exist for a given message route. Otherwise, a reference to a
// Result is returned. The caller must not commit state if an error is returned.
func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, msgsV2 []protov2.Message, mode sdk.ExecMode) (*sdk.Result, error) {
// extract the tracer from the context
tracer, ok := ctx.Logger().(log.Tracer)
if !ok {
tracer = log.NewNopTracer()
}
span := tracer.StartSpan("runMsgs")
defer span.End()

events := sdk.EmptyEvents()
var msgResponses []*codectypes.Any

Expand All @@ -984,11 +1013,13 @@ func (app *BaseApp) runMsgs(ctx sdk.Context, msgs []sdk.Msg, msgsV2 []protov2.Me
return nil, errorsmod.Wrapf(sdkerrors.ErrUnknownRequest, "no message handler found for %T", msg)
}

msgSpan := span.StartSpan("msgHandler", "msgType", sdk.MsgTypeURL(msg), "msgIndex", i)
// ADR 031 request type routing
msgResult, err := handler(ctx, msg)
if err != nil {
return nil, errorsmod.Wrapf(err, "failed to execute message; message index: %d", i)
}
msgSpan.End()

// create message events
msgEvents, err := createEvents(app.cdc, msgResult.GetEvents(), msg, msgsV2[i])
Expand Down Expand Up @@ -1077,7 +1108,7 @@ func (app *BaseApp) PrepareProposalVerifyTx(tx sdk.Tx) ([]byte, error) {
return nil, err
}

_, _, _, err = app.RunTx(execModePrepareProposal, bz, tx, -1, nil, nil)
_, _, _, err = app.RunTx(execModePrepareProposal, bz, tx, -1, nil, nil, app.tracer)
if err != nil {
return nil, err
}
Expand All @@ -1096,7 +1127,7 @@ func (app *BaseApp) ProcessProposalVerifyTx(txBz []byte) (sdk.Tx, error) {
return nil, err
}

_, _, _, err = app.RunTx(execModeProcessProposal, txBz, tx, -1, nil, nil)
_, _, _, err = app.RunTx(execModeProcessProposal, txBz, tx, -1, nil, nil, app.tracer)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion baseapp/genesis.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ var _ genesis.TxHandler = (*BaseApp)(nil)
// ExecuteGenesisTx implements genesis.GenesisState from
// cosmossdk.io/core/genesis to set initial state in genesis
func (ba *BaseApp) ExecuteGenesisTx(tx []byte) error {
res := ba.deliverTx(tx, nil, -1, nil)
res := ba.deliverTx(tx, nil, -1, nil, ba.tracer)

if res.Code != types.CodeTypeOK {
return errors.New(res.Log)
Expand Down
9 changes: 5 additions & 4 deletions baseapp/test_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

errorsmod "cosmossdk.io/errors"

"cosmossdk.io/log"
sdk "github.com/cosmos/cosmos-sdk/types"
sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
)
Expand All @@ -19,13 +20,13 @@ func (app *BaseApp) SimCheck(txEncoder sdk.TxEncoder, tx sdk.Tx) (sdk.GasInfo, *
return sdk.GasInfo{}, nil, errorsmod.Wrapf(sdkerrors.ErrInvalidRequest, "%s", err)
}

gasInfo, result, _, err := app.RunTx(execModeCheck, bz, tx, -1, nil, nil)
gasInfo, result, _, err := app.RunTx(execModeCheck, bz, tx, -1, nil, nil, log.NewNopTracer())
return gasInfo, result, err
}

// Simulate executes a tx in simulate mode to get result and gas info.
func (app *BaseApp) Simulate(txBytes []byte) (sdk.GasInfo, *sdk.Result, error) {
gasInfo, result, _, err := app.RunTx(execModeSimulate, txBytes, nil, -1, nil, nil)
gasInfo, result, _, err := app.RunTx(execModeSimulate, txBytes, nil, -1, nil, nil, log.NewNopTracer())
return gasInfo, result, err
}

Expand All @@ -36,7 +37,7 @@ func (app *BaseApp) SimDeliver(txEncoder sdk.TxEncoder, tx sdk.Tx) (sdk.GasInfo,
return sdk.GasInfo{}, nil, errorsmod.Wrapf(sdkerrors.ErrInvalidRequest, "%s", err)
}

gasInfo, result, _, err := app.RunTx(execModeFinalize, bz, tx, -1, nil, nil)
gasInfo, result, _, err := app.RunTx(execModeFinalize, bz, tx, -1, nil, nil, log.NewNopTracer())
return gasInfo, result, err
}

Expand All @@ -47,7 +48,7 @@ func (app *BaseApp) SimTxFinalizeBlock(txEncoder sdk.TxEncoder, tx sdk.Tx) (sdk.
return sdk.GasInfo{}, nil, errorsmod.Wrapf(sdkerrors.ErrInvalidRequest, "%s", err)
}

gasInfo, result, _, err := app.RunTx(execModeFinalize, bz, tx, -1, nil, nil)
gasInfo, result, _, err := app.RunTx(execModeFinalize, bz, tx, -1, nil, nil, log.NewNopTracer())
return gasInfo, result, err
}

Expand Down
Loading
Loading