Skip to content

Commit 5aff1fc

Browse files
move "batch size estimation" biz-logic from SD to Exec (#19011)
reason: - it creating chicken-egg problem. Can't print in logs size before commitment, then print after commitment - and understand how much data produced commitment. Because "print after" will show over-estimated size. - SD is now having less and less biz-logic. So, `EstimateSize` it's clearly abstraction-leak Before this change i see: ``` buf=512M before commitment buf=9G after commitment ``` but it's not because Commitment producing so much data it's because we "double-estimate": using estimation logic which "predicting RAM usage after Commitment - before commitment", use same prediction "after Commitment" is incorrect - because "we just know how much RAM we using now".
1 parent a2eb996 commit 5aff1fc

File tree

6 files changed

+27
-13
lines changed

6 files changed

+27
-13
lines changed

db/state/execctx/domain_shared.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ func (sd *SharedDomains) ClearRam(resetCommitment bool) {
240240
sd.mem.ClearRam()
241241
}
242242

243-
func (sd *SharedDomains) SizeEstimate() uint64 {
243+
func (sd *SharedDomains) Size() uint64 {
244244
return sd.mem.SizeEstimate()
245245
}
246246

db/state/temporal_mem_batch.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -254,10 +254,7 @@ func (sd *TemporalMemBatch) GetAsOf(domain kv.Domain, key []byte, ts uint64) (v
254254
func (sd *TemporalMemBatch) SizeEstimate() uint64 {
255255
sd.latestStateLock.RLock()
256256
defer sd.latestStateLock.RUnlock()
257-
258-
// multiply 2: to cover data-structures overhead (and keep accounting cheap)
259-
// and muliply 8 more: for Commitment calculation when batch is full
260-
return uint64(sd.metrics.CachePutSize) * 16
257+
return uint64(sd.metrics.CachePutSize)
261258
}
262259

263260
func (sd *TemporalMemBatch) ClearRam() {

execution/stagedsync/exec3_metrics.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,7 @@ func (p *Progress) LogExecution(rs *state.StateV3, ex executor) {
592592
curTaskGasPerSec := int64(float64(curTaskGas) / seconds)
593593

594594
uncommitedGas := uint64(te.executedGas.Load() - te.committedGas.Load())
595-
sizeEstimate := rs.SizeEstimate()
595+
sizeEstimate := rs.SizeEstimateBeforeCommitment()
596596

597597
switch ex.(type) {
598598
case *parallelExecutor:
@@ -874,7 +874,10 @@ func (p *Progress) log(mode string, suffix string, te *txExecutor, rs *state.Sta
874874
var vals []any
875875

876876
if mode == "done" {
877-
vals = []any{"in", interval}
877+
vals = []any{
878+
"in", interval,
879+
"buf", fmt.Sprintf("%s/%s", common.ByteCount(rs.SizeEstimateAfterCommitment()), common.ByteCount(p.commitThreshold)),
880+
}
878881
}
879882

880883
vals = append(vals, []any{

execution/stagedsync/exec3_parallel.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ func (pe *parallelExecutor) exec(ctx context.Context, execStage *StageState, u U
257257
lastBlockResult = *applyResult
258258
}
259259

260-
flushPending = pe.rs.SizeEstimate() > pe.cfg.batchSize.Bytes()
260+
flushPending = pe.rs.SizeEstimateBeforeCommitment() > pe.cfg.batchSize.Bytes()
261261

262262
if !dbg.DiscardCommitment() {
263263
if !dbg.BatchCommitments || shouldGenerateChangesets || lastBlockResult.BlockNum == maxBlockNum ||

execution/stagedsync/exec3_serial.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,7 @@ func (se *serialExecutor) exec(ctx context.Context, execStage *StageState, u Unw
215215
break
216216
}
217217
se.LogExecution()
218-
isBatchFull := se.readState().SizeEstimate() >= se.cfg.batchSize.Bytes()
218+
isBatchFull := se.readState().SizeEstimateBeforeCommitment() >= se.cfg.batchSize.Bytes()
219219
needCalcRoot := isBatchFull || havePartialBlock
220220
// If we have a partial first block it may not be validated, then we should compute root hash ASAP for fail-fast
221221
// this will only happen for the first executed block

execution/state/rw_v3.go

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -275,11 +275,25 @@ func (rs *StateV3) applyLogsAndTraces4(tx kv.TemporalTx, txNum uint64, receipt *
275275
return nil
276276
}
277277

278-
func (rs *StateV3) SizeEstimate() (r uint64) {
279-
if rs.domains != nil {
280-
r += rs.domains.SizeEstimate()
278+
// SizeEstimateBeforeCommitment - including esitmation of future ComputeCommitment on current state changes
279+
func (rs *StateV3) SizeEstimateBeforeCommitment() uint64 {
280+
if rs.domains == nil {
281+
return 0
281282
}
282-
return r
283+
sz := rs.domains.Size()
284+
sz *= 2 // to cover data-structures overhead: map, btree, etc... and GC overhead (clean happening periodically)
285+
sz *= 2 // for Commitment calculation when batch is full
286+
return sz
287+
}
288+
289+
// SizeEstimateAfterCommitment - not including any additional estimations. Use it after ComputeCommitment calc - to see
290+
func (rs *StateV3) SizeEstimateAfterCommitment() uint64 {
291+
if rs.domains == nil {
292+
return 0
293+
}
294+
sz := rs.domains.Size()
295+
sz *= 2 // to cover data-structures overhead: map, btree, etc... and GC overhead (clean happening periodically)
296+
return sz
283297
}
284298

285299
type storageItem struct {

0 commit comments

Comments
 (0)