Skip to content

Commit f4597a3

Browse files
committed
feat(network-fabric): make the ledger-height retry budget configurable
finality.Delivery bounds the GetLedgerInfo retries that decide where a block scan starts, but neither construction site set LedgerInfoAttempts or LedgerInfoRetryDelay, so the budget was effectively a compile-time constant and only tests could vary it. The budget is operationally significant: nothing retries ScanBlock, so a height that stays unreadable costs the channel its block-based finality until the process restarts. How long a peer takes to come back is a deployment property, so expose both values through the existing token.finality.delivery.* mechanism rather than picking one number for everyone. - token.finality.delivery.ledgerInfoAttempts (default 7) - token.finality.delivery.ledgerInfoRetryDelay (default 500ms, doubling) Non-positive values fall back to the defaults: zero attempts would refuse every scan and a non-positive delay would busy loop. Both finality and lookup construct a Delivery, so both providers carry the pair as a finality.LedgerInfoRetry. The exported provider constructors take it through a variadic option, leaving their existing signatures source-compatible. Signed-off-by: AkramBitar <akram@il.ibm.com>
1 parent 52b760b commit f4597a3

8 files changed

Lines changed: 221 additions & 31 deletions

File tree

docs/services/network-fabric.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,9 @@ sequenceDiagram
299299
The scan resumes at the peer's current ledger height, read via `GetLedgerInfo`
300300
([`delivery.go`](../../token/services/network/fabric/finality/delivery.go)). Because that
301301
one RPC decides the starting block, a failure is retried with an exponentially growing
302-
delay — 7 attempts spanning ~31.5s, aborting early if the context is cancelled.
302+
delay — 7 attempts spanning ~31.5s by default, aborting early if the context is cancelled.
303+
Both are configurable via `token.finality.delivery.ledgerInfoAttempts` and
304+
`ledgerInfoRetryDelay` (see [Finality Configuration](#finality-configuration)).
303305

304306
That budget is deliberately long. Nothing retries `ScanBlock`: FSC's
305307
`events.ListenerManager` calls it once from a goroutine that only logs the result, so an
@@ -477,8 +479,15 @@ token:
477479
blockProcessParallelism: 10 # Parallel block processors
478480
lruSize: 30 # Cache size for recent transactions
479481
listenerTimeout: 10s # Timeout for listener notifications
482+
ledgerInfoAttempts: 7 # Attempts at reading the starting ledger height
483+
ledgerInfoRetryDelay: 500ms # First retry pause; doubles each attempt (~31.5s total)
480484
```
481485
486+
`ledgerInfoAttempts` and `ledgerInfoRetryDelay` bound the ledger-height read that decides
487+
where a block scan starts — see [Choosing the Starting Block](#choosing-the-starting-block).
488+
Non-positive values are ignored in favour of the defaults: zero attempts would refuse every
489+
scan, and a non-positive delay would busy loop.
490+
482491
### Endorsement Configuration
483492

484493
```yaml

token/services/network/fabric/config/config.go

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,34 @@ type ListenerManagerConfig interface {
1919
DeliveryListenerTimeout() time.Duration
2020
DeliveryLRUSize() int
2121
DeliveryLRUBuffer() int
22+
DeliveryLedgerInfoAttempts() int
23+
DeliveryLedgerInfoRetryDelay() time.Duration
2224
}
2325

2426
const (
25-
DeliveryMapperParallelism = "token.finality.delivery.mapperParallelism"
26-
DeliveryBlockProcessParallelism = "token.finality.delivery.blockProcessParallelism"
27-
DeliveryLRUSize = "token.finality.delivery.lruSize"
28-
DeliveryLRUBuffer = "token.finality.delivery.lruBuffer"
29-
DeliveryListenerTimeout = "token.finality.delivery.listenerTimeout"
27+
DeliveryMapperParallelism = "token.finality.delivery.mapperParallelism"
28+
DeliveryBlockProcessParallelism = "token.finality.delivery.blockProcessParallelism"
29+
DeliveryLRUSize = "token.finality.delivery.lruSize"
30+
DeliveryLRUBuffer = "token.finality.delivery.lruBuffer"
31+
DeliveryListenerTimeout = "token.finality.delivery.listenerTimeout"
32+
// DeliveryLedgerInfoAttempts bounds how many times the current ledger height is
33+
// read before the block scan is refused. See finality.Delivery.
34+
DeliveryLedgerInfoAttempts = "token.finality.delivery.ledgerInfoAttempts"
35+
// DeliveryLedgerInfoRetryDelay is the pause before the first retry of that read;
36+
// it doubles on each further attempt.
37+
DeliveryLedgerInfoRetryDelay = "token.finality.delivery.ledgerInfoRetryDelay"
3038
DefaultDeliveryMapperParallelism = 10
3139
DefaultDeliveryBlockProcessParallelism = 10
3240
DefaultDeliveryLRUSize = 30
3341
DefaultDeliveryLRUBuffer = 15
3442
DefaultDeliveryListenerTimeout = 10 * time.Second
43+
// DefaultDeliveryLedgerInfoAttempts and DefaultDeliveryLedgerInfoRetryDelay
44+
// span ~31.5s of retries (0.5s + 1s + 2s + 4s + 8s + 16s). The budget has to
45+
// outlast a peer restart rather than a dropped packet: nothing retries the
46+
// block scan, so a height that stays unreadable costs the channel its
47+
// block-based finality until the process restarts.
48+
DefaultDeliveryLedgerInfoAttempts = 7
49+
DefaultDeliveryLedgerInfoRetryDelay = 500 * time.Millisecond
3550
)
3651

3752
type ManagerType string
@@ -88,6 +103,22 @@ func (c *serviceListenerManagerConfig) DeliveryListenerTimeout() time.Duration {
88103
return DefaultDeliveryListenerTimeout
89104
}
90105

106+
func (c *serviceListenerManagerConfig) DeliveryLedgerInfoAttempts() int {
107+
if v := c.c.GetInt(DeliveryLedgerInfoAttempts); v > 0 {
108+
return v
109+
}
110+
111+
return DefaultDeliveryLedgerInfoAttempts
112+
}
113+
114+
func (c *serviceListenerManagerConfig) DeliveryLedgerInfoRetryDelay() time.Duration {
115+
if v := c.c.GetDuration(DeliveryLedgerInfoRetryDelay); v > 0 {
116+
return v
117+
}
118+
119+
return DefaultDeliveryLedgerInfoRetryDelay
120+
}
121+
91122
func (c *serviceListenerManagerConfig) String() string {
92-
return fmt.Sprintf("Delivery [mapperParalellism: %d, lru: (%d, %d), listenerTimeout: %v]", c.DeliveryMapperParallelism(), c.DeliveryLRUSize(), c.DeliveryLRUBuffer(), c.DeliveryListenerTimeout())
123+
return fmt.Sprintf("Delivery [mapperParalellism: %d, lru: (%d, %d), listenerTimeout: %v, ledgerInfo: (%d, %v)]", c.DeliveryMapperParallelism(), c.DeliveryLRUSize(), c.DeliveryLRUBuffer(), c.DeliveryListenerTimeout(), c.DeliveryLedgerInfoAttempts(), c.DeliveryLedgerInfoRetryDelay())
93124
}
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
Copyright IBM Corp. All Rights Reserved.
3+
4+
SPDX-License-Identifier: Apache-2.0
5+
*/
6+
7+
package config_test
8+
9+
import (
10+
"testing"
11+
"time"
12+
13+
"github.com/LFDT-Panurus/panurus/token/services/network/fabric/config"
14+
"github.com/stretchr/testify/assert"
15+
)
16+
17+
// mapConfigService is a driver.ConfigService backed by two maps: whatever the test
18+
// puts in them is "set", everything else reads as a zero value, which is what an
19+
// absent YAML key looks like.
20+
type mapConfigService struct {
21+
ints map[string]int
22+
durations map[string]time.Duration
23+
}
24+
25+
func (c *mapConfigService) GetInt(key string) int { return c.ints[key] }
26+
func (c *mapConfigService) GetDuration(key string) time.Duration { return c.durations[key] }
27+
func (c *mapConfigService) GetString(string) string { return "" }
28+
func (c *mapConfigService) GetBool(string) bool { return false }
29+
func (c *mapConfigService) GetStringSlice(string) []string { return nil }
30+
func (c *mapConfigService) IsSet(string) bool { return false }
31+
func (c *mapConfigService) UnmarshalKey(string, any) error { return nil }
32+
func (c *mapConfigService) ConfigFileUsed() string { return "" }
33+
func (c *mapConfigService) GetPath(string) string { return "" }
34+
func (c *mapConfigService) TranslatePath(path string) string { return path }
35+
36+
// TestLedgerInfoRetryDefaults pins the defaults an unconfigured deployment gets.
37+
// The attempt budget matters beyond taste: nothing retries the block scan, so a
38+
// height that stays unreadable costs the channel its block-based finality until
39+
// the process restarts, and the budget has to outlast a peer restart.
40+
func TestLedgerInfoRetryDefaults(t *testing.T) {
41+
c := config.NewListenerManagerConfig(&mapConfigService{})
42+
43+
assert.Equal(t, config.DefaultDeliveryLedgerInfoAttempts, c.DeliveryLedgerInfoAttempts())
44+
assert.Equal(t, config.DefaultDeliveryLedgerInfoRetryDelay, c.DeliveryLedgerInfoRetryDelay())
45+
assert.GreaterOrEqual(t, totalRetryWait(c.DeliveryLedgerInfoAttempts(), c.DeliveryLedgerInfoRetryDelay()), 20*time.Second,
46+
"the default budget must outlast a peer restart, not just a dropped packet")
47+
}
48+
49+
// TestLedgerInfoRetryReadsConfiguredValues covers the point of the exercise: an
50+
// operator can shorten or lengthen the budget without a rebuild.
51+
func TestLedgerInfoRetryReadsConfiguredValues(t *testing.T) {
52+
c := config.NewListenerManagerConfig(&mapConfigService{
53+
ints: map[string]int{config.DeliveryLedgerInfoAttempts: 12},
54+
durations: map[string]time.Duration{config.DeliveryLedgerInfoRetryDelay: 250 * time.Millisecond},
55+
})
56+
57+
assert.Equal(t, 12, c.DeliveryLedgerInfoAttempts())
58+
assert.Equal(t, 250*time.Millisecond, c.DeliveryLedgerInfoRetryDelay())
59+
}
60+
61+
// TestLedgerInfoRetryRejectsNonPositiveValues covers the values a hand-edited YAML
62+
// can hold: 0 attempts would refuse every scan and a negative delay would busy
63+
// loop, so both fall back to the default rather than being honoured.
64+
func TestLedgerInfoRetryRejectsNonPositiveValues(t *testing.T) {
65+
for _, attempts := range []int{0, -1} {
66+
c := config.NewListenerManagerConfig(&mapConfigService{ints: map[string]int{config.DeliveryLedgerInfoAttempts: attempts}})
67+
assert.Equal(t, config.DefaultDeliveryLedgerInfoAttempts, c.DeliveryLedgerInfoAttempts(), "attempts %d must not be honoured", attempts)
68+
}
69+
70+
for _, delay := range []time.Duration{0, -time.Second} {
71+
c := config.NewListenerManagerConfig(&mapConfigService{durations: map[string]time.Duration{config.DeliveryLedgerInfoRetryDelay: delay}})
72+
assert.Equal(t, config.DefaultDeliveryLedgerInfoRetryDelay, c.DeliveryLedgerInfoRetryDelay(), "delay %v must not be honoured", delay)
73+
}
74+
}
75+
76+
// TestStringReportsLedgerInfoBudget keeps the startup log line informative: the
77+
// budget is otherwise invisible to an operator diagnosing a missing finality.
78+
func TestStringReportsLedgerInfoBudget(t *testing.T) {
79+
s := config.NewListenerManagerConfig(&mapConfigService{}).String()
80+
81+
assert.Contains(t, s, "ledgerInfo:")
82+
}
83+
84+
// totalRetryWait sums the doubling schedule ledgerHeight sleeps through: one wait
85+
// less than the number of attempts, each twice the previous.
86+
func totalRetryWait(attempts int, first time.Duration) time.Duration {
87+
var total time.Duration
88+
for i, delay := 0, first; i < attempts-1; i, delay = i+1, delay*2 {
89+
total += delay
90+
}
91+
92+
return total
93+
}

token/services/network/fabric/finality/delivery.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ type ledgerHeightProvider interface {
5656
GetLedgerInfo() (*fabric.LedgerInfo, error)
5757
}
5858

59+
// LedgerInfoRetry is the retry budget for the ledger-height read that decides
60+
// where a block scan starts, as carried from configuration to a Delivery. A zero
61+
// value selects the defaults.
62+
type LedgerInfoRetry struct {
63+
// Attempts bounds the number of GetLedgerInfo calls.
64+
Attempts int
65+
// Delay is the pause before the first retry; it doubles on each further attempt.
66+
Delay time.Duration
67+
}
68+
5969
type Delivery struct {
6070
Delivery blockFromScanner
6171
Ledger ledgerHeightProvider

token/services/network/fabric/finality/deliveryflm.go

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -103,28 +103,46 @@ func (i TxInfo) ID() driver2.TxID {
103103
}
104104

105105
type deliveryBasedFLMProvider struct {
106-
fnsp *fabric.NetworkServiceProvider
107-
tracerProvider trace.TracerProvider
108-
config events.DeliveryListenerManagerConfig
109-
newMapper newTxInfoMapper
106+
fnsp *fabric.NetworkServiceProvider
107+
tracerProvider trace.TracerProvider
108+
config events.DeliveryListenerManagerConfig
109+
newMapper newTxInfoMapper
110+
ledgerInfoRetry LedgerInfoRetry
110111
}
111112

112-
func NewDeliveryBasedFLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, config events.DeliveryListenerManagerConfig, newMapper newTxInfoMapper) *deliveryBasedFLMProvider {
113-
return &deliveryBasedFLMProvider{
113+
// FLMProviderOption customizes a delivery-based finality listener manager provider.
114+
type FLMProviderOption func(*deliveryBasedFLMProvider)
115+
116+
// WithLedgerInfoRetry sets the retry budget each manager applies to the
117+
// ledger-height read that decides where its block scan starts. Zero fields select
118+
// the Delivery defaults.
119+
func WithLedgerInfoRetry(retry LedgerInfoRetry) FLMProviderOption {
120+
return func(p *deliveryBasedFLMProvider) {
121+
p.ledgerInfoRetry = retry
122+
}
123+
}
124+
125+
func NewDeliveryBasedFLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, config events.DeliveryListenerManagerConfig, newMapper newTxInfoMapper, opts ...FLMProviderOption) *deliveryBasedFLMProvider {
126+
p := &deliveryBasedFLMProvider{
114127
fnsp: fnsp,
115128
tracerProvider: tracerProvider,
116129
config: config,
117130
newMapper: newMapper,
118131
}
132+
for _, opt := range opts {
133+
opt(p)
134+
}
135+
136+
return p
119137
}
120138

121-
func newEndorserDeliveryBasedFLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, keyTranslator translator.KeyTranslator, config events.DeliveryListenerManagerConfig) *deliveryBasedFLMProvider {
139+
func newEndorserDeliveryBasedFLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, keyTranslator translator.KeyTranslator, config events.DeliveryListenerManagerConfig, opts ...FLMProviderOption) *deliveryBasedFLMProvider {
122140
return NewDeliveryBasedFLMProvider(fnsp, tracerProvider, config, func(network, _ string) events.EventInfoMapper[TxInfo] {
123141
return &EndorserTxInfoMapper{
124142
Network: network,
125143
KeyTranslator: keyTranslator,
126144
}
127-
})
145+
}, opts...)
128146
}
129147

130148
func (p *deliveryBasedFLMProvider) NewManager(network, channel string) (ListenerManager, error) {
@@ -142,9 +160,11 @@ func (p *deliveryBasedFLMProvider) NewManager(network, channel string) (Listener
142160
logger,
143161
p.config,
144162
&Delivery{
145-
Delivery: ch.Delivery(),
146-
Ledger: ch.Ledger(),
147-
Logger: logger,
163+
Delivery: ch.Delivery(),
164+
Ledger: ch.Ledger(),
165+
Logger: logger,
166+
LedgerInfoAttempts: p.ledgerInfoRetry.Attempts,
167+
LedgerInfoRetryDelay: p.ledgerInfoRetry.Delay,
148168
},
149169
&DeliveryScanQueryByID{
150170
Delivery: ch.Delivery(),

token/services/network/fabric/finality/manager.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,8 @@ func NewListenerManagerProvider(fnsp *fabric.NetworkServiceProvider, tracerProvi
3131
ListenerTimeout: lmConfig.DeliveryListenerTimeout(),
3232
LRUSize: lmConfig.DeliveryLRUSize(),
3333
LRUBuffer: lmConfig.DeliveryLRUBuffer(),
34-
})
34+
}, WithLedgerInfoRetry(LedgerInfoRetry{
35+
Attempts: lmConfig.DeliveryLedgerInfoAttempts(),
36+
Delay: lmConfig.DeliveryLedgerInfoRetryDelay(),
37+
}))
3538
}

token/services/network/fabric/lookup/deliveryllm.go

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -76,22 +76,40 @@ func (i KeyInfo) ID() driver2.PKey {
7676
}
7777

7878
type deliveryBasedLLMProvider struct {
79-
fnsp *fabric.NetworkServiceProvider
80-
tracerProvider trace.TracerProvider
81-
config events.DeliveryListenerManagerConfig
82-
newMapper newTxInfoMapper
79+
fnsp *fabric.NetworkServiceProvider
80+
tracerProvider trace.TracerProvider
81+
config events.DeliveryListenerManagerConfig
82+
newMapper newTxInfoMapper
83+
ledgerInfoRetry finality2.LedgerInfoRetry
84+
}
85+
86+
// LLMProviderOption customizes a delivery-based lookup listener manager provider.
87+
type LLMProviderOption func(*deliveryBasedLLMProvider)
88+
89+
// WithLedgerInfoRetry sets the retry budget each manager applies to the
90+
// ledger-height read that decides where its block scan starts. Zero fields select
91+
// the finality.Delivery defaults.
92+
func WithLedgerInfoRetry(retry finality2.LedgerInfoRetry) LLMProviderOption {
93+
return func(p *deliveryBasedLLMProvider) {
94+
p.ledgerInfoRetry = retry
95+
}
8396
}
8497

85-
func NewDeliveryBasedLLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, config events.DeliveryListenerManagerConfig, newMapper newTxInfoMapper) *deliveryBasedLLMProvider {
86-
return &deliveryBasedLLMProvider{
98+
func NewDeliveryBasedLLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, config events.DeliveryListenerManagerConfig, newMapper newTxInfoMapper, opts ...LLMProviderOption) *deliveryBasedLLMProvider {
99+
p := &deliveryBasedLLMProvider{
87100
fnsp: fnsp,
88101
tracerProvider: tracerProvider,
89102
config: config,
90103
newMapper: newMapper,
91104
}
105+
for _, opt := range opts {
106+
opt(p)
107+
}
108+
109+
return p
92110
}
93111

94-
func newEndorserDeliveryBasedLLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, keyTranslator translator.KeyTranslator, config events.DeliveryListenerManagerConfig) *deliveryBasedLLMProvider {
112+
func newEndorserDeliveryBasedLLMProvider(fnsp *fabric.NetworkServiceProvider, tracerProvider trace.TracerProvider, keyTranslator translator.KeyTranslator, config events.DeliveryListenerManagerConfig, opts ...LLMProviderOption) *deliveryBasedLLMProvider {
95113
prefix, err := keyTranslator.TransferActionMetadataKeyPrefix()
96114
if err != nil {
97115
panic(err)
@@ -106,7 +124,7 @@ func newEndorserDeliveryBasedLLMProvider(fnsp *fabric.NetworkServiceProvider, tr
106124
network: network,
107125
prefixes: []string{prefix, setupKey},
108126
}
109-
})
127+
}, opts...)
110128
}
111129

112130
func (p *deliveryBasedLLMProvider) NewManager(network, channel string) (ListenerManager, error) {
@@ -123,9 +141,11 @@ func (p *deliveryBasedLLMProvider) NewManager(network, channel string) (Listener
123141
logger,
124142
p.config,
125143
&finality2.Delivery{
126-
Delivery: ch.Delivery(),
127-
Ledger: ch.Ledger(),
128-
Logger: logger,
144+
Delivery: ch.Delivery(),
145+
Ledger: ch.Ledger(),
146+
Logger: logger,
147+
LedgerInfoAttempts: p.ledgerInfoRetry.Attempts,
148+
LedgerInfoRetryDelay: p.ledgerInfoRetry.Delay,
129149
},
130150
&DeliveryScanQueryByID{
131151
Delivery: ch.Delivery(),

token/services/network/fabric/lookup/manager.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/LFDT-Panurus/panurus/token/services/logging"
1111
"github.com/LFDT-Panurus/panurus/token/services/network/common/rws/translator"
1212
"github.com/LFDT-Panurus/panurus/token/services/network/fabric/config"
13+
finality2 "github.com/LFDT-Panurus/panurus/token/services/network/fabric/finality"
1314
"github.com/hyperledger-labs/fabric-smart-client/platform/fabric"
1415
"github.com/hyperledger-labs/fabric-smart-client/platform/fabric/core/generic/events"
1516
"go.opentelemetry.io/otel/trace"
@@ -35,5 +36,8 @@ func NewListenerManagerProvider(fnsp *fabric.NetworkServiceProvider, tracerProvi
3536
ListenerTimeout: lmConfig.DeliveryListenerTimeout(),
3637
LRUSize: lmConfig.DeliveryLRUSize(),
3738
LRUBuffer: lmConfig.DeliveryLRUBuffer(),
38-
})
39+
}, WithLedgerInfoRetry(finality2.LedgerInfoRetry{
40+
Attempts: lmConfig.DeliveryLedgerInfoAttempts(),
41+
Delay: lmConfig.DeliveryLedgerInfoRetryDelay(),
42+
}))
3943
}

0 commit comments

Comments
 (0)