Added Backoff (#2140)

Signed-off-by: Effi-S <effi.szt@gmail.com>

Author: Effi-S

docs/configuration.md

@@ -415,7 +415,24 @@ token:
           # Adds random jitter to break symmetry when multiple auditors retry simultaneously
           # Default: 0.3 (30%)
           jitterFactor: 0.3
-                  Security: 256
+
+        # auditTokensRetry controls how the auditor's early validation gate
+        # (AuditorCheck) tolerates the read-timing race in which a referenced
+        # token's producing transaction is still pending, so its outputs have not
+        # yet been persisted to the token store by the asynchronous finality
+        # listener. When a lookup misses and the producing tx is still pending,
+        # the gate waits and retries instead of spuriously rejecting the request.
+        auditTokensRetry:
+          # numRetries is the number of token-lookup attempts before giving up.
+          # There are numRetries-1 delays between attempts. Values <= 0 (or an
+          # omitted key) keep the default; a single attempt is always made.
+          # Default: 3
+          numRetries: 3
+          # retryDelay is the backoff slept between attempts, as a duration string
+          # (e.g. 500ms, 3s). Must be > 0; an invalid or non-positive value falls
+          # back to the default. The backoff honors context cancellation.
+          # Default: 3s
+          retryDelay: 3s
 ```
 
 ## Minimal Configuration
@@ -821,6 +838,59 @@ Default values:
 
 ---
 
+### Optional: token.tms.<name>.auditor.auditTokensRetry
+
+Controls how the auditor's early validation gate (`AuditorCheck`) tolerates the
+read-timing race in which a referenced token's producing transaction is still
+pending, so its outputs have not yet been persisted to the token store by the
+asynchronous finality listener. On a missing lookup whose producing transaction is
+still `Pending`, the gate waits `retryDelay` and retries up to `numRetries`
+attempts before failing, instead of spuriously rejecting a validly-audited,
+quickly-chained transaction. This mirrors the tolerance already applied on the
+sibling `Audit()` path. Applies to both the `fabtoken` and `dlog` drivers.
+
+If not specified, the default configuration is:
+
+```yaml
+token:
+  tms:
+    <name>:
+      auditor:
+        auditTokensRetry:
+          numRetries: 3
+          retryDelay: 3s
+```
+
+Default values:
+
+- numRetries: 3
+- retryDelay: 3s
+
+**Parameter Descriptions:**
+
+- **numRetries**: Number of token-lookup attempts made before giving up, with
+  `numRetries - 1` backoff delays between them (so the defaults give a ~6s grace
+  window: 3 attempts, 2 delays of 3s). A value `<= 0`, or an omitted key, keeps the
+  default; at least one attempt is always made. Retries are only spent while a
+  referenced transaction is genuinely `Pending` — a hard lookup failure (or a
+  failure to determine pending status) fails fast without consuming the budget.
+- **retryDelay**: Backoff slept between attempts, expressed as a Go duration string
+  (e.g. `500ms`, `3s`). Must be `> 0`; an invalid or non-positive value logs a
+  warning and falls back to the default. The backoff honors context cancellation,
+  so a cancelled or timed-out request returns immediately rather than pinning a
+  goroutine for the full grace window.
+
+**Tuning Recommendations:**
+
+- **For quickly-chained, high-throughput workloads** (an output spent shortly after
+  it was audited), raise `numRetries` and/or `retryDelay` to widen the grace window
+  and further reduce spurious rejections when finality persistence lags.
+- **For latency-sensitive gates**, lower `retryDelay` (e.g. `500ms`) so the audit
+  gate fails faster when the producing transaction never becomes final, at the cost
+  of tolerating a shorter persistence lag.
+
+---
+
 ### Optional: token.tms.<name>.auditor.locker
 
 Controls the distributed locking strategy used by the auditor to serialise

token/core/common/audit_retry_config.go

@@ -0,0 +1,93 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package common
+
+import (
+	"time"
+
+	"github.com/LFDT-Panurus/panurus/token/services/logging"
+)
+
+// AuditRetryConfigKey is the per-TMS configuration key (relative to the TMS
+// configuration block) holding the audit-token retry/backoff settings consumed by
+// AuditorCheck's RetrieveAuditTokens call.
+const AuditRetryConfigKey = "auditor.auditTokensRetry"
+
+// AuditRetryConfig holds the retry budget and backoff used by RetrieveAuditTokens
+// to tolerate the pending-transaction read-timing race (issue #2105).
+type AuditRetryConfig struct {
+	// NumRetries is the number of ListAuditTokens attempts made before giving up.
+	// A value <= 0 is clamped to a single attempt by RetrieveAuditTokens.
+	NumRetries int
+	// RetryDelay is the backoff slept between attempts (NumRetries-1 delays).
+	RetryDelay time.Duration
+}
+
+// DefaultAuditRetryConfig returns the audit-token retry configuration using the
+// package default constants.
+func DefaultAuditRetryConfig() AuditRetryConfig {
+	return AuditRetryConfig{
+		NumRetries: DefaultAuditTokensNumRetries,
+		RetryDelay: DefaultAuditTokensRetryDelay,
+	}
+}
+
+// AuditConfigProvider is the minimal configuration surface needed to load the
+// audit-token retry configuration. It is satisfied by driver.Configuration (the
+// per-TMS config) and is kept minimal so it is trivial to mock in tests.
+type AuditConfigProvider interface {
+	// IsSet checks whether a configuration key is defined.
+	IsSet(key string) bool
+	// UnmarshalKey decodes the configuration value associated with a key into rawVal.
+	UnmarshalKey(key string, rawVal any) error
+}
+
+// auditRetryConfigRaw is the yaml-facing shape of AuditRetryConfigKey. RetryDelay
+// is decoded as a duration string (e.g. "3s") so operators can express it in
+// human-readable units.
+type auditRetryConfigRaw struct {
+	NumRetries int    `yaml:"numRetries"`
+	RetryDelay string `yaml:"retryDelay"`
+}
+
+// LoadAuditRetryConfig loads the audit-token retry configuration from the passed
+// provider, overlaying any valid configured value onto DefaultAuditRetryConfig().
+// A missing key, an unmarshal failure, or an individually invalid field leaves the
+// corresponding default in place (a warning is logged for invalid values), so this
+// function never fails: the audit gate always has a usable configuration.
+func LoadAuditRetryConfig(cp AuditConfigProvider) AuditRetryConfig {
+	cfg := DefaultAuditRetryConfig()
+
+	if cp == nil || !cp.IsSet(AuditRetryConfigKey) {
+		return cfg
+	}
+
+	var raw auditRetryConfigRaw
+	if err := cp.UnmarshalKey(AuditRetryConfigKey, &raw); err != nil {
+		logging.MustGetLogger().Warnf("failed to unmarshal audit-token retry configuration [%s], using defaults: %v", AuditRetryConfigKey, err)
+
+		return cfg
+	}
+
+	// Apply the retry count if valid. A value <= 0 keeps the default; disabling
+	// retries entirely is intentionally not expressible here, matching the
+	// "at least one attempt" clamp in RetrieveAuditTokens.
+	if raw.NumRetries > 0 {
+		cfg.NumRetries = raw.NumRetries
+	}
+
+	// Apply the retry delay if valid.
+	if raw.RetryDelay != "" {
+		if duration, err := time.ParseDuration(raw.RetryDelay); err == nil && duration > 0 {
+			cfg.RetryDelay = duration
+		} else {
+			logging.MustGetLogger().Warnf("invalid retryDelay value [%s] for key [%s], using default", raw.RetryDelay, AuditRetryConfigKey)
+		}
+	}
+
+	return cfg
+}

token/core/common/audit_retry_config_test.go

@@ -0,0 +1,132 @@
+/*
+Copyright IBM Corp. All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/hyperledger-labs/fabric-smart-client/pkg/utils/errors"
+	"github.com/stretchr/testify/assert"
+)
+
+// stubAuditConfigProvider is a minimal AuditConfigProvider whose IsSet result and
+// UnmarshalKey behavior are supplied per test.
+type stubAuditConfigProvider struct {
+	set         bool
+	unmarshal   func(key string, rawVal any) error
+	lastKey     string
+	unmarshaled bool
+}
+
+func (s *stubAuditConfigProvider) IsSet(string) bool { return s.set }
+
+func (s *stubAuditConfigProvider) UnmarshalKey(key string, rawVal any) error {
+	s.lastKey = key
+	s.unmarshaled = true
+	if s.unmarshal == nil {
+		return nil
+	}
+
+	return s.unmarshal(key, rawVal)
+}
+
+func TestDefaultAuditRetryConfig(t *testing.T) {
+	cfg := DefaultAuditRetryConfig()
+	assert.Equal(t, DefaultAuditTokensNumRetries, cfg.NumRetries)
+	assert.Equal(t, DefaultAuditTokensRetryDelay, cfg.RetryDelay)
+}
+
+func TestLoadAuditRetryConfig(t *testing.T) {
+	t.Run("NilProviderReturnsDefaults", func(t *testing.T) {
+		cfg := LoadAuditRetryConfig(nil)
+		assert.Equal(t, DefaultAuditRetryConfig(), cfg)
+	})
+
+	t.Run("KeyNotSetReturnsDefaults", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: false}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, DefaultAuditRetryConfig(), cfg)
+		assert.False(t, cp.unmarshaled, "UnmarshalKey must not be called when the key is unset")
+	})
+
+	t.Run("UsesConfiguredKey", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true}
+		LoadAuditRetryConfig(cp)
+		assert.Equal(t, AuditRetryConfigKey, cp.lastKey)
+	})
+
+	t.Run("OverridesBothFields", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, rawVal any) error {
+			raw := rawVal.(*auditRetryConfigRaw)
+			raw.NumRetries = 7
+			raw.RetryDelay = "500ms"
+
+			return nil
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, 7, cfg.NumRetries)
+		assert.Equal(t, 500*time.Millisecond, cfg.RetryDelay)
+	})
+
+	t.Run("UnmarshalErrorReturnsDefaults", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, _ any) error {
+			return errors.New("boom")
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, DefaultAuditRetryConfig(), cfg)
+	})
+
+	t.Run("NonPositiveNumRetriesKeepsDefault", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, rawVal any) error {
+			raw := rawVal.(*auditRetryConfigRaw)
+			raw.NumRetries = 0
+			raw.RetryDelay = "2s"
+
+			return nil
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, DefaultAuditTokensNumRetries, cfg.NumRetries)
+		assert.Equal(t, 2*time.Second, cfg.RetryDelay)
+	})
+
+	t.Run("InvalidRetryDelayKeepsDefault", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, rawVal any) error {
+			raw := rawVal.(*auditRetryConfigRaw)
+			raw.NumRetries = 5
+			raw.RetryDelay = "not-a-duration"
+
+			return nil
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, 5, cfg.NumRetries)
+		assert.Equal(t, DefaultAuditTokensRetryDelay, cfg.RetryDelay)
+	})
+
+	t.Run("NonPositiveRetryDelayKeepsDefault", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, rawVal any) error {
+			raw := rawVal.(*auditRetryConfigRaw)
+			raw.RetryDelay = "0s"
+
+			return nil
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, DefaultAuditTokensRetryDelay, cfg.RetryDelay)
+	})
+
+	t.Run("EmptyRetryDelayKeepsDefault", func(t *testing.T) {
+		cp := &stubAuditConfigProvider{set: true, unmarshal: func(_ string, rawVal any) error {
+			raw := rawVal.(*auditRetryConfigRaw)
+			raw.NumRetries = 4
+
+			return nil
+		}}
+		cfg := LoadAuditRetryConfig(cp)
+		assert.Equal(t, 4, cfg.NumRetries)
+		assert.Equal(t, DefaultAuditTokensRetryDelay, cfg.RetryDelay)
+	})
+}