add random backoff to avoid livelock (#1647)

Signed-off-by: Hayim.Shaul@ibm.com <hayimsha@fhe03.vpc.cloud9.ibm.com>

Author: HayimShaul

docs/configuration.md

@@ -292,6 +292,31 @@ token:
                   Security: 256
                 SW:
                   Hash: SHA2
+      # Auditor lock configuration for enrollment ID locking during audit operations
+      # These settings control the retry behavior when multiple auditors compete for locks
+      auditor:
+        lock:
+          # maxRetries is the maximum number of retry attempts for lock acquisition
+          # Default: 10
+          maxRetries: 10
+          
+          # initialBackoff is the initial backoff delay before the first retry
+          # Default: 10ms
+          initialBackoff: 10ms
+          
+          # maxBackoff is the maximum backoff delay between retries
+          # Default: 5s
+          maxBackoff: 5s
+          
+          # backoffMultiplier is the exponential backoff multiplier
+          # Each retry delay is multiplied by this factor
+          # Default: 2.0
+          backoffMultiplier: 2.0
+          
+          # jitterFactor is the randomization factor to prevent thundering herd (0.0 to 1.0)
+          # Adds random jitter to break symmetry when multiple auditors retry simultaneously
+          # Default: 0.3 (30%)
+          jitterFactor: 0.3
                   Security: 256
 ```
 
@@ -420,6 +445,57 @@ Default values:
    - Increase `workerCount` to 8-16 to improve parallel processing
    - Decrease `scanInterval` to 2-3s for faster recovery detection
 
+---
+
+### Optional: token.tms.<name>.auditor.lock
+
+If not specified, the default configuration is:
+
+```yaml
+token:
+  tms:
+    <name>:
+      auditor:
+        lock:
+          maxRetries: 10
+          initialBackoff: 10ms
+          maxBackoff: 5s
+          backoffMultiplier: 2.0
+          jitterFactor: 0.3
+```
+
+Default values:
+
+- maxRetries: 10
+- initialBackoff: 10ms
+- maxBackoff: 5s
+- backoffMultiplier: 2.0
+- jitterFactor: 0.3
+
+**Parameter Descriptions:**
+
+- **maxRetries**: Maximum number of retry attempts when acquiring locks on enrollment IDs during audit operations
+- **initialBackoff**: Initial delay before the first retry attempt
+- **maxBackoff**: Maximum delay between retry attempts (exponential backoff is capped at this value)
+- **backoffMultiplier**: Factor by which the backoff delay increases after each retry (exponential growth)
+- **jitterFactor**: Randomization factor (0.0 to 1.0) added to backoff delays to prevent multiple auditors from retrying simultaneously (prevents thundering herd problem)
+
+**Tuning Recommendations:**
+
+1. **For High-Contention Environments:**
+   - Increase `maxRetries` to 15-20 to handle more lock conflicts
+   - Increase `maxBackoff` to 10s to spread out retry attempts
+   - Keep `jitterFactor` at 0.3 or higher to maintain randomization
+
+2. **For Low-Latency Requirements:**
+   - Decrease `initialBackoff` to 5ms for faster initial retries
+   - Decrease `maxBackoff` to 2s to avoid long waits
+   - Increase `backoffMultiplier` to 3.0 for faster exponential growth
+
+3. **For Resource-Constrained Environments:**
+   - Decrease `maxRetries` to 5 to fail faster
+   - Keep default backoff settings to balance retry attempts with resource usage
+
 2. **For Resource-Constrained Environments:**
    - Decrease `batchSize` to 50 to reduce memory usage
    - Decrease `workerCount` to 2 to reduce CPU load

token/services/auditor/auditor.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hyperledger-labs/fabric-token-sdk/token/services/tokens"
 	"github.com/hyperledger-labs/fabric-token-sdk/token/services/ttx/dep"
 	"github.com/hyperledger-labs/fabric-token-sdk/token/services/ttx/finality"
+	"github.com/hyperledger-labs/fabric-token-sdk/token/services/utils"
 	token2 "github.com/hyperledger-labs/fabric-token-sdk/token/token"
 	"go.opentelemetry.io/otel/trace"
 )
@@ -81,9 +82,11 @@ type Service struct {
 	metricsProvider metrics.Provider
 	metrics         *Metrics
 	checkService    CheckService
+	lockConfig      *LockConfig
 }
 
 // NewService creates a new auditor Service with the provided dependencies.
+// If lockConfig is nil, default lock configuration will be used.
 func NewService(
 	tmsID token.TMSID,
 	networkProvider NetworkProvider,
@@ -93,7 +96,12 @@ func NewService(
 	finalityTracer trace.Tracer,
 	metricsProvider metrics.Provider,
 	checkService CheckService,
+	lockConfig *LockConfig,
 ) *Service {
+	if lockConfig == nil {
+		lockConfig = DefaultLockConfig()
+	}
+
 	return &Service{
 		tmsID:           tmsID,
 		networkProvider: networkProvider,
@@ -104,6 +112,7 @@ func NewService(
 		metricsProvider: metricsProvider,
 		metrics:         newMetrics(metricsProvider),
 		checkService:    checkService,
+		lockConfig:      lockConfig,
 	}
 }
 
@@ -113,7 +122,8 @@ func (a *Service) Validate(ctx context.Context, request *token.Request) error {
 }
 
 // Audit extracts the list of inputs and outputs from the passed transaction.
-// In addition, Audit acquires locks on the enrollment IDs involved in the transaction.
+// In addition, the Audit locks the enrollment named ids with retry logic and exponential backoff
+// to prevent livelock conditions.
 // The caller MUST call Release() to unlock these enrollment IDs after processing.
 //
 // IMPORTANT: The defer Release() statement MUST be placed immediately after checking
@@ -141,18 +151,47 @@ func (a *Service) Audit(ctx context.Context, tx Transaction) (*token.InputStream
 	var eids []string
 	eids = append(eids, record.Inputs.EnrollmentIDs()...)
 	eids = append(eids, record.Outputs.EnrollmentIDs()...)
-	logger.DebugfContext(ctx, "audit transaction [%s], acquire locks", tx.ID())
-	if err := a.auditDB.AcquireLocks(ctx, string(request.Anchor), eids...); err != nil {
+
+	// Acquire locks with retry and exponential backoff to prevent livelock
+	logger.DebugfContext(ctx, "audit transaction [%s], acquire locks with retry", tx.ID())
+	if err := a.acquireLocksWithRetry(ctx, string(request.Anchor), eids); err != nil {
 		a.metrics.AuditLockConflicts.Add(1)
 
 		return nil, nil, err
 	}
+
 	logger.DebugfContext(ctx, "audit transaction [%s], acquire locks done", tx.ID())
 	a.metrics.AuditDuration.Observe(time.Since(start).Seconds())
 
 	return record.Inputs, record.Outputs, nil
 }
 
+// acquireLocksWithRetry attempts to acquire locks with exponential backoff and randomized jitter
+// to prevent livelock conditions when multiple auditors compete for the same enrollment IDs.
+// This implements the mitigation strategy for deadlock/livelock prevention.
+func (a *Service) acquireLocksWithRetry(ctx context.Context, anchor string, eids []string) error {
+	// Create a retry runner with jitter support
+	retryRunner := utils.NewRetryRunnerWithJitter(
+		logger,
+		a.lockConfig.MaxRetries,
+		a.lockConfig.InitialBackoff,
+		a.lockConfig.MaxBackoff,
+		a.lockConfig.BackoffMultiplier,
+		a.lockConfig.JitterFactor,
+	)
+
+	// Use the retry runner to acquire locks
+	err := retryRunner.RunWithContext(ctx, func() error {
+		return a.auditDB.AcquireLocks(ctx, anchor, eids...)
+	})
+
+	if err != nil {
+		return errors.WithMessagef(err, "failed to acquire locks for anchor [%s]", anchor)
+	}
+
+	return nil
+}
+
 // Append adds the passed transaction to the auditor database.
 // It also releases the locks acquired by Audit.
 func (a *Service) Append(ctx context.Context, tx Transaction) error {