Merge pull request #91 from AdaBliss/feat/rpc-retry-backoff

feat(rpc): add exponential backoff retry for Soroban RPC calls

Author: Tybravo

.env.example

@@ -5,15 +5,17 @@ JWT_SECRET=your-super-secret-jwt-key-change-this
 JWT_EXPIRES_IN=7d
 BCRYPT_ROUNDS=10
 LOG_LEVEL=debug
+
 # Comma-separated list of allowed frontend origins. Use "*" to allow any origin (not recommended for production).
 CORS_ORIGIN=http://localhost:3000,http://localhost:5173
+
 RATE_LIMIT_WINDOW_MS=900000
 RATE_LIMIT_MAX_REQUESTS=100
 
 # ─── Stellar / Soroban ─────────────────────────────────────────────────────────
 # Soroban RPC endpoint.
 # Testnet  : https://soroban-testnet.stellar.org
-# Mainnet  : https://soroban-mainnet.stellar.org  (or a custom Horizon/RPC node)
+# Mainnet  : https://soroban-mainnet.stellar.org (or a custom Horizon/RPC node)
 SOROBAN_RPC_URL=https://soroban-testnet.stellar.org
 
 # Network passphrase — must match SOROBAN_RPC_URL.
@@ -27,22 +29,20 @@ STELLAR_NETWORK=testnet
 # Request timeout (ms) for Soroban RPC calls. Default: 10000
 SOROBAN_RPC_TIMEOUT_MS=10000
 
-# Deployed SwiftChain escrow contract id (starts with "C"). Required by
-# POST /api/v1/transactions/escrow-lock; endpoints that need it return 503 when unset.
-SOROBAN_ESCROW_CONTRACT_ID=
+# Retry/backoff behaviour for Soroban RPC calls (rate limits, transient outages).
+# Maximum attempts per call, including the first. Default: 5
+SOROBAN_RPC_MAX_RETRIES=5
 
-# Contract function invoked to lock escrow funds. Default: lock_escrow
-SOROBAN_ESCROW_LOCK_FUNCTION=lock_escrow
+# Base delay (ms) for exponential backoff between retries. Default: 250
+SOROBAN_RPC_RETRY_BASE_MS=250
 
-# Base fee (stroops) used when building transactions. Default: 100
-STELLAR_BASE_FEE=100
+# Maximum backoff delay (ms) between retries. Default: 8000
+SOROBAN_RPC_RETRY_MAX_MS=8000
 
-# Validity window (seconds) of generated unsigned transactions. Default: 300
-STELLAR_TRANSACTION_TIMEOUT_SECONDS=300
 # ─── Escrow indexer ─────────────────────────────────────────────────────────────
 # Deployed escrow Soroban contract id (the "C..." address) whose events the
 # escrow indexer subscribes to.
 ESCROW_CONTRACT_ID=
 
 # Event topic emitted by the escrow contract when funds are locked. Default: escrow_funded
-ESCROW_FUNDED_EVENT_TOPIC=escrow_funded
+ESCROW_FUNDED_EVENT_TOPIC=escrow_funded

src/blockchain/soroban.service.ts

@@ -1,6 +1,8 @@
 import { rpc as StellarRpc } from '@stellar/stellar-sdk';
 import logger from '../config/logger';
 import { sorobanRpcClient, stellarConfig } from '../config/stellar';
+import env from '../config/env';
+import { withRetry, RetryOptions } from '../utils/rpcRetry';
 
 /**
  * Result returned by a successful connectivity check.
@@ -52,6 +54,27 @@ export class SorobanService {
     this.client = client;
   }
 
+  /**
+   * Retry configuration derived from environment settings, shared by every
+   * RPC call this service makes.
+   */
+  private get retryOptions(): Pick<RetryOptions, 'maxAttempts' | 'baseDelayMs' | 'maxDelayMs'> {
+    return {
+      maxAttempts: env.SOROBAN_RPC_MAX_RETRIES,
+      baseDelayMs: env.SOROBAN_RPC_RETRY_BASE_MS,
+      maxDelayMs: env.SOROBAN_RPC_RETRY_MAX_MS,
+    };
+  }
+
+  /**
+   * Wrap a Soroban RPC call with exponential-backoff retry so transient
+   * failures (rate limiting, temporary node outages) are absorbed instead
+   * of propagating on the first failure.
+   */
+  private callWithRetry<T>(operationName: string, fn: () => Promise<T>): Promise<T> {
+    return withRetry(fn, { ...this.retryOptions, operationName });
+  }
+
   /**
    * Perform a connectivity check against the Soroban RPC node.
    *
@@ -71,8 +94,8 @@ export class SorobanService {
 
     try {
       const [health, ledger] = await Promise.all([
-        this.client.getHealth(),
-        this.client.getLatestLedger(),
+        this.callWithRetry('getHealth', () => this.client.getHealth()),
+        this.callWithRetry('getLatestLedger', () => this.client.getLatestLedger()),
       ]);
 
       const latencyMs = Date.now() - start;
@@ -122,7 +145,7 @@ export class SorobanService {
    * @throws  If the RPC call fails.
    */
   public async getLatestLedger(): Promise<number> {
-    const ledger = await this.client.getLatestLedger();
+    const ledger = await this.callWithRetry('getLatestLedger', () => this.client.getLatestLedger());
     return ledger.sequence;
   }
 
@@ -132,7 +155,7 @@ export class SorobanService {
    * @returns The raw `getNetwork` response from the SDK.
    */
   public async getNetworkInfo(): Promise<StellarRpc.Api.GetNetworkResponse> {
-    return this.client.getNetwork();
+    return this.callWithRetry('getNetwork', () => this.client.getNetwork());
   }
 }
 

src/config/env.ts

@@ -14,6 +14,9 @@ interface EnvConfig {
   CORS_ORIGIN: string;
   RATE_LIMIT_WINDOW_MS: number;
   RATE_LIMIT_MAX_REQUESTS: number;
+  SOROBAN_RPC_MAX_RETRIES: number;
+  SOROBAN_RPC_RETRY_BASE_MS: number;
+  SOROBAN_RPC_RETRY_MAX_MS: number;
 }
 
 const envSchema = z.object({
@@ -27,6 +30,9 @@ const envSchema = z.object({
   CORS_ORIGIN: z.string().default('*'),
   RATE_LIMIT_WINDOW_MS: z.coerce.number().int().min(1000).default(900000),
   RATE_LIMIT_MAX_REQUESTS: z.coerce.number().int().min(1).default(100),
+  SOROBAN_RPC_MAX_RETRIES: z.coerce.number().int().min(1).max(10).default(5),
+  SOROBAN_RPC_RETRY_BASE_MS: z.coerce.number().int().min(1).default(250),
+  SOROBAN_RPC_RETRY_MAX_MS: z.coerce.number().int().min(1).default(8000),
 });
 
 let env: EnvConfig;

src/utils/rpcRetry.ts

@@ -0,0 +1,99 @@
+import logger from '../config/logger';
+
+/**
+ * Options controlling retry/backoff behaviour for `withRetry`.
+ */
+export interface RetryOptions {
+  /** Maximum number of attempts (including the first). Default: 5. */
+  maxAttempts?: number;
+  /** Base delay in milliseconds used for exponential backoff. Default: 250. */
+  baseDelayMs?: number;
+  /** Upper bound on any single backoff delay. Default: 8000. */
+  maxDelayMs?: number;
+  /** Multiplier applied to the delay on each retry. Default: 2. */
+  factor?: number;
+  /** Fraction of jitter (0-1) applied to each computed delay. Default: 0.2. */
+  jitter?: number;
+  /** Label used in log messages to identify the operation being retried. */
+  operationName?: string;
+  /** Predicate deciding whether a given error should trigger a retry. Defaults to retrying everything. */
+  isRetryable?: (error: unknown) => boolean;
+}
+
+const DEFAULT_OPTIONS: Required<Omit<RetryOptions, 'operationName' | 'isRetryable'>> = {
+  maxAttempts: 5,
+  baseDelayMs: 250,
+  maxDelayMs: 8000,
+  factor: 2,
+  jitter: 0.2,
+};
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Compute the delay for a given retry attempt using exponential backoff with
+ * full jitter, capped at `maxDelayMs`.
+ *
+ * @param attempt Zero-based retry attempt number (0 = first retry).
+ */
+export function computeBackoffDelay(
+  attempt: number,
+  options: Required<Omit<RetryOptions, 'operationName' | 'isRetryable'>>,
+): number {
+  const exponential = options.baseDelayMs * Math.pow(options.factor, attempt);
+  const capped = Math.min(exponential, options.maxDelayMs);
+  const jitterRange = capped * options.jitter;
+  const jitterOffset = (Math.random() * 2 - 1) * jitterRange;
+  return Math.max(0, Math.round(capped + jitterOffset));
+}
+
+/**
+ * Execute `fn`, retrying with exponential backoff on failure.
+ *
+ * Intended for wrapping Soroban RPC calls that may fail transiently due to
+ * rate limiting (HTTP 429) or temporary node outages. Every failed attempt
+ * is logged; once all attempts are exhausted the last error is rethrown so
+ * callers can handle it as they would an unwrapped RPC failure.
+ *
+ * @param fn      The async operation to execute.
+ * @param options Retry/backoff configuration.
+ */
+export async function withRetry<T>(fn: () => Promise<T>, options: RetryOptions = {}): Promise<T> {
+  const resolved = { ...DEFAULT_OPTIONS, ...options };
+  const operationName = options.operationName ?? 'rpc-call';
+  const isRetryable = options.isRetryable ?? ((): boolean => true);
+
+  let lastError: unknown;
+
+  for (let attempt = 0; attempt < resolved.maxAttempts; attempt += 1) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+      const attemptNumber = attempt + 1;
+      const isLastAttempt = attemptNumber >= resolved.maxAttempts;
+      const message = err instanceof Error ? err.message : String(err);
+
+      if (!isRetryable(err) || isLastAttempt) {
+        logger.error(
+          `[RPC Retry] ${operationName} failed permanently after ${attemptNumber} attempt(s) — error="${message}"`,
+        );
+        throw err;
+      }
+
+      const delayMs = computeBackoffDelay(attempt, resolved);
+
+      logger.warn(
+        `[RPC Retry] ${operationName} attempt ${attemptNumber}/${resolved.maxAttempts} failed ` +
+          `— error="${message}" — retrying in ${delayMs}ms`,
+      );
+
+      await sleep(delayMs);
+    }
+  }
+
+  // Unreachable in practice (loop always returns or throws), kept for type safety.
+  throw lastError;
+}

tests/rpcRetry.test.ts

@@ -0,0 +1,73 @@
+/**
+ * Unit tests for the withRetry exponential-backoff helper.
+ */
+
+import { withRetry, computeBackoffDelay } from '../src/utils/rpcRetry';
+
+jest.mock('../src/config/logger', () => ({
+  info: jest.fn(),
+  warn: jest.fn(),
+  error: jest.fn(),
+  debug: jest.fn(),
+}));
+
+describe('withRetry', () => {
+  it('returns the result immediately when the operation succeeds on the first try', async () => {
+    const fn = jest.fn().mockResolvedValue('ok');
+
+    const result = await withRetry(fn, { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 });
+
+    expect(result).toBe('ok');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries on failure and eventually succeeds', async () => {
+    const fn = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('timeout'))
+      .mockRejectedValueOnce(new Error('timeout'))
+      .mockResolvedValue('recovered');
+
+    const result = await withRetry(fn, { maxAttempts: 5, baseDelayMs: 1, maxDelayMs: 2 });
+
+    expect(result).toBe('recovered');
+    expect(fn).toHaveBeenCalledTimes(3);
+  });
+
+  it('throws the last error once maxAttempts is exhausted', async () => {
+    const fn = jest.fn().mockRejectedValue(new Error('persistent failure'));
+
+    await expect(withRetry(fn, { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 2 })).rejects.toThrow(
+      'persistent failure',
+    );
+    expect(fn).toHaveBeenCalledTimes(3);
+  });
+
+  it('does not retry when isRetryable returns false', async () => {
+    const fn = jest.fn().mockRejectedValue(new Error('not retryable'));
+
+    await expect(
+      withRetry(fn, {
+        maxAttempts: 5,
+        baseDelayMs: 1,
+        maxDelayMs: 2,
+        isRetryable: () => false,
+      }),
+    ).rejects.toThrow('not retryable');
+    expect(fn).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('computeBackoffDelay', () => {
+  const options = { maxAttempts: 5, baseDelayMs: 100, maxDelayMs: 1000, factor: 2, jitter: 0 };
+
+  it('grows exponentially with the attempt number', () => {
+    expect(computeBackoffDelay(0, options)).toBe(100);
+    expect(computeBackoffDelay(1, options)).toBe(200);
+    expect(computeBackoffDelay(2, options)).toBe(400);
+  });
+
+  it('caps the delay at maxDelayMs', () => {
+    expect(computeBackoffDelay(10, options)).toBe(1000);
+  });
+});

tests/soroban.service.test.ts

@@ -36,6 +36,16 @@ jest.mock('../src/config/stellar', () => ({
   createSorobanRpcClient: jest.fn(),
 }));
 
+// Keep retry backoff delays effectively instant so retry tests run fast.
+jest.mock('../src/config/env', () => ({
+  __esModule: true,
+  default: {
+    SOROBAN_RPC_MAX_RETRIES: 3,
+    SOROBAN_RPC_RETRY_BASE_MS: 1,
+    SOROBAN_RPC_RETRY_MAX_MS: 2,
+  },
+}));
+
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 
 /** Build a minimal mock of rpc.Server with controllable method responses. */
@@ -200,4 +210,32 @@ describe('SorobanService', () => {
       await expect(service.getNetworkInfo()).rejects.toThrow('network error');
     });
   });
+
+  // ── retry / backoff behaviour ──────────────────────────────────────────────
+
+  describe('retry on transient RPC failures', () => {
+    it('retries getLatestLedger and succeeds once the node recovers', async () => {
+      const getLatestLedger = jest
+        .fn()
+        .mockRejectedValueOnce(new Error('429 rate limited'))
+        .mockResolvedValue({ sequence: 999, id: 'xyz', protocolVersion: 21 });
+      const client = makeMockClient({ getLatestLedger });
+      const service = new SorobanService(client);
+
+      const seq = await service.getLatestLedger();
+
+      expect(seq).toBe(999);
+      expect(getLatestLedger).toHaveBeenCalledTimes(2);
+    });
+
+    it('gives up and throws after exhausting configured retry attempts', async () => {
+      const getNetwork = jest.fn().mockRejectedValue(new Error('ECONNRESET'));
+      const client = makeMockClient({ getNetwork });
+      const service = new SorobanService(client);
+
+      await expect(service.getNetworkInfo()).rejects.toThrow('ECONNRESET');
+      // SOROBAN_RPC_MAX_RETRIES is mocked to 3 attempts total.
+      expect(getNetwork).toHaveBeenCalledTimes(3);
+    });
+  });
 });