[#1145] Add fail-safe rate limiting for distributed store outages

.env.example

@@ -84,6 +84,12 @@ RATE_LIMIT_MAX_REQUESTS=5
 RATE_LIMIT_WINDOW_MS=60000
 RATE_LIMIT_STORE=memory
 RATE_LIMIT_PG_TABLE=gateway_rate_limit_buckets
+# Behavior when the distributed rate-limit store is unavailable. `fail-closed`
+# rejects protected requests; `fallback` allows only the bounded local policy.
+RATE_LIMIT_OUTAGE_MODE=fail-closed
+RATE_LIMIT_FALLBACK_MAX_REQUESTS=10
+RATE_LIMIT_FALLBACK_WINDOW_MS=60000
+RATE_LIMIT_FALLBACK_MAX_BUCKETS=10000
 
 # -----------------------------------------------------------------------------
 # Credits endpoint token-bucket rate limiting (GET /api/billing/credits)

README.md

@@ -463,6 +463,10 @@ For request-id validation, AsyncLocalStorage propagation, structured logging, an
 | `RATE_LIMIT_WINDOW_MS` | No | `60000` | Token-bucket refill window for `RATE_LIMIT_MAX_REQUESTS` (ms) |
 | `RATE_LIMIT_STORE` | No | `memory` | `memory` or `postgres`. Use `postgres` to share bucket state across multiple gateway instances |
 | `RATE_LIMIT_PG_TABLE` | No | `gateway_rate_limit_buckets` | Table name used when `RATE_LIMIT_STORE=postgres` (auto-created) |
+| `RATE_LIMIT_OUTAGE_MODE` | No | `fail-closed` | Distributed-store outage policy: reject protected requests or use the bounded local fallback (`fallback`) |
+| `RATE_LIMIT_FALLBACK_MAX_REQUESTS` | No | `10` | Maximum requests per key during fallback mode; never exceeds the distributed request policy |
+| `RATE_LIMIT_FALLBACK_WINDOW_MS` | No | `60000` | Fallback window length in milliseconds |
+| `RATE_LIMIT_FALLBACK_MAX_BUCKETS` | No | `10000` | Hard cap on local fallback keys; oldest keys are evicted during an outage |
 | `QUOTA_RATE_LIMIT_CAPACITY` | No | `60` | Token-bucket burst capacity for all `/api/quotas` endpoints (per user / IP) |
 | `QUOTA_RATE_LIMIT_REFILL_RATE` | No | `1` | Tokens added per second to each `/api/quotas` bucket; governs steady-state request rate |
 | `CORS_ALLOWED_ORIGINS` | No | `http://localhost:5173` | Comma-separated allowed origins |

src/config/env.test.ts

@@ -153,6 +153,10 @@ describe('env schema — gateway rate limit config', () => {
       expect(result.data.RATE_LIMIT_WINDOW_MS).toBe(60_000);
       expect(result.data.RATE_LIMIT_STORE).toBe('memory');
       expect(result.data.RATE_LIMIT_PG_TABLE).toBe('gateway_rate_limit_buckets');
+      expect(result.data.RATE_LIMIT_OUTAGE_MODE).toBe('fail-closed');
+      expect(result.data.RATE_LIMIT_FALLBACK_MAX_REQUESTS).toBe(10);
+      expect(result.data.RATE_LIMIT_FALLBACK_WINDOW_MS).toBe(60_000);
+      expect(result.data.RATE_LIMIT_FALLBACK_MAX_BUCKETS).toBe(10_000);
     }
   });
 
@@ -173,6 +177,35 @@ describe('env schema — gateway rate limit config', () => {
     }
   });
 
+  it('accepts explicit fallback outage policy and bounds', () => {
+    const result = envSchema.safeParse({
+      ...baseEnv,
+      RATE_LIMIT_STORE: 'postgres',
+      RATE_LIMIT_OUTAGE_MODE: 'fallback',
+      RATE_LIMIT_FALLBACK_MAX_REQUESTS: '7',
+      RATE_LIMIT_FALLBACK_WINDOW_MS: '15000',
+      RATE_LIMIT_FALLBACK_MAX_BUCKETS: '250',
+    });
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.RATE_LIMIT_OUTAGE_MODE).toBe('fallback');
+      expect(result.data.RATE_LIMIT_FALLBACK_MAX_REQUESTS).toBe(7);
+      expect(result.data.RATE_LIMIT_FALLBACK_WINDOW_MS).toBe(15_000);
+      expect(result.data.RATE_LIMIT_FALLBACK_MAX_BUCKETS).toBe(250);
+    }
+  });
+
+  it('rejects an unsupported outage mode and unsafe fallback dimensions', () => {
+    const result = envSchema.safeParse({
+      ...baseEnv,
+      RATE_LIMIT_OUTAGE_MODE: 'allow-all',
+      RATE_LIMIT_FALLBACK_MAX_REQUESTS: '0',
+      RATE_LIMIT_FALLBACK_WINDOW_MS: '-1',
+      RATE_LIMIT_FALLBACK_MAX_BUCKETS: '0',
+    });
+    expect(result.success).toBe(false);
+  });
+
   it('rejects a store value other than "memory" or "postgres"', () => {
     const result = envSchema.safeParse({
       ...baseEnv,

src/config/env.ts

@@ -108,6 +108,10 @@ export const envSchema = z
     RATE_LIMIT_MAX_REQUESTS: z.coerce.number().int().positive().default(5),
     RATE_LIMIT_WINDOW_MS: z.coerce.number().int().positive().default(60_000),
     RATE_LIMIT_STORE: z.enum(["memory", "postgres"]).default("memory"),
+    RATE_LIMIT_OUTAGE_MODE: z.enum(["fail-closed", "fallback"]).default("fail-closed"),
+    RATE_LIMIT_FALLBACK_MAX_REQUESTS: z.coerce.number().int().positive().default(10),
+    RATE_LIMIT_FALLBACK_WINDOW_MS: z.coerce.number().int().positive().default(60_000),
+    RATE_LIMIT_FALLBACK_MAX_BUCKETS: z.coerce.number().int().positive().default(10_000),
     RATE_LIMIT_PG_TABLE: z
       .string()
       .regex(

src/config/index.test.ts

@@ -91,6 +91,10 @@ describe('config validation', () => {
               windowMs: number;
               store: 'memory' | 'postgres';
               postgresTable: string;
+              outageMode: 'fail-closed' | 'fallback';
+              fallbackMaxRequests: number;
+              fallbackWindowMs: number;
+              maxFallbackBuckets: number;
             };
           };
         }
@@ -104,6 +108,10 @@ describe('config validation', () => {
       windowMs: 60_000,
       store: 'memory',
       postgresTable: 'gateway_rate_limit_buckets',
+      outageMode: 'fail-closed',
+      fallbackMaxRequests: 10,
+      fallbackWindowMs: 60_000,
+      maxFallbackBuckets: 10_000,
     });
   });
 
@@ -125,6 +133,10 @@ describe('config validation', () => {
               windowMs: number;
               store: 'memory' | 'postgres';
               postgresTable: string;
+              outageMode: 'fail-closed' | 'fallback';
+              fallbackMaxRequests: number;
+              fallbackWindowMs: number;
+              maxFallbackBuckets: number;
             };
           };
         }
@@ -138,6 +150,10 @@ describe('config validation', () => {
       windowMs: 10_000,
       store: 'postgres',
       postgresTable: 'custom_rate_limit_buckets',
+      outageMode: 'fail-closed',
+      fallbackMaxRequests: 10,
+      fallbackWindowMs: 60_000,
+      maxFallbackBuckets: 10_000,
     });
   });
 

src/config/index.ts

@@ -201,6 +201,10 @@ export const config = {
     windowMs: env.RATE_LIMIT_WINDOW_MS,
     store: env.RATE_LIMIT_STORE,
     postgresTable: env.RATE_LIMIT_PG_TABLE,
+    outageMode: env.RATE_LIMIT_OUTAGE_MODE,
+    fallbackMaxRequests: env.RATE_LIMIT_FALLBACK_MAX_REQUESTS,
+    fallbackWindowMs: env.RATE_LIMIT_FALLBACK_WINDOW_MS,
+    maxFallbackBuckets: env.RATE_LIMIT_FALLBACK_MAX_BUCKETS,
   },
 
   sorobanRpc:

src/metrics.ts

@@ -7,6 +7,29 @@ import { UnauthorizedError } from './errors/index.js';
 export const register = new client.Registry();
 client.collectDefaultMetrics({ register });
 
+const rateLimiterStoreOutages = new client.Counter({
+  name: 'rate_limiter_store_outages_total',
+  help: 'Number of distributed rate-limiter store outages observed',
+  labelNames: ['outage_mode'],
+});
+
+const rateLimiterStoreDegraded = new client.Gauge({
+  name: 'rate_limiter_store_degraded',
+  help: 'Whether the distributed rate-limiter store is currently degraded',
+});
+
+register.registerMetric(rateLimiterStoreOutages);
+register.registerMetric(rateLimiterStoreDegraded);
+
+export function recordRateLimiterStoreOutage(outageMode: 'fail-closed' | 'fallback'): void {
+  rateLimiterStoreOutages.inc({ outage_mode: outageMode });
+  rateLimiterStoreDegraded.set(1);
+}
+
+export function recordRateLimiterStoreRecovery(): void {
+  rateLimiterStoreDegraded.set(0);
+}
+
 // ── Route groups ──────────────────────────────────────────────────────────────
 //
 // A `route_group` label is added to every HTTP metric so dashboards can slice