Skip to content

Commit 01c54f1

Browse files
authored
feat: add response compression and stream negotiation for API gateway (#627)
1 parent e3aa8d6 commit 01c54f1

10 files changed

Lines changed: 867 additions & 4 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
# dependencies
44
node_modules/
5+
pnpm-lock.yaml
56

67
# Expo
78
.expo/

backend/config/compression.ts

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/**
2+
* Compression configuration for the API gateway.
3+
*
4+
* Controls algorithm negotiation, default levels, per-endpoint overrides,
5+
* the minimum payload threshold, and endpoint skip patterns.
6+
*
7+
* Route handlers can override the per-request level by setting the response
8+
* header X-Compression-Level before the middleware processes the body.
9+
*/
10+
11+
export type CompressionAlgorithm = 'br' | 'gzip' | 'identity';
12+
13+
export interface EndpointCompressionOverride {
14+
algorithm: CompressionAlgorithm;
15+
level: number;
16+
threshold: number;
17+
}
18+
19+
export interface GlobalCompressionConfig {
20+
default: EndpointCompressionOverride;
21+
/** URL path patterns mapped to overrides. Evaluated in insertion order. */
22+
endpointOverrides: Map<string, Partial<EndpointCompressionOverride>>;
23+
/** Regex patterns for paths that must never be compressed. */
24+
skipPatterns: RegExp[];
25+
}
26+
27+
export const X_COMPRESSION_LEVEL_HEADER = 'X-Compression-Level';
28+
29+
export const DEFAULT_COMPRESSION_CONFIG: GlobalCompressionConfig = {
30+
default: {
31+
algorithm: 'br',
32+
level: 4,
33+
threshold: 1024,
34+
},
35+
endpointOverrides: new Map([
36+
['/api/exports/invoices', { level: 5 }],
37+
['/api/exports/dump', { level: 6, threshold: 512 }],
38+
['/api/analytics/reports', { level: 3 }],
39+
['/api/analytics/export', { level: 5 }],
40+
['/api/subscriptions/list', { level: 4, threshold: 2048 }],
41+
]),
42+
skipPatterns: [
43+
/\/stream\/video\//,
44+
/\/downloads\/.*\.(gz|br|zip|mp4|webm|webp|avif)$/,
45+
/\/realtime\/events/,
46+
/^\/ws(\/|$)/,
47+
/\/health$/,
48+
],
49+
};
50+
51+
/**
52+
* Resolve the compression config for a given request path.
53+
*
54+
* @param path - URL pathname (e.g. "/api/exports/invoices/2025-01.csv")
55+
* @param runtimeLevel - Optional value from the X-Compression-Level response header
56+
*/
57+
export function resolveCompressionConfig(
58+
config: GlobalCompressionConfig,
59+
path: string,
60+
runtimeLevel?: number,
61+
): EndpointCompressionOverride {
62+
const resolved: EndpointCompressionOverride = { ...config.default };
63+
64+
for (const [pattern, override] of config.endpointOverrides) {
65+
if (path.startsWith(pattern)) {
66+
Object.assign(resolved, override);
67+
break;
68+
}
69+
}
70+
71+
if (runtimeLevel !== undefined && runtimeLevel >= 0 && runtimeLevel <= 11) {
72+
resolved.level = runtimeLevel;
73+
}
74+
75+
return resolved;
76+
}
77+
78+
/**
79+
* Check whether compression should be skipped entirely for this path.
80+
*/
81+
export function shouldSkipCompression(
82+
config: GlobalCompressionConfig,
83+
path: string,
84+
): boolean {
85+
return config.skipPatterns.some((pattern) => pattern.test(path));
86+
}

backend/gateway/index.ts

Lines changed: 188 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,190 @@
1+
/**
2+
* API Gateway
3+
*
4+
* Express application factory that assembles the middleware pipeline:
5+
* 1. Streaming support (chunked transfer for large payloads)
6+
* 2. Compression negotiation (Brotli/gzip via Accept-Encoding)
7+
* 3. Idempotency (payment route safety)
8+
* 4. Rate limiting
9+
* 5. Standardised response envelope
10+
*
11+
* Usage:
12+
* import { createGateway } from './gateway';
13+
* const app = createGateway();
14+
* app.listen(3000);
15+
*/
16+
17+
import express from 'express';
18+
import type { Application, Request, Response, NextFunction } from 'express';
19+
import { compressionMiddleware } from '../shared/middleware/compression';
20+
import { streamingMiddleware } from '../shared/middleware/streaming';
21+
import { idempotencyMiddleware } from '../services/idempotencyMiddleware';
22+
import { API_VERSION_HEADER, API_VERSION_VALUE } from '../services/shared/apiResponse';
23+
import { REQUEST_ID_HEADER } from '../services/shared/apiResponse';
24+
25+
export interface GatewayOptions {
26+
/** Trust proxy headers (X-Forwarded-For, etc.). Default true. */
27+
trustProxy?: boolean;
28+
/** Disable compression middleware entirely. Default false. */
29+
disableCompression?: boolean;
30+
/** Disable streaming middleware. Default false. */
31+
disableStreaming?: boolean;
32+
}
33+
34+
export function createGateway(options: GatewayOptions = {}): Application {
35+
const app = express();
36+
37+
if (options.trustProxy !== false) {
38+
app.set('trust proxy', true);
39+
}
40+
41+
app.disable('x-powered-by');
42+
43+
app.use(express.json({ limit: '10mb' }));
44+
45+
// ── Response envelope header ──────────────────────────────────────────
46+
app.use((_req: Request, res: Response, next: NextFunction) => {
47+
res.setHeader(API_VERSION_HEADER, API_VERSION_VALUE);
48+
res.setHeader('X-Content-Type-Options', 'nosniff');
49+
res.setHeader('X-Frame-Options', 'DENY');
50+
next();
51+
});
52+
53+
// ── Request ID injection ──────────────────────────────────────────────
54+
app.use((req: Request, _res: Response, next: NextFunction) => {
55+
if (!req.headers[REQUEST_ID_HEADER.toLowerCase()]) {
56+
const { randomUUID } = require('crypto');
57+
req.headers[REQUEST_ID_HEADER.toLowerCase()] = randomUUID();
58+
}
59+
next();
60+
});
61+
62+
// ── Streaming ─────────────────────────────────────────────────────────
63+
if (!options.disableStreaming) {
64+
app.use(streamingMiddleware);
65+
}
66+
67+
// ── Compression ───────────────────────────────────────────────────────
68+
if (!options.disableCompression) {
69+
app.use(compressionMiddleware());
70+
}
71+
72+
// ── Idempotency on payment routes ─────────────────────────────────────
73+
app.post('/api/payments/charge', idempotencyMiddleware);
74+
75+
// ── Export routes (demonstrate streaming + compression) ───────────────
76+
app.get('/api/exports/invoices', async (req: Request, res: Response) => {
77+
res.setHeader('X-Compression-Level', '5');
78+
res.setHeader('Content-Type', 'text/csv; charset=utf-8');
79+
80+
if ((res as any).stream) {
81+
const rows = generateSampleCSVRows(5000);
82+
await (res as any).stream(rows, {
83+
contentType: 'text/csv; charset=utf-8',
84+
contentDisposition: 'attachment; filename="invoices.csv"',
85+
});
86+
} else {
87+
const all = Array.from(generateSampleCSVRows(5000)).join('');
88+
res.send(all);
89+
}
90+
});
91+
92+
app.get('/api/exports/dump', async (req: Request, res: Response) => {
93+
res.setHeader('X-Compression-Level', '6');
94+
res.setHeader('Content-Type', 'application/json; charset=utf-8');
95+
96+
const data = generateSampleJSON(2000);
97+
res.json(data);
98+
});
99+
100+
// ── Health (skip list — no compression) ───────────────────────────────
101+
app.get('/health', (_req: Request, res: Response) => {
102+
res.json({ status: 'ok', uptime: process.uptime() });
103+
});
104+
105+
// ── 404 fallback ──────────────────────────────────────────────────────
106+
app.use((_req: Request, res: Response) => {
107+
res.status(404).json({
108+
success: false,
109+
error: { code: 'NOT_FOUND', message: 'Route not found' },
110+
meta: {
111+
timestamp: new Date().toISOString(),
112+
requestId: '',
113+
apiVersion: 1,
114+
},
115+
});
116+
});
117+
118+
return app;
119+
}
120+
121+
// ── Sample data generators (for demo routes) ──────────────────────────────
122+
123+
function* generateSampleCSVRows(count: number): Generator<string> {
124+
const header = 'id,date,amount,currency,status,customer_id,plan,payment_method\n';
125+
yield header;
126+
127+
for (let i = 1; i <= count; i++) {
128+
const date = new Date(2025, 0, 1 + (i % 365)).toISOString().split('T')[0];
129+
const amount = (Math.random() * 200 + 5).toFixed(2);
130+
const status = ['paid', 'pending', 'failed', 'refunded'][i % 4];
131+
const plan = ['starter', 'pro', 'enterprise', 'pro', 'starter'][i % 5];
132+
const method = ['credit_card', 'paypal', 'stellar', 'bank_transfer'][i % 4];
133+
yield `${i},${date},${amount},USD,${status},cust_${1000 + i},${plan},${method}\n`;
134+
}
135+
}
136+
137+
function generateSampleJSON(count: number): Record<string, unknown> {
138+
const items: Record<string, unknown>[] = [];
139+
for (let i = 1; i <= count; i++) {
140+
items.push({
141+
id: i,
142+
timestamp: new Date(2025, 0, 1 + (i % 365)).toISOString(),
143+
customer: {
144+
id: `cust_${1000 + i}`,
145+
name: `Customer ${i}`,
146+
email: `user${i}@example.com`,
147+
plan: ['starter', 'pro', 'enterprise'][i % 3],
148+
},
149+
subscription: {
150+
status: ['active', 'paused', 'cancelled'][i % 3],
151+
nextBilling: new Date(2025, i % 12, 15).toISOString(),
152+
amount: (Math.random() * 100 + 5).toFixed(2),
153+
currency: 'USD',
154+
},
155+
metadata: {
156+
source: 'api_export',
157+
region: ['us-east', 'eu-west', 'ap-southeast'][i % 3],
158+
version: '1.0',
159+
},
160+
});
161+
}
162+
return { total: count, items, exportedAt: new Date().toISOString() };
163+
}
164+
165+
/**
166+
* Start the gateway server.
167+
*
168+
* @param port - Port to listen on (default from PORT env var or 3000)
169+
* @param options - Gateway options
170+
*/
171+
export function startGateway(
172+
port?: number,
173+
options?: GatewayOptions,
174+
): Application {
175+
const app = createGateway(options);
176+
const listenPort = port ?? parseInt(process.env.PORT || '3000', 10);
177+
app.listen(listenPort, () => {
178+
console.log(`SubTrackr API gateway listening on port ${listenPort}`);
179+
});
180+
return app;
181+
}
182+
183+
// Allow running directly: node backend/gateway/index.js
184+
if (require.main === module) {
185+
startGateway();
186+
}
187+
1188
/**
2189
* Rate-limit anomaly detection gateway (#615).
3190
*
@@ -23,4 +210,4 @@ export {
23210
type LimitAction,
24211
type Severity,
25212
} from "./adaptiveRateLimit";
26-
export { createAdaptiveRateLimitMiddleware } from "./middleware/adaptiveRateLimitMiddleware";
213+
export { createAdaptiveRateLimitMiddleware } from "./middleware/adaptiveRateLimitMiddleware";

backend/services/idempotencyMiddleware.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import {
2121
IdempotencyKeyCollisionError,
2222
IdempotencyRequestInFlightError,
2323
} from './idempotencyService';
24-
import { fail } from './apiResponse';
24+
import { fail } from './shared/apiResponse';
2525

2626
export function idempotencyMiddleware(req: Request, res: Response, next: NextFunction): void {
2727
const key = req.headers[IDEMPOTENCY_KEY_HEADER.toLowerCase()] as string | undefined;

0 commit comments

Comments
 (0)