Skip to content

Commit 81d8f9c

Browse files
authored
Merge pull request #681 from sweetesty/feature/BE-IDX-121-event-indexer-controls
feat(backend): implement event indexer pagination and backpressure co…
2 parents 9d6785d + 855e124 commit 81d8f9c

2 files changed

Lines changed: 245 additions & 11 deletions

File tree

backend/src/indexer/ledger_follower.ts

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,17 @@ export interface LedgerFollowerConfig {
1919
retryDelayMs?: number;
2020
maxRetries?: number;
2121
ledgerGapWarningThreshold?: number;
22+
maxBlockLimit?: number;
23+
highEventDensityThreshold?: number;
2224
}
2325

2426
export interface IndexerStatus {
2527
running: boolean;
2628
lastProcessedLedger: number;
2729
lastPollAt: string | null;
2830
consecutiveErrors: number;
31+
currentBatchSize: number;
32+
isThrottled: boolean;
2933
}
3034

3135
const INDEXER_STATE_ID = 1;
@@ -34,6 +38,8 @@ const DEFAULT_MAX_POLL_INTERVAL_MS = 60_000;
3438
const DEFAULT_RETRY_DELAY_MS = 1_000;
3539
const DEFAULT_MAX_RETRIES = 5;
3640
const DEFAULT_GAP_THRESHOLD = 10;
41+
const DEFAULT_MAX_BLOCK_LIMIT = 100;
42+
const DEFAULT_HIGH_EVENT_DENSITY_THRESHOLD = 5.0;
3743

3844
async function fetchWithRetry(
3945
url: string,
@@ -91,6 +97,8 @@ export class LedgerFollower {
9197
private lastPollAt: string | null = null;
9298
private consecutiveErrors = 0;
9399
private currentPollIntervalMs: number;
100+
private currentBatchSize: number;
101+
private isThrottled = false;
94102

95103
constructor(
96104
private readonly pool: Pool,
@@ -105,9 +113,12 @@ export class LedgerFollower {
105113
retryDelayMs: config.retryDelayMs ?? DEFAULT_RETRY_DELAY_MS,
106114
maxRetries: config.maxRetries ?? DEFAULT_MAX_RETRIES,
107115
ledgerGapWarningThreshold: config.ledgerGapWarningThreshold ?? DEFAULT_GAP_THRESHOLD,
116+
maxBlockLimit: config.maxBlockLimit ?? DEFAULT_MAX_BLOCK_LIMIT,
117+
highEventDensityThreshold: config.highEventDensityThreshold ?? DEFAULT_HIGH_EVENT_DENSITY_THRESHOLD,
108118
};
109119

110120
this.currentPollIntervalMs = this.config.pollIntervalMs;
121+
this.currentBatchSize = this.config.maxBlockLimit;
111122
this.guard = new IdempotencyGuard(pool);
112123
this.topicFilter = buildTopicFilter(config.contractIds, this.config.allowedTopicHashes);
113124
}
@@ -118,6 +129,8 @@ export class LedgerFollower {
118129
lastProcessedLedger: this.lastProcessedLedger,
119130
lastPollAt: this.lastPollAt,
120131
consecutiveErrors: this.consecutiveErrors,
132+
currentBatchSize: this.currentBatchSize,
133+
isThrottled: this.isThrottled,
121134
};
122135
}
123136

@@ -137,9 +150,9 @@ export class LedgerFollower {
137150
private async loop(): Promise<void> {
138151
while (this.running) {
139152
try {
140-
const newEvents = await this.pollOnce();
153+
const { processed, hasMore } = await this.pollOnce();
141154

142-
if (newEvents === 0) {
155+
if (processed === 0 && !hasMore) {
143156
this.currentPollIntervalMs = Math.min(
144157
this.currentPollIntervalMs * 1.5,
145158
this.config.maxPollIntervalMs,
@@ -149,6 +162,13 @@ export class LedgerFollower {
149162
}
150163

151164
this.consecutiveErrors = 0;
165+
this.lastPollAt = new Date().toISOString();
166+
167+
if (hasMore) {
168+
// Catching up: proceed to next batch immediately with a minor delay to prevent event loop exhaustion
169+
await sleep(50);
170+
continue;
171+
}
152172
} catch (err) {
153173
this.consecutiveErrors++;
154174
const backoff = Math.min(
@@ -164,16 +184,15 @@ export class LedgerFollower {
164184
continue;
165185
}
166186

167-
this.lastPollAt = new Date().toISOString();
168187
await sleep(this.currentPollIntervalMs);
169188
}
170189
}
171190

172-
private async pollOnce(): Promise<number> {
191+
private async pollOnce(): Promise<{ processed: number; hasMore: boolean }> {
173192
const latestLedger = await this.fetchLatestLedgerSequence();
174193

175194
if (latestLedger <= this.lastProcessedLedger) {
176-
return 0;
195+
return { processed: 0, hasMore: false };
177196
}
178197

179198
const gap = latestLedger - this.lastProcessedLedger;
@@ -186,7 +205,8 @@ export class LedgerFollower {
186205
});
187206
}
188207

189-
const events = await this.fetchEvents(this.lastProcessedLedger + 1, latestLedger);
208+
const targetLedger = Math.min(latestLedger, this.lastProcessedLedger + this.currentBatchSize);
209+
const events = await this.fetchEvents(this.lastProcessedLedger + 1, targetLedger);
190210
const relevant = events.filter(this.topicFilter);
191211
let processed = 0;
192212

@@ -202,18 +222,43 @@ export class LedgerFollower {
202222
processed++;
203223
}
204224

205-
await this.saveLastProcessedLedger(latestLedger);
206-
this.lastProcessedLedger = latestLedger;
225+
await this.saveLastProcessedLedger(targetLedger);
226+
const prevProcessedLedger = this.lastProcessedLedger;
227+
this.lastProcessedLedger = targetLedger;
228+
229+
// Calculate event density in this batch
230+
const batchRange = targetLedger - prevProcessedLedger;
231+
const density = events.length / Math.max(1, batchRange);
232+
233+
if (density > this.config.highEventDensityThreshold) {
234+
this.isThrottled = true;
235+
// Multiplicative decrease
236+
this.currentBatchSize = Math.max(10, Math.floor(this.currentBatchSize * 0.5));
237+
logger.warn("High event density detected. Applying backpressure and throttling batch size", {
238+
density,
239+
threshold: this.config.highEventDensityThreshold,
240+
newBatchSize: this.currentBatchSize,
241+
eventsCount: events.length,
242+
});
243+
// Apply backpressure throttle delay
244+
await sleep(500);
245+
} else {
246+
this.isThrottled = false;
247+
// Additive increase
248+
this.currentBatchSize = Math.min(this.config.maxBlockLimit, this.currentBatchSize + 10);
249+
}
207250

208251
logger.info("Poll cycle complete", {
209-
fromLedger: this.lastProcessedLedger - (latestLedger - this.lastProcessedLedger),
210-
toLedger: latestLedger,
252+
fromLedger: prevProcessedLedger + 1,
253+
toLedger: targetLedger,
211254
totalEvents: events.length,
212255
relevantEvents: relevant.length,
213256
newlyProcessed: processed,
257+
currentBatchSize: this.currentBatchSize,
258+
isThrottled: this.isThrottled,
214259
});
215260

216-
return processed;
261+
return { processed, hasMore: targetLedger < latestLedger };
217262
}
218263

219264
private async fetchLatestLedgerSequence(): Promise<number> {
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
import test from "node:test";
2+
import assert from "node:assert/strict";
3+
import Module from "node:module";
4+
import { EventEmitter } from "node:events";
5+
6+
// ── Mock network clients ──
7+
let mockGetHandler: (url: string, options: any, callback: (res: any) => void) => any = () => {
8+
throw new Error("mockGetHandler not set");
9+
};
10+
11+
const originalLoad = (Module as any)._load;
12+
(Module as any)._load = function patchedLoad(request: string, parent: unknown, isMain: boolean) {
13+
if (request === "http" || request === "https") {
14+
return {
15+
get: (url: string, options: any, callback: (res: any) => void) => {
16+
return mockGetHandler(url, options, callback);
17+
}
18+
};
19+
}
20+
return originalLoad.apply(this, [request, parent, isMain]);
21+
};
22+
23+
// Now import the LedgerFollower
24+
import { LedgerFollower } from "../src/indexer/ledger_follower";
25+
26+
function createMockResponse(body: string) {
27+
const res = new EventEmitter() as any;
28+
process.nextTick(() => {
29+
res.emit("data", Buffer.from(body));
30+
res.emit("end");
31+
});
32+
return res;
33+
}
34+
35+
function createMockRequest() {
36+
const req = new EventEmitter() as any;
37+
req.destroy = () => {};
38+
return req;
39+
}
40+
41+
// ── Test Cases ──
42+
43+
test("LedgerFollower uses dynamic pagination and batches query ranges correctly", async () => {
44+
let dbSavedLedger = 0;
45+
const mockPool = {
46+
query: async (queryText: string, params?: any[]) => {
47+
if (queryText.includes("SELECT last_processed_ledger")) {
48+
return { rowCount: 1, rows: [{ last_processed_ledger: "500" }] };
49+
}
50+
if (queryText.includes("INSERT INTO indexer_state")) {
51+
dbSavedLedger = params?.[1];
52+
return { rowCount: 1, rows: [] };
53+
}
54+
return { rowCount: 0, rows: [] };
55+
}
56+
} as any;
57+
58+
// Set up mock HTTP response
59+
mockGetHandler = (url: string, options: any, callback: any) => {
60+
const req = createMockRequest();
61+
if (url.includes("/ledgers?")) {
62+
callback(createMockResponse(JSON.stringify({
63+
_embedded: { records: [{ sequence: 1000 }] }
64+
})));
65+
} else if (url.includes("/soroban/events?")) {
66+
// Return 0 events
67+
callback(createMockResponse(JSON.stringify({
68+
events: []
69+
})));
70+
}
71+
return req;
72+
};
73+
74+
const follower = new LedgerFollower(mockPool, {
75+
stellarRpcUrl: "https://mock-stellar.com",
76+
contractIds: ["mock-contract"],
77+
maxBlockLimit: 100,
78+
pollIntervalMs: 1000,
79+
});
80+
81+
// Manually set internal state instead of start() to avoid background loop
82+
(follower as any).lastProcessedLedger = 500;
83+
(follower as any).running = true;
84+
85+
const result = await (follower as any).pollOnce();
86+
87+
assert.equal(result.processed, 0);
88+
assert.equal(result.hasMore, true); // Since 600 < 1000
89+
assert.equal(follower.status.lastProcessedLedger, 600); // 500 + maxBlockLimit (100)
90+
assert.equal(dbSavedLedger, 600); // Sequence saved persistently to indexer_state
91+
});
92+
93+
test("LedgerFollower applies multiplicative decrease throttling when event density is high", async () => {
94+
let dbSavedLedger = 0;
95+
const mockPool = {
96+
query: async (queryText: string, params?: any[]) => {
97+
if (queryText.includes("SELECT last_processed_ledger")) {
98+
return { rowCount: 1, rows: [{ last_processed_ledger: "500" }] };
99+
}
100+
if (queryText.includes("INSERT INTO indexer_state")) {
101+
dbSavedLedger = params?.[1];
102+
return { rowCount: 1, rows: [] };
103+
}
104+
return { rowCount: 0, rows: [] };
105+
}
106+
} as any;
107+
108+
// Mock high density events return: 600 events inside the batch of 100 ledgers
109+
mockGetHandler = (url: string, options: any, callback: any) => {
110+
const req = createMockRequest();
111+
if (url.includes("/ledgers?")) {
112+
callback(createMockResponse(JSON.stringify({
113+
_embedded: { records: [{ sequence: 1000 }] }
114+
})));
115+
} else if (url.includes("/soroban/events?")) {
116+
const mockEvents = Array.from({ length: 600 }, (_, i) => ({
117+
id: `event-${i}`,
118+
ledger: 550,
119+
contractId: "mock-contract",
120+
topic: ["test"],
121+
value: "value"
122+
}));
123+
callback(createMockResponse(JSON.stringify({
124+
events: mockEvents
125+
})));
126+
}
127+
return req;
128+
};
129+
130+
const follower = new LedgerFollower(mockPool, {
131+
stellarRpcUrl: "https://mock-stellar.com",
132+
contractIds: ["mock-contract"],
133+
maxBlockLimit: 100,
134+
highEventDensityThreshold: 5.0, // Throttles if density > 5.0 events/block
135+
pollIntervalMs: 1000,
136+
});
137+
138+
(follower as any).lastProcessedLedger = 500;
139+
(follower as any).running = true;
140+
141+
await (follower as any).pollOnce();
142+
143+
// Halved from 100 to 50
144+
assert.equal(follower.status.currentBatchSize, 50);
145+
assert.equal(follower.status.isThrottled, true);
146+
assert.equal(dbSavedLedger, 600);
147+
});
148+
149+
test("LedgerFollower applies additive increase recovery when event density is low", async () => {
150+
const mockPool = {
151+
query: async (queryText: string) => {
152+
if (queryText.includes("SELECT last_processed_ledger")) {
153+
return { rowCount: 1, rows: [{ last_processed_ledger: "500" }] };
154+
}
155+
return { rowCount: 0, rows: [] };
156+
}
157+
} as any;
158+
159+
mockGetHandler = (url: string, options: any, callback: any) => {
160+
const req = createMockRequest();
161+
if (url.includes("/ledgers?")) {
162+
callback(createMockResponse(JSON.stringify({
163+
_embedded: { records: [{ sequence: 1000 }] }
164+
})));
165+
} else if (url.includes("/soroban/events?")) {
166+
callback(createMockResponse(JSON.stringify({ events: [] })));
167+
}
168+
return req;
169+
};
170+
171+
const follower = new LedgerFollower(mockPool, {
172+
stellarRpcUrl: "https://mock-stellar.com",
173+
contractIds: ["mock-contract"],
174+
maxBlockLimit: 100,
175+
pollIntervalMs: 1000,
176+
});
177+
178+
(follower as any).lastProcessedLedger = 500;
179+
(follower as any).running = true;
180+
181+
// Set initial batch size to a low value to test additive increase
182+
(follower as any).currentBatchSize = 50;
183+
184+
await (follower as any).pollOnce();
185+
186+
// Additively increased from 50 to 60 (+10)
187+
assert.equal(follower.status.currentBatchSize, 60);
188+
assert.equal(follower.status.isThrottled, false);
189+
});

0 commit comments

Comments
 (0)