Skip to content

Commit 00e747f

Browse files
daywissdaywiss
authored
improve(indexer): refactor configuration parsing (#45)
Signed-off-by: david <[email protected]>
1 parent a348302 commit 00e747f

File tree

10 files changed

+230
-205
lines changed

10 files changed

+230
-205
lines changed

Diff for: ‎apps/node/src/app.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ async function run() {
3737
void (await Template.Main(process.env));
3838
return "Example template app running";
3939
case "indexer":
40-
void (await Indexer.Main(process.env, logger));
40+
void (await Indexer.Main(Indexer.envToConfig(process.env), logger));
4141
break;
4242
case "persistence-example":
4343
void (await PersistenceExample.Main(process.env));

Diff for: ‎packages/indexer/README.md

+27
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,30 @@ Without docker:
1717

1818
## Test
1919
In this package run `pnpm test`
20+
21+
## ENV
22+
```
23+
DATABASE_HOST=localhost
24+
DATABASE_PORT=5432
25+
DATABASE_USER=user
26+
DATABASE_PASSWORD=password
27+
DATABASE_NAME=mydatabase
28+
29+
REDIS_HOST=localhost
30+
REDIS_PORT=6380
31+
RPC_PROVIDER_URLS_1=https://mainnet.infura.io/v3/xxx
32+
RPC_PROVIDER_URLS_10=https://optimism-mainnet.infura.io/v3/xxx
33+
RPC_PROVIDER_URLS_137=https://polygon-mainnet.infura.io/v3/xxx
34+
HUBPOOL_CHAIN=1
35+
SPOKEPOOL_CHAINS_ENABLED=1,2
36+
37+
PROVIDER_CACHE_NAMESPACE=indexer_provider_cache
38+
MAX_CONCURRENCY=1
39+
PCT_RPC_CALLS_LOGGED=100
40+
STANDARD_TTL_BLOCK_DISTANCE=1
41+
NO_TTL_BLOCK_DISTANCE=1000
42+
PROVIDER_CACHE_TTL=100000
43+
NODE_QUORUM_THRESHOLD=1
44+
RETRIES=2
45+
DELAY=1000
46+
```

Diff for: ‎packages/indexer/src/generics/BaseIndexer.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ export abstract class BaseIndexer {
3737
this.logger.error({
3838
at: "BaseIndexer#start",
3939
message: `Failed to initialize ${this.name}`,
40-
error: e,
40+
error: (e as unknown as Error).message,
4141
});
4242
return;
4343
}

Diff for: ‎packages/indexer/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
export * from "./main";
2+
export * from "./parseEnv";

Diff for: ‎packages/indexer/src/main.ts

+13-185
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,14 @@
1-
import assert from "assert";
21
import * as services from "./services";
32
import winston from "winston";
43
import Redis from "ioredis";
5-
import * as s from "superstruct";
6-
import * as acrossConstants from "@across-protocol/constants";
74
import * as across from "@across-protocol/sdk";
85
import { connectToDatabase } from "./database/database.provider";
9-
import { providers } from "ethers";
10-
import { DatabaseConfig } from "@repo/indexer-database";
6+
import * as parseEnv from "./parseEnv";
117

12-
type RedisConfig = {
13-
host: string;
14-
port: number;
15-
};
16-
async function initializeRedis(config: RedisConfig, logger: winston.Logger) {
8+
async function initializeRedis(
9+
config: parseEnv.RedisConfig,
10+
logger: winston.Logger,
11+
) {
1712
const redis = new Redis({
1813
...config,
1914
});
@@ -31,197 +26,30 @@ async function initializeRedis(config: RedisConfig, logger: winston.Logger) {
3126
});
3227
}
3328

34-
function getPostgresConfig(
35-
env: Record<string, string | undefined>,
36-
): DatabaseConfig {
37-
assert(env.DATABASE_HOST, "requires DATABASE_HOST");
38-
assert(env.DATABASE_PORT, "requires DATABASE_PORT");
39-
assert(env.DATABASE_USER, "requires DATABASE_USER");
40-
assert(env.DATABASE_PASSWORD, "requires DATABASE_PASSWORD");
41-
assert(env.DATABASE_NAME, "requires DATABASE_NAME");
42-
return {
43-
host: env.DATABASE_HOST,
44-
port: env.DATABASE_PORT,
45-
user: env.DATABASE_USER,
46-
password: env.DATABASE_PASSWORD,
47-
dbName: env.DATABASE_NAME,
48-
};
49-
}
50-
51-
type RetryProviderConfig = {
52-
providerCacheNamespace: string;
53-
maxConcurrency: number;
54-
pctRpcCallsLogged: number;
55-
standardTtlBlockDistance: number;
56-
noTtlBlockDistance: number;
57-
providerCacheTtl: number;
58-
nodeQuorumThreshold: number;
59-
retries: number;
60-
delay: number;
61-
};
62-
// superstruct coersion to turn string into an int and validate
63-
const stringToInt = s.coerce(s.number(), s.string(), (value) =>
64-
parseInt(value),
65-
);
66-
function getRetryProviderConfig(
67-
env: Record<string, string | undefined>,
68-
): RetryProviderConfig {
69-
assert(env.PROVIDER_CACHE_NAMESPACE, "requires PROVIDER_CACHE_NAMESPACE");
70-
assert(env.MAX_CONCURRENCY, "requires MAX_CONCURRENCY");
71-
assert(env.PCT_RPC_CALLS_LOGGED, "requires PCT_RPC_CALLS_LOGGED");
72-
assert(
73-
env.STANDARD_TTL_BLOCK_DISTANCE,
74-
"requires STANDARD_TTL_BLOCK_DISTANCE",
75-
);
76-
assert(env.NO_TTL_BLOCK_DISTANCE, "requires NO_TTL_BLOCK_DISTANCE");
77-
assert(env.PROVIDER_CACHE_TTL, "requires PROVIDER_CACHE_TTL");
78-
assert(env.NODE_QUORUM_THRESHOLD, "requires NODE_QUORUM_THRESHOLD");
79-
assert(env.RETRIES, "requires RETRIES");
80-
assert(env.DELAY, "requires DELAY");
81-
return {
82-
providerCacheNamespace: env.PROVIDER_CACHE_NAMESPACE,
83-
maxConcurrency: s.create(env.MAX_CONCURRENCY, stringToInt),
84-
pctRpcCallsLogged: s.create(env.PCT_RPC_CALLS_LOGGED, stringToInt),
85-
standardTtlBlockDistance: s.create(
86-
env.STANDARD_TTL_BLOCK_DISTANCE,
87-
stringToInt,
88-
),
89-
noTtlBlockDistance: s.create(env.NO_TTL_BLOCK_DISTANCE, stringToInt),
90-
providerCacheTtl: s.create(env.PROVIDER_CACHE_TTL, stringToInt),
91-
nodeQuorumThreshold: s.create(env.NODE_QUORUM_THRESHOLD, stringToInt),
92-
retries: s.create(env.RETRIES, stringToInt),
93-
delay: s.create(env.DELAY, stringToInt),
94-
};
95-
}
96-
97-
// utility call to create the spoke pool event indexer config
98-
async function getSpokePoolIndexerConfig(params: {
99-
retryProviderConfig: RetryProviderConfig;
100-
spokePoolProviderUrl: string;
101-
hubPoolNetworkInfo: providers.Network;
102-
hubPoolProviderUrl: string;
103-
}) {
104-
const {
105-
retryProviderConfig,
106-
spokePoolProviderUrl,
107-
hubPoolProviderUrl,
108-
hubPoolNetworkInfo,
109-
} = params;
110-
const tempSpokeProvider = new providers.JsonRpcProvider(spokePoolProviderUrl);
111-
const spokePoolNetworkInfo = await tempSpokeProvider.getNetwork();
112-
return {
113-
retryProviderConfig,
114-
configStoreConfig: {
115-
chainId: hubPoolNetworkInfo.chainId,
116-
providerUrl: hubPoolProviderUrl,
117-
maxBlockLookBack: 10000,
118-
},
119-
hubConfig: {
120-
chainId: hubPoolNetworkInfo.chainId,
121-
providerUrl: hubPoolProviderUrl,
122-
maxBlockLookBack: 10000,
123-
},
124-
spokeConfig: {
125-
chainId: spokePoolNetworkInfo.chainId,
126-
providerUrl: spokePoolProviderUrl,
127-
// TODO: Set this per chain
128-
maxBlockLookBack: 10000,
129-
},
130-
redisKeyPrefix: `spokePoolIndexer:${spokePoolNetworkInfo.chainId}`,
131-
};
132-
}
133-
// utility call to create the hubpool event indexer config
134-
async function getHubPoolIndexerConfig(params: {
135-
retryProviderConfig: RetryProviderConfig;
136-
hubPoolNetworkInfo: providers.Network;
137-
hubPoolProviderUrl: string;
138-
}) {
139-
const { retryProviderConfig, hubPoolProviderUrl, hubPoolNetworkInfo } =
140-
params;
141-
return {
142-
retryProviderConfig,
143-
hubConfig: {
144-
chainId: hubPoolNetworkInfo.chainId,
145-
providerUrl: hubPoolProviderUrl,
146-
maxBlockLookBack: 10000,
147-
},
148-
redisKeyPrefix: `hubPoolIndexer:${hubPoolNetworkInfo.chainId}`,
149-
};
150-
}
151-
152-
export async function Main(
153-
env: Record<string, string | undefined>,
154-
logger: winston.Logger,
155-
) {
156-
const spokePoolProviderUrls: string[] = Object.values(
157-
acrossConstants.MAINNET_CHAIN_IDs,
158-
)
159-
.map((chainId) => env[`INDEXER_SPOKEPOOL_PROVIDER_URL_${chainId}`])
160-
.filter((x): x is string => !!x);
161-
162-
assert(
163-
spokePoolProviderUrls.length > 0,
164-
"Must provide a url for at least one provider on one chain, for example: INDEXER_SPOKEPOOL_PROVIDER_URL_1",
165-
);
166-
167-
assert(
168-
env.INDEXER_HUBPOOL_PROVIDER_URL,
169-
"requires INDEXER_HUBPOOL_PROVIDER_URL",
170-
);
171-
const hubPoolProviderUrl = env.INDEXER_HUBPOOL_PROVIDER_URL;
172-
assert(env.INDEXER_REDIS_HOST, "requires INDEXER_REDIS_HOST");
173-
assert(env.INDEXER_REDIS_PORT, "requires INDEXER_REDIS_PORT");
174-
const redisConfig = {
175-
host: env.INDEXER_REDIS_HOST,
176-
port: Number(env.INDEXER_REDIS_PORT),
177-
};
29+
export async function Main(config: parseEnv.Config, logger: winston.Logger) {
30+
const { redisConfig, postgresConfig, hubConfig, spokeConfigs } = config;
17831

17932
const redis = await initializeRedis(redisConfig, logger);
180-
181-
const postgresConfig = getPostgresConfig(env);
18233
const postgres = await connectToDatabase(postgresConfig, logger);
183-
184-
const retryProviderConfig = getRetryProviderConfig(env);
185-
const tempHubProvider = new providers.JsonRpcProvider(hubPoolProviderUrl);
186-
const hubPoolNetworkInfo = await tempHubProvider.getNetwork();
18734
const bundleProcessor = new services.bundles.Processor({
18835
logger,
18936
redis,
19037
postgres,
19138
});
192-
const spokePoolIndexers: Array<services.spokePoolIndexer.Indexer> = [];
193-
const hubPoolIndexerConfig = await getHubPoolIndexerConfig({
194-
hubPoolNetworkInfo,
195-
hubPoolProviderUrl,
196-
retryProviderConfig,
197-
});
198-
// canonical hubpool indexer
19939
const hubPoolIndexer = new services.hubPoolIndexer.Indexer({
20040
logger,
20141
redis,
20242
postgres,
203-
...hubPoolIndexerConfig,
43+
...hubConfig,
20444
});
205-
// instanciate multiple spoke pool event indexers
206-
for (const spokePoolProviderUrl of spokePoolProviderUrls) {
207-
const config = await getSpokePoolIndexerConfig({
208-
hubPoolNetworkInfo,
209-
spokePoolProviderUrl,
210-
hubPoolProviderUrl,
211-
retryProviderConfig,
212-
});
213-
logger.info({
214-
message: "Starting indexer",
215-
...config,
216-
});
217-
const spokeIndexer = new services.spokePoolIndexer.Indexer({
45+
const spokePoolIndexers = spokeConfigs.map((spokeConfig) => {
46+
return new services.spokePoolIndexer.Indexer({
21847
logger,
21948
redis,
22049
postgres,
221-
...config,
50+
...spokeConfig,
22251
});
223-
spokePoolIndexers.push(spokeIndexer);
224-
}
52+
});
22553

22654
let exitRequested = false;
22755
process.on("SIGINT", () => {
@@ -260,7 +88,7 @@ export async function Main(
26088
(r) => r.status === "fulfilled",
26189
),
26290
bundleProcessorRunSuccess: bundleResults.status === "fulfilled",
263-
hubPoolRunSucccess: hubPoolResult.status === "fulfilled",
91+
hubPoolRunSuccess: hubPoolResult.status === "fulfilled",
26492
},
26593
});
26694

0 commit comments

Comments
 (0)