Skip to content

Commit 6c16ec2

Browse files
djwhittdjwhitt
committed
WIP parallel ArNS fixes and cleanup
1 parent 2ee1c80 commit 6c16ec2

File tree

8 files changed

+491
-136
lines changed

8 files changed

+491
-136
lines changed

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"@clickhouse/client": "^1.10.1",
1515
"@dha-team/arbundles": "^1.0.1",
1616
"@permaweb/aoconnect": "^0.0.63",
17+
"any-signal": "^4.1.1",
1718
"apollo-server-express": "^3.13.0",
1819
"arweave": "^2.0.0-ec.0",
1920
"axios": "^1.7.9",

‎src/middleware/arns.ts

+7-29
Original file line numberDiff line numberDiff line change
@@ -22,21 +22,13 @@ import * as config from '../config.js';
2222
import { headerNames } from '../constants.js';
2323
import { sendNotFound, sendPaymentRequired } from '../routes/data/handlers.js';
2424
import { DATA_PATH_REGEX } from '../constants.js';
25-
import { NameResolution, NameResolver } from '../types.js';
25+
import { NameResolver } from '../types.js';
2626
import * as metrics from '../metrics.js';
2727
import * as system from '../system.js';
28-
import NodeCache from 'node-cache';
2928

3029
const EXCLUDED_SUBDOMAINS = new Set('www');
3130
const MAX_ARNS_NAME_LENGTH = 51;
3231

33-
// simple cache that stores the arns resolution promises to avoid duplicate requests to the name resolver
34-
const arnsRequestCache = new NodeCache({
35-
stdTTL: 60, // short cache in case we forget to delete
36-
checkperiod: 60,
37-
useClones: false, // cloning promises is unsafe
38-
});
39-
4032
export const createArnsMiddleware = ({
4133
dataHandler,
4234
nameResolver,
@@ -85,28 +77,14 @@ export const createArnsMiddleware = ({
8577
return;
8678
}
8779

88-
const getArnsResolutionPromise = async (): Promise<NameResolution> => {
89-
if (arnsRequestCache.has(arnsSubdomain)) {
90-
const arnsResolutionPromise =
91-
arnsRequestCache.get<Promise<NameResolution>>(arnsSubdomain);
92-
if (arnsResolutionPromise) {
93-
return arnsResolutionPromise;
94-
}
95-
}
96-
const arnsResolutionPromise = nameResolver.resolve({
97-
name: arnsSubdomain,
98-
});
99-
arnsRequestCache.set(arnsSubdomain, arnsResolutionPromise);
100-
return arnsResolutionPromise;
101-
};
102-
103-
const start = Date.now();
80+
// NOTE: Errors and request deduplication are expected to be handled by the
81+
// resolver
82+
const end = metrics.arnsResolutionTime.startTimer();
10483
const { resolvedId, ttl, processId, resolvedAt, limit, index } =
105-
await getArnsResolutionPromise().finally(() => {
106-
// remove from cache after resolution
107-
arnsRequestCache.del(arnsSubdomain);
84+
await nameResolver.resolve({
85+
name: arnsSubdomain,
10886
});
109-
metrics.arnsResolutionTime.observe(Date.now() - start);
87+
end();
11088
if (resolvedId === undefined) {
11189
sendNotFound(res);
11290
return;

‎src/resolution/composite-arns-resolver.test.ts

+258
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
/**
2+
* AR.IO Gateway
3+
* Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
4+
*
5+
* This program is free software: you can redistribute it and/or modify
6+
* it under the terms of the GNU Affero General Public License as published by
7+
* the Free Software Foundation, either version 3 of the License, or
8+
* (at your option) any later version.
9+
*
10+
* This program is distributed in the hope that it will be useful,
11+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
* GNU Affero General Public License for more details.
14+
*
15+
* You should have received a copy of the GNU Affero General Public License
16+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
*/
18+
import { strict as assert } from 'node:assert';
19+
import { describe, it, beforeEach, mock } from 'node:test';
20+
import winston from 'winston';
21+
import { CompositeArNSResolver } from './composite-arns-resolver.js';
22+
import { NameResolution, NameResolver } from '../types.js';
23+
import { KvArNSResolutionStore } from '../store/kv-arns-name-resolution-store.js';
24+
import { KvArNSRegistryStore } from '../store/kv-arns-base-name-store.js';
25+
26+
const log = winston.createLogger({ silent: true });
27+
const mockResolution: NameResolution = {
28+
name: 'test.ar',
29+
resolvedId: 'tx1',
30+
resolvedAt: Date.now(),
31+
ttl: 300,
32+
processId: 'process1',
33+
limit: 1,
34+
index: 1,
35+
};
36+
37+
describe('CompositeArNSResolver', () => {
38+
const mockResolutionCache = {
39+
get: mock.fn(),
40+
set: mock.fn(),
41+
} as unknown as KvArNSResolutionStore;
42+
43+
const mockRegistryCache = {
44+
get: mock.fn(),
45+
set: mock.fn(),
46+
} as unknown as KvArNSRegistryStore;
47+
48+
beforeEach(() => {
49+
mock.reset();
50+
});
51+
52+
it('should return early when first resolver succeeds', async () => {
53+
const resolver1: NameResolver = {
54+
resolve: mock.fn(async () => mockResolution),
55+
};
56+
57+
const resolver2: NameResolver = {
58+
resolve: mock.fn(async () => {
59+
throw new Error();
60+
}),
61+
};
62+
63+
const compositeResolver = new CompositeArNSResolver({
64+
log,
65+
resolvers: [resolver1, resolver2],
66+
resolutionCache: mockResolutionCache,
67+
registryCache: mockRegistryCache,
68+
maxConcurrentResolutions: 2,
69+
});
70+
71+
const result = await compositeResolver.resolve({ name: 'test.ar' });
72+
73+
assert.strictEqual((resolver1.resolve as any).mock.calls.length, 1);
74+
assert.strictEqual((resolver2.resolve as any).mock.calls.length, 1);
75+
assert.deepEqual(result, mockResolution);
76+
});
77+
78+
it('should try all resolvers when earlier ones fail', async () => {
79+
const resolver1: NameResolver = {
80+
resolve: mock.fn(async () => {
81+
throw new Error('Failed');
82+
}),
83+
};
84+
85+
const resolver2: NameResolver = {
86+
resolve: mock.fn(async () => mockResolution),
87+
};
88+
89+
const compositeResolver = new CompositeArNSResolver({
90+
log,
91+
resolvers: [resolver1, resolver2],
92+
resolutionCache: mockResolutionCache,
93+
registryCache: mockRegistryCache,
94+
maxConcurrentResolutions: 2,
95+
resolverTimeoutMs: 50,
96+
});
97+
98+
const result = await compositeResolver.resolve({ name: 'test.ar' });
99+
100+
assert.strictEqual((resolver1.resolve as any).mock.calls.length, 1);
101+
assert.strictEqual((resolver2.resolve as any).mock.calls.length, 1);
102+
assert.deepEqual(result, mockResolution);
103+
});
104+
105+
it('should use cached resolution when available and not expired', async () => {
106+
const now = Date.now();
107+
const mockResolution: NameResolution = {
108+
name: 'test.ar',
109+
resolvedId: 'tx1',
110+
resolvedAt: now - 100000,
111+
ttl: 300,
112+
processId: 'process1',
113+
limit: 1,
114+
index: 1,
115+
};
116+
117+
mock.method(mockResolutionCache, 'get', async () =>
118+
Buffer.from(JSON.stringify(mockResolution)),
119+
);
120+
121+
const resolver1: NameResolver = {
122+
resolve: mock.fn(),
123+
};
124+
125+
const compositeResolver = new CompositeArNSResolver({
126+
log,
127+
resolvers: [resolver1],
128+
resolutionCache: mockResolutionCache,
129+
registryCache: mockRegistryCache,
130+
maxConcurrentResolutions: 2,
131+
resolverTimeoutMs: 50,
132+
});
133+
134+
const result = await compositeResolver.resolve({ name: 'test.ar' });
135+
136+
assert.strictEqual((resolver1.resolve as any).mock.calls.length, 0);
137+
assert.deepEqual(result, mockResolution);
138+
});
139+
140+
it('should handle resolver timeouts correctly', async () => {
141+
const resolver1: NameResolver = {
142+
resolve: mock.fn(
143+
async () => new Promise((resolve) => setTimeout(resolve, 1000)),
144+
),
145+
};
146+
147+
const resolver2: NameResolver = {
148+
resolve: mock.fn(async () => mockResolution),
149+
};
150+
151+
const compositeResolver = new CompositeArNSResolver({
152+
log,
153+
resolvers: [resolver1, resolver2],
154+
resolutionCache: mockResolutionCache,
155+
registryCache: mockRegistryCache,
156+
maxConcurrentResolutions: 2,
157+
resolverTimeoutMs: 50,
158+
});
159+
160+
const result = await compositeResolver.resolve({ name: 'test.ar' });
161+
162+
assert.strictEqual((resolver1.resolve as any).mock.calls.length, 1);
163+
assert.strictEqual((resolver2.resolve as any).mock.calls.length, 1);
164+
assert.deepEqual(result, mockResolution);
165+
});
166+
167+
it('should return undefined resolution when all resolvers fail', async () => {
168+
const resolver1: NameResolver = {
169+
resolve: mock.fn(async () => {
170+
throw new Error('Failed 1');
171+
}),
172+
};
173+
174+
const resolver2: NameResolver = {
175+
resolve: mock.fn(async () => {
176+
throw new Error('Failed 2');
177+
}),
178+
};
179+
180+
const compositeResolver = new CompositeArNSResolver({
181+
log,
182+
resolvers: [resolver1, resolver2],
183+
resolutionCache: mockResolutionCache,
184+
registryCache: mockRegistryCache,
185+
maxConcurrentResolutions: 2,
186+
resolverTimeoutMs: 50,
187+
});
188+
189+
const result = await compositeResolver.resolve({ name: 'test.ar' });
190+
191+
assert.deepEqual(result, {
192+
name: 'test.ar',
193+
resolvedId: undefined,
194+
resolvedAt: undefined,
195+
ttl: undefined,
196+
processId: undefined,
197+
limit: undefined,
198+
index: undefined,
199+
});
200+
});
201+
202+
it('should respect maxConcurrentResolutions limit', async () => {
203+
let activeResolutions = 0;
204+
let maxActiveResolutions = 0;
205+
let resolversCalled = 0;
206+
const totalResolvers = 4;
207+
208+
const createResolver = (): NameResolver => ({
209+
resolve: mock.fn(async () => {
210+
resolversCalled++;
211+
activeResolutions++;
212+
maxActiveResolutions = Math.max(
213+
maxActiveResolutions,
214+
activeResolutions,
215+
);
216+
217+
await Promise.resolve();
218+
219+
activeResolutions--;
220+
return {
221+
name: 'test.ar',
222+
resolvedId: undefined,
223+
resolvedAt: undefined,
224+
ttl: undefined,
225+
processId: undefined,
226+
limit: undefined,
227+
index: undefined,
228+
};
229+
}),
230+
});
231+
232+
const resolvers = Array(totalResolvers)
233+
.fill(null)
234+
.map(() => createResolver());
235+
236+
const compositeResolver = new CompositeArNSResolver({
237+
log,
238+
resolvers,
239+
resolutionCache: mockResolutionCache,
240+
registryCache: mockRegistryCache,
241+
maxConcurrentResolutions: 2,
242+
resolverTimeoutMs: 50,
243+
});
244+
245+
await compositeResolver.resolve({ name: 'test.ar' });
246+
247+
assert.equal(
248+
maxActiveResolutions,
249+
2,
250+
'Max concurrent resolutions exceeded limit',
251+
);
252+
assert.equal(
253+
resolversCalled,
254+
totalResolvers,
255+
'Not all resolvers were called',
256+
);
257+
});
258+
});

0 commit comments

Comments
 (0)