Skip to content

Commit 564b30d

Browse files
joefitterjoefitter
committed
Feature - add helpers to optimise fetching and parsing
multiple metadata accounts I am working on an app which parses many transactions using codama and as part of this I need to fetch IDLs for all programs in all instructions. As the only published method for fetching and parsing program-metadata content only supports single calls this results in many getAccountInfo calls which can exceed RPC rate limits for some suppliers. The addition of fetchAndParseAllMetadataContent which uses getMultipleAccounts to prefetch all PDA accounts, and additionally fetches all external accounts in a total of 2 RPC calls instead of potentially hundreds.
1 parent 3bca513 commit 564b30d

File tree

3 files changed

+242
-21
lines changed

3 files changed

+242
-21
lines changed

‎clients/js/src/fetchMetadataContent.ts‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
11
import { parse as parseToml } from '@iarna/toml';
2-
import { Address, GetAccountInfoApi, Rpc } from '@solana/kit';
2+
import {
3+
Address,
4+
assertAccountsExist,
5+
GetAccountInfoApi,
6+
GetMultipleAccountsApi,
7+
Rpc,
8+
} from '@solana/kit';
39
import { parse as parseYaml } from 'yaml';
4-
import { fetchMetadataFromSeeds, Format, SeedArgs } from './generated';
5-
import { unpackAndFetchData } from './packData';
10+
import {
11+
fetchAllMaybeMetadata,
12+
fetchMetadataFromSeeds,
13+
findMetadataPda,
14+
Format,
15+
SeedArgs,
16+
} from './generated';
17+
import { unpackAndFetchAllData, unpackAndFetchData } from './packData';
618

719
export async function fetchMetadataContent(
820
rpc: Rpc<GetAccountInfoApi>,
@@ -18,6 +30,28 @@ export async function fetchMetadataContent(
1830
return await unpackAndFetchData({ rpc, ...account.data });
1931
}
2032

33+
type FetchAllMetadataContentInput = {
34+
program: Address;
35+
seed: SeedArgs;
36+
authority: Address | null;
37+
}[];
38+
39+
export async function fetchAllMetadataContent(
40+
rpc: Rpc<GetMultipleAccountsApi>,
41+
input: FetchAllMetadataContentInput
42+
): Promise<string[]> {
43+
const addresses = await Promise.all(
44+
input.map(async ({ program, authority, seed }) => {
45+
const [address] = await findMetadataPda({ program, authority, seed });
46+
47+
return address;
48+
})
49+
);
50+
const accounts = await fetchAllMaybeMetadata(rpc, addresses);
51+
assertAccountsExist(accounts);
52+
return await unpackAndFetchAllData({ rpc, accounts });
53+
}
54+
2155
export async function fetchAndParseMetadataContent(
2256
rpc: Rpc<GetAccountInfoApi>,
2357
program: Address,
@@ -42,3 +76,34 @@ export async function fetchAndParseMetadataContent(
4276
return content;
4377
}
4478
}
79+
80+
type FetchAndParseAllMetadataContentInput = FetchAllMetadataContentInput;
81+
82+
export async function fetchAndParseAllMetadataContent(
83+
rpc: Rpc<GetMultipleAccountsApi>,
84+
input: FetchAndParseAllMetadataContentInput
85+
): Promise<unknown[]> {
86+
const addresses = await Promise.all(
87+
input.map(async ({ program, authority, seed }) => {
88+
const [address] = await findMetadataPda({ program, authority, seed });
89+
90+
return address;
91+
})
92+
);
93+
const accounts = await fetchAllMaybeMetadata(rpc, addresses);
94+
assertAccountsExist(accounts);
95+
const unpacked = await unpackAndFetchAllData({ rpc, accounts });
96+
return unpacked.map((content, index) => {
97+
switch (accounts[index].data.format) {
98+
case Format.Json:
99+
return JSON.parse(content);
100+
case Format.Yaml:
101+
return parseYaml(content);
102+
case Format.Toml:
103+
return parseToml(content);
104+
case Format.None:
105+
default:
106+
return content;
107+
}
108+
});
109+
}

‎clients/js/src/packData.ts‎

Lines changed: 88 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
import {
2+
Account,
23
Address,
34
assertAccountExists,
5+
assertAccountsExist,
46
fetchEncodedAccount,
7+
fetchEncodedAccounts,
58
GetAccountInfoApi,
69
getBase16Decoder,
710
getBase16Encoder,
811
getBase58Decoder,
912
getBase58Encoder,
1013
getBase64Decoder,
1114
getBase64Encoder,
15+
GetMultipleAccountsApi,
1216
getUtf8Decoder,
1317
getUtf8Encoder,
1418
pipe,
@@ -23,6 +27,7 @@ import {
2327
Encoding,
2428
getExternalDataDecoder,
2529
getExternalDataEncoder,
30+
Metadata,
2631
} from './generated';
2732

2833
export type PackedData = {
@@ -32,6 +37,12 @@ export type PackedData = {
3237
data: ReadonlyUint8Array;
3338
};
3439

40+
export type UnpackedData = {
41+
address: Address;
42+
offset?: number;
43+
length?: number;
44+
};
45+
3546
export function packDirectData(input: {
3647
content: string;
3748
/** Defaults to `Compression.Zlib`. */
@@ -120,11 +131,7 @@ export async function unpackAndFetchUrlData(
120131
return await response.text();
121132
}
122133

123-
export function unpackExternalData(data: ReadonlyUint8Array): {
124-
address: Address;
125-
offset?: number;
126-
length?: number;
127-
} {
134+
export function unpackExternalData(data: ReadonlyUint8Array): UnpackedData {
128135
const externalData = getExternalDataDecoder().decode(data);
129136
return {
130137
address: externalData.address,
@@ -133,27 +140,47 @@ export function unpackExternalData(data: ReadonlyUint8Array): {
133140
};
134141
}
135142

136-
export async function unpackAndFetchExternalData(
137-
input: Omit<PackedData, 'dataSource'> & { rpc: Rpc<GetAccountInfoApi> }
138-
): Promise<string> {
139-
const externalData = unpackExternalData(input.data);
140-
const account = await fetchEncodedAccount(input.rpc, externalData.address);
141-
assertAccountExists(account);
143+
export function uncompressAndDecodeExternalData({
144+
compression,
145+
encoding,
146+
account,
147+
unpackedExternalData,
148+
}: Pick<PackedData, 'compression' | 'encoding'> & {
149+
account: Account<Uint8Array>;
150+
unpackedExternalData: UnpackedData;
151+
}): string {
142152
let data = account.data;
143-
if (externalData.offset !== undefined) {
144-
data = data.slice(externalData.offset);
153+
if (unpackedExternalData.offset !== undefined) {
154+
data = data.slice(unpackedExternalData.offset);
145155
}
146-
if (externalData.length !== undefined) {
147-
data = data.slice(0, externalData.length);
156+
if (unpackedExternalData.length !== undefined) {
157+
data = data.slice(0, unpackedExternalData.length);
148158
}
149159
if (data.length === 0) {
150160
return '';
151161
}
152162
return pipe(
153163
data,
154-
(d) => uncompressData(d, input.compression),
155-
(d) => decodeData(d, input.encoding)
164+
(d) => uncompressData(d, compression),
165+
(d) => decodeData(d, encoding)
166+
);
167+
}
168+
169+
export async function unpackAndFetchExternalData(
170+
input: Omit<PackedData, 'dataSource'> & { rpc: Rpc<GetAccountInfoApi> }
171+
): Promise<string> {
172+
const unpackedExternalData = unpackExternalData(input.data);
173+
const account = await fetchEncodedAccount(
174+
input.rpc,
175+
unpackedExternalData.address
156176
);
177+
assertAccountExists(account);
178+
return uncompressAndDecodeExternalData({
179+
compression: input.compression,
180+
encoding: input.encoding,
181+
account,
182+
unpackedExternalData,
183+
});
157184
}
158185

159186
export async function unpackAndFetchData(
@@ -171,6 +198,50 @@ export async function unpackAndFetchData(
171198
}
172199
}
173200

201+
export async function unpackAndFetchAllData({
202+
accounts,
203+
rpc,
204+
}: {
205+
accounts: Account<Metadata>[];
206+
rpc: Rpc<GetMultipleAccountsApi>;
207+
}) {
208+
const unpackedExternalAccounts = accounts
209+
.filter((account) => account.data.dataSource === DataSource.External)
210+
.map((account) => ({
211+
address: account.address,
212+
unpacked: unpackExternalData(account.data.data),
213+
}));
214+
215+
const fetchedExternalAccounts = await fetchEncodedAccounts(
216+
rpc,
217+
unpackedExternalAccounts.map((account) => account.unpacked.address)
218+
);
219+
assertAccountsExist(fetchedExternalAccounts);
220+
221+
return Promise.all(
222+
accounts.map(async (account) => {
223+
switch (account.data.dataSource) {
224+
case DataSource.Direct:
225+
return unpackDirectData(account.data);
226+
case DataSource.Url:
227+
return await unpackAndFetchUrlData(account.data);
228+
case DataSource.External: {
229+
const accountIndex = unpackedExternalAccounts.findIndex(
230+
(acc) => acc.address === account.address
231+
);
232+
return uncompressAndDecodeExternalData({
233+
compression: account.data.compression,
234+
encoding: account.data.encoding,
235+
account: fetchedExternalAccounts[accountIndex],
236+
unpackedExternalData:
237+
unpackedExternalAccounts[accountIndex].unpacked,
238+
});
239+
}
240+
}
241+
})
242+
);
243+
}
244+
174245
export function compressData(
175246
data: ReadonlyUint8Array,
176247
compression: Compression

‎clients/js/test/fetchMetadataContent.test.ts‎

Lines changed: 86 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1-
import { address } from '@solana/kit';
1+
import { address, generateKeyPairSigner, getUtf8Encoder } from '@solana/kit';
22
import test from 'ava';
33
import {
4+
Compression,
5+
Encoding,
6+
fetchAndParseAllMetadataContent,
47
fetchAndParseMetadataContent,
58
Format,
69
packDirectData,
10+
packExternalData,
711
writeMetadata,
812
} from '../src';
913
import {
14+
createBuffer,
1015
createDefaultSolanaClient,
1116
createDeployedProgram,
1217
generateKeyPairSignerWithSol,
@@ -74,3 +79,83 @@ test('it fetches and parses direct IDLs from non-canonical metadata accounts', a
7479
version: '1.0.0',
7580
});
7681
});
82+
83+
test.only('it fetches and parses multiple direct IDLs from metadata accounts', async (t) => {
84+
t.timeout(30_000);
85+
// Given the following authority and deployed program.
86+
const client = createDefaultSolanaClient();
87+
const authority = await generateKeyPairSignerWithSol(client);
88+
const program = address('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA');
89+
90+
const metadata1 = await generateKeyPairSigner();
91+
const metadata2 = await generateKeyPairSigner();
92+
93+
// And given the following IDLs exist for the programs.
94+
const idl1 = '{"kind":"rootNode","standard":"codama","version":"1.0.0"}';
95+
const idl2 = '{"kind":"rootNode","standard":"codama","version":"1.0.1"}';
96+
const buffer = await generateKeyPairSigner();
97+
98+
// We create a buffer account to hold the IDL data
99+
await createBuffer(client, {
100+
buffer: buffer.address,
101+
authority: buffer,
102+
payer: authority,
103+
data: getUtf8Encoder().encode(idl2),
104+
});
105+
106+
// And we create metadata accounts for direct and external data
107+
await Promise.all([
108+
writeMetadata({
109+
...client,
110+
...packDirectData({ content: idl1 }),
111+
payer: authority,
112+
authority: metadata1,
113+
program,
114+
seed: 'idl',
115+
format: Format.Json,
116+
}),
117+
writeMetadata({
118+
...client,
119+
...packExternalData({
120+
address: buffer.address,
121+
offset: 96,
122+
length: idl2.length,
123+
compression: Compression.None,
124+
encoding: Encoding.Utf8,
125+
}),
126+
payer: authority,
127+
authority: metadata2,
128+
program,
129+
seed: 'idl',
130+
format: Format.Json,
131+
}),
132+
]);
133+
134+
// When we fetch the IDLs for the programs.
135+
const result = await fetchAndParseAllMetadataContent(client.rpc, [
136+
{
137+
program,
138+
seed: 'idl',
139+
authority: metadata1.address,
140+
},
141+
{
142+
program,
143+
seed: 'idl',
144+
authority: metadata2.address,
145+
},
146+
]);
147+
148+
// Then we expect the following IDLs to be fetched and parsed.
149+
t.deepEqual(result, [
150+
{
151+
kind: 'rootNode',
152+
standard: 'codama',
153+
version: '1.0.0',
154+
},
155+
{
156+
kind: 'rootNode',
157+
standard: 'codama',
158+
version: '1.0.1',
159+
},
160+
]);
161+
});

0 commit comments

Comments
 (0)