Feature - add helpers to optimise fetching and parsing multiple metadata accounts

clients/js/src/fetchMetadataContent.ts

@@ -1,8 +1,20 @@
 import { parse as parseToml } from '@iarna/toml';
-import { Address, GetAccountInfoApi, Rpc } from '@solana/kit';
+import {
+  Address,
+  assertAccountsExist,
+  GetAccountInfoApi,
+  GetMultipleAccountsApi,
+  Rpc,
+} from '@solana/kit';
 import { parse as parseYaml } from 'yaml';
-import { fetchMetadataFromSeeds, Format, SeedArgs } from './generated';
-import { unpackAndFetchData } from './packData';
+import {
+  fetchAllMaybeMetadata,
+  fetchMetadataFromSeeds,
+  findMetadataPda,
+  Format,
+  SeedArgs,
+} from './generated';
+import { unpackAndFetchAllData, unpackAndFetchData } from './packData';
 
 export async function fetchMetadataContent(
   rpc: Rpc<GetAccountInfoApi>,
@@ -18,6 +30,28 @@ export async function fetchMetadataContent(
   return await unpackAndFetchData({ rpc, ...account.data });
 }
 
+type FetchAllMetadataContentInput = {
+  program: Address;
+  seed: SeedArgs;
+  authority: Address | null;
+}[];
+
+export async function fetchAllMetadataContent(
+  rpc: Rpc<GetMultipleAccountsApi>,
+  input: FetchAllMetadataContentInput
+): Promise<string[]> {
+  const addresses = await Promise.all(
+    input.map(async ({ program, authority, seed }) => {
+      const [address] = await findMetadataPda({ program, authority, seed });
+
+      return address;
+    })
+  );
+  const accounts = await fetchAllMaybeMetadata(rpc, addresses);
+  assertAccountsExist(accounts);
+  return await unpackAndFetchAllData({ rpc, accounts });
+}
+
 export async function fetchAndParseMetadataContent(
   rpc: Rpc<GetAccountInfoApi>,
   program: Address,
@@ -42,3 +76,34 @@ export async function fetchAndParseMetadataContent(
       return content;
   }
 }
+
+type FetchAndParseAllMetadataContentInput = FetchAllMetadataContentInput;
+
+export async function fetchAndParseAllMetadataContent(
+  rpc: Rpc<GetMultipleAccountsApi>,
+  input: FetchAndParseAllMetadataContentInput
+): Promise<unknown[]> {
+  const addresses = await Promise.all(
+    input.map(async ({ program, authority, seed }) => {
+      const [address] = await findMetadataPda({ program, authority, seed });
+
+      return address;
+    })
+  );
+  const maybeAccounts = await fetchAllMaybeMetadata(rpc, addresses);
+  const accounts = maybeAccounts.filter((acc) => acc.exists);
+  const unpacked = await unpackAndFetchAllData({ rpc, accounts });
+  return unpacked.map((content, index) => {
+    switch (accounts[index].data.format) {
+      case Format.Json:
+        return JSON.parse(content);
+      case Format.Yaml:
+        return parseYaml(content);
+      case Format.Toml:
+        return parseToml(content);
+      case Format.None:
+      default:
+        return content;
+    }
+  });
+}

clients/js/src/packData.ts

@@ -1,14 +1,18 @@
 import {
+  Account,
   Address,
   assertAccountExists,
+  assertAccountsExist,
   fetchEncodedAccount,
+  fetchEncodedAccounts,
   GetAccountInfoApi,
   getBase16Decoder,
   getBase16Encoder,
   getBase58Decoder,
   getBase58Encoder,
   getBase64Decoder,
   getBase64Encoder,
+  GetMultipleAccountsApi,
   getUtf8Decoder,
   getUtf8Encoder,
   pipe,
@@ -23,6 +27,7 @@ import {
   Encoding,
   getExternalDataDecoder,
   getExternalDataEncoder,
+  Metadata,
 } from './generated';
 
 export type PackedData = {
@@ -32,6 +37,12 @@ export type PackedData = {
   data: ReadonlyUint8Array;
 };
 
+export type UnpackedData = {
+  address: Address;
+  offset?: number;
+  length?: number;
+};
+
 export function packDirectData(input: {
   content: string;
   /** Defaults to `Compression.Zlib`. */
@@ -120,11 +131,7 @@ export async function unpackAndFetchUrlData(
   return await response.text();
 }
 
-export function unpackExternalData(data: ReadonlyUint8Array): {
-  address: Address;
-  offset?: number;
-  length?: number;
-} {
+export function unpackExternalData(data: ReadonlyUint8Array): UnpackedData {
   const externalData = getExternalDataDecoder().decode(data);
   return {
     address: externalData.address,
@@ -133,27 +140,47 @@ export function unpackExternalData(data: ReadonlyUint8Array): {
   };
 }
 
-export async function unpackAndFetchExternalData(
-  input: Omit<PackedData, 'dataSource'> & { rpc: Rpc<GetAccountInfoApi> }
-): Promise<string> {
-  const externalData = unpackExternalData(input.data);
-  const account = await fetchEncodedAccount(input.rpc, externalData.address);
-  assertAccountExists(account);
+export function uncompressAndDecodeExternalData({
+  compression,
+  encoding,
+  account,
+  unpackedExternalData,
+}: Pick<PackedData, 'compression' | 'encoding'> & {
+  account: Account<Uint8Array>;
+  unpackedExternalData: UnpackedData;
+}): string {
   let data = account.data;
-  if (externalData.offset !== undefined) {
-    data = data.slice(externalData.offset);
+  if (unpackedExternalData.offset !== undefined) {
+    data = data.slice(unpackedExternalData.offset);
   }
-  if (externalData.length !== undefined) {
-    data = data.slice(0, externalData.length);
+  if (unpackedExternalData.length !== undefined) {
+    data = data.slice(0, unpackedExternalData.length);
   }
   if (data.length === 0) {
     return '';
   }
   return pipe(
     data,
-    (d) => uncompressData(d, input.compression),
-    (d) => decodeData(d, input.encoding)
+    (d) => uncompressData(d, compression),
+    (d) => decodeData(d, encoding)
+  );
+}
+
+export async function unpackAndFetchExternalData(
+  input: Omit<PackedData, 'dataSource'> & { rpc: Rpc<GetAccountInfoApi> }
+): Promise<string> {
+  const unpackedExternalData = unpackExternalData(input.data);
+  const account = await fetchEncodedAccount(
+    input.rpc,
+    unpackedExternalData.address
   );
+  assertAccountExists(account);
+  return uncompressAndDecodeExternalData({
+    compression: input.compression,
+    encoding: input.encoding,
+    account,
+    unpackedExternalData,
+  });
 }
 
 export async function unpackAndFetchData(
@@ -171,6 +198,50 @@ export async function unpackAndFetchData(
   }
 }
 
+export async function unpackAndFetchAllData({
+  accounts,
+  rpc,
+}: {
+  accounts: Account<Metadata>[];
+  rpc: Rpc<GetMultipleAccountsApi>;
+}) {
+  const unpackedExternalAccounts = accounts
+    .filter((account) => account.data.dataSource === DataSource.External)
+    .map((account) => ({
+      address: account.address,
+      unpacked: unpackExternalData(account.data.data),
+    }));
+
+  const fetchedExternalAccounts = await fetchEncodedAccounts(
+    rpc,
+    unpackedExternalAccounts.map((account) => account.unpacked.address)
+  );
+  assertAccountsExist(fetchedExternalAccounts);
+
+  return Promise.all(
+    accounts.map(async (account) => {
+      switch (account.data.dataSource) {
+        case DataSource.Direct:
+          return unpackDirectData(account.data);
+        case DataSource.Url:
+          return await unpackAndFetchUrlData(account.data);
+        case DataSource.External: {
+          const accountIndex = unpackedExternalAccounts.findIndex(
+            (acc) => acc.address === account.address
+          );
+          return uncompressAndDecodeExternalData({
+            compression: account.data.compression,
+            encoding: account.data.encoding,
+            account: fetchedExternalAccounts[accountIndex],
+            unpackedExternalData:
+              unpackedExternalAccounts[accountIndex].unpacked,
+          });
+        }
+      }
+    })
+  );
+}
+
 export function compressData(
   data: ReadonlyUint8Array,
   compression: Compression

clients/js/test/fetchMetadataContent.test.ts

@@ -1,12 +1,17 @@
-import { address } from '@solana/kit';
+import { address, generateKeyPairSigner, getUtf8Encoder } from '@solana/kit';
 import test from 'ava';
 import {
+  Compression,
+  Encoding,
+  fetchAndParseAllMetadataContent,
   fetchAndParseMetadataContent,
   Format,
   packDirectData,
+  packExternalData,
   writeMetadata,
 } from '../src';
 import {
+  createBuffer,
   createDefaultSolanaClient,
   createDeployedProgram,
   generateKeyPairSignerWithSol,
@@ -74,3 +79,83 @@ test('it fetches and parses direct IDLs from non-canonical metadata accounts', a
     version: '1.0.0',
   });
 });
+
+test('it fetches and parses multiple direct IDLs from metadata accounts', async (t) => {
+  t.timeout(30_000);
+  // Given the following authority and deployed program.
+  const client = createDefaultSolanaClient();
+  const authority = await generateKeyPairSignerWithSol(client);
+  const program = address('TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA');
+
+  const metadata1 = await generateKeyPairSigner();
+  const metadata2 = await generateKeyPairSigner();
+
+  // And given the following IDLs exist for the programs.
+  const idl1 = '{"kind":"rootNode","standard":"codama","version":"1.0.0"}';
+  const idl2 = '{"kind":"rootNode","standard":"codama","version":"1.0.1"}';
+  const buffer = await generateKeyPairSigner();
+
+  // We create a buffer account to hold the IDL data
+  await createBuffer(client, {
+    buffer: buffer.address,
+    authority: buffer,
+    payer: authority,
+    data: getUtf8Encoder().encode(idl2),
+  });
+
+  // And we create metadata accounts for direct and external data
+  await Promise.all([
+    writeMetadata({
+      ...client,
+      ...packDirectData({ content: idl1 }),
+      payer: authority,
+      authority: metadata1,
+      program,
+      seed: 'idl',
+      format: Format.Json,
+    }),
+    writeMetadata({
+      ...client,
+      ...packExternalData({
+        address: buffer.address,
+        offset: 96,
+        length: idl2.length,
+        compression: Compression.None,
+        encoding: Encoding.Utf8,
+      }),
+      payer: authority,
+      authority: metadata2,
+      program,
+      seed: 'idl',
+      format: Format.Json,
+    }),
+  ]);
+
+  // When we fetch the IDLs for the programs.
+  const result = await fetchAndParseAllMetadataContent(client.rpc, [
+    {
+      program,
+      seed: 'idl',
+      authority: metadata1.address,
+    },
+    {
+      program,
+      seed: 'idl',
+      authority: metadata2.address,
+    },
+  ]);
+
+  // Then we expect the following IDLs to be fetched and parsed.
+  t.deepEqual(result, [
+    {
+      kind: 'rootNode',
+      standard: 'codama',
+      version: '1.0.0',
+    },
+    {
+      kind: 'rootNode',
+      standard: 'codama',
+      version: '1.0.1',
+    },
+  ]);
+});