feat(ts/idl): Update IDL fetching for the new PMP metadata (#4253)

* feat(ts/idl): Update IDL fetching for the new PMP metadata

* chore(ts): Update some uses of `Buffer` to TS 5.9 compatibility

* tests(idl): Use TS API in tests to fetch IDL

* cli(idl): Force install PMP CLI on first use

* refactor: Don't use extra dependencies for decoding IDL metadata

Author: jamie-osec

cli/src/metadata.rs

@@ -41,9 +41,14 @@ impl IdlCommand {
 
     pub fn status(self) -> io::Result<ExitStatus> {
         let mut command = Command::new("npx");
+        // Force on first-time install
+        command.arg("--yes");
+        // Use pinned version
         command.arg(format!(
-            "@solana-program/program-metadata@{PMP_CLIENT_VERSION}"
+            "--package=@solana-program/program-metadata@{PMP_CLIENT_VERSION}"
         ));
+        command.arg("--");
+        command.arg("program-metadata");
         command.args(["--rpc", &self.rpc_url]);
         command.args(self.subcommand.args());
         command.status()

tests/anchor-cli-idl/.gitignore

@@ -0,0 +1 @@
+testLargeIdl.json

tests/anchor-cli-idl/test.sh

@@ -2,6 +2,10 @@
 
 set -euo pipefail
 
+# FIXME: For some reason, in CI, executing with NPX results in `sh: program-metadata: not found`
+# Installing this globally fixes this in CI, but this should be investigated and fixed properly
+# Implementing IDL fetching via Rust client will make this redundant
+npm install --global @solana-program/program-metadata@0.5.1
 DEPLOYER_KEYPAIR="keypairs/deployer-keypair.json"
 PROGRAM_ONE="2uA3amp95zsEHUpo8qnLMhcFAUsiKVEcKHXS1JetFjU5"
 

tests/anchor-cli-idl/tests/idl.ts

@@ -15,17 +15,7 @@ describe("Test CLI IDL commands", () => {
   const programOne = anchor.workspace.IdlCommandsOne as Program<IdlCommandsOne>;
   const programTwo = anchor.workspace.IdlCommandsTwo as Program<IdlCommandsTwo>;
 
-  // FIXME: Once the TS client is updated to use PMP, this should use the TS API and not CLI
-  const fetchIdl = async (programId) => {
-    let idl;
-    try {
-      idl = execSync(`anchor idl fetch ${programId}`).toString();
-    } catch {
-      // CLI errors on failure, TS API returns null
-      return null;
-    }
-    return JSON.parse(idl);
-  };
+  const fetchIdl = anchor.Program.fetchIdl;
 
   it("Can initialize IDL account", async () => {
     execSync(

ts/packages/anchor/package.json

@@ -33,8 +33,8 @@
     "test": "jest tests --detectOpenHandles"
   },
   "dependencies": {
-    "@anchor-lang/errors": "^0.32.1",
     "@anchor-lang/borsh": "^0.32.1",
+    "@anchor-lang/errors": "^0.32.1",
     "@noble/hashes": "^1.3.1",
     "@solana/web3.js": "^1.69.0",
     "bn.js": "^5.1.2",

ts/packages/anchor/src/idl.ts

@@ -1,7 +1,8 @@
+import { bs58, utf8 } from "./utils/bytes/index.js";
+import { inflate, ungzip } from "pako";
 import camelCase from "camelcase";
 import { Buffer } from "buffer";
 import { PublicKey } from "@solana/web3.js";
-import * as borsh from "@anchor-lang/borsh";
 
 export type Idl = {
   address: string;
@@ -271,36 +272,167 @@ export function isCompositeAccounts(
   return "accounts" in accountItem;
 }
 
-// Deterministic IDL address as a function of the program id.
-export async function idlAddress(programId: PublicKey): Promise<PublicKey> {
-  const base = (await PublicKey.findProgramAddress([], programId))[0];
-  return await PublicKey.createWithSeed(base, seed(), programId);
+// Account format defined at
+// https://github.com/solana-program/program-metadata/blob/734e947d/clients/js/src/generated/accounts/metadata.ts#L123-L138
+const PROGRAM_METADATA_PROGRAM_ID = new PublicKey(
+  "ProgM6JCCvbYkfKqJYHePx4xxSUSqJp7rh8Lyv7nk7S"
+);
+const IDL_METADATA_SEED = "idl";
+const ACCOUNT_DISCRIMINATOR_METADATA = 2;
+const DATA_SOURCE_DIRECT = 0;
+// Only JSON formatted data is currently supported
+export const FORMAT_JSON = 1;
+const SEED_SIZE = 16;
+const DATA_LENGTH_SIZE = 4;
+const DATA_LENGTH_PADDING = 5;
+const ZEROABLE_OPTION_PUBKEY_SIZE = 32;
+const METADATA_HEADER_SIZE =
+  1 + 32 + ZEROABLE_OPTION_PUBKEY_SIZE + 1 + 1 + SEED_SIZE + 1 + 1 + 1 + 1;
+
+export enum MetadataCompression {
+  None = 0,
+  Gzip = 1,
+  Zlib = 2,
 }
 
-// Seed for generating the idlAddress.
-export function seed(): string {
-  return "anchor:idl";
+export enum MetadataEncoding {
+  None = 0,
+  Utf8 = 1,
+  Base58 = 2,
+  Base64 = 3,
 }
 
-// The on-chain account of the IDL.
-export interface IdlProgramAccount {
-  authority: PublicKey;
+export type MetadataAccount = {
+  format: number;
+  dataSource: number;
+  compression: MetadataCompression;
+  encoding: MetadataEncoding;
   data: Buffer;
+};
+
+function encodeMetadataSeed(seed: string): Buffer {
+  const encodedSeed = Buffer.from(utf8.encode(seed));
+  if (encodedSeed.length > SEED_SIZE) {
+    throw new Error(`Metadata seed '${seed}' exceeds ${SEED_SIZE} bytes`);
+  }
+
+  const paddedSeed = Buffer.alloc(SEED_SIZE);
+  encodedSeed.copy(paddedSeed);
+  return paddedSeed;
 }
 
-const IDL_ACCOUNT_LAYOUT: borsh.Layout<IdlProgramAccount> = borsh.struct([
-  borsh.publicKey("authority"),
-  borsh.vecU8("data"),
-]);
+export function idlAddress(programId: PublicKey): PublicKey {
+  // Canonical metadata uses a null authority seed, which is serialized as `[]`.
+  return PublicKey.findProgramAddressSync(
+    [
+      programId.toBuffer(),
+      Buffer.alloc(0),
+      encodeMetadataSeed(IDL_METADATA_SEED),
+    ],
+    PROGRAM_METADATA_PROGRAM_ID
+  )[0];
+}
 
-export function decodeIdlAccount(data: Buffer): IdlProgramAccount {
-  return IDL_ACCOUNT_LAYOUT.decode(data);
+export function seed(): string {
+  return "idl";
 }
 
-export function encodeIdlAccount(acc: IdlProgramAccount): Buffer {
-  const buffer = Buffer.alloc(1000); // TODO: use a tighter buffer.
-  const len = IDL_ACCOUNT_LAYOUT.encode(acc, buffer);
-  return buffer.slice(0, len);
+export function decodeIdlAccount<IDL extends Idl = Idl>(data: Buffer): IDL {
+  const minimumSize =
+    METADATA_HEADER_SIZE + DATA_LENGTH_SIZE + DATA_LENGTH_PADDING;
+  if (data.length < minimumSize) {
+    throw new Error("Metadata account is too small");
+  }
+
+  let offset = 0;
+  const discriminator = data.readUInt8(offset);
+  offset += 1;
+  if (discriminator !== ACCOUNT_DISCRIMINATOR_METADATA) {
+    throw new Error(
+      `Invalid metadata account discriminator: ${discriminator.toString()}`
+    );
+  }
+
+  offset += 32; // program
+  offset += ZEROABLE_OPTION_PUBKEY_SIZE; // authority
+  offset += 1; // mutable
+  offset += 1; // canonical
+  offset += SEED_SIZE; // seed
+
+  const encoding = data.readUInt8(offset) as MetadataEncoding;
+  offset += 1;
+
+  const compression = data.readUInt8(offset) as MetadataCompression;
+  offset += 1;
+
+  const format = data.readUInt8(offset);
+  if (format !== FORMAT_JSON) {
+    throw new Error(
+      `IDL has data format '${format}', only JSON IDLs (${FORMAT_JSON}) are supported`
+    );
+  }
+  offset += 1;
+
+  const dataSource = data.readUInt8(offset);
+  if (dataSource !== DATA_SOURCE_DIRECT) {
+    throw new Error(
+      `IDL has source '${dataSource}', only directly embedded data (${DATA_SOURCE_DIRECT}) is supported`
+    );
+  }
+  offset += 1;
+
+  const dataLength = data.readUInt32LE(offset);
+  offset += DATA_LENGTH_SIZE + DATA_LENGTH_PADDING;
+
+  if (data.length < offset + dataLength) {
+    throw new Error("Metadata account data is truncated");
+  }
+
+  const blob = data.subarray(offset, offset + dataLength);
+  const decoded = decodeMetadataData(
+    uncompressMetadataData(blob, compression),
+    encoding
+  );
+  return JSON.parse(decoded);
+}
+
+export function uncompressMetadataData(
+  data: Buffer,
+  compression: MetadataCompression
+): Buffer {
+  switch (compression) {
+    case MetadataCompression.None:
+      return data;
+    case MetadataCompression.Gzip:
+      return Buffer.from(ungzip(data));
+    case MetadataCompression.Zlib:
+      return Buffer.from(inflate(data));
+    default:
+      throw new Error(
+        `Unsupported metadata compression: ${String(compression as number)}`
+      );
+  }
+}
+
+export function decodeMetadataData(
+  data: Buffer,
+  encoding: MetadataEncoding
+): string {
+  switch (encoding) {
+    // 'None' is actually hex-encoded
+    case MetadataEncoding.None:
+      return data.toString("hex");
+    case MetadataEncoding.Utf8:
+      return utf8.decode(data);
+    case MetadataEncoding.Base58:
+      return bs58.encode(data);
+    case MetadataEncoding.Base64:
+      return data.toString("base64");
+    default:
+      throw new Error(
+        `Unsupported metadata encoding: ${String(encoding as number)}`
+      );
+  }
 }
 
 /**

ts/packages/anchor/src/program/index.ts

@@ -1,5 +1,5 @@
+import { Buffer } from "buffer";
 import { Commitment, PublicKey } from "@solana/web3.js";
-import { inflate } from "pako";
 import { BorshCoder, Coder } from "../coder/index.js";
 import {
   Idl,
@@ -9,7 +9,6 @@ import {
   idlAddress,
 } from "../idl.js";
 import Provider, { getProvider } from "../provider.js";
-import { utf8 } from "../utils/bytes/index.js";
 import { CustomAccountResolver } from "./accounts-resolver.js";
 import { Address, translateAddress } from "./common.js";
 import { EventManager } from "./event.js";
@@ -345,21 +344,16 @@ export class Program<IDL extends Idl = Idl> {
    * @param provider  The network and wallet context.
    */
   public static async fetchIdl<IDL extends Idl = Idl>(
-    address: Address,
+    programAddress: Address,
     provider?: Provider
   ): Promise<IDL | null> {
     provider = provider ?? getProvider();
-    const programId = translateAddress(address);
-
-    const idlAddr = await idlAddress(programId);
+    const programId = translateAddress(programAddress);
+    const idlAddr = idlAddress(programId);
     const accountInfo = await provider.connection.getAccountInfo(idlAddr);
-    if (!accountInfo) {
-      return null;
-    }
-    // Chop off account discriminator.
-    let idlAccount = decodeIdlAccount(accountInfo.data.slice(8));
-    const inflatedIdl = inflate(idlAccount.data);
-    return JSON.parse(utf8.decode(inflatedIdl));
+    if (!accountInfo) return null;
+
+    return decodeIdlAccount<IDL>(accountInfo.data);
   }
 
   /**

ts/packages/anchor/src/utils/pubkey.ts

@@ -1,4 +1,3 @@
-import { Buffer } from "buffer";
 import { PublicKey } from "@solana/web3.js";
 import { sha256 } from "@noble/hashes/sha256";