Merge pull request #35 from BeanstalkFarms/codex/rpc-cost-optimization-port

[codex] Optimize API RPC usage

Author: Brean0

docs/rpc-cost-optimization-port-plan.md

@@ -0,0 +1,58 @@
+# RPC Cost Optimization Port Plan
+
+Date: 2026-05-09
+
+This captures the initial Pinto-to-Beanstalk port plan before code changes. It is intentionally redacted: exposed key material is described by source and protection behavior, not copied.
+
+## Pinto Reference Changes
+
+- Pinto frontend:
+  - `Reduce conservative RPC waste`
+  - `Add app version refresh guard`
+- Pinto API:
+  - `Optimize RPC usage`
+- Pinto bots:
+  - `Optimize event RPC polling`
+  - `Add shared event dispatcher`
+  - `Fix dispatcher routing regressions`
+  - `Add Phase 3A RPC cleanup`
+
+## Live Key Exposure Audit
+
+- `https://pinto.money/` currently did not expose full Alchemy RPC URLs in the initial deployed JS/CSS bundle. The source still builds RPC URLs from `VITE_ALCHEMY_API_KEY`, so any configured Vite key is public by design.
+- `https://bean.money/` and `https://www.bean.money/` did not expose Alchemy RPC URLs in the initial deployed JS/CSS bundles checked.
+- `https://app.bean.money/` exposes Alchemy RPC URLs in the deployed app bundle. The Beanstalk production ETH/Arbitrum key matched the local UI production key and rejected requests with no Origin or a foreign Origin, while accepting the app Origin.
+- The Beanstalk app bundle also contains provider-library default/community Alchemy endpoints. One Optimism endpoint accepted a foreign Origin, but it did not match the Beanstalk UI production key.
+- Local Beanstalk UI source contains a hardcoded Alchemy NFT API URL in `projects/ui/src/pages/nft.tsx`. It was not seen in the initial live app bundle scan, but it should still be treated as exposed source key material if the route or code is shipped elsewhere.
+- Local Pinto UI source contains Alchemy URL construction in `src/utils/wagmi/chains.ts` and `src/pages/DevToolsInstall.tsx`, plus literal Alchemy URL examples/comments in the dev tools page. These should be removed or rotated if they point at live apps.
+
+## Security Posture
+
+- All `VITE_*`/client-side RPC keys should be treated as public identifiers, not secrets.
+- Frontend Alchemy keys are acceptable only if separately scoped for browser use, origin/domain restricted, quota limited, and monitored.
+- Backend/API/bot keys should use separate Alchemy apps with no browser origin allowlist dependency and should never be bundled into frontends.
+- Rotate any key that appears literally in source, comments, docs, old bundles, or test fixtures if it has real permissions or billable quota.
+
+## Port Order
+
+1. Beanstalk API:
+   - Port provider request batching and optional RPC request logging.
+   - Add historical RPC call caching for explicit `blockTag` reads.
+   - Add Multicall3 helpers where Beanstalk endpoints perform repeated per-token reads.
+   - Avoid Pinto's Base-only timestamp lookup shortcut unless Beanstalk chain behavior is handled explicitly.
+2. Beanstalk bots:
+   - Port HTTP RPC request counting.
+   - Add a shared event dispatcher for Well, Beanstalk, and Market monitors.
+   - De-duplicate contract/event filters and reuse receipts instead of fetching the same transaction receipts repeatedly.
+   - Keep Beanstalk-only monitors such as Barn Raise and Contracts Migrated separate until routing is verified.
+3. Beanstalk frontend:
+   - Port the app version refresh guard.
+   - Audit polling intervals and heavy reads against Pinto's lower-cost query defaults.
+   - Remove literal Alchemy URL/key examples from UI source where practical.
+
+## Verification Targets
+
+- API: targeted Jest tests around provider creation, cache behavior, and multicall-compatible service paths.
+- Bots: targeted unit/integration tests for event source construction, dispatcher routing, and receipt reuse.
+- Frontend: build/typecheck if the existing Beanstalk UI dependency state allows it; otherwise, static verification and minimal focused checks.
+

src/datasources/alchemy.js

@@ -1,34 +1,111 @@
 const { Alchemy } = require('alchemy-sdk');
 const EnvUtil = require('../utils/env');
+const { ethers } = require('ethers');
+const Log = require('../utils/logging');
+
+const RPC_BATCH_OPTIONS = {
+  batchMaxCount: Number(process.env.RPC_BATCH_MAX_COUNT ?? 10),
+  batchStallTime: Number(process.env.RPC_BATCH_STALL_TIME_MS ?? 10),
+  batchMaxSize: Number(process.env.RPC_BATCH_MAX_SIZE ?? 1024 * 1024)
+};
+const ETHERS_NETWORKS = {
+  eth: { name: 'mainnet', chainId: 1 },
+  arb: { name: 'arbitrum', chainId: 42161 }
+};
 
 class AlchemyUtil {
+  // Contains alchemy object
+  static _alchemies = {};
   // Contains a provider by chain
   static _providers = {};
   // Access to the underlying promise whos execution populates _providers.
   // Allows flexibility in awaiting when necessary (i.e. once at application startup)
   static _providerPromises = {};
+  static _rpcCounters = {};
 
   static {
     for (const chain of EnvUtil.getEnabledChains()) {
-      const settings = {
-        apiKey: EnvUtil.getAlchemyKey(),
-        network: `${chain}-mainnet` // Of type alchemy-sdk.Network
-      };
-      const alchemy = new Alchemy(settings);
-      this._providerPromises[chain] = alchemy.config.getProvider().then((p) => {
-        this._providers[chain] = p;
-      });
+      if (EnvUtil.getCustomRpcUrl(chain)) {
+        this._providers[chain] = this._makeProvider(chain, EnvUtil.getCustomRpcUrl(chain));
+      } else {
+        const settings = {
+          apiKey: EnvUtil.getAlchemyKey(),
+          network: `${chain}-mainnet` // Of type alchemy-sdk.Network
+        };
+        this._alchemies[chain] = new Alchemy(settings);
+        this._providers[chain] = this._makeProvider(
+          chain,
+          `https://${chain}-mainnet.g.alchemy.com/v2/${EnvUtil.getAlchemyKey()}`
+        );
+      }
     }
   }
 
+  static alchemyForChain(chain) {
+    return AlchemyUtil._alchemies[chain];
+  }
+
   static providerForChain(chain) {
     return AlchemyUtil._providers[chain];
   }
 
+  static rpcCounts(chain) {
+    return AlchemyUtil._rpcCounters[chain];
+  }
+
   // Returns immediately if already resolved (and _providers is populated)
   static async ready(chain) {
     await AlchemyUtil._providerPromises[chain];
   }
+
+  static _makeProvider(chain, url) {
+    const provider = new ethers.JsonRpcProvider(url, ETHERS_NETWORKS[chain], RPC_BATCH_OPTIONS);
+    // Needed to get the alchemy-sdk Contract constructor to work with an ethers v6 provider.
+    provider._isProvider = true;
+    return AlchemyUtil._withRequestCounter(chain, provider);
+  }
+
+  static _withRequestCounter(chain, provider) {
+    if (process.env.LOG_RPC !== '1' || !provider._send) {
+      return provider;
+    }
+
+    const counter = {
+      total: 0,
+      methods: {}
+    };
+    AlchemyUtil._rpcCounters[chain] = counter;
+
+    const originalSend = provider._send.bind(provider);
+    provider._send = async (payload) => {
+      const requests = Array.isArray(payload) ? payload : [payload];
+      for (const request of requests) {
+        counter.total++;
+        counter.methods[request.method] = (counter.methods[request.method] ?? 0) + 1;
+      }
+      return await originalSend(payload);
+    };
+
+    let didLog = false;
+    const logCounts = () => {
+      if (didLog) {
+        return;
+      }
+      didLog = true;
+      Log.info(`RPC request counts for ${chain}`, JSON.stringify(counter));
+    };
+    process.once('beforeExit', logCounts);
+    process.once('exit', logCounts);
+    process.once('SIGINT', () => {
+      logCounts();
+      process.exit(0);
+    });
+    process.once('SIGTERM', () => {
+      logCounts();
+      process.exit(0);
+    });
+    return provider;
+  }
 }
 
 module.exports = AlchemyUtil;

src/datasources/contracts/contracts.js

@@ -1,5 +1,6 @@
 const { Contract: AlchemyContract } = require('alchemy-sdk');
 const { C } = require('../../constants/runtime-constants');
+const RpcCache = require('../rpc-cache');
 const SuperContract = require('./super-contract');
 const wellFunctionAbi = require('../../datasources/abi/basin/WellFunction.json');
 
@@ -20,10 +21,11 @@ class Contracts {
 
   static makeContract(address, abi, provider) {
     const underlyingContract = new AlchemyContract(address, abi, provider);
-    return new SuperContract(underlyingContract);
+    return RpcCache.wrapContract(new SuperContract(underlyingContract), address);
   }
 
   static _getDefaultContract(address, c = C()) {
+    address = address.toLowerCase();
     const network = c.CHAIN;
     const key = JSON.stringify({ address, network });
     if (!Contracts._contracts[key]) {

src/datasources/contracts/multicall.js

@@ -0,0 +1,91 @@
+const { C } = require('../../constants/runtime-constants');
+const retryable = require('../../utils/async/retryable');
+const SuperContract = require('./super-contract');
+const Contracts = require('./contracts');
+
+const MULTICALL3_ADDRESS = '0xcA11bde05977b3631167028862bE2a173976CA11';
+const MULTICALL3_ABI = [
+  {
+    inputs: [
+      {
+        components: [
+          { internalType: 'address', name: 'target', type: 'address' },
+          { internalType: 'bool', name: 'allowFailure', type: 'bool' },
+          { internalType: 'bytes', name: 'callData', type: 'bytes' }
+        ],
+        internalType: 'struct Multicall3.Call3[]',
+        name: 'calls',
+        type: 'tuple[]'
+      }
+    ],
+    name: 'aggregate3',
+    outputs: [
+      {
+        components: [
+          { internalType: 'bool', name: 'success', type: 'bool' },
+          { internalType: 'bytes', name: 'returnData', type: 'bytes' }
+        ],
+        internalType: 'struct Multicall3.Result[]',
+        name: 'returnData',
+        type: 'tuple[]'
+      }
+    ],
+    stateMutability: 'view',
+    type: 'function'
+  }
+];
+
+class Multicall {
+  static async aggregate(calls, { blockTag = 'latest', c = C() } = {}) {
+    if (calls.length === 0) {
+      return [];
+    }
+
+    const results = new Array(calls.length);
+    const callsByBlock = calls.reduce((acc, call, index) => {
+      const callBlock = call.blockTag ?? blockTag;
+      if (!acc.has(callBlock)) {
+        acc.set(callBlock, []);
+      }
+      acc.get(callBlock).push({ call, index });
+      return acc;
+    }, new Map());
+
+    await Promise.all(
+      [...callsByBlock.entries()].map(async ([callBlock, entries]) => {
+        const multicall = Contracts.makeContract(MULTICALL3_ADDRESS, MULTICALL3_ABI, c.RPC);
+        const aggregateCalls = entries.map(({ call }) => ({
+          target: call.contract.address,
+          allowFailure: call.allowFailure ?? false,
+          callData: call.contract.interface.encodeFunctionData(call.method, call.args ?? [])
+        }));
+
+        const aggregateResults = await retryable(() =>
+          multicall.aggregate3({ target: 'SuperContract', skipTransform: true }, aggregateCalls, {
+            blockTag: callBlock
+          })
+        );
+
+        for (let i = 0; i < entries.length; ++i) {
+          const { call, index } = entries[i];
+          const result = aggregateResults[i];
+          if (!result.success) {
+            if (call.allowFailure) {
+              results[index] = null;
+              continue;
+            }
+            throw new Error(`Multicall failed for ${call.contract.address}.${call.method}`);
+          }
+
+          const fragment = call.contract.interface.getFunction(call.method);
+          const decoded = call.contract.interface.decodeFunctionResult(fragment, result.returnData);
+          results[index] = SuperContract._transformAll(fragment.outputs.length === 1 ? decoded[0] : decoded);
+        }
+      })
+    );
+
+    return results;
+  }
+}
+
+module.exports = Multicall;

src/datasources/contracts/super-contract.js

@@ -17,7 +17,17 @@ class SuperContract {
         }
 
         return async (...args) => {
-          const rawResult = await retryable(() => contract[property](...args));
+          const { superOptions, contractArgs } = SuperContract._identifySuperArgs(args);
+
+          const retryableOptions = {};
+          if (superOptions?.skipRetry) {
+            retryableOptions.earlyTerminate = superOptions.skipRetry;
+          }
+          const rawResult = await retryable(() => contract[property](...contractArgs), retryableOptions);
+
+          if (superOptions?.skipTransform) {
+            return rawResult;
+          }
           return SuperContract._transformAll(rawResult);
         };
       }
@@ -26,6 +36,15 @@ class SuperContract {
     return new Proxy(this, proxyHandler);
   }
 
+  static _identifySuperArgs(args) {
+    const superArgsIndex = args.findIndex((a) => a?.target === 'SuperContract');
+    if (superArgsIndex !== -1) {
+      const superOptions = args.splice(superArgsIndex, 1)[0];
+      return { superOptions, contractArgs: args };
+    }
+    return { contractArgs: args };
+  }
+
   // Transforms everything in this result to be a BigInt.
   // Handles arrays/tuples (with named fields), and single values
   static _transformAll(rawResult) {