fix(generation): fix filtering being dismissed (#189)

* fix(generation): fix filtering being dismissed

* Remove early return

* Restore early return, remove logs on invalid list

Author: chrstph-dvx

__test__/unit/removeInvalidTokens.test.ts

@@ -2,6 +2,42 @@ import { removeInvalidTokensFromList } from '../../src/lib/utils';
 import { ArbTokenList } from '../../src/lib/types';
 
 describe('removeInvalidTokensFromList - performance bug', () => {
+  it('should filter out tokens with non-EVM addresses such as Solana tokens', () => {
+    const listWithSolanaToken: ArbTokenList = {
+      name: 'Test List',
+      timestamp: '2025-01-01T00:00:00.000Z',
+      version: { major: 1, minor: 0, patch: 0 },
+      tokens: [
+        {
+          chainId: 42161,
+          address: '0x1111111111111111111111111111111111111111',
+          name: 'Valid Token',
+          symbol: 'VALID',
+          decimals: 18,
+        },
+        {
+          chainId: 501000101,
+          address: '5mbK36SZ7J19An8jFochhQS4of8g6BwUjbeCSxBSoWdp',
+          name: 'michi',
+          symbol: '$MICHI',
+          decimals: 6,
+        },
+      ],
+    };
+
+    const result = removeInvalidTokensFromList(listWithSolanaToken);
+
+    expect(result.tokens).toHaveLength(1);
+    expect(result.tokens[0].symbol).toBe('VALID');
+    expect(result.tokens).not.toContainEqual({
+      chainId: 501000101,
+      address: '5mbK36SZ7J19An8jFochhQS4of8g6BwUjbeCSxBSoWdp',
+      name: 'michi',
+      symbol: '$MICHI',
+      decimals: 6,
+    });
+  });
+
   it('should remove token with name > 40 characters without hanging', () => {
     const listWithLongName: ArbTokenList = {
       name: 'Test List',

src/lib/token_list_gen.ts

@@ -376,9 +376,9 @@ export const arbifyL1List = async (
   newList: ArbTokenList;
   l1ListName: string;
 }> => {
-  const l1TokenList = await getTokenListObj(pathOrUrl);
-
-  removeInvalidTokensFromList(l1TokenList);
+  const l1TokenList = removeInvalidTokensFromList(
+    await getTokenListObj(pathOrUrl),
+  ) as TokenList;
   const prevArbTokenList = ignorePreviousList
     ? null
     : await getPrevList(prevArbifiedList);
@@ -429,8 +429,9 @@ export const updateArbifiedList = async (
     prevArbifiedList: string | undefined;
   },
 ) => {
-  const arbTokenList = await getTokenListObj(pathOrUrl);
-  removeInvalidTokensFromList(arbTokenList);
+  const arbTokenList = removeInvalidTokensFromList(
+    await getTokenListObj(pathOrUrl),
+  ) as ArbTokenList;
   const prevArbTokenList = ignorePreviousList
     ? null
     : await getPrevList(prevArbifiedList);

src/lib/utils.ts

@@ -8,7 +8,7 @@ import { L2GatewayRouter__factory } from '@arbitrum/sdk/dist/lib/abi/factories/L
 
 import { ArbTokenList, GraphTokenResult } from './types';
 import path from 'path';
-import { tokenListIsValid } from './validateTokenList';
+import { tokenInfoIsValid, tokenListIsValid } from './validateTokenList';
 import {
   l2ToL1GatewayAddresses,
   l2ToL1GatewayAddressesNova,
@@ -246,31 +246,31 @@ export const getTokenListObjFromLocalPath = async (path: string) => {
 export const removeInvalidTokensFromList = (
   tokenList: ArbTokenList | TokenList,
 ): ArbTokenList | TokenList => {
-  // Quick check: if valid, return immediately
-  if (tokenListIsValid(tokenList)) {
+  if (tokenListIsValid(tokenList, { logErrors: false })) {
     return tokenList;
   }
 
-  console.log('Invalid token list detected, filtering invalid tokens...');
+  let removedCount = 0;
+  let loggedInvalidList = false;
 
-  // Validate each token individually
   const validTokens = tokenList.tokens.filter((token) => {
-    const singleTokenList = {
-      ...tokenList,
-      tokens: [token],
-    };
-
-    const isValid = tokenListIsValid(singleTokenList);
+    const isValid = tokenInfoIsValid(token);
 
     if (!isValid) {
+      if (!loggedInvalidList) {
+        console.log('Invalid token list detected, filtering invalid tokens...');
+        loggedInvalidList = true;
+      }
+      removedCount++;
       console.log('Removing invalid token:', token.name || token.address);
     }
 
     return isValid;
   });
 
-  const removedCount = tokenList.tokens.length - validTokens.length;
-  console.log(`Removed ${removedCount} invalid token(s)`);
+  if (removedCount > 0) {
+    console.log(`Removed ${removedCount} invalid token(s)`);
+  }
 
   // Create cleaned list
   const cleanedList = {

src/lib/validateTokenList.ts

@@ -1,25 +1,36 @@
 import Ajv from 'ajv';
 import betterAjvErrors from 'better-ajv-errors';
 import addFormats from 'ajv-formats';
-import { schema, TokenList } from '@uniswap/token-lists';
+import { schema, TokenInfo, TokenList } from '@uniswap/token-lists';
 import { ArbTokenList } from './types';
 
-export const tokenListIsValid = (tokenList: ArbTokenList | TokenList) => {
-  const ajv = new Ajv();
-  addFormats(ajv);
-  schema.properties.tokens.minItems = 0;
-  schema.properties.tokens.maxItems = 15_000;
-  const validate = ajv.compile(schema);
+const ajv = new Ajv({ allErrors: true });
+addFormats(ajv);
+schema.properties.tokens.minItems = 0;
+schema.properties.tokens.maxItems = 15_000;
+const validate = ajv.compile(schema);
+const validateTokenInfo = ajv.compile({
+  ...schema.definitions.TokenInfo,
+  definitions: schema.definitions,
+});
 
-  const res = validate(tokenList);
-  if (validate.errors) {
+export const tokenInfoIsValid = (tokenInfo: TokenInfo) =>
+  validateTokenInfo(tokenInfo) as boolean;
+
+export const tokenListIsValid = (
+  tokenList: ArbTokenList | TokenList,
+  { logErrors = true }: { logErrors?: boolean } = {},
+) => {
+  const isValid = validate(tokenList) as boolean;
+
+  if (logErrors && validate.errors) {
     const output = betterAjvErrors(schema, tokenList, validate.errors, {
       indent: 2,
     });
     console.log(output);
   }
 
-  return res;
+  return isValid;
 };
 
 export const validateTokenListWithErrorThrowing = (