Skip to content

Commit 1604251

Browse files
isArlekinisArlekin
authored
feat: pass Alchemy api key and parse cache by batch with size 20 (#110)
1 parent a4790ab commit 1604251

File tree

2 files changed

+141
-120
lines changed

2 files changed

+141
-120
lines changed

‎src/scripts/parse-cache.ts‎

Lines changed: 138 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ async function getVotingData(initialProposals: InitialProposal[]) {
9797
] as any,
9898
rpcUrl: getRPCUrl(
9999
appConfig.votingMachineChainIds[0] as SupportedChainIds,
100+
{
101+
alchemyKey: process.env.ALCHEMY_API_KEY,
102+
},
100103
),
101104
}),
102105
}),
@@ -109,7 +112,9 @@ async function getVotingData(initialProposals: InitialProposal[]) {
109112
abi: IVotingMachineDataHelper_ABI,
110113
client: createViemClient({
111114
chain: ChainList[chainId as SupportedChainIds] as any,
112-
rpcUrl: getRPCUrl(chainId as SupportedChainIds),
115+
rpcUrl: getRPCUrl(chainId as SupportedChainIds, {
116+
alchemyKey: process.env.ALCHEMY_API_KEY,
117+
}),
113118
}),
114119
});
115120
});
@@ -449,7 +454,9 @@ async function parseProposalEvents(
449454
chain: ChainList[
450455
appConfig.govCoreChainId as SupportedChainIds
451456
] as any,
452-
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds),
457+
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds, {
458+
alchemyKey: process.env.ALCHEMY_API_KEY,
459+
}),
453460
}),
454461
{
455462
blockHash: governanceEvents.find(
@@ -628,7 +635,9 @@ async function parseCache() {
628635
appConfig.govCoreConfig.dataHelperContractAddress,
629636
client: createViemClient({
630637
chain: ChainList[appConfig.govCoreChainId as SupportedChainIds],
631-
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds),
638+
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds, {
639+
alchemyKey: process.env.ALCHEMY_API_KEY,
640+
}),
632641
}),
633642
});
634643

@@ -639,7 +648,9 @@ async function parseCache() {
639648
abi: IVotingPortal_ABI,
640649
client: createViemClient({
641650
chain: ChainList[appConfig.govCoreChainId as SupportedChainIds],
642-
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds),
651+
rpcUrl: getRPCUrl(appConfig.govCoreChainId as SupportedChainIds, {
652+
alchemyKey: process.env.ALCHEMY_API_KEY,
653+
}),
643654
}),
644655
address: proposal[1].votingPortal,
645656
});
@@ -679,127 +690,135 @@ async function parseCache() {
679690
};
680691
});
681692

682-
// start parsing
683-
await Promise.allSettled(
684-
proposalsData.map(async (proposal) => {
685-
const proposalIpfsData = ipfsCache[proposal.ipfsHash];
686-
687-
const path = `${proposal.votingChainId}/events`;
688-
const votingMachineAddress = getVotingMachineAddress(
689-
proposal.votingChainId,
690-
proposal.id,
691-
);
692-
const votesCache =
693-
readJSONCache<Awaited<ReturnType<typeof getVotingMachineEvents>>>(
694-
path,
695-
votingMachineAddress,
696-
) || [];
697-
698-
// format VoteEmitted events to UI data format
699-
const proposalVoters: VotersData[] = votesCache
700-
.filter(
701-
(data) =>
702-
data.eventName === 'VoteEmitted' &&
703-
Number(data.args.proposalId) === proposal.id,
704-
)
705-
.sort((a, b) => Number(b.blockNumber) - Number(a.blockNumber))
706-
.map((event) => ({
707-
proposalId: Number(event.args.proposalId),
708-
address: (
709-
(event.eventName === 'VoteEmitted' && event.args.voter) ||
710-
''
711-
).toString() as Hex,
712-
support:
713-
(event.eventName === 'VoteEmitted' && event.args.support) || false,
714-
votingPower: normalizeBN(
715-
(
716-
(event.eventName === 'VoteEmitted' && event.args.votingPower) ||
693+
// start parsing in batches to reduce peak memory usage
694+
const batchSize = 20;
695+
for (let i = 0; i < proposalsData.length; i += batchSize) {
696+
const batch = proposalsData.slice(i, i + batchSize);
697+
await Promise.allSettled(
698+
batch.map(async (proposal) => {
699+
const proposalIpfsData = ipfsCache[proposal.ipfsHash];
700+
701+
const path = `${proposal.votingChainId}/events`;
702+
const votingMachineAddress = getVotingMachineAddress(
703+
proposal.votingChainId,
704+
proposal.id,
705+
);
706+
const votesCache =
707+
readJSONCache<Awaited<ReturnType<typeof getVotingMachineEvents>>>(
708+
path,
709+
votingMachineAddress,
710+
) || [];
711+
712+
// format VoteEmitted events to UI data format
713+
const proposalVoters: VotersData[] = votesCache
714+
.filter(
715+
(data) =>
716+
data.eventName === 'VoteEmitted' &&
717+
Number(data.args.proposalId) === proposal.id,
718+
)
719+
.sort((a, b) => Number(b.blockNumber) - Number(a.blockNumber))
720+
.map((event) => ({
721+
proposalId: Number(event.args.proposalId),
722+
address: (
723+
(event.eventName === 'VoteEmitted' && event.args.voter) ||
717724
''
718-
).toString(),
719-
18,
720-
).toNumber(),
721-
transactionHash: event.transactionHash,
722-
blockNumber: Number(event.blockNumber),
723-
chainId: proposal.votingChainId,
724-
}));
725-
726-
// get ens name for voters
727-
const proposalVotersWithEnsName = await Promise.all(
728-
proposalVoters.map(async (vote) => {
729-
try {
730-
const name = await getEnsName(
731-
createViemClient({
732-
chain: ChainList[ChainId.mainnet],
733-
rpcUrl: getRPCUrl(ChainId.mainnet),
734-
}),
735-
{
736-
address: vote.address,
737-
},
738-
);
739-
740-
return {
741-
...vote,
742-
ensName: name ? name : undefined,
743-
};
744-
} catch {
745-
return {
746-
...vote,
747-
ensName: undefined,
748-
};
749-
}
750-
}),
751-
);
725+
).toString() as Hex,
726+
support:
727+
(event.eventName === 'VoteEmitted' && event.args.support) ||
728+
false,
729+
votingPower: normalizeBN(
730+
(
731+
(event.eventName === 'VoteEmitted' && event.args.votingPower) ||
732+
''
733+
).toString(),
734+
18,
735+
).toNumber(),
736+
transactionHash: event.transactionHash,
737+
blockNumber: Number(event.blockNumber),
738+
chainId: proposal.votingChainId,
739+
}));
740+
741+
// get ens name for voters
742+
const proposalVotersWithEnsName = await Promise.all(
743+
proposalVoters.map(async (vote) => {
744+
try {
745+
const name = await getEnsName(
746+
createViemClient({
747+
chain: ChainList[ChainId.mainnet],
748+
rpcUrl: getRPCUrl(ChainId.mainnet, {
749+
alchemyKey: process.env.ALCHEMY_API_KEY,
750+
}),
751+
}),
752+
{
753+
address: vote.address,
754+
},
755+
);
756+
757+
return {
758+
...vote,
759+
ensName: name ? name : undefined,
760+
};
761+
} catch {
762+
return {
763+
...vote,
764+
ensName: undefined,
765+
};
766+
}
767+
}),
768+
);
752769

753-
// get payloads data for proposal
754-
const { proposalPayloadsData } = getProposalPayloads(proposal);
770+
// get payloads data for proposal
771+
const { proposalPayloadsData } = getProposalPayloads(proposal);
755772

756-
const dataToWrite = {
757-
payloads: proposalPayloadsData,
758-
ipfs: proposalIpfsData,
759-
proposal: proposal,
760-
};
773+
const dataToWrite = {
774+
payloads: proposalPayloadsData,
775+
ipfs: proposalIpfsData,
776+
proposal: proposal,
777+
};
761778

762-
const proposalCache =
763-
readJSONCache<{
764-
payloads: Payload[];
765-
ipfs: ProposalMetadata;
766-
proposal: BasicProposal;
767-
}>(`${initDirName}/proposals`, `proposal_${proposal.id}`) || undefined;
768-
if (
769-
!proposalCache ||
770-
!proposalCache?.proposal.isFinished ||
771-
Number(proposalCache.proposal.cancellationFee) > 0
772-
) {
773-
writeJSONCache(
774-
`${initDirName}/proposals`,
775-
`proposal_${proposal.id}`,
776-
dataToWrite,
777-
);
778-
console.log(`Proposal ${proposal.id} data cached.`);
779-
}
779+
const proposalCache =
780+
readJSONCache<{
781+
payloads: Payload[];
782+
ipfs: ProposalMetadata;
783+
proposal: BasicProposal;
784+
}>(`${initDirName}/proposals`, `proposal_${proposal.id}`) ||
785+
undefined;
786+
if (
787+
!proposalCache ||
788+
!proposalCache?.proposal.isFinished ||
789+
Number(proposalCache.proposal.cancellationFee) > 0
790+
) {
791+
writeJSONCache(
792+
`${initDirName}/proposals`,
793+
`proposal_${proposal.id}`,
794+
dataToWrite,
795+
);
796+
console.log(`Proposal ${proposal.id} data cached.`);
797+
}
780798

781-
const proposalVotersCache =
782-
readJSONCache<{
783-
votes: VotersData[];
784-
}>(`${initDirName}/votes`, `vote_for_proposal_${proposal.id}`) ||
785-
undefined;
786-
if (
787-
!proposalVotersCache ||
788-
!isProposalFinal(proposalCache ? proposalCache.proposal.state : 0)
789-
) {
790-
writeJSONCache(
791-
`${initDirName}/votes`,
792-
`vote_for_proposal_${proposal.id}`,
793-
{
794-
votes: proposalVotersWithEnsName,
795-
},
796-
);
797-
console.log(`Proposal ${proposal.id} voters data cached.`);
798-
}
799+
const proposalVotersCache =
800+
readJSONCache<{
801+
votes: VotersData[];
802+
}>(`${initDirName}/votes`, `vote_for_proposal_${proposal.id}`) ||
803+
undefined;
804+
if (
805+
!proposalVotersCache ||
806+
!isProposalFinal(proposalCache ? proposalCache.proposal.state : 0)
807+
) {
808+
writeJSONCache(
809+
`${initDirName}/votes`,
810+
`vote_for_proposal_${proposal.id}`,
811+
{
812+
votes: proposalVotersWithEnsName,
813+
},
814+
);
815+
console.log(`Proposal ${proposal.id} voters data cached.`);
816+
}
799817

800-
await parseProposalEvents(proposal, configs, contractsConstants);
801-
}),
802-
);
818+
await parseProposalEvents(proposal, configs, contractsConstants);
819+
}),
820+
);
821+
}
803822

804823
const proposalIds = proposalsData
805824
.filter((proposal) => proposal.isFinished)

‎src/scripts/update-cache-package.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,9 @@ async function updateVMEvents(
237237
abi: IVotingPortal_ABI,
238238
client: createViemClient({
239239
chain: ChainList[govCoreChainId as SupportedChainIds] as any,
240-
rpcUrl: getRPCUrl(govCoreChainId as SupportedChainIds),
240+
rpcUrl: getRPCUrl(govCoreChainId as SupportedChainIds, {
241+
alchemyKey: process.env.ALCHEMY_API_KEY,
242+
}),
241243
}),
242244
address: portal,
243245
});

0 commit comments

Comments
 (0)