diff --git a/packages/retryable-monitor/handlers/reportFailedRetryables.ts b/packages/retryable-monitor/handlers/reportFailedRetryables.ts index 46ff085..5f037f1 100644 --- a/packages/retryable-monitor/handlers/reportFailedRetryables.ts +++ b/packages/retryable-monitor/handlers/reportFailedRetryables.ts @@ -6,7 +6,7 @@ import { TokenDepositData, } from '../core/types' import { postSlackMessage } from './slack/postSlackMessage' -import { generateFailedRetryableSlackMessage } from './slack/slackMessageGenerator' +import { generateRetryableSlackBlocks } from './slack/slackMessageFormattingUtils' export const reportFailedRetryables = async ({ parentChainRetryableReport, @@ -49,7 +49,7 @@ export const reportFailedRetryables = async ({ ) try { - const reportStr = await generateFailedRetryableSlackMessage({ + const blocks = await generateRetryableSlackBlocks({ parentChainRetryableReport, childChainRetryableReport, tokenDepositData, @@ -58,7 +58,10 @@ export const reportFailedRetryables = async ({ childChainProvider, }) - postSlackMessage({ message: reportStr }) + await postSlackMessage({ + blocks, + message: `Failed Retryable Alert - [${childChain.name}] ${t.status}`, + }) } catch (e) { console.log('Could not send slack message', e) } diff --git a/packages/retryable-monitor/handlers/slack/postSlackMessage.ts b/packages/retryable-monitor/handlers/slack/postSlackMessage.ts index 1d9b29b..877b576 100644 --- a/packages/retryable-monitor/handlers/slack/postSlackMessage.ts +++ b/packages/retryable-monitor/handlers/slack/postSlackMessage.ts @@ -1,18 +1,38 @@ -import { postSlackMessage as commonPostSlackMessage } from '../../../utils/postSlackMessage' +import { + postSlackMessage as commonPostSlackMessage, + postSlackBlocks as commonPostSlackBlocks, +} from '../../../utils/postSlackMessage' const slackToken = process.env.RETRYABLE_MONITORING_SLACK_TOKEN const slackChannel = process.env.RETRYABLE_MONITORING_SLACK_CHANNEL -export const postSlackMessage = ({ message }: { message: string }) => { +export const postSlackMessage = ({ + message, + blocks, +}: { + message?: string + blocks?: any[] +}) => { if (!slackToken) throw new Error(`Slack token is required.`) if (!slackChannel) throw new Error(`Slack channel is required.`) if (process.env.NODE_ENV === 'DEV') return if (process.env.NODE_ENV === 'CI' && message === 'success') return - commonPostSlackMessage({ - slackToken, - slackChannel, - message, - }) + if (blocks) { + return commonPostSlackBlocks({ + slackToken, + slackChannel, + blocks, + text: message || 'Failed Retryable Alert', + }) + } else if (message) { + return commonPostSlackMessage({ + slackToken, + slackChannel, + message, + }) + } else { + throw new Error('Either message or blocks must be provided') + } } diff --git a/packages/retryable-monitor/handlers/slack/slackMessageFormattingUtils.ts b/packages/retryable-monitor/handlers/slack/slackMessageFormattingUtils.ts index fdb825c..d848645 100644 --- a/packages/retryable-monitor/handlers/slack/slackMessageFormattingUtils.ts +++ b/packages/retryable-monitor/handlers/slack/slackMessageFormattingUtils.ts @@ -25,6 +25,8 @@ import { ChildNetwork, getExplorerUrlPrefixes } from '../../../utils' * */ +type Priority = 'Critical' | 'High' | 'Medium' | 'Low' + let ethPriceCache: number let tokenPriceCache: { [key: string]: number } = {} @@ -48,121 +50,16 @@ export const getTimeDifference = (timestampInSeconds: number) => { } } -export const formatPrefix = ( - ticket: ChildChainTicketReport, - childChainName: string -) => { - const now = Math.floor(new Date().getTime() / 1000) // now in s - - let prefix - switch (ticket.status) { - case ParentToChildMessageStatus[ - ParentToChildMessageStatus.FUNDS_DEPOSITED_ON_CHILD - ]: - prefix = `*[${childChainName}] Redeem failed for ticket:*` - break - case ParentToChildMessageStatus[ParentToChildMessageStatus.EXPIRED]: - prefix = `*[${childChainName}] Retryable ticket expired:*` - break - case ParentToChildMessageStatus[ParentToChildMessageStatus.NOT_YET_CREATED]: - prefix = `*[${childChainName}] Retryable ticket hasn't been scheduled:*` - break - default: - prefix = `*[${childChainName}] Found retryable ticket in unrecognized state:*` - } - - // if ticket is about to expire in less than 48h make it a bit dramatic - if (ticket.status == 'RedeemFailed' || ticket.status == 'Created') { - const criticalSoonToExpirePeriod = 2 * 24 * 60 * 60 // 2 days in s - const expiresIn = +ticket.timeoutTimestamp - now - if (expiresIn < criticalSoonToExpirePeriod) { - prefix = `🆘📣 ${prefix} 📣🆘` - } - } - - return prefix -} - -export const formatInitiator = async ( - deposit: TokenDepositData | undefined, - l1Report: ParentChainTicketReport | undefined, - childChain: ChildNetwork -) => { - const { PARENT_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes(childChain) - - if (deposit !== undefined) { - let msg = '\n\t *Deposit initiated by:* ' - // let text = await getContractName(Chain.ETHEREUM, deposit.sender) - return `${msg}<${PARENT_CHAIN_ADDRESS_PREFIX + deposit.sender}|${ - deposit.sender - }>` - } - - if (l1Report !== undefined) { - let msg = '\n\t *Retryable sender:* ' - // let text = await getContractName(Chain.ETHEREUM, l1Report.sender) - return `${msg}<${PARENT_CHAIN_ADDRESS_PREFIX + l1Report.sender}|${ - l1Report.sender - }>` - } - - return '' -} - -export const formatId = ( - ticket: ChildChainTicketReport, - childChain: ChildNetwork -) => { - let msg = '\n\t *Child chain ticket creation TX:* ' - - if (ticket.id == null) { - return msg + '-' - } - - const { CHILD_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) - - return `${msg}<${CHILD_CHAIN_TX_PREFIX + ticket.id}|${ticket.id}>` -} - -export const formatL1TX = ( - l1Report: ParentChainTicketReport | undefined, - childChain: ChildNetwork -) => { - let msg = '\n\t *Parent Chain TX:* ' - - if (l1Report == undefined) { - return msg + '-' - } - - const { PARENT_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) - - return `${msg}<${PARENT_CHAIN_TX_PREFIX + l1Report.transactionHash}|${ - l1Report.transactionHash - }>` -} - -export const formatL2ExecutionTX = ( - ticket: ChildChainTicketReport, - childChain: ChildNetwork -) => { - let msg = '\n\t *Child chain execution TX:* ' - - if (!ticket.retryTxHash) { - return msg + ': No auto-redeem attempt found' - } - - const { CHILD_CHAIN_TX_PREFIX } = getExplorerUrlPrefixes(childChain) - - return `${msg}<${CHILD_CHAIN_TX_PREFIX + ticket.retryTxHash}|${ - ticket.retryTxHash - }>` +export const timestampToDate = (timestampInSeconds: number) => { + const date = new Date(timestampInSeconds * 1000) + return date.toUTCString() } -export const formatL2Callvalue = async ( +export const getCallValueInfo = async ( ticket: ChildChainTicketReport, childChain: ChildNetwork, parentChainProvider: Provider -) => { +): Promise<{ valueText: string; valueUsd: number }> => { if (childChain.nativeToken) { const erc20 = ERC20__factory.connect( childChain.nativeToken, @@ -173,107 +70,42 @@ export const formatL2Callvalue = async ( erc20.decimals(), ]) - const nativeTokenAmount = ethers.utils.formatUnits(ticket.deposit, decimals) - return `\n\t *Child chain callvalue:* ${nativeTokenAmount} ${symbol} (Gas token: ${symbol})` + const amount = ethers.utils.formatUnits(ticket.deposit, decimals) + return { + valueText: `${parseFloat(amount).toFixed(4)} ${symbol}`, + valueUsd: 0, // Custom tokens don't have USD value by default + } } else { const ethAmount = ethers.utils.formatEther(ticket.deposit) - const depositWorthInUsd = (+ethAmount * (await getEthPrice())).toFixed(2) - return `\n\t *Child chain callvalue:* ${ethAmount} ETH ($${depositWorthInUsd})` - } -} - -export const formatTokenDepositData = async ( - deposit: TokenDepositData | undefined -) => { - let msg = '\n\t *Tokens deposited:* ' - - if (deposit === undefined) { - return msg + '-' - } - - const amount = deposit.tokenAmount - ? ethers.utils.formatUnits(deposit.tokenAmount, deposit.l1Token.decimals) - : '-' - - const tokenPriceInUSD = await getTokenPrice(deposit.l1Token.id) - if (tokenPriceInUSD !== undefined) { - const depositWorthInUSD = (+amount * tokenPriceInUSD).toFixed(2) - msg = `${msg} ${amount} ${deposit.l1Token.symbol} (\$${depositWorthInUSD}) (${deposit.l1Token.id})` - } else { - msg = `${msg} ${amount} ${deposit.l1Token.symbol} (${deposit.l1Token.id})` + const ethPrice = await getEthPrice() + const valueUsd = +ethAmount * ethPrice + return { + valueText: `${parseFloat(ethAmount).toFixed(4)} ETH`, + valueUsd, + } } - - return msg -} - -export const formatDestination = async ( - ticket: ChildChainTicketReport, - childChain: ChildNetwork -) => { - let msg = `\n\t *Destination:* ` - const { CHILD_CHAIN_ADDRESS_PREFIX } = getExplorerUrlPrefixes(childChain) - - return `${msg}<${CHILD_CHAIN_ADDRESS_PREFIX + ticket.retryTo}|${ - ticket.retryTo - }>` } -export const formatGasData = async ( +export const formatL2Callvalue = async ( ticket: ChildChainTicketReport, - childChainProvider: Provider + childChain: ChildNetwork, + parentChainProvider: Provider ) => { - const { l2GasPrice, l2GasPriceAtCreation, redeemEstimate } = await getGasInfo( - +ticket.createdAtBlockNumber, - ticket.id, - childChainProvider + const { valueText, valueUsd } = await getCallValueInfo( + ticket, + childChain, + parentChainProvider ) - let msg = `\n\t *Gas params:* ` - msg += `\n\t\t gas price provided: ${ethers.utils.formatUnits( - ticket.gasFeeCap, - 'gwei' - )} gwei` - - if (l2GasPriceAtCreation) { - msg += `\n\t\t gas price at ticket creation block: ${ethers.utils.formatUnits( - l2GasPriceAtCreation, - 'gwei' - )} gwei` - } else { - msg += `\n\t\t gas price at ticket creation block: unable to fetch (missing data)` - } - - msg += `\n\t\t gas price now: ${ethers.utils.formatUnits( - l2GasPrice, - 'gwei' - )} gwei` - msg += `\n\t\t gas limit provided: ${ticket.gasLimit}` - - if (redeemEstimate) { - msg += `\n\t\t redeem gas estimate: ${redeemEstimate} ` + if (childChain.nativeToken) { + const parts = valueText.split(' ') + const symbol = parts[1] + return `\n\t *Child chain callvalue:* ${valueText} (Gas token: ${symbol})` } else { - msg += `\n\t\t redeem gas estimate: estimateGas call reverted` - } - - return msg -} - -export const formatCreatedAt = (ticket: ChildChainTicketReport) => { - return `\n\t *Created at:* ${timestampToDate(+ticket.createdAtTimestamp)}` -} - -export const formatExpiration = (ticket: ChildChainTicketReport) => { - let msg = `\n\t *${ - ticket.status == 'Expired' ? `Expired` : `Expires` - } at:* ${timestampToDate(+ticket.timeoutTimestamp)}` - - if (ticket.status == 'RedeemFailed' || ticket.status == 'Created') { - msg = `${msg} (that's ${getTimeDifference( - +ticket.timeoutTimestamp - )} from now)` + return `\n\t *Child chain callvalue:* ${valueText} ($${valueUsd.toFixed( + 2 + )})` } - - return msg } export const getEthPrice = async () => { @@ -304,17 +136,6 @@ export const getTokenPrice = async (tokenAddress: string) => { return tokenPriceCache[tokenAddress] } -// Unix timestamp -export const getPastTimestamp = (daysAgoInMs: number) => { - const now = new Date().getTime() - return Math.floor((now - daysAgoInMs) / 1000) -} - -export const timestampToDate = (timestampInSeconds: number) => { - const date = new Date(timestampInSeconds * 1000) - return date.toUTCString() -} - /** * Call precompiles to get info about gas price and gas estimation for the TX execution. * @@ -362,3 +183,191 @@ export async function getGasInfo( return { l2GasPrice, l2GasPriceAtCreation, redeemEstimate } } + +export const generateRetryableSlackBlocks = async ({ + parentChainRetryableReport, + childChainRetryableReport, + tokenDepositData, + childChain, + parentChainProvider, + childChainProvider, +}: { + parentChainRetryableReport: ParentChainTicketReport + childChainRetryableReport: ChildChainTicketReport + tokenDepositData?: TokenDepositData + childChain: ChildNetwork + parentChainProvider: ethers.providers.Provider + childChainProvider: ethers.providers.Provider +}): Promise => { + const ticket = childChainRetryableReport + + const { valueText: baseValueText, valueUsd } = await getCallValueInfo( + ticket, + childChain, + parentChainProvider + ) + let valueText = baseValueText + + if (tokenDepositData?.tokenAmount && tokenDepositData?.l1Token) { + const amount = ethers.utils.formatUnits( + tokenDepositData.tokenAmount, + tokenDepositData.l1Token.decimals + ) + valueText += ` + ${parseFloat(amount).toFixed(2)} ${ + tokenDepositData.l1Token.symbol + }` + } + + const now = Math.floor(Date.now() / 1000) + const hoursLeft = (+ticket.timeoutTimestamp - now) / 3600 + const priority = calculatePriority(valueUsd, hoursLeft) + const state = getSimpleState(ticket.status) + const timeLeft = formatTimeLeft(+ticket.timeoutTimestamp) + const ticketId = ticket.id.slice(-8).toUpperCase() + + const priorityEmoji = { + Critical: '🚨', + High: '🔥', + Medium: '⚠️', + Low: '💡', + }[priority] + + const blocks: any[] = [ + { + type: 'header', + text: { + type: 'plain_text', + text: `${priorityEmoji} Retryable Alert`, + }, + }, + + { + type: 'section', + fields: [ + { + type: 'mrkdwn', + text: `*Ticket*\n\`${ticketId}\`\n\u00A0`, + }, + { + type: 'mrkdwn', + text: `*Priority*\n${priority}\n\u00A0`, + }, + { + type: 'mrkdwn', + text: `*State*\n${state}\n\u00A0`, + }, + { + type: 'mrkdwn', + text: `*Value*\n${valueText}\n\u00A0`, + }, + { + type: 'mrkdwn', + text: `*Network*\n${childChain.name}\n\u00A0`, + }, + { + type: 'mrkdwn', + text: `*Time Left*\n${timeLeft}\n\u00A0`, + }, + ], + }, + ] + + const actionElements: any[] = [] + + if (hoursLeft > 0) { + if (state === 'Pending') { + actionElements.push({ + type: 'button', + action_id: 'redeem_now', + text: { type: 'plain_text', text: 'Redeem now' }, + value: ticketId, + style: 'primary', + }) + } else if (state === 'Failed') { + actionElements.push({ + type: 'button', + action_id: 'retry_redeem', + text: { type: 'plain_text', text: 'Retry redeem' }, + value: ticketId, + style: 'primary', + }) + } + } + + actionElements.push({ + type: 'button', + text: { type: 'plain_text', text: 'Open dashboard' }, + url: `https://retryable-dashboard.arbitrum.io/tx/${ticket.id}`, + action_id: 'open_dashboard', + }) + + const { PARENT_CHAIN_TX_PREFIX, CHILD_CHAIN_TX_PREFIX } = + getExplorerUrlPrefixes(childChain) + + actionElements.push({ + type: 'overflow', + action_id: `overflow_${ticketId}`, + options: [ + { + text: { type: 'plain_text', text: 'Parent transaction' }, + value: `parent_tx_${ticketId}`, + url: `${PARENT_CHAIN_TX_PREFIX}${parentChainRetryableReport.transactionHash}`, + }, + { + text: { type: 'plain_text', text: 'Child transaction' }, + value: `child_tx_${ticketId}`, + url: `${CHILD_CHAIN_TX_PREFIX}${ticket.id}`, + }, + { + text: { type: 'plain_text', text: 'GitHub CI run' }, + value: `github_ci_${ticketId}`, + url: 'https://github.com/OffchainLabs/arbitrum-monitoring/actions', + }, + ], + }) + + if (actionElements.length > 0) { + blocks.push({ + type: 'actions', + elements: actionElements, + }) + } + + return blocks +} + +const calculatePriority = (valueUsd: number, hoursLeft: number): Priority => { + if (valueUsd >= 1000 || hoursLeft < 2) return 'Critical' + if (valueUsd >= 100 || hoursLeft < 24) return 'High' + if (hoursLeft < 72) return 'Medium' + return 'Low' +} + +const getSimpleState = (status: string): string => { + switch (status) { + case ParentToChildMessageStatus[ParentToChildMessageStatus.NOT_YET_CREATED]: + return 'Pending' + case ParentToChildMessageStatus[ + ParentToChildMessageStatus.FUNDS_DEPOSITED_ON_CHILD + ]: + return 'Failed' + case ParentToChildMessageStatus[ParentToChildMessageStatus.EXPIRED]: + return 'Expired' + default: + return 'Unknown' + } +} + +const formatTimeLeft = (timestampInSeconds: number): string => { + const now = Math.floor(Date.now() / 1000) + const diff = timestampInSeconds - now + + if (diff <= 0) return 'Expired' + + const hours = Math.floor(diff / 3600) + const days = Math.floor(hours / 24) + + if (days > 0) return `${days}d ${hours % 24}h` + if (hours > 0) return `${hours}h` + return '<1h' +} diff --git a/packages/retryable-monitor/handlers/slack/slackMessageGenerator.ts b/packages/retryable-monitor/handlers/slack/slackMessageGenerator.ts deleted file mode 100644 index e347ba9..0000000 --- a/packages/retryable-monitor/handlers/slack/slackMessageGenerator.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { providers } from 'ethers' -import { ChildNetwork } from '../../../utils' -import { - ChildChainTicketReport, - ParentChainTicketReport, - TokenDepositData, -} from '../../core/types' -import { - formatPrefix, - formatInitiator, - formatDestination, - formatL1TX, - formatId, - formatL2ExecutionTX, - formatL2Callvalue, - formatTokenDepositData, - formatGasData, - formatCreatedAt, - formatExpiration, -} from './slackMessageFormattingUtils' - -export const generateFailedRetryableSlackMessage = async ({ - parentChainRetryableReport, - childChainRetryableReport, - tokenDepositData, - childChain, - parentChainProvider, - childChainProvider, -}: { - parentChainRetryableReport: ParentChainTicketReport - childChainRetryableReport: ChildChainTicketReport - tokenDepositData?: TokenDepositData - childChain: ChildNetwork - parentChainProvider: providers.Provider - childChainProvider: providers.Provider -}): Promise => { - const t = childChainRetryableReport - const l1Report = parentChainRetryableReport - - // build message to report - return ( - formatPrefix(t, childChain.name) + - (await formatInitiator(tokenDepositData, l1Report, childChain)) + - (await formatDestination(t, childChain)) + - formatL1TX(l1Report, childChain) + - formatId(t, childChain) + - formatL2ExecutionTX(t, childChain) + - (await formatL2Callvalue(t, childChain, parentChainProvider)) + - (await formatTokenDepositData(tokenDepositData)) + - (await formatGasData(t, childChainProvider)) + - formatCreatedAt(t) + - formatExpiration(t) + - '\n=================================================================' - ) -} diff --git a/packages/retryable-monitor/package.json b/packages/retryable-monitor/package.json index 4f4c2ca..0ba0547 100644 --- a/packages/retryable-monitor/package.json +++ b/packages/retryable-monitor/package.json @@ -9,6 +9,7 @@ "@ethersproject/abstract-provider": "^5.5.1", "@notionhq/client": "^2.3.0", "axios": "^1.7.2", + "dotenv": "^16.0.0", "ethers": "^5.5.4", "graphql": "^16.6.0", "graphql-request": "^6.1.0", diff --git a/packages/utils/postSlackMessage.ts b/packages/utils/postSlackMessage.ts index c815ce0..585ce4a 100644 --- a/packages/utils/postSlackMessage.ts +++ b/packages/utils/postSlackMessage.ts @@ -25,3 +25,28 @@ export const postSlackMessage = ({ unfurl_links: false, }) } + +export const postSlackBlocks = ({ + slackToken, + slackChannel, + blocks, + text = 'New notification', +}: { + slackToken: string + slackChannel: string + blocks: any[] + text?: string +}) => { + const web = new WebClient(slackToken) + + console.log( + `>>> Posting blocks to Slack -> ${JSON.stringify(blocks, null, 2)}` + ) + + return web.chat.postMessage({ + text: text, // Fallback text for notifications + channel: slackChannel, + blocks: blocks, + unfurl_links: false, + }) +} diff --git a/yarn.lock b/yarn.lock index 40fb781..7e1173b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1913,6 +1913,11 @@ dotenv@^10.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== +dotenv@^16.0.0: + version "16.5.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.5.0.tgz#092b49f25f808f020050051d1ff258e404c78692" + integrity sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg== + dotenv@^16.3.1: version "16.4.5" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f"