diff --git a/.env.sample b/.env.sample index 8626c01..1168122 100644 --- a/.env.sample +++ b/.env.sample @@ -2,6 +2,7 @@ NODE_ENV= RETRYABLE_MONITORING_SLACK_TOKEN= RETRYABLE_MONITORING_SLACK_CHANNEL= +RETRYABLE_MONITORING_PRIVATE_KEY= BATCH_POSTER_MONITORING_SLACK_TOKEN= BATCH_POSTER_MONITORING_SLACK_CHANNEL= diff --git a/packages/retryable-monitor/core/redeemRetryable.ts b/packages/retryable-monitor/core/redeemRetryable.ts new file mode 100644 index 0000000..c678601 --- /dev/null +++ b/packages/retryable-monitor/core/redeemRetryable.ts @@ -0,0 +1,77 @@ +import { providers, Wallet } from 'ethers' +import { + ParentTransactionReceipt, + ParentToChildMessageStatus, +} from '@arbitrum/sdk' +import { getConfig, DEFAULT_CONFIG_PATH } from '../../utils' +import dotenv from 'dotenv' + +dotenv.config() + +export const redeemRetryable = async (parentTxHash: string): Promise => { + const config = getConfig({ configPath: DEFAULT_CONFIG_PATH }) + + const pk = process.env.RETRYABLE_MONITORING_PRIVATE_KEY + if (!pk) { + throw new Error('RETRYABLE_MONITORING_PRIVATE_KEY env var is required for redeemRetryable') + } + + let lastError: unknown + + for (const childChain of config.childChains) { + try { + // 1) Check parent chain for the tx + const parentChainProvider = new providers.JsonRpcProvider( + childChain.parentRpcUrl + ) + const receipt = await parentChainProvider.getTransactionReceipt( + parentTxHash + ) + if (!receipt) { + // not on this parent chain; try the next one + continue + } + + // 2) We found the parent receipt -> attempt on its configured child + const childChainProvider = new providers.JsonRpcProvider( + childChain.orbitRpcUrl + ) + const wallet = new Wallet(pk, childChainProvider) + + const parentReceipt = new ParentTransactionReceipt(receipt) + const messages = await parentReceipt.getParentToChildMessages(wallet) + + if (!messages || messages.length === 0) { + // no L1->L2 messages associated; try next chain + continue + } + + // If multiple, redeem the first (adjust selection logic if needed) + const message = messages[0] + + // 3) If already redeemed, return its tx hash instead of throwing + const already = await message.getSuccessfulRedeem().catch(() => null) + if (already && already.status === ParentToChildMessageStatus.REDEEMED) { + const existingHash = + (already as any)?.childTxReceipt?.transactionHash ?? + (already as any)?.txHash + if (existingHash) return existingHash + } + + // 4) Otherwise redeem now + const tx = await message.redeem() + const redeemReceipt = await tx.waitForRedeem() + return redeemReceipt.transactionHash + } catch (err) { + // Record and continue probing other configured (parent, child) pairs + lastError = err + continue + } + } + + const suffix = + lastError instanceof Error ? ` Last error: ${lastError.message}` : '' + throw new Error( + `❌ Parent tx ${parentTxHash} not found/redeemable on any configured chain.${suffix}` + ) +} diff --git a/packages/retryable-monitor/core/types.ts b/packages/retryable-monitor/core/types.ts index 34206ef..65c2f79 100644 --- a/packages/retryable-monitor/core/types.ts +++ b/packages/retryable-monitor/core/types.ts @@ -9,6 +9,7 @@ export interface FindRetryablesOptions { configPath: string enableAlerting: boolean writeToNotion: boolean + autoRedeem?: boolean } export interface CheckRetryablesOneOffParams { @@ -77,6 +78,7 @@ export interface OnRetryableFoundParams { l2CallValue: string createdAt?: number decision?: string + botRedemptionStatus?: string } } diff --git a/packages/retryable-monitor/handlers/notion/alertUntriagedRetraybles.ts b/packages/retryable-monitor/handlers/notion/alertUntriagedRetraybles.ts index 760f841..13279cb 100644 --- a/packages/retryable-monitor/handlers/notion/alertUntriagedRetraybles.ts +++ b/packages/retryable-monitor/handlers/notion/alertUntriagedRetraybles.ts @@ -1,6 +1,7 @@ import { notionClient, databaseId } from './createNotionClient' import { postSlackMessage } from '../slack/postSlackMessage' -import { ChildNetwork } from '../../../utils' +import { redeemRetryable } from '../../core/redeemRetryable' +import type { ChildNetwork } from '../../../utils' const formatDate = (iso: string | undefined) => { if (!iso) return '(unknown)' @@ -26,7 +27,8 @@ const isNearExpiry = (iso: string | undefined, hours = 24) => { } export const alertUntriagedNotionRetryables = async ( - childChains: ChildNetwork[] = [] + childChains: ChildNetwork[] = [], + enableAutoRedeem = false // controls >96h silent redemption ) => { const allowedChainIds = childChains.map(c => c.chainId) const response = await notionClient.databases.query({ @@ -82,6 +84,9 @@ export const alertUntriagedNotionRetryables = async ( const expiryTime = timeoutRaw ? new Date(timeoutRaw).getTime() : Infinity const hoursLeft = (expiryTime - now) / (1000 * 60 * 60) + // If a timeout exists and it's already past, skip + if (Number.isFinite(hoursLeft) && hoursLeft < 0) continue + let message = '' if (decision === 'Triage') { @@ -91,12 +96,50 @@ export const alertUntriagedNotionRetryables = async ( message = `⚠️ Retryable ticket needs triage:\n• Retryable: ${retryableUrl}\n• Timeout: ${timeoutStr}\n• Parent Tx: ${parentTx}\n• Total value deposited: ${deposit}\n→ Please review and decide whether to redeem or ignore.` } } else if (decision === 'Should Redeem') { - if (!isNearExpiry(timeoutRaw)) continue - message = `🚨 Retryable marked for redemption and nearing expiry:\n• Retryable: ${retryableUrl}\n• Timeout: ${timeoutStr}\n• Parent Tx: ${parentTx}\n• Total value deposited: ${deposit}\n→ Check why it hasn't been executed.` - } else { - continue + const under24HoursLeftToExpire = timeoutRaw + ? isNearExpiry(timeoutRaw, 24) + : false + const moreThan4DaysLeftToExpire = timeoutRaw ? hoursLeft > 96 : false + + if (under24HoursLeftToExpire) { + // urgent alert path + message = `🚨 Retryable marked for redemption and nearing expiry:\n• Retryable: ${retryableUrl}\n• Timeout: ${timeoutStr}\n• Parent Tx: ${parentTx}\n• Total value deposited: ${deposit}\n→ Check why it hasn't been executed.` + } else if (!enableAutoRedeem && hoursLeft <= 72) { + // NEW: early alert when auto-redeem is disabled + message = `⚠️ Retryable marked for redemption, approaching window (auto-redeem disabled):\n• Retryable: ${retryableUrl}\n• Timeout: ${timeoutStr}\n• Parent Tx: ${parentTx}\n• Total value deposited: ${deposit}\n→ Consider redeeming ahead of time.` + } else if (moreThan4DaysLeftToExpire) { + if (!enableAutoRedeem) continue + try { + await redeemRetryable(parentTx) + await notionClient.pages.update({ + page_id: page.id, + properties: { + 'Bot Redemption Status': { + select: { name: 'Bot Success' }, + }, + }, + }) + } catch { + await notionClient.pages.update({ + page_id: page.id, + properties: { + 'Bot Redemption Status': { + select: { name: 'Bot Failed' }, + }, + }, + }) + } + continue // no Slack message for auto-redeem path + } else { + continue // between 72h–96h (or >96h with auto-redeem off) → no action + } } - await postSlackMessage({ message }) + if (!message) continue + try { + await postSlackMessage({ message }) + } catch { + // swallow Slack errors to keep loop running + } } -} \ No newline at end of file +} diff --git a/packages/retryable-monitor/handlers/notion/syncRetryableToNotion.ts b/packages/retryable-monitor/handlers/notion/syncRetryableToNotion.ts index b0ad9e6..36becce 100644 --- a/packages/retryable-monitor/handlers/notion/syncRetryableToNotion.ts +++ b/packages/retryable-monitor/handlers/notion/syncRetryableToNotion.ts @@ -69,6 +69,12 @@ export async function syncRetryableToNotion( rich_text: [{ text: { content: metadata.tokensDeposited } }], } } + // support verbose column for bot redemption outcome if provided + if (metadata.botRedemptionStatus) { + notionProps['Bot Redemption Status'] = { + select: { name: metadata.botRedemptionStatus }, + } + } } if (isRetryableFoundInNotion) { @@ -112,6 +118,13 @@ export async function syncRetryableToNotion( } } + // carry Bot Redemption Status if present on this update + if (metadata?.botRedemptionStatus) { + executedProps['Bot Redemption Status'] = { + select: { name: metadata.botRedemptionStatus }, + } + } + await notionClient.pages.update({ page_id: page.id, properties: executedProps, @@ -150,6 +163,13 @@ export async function syncRetryableToNotion( ...(metadata?.decision ? { Decision: { select: { name: metadata.decision } } } : {}), + ...(metadata?.botRedemptionStatus + ? { + 'Bot Redemption Status': { + select: { name: metadata.botRedemptionStatus }, + }, + } + : {}), ...notionProps, }, }) diff --git a/packages/retryable-monitor/index.ts b/packages/retryable-monitor/index.ts index ef9cfd9..3932aee 100644 --- a/packages/retryable-monitor/index.ts +++ b/packages/retryable-monitor/index.ts @@ -60,10 +60,18 @@ const options: FindRetryablesOptions = yargs(process.argv.slice(2)) configPath: { type: 'string', default: DEFAULT_CONFIG_PATH }, enableAlerting: { type: 'boolean', default: false }, writeToNotion: { type: 'boolean', default: false }, + autoRedeem: { type: 'boolean', default: false }, }) .strict() .parseSync() as FindRetryablesOptions +if (options.autoRedeem && !options.writeToNotion) { + console.warn( + '[retryable-monitor] --autoRedeem has no effect unless the Notion sweep runs. ' + + 'You can enable it with --writeToNotion.' + ) +} + const config = getConfig({ configPath: options.configPath }) // Function to process a child chain and check for retryable transactions @@ -99,7 +107,10 @@ const processChildChain = async ( if (writeToNotion) { console.log('Activating continuous sweep of Notion database...') setInterval(async () => { - await alertUntriagedNotionRetryables(config.childChains) + await alertUntriagedNotionRetryables( + config.childChains, + options.autoRedeem + ) }, 1000 * 60 * 60) // Run every hour } } else { @@ -177,11 +188,9 @@ const processOrbitChainsConcurrently = async () => { // once we process all the chains go through the Notion database once to alert on any `Unresolved` tickets found if (options.writeToNotion) { - await alertUntriagedNotionRetryables(config.childChains) - + await alertUntriagedNotionRetryables(config.childChains, options.autoRedeem) } } - // Start processing child chains concurrently processOrbitChainsConcurrently()