feat: alerts

alerts/.dockerignore

@@ -0,0 +1,106 @@
+# Dependencies
+node_modules
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage
+.grunt
+
+# Bower dependency directory
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons
+build/Release
+
+# Dependency directories
+jspm_packages/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# Environment files
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# Runtime logs
+logs
+*.log
+
+# Build outputs (already copied from builder stage)
+dist
+
+# Development files
+src/**/*.test.ts
+src/**/*.spec.ts
+**/*.test.js
+**/*.spec.js
+
+# IDE files
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# OS files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Git
+.git
+.gitignore
+.gitattributes
+
+# Docker
+Dockerfile
+docker-compose.yml
+docker-compose.*.yml
+.dockerignore
+
+# Documentation
+README.md
+*.md
+docs/
+
+# Alert state data (will be mounted as volume)
+data/
+
+# Monitoring config
+monitoring/

alerts/.env.example

@@ -4,10 +4,38 @@ SLACK_TOKEN=
 # Slack Channel to send a message
 SLACK_CHANNEL=
 
+# self-contained slack webhook, doesn't required SLACK_TOKEN & SLACK_CHANNEL if SLACK_WEBHOOK_URL is created
+SLACK_WEBHOOK_URL=
+
 # Providers URLs
 MAINNET_RPC_URL=https://rpc.ankr.com/eth
 GNOSIS_RPC_URL=https://rpc.ankr.com/gnosis
 
 # Subgraph URLs
-SUBGRAPH_API_NATIVE=https://api.thegraph.com/subgraphs/name/laimejesus/bridge-monitor-native
-SUBGRAPH_API_FOREIGN=https://api.thegraph.com/subgraphs/name/laimejesus/bridge-monitor-foreign
+SUBGRAPH_API_NATIVE=
+SUBGRAPH_API_FOREIGN=
+SUBGRAPH_API_KEY=
+
+## Validator balance
+# true= only track validator balance on Gnosis Chain
+IS_VALIDATOR_BALANCE_ON_GC=true
+MIN_XDAI_BALANCE_THRESHOLD=1 # 1xdai
+MIN_ETH_BALANCE_THRESHOLD=0 # set to 0 because the validator doesn't require balance to claim on Ethereum
+
+## Inactive validator 
+# max hrs to consider a validator inactive (not calling on chain function)
+INACTIVITY_THRESHOLD_HOURS=12
+
+## Stucked tx
+# max hrs after first initiated a tx but still at 'COLLECTING' status
+TRANSACTION_TIMEOUT_HOURS=1
+
+# Scheduler Configuration
+SCHEDULE_ENABLED=true
+SCHEDULE_CRON=*/15 * * * *
+RUN_ONCE_ON_START=true
+
+# Alert State Management
+ALERT_STATE_FILE=./data/stuck-tx-alerts.json
+ALERT_CLEANUP_HOURS=48
+NODE_ENV=PROD # or DEV

alerts/.gitignore

@@ -10,5 +10,10 @@
 # custom logs
 /logs
 
+# alert state data
+/data
+
 #types
 src/types/
+
+/dist

alerts/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

alerts/Dockerfile

@@ -0,0 +1,42 @@
+# Production stage
+FROM node:18-alpine
+
+# Install dumb-init for proper signal handling
+RUN apk add --no-cache dumb-init
+
+# Create app user
+RUN addgroup -g 1001 -S nodejs && \
+    adduser -S alerts -u 1001
+
+# Set working directory
+WORKDIR /app
+
+# Copy package files
+COPY package.json ./
+COPY yarn.lock ./
+COPY .yarnrc.yml ./
+
+# Install dependencies
+RUN yarn install && \
+    yarn cache clean
+
+# Copy source code
+COPY . .
+
+# Generate types
+RUN yarn typechain
+RUN yarn subgraph --verbose || true
+RUN yarn build
+
+# Create data directory for alert state
+RUN mkdir -p /app/data && \
+    chown -R alerts:nodejs /app
+
+# Switch to non-root user
+USER alerts
+
+# Use dumb-init to handle signals properly
+ENTRYPOINT ["dumb-init", "--"]
+
+# Default command
+CMD ["yarn", "start"]

alerts/README.md

@@ -6,7 +6,22 @@ The alerts that will be sent are:
 
 - Low Balance (XDAI or ETH) for validators
 - Low Limits for XDAI Native and Foreign Bridge Contracts (dailyLimit and executionDailyLimit)
-- ....
+- Invalid validator: When validator is not signing transactions for threshold amount of time
+- Stucked Transaction: When a transaction is initiated but still in 'Collecting' status for threshold amount of time
+
+# Dev
+
+```shell
+cp .env.example .env
+yarn install && yarn typechain & yarn subgraph & yarn build
+yarn dev # or yarn start
+```
+
+# Run docker
+
+```shell
+docker compose up --build
+```
 
 ### Validators
 
@@ -19,6 +34,7 @@ A limit is low when it is under 25%? of the total limit per day. (TBD)
 ## Configuration
 
 Remember to configure Slack App:
+
 - create slack app
 - add slack into workspace
 - create scope permissions for slack

alerts/docker-compose.yml

@@ -0,0 +1,30 @@
+version: "3.8"
+
+services:
+  alerts:
+    build:
+      context: .
+      args:
+        - SUBGRAPH_API_KEY=${SUBGRAPH_API_KEY}
+    environment:
+      - SLACK_TOKEN=${SLACK_TOKEN}
+      - SLACK_CHANNEL=${SLACK_CHANNEL}
+      - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
+      - MAINNET_RPC_URL=${MAINNET_RPC_URL:-https://rpc.ankr.com/eth}
+      - GNOSIS_RPC_URL=${GNOSIS_RPC_URL:-https://rpc.ankr.com/gnosis}
+      - SUBGRAPH_API_NATIVE=${SUBGRAPH_API_NATIVE}
+      - SUBGRAPH_API_FOREIGN=${SUBGRAPH_API_FOREIGN}
+      - SUBGRAPH_API_KEY=${SUBGRAPH_API_KEY}
+      - IS_VALIDATOR_BALANCE_ON_GC=${IS_VALIDATOR_BALANCE_ON_GC:-true}
+      - INACTIVITY_THRESHOLD_HOURS=${INACTIVITY_THRESHOLD_HOURS:-2}
+      - TRANSACTION_TIMEOUT_HOURS=${TRANSACTION_TIMEOUT_HOURS:-2}
+      - SCHEDULE_ENABLED=${SCHEDULE_ENABLED:-true}
+      - SCHEDULE_CRON=${SCHEDULE_CRON}
+      - RUN_ONCE_ON_START=${RUN_ONCE_ON_START}
+      - ALERT_STATE_FILE=${ALERT_STATE_FILE:-./data/stuck-tx-alerts.json}
+      - ALERT_CLEANUP_HOURS=${ALERT_CLEANUP_HOURS:-48}
+    env_file:
+      - .env
+    volumes:
+      - ./data:/app/data
+    restart: unless-stopped

alerts/package.json

@@ -9,12 +9,15 @@
   ],
   "scripts": {
     "dev": "ts-node src/index.ts",
+    "dev:schedule": "SCHEDULE_ENABLED=true RUN_ONCE_ON_START=true ts-node src/index.ts",
+    "dev:once": "SCHEDULE_ENABLED=false ts-node src/index.ts",
     "start": "node ./dist/index.js",
-    "build": "tsc --build --clean",
+    "start:schedule": "SCHEDULE_ENABLED=true node ./dist/index.js",
+    "start:once": "SCHEDULE_ENABLED=false node ./dist/index.js",
+    "build": "tsc",
     "typechain": "typechain --target=ethers-v5 ./src/abis/**/*.json --out-dir ./src/types/typechain",
-    "subgraph": "graphql-codegen --config subgraph-config.js",
+    "subgraph": "graphql-codegen --config ./subgraph-config.js",
     "types": "tsc --project tsconfig.json",
-    "postinstall": "yarn typechain && yarn subgraph",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "keywords": [],
@@ -30,7 +33,8 @@
     "ethers": "^5.7.1",
     "graphql": "^16.6.0",
     "graphql-request": "^5.0.0",
-    "graphql-tag": "^2.12.6"
+    "graphql-tag": "^2.12.6",
+    "node-cron": "^3.0.3"
   },
   "devDependencies": {
     "@graphql-codegen/cli": "^2.13.6",
@@ -40,6 +44,8 @@
     "@graphql-codegen/typescript-operations": "^2.5.4",
     "@typechain/ethers-v5": "^10.1.0",
     "@types/node": "^18.8.3",
+    "@types/node-cron": "^3.0.11",
+    "@types/node-fetch": "^2.6.13",
     "ts-node": "^10.9.1",
     "typechain": "^8.1.0",
     "typescript": "^4.8.4",

alerts/src/alerts/inactiveValidators.ts

@@ -0,0 +1,65 @@
+import { title } from 'process';
+import { fetchValidators, Validator } from '../validators';
+import { Message, MessageType } from "./messages"
+
+const INACTIVITY_THRESHOLD_HOURS = parseInt(process.env.INACTIVITY_THRESHOLD_HOURS) || 0.01;
+
+
+const createInactiveValidatorMessage = (validator: Validator, inactiveSince: string, lastActivityUTC: string) =>{
+  return{
+    title: `Inactive validator alert: ${validator.name} on ${validator.bridgeType}`,
+    type: MessageType.INACTIVE_VALIDATOR,
+    createdBy: validator.name,
+    createdByLink: `https://gnosisscan.io/address/${validator.address}`,
+    timestamp: new Date(),
+    body: `Inactive since: ${inactiveSince}, last activity in ${lastActivityUTC}`
+
+  }
+}
+
+export const checkInactiveValidators = async (): Promise<Message[] | null> => {
+  console.log('Checking for inactive validators...');
+  try {
+    const allValidators = await fetchValidators();
+    const now = Math.floor(Date.now() / 1000);
+    const inactivityCutoff = now - INACTIVITY_THRESHOLD_HOURS * 60 * 60;
+
+    const inactiveValidators = allValidators.filter(validator => {
+      if (!validator.lastActivity) {
+        return true; // Consider validators never seen as inactive
+      }
+      return parseInt(validator.lastActivity, 10) < inactivityCutoff;
+    });
+
+    if (inactiveValidators.length > 0) {
+      console.log(`Found ${inactiveValidators.length} inactive validators.`);
+
+        const message = inactiveValidators.map(( validator: Validator) => {
+
+        // Calculate inactivity duration
+        const diffInSeconds = now - parseInt(validator.lastActivity, 10);
+        const diffInHours = Math.round(diffInSeconds / 3600);
+        const hourText = diffInHours === 1 ? 'hr' : 'hrs';
+        const inactiveSince = `${diffInHours} ${hourText} ago`;
+        const lastActivityUTC = new Date(parseInt(validator.lastActivity, 10) * 1000).toUTCString()
+
+
+        return createInactiveValidatorMessage(validator,inactiveSince, lastActivityUTC)
+
+      });
+      return message
+
+
+
+    } else {
+      console.log('All validators are active.');
+    }
+  } catch (e) {
+    if (e instanceof Error) {
+      console.error('Could not check for inactive validators due to an error:', e.message);
+    } else {
+      console.error('An unknown error occurred while checking for inactive validators.');
+    }
+  }
+  return null;
+};