diff --git a/deploy/grafana/README.md b/deploy/grafana/README.md new file mode 100644 index 0000000..f94c995 --- /dev/null +++ b/deploy/grafana/README.md @@ -0,0 +1,206 @@ +# Grafana Dashboards for Rug Pull Detector + +This directory contains pre-configured Grafana dashboard JSON templates for monitoring the Rug Pull Detector WebSocket server metrics. + +## Dashboards + +### 1. Overview Dashboard (`overview-dashboard.json`) +**UID:** `rug-pull-overview` + +Provides a high-level view of system health including: +- Active WebSocket connections +- Total blocks processed +- Cache hit statistics +- RPC error rates +- Block processing latency +- Actor queue depths + +### 2. RPC Performance Dashboard (`rpc-performance-dashboard.json`) +**UID:** `rug-pull-rpc` + +Detailed RPC node performance metrics: +- RPC error rate gauge +- RPC timeout rate gauge +- RPC latency percentiles (p50, p95, p99) +- RPC success rate by chain +- RPC errors by type +- RPC timeouts by chain + +### 3. Cache Performance Dashboard (`cache-performance-dashboard.json`) +**UID:** `rug-pull-cache` + +Cache performance and efficiency metrics: +- Risk cache hit ratio gauge +- Cache size in bytes +- Cache hit/miss rate +- Cache eviction rate +- Cache hit ratio over time + +### 4. WebSocket & Inference Dashboard (`websocket-inference-dashboard.json`) +**UID:** `rug-pull-websocket-inference` + +WebSocket and model inference metrics: +- Active WebSocket connections +- Message rate (sent/received) +- Subscription rate +- Inference latency (p95) +- WebSocket connection events +- Model inference throughput +- Processing throughput (transactions/events) + +## Installation + +### Prerequisites +- Grafana instance (v8.5+) +- Prometheus instance configured to scrape metrics from the WebSocket server + +### Importing Dashboards + +1. Navigate to your Grafana instance +2. Go to **Dashboards** → **Import** +3. Choose **Upload JSON file** +4. Select the dashboard JSON file from this directory +5. Configure the Prometheus data source if prompted +6. Click **Import** + +### Alternative: Programmatic Import + +You can import dashboards programmatically using the Grafana API: + +```bash +# Import overview dashboard +curl -X POST \ + http://your-grafana:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -d @overview-dashboard.json + +# Import RPC performance dashboard +curl -X POST \ + http://your-grafana:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -d @rpc-performance-dashboard.json + +# Import cache performance dashboard +curl -X POST \ + http://your-grafana:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -d @cache-performance-dashboard.json + +# Import WebSocket & inference dashboard +curl -X POST \ + http://your-grafana:3000/api/dashboards/db \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_KEY" \ + -d @websocket-inference-dashboard.json +``` + +## Metrics Endpoint Configuration + +The WebSocket server exposes Prometheus metrics on a protected endpoint: + +- **Endpoint:** `http://localhost:9090/metrics` +- **Authentication:** Bearer token (optional, configured via `METRICS_API_KEY` environment variable) + +### Prometheus Configuration + +Add the following to your `prometheus.yml`: + +```yaml +scrape_configs: + - job_name: 'rug-pull-detector' + scrape_interval: 15s + metrics_path: '/metrics' + static_configs: + - targets: ['localhost:9090'] + # If authentication is enabled + authorization: + type: Bearer + credentials: YOUR_METRICS_API_KEY +``` + +## Available Metrics + +### RPC Metrics +- `rpc_calls_total` - Total RPC calls by chain, operation, and status +- `rpc_errors_total` - Total RPC errors by chain, operation, and error type +- `rpc_timeouts_total` - Total RPC timeouts by chain and operation +- `rpc_duration_ms` - RPC call duration histogram + +### Cache Metrics +- `cache_operations_total` - Total cache operations by cache, operation, and result +- `cache_hits_total` - Total cache hits by cache +- `cache_misses_total` - Total cache misses by cache +- `cache_size_bytes` - Current cache size in bytes +- `cache_hit_ratio` - Cache hit ratio gauge +- `cache_evictions_total` - Total cache evictions by cache + +### Inference Metrics +- `inference_total` - Total inference operations by model and status +- `inference_duration_ms` - Inference duration histogram by model +- `transactions_processed_total` - Total transactions processed +- `events_processed_total` - Total events processed + +### WebSocket Metrics +- `websocket_active_connections` - Current active WebSocket connections +- `websocket_connections_total` - Total connection events by status and reason +- `websocket_messages_total` - Total messages by direction and type +- `websocket_subscriptions_total` - Total subscription events by action + +### Actor Metrics +- `blocks_processed_total` - Total blocks processed by chain +- `block_processing_duration_ms` - Block processing duration histogram by chain +- `actor_queue_depth` - Current actor queue depth by chain +- `backpressure_events_total` - Total backpressure events by chain +- `actor_timeouts_total` - Total actor timeouts by chain +- `inter_actor_latency_ms` - Inter-actor message latency histogram by chain + +### Database Metrics +- `database_queries_total` - Total database queries by operation and status +- `database_query_duration_ms` - Database query duration histogram by operation +- `database_active_connections` - Active database connections +- `database_idle_connections` - Idle database connections + +### Alert Metrics +- `alerts_generated_total` - Total alerts generated by risk level and type +- `alert_broadcast_duration_ms` - Alert broadcast duration histogram +- `alerts_delivered_total` - Total alerts delivered to subscribers + +## Environment Variables + +Configure the metrics server using the following environment variables: + +- `METRICS_API_KEY` - Optional Bearer token for protecting the `/metrics` endpoint +- `METRICS_PORT` - Port for the metrics server (default: 9090) + +## Customization + +All dashboards are fully editable within Grafana. You can: +- Add additional panels +- Modify queries +- Adjust thresholds and alerts +- Change visualization types +- Add variables for dynamic filtering + +## Alerting + +To set up alerts based on these metrics: + +1. Open a dashboard +2. Click the panel you want to alert on +3. Click the **Alert** icon (bell) +4. Configure alert conditions and notifications +5. Save the alert rule + +Recommended alert thresholds: +- RPC error rate > 5% +- Cache hit ratio < 70% +- WebSocket active connections = 0 (if service should be available) +- Block processing latency p95 > 1000ms +- Actor queue depth > 800 (80% of capacity) + +## Support + +For issues or questions about the metrics implementation, refer to the main project documentation or create an issue in the repository. diff --git a/deploy/grafana/cache-performance-dashboard.json b/deploy/grafana/cache-performance-dashboard.json new file mode 100644 index 0000000..7ac8204 --- /dev/null +++ b/deploy/grafana/cache-performance-dashboard.json @@ -0,0 +1,359 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "cache_hit_ratio{cache=\"risk_cache\"}", + "refId": "A" + } + ], + "title": "Risk Cache Hit Ratio", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "cache_size_bytes{cache=\"risk_cache\"}", + "refId": "A" + } + ], + "title": "Cache Size (Bytes)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "rate(cache_hits_total[5m])", + "legendFormat": "{{cache}}", + "refId": "A" + }, + { + "expr": "rate(cache_misses_total[5m])", + "legendFormat": "{{cache}} (misses)", + "refId": "B" + } + ], + "title": "Cache Hit/Miss Rate", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["sum", "lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "rate(cache_evictions_total[5m])", + "legendFormat": "{{cache}}", + "refId": "A" + } + ], + "title": "Cache Eviction Rate", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "min", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "cache_hit_ratio", + "legendFormat": "{{cache}}", + "refId": "A" + } + ], + "title": "Cache Hit Ratio Over Time", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": ["rug-pull-detector", "cache", "performance"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rug Pull Detector - Cache Performance", + "uid": "rug-pull-cache", + "version": 1 +} diff --git a/deploy/grafana/overview-dashboard.json b/deploy/grafana/overview-dashboard.json new file mode 100644 index 0000000..63cf3f2 --- /dev/null +++ b/deploy/grafana/overview-dashboard.json @@ -0,0 +1,598 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "websocket_active_connections", + "refId": "A" + } + ], + "title": "Active WebSocket Connections", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(blocks_processed_total)", + "refId": "A" + } + ], + "title": "Total Blocks Processed", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(cache_hits_total)", + "refId": "A" + } + ], + "title": "Total Cache Hits", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rpc_errors_total)", + "refId": "A" + } + ], + "title": "Total RPC Errors", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "rate(rpc_calls_total[5m])", + "legendFormat": "{{chain}} - {{operation}} - {{status}}", + "refId": "A" + } + ], + "title": "RPC Call Rate", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(rpc_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} - {{operation}} (p95)", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, rate(rpc_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} - {{operation}} (p50)", + "refId": "B" + } + ], + "title": "RPC Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "cache_hit_ratio", + "legendFormat": "{{cache}}", + "refId": "A" + } + ], + "title": "Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(block_processing_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} (p95)", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, rate(block_processing_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} (p50)", + "refId": "B" + } + ], + "title": "Block Processing Latency", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 9, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "actor_queue_depth", + "legendFormat": "{{chain}}", + "refId": "A" + } + ], + "title": "Actor Queue Depth", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": ["rug-pull-detector", "websocket", "metrics"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rug Pull Detector - Overview", + "uid": "rug-pull-overview", + "version": 1 +} diff --git a/deploy/grafana/rpc-performance-dashboard.json b/deploy/grafana/rpc-performance-dashboard.json new file mode 100644 index 0000000..e1642ed --- /dev/null +++ b/deploy/grafana/rpc-performance-dashboard.json @@ -0,0 +1,440 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(rpc_errors_total[5m])) / sum(rate(rpc_calls_total[5m]))", + "refId": "A" + } + ], + "title": "RPC Error Rate", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(rpc_timeouts_total[5m])) / sum(rate(rpc_calls_total[5m]))", + "refId": "A" + } + ], + "title": "RPC Timeout Rate", + "type": "gauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(rpc_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} - {{operation}} (p99)", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(rpc_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} - {{operation}} (p95)", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, rate(rpc_duration_ms_bucket[5m]))", + "legendFormat": "{{chain}} - {{operation}} (p50)", + "refId": "C" + } + ], + "title": "RPC Latency Distribution", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(rpc_calls_total{status=\"success\"}[5m])) by (chain, operation)", + "legendFormat": "{{chain}} - {{operation}}", + "refId": "A" + } + ], + "title": "RPC Success Rate by Chain", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["sum", "lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(rpc_errors_total[5m])) by (error_type)", + "legendFormat": "{{error_type}}", + "refId": "A" + } + ], + "title": "RPC Errors by Type", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(rpc_timeouts_total[5m])) by (chain, operation)", + "legendFormat": "{{chain}} - {{operation}}", + "refId": "A" + } + ], + "title": "RPC Timeouts by Chain", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": ["rug-pull-detector", "rpc", "performance"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rug Pull Detector - RPC Performance", + "uid": "rug-pull-rpc", + "version": 1 +} diff --git a/deploy/grafana/websocket-inference-dashboard.json b/deploy/grafana/websocket-inference-dashboard.json new file mode 100644 index 0000000..fcecbb9 --- /dev/null +++ b/deploy/grafana/websocket-inference-dashboard.json @@ -0,0 +1,678 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "websocket_active_connections", + "refId": "A" + } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(websocket_messages_total[5m]))", + "refId": "A" + } + ], + "title": "Message Rate", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(websocket_subscriptions_total[5m]))", + "refId": "A" + } + ], + "title": "Subscription Rate", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(inference_duration_ms_bucket[5m]))", + "refId": "A" + } + ], + "title": "Inference Latency (p95)", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "websocket_active_connections", + "legendFormat": "Active Connections", + "refId": "A" + } + ], + "title": "WebSocket Active Connections", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(websocket_messages_total{direction=\"received\"}[5m]))", + "legendFormat": "Received", + "refId": "A" + }, + { + "expr": "sum(rate(websocket_messages_total{direction=\"sent\"}[5m]))", + "legendFormat": "Sent", + "refId": "B" + } + ], + "title": "WebSocket Message Rate", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(inference_duration_ms_bucket[5m]))", + "legendFormat": "{{model}} (p99)", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(inference_duration_ms_bucket[5m]))", + "legendFormat": "{{model}} (p95)", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.50, rate(inference_duration_ms_bucket[5m]))", + "legendFormat": "{{model}} (p50)", + "refId": "C" + } + ], + "title": "Model Inference Latency Distribution", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(inference_total{status=\"success\"}[5m])) by (model)", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "title": "Model Inference Throughput", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 9, + "options": { + "legend": { + "calcs": ["sum", "lastNotNull"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "sum(rate(websocket_connections_total[5m])) by (status)", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "WebSocket Connection Events", + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 10, + "options": { + "legend": { + "calcs": ["mean", "lastNotNull", "max"], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.5.0", + "targets": [ + { + "expr": "rate(transactions_processed_total[5m])", + "legendFormat": "Transactions/sec", + "refId": "A" + }, + { + "expr": "rate(events_processed_total[5m])", + "legendFormat": "Events/sec", + "refId": "B" + } + ], + "title": "Processing Throughput", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 27, + "style": "dark", + "tags": ["rug-pull-detector", "websocket", "inference"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Rug Pull Detector - WebSocket & Inference", + "uid": "rug-pull-websocket-inference", + "version": 1 +} diff --git a/websocket-server/Cargo.toml b/websocket-server/Cargo.toml index 76e5e84..172c769 100644 --- a/websocket-server/Cargo.toml +++ b/websocket-server/Cargo.toml @@ -24,3 +24,8 @@ sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "postgres", "chron dotenvy = "0.15" hex = "0.4" rand = "0.8" +metrics = "0.21" +metrics-exporter-prometheus = "0.12" +axum = "0.7" +tower = "0.4" +tower-http = { version = "0.5", features = ["trace", "cors"] } diff --git a/websocket-server/src/chain_actor.rs b/websocket-server/src/chain_actor.rs index 40d585c..5d6edb4 100644 --- a/websocket-server/src/chain_actor.rs +++ b/websocket-server/src/chain_actor.rs @@ -13,6 +13,7 @@ use std::time::{Duration, Instant}; use tokio::sync::{mpsc, RwLock, Semaphore}; use tokio::time::timeout; use tracing::{debug, error, info, warn, instrument}; +use crate::metrics::actor; /// Maximum number of messages in bounded channels before backpressure kicks in const CHANNEL_CAPACITY: usize = 1000; @@ -256,14 +257,16 @@ impl ChainActor { #[instrument(skip(self, payload))] async fn process_block(&mut self, payload: BlockPayload) { let start_time = Instant::now(); - + let chain = self.chain_id.as_str(); + // Acquire semaphore permit for backpressure handling let _permit = match self.semaphore.clone().acquire_owned().await { Ok(permit) => permit, Err(_) => { - error!("Failed to acquire semaphore for {}", self.chain_id.as_str()); + error!("Failed to acquire semaphore for {}", chain); let mut metrics = self.metrics.write().await; metrics.record_backpressure(); + actor::record_backpressure(chain); return; } }; @@ -276,20 +279,28 @@ impl ChainActor { let processing_time = start_time.elapsed(); let latency_ns = processing_time.as_nanos() as u64; + let latency_ms = processing_time.as_secs_f64() * 1000.0; - // Update metrics + // Update internal metrics { let mut metrics = self.metrics.write().await; match processing_result { Ok(_) => metrics.record_latency(latency_ns), Err(_) => { metrics.record_timeout(); + actor::record_actor_timeout(chain); warn!("Processing timeout for block {} on {}", - payload.block_number, self.chain_id.as_str()); + payload.block_number, chain); } } } + // Update Prometheus metrics + actor::record_block_processing_duration(chain, latency_ms); + actor::record_blocks_processed(chain, 1); + actor::record_inter_actor_latency(chain, latency_ms); + actor::record_queue_depth(chain, self.receiver.len()); + // Update status { let mut status = self.status.write().await; diff --git a/websocket-server/src/lib.rs b/websocket-server/src/lib.rs index e94b324..8c752dc 100644 --- a/websocket-server/src/lib.rs +++ b/websocket-server/src/lib.rs @@ -5,3 +5,5 @@ pub mod evm_disassembler; pub mod chain_actor; pub mod actor_monitor; pub mod block_ingestion; +pub mod metrics; +pub mod metrics_server; diff --git a/websocket-server/src/main.rs b/websocket-server/src/main.rs index 96da70e..4966f69 100644 --- a/websocket-server/src/main.rs +++ b/websocket-server/src/main.rs @@ -5,6 +5,8 @@ use tokio_tungstenite::tungstenite::Message; use tracing::{error, info, warn}; use uuid::Uuid; use tracing_subscriber::prelude::*; +use rug_pull_websocket_server::metrics::{self, rpc, cache, inference, websocket, actor, database, alerts}; +use rug_pull_websocket_server::metrics_server::MetricsServer; mod broadcast; mod subscription; @@ -25,6 +27,19 @@ use rug_pull_websocket_server::block_ingestion::BlockIngestionBridge; #[tokio::main] async fn main() -> Result<()> { + // Initialize Prometheus metrics exporter + let metrics_handle = metrics::init_metrics(); + info!("Prometheus metrics exporter initialized"); + + // Start the protected metrics server + let metrics_server = MetricsServer::new(); + let metrics_handle_clone = metrics_handle.clone(); + tokio::spawn(async move { + if let Err(e) = metrics_server.run(metrics_handle_clone).await { + error!("Metrics server error: {}", e); + } + }); + // Initialize telemetry with tokio-console support let console_layer = console_subscriber::ConsoleLayer::builder() .with_default_env() @@ -121,6 +136,7 @@ async fn main() -> Result<()> { while let Some(msg) = read.next().await { match msg { Ok(Message::Text(text)) => { + websocket::record_message_received("text"); let cpu_pool = cpu_pool.clone(); let parse_res = cpu_pool.spawn(move || { serde_json::from_str::(&text) @@ -144,14 +160,20 @@ async fn main() -> Result<()> { } } Ok(Message::Close(_)) => { + websocket::record_message_received("close"); info!("Client {} sent close frame", client_id); + websocket::record_disconnection("close_frame"); break; } Err(e) => { + websocket::record_message_received("error"); error!("Error receiving message: {}", e); + websocket::record_disconnection("error"); break; } - _ => {} + _ => { + websocket::record_message_received("other"); + } } } }; @@ -163,7 +185,9 @@ async fn main() -> Result<()> { if subscription_manager.is_subscribed(client_id, &alert.address).await { let server_msg = ServerMessage::Alert(alert.clone()); if let Ok(json) = serde_json::to_string(&server_msg) { + websocket::record_message_sent("alert"); if write.send(Message::Text(json)).await.is_err() { + websocket::record_disconnection("send_error"); break; } } diff --git a/websocket-server/src/metrics.rs b/websocket-server/src/metrics.rs new file mode 100644 index 0000000..cc5f159 --- /dev/null +++ b/websocket-server/src/metrics.rs @@ -0,0 +1,252 @@ +//! Prometheus metrics instrumentation for the Rug Pull Detector WebSocket server +//! +//! This module provides comprehensive metrics for: +//! - RPC node response latencies +//! - Cache hit ratios +//! - Model inference computation durations +//! - Active WebSocket connections +//! - Error rates + +use metrics::{counter, gauge, histogram}; +use metrics_exporter_prometheus::{PrometheusBuilder, PrometheusHandle}; +use std::time::Duration; + +/// Initialize the Prometheus metrics exporter +pub fn init_metrics() -> PrometheusHandle { + PrometheusBuilder::new() + .with_http_listener(([0, 0, 0, 0], 9090)) + .install() + .expect("Failed to install Prometheus exporter") +} + +/// Metrics for RPC operations +pub mod rpc { + use metrics::{counter, histogram}; + + /// Record a successful RPC call + pub fn record_rpc_success(chain: &str, operation: &str, duration_ms: f64) { + histogram!("rpc_duration_ms", duration_ms, "chain" => chain, "operation" => operation); + counter!("rpc_calls_total", 1, "chain" => chain, "operation" => operation, "status" => "success"); + } + + /// Record a failed RPC call + pub fn record_rpc_error(chain: &str, operation: &str, error_type: &str) { + counter!("rpc_calls_total", 1, "chain" => chain, "operation" => operation, "status" => "error", "error_type" => error_type); + counter!("rpc_errors_total", 1, "chain" => chain, "operation" => operation, "error_type" => error_type); + } + + /// Record RPC timeout + pub fn record_rpc_timeout(chain: &str, operation: &str) { + counter!("rpc_calls_total", 1, "chain" => chain, "operation" => operation, "status" => "timeout"); + counter!("rpc_timeouts_total", 1, "chain" => chain, "operation" => operation); + } +} + +/// Metrics for cache operations +pub mod cache { + use metrics::{counter, gauge, histogram}; + + /// Record a cache hit + pub fn record_cache_hit(cache_name: &str, operation: &str) { + counter!("cache_operations_total", 1, "cache" => cache_name, "operation" => operation, "result" => "hit"); + counter!("cache_hits_total", 1, "cache" => cache_name); + } + + /// Record a cache miss + pub fn record_cache_miss(cache_name: &str, operation: &str) { + counter!("cache_operations_total", 1, "cache" => cache_name, "operation" => operation, "result" => "miss"); + counter!("cache_misses_total", 1, "cache" => cache_name); + } + + /// Record cache entry size + pub fn record_cache_size(cache_name: &str, size: u64) { + gauge!("cache_size_bytes", size as f64, "cache" => cache_name); + } + + /// Record cache eviction + pub fn record_cache_eviction(cache_name: &str) { + counter!("cache_evictions_total", 1, "cache" => cache_name); + } + + /// Calculate and record cache hit ratio + pub fn record_cache_hit_ratio(cache_name: &str, ratio: f64) { + gauge!("cache_hit_ratio", ratio, "cache" => cache_name); + } +} + +/// Metrics for model inference +pub mod inference { + use metrics::{counter, histogram}; + + /// Record model inference duration + pub fn record_inference_duration(model_name: &str, duration_ms: f64) { + histogram!("inference_duration_ms", duration_ms, "model" => model_name); + } + + /// Record model inference success + pub fn record_inference_success(model_name: &str) { + counter!("inference_total", 1, "model" => model_name, "status" => "success"); + } + + /// Record model inference error + pub fn record_inference_error(model_name: &str, error_type: &str) { + counter!("inference_total", 1, "model" => model_name, "status" => "error", "error_type" => error_type); + } + + /// Record number of transactions processed + pub fn record_transactions_processed(count: u64) { + counter!("transactions_processed_total", count); + } + + /// Record number of events processed + pub fn record_events_processed(count: u64) { + counter!("events_processed_total", count); + } +} + +/// Metrics for WebSocket connections +pub mod websocket { + use metrics::{counter, gauge}; + + /// Increment active WebSocket connections + pub fn increment_active_connections() { + gauge!("websocket_active_connections", |val| val + 1.0); + } + + /// Decrement active WebSocket connections + pub fn decrement_active_connections() { + gauge!("websocket_active_connections", |val| (val - 1.0).max(0.0)); + } + + /// Record new WebSocket connection + pub fn record_connection() { + counter!("websocket_connections_total", 1, "status" => "established"); + } + + /// Record WebSocket disconnection + pub fn record_disconnection(reason: &str) { + counter!("websocket_connections_total", 1, "status" => "disconnected", "reason" => reason); + } + + /// Record WebSocket message received + pub fn record_message_received(message_type: &str) { + counter!("websocket_messages_total", 1, "direction" => "received", "type" => message_type); + } + + /// Record WebSocket message sent + pub fn record_message_sent(message_type: &str) { + counter!("websocket_messages_total", 1, "direction" => "sent", "type" => message_type); + } + + /// Record subscription + pub fn record_subscription() { + counter!("websocket_subscriptions_total", 1, "action" => "subscribe"); + } + + /// Record unsubscription + pub fn record_unsubscription() { + counter!("websocket_subscriptions_total", 1, "action" => "unsubscribe"); + } +} + +/// Metrics for actor system +pub mod actor { + use metrics::{counter, gauge, histogram}; + + /// Record block processing duration + pub fn record_block_processing_duration(chain: &str, duration_ms: f64) { + histogram!("block_processing_duration_ms", duration_ms, "chain" => chain); + } + + /// Record blocks processed + pub fn record_blocks_processed(chain: &str, count: u64) { + counter!("blocks_processed_total", count, "chain" => chain); + } + + /// Record actor queue depth + pub fn record_queue_depth(chain: &str, depth: usize) { + gauge!("actor_queue_depth", depth as f64, "chain" => chain); + } + + /// Record backpressure event + pub fn record_backpressure(chain: &str) { + counter!("backpressure_events_total", 1, "chain" => chain); + } + + /// Record actor timeout + pub fn record_actor_timeout(chain: &str) { + counter!("actor_timeouts_total", 1, "chain" => chain); + } + + /// Record inter-actor message latency + pub fn record_inter_actor_latency(chain: &str, latency_ms: f64) { + histogram!("inter_actor_latency_ms", latency_ms, "chain" => chain); + } +} + +/// Metrics for database operations +pub mod database { + use metrics::{counter, histogram}; + + /// Record database query duration + pub fn record_query_duration(operation: &str, duration_ms: f64) { + histogram!("database_query_duration_ms", duration_ms, "operation" => operation); + } + + /// Record database query success + pub fn record_query_success(operation: &str) { + counter!("database_queries_total", 1, "operation" => operation, "status" => "success"); + } + + /// Record database query error + pub fn record_query_error(operation: &str, error_type: &str) { + counter!("database_queries_total", 1, "operation" => operation, "status" => "error", "error_type" => error_type); + } + + /// Record active database connections + pub fn record_active_connections(count: u64) { + gauge!("database_active_connections", count as f64); + } + + /// Record database connection pool idle connections + pub fn record_idle_connections(count: u64) { + gauge!("database_idle_connections", count as f64); + } +} + +/// Metrics for alerts +pub mod alerts { + use metrics::{counter, histogram}; + + /// Record alert generated + pub fn record_alert_generated(risk_level: &str, alert_type: &str) { + counter!("alerts_generated_total", 1, "risk_level" => risk_level, "alert_type" => alert_type); + } + + /// Record alert broadcast duration + pub fn record_alert_broadcast_duration(duration_ms: f64) { + histogram!("alert_broadcast_duration_ms", duration_ms); + } + + /// Record alerts delivered to subscribers + pub fn record_alerts_delivered(count: u64) { + counter!("alerts_delivered_total", count); + } +} + +/// Helper trait for timing operations +pub trait TimedExt { + fn timed(self, operation: F) -> R + where + F: FnOnce(Duration) -> R; +} + +impl TimedExt for std::time::Instant { + fn timed(self, operation: F) -> R + where + F: FnOnce(Duration) -> R, + { + let duration = self.elapsed(); + operation(duration) + } +} diff --git a/websocket-server/src/metrics_server.rs b/websocket-server/src/metrics_server.rs new file mode 100644 index 0000000..efc51ac --- /dev/null +++ b/websocket-server/src/metrics_server.rs @@ -0,0 +1,207 @@ +//! Protected HTTP server for Prometheus metrics endpoint +//! +//! This module provides a protected /metrics endpoint using Axum with +//! authentication support to ensure metrics are only accessible by authorized systems. + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::get, + Router, +}; +use metrics_exporter_prometheus::PrometheusHandle; +use std::net::SocketAddr; +use tower_http::trace::TraceLayer; +use tracing::{info, warn}; + +/// Metrics server configuration +pub struct MetricsServerConfig { + /// Address to bind the metrics server to + pub bind_address: SocketAddr, + /// Optional API key for authentication (if None, no auth required) + pub api_key: Option, +} + +impl Default for MetricsServerConfig { + fn default() -> Self { + Self { + bind_address: SocketAddr::from(([0, 0, 0, 0], 9090)), + api_key: std::env::var("METRICS_API_KEY").ok(), + } + } +} + +/// Create the metrics router with optional authentication +fn create_metrics_router(handle: PrometheusHandle, api_key: Option) -> Router { + Router::new() + .route("/metrics", get(metrics_handler)) + .layer(TraceLayer::new_for_http()) + .with_state((handle, api_key)) +} + +/// Handler for the /metrics endpoint with optional authentication +async fn metrics_handler( + State((handle, api_key)): State<(PrometheusHandle, Option)>, + headers: HeaderMap, +) -> Result { + // Check authentication if API key is configured + if let Some(expected_key) = api_key { + let auth_header = headers + .get("authorization") + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.strip_prefix("Bearer ")); + + match auth_header { + Some(key) if key == expected_key => { + // Authentication successful + } + _ => { + warn!("Unauthorized metrics access attempt"); + return Err(StatusCode::UNAUTHORIZED); + } + } + } + + // Render and return Prometheus metrics + let metrics = handle.render(); + Ok(metrics.into_response()) +} + +/// Metrics server that runs the HTTP server +pub struct MetricsServer { + config: MetricsServerConfig, +} + +impl MetricsServer { + /// Create a new metrics server with default configuration + pub fn new() -> Self { + Self { + config: MetricsServerConfig::default(), + } + } + + /// Create a new metrics server with custom configuration + pub fn with_config(config: MetricsServerConfig) -> Self { + Self { config } + } + + /// Start the metrics server + pub async fn run(self, handle: PrometheusHandle) -> anyhow::Result<()> { + let app = create_metrics_router(handle, self.config.api_key.clone()); + + info!( + "Starting metrics server on {} (authentication: {})", + self.config.bind_address, + if self.config.api_key.is_some() { + "enabled" + } else { + "disabled" + } + ); + + let listener = tokio::net::TcpListener::bind(self.config.bind_address).await?; + axum::serve(listener, app).await?; + + Ok(()) + } +} + +impl Default for MetricsServer { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use tower::ServiceExt; + + #[tokio::test] + async fn test_metrics_handler_no_auth() { + let handle = PrometheusBuilder::new() + .install() + .expect("Failed to install Prometheus exporter"); + + let app = create_metrics_router(handle, None); + + let response = app + .oneshot( + axum::http::Request::builder() + .uri("/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_metrics_handler_with_auth_success() { + let handle = PrometheusBuilder::new() + .install() + .expect("Failed to install Prometheus exporter"); + + let app = create_metrics_router(handle, Some("test-key".to_string())); + + let response = app + .oneshot( + axum::http::Request::builder() + .uri("/metrics") + .header("authorization", "Bearer test-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_metrics_handler_with_auth_failure() { + let handle = PrometheusBuilder::new() + .install() + .expect("Failed to install Prometheus exporter"); + + let app = create_metrics_router(handle, Some("test-key".to_string())); + + let response = app + .oneshot( + axum::http::Request::builder() + .uri("/metrics") + .header("authorization", "Bearer wrong-key") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_metrics_handler_without_auth_header() { + let handle = PrometheusBuilder::new() + .install() + .expect("Failed to install Prometheus exporter"); + + let app = create_metrics_router(handle, Some("test-key".to_string())); + + let response = app + .oneshot( + axum::http::Request::builder() + .uri("/metrics") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } +} diff --git a/websocket-server/src/subscription.rs b/websocket-server/src/subscription.rs index 02e569f..937a4fb 100644 --- a/websocket-server/src/subscription.rs +++ b/websocket-server/src/subscription.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tokio::sync::RwLock; use uuid::Uuid; +use crate::metrics::websocket; #[derive(Clone)] pub struct SubscriptionManager { @@ -18,11 +19,15 @@ impl SubscriptionManager { pub async fn add_client(&self, client_id: Uuid) { let mut clients = self.clients.write().await; clients.insert(client_id, HashSet::new()); + websocket::increment_active_connections(); + websocket::record_connection(); } pub async fn remove_client(&self, client_id: Uuid) { let mut clients = self.clients.write().await; clients.remove(&client_id); + websocket::decrement_active_connections(); + websocket::record_disconnection("client_removed"); } pub async fn subscribe(&self, client_id: Uuid, address: &str) { @@ -30,6 +35,7 @@ impl SubscriptionManager { if let Some(subscriptions) = clients.get_mut(&client_id) { subscriptions.insert(address.to_lowercase()); } + websocket::record_subscription(); } pub async fn unsubscribe(&self, client_id: Uuid, address: &str) { @@ -37,6 +43,7 @@ impl SubscriptionManager { if let Some(subscriptions) = clients.get_mut(&client_id) { subscriptions.remove(&address.to_lowercase()); } + websocket::record_unsubscription(); } pub async fn is_subscribed(&self, client_id: Uuid, address: &str) -> bool {