diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c9f1abc --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# VeriNode-Backend developer tasks. +# +# The `localnet` target brings up a one-command local development environment +# so new contributors don't have to wire Postgres, the OTel collector, +# Prometheus and Grafana by hand. See deploy/localnet/README.md. + +COMPOSE := docker compose -f deploy/localnet/docker-compose.yml +LOCALNET_DB_ENV := DB_HOST=localhost DB_PORT=$(or $(LOCALNET_PG_PORT),5432) DB_USER=verinode DB_PASSWORD=verinode DB_NAME=verinode + +.DEFAULT_GOAL := help + +.PHONY: help +help: ## Show this help + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' + +.PHONY: localnet +localnet: ## Boot the full local stack (API + Postgres + OTel + Prometheus + Grafana) and seed data + $(COMPOSE) up -d --build --wait + $(MAKE) localnet-seed + $(MAKE) localnet-mock + @echo "" + @echo "localnet is up:" + @echo " API http://localhost:3000 (/metrics for Prometheus scrape)" + @echo " Prometheus http://localhost:9090" + @echo " Grafana http://localhost:3001 (anonymous viewer; login admin/admin)" + @echo " Postgres localhost:5432 (verinode / verinode)" + +.PHONY: localnet-seed +localnet-seed: ## Seed validators / stakes / reputations / pending rewards (idempotent) + $(LOCALNET_DB_ENV) npx ts-node scripts/seed-localnet.ts + +.PHONY: localnet-mock +localnet-mock: ## Generate mock heartbeat + reward telemetry for the seeded validators + $(LOCALNET_DB_ENV) npx ts-node scripts/mock-telemetry.ts + +.PHONY: localnet-logs +localnet-logs: ## Tail all localnet service logs + $(COMPOSE) logs -f + +.PHONY: localnet-clean +localnet-clean: ## Tear down the stack and remove all state (volumes) + $(COMPOSE) down -v --remove-orphans diff --git a/deploy/localnet/README.md b/deploy/localnet/README.md new file mode 100644 index 0000000..0d6bab9 --- /dev/null +++ b/deploy/localnet/README.md @@ -0,0 +1,53 @@ +# VeriNode localnet + +A one-command local development environment for the VeriNode backend, so a new +contributor can go from a fresh clone to a running, populated, observable stack +in minutes instead of days. + +```bash +make localnet # build + boot everything, then seed data +make localnet-clean # tear it all down and remove state +``` + +## What comes up + +| Service | URL / port | Purpose | +|---|---|---| +| API | http://localhost:3000 | The backend (`/metrics` for Prometheus) | +| Postgres (TimescaleDB) | localhost:5432 | `verinode` / `verinode`; migrations auto-applied | +| OTel collector | localhost:4317 (OTLP gRPC) | Receives traces from the API | +| Prometheus | http://localhost:9090 | Scrapes the API `/metrics` + collector | +| Grafana | http://localhost:3001 | Pre-provisioned Prometheus datasource + the repo's dashboards | + +Migrations under `src/database/migrations/` are mounted into the Postgres +container's `docker-entrypoint-initdb.d`, so they run in order on first boot +(the image is `timescale/timescaledb-ha` because `002_uptime_schema.sql` +requires TimescaleDB + pg_cron). Grafana loads the existing dashboards from +`deploy/observability/` and `deploy/monitoring/` via file provisioning. + +## Seed data + +Run automatically by `make localnet` (and re-runnable on their own): + +- `make localnet-seed` — provisions a bond pool and **8 validators**, each with + a stake, a reputation score, and a pending reward balance. +- `make localnet-mock` — emits a window of `uptime_heartbeat` rows and a few + `reward_tx` rows per validator, so the time-series panels have live data. + +## A note on scope + +The original issue was framed as a blockchain **validator testnet** (genesis +file, pre-funded accounts, mock attestations). This repo is the ROSCA +savings-circle API backend, which has no genesis/accounts primitives — so, per +the maintainer's confirmation on the issue, those steps are mapped to their +real equivalents here: the **seed script** provisions validator identities with +stakes/reputations/rewards (the "pre-funded accounts" analogue), and the +**mock-telemetry generator** produces heartbeat/reward data (the "mock +attestation" analogue). Everything is grounded in tables that actually exist +(`bond_pools`, `validator_stakes`, `reputations`, `reward_pending_amounts`, +`reward_tx`, `uptime_heartbeat`). + +## Requirements + +- Docker + Docker Compose v2 (`up --wait` support) +- Node.js (for the seed/mock scripts, run on the host via `ts-node`) diff --git a/deploy/localnet/docker-compose.yml b/deploy/localnet/docker-compose.yml new file mode 100644 index 0000000..6c4d7a6 --- /dev/null +++ b/deploy/localnet/docker-compose.yml @@ -0,0 +1,102 @@ +# VeriNode local development stack ("localnet"). +# +# One command (`make localnet`) brings up the full backend with its +# dependencies and monitoring, so a new contributor does not have to wire +# Postgres, the OTel collector, Prometheus and Grafana by hand. +# +# NOTE on scope: this repo is the ROSCA (savings-circle) API backend, not an +# L1 validator client. "Validators" here are the node/validator identities the +# backend tracks (bond_pools, validator_stakes, reputations); the seed script +# provisions 8 of them with stakes + balances (the "pre-funded accounts" +# equivalent), and the mock generator emits heartbeat/reward telemetry (the +# "mock attestation" equivalent). See README.md in this directory. + +name: verinode-localnet + +services: + postgres: + # timescaledb-ha bundles TimescaleDB + pg_cron, both required by the + # uptime_heartbeat migration (002_uptime_schema.sql). pg_cron must be + # preloaded (and pointed at the app database) BEFORE initdb runs the + # migrations, or CREATE EXTENSION pg_cron aborts the init. + image: timescale/timescaledb-ha:pg16 + command: + - -c + - shared_preload_libraries=timescaledb,pg_cron + - -c + - cron.database_name=verinode + environment: + POSTGRES_USER: verinode + POSTGRES_PASSWORD: verinode + POSTGRES_DB: verinode + ports: + # Override with LOCALNET_PG_PORT if 5432 is taken on the host. + - "${LOCALNET_PG_PORT:-5432}:5432" + volumes: + - pgdata:/var/lib/postgresql/data + # Migrations are pure, idempotent SQL — run them on first init, in + # numeric order, before anything connects. + - ../../src/database/migrations:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U verinode -d verinode"] + interval: 3s + timeout: 5s + retries: 20 + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.109.0 + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "8889:8889" # Prometheus exporter (collector's own metrics) + + api: + build: + context: ../.. + dockerfile: Dockerfile + environment: + NODE_ENV: development + PORT: "3000" + DB_HOST: postgres + DB_PORT: "5432" + DB_USER: verinode + DB_PASSWORD: verinode + DB_NAME: verinode + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + OTEL_SERVICE_NAME: verinode-api + ports: + - "3000:3000" + depends_on: + postgres: + condition: service_healthy + otel-collector: + condition: service_started + + prometheus: + image: prom/prometheus:v2.54.1 + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - "9090:9090" + depends_on: + - api + + grafana: + image: grafana/grafana:11.2.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + ports: + - "3001:3000" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + +volumes: + pgdata: diff --git a/deploy/localnet/grafana/dashboards/config-management-dashboard.json b/deploy/localnet/grafana/dashboards/config-management-dashboard.json new file mode 100644 index 0000000..31752c6 --- /dev/null +++ b/deploy/localnet/grafana/dashboards/config-management-dashboard.json @@ -0,0 +1,8 @@ +{ + "title": "VeriNode Configuration Management", + "panels": [ + { "title": "Config reload successes", "expr": "sum(rate(verinode_config_reload_total[5m]))" }, + { "title": "Config reload failures", "expr": "sum(rate(verinode_config_reload_failed_total[5m]))" }, + { "title": "Config reload P99", "expr": "histogram_quantile(0.99, sum(rate(verinode_config_reload_duration_ms_bucket[5m])) by (le))" } + ] +} diff --git a/deploy/localnet/grafana/dashboards/structured-logging-dashboard.json b/deploy/localnet/grafana/dashboards/structured-logging-dashboard.json new file mode 100644 index 0000000..1493b19 --- /dev/null +++ b/deploy/localnet/grafana/dashboards/structured-logging-dashboard.json @@ -0,0 +1,9 @@ +{ + "title": "VeriNode Structured Logging", + "schemaVersion": 39, + "panels": [ + { "type": "timeseries", "title": "Log records by severity", "targets": [{ "expr": "sum(rate(log_records_total{service_name=\"verinode-backend\"}[5m])) by (severity_text)" }] }, + { "type": "timeseries", "title": "Log ingestion failures", "targets": [{ "expr": "rate(otelcol_receiver_refused_log_records[5m])" }] }, + { "type": "timeseries", "title": "HTTP server P99 latency", "targets": [{ "expr": "histogram_quantile(0.99, sum(rate(http_server_duration_seconds_bucket{service_name=\"verinode-backend\"}[5m])) by (le))" }] } + ] +} diff --git a/deploy/localnet/grafana/provisioning/dashboards/dashboards.yml b/deploy/localnet/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..f97207c --- /dev/null +++ b/deploy/localnet/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: VeriNode localnet dashboards + orgId: 1 + folder: VeriNode + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/deploy/localnet/grafana/provisioning/datasources/datasource.yml b/deploy/localnet/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..0b304bc --- /dev/null +++ b/deploy/localnet/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/deploy/localnet/otel-collector-config.yaml b/deploy/localnet/otel-collector-config.yaml new file mode 100644 index 0000000..0126d9f --- /dev/null +++ b/deploy/localnet/otel-collector-config.yaml @@ -0,0 +1,30 @@ +# Minimal OpenTelemetry collector for local development. +# Receives OTLP/gRPC traces from the API (OTEL_EXPORTER_OTLP_ENDPOINT points +# here), logs them to the collector's stdout for easy inspection, and exposes +# its own metrics for Prometheus to scrape. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: {} + +exporters: + debug: + verbosity: normal + prometheus: + endpoint: 0.0.0.0:8889 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] diff --git a/deploy/localnet/prometheus.yml b/deploy/localnet/prometheus.yml new file mode 100644 index 0000000..e3a48c7 --- /dev/null +++ b/deploy/localnet/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 10s + evaluation_interval: 10s + +scrape_configs: + # The VeriNode API exposes Prometheus text-format metrics at GET /metrics + # (see index.js and src/api/metrics/prometheus.ts). + - job_name: verinode-api + metrics_path: /metrics + static_configs: + - targets: ["api:3000"] + + # The OTel collector's own Prometheus exporter. + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8889"] + + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/scripts/mock-telemetry.ts b/scripts/mock-telemetry.ts new file mode 100644 index 0000000..7aeea85 --- /dev/null +++ b/scripts/mock-telemetry.ts @@ -0,0 +1,79 @@ +/** + * Generate mock telemetry for the seeded validators. + * + * This is the ROSCA-backend equivalent of a "mock attestation generator": it + * emits a window of uptime_heartbeat rows (the TimescaleDB hypertable the node + * monitoring is built on) plus a few reward_tx rows per validator, so Grafana + * panels and API endpoints have live-looking time-series data to show. + * + * Idempotent-ish: re-running appends a fresh window of heartbeats. + * + * Usage: npx ts-node scripts/mock-telemetry.ts (DB_* env, defaults to localnet) + */ +import { Client } from 'pg'; +import { validatorIds } from './seed-localnet'; + +const VALIDATOR_COUNT = Number(process.env.LOCALNET_VALIDATORS ?? 8); +const HEARTBEAT_MINUTES = Number(process.env.LOCALNET_HEARTBEAT_MINUTES ?? 60); + +function dbConfig() { + return { + host: process.env.DB_HOST ?? 'localhost', + port: Number(process.env.DB_PORT ?? 5432), + user: process.env.DB_USER ?? 'verinode', + password: process.env.DB_PASSWORD ?? 'verinode', + database: process.env.DB_NAME ?? 'verinode', + }; +} + +async function main(): Promise { + const client = new Client(dbConfig()); + await client.connect(); + try { + const validators = validatorIds(VALIDATOR_COUNT); + const now = Date.now(); + let heartbeats = 0; + let rewards = 0; + + for (const [idx, v] of validators.entries()) { + // One heartbeat per minute over the window. + for (let m = HEARTBEAT_MINUTES; m >= 0; m--) { + const ts = new Date(now - m * 60_000).toISOString(); + // Deterministic-ish variation: mostly up, occasional degraded. + const degraded = (m + idx) % 17 === 0; + const latency = 20 + ((m * 7 + idx * 3) % 40) + (degraded ? 120 : 0); + const status = degraded ? 'degraded' : 'up'; + const uptimePct = degraded ? 98.5 : 100.0; + const blockHeight = 1_000_000 + (HEARTBEAT_MINUTES - m) * 12 + idx; + await client.query( + `INSERT INTO uptime_heartbeat + (time, node_id, latency_ms, status, uptime_pct, block_height) + VALUES ($1, $2, $3, $4, $5, $6)`, + [ts, v, latency, status, uptimePct, blockHeight], + ); + heartbeats++; + } + + // A couple of reward transactions per validator (references reward_pending_amounts). + for (let r = 0; r < 3; r++) { + await client.query( + `INSERT INTO reward_tx (node_id, amount) VALUES ($1, $2)`, + [v, (0.5 + r * 0.25).toFixed(7)], + ); + rewards++; + } + } + + console.log( + `Generated ${heartbeats} heartbeat rows and ${rewards} reward_tx rows ` + + `across ${VALIDATOR_COUNT} validators (${HEARTBEAT_MINUTES}m window).`, + ); + } finally { + await client.end(); + } +} + +main().catch((err) => { + console.error('[mock-telemetry] failed:', err); + process.exit(1); +}); diff --git a/scripts/seed-localnet.ts b/scripts/seed-localnet.ts new file mode 100644 index 0000000..6f91297 --- /dev/null +++ b/scripts/seed-localnet.ts @@ -0,0 +1,77 @@ +/** + * Seed the localnet database with a set of validator identities. + * + * This is the ROSCA-backend equivalent of a testnet "genesis file with + * pre-funded accounts": it provisions a bond pool and N validators, each with + * a stake (validator_stakes), a reputation score (reputations), and a pending + * reward balance (reward_pending_amounts) — so a fresh checkout has realistic + * data to develop and demo against. + * + * Idempotent: safe to run repeatedly (ON CONFLICT upserts). + * + * Usage: npx ts-node scripts/seed-localnet.ts (DB_* env, defaults to localnet) + */ +import { Client } from 'pg'; + +const VALIDATOR_COUNT = Number(process.env.LOCALNET_VALIDATORS ?? 8); +const POOL_ID = 'localnet-pool'; +const STAKE_PER_VALIDATOR = 1_000_000; + +function dbConfig() { + return { + host: process.env.DB_HOST ?? 'localhost', + port: Number(process.env.DB_PORT ?? 5432), + user: process.env.DB_USER ?? 'verinode', + password: process.env.DB_PASSWORD ?? 'verinode', + database: process.env.DB_NAME ?? 'verinode', + }; +} + +export function validatorIds(count: number): string[] { + return Array.from({ length: count }, (_, i) => `validator-${String(i + 1).padStart(2, '0')}`); +} + +async function main(): Promise { + const client = new Client(dbConfig()); + await client.connect(); + try { + const validators = validatorIds(VALIDATOR_COUNT); + + await client.query( + `INSERT INTO bond_pools (id, balance) VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE SET balance = EXCLUDED.balance`, + [POOL_ID, STAKE_PER_VALIDATOR * VALIDATOR_COUNT], + ); + + for (const [idx, v] of validators.entries()) { + await client.query( + `INSERT INTO validator_stakes (pool_id, validator_id, amount) VALUES ($1, $2, $3) + ON CONFLICT (pool_id, validator_id) DO UPDATE SET amount = EXCLUDED.amount`, + [POOL_ID, v, STAKE_PER_VALIDATOR], + ); + // Varied but valid reputation scores (schema bounds: -1000..1000). + await client.query( + `INSERT INTO reputations (node_id, score) VALUES ($1, $2) + ON CONFLICT (node_id) DO UPDATE SET score = EXCLUDED.score, updated_at = NOW()`, + [v, 500 + idx * 40], + ); + await client.query( + `INSERT INTO reward_pending_amounts (node_id, amount) VALUES ($1, $2) + ON CONFLICT (node_id) DO UPDATE SET amount = EXCLUDED.amount`, + [v, (10 + idx).toFixed(7)], + ); + } + + console.log( + `Seeded ${VALIDATOR_COUNT} validators into bond pool "${POOL_ID}" ` + + `(stake ${STAKE_PER_VALIDATOR} each) with reputations and pending rewards.`, + ); + } finally { + await client.end(); + } +} + +main().catch((err) => { + console.error('[seed-localnet] failed:', err); + process.exit(1); +}); diff --git a/src/database/migrations/011_distributed_jobs.sql b/src/database/migrations/011_distributed_jobs.sql index 2777279..6e826c9 100644 --- a/src/database/migrations/011_distributed_jobs.sql +++ b/src/database/migrations/011_distributed_jobs.sql @@ -18,12 +18,16 @@ CREATE TABLE IF NOT EXISTS distributed_jobs ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); --- Critical partial index for < 100ms P99 lease acquisition --- Only indexes jobs that are ready for execution +-- Critical partial index for < 100ms P99 lease acquisition. +-- NOTE: the predicate must be IMMUTABLE, so the lock-expiry condition +-- (locked_until <= NOW()) cannot live here — Postgres rejects NOW() in index +-- predicates because index membership is fixed at write time while NOW() +-- changes continuously. The static status filter below still excludes the +-- bulk of the table (completed/failed jobs); lease queries apply the +-- time-dependent locked_until check at query time and can use this index. CREATE INDEX IF NOT EXISTS idx_distributed_jobs_ready ON distributed_jobs (run_at, job_type) - WHERE status IN ('pending', 'running') - AND (locked_until IS NULL OR locked_until <= NOW()); + WHERE status IN ('pending', 'running'); -- Index for monitoring queries CREATE INDEX IF NOT EXISTS idx_distributed_jobs_status