Skip to content

Commit eeaabee

Browse files
committed
Merge remote-tracking branch 'upstream/master'
2 parents 0e0d25b + 6ee80c8 commit eeaabee

14 files changed

Lines changed: 6355 additions & 547 deletions

File tree

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,5 @@ __debug*
2020
__pycache__/
2121
.dev/
2222
.spec/
23-
static/api-docs/swagger-ui/
23+
static/api-docs/swagger-ui/
24+
configs/grafana/grafana.json

AGENTS.md

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,22 @@ the coin config, prints a compact sync/metrics snapshot, downloads the selected
5151
profile, and runs `go tool pprof -top`. Start with CPU for throughput issues and
5252
`--profile goroutine` for deadlock/stall investigations.
5353

54+
## Metrics
55+
56+
Prometheus metrics and the Grafana dashboard share one source of truth, `configs/metrics.yaml`.
57+
58+
- **Add a metric:** add an entry to `configs/metrics.yaml` (stable key + `name`/`type`/`help`;
59+
`labels` for `*_vec`, `buckets` for histograms), then a `Metrics` field in `common/metrics.go` tagged `metric:"<key>"`.
60+
- **Add a panel:** add the viz skeleton (type/`fieldConfig`/`options`, new `id` + a semantic
61+
`x-panel-key`, and an `x-query-key` per target — no `gridPos` or `datasource`) to
62+
`configs/grafana/template.json`, then its `title`/`description`/`queries` under that `x-panel-key`
63+
in `configs/grafana/panels.yaml` (queries keyed by `x-query-key`, each with `promql`/`legend`;
64+
write metric names as `{{name:<key>}}`). Panels pack left-to-right in `template.json` order at 8×8;
65+
set `width`/`height` in the panels.yaml entry to override.
66+
- Prefer stable panel keys like `<section>.<subject>[_stat]` (for example `rpc.request_duration_p95`)
67+
and query keys that name the plotted series (`requests`, `errors`, `p95`, `total`, `threshold`).
68+
- After any of these, run `python3 contrib/scripts/render_grafana.py` (CI gates with `--check`).
69+
5470
## Facts to keep in mind to avoid regressions and waste
5571

5672
- Blockbook instance should be able to :

common/metrics.go

Lines changed: 192 additions & 539 deletions
Large diffs are not rendered by default.

common/metrics_test.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
//go:build unittest
2+
3+
package common
4+
5+
import (
6+
"reflect"
7+
"strings"
8+
"sync"
9+
"testing"
10+
11+
"github.com/prometheus/client_golang/prometheus"
12+
"github.com/trezor/blockbook/configs"
13+
yaml "gopkg.in/yaml.v3"
14+
)
15+
16+
var prometheusRegistryMu sync.Mutex
17+
18+
func useTestPrometheusRegistry(t *testing.T) {
19+
t.Helper()
20+
21+
prometheusRegistryMu.Lock()
22+
oldRegisterer := prometheus.DefaultRegisterer
23+
oldGatherer := prometheus.DefaultGatherer
24+
registry := prometheus.NewRegistry()
25+
prometheus.DefaultRegisterer = registry
26+
prometheus.DefaultGatherer = registry
27+
28+
t.Cleanup(func() {
29+
prometheus.DefaultRegisterer = oldRegisterer
30+
prometheus.DefaultGatherer = oldGatherer
31+
prometheusRegistryMu.Unlock()
32+
})
33+
}
34+
35+
// TestGetMetrics verifies that every field of the Metrics struct is bound to a
36+
// definition in configs/metrics.yaml, of a matching type, and that the resulting
37+
// collectors are all constructed (non-nil) after loading.
38+
func TestGetMetrics(t *testing.T) {
39+
useTestPrometheusRegistry(t)
40+
41+
m, err := GetMetrics("metrics_unittest")
42+
if err != nil {
43+
t.Fatalf("GetMetrics: %v", err)
44+
}
45+
v := reflect.ValueOf(m).Elem()
46+
tp := v.Type()
47+
for i := 0; i < tp.NumField(); i++ {
48+
if v.Field(i).IsNil() {
49+
t.Errorf("field %s was not initialized from configs/metrics.yaml", tp.Field(i).Name)
50+
}
51+
if tag := tp.Field(i).Tag.Get("metric"); tag == "" {
52+
t.Errorf("field %s is missing its `metric` tag", tp.Field(i).Name)
53+
}
54+
}
55+
}
56+
57+
// TestMetricsYAMLInvariants checks the embedded single-source-of-truth file for the
58+
// invariants the loader and the Grafana renderer both rely on: 1:1 correspondence with
59+
// the struct, unique prometheus names, the common prefix, and key/name being distinct
60+
// enough that the stable-key indirection holds (key never carries the prefix).
61+
func TestMetricsYAMLInvariants(t *testing.T) {
62+
var cfg metricsConfig
63+
if err := yaml.Unmarshal(configs.MetricsYAML, &cfg); err != nil {
64+
t.Fatalf("parsing embedded metrics.yaml: %v", err)
65+
}
66+
if cfg.Prefix == "" {
67+
t.Fatal("metrics.yaml: prefix must be set")
68+
}
69+
70+
numFields := reflect.TypeOf(Metrics{}).NumField()
71+
if len(cfg.Metrics) != numFields {
72+
t.Errorf("metrics.yaml has %d entries but Metrics struct has %d fields (must be 1:1)", len(cfg.Metrics), numFields)
73+
}
74+
75+
names := make(map[string]string, len(cfg.Metrics))
76+
for key, def := range cfg.Metrics {
77+
if !strings.HasPrefix(def.Name, cfg.Prefix) {
78+
t.Errorf("metric %q: name %q does not start with prefix %q", key, def.Name, cfg.Prefix)
79+
}
80+
if strings.HasPrefix(key, cfg.Prefix) {
81+
t.Errorf("metric %q: stable key must not carry the %q prefix", key, cfg.Prefix)
82+
}
83+
if prev, dup := names[def.Name]; dup {
84+
t.Errorf("duplicate prometheus name %q (keys %q and %q)", def.Name, prev, key)
85+
}
86+
names[def.Name] = key
87+
}
88+
}

configs/grafana/README.md

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# Metrics & Grafana — single source of truth
2+
3+
Blockbook's prometheus metrics and its Grafana dashboard are generated from a few
4+
source files, so metric names/help and panel queries/descriptions are never hand-synced.
5+
6+
```mermaid
7+
flowchart TD
8+
M["configs/metrics.yaml<br/>name · type · help · labels · buckets"]
9+
T["configs/grafana/template.json<br/>viz skeleton + x-panel-key / x-query-key"]
10+
P["configs/grafana/panels.yaml<br/>per x-panel-key: title · description · queries · width/height"]
11+
G["common.GetMetrics<br/>builds + registers collectors at startup"]
12+
R(["contrib/scripts/render_grafana.py"])
13+
D["configs/grafana/grafana.json<br/>import into Grafana — generated, git-ignored"]
14+
M -->|go:embed| G
15+
M --> R
16+
T --> R
17+
P --> R
18+
R --> D
19+
```
20+
21+
## Files
22+
23+
| file | holds | committed |
24+
|---|---|---|
25+
| `../metrics.yaml` | every metric, keyed by a **stable id**: `name`, `type`, `help`, `labels`, `buckets` | yes |
26+
| `template.json` | dashboard **skeleton** — rows, panel type, `fieldConfig`, `options`, plus a semantic `x-panel-key` per panel and `x-query-key` per target (the join keys). No titles/descriptions/exprs/legends, no `gridPos`, no `datasource`. | yes |
27+
| `panels.yaml` | per-panel **content**, keyed by `x-panel-key` (e.g. `rpc.request_rate`): `title`, `description`, `queries` keyed by `x-query-key` (each with `promql` + `legend`), and optional `width`/`height` (default `8`×`8`; rows fill the row) | yes |
28+
| `grafana.json` | the rendered dashboard you import into Grafana | **no** (git-ignored) |
29+
30+
`render_grafana.py` packs panels into Grafana's 24-column grid from template order and each panel's
31+
`width`/`height` (panels.yaml), so the committed template carries no brittle `x/y` positions; it also
32+
injects the single Prometheus `datasource` onto every panel and target, so the template repeats none.
33+
It joins each `panels.yaml` entry to its template panel by `x-panel-key`, and each `queries:` entry to
34+
a template target by `x-query-key` (Grafana's own `id`/`refId` stay in the template; the x-keys are
35+
stripped from the rendered `grafana.json`). Inside `promql` / `description`, `{{name:<key>}}` /
36+
`{{help:<key>}}` expand from `../metrics.yaml`, so a metric's name lives in one place and a rename
37+
propagates to the Go binary and every panel.
38+
39+
Use stable, descriptive keys: `x-panel-key` should look like `<section>.<subject>[_stat]`
40+
(for example `rpc.request_duration_p95`), and `x-query-key` should name the plotted series
41+
(`requests`, `errors`, `p95`, `total`, `threshold`). Rename titles freely, but keep these keys
42+
stable once other files refer to them.
43+
44+
## Render
45+
46+
```bash
47+
python3 contrib/scripts/render_grafana.py # write configs/grafana/grafana.json
48+
python3 contrib/scripts/render_grafana.py --check # validate alignment only, no write (CI)
49+
```
50+
51+
`--check` fails on an unknown metric key, an invalid `width`/`height`, a `gridPos` or `datasource`
52+
that leaked into `template.json`, a template ↔ `panels.yaml` `x-panel-key` or `x-query-key` mismatch,
53+
a leftover placeholder, or any per-panel title/description/expr/legend that leaked into `template.json`.
54+
55+
> How to add or rename a metric or panel: see the **Metrics** section in `AGENTS.md`.
56+
> The Grafana UI is preview-only — `template.json` + `panels.yaml` are the source.

0 commit comments

Comments
 (0)