feat: add monitor subcommand with Prometheus metrics (#7)

Author: kilianglas

ohttp-probe/README.md

@@ -17,7 +17,7 @@ go build .
 
 ## Commands
 
-The probe has three subcommands. Each has its own `-h` for command-specific
+The probe has four subcommands. Each has its own `-h` for command-specific
 flags. There is no `-mode` flag — relay vs direct-to-gateway is chosen by
 which URL you pass to `-relay-url`.
 
@@ -84,6 +84,48 @@ Error categories in the summary:
 - `decrypt` — content-type mismatches, OHTTP/BHTTP unmarshal, HPKE decapsulate failures
 - `inner HTTP` — successful OHTTP round-trip where the decrypted inner response is non-2xx
 
+### `monitor`
+
+Long-running probe loop intended for in-cluster deployment. Fetches the
+gateway's key config once at startup, then on every `-delay` runs one probe
+per `-target-url` (concurrently, one goroutine per target) and records
+latency and outcome to Prometheus metrics on a side server. Latency
+observation is the outer round-trip only — the startup key fetch is *not*
+included. If keys rotate during the probe's lifetime, probes start failing
+(`transport_err`) the same way real clients with stale keys do; restart the
+pod to refresh the config.
+
+No log-on-failure by default — alerting is consumer-side, driven off the
+counter; `-v` opts into per-iteration stderr lines for local diagnosis.
+
+```sh
+./ohttp-probe monitor \
+  -relay-url https://gateway.example.com/gateway \
+  -keys-url https://gateway.example.com/ohttp-keys \
+  -target-url https://api.example.com/health \
+  -target-url https://other-api.example.com/health \
+  -delay 30s \
+  -metrics-addr :9090
+```
+
+Metrics on `<metrics-addr>/metrics`:
+
+| Metric | Type | Labels |
+|---|---|---|
+| `ohttp_probe_duration_seconds` | Histogram | `target` |
+| `ohttp_probe_requests_total` | Counter | `target`, `outcome` |
+
+`outcome` ∈ `{ok, transport_err, decrypt_err}`. Inner non-2xx (e.g. backend
+`/health` returns 503) is bucketed as `transport_err` — the probe didn't
+return a healthy result. All three outcome series are pre-registered at 0
+so absent series don't surprise alerting on a fresh probe.
+
+`/healthz` on the same address responds 200 for liveness probes.
+
+Cluster-side labels (`env`, `region`, `cluster`) are intentionally omitted
+from the probe metrics — scrape-side relabeling (e.g. Prometheus
+`relabel_configs`) handles those.
+
 ## Shared concepts
 
 Three URL flags appear across the subcommands. The relationship between
@@ -99,7 +141,7 @@ client → POST <relay-url> → gateway → GET <target-url>
 |---|---|
 | `-relay-url` | URL the OHTTP request is POSTed to. Privacy relay, gateway `/gateway` endpoint, or any RFC 9458 server. |
 | `-keys-url` | URL of the gateway's OHTTP key config (e.g. `https://<gateway>/ohttp-keys`). |
-| `-target-url` | Full URL of the inner target the BHTTP request is forwarded to. Repeatable in `probe`; single in `load`. |
+| `-target-url` | Full URL of the inner target the BHTTP request is forwarded to. Repeatable in `probe` and `monitor`; single in `load`. |
 | `-gateway-url` | Echo-only: gateway base URL. Probe POSTs to `<gateway-url>/gateway-echo`; keys are read from `<gateway-url>/ohttp-keys`. Repeatable. |
 
 Inner request method is always `GET` — the tool tests OHTTP setup, not
@@ -114,7 +156,7 @@ arbitrary endpoint behavior.
 3. **Send** — `POST <gateway-url>/gateway-echo` with `Content-Type: message/ohttp-req`
 4. **Decrypt** — the encrypted response is decapsulated and compared to the original payload
 
-### Probe (`probe`) and load (`load`)
+### Probe (`probe`), load (`load`), monitor (`monitor`)
 
 1. **Fetch keys** — `GET <keys-url>` returns the gateway's HPKE public key config
 2. **Build inner request** — a BHTTP-encoded `GET` of `-target-url`
@@ -124,6 +166,6 @@ arbitrary endpoint behavior.
 
 ## Exit codes
 
-- `0` — all probes passed
-- `1` — one or more probes failed
+- `0` — all probes passed (or `monitor` shut down cleanly)
+- `1` — one or more probes failed (or `monitor`'s metrics server failed to start)
 - `2` — flag parse error / missing required argument

ohttp-probe/echo.go

@@ -3,16 +3,16 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"net/http"
 	"os"
 	"time"
 )
 
-// runEcho parses the echo subcommand's flags and probes each -gateway-url
-// in sequence. Returns process exit code: 0 if all probed successfully, 1
-// if any failed, 2 on flag-parse / argument errors.
+// runEcho probes each -gateway-url in sequence. Exits 0 if all succeed,
+// 1 if any fail, 2 on flag/arg errors.
 func runEcho(ctx context.Context, args []string) int {
 	fs := flag.NewFlagSet("echo", flag.ContinueOnError)
 	fs.Usage = func() {
@@ -35,6 +35,9 @@ Flags:
 	verbose := fs.Bool("v", false, "verbose output")
 
 	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
 		return 2
 	}
 	if len(gateways) == 0 {
@@ -62,9 +65,8 @@ Flags:
 	return exit
 }
 
-// probeEcho fetches the gateway's OHTTP key config, encapsulates payload,
-// POSTs to <gateway>/gateway-echo, and asserts the decapsulated response
-// matches.
+// probeEcho POSTs an HPKE-encrypted payload to <gateway>/gateway-echo and
+// asserts the decapsulated response matches the original.
 func probeEcho(ctx context.Context, client *http.Client, gateway string, payload []byte, verbose bool) error {
 	config, err := fetchKeys(ctx, client, joinURL(gateway, pathOHTTPKeys), verbose)
 	if err != nil {

ohttp-probe/go.mod

@@ -5,18 +5,26 @@ go 1.26.1
 require (
 	github.com/chris-wood/ohttp-go v0.0.0-20260205154755-776f22a178b8
 	github.com/cloudflare/circl v1.6.3
+	github.com/prometheus/client_golang v1.20.5
 	github.com/tsenart/vegeta/v12 v12.13.0
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/influxdata/tdigest v0.0.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.42.0 // indirect
 	golang.org/x/text v0.35.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

ohttp-probe/go.sum

@@ -1,6 +1,10 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e h1:mWOqoK5jV13ChKf/aF3plwQ96laasTJgZi4f1aSOu+M=
 github.com/bmizerany/perks v0.0.0-20230307044200-03f9df79da1e/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chris-wood/ohttp-go v0.0.0-20260205154755-776f22a178b8 h1:M3jyHgFxzohWoxsZB1rTlfEN9qG8xMthhfBFP1kZtqo=
 github.com/chris-wood/ohttp-go v0.0.0-20260205154755-776f22a178b8/go.mod h1:P/sVWl8F9KHJ1esPj/g1A5h8vfA3Ps9n6JOMNf6TszU=
 github.com/cloudflare/circl v1.3.3-0.20230418220640-795540340d5c/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
@@ -18,10 +22,24 @@ github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/Z
 github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8=
 github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/streadway/quantile v0.0.0-20220407130108-4246515d968d h1:X4+kt6zM/OVO6gbJdAfJR60MGPsqCzbtXNnjoGqdfAs=
@@ -78,6 +96,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
 gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

ohttp-probe/load.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -15,16 +16,14 @@ import (
 	vegeta "github.com/tsenart/vegeta/v12/lib"
 )
 
-// Error prefixes attached to ohttpRoundTripper failures. The prefixes survive
-// http.Client error wrapping so the summary bucketing can scan
-// vegeta.Result.Error and keep transport vs decrypt failures distinguishable.
+// Error prefixes survive http.Client wrapping so the summary bucketing can
+// scan vegeta.Result.Error to distinguish transport from decrypt failures.
 const (
 	errPrefixTransport = "ohttp-transport: "
 	errPrefixDecrypt   = "ohttp-decrypt: "
 )
 
-// prefixForKind maps a doOHTTPRoundTrip error kind to its vegeta-error prefix.
-// errKindNone returns "" — caller should only invoke this when err != nil.
+// prefixForKind maps an error kind to its vegeta-error prefix.
 func prefixForKind(kind ohttpErrKind) string {
 	switch kind {
 	case errKindTransport:
@@ -51,12 +50,10 @@ type loadSummary struct {
 	Latencies  vegeta.LatencyMetrics
 }
 
-// ohttpRoundTripper turns each outbound inner request into an OHTTP-wrapped
-// POST: marshal to BHTTP, HPKE-encapsulate with the pre-fetched key config,
-// POST the ciphertext to relayURL, and synthesize an *http.Response from the
-// decapsulated inner BHTTP response. Transport and decrypt failures surface
-// as errors with distinct prefixes; inner non-2xx flows through as a normal
-// response so the status code shows up in vegeta.Result.Code.
+// ohttpRoundTripper wraps each outbound request in OHTTP, POSTs to relayURL,
+// and synthesizes an *http.Response from the decapsulated inner BHTTP. Inner
+// non-2xx surfaces in vegeta.Result.Code; transport/decrypt failures get
+// prefixed in the error string.
 type ohttpRoundTripper struct {
 	inner    *http.Client
 	relayURL string
@@ -91,9 +88,8 @@ func (t *ohttpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
 	return synth, nil
 }
 
-// runLoad parses the load subcommand's flags and drives the OHTTP probe path
-// at constant QPS for a fixed duration. Returns process exit code: 0 on
-// success (no failures), 1 if any requests failed, 2 on flag errors.
+// runLoad drives the OHTTP path at constant QPS for a fixed duration.
+// Exits 0 on no failures, 1 if any failed, 2 on flag/arg errors.
 func runLoad(ctx context.Context, args []string) int {
 	fs := flag.NewFlagSet("load", flag.ContinueOnError)
 	fs.Usage = func() {
@@ -119,6 +115,9 @@ Flags:
 	verbose := fs.Bool("v", false, "verbose output")
 
 	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
 		return 2
 	}
 	if *relayURL == "" || *keysURL == "" || *targetURL == "" {
@@ -149,8 +148,8 @@ Flags:
 	return 0
 }
 
-// executeLoad does the actual vegeta run. Split from runLoad so tests can
-// drive the load logic directly without going through flag parsing.
+// executeLoad runs the vegeta attack. Split from runLoad so tests can drive
+// it directly without flag parsing.
 func executeLoad(ctx context.Context, client *http.Client, relayURL, keysURL, targetURL string, qps int, duration time.Duration, verbose bool) error {
 	fmt.Fprintf(os.Stderr, "load: fetching keys from %s\n", keysURL)
 	config, err := fetchKeys(ctx, client, keysURL, verbose)
@@ -231,8 +230,8 @@ func (c *bucketCounts) classify(res *vegeta.Result) {
 	case strings.Contains(res.Error, errPrefixDecrypt):
 		c.decrypt++
 	case res.Error != "":
-		// Shouldn't happen — ohttpRoundTripper always prefixes its errors.
-		// Bucket as transport so unexpected failures aren't silently dropped.
+		// Catch-all: ohttpRoundTripper should always prefix; bucket
+		// unexpected unprefixed errors as transport.
 		c.transport++
 	case res.Code < 200 || res.Code >= 300:
 		c.inner++

ohttp-probe/main.go

@@ -32,6 +32,8 @@ func main() {
 		os.Exit(runProbe(rootContext(), args))
 	case "load":
 		os.Exit(runLoad(rootContext(), args))
+	case "monitor":
+		os.Exit(runMonitor(rootContext(), args))
 	case "-h", "--help", "help":
 		usage()
 		os.Exit(0)
@@ -54,22 +56,20 @@ Commands:
           gateway) against one or more inner target URLs.
   load    Drive the probe path at a fixed QPS for a fixed duration against
           a single target.
+  monitor Long-running probe loop that exposes Prometheus metrics on a
+          side server. Intended for in-cluster deployment.
 
 Run "ohttp-probe <command> -h" for command-specific flags.
 `)
 }
 
-// rootContext returns a context that's cancelled by SIGINT. The cancel is
-// owned by the process; subcommands rely on the context to interrupt
-// in-flight requests on Ctrl-C.
+// rootContext returns a context cancelled by SIGINT.
 func rootContext() context.Context {
 	ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt)
 	return ctx
 }
 
-// urlList is a repeatable string flag that accumulates one URL per
-// occurrence. Used by probe (-target-url), echo (-gateway-url) — anywhere
-// the design takes a list of URLs.
+// urlList is a repeatable string flag, accumulating one URL per occurrence.
 type urlList []string
 
 func (u *urlList) String() string { return strings.Join(*u, ",") }
@@ -108,15 +108,12 @@ func validateURL(raw string) error {
 	return nil
 }
 
-// joinURL appends path to base, taking care of slashes. Used by echo to
-// build /gateway-echo and /ohttp-keys URLs from a gateway base.
+// joinURL appends path to base, normalising slashes.
 func joinURL(base, path string) string {
 	u, _ := url.Parse(base)
 	u = u.JoinPath(path)
 
 	return u.String()
 }
 
-// trimRightSlash strips trailing "/" from base URLs so subsequent path joins
-// produce canonical output.
 func trimRightSlash(s string) string { return strings.TrimRight(s, "/") }