litewitness: add Prometheus metrics.

Patterns heavily riffed from the sunlight codebase.

Standard go and process stats are collected,
and custom stats: these include the number of logs monitored,
how large they are, how long requests are taking to service, etc.

The prefix levels on the metrics might be a little high;
what I'm following here is the sunlight project's distinction
between the command's whole metrics, and the witness component.
Right now, the only metrics I've introduced to litewitness are on
the witness component (and that sounds doubly redundant due to
the project name), but that might not remain the case forever.

I disable the exposure of the metrics endpoint if the obscurity mode flag is engaged.
Some metrics do contain log names, and so it matches the same enumeration
concern that obscurity mode describes as its purpose.

(Possibly future work: I'm not super satisfied with how sometimes
the logging labels and the prometheus labels have parallel evolution,
for example in `processAddCheckpointRequest`.
But attempting to unify those feels like a potential portal to frameworkitis,
so leaving that as a future itch to scratch.  Or, perhaps not.)

Author: warpfork

cmd/litewitness/README.md

@@ -69,13 +69,13 @@ until one connects successfully. If the connection drops after establishing,
 litewitness exits.
 
     -obscurity
-            enable obscurity mode (disable / and /logz endpoints)
+            enable obscurity mode (disable / and /logz and /metrics endpoints)
 
 Note that the c2sp.org/tlog-witness protocol is not designed to keep the
 supported logs or their tree states secret. Moreover, litewitness has no access
-to any secrets (becuase the private key is in ssh-agent) except arguably the IP
+to any secrets (because the private key is in ssh-agent) except arguably the IP
 addresses of its clients (which are always redacted from /logz). Obscurity mode
-disables the `/` and `/logz` endpoints to make it harder to enumerate the logs
+disables the `/` and `/logz` and `/metrics` endpoints to make it harder to enumerate the logs
 known to the witness.
 
 ## witnessctl

cmd/litewitness/litewitness.go

@@ -26,6 +26,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
 	"golang.org/x/net/http2"
@@ -128,11 +131,23 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
+	metricsRegistry := prometheus.NewRegistry()
+	metricsRegistry.MustRegister(collectors.NewGoCollector())
+	metricsRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+	litewitnessMetrics := prometheus.WrapRegistererWithPrefix("litewitness_", metricsRegistry)
+	witnessMetrics := prometheus.WrapRegistererWithPrefix("witness_", litewitnessMetrics)
+	witnessMetrics.MustRegister(w.Metrics()...)
+
 	mux := http.NewServeMux()
 	mux.Handle("/", w)
 	if !*obscurityFlag {
 		mux.Handle("/logz", console)
 		mux.Handle("/{$}", indexHandler(w))
+		mux.Handle("/metrics", promhttp.HandlerFor(metricsRegistry, promhttp.HandlerOpts{
+			ErrorLog: slog.NewLogLogger(slog.Default().Handler().WithAttrs(
+				[]slog.Attr{slog.String("source", "metrics")},
+			), slog.LevelWarn),
+		}))
 	}
 
 	srv := &http.Server{

go.mod

@@ -21,6 +21,7 @@ require (
 	filippo.io/edwards25519 v1.1.0 // indirect
 	filippo.io/hpke v0.4.0 // indirect
 	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -34,7 +35,12 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.17 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/prometheus/client_golang v1.23.2 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/transparency-dev/formats v0.0.0-20251208091212-1378f9e1b1b7 // indirect
@@ -43,11 +49,13 @@ require (
 	go.opentelemetry.io/otel v1.39.0 // indirect
 	go.opentelemetry.io/otel/metric v1.39.0 // indirect
 	go.opentelemetry.io/otel/trace v1.39.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
 	golang.org/x/term v0.38.0 // indirect
 	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/tools v0.39.0 // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	modernc.org/libc v1.65.7 // indirect
 	modernc.org/mathutil v1.7.1 // indirect

go.sum

@@ -10,6 +10,8 @@ filippo.io/mostly-harmless/vrf-r255 v0.0.0-20251110151915-f587ba8b0f82 h1:ZYps1v
 filippo.io/mostly-harmless/vrf-r255 v0.0.0-20251110151915-f587ba8b0f82/go.mod h1:ac5Gah0LmA0/YD4SHdO2M+WUjScWsc99zrAfJK4QViY=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -44,10 +46,20 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.17 h1:78v8ZlW0bP43XfmAfPsdXcoNCelfMHsDmd/pkENfrjQ=
 github.com/mattn/go-runewidth v0.0.17/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -71,6 +83,8 @@ go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF
 go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs=
 go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI=
 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
 golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
@@ -91,6 +105,9 @@ golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
 golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
 golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=

internal/witness/metrics.go

@@ -0,0 +1,80 @@
+package witness
+
+import (
+	"reflect"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type metrics struct {
+	KnownLogs          prometheus.Gauge
+	LogSize            *prometheus.GaugeVec
+	AddCheckpointCount *prometheus.CounterVec
+
+	ReqCount    *prometheus.CounterVec
+	ReqInFlight *prometheus.GaugeVec
+	ReqDuration *prometheus.SummaryVec
+}
+
+func initMetrics() metrics {
+	return metrics{
+		KnownLogs: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "known_logs",
+			Help: "Number of logs known to the witness.",
+		}),
+		LogSize: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "log_tree_size",
+				Help: "Latest cosigned tree size for each log, by origin.",
+				// n.b. yes "tree size" is the term for number of entries, per language in c2sp.org/tlog-checkpoint .
+			},
+			[]string{"origin"},
+		),
+		AddCheckpointCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "add_checkpoint_requests_total",
+				Help: "Total add-checkpoint requests processed, by log origin.",
+			},
+			// error is empty on success;
+			// origin is empty for requests that fail before the origin is parsed (e.g. malformed input);
+			// progress is true if the new checkpoint is for a larger tree, false if same, or empty if earlier errors.
+			[]string{"error", "origin", "progress"},
+		),
+
+		ReqInFlight: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Name: "http_in_flight_requests",
+				Help: "HTTP requests currently being served, by endpoint.",
+			},
+			[]string{"endpoint"},
+		),
+		ReqCount: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "http_requests_total",
+				Help: "HTTP requests served, by endpoint and response code.",
+			},
+			[]string{"endpoint", "code"},
+		),
+		ReqDuration: prometheus.NewSummaryVec(
+			prometheus.SummaryOpts{
+				Name:       "http_request_duration_seconds",
+				Help:       "HTTP request latency in seconds, by endpoint.",
+				Objectives: map[float64]float64{0.5: 0.05, 0.75: 0.025, 0.9: 0.01, 0.99: 0.001},
+				MaxAge:     1 * time.Minute,
+				AgeBuckets: 6,
+			},
+			[]string{"endpoint"},
+		),
+	}
+}
+
+// Metrics returns all Prometheus collectors owned by this Witness.
+// The caller should register them to a [prometheus.Registry].
+func (w *Witness) Metrics() []prometheus.Collector {
+	var collectors []prometheus.Collector
+	for i := 0; i < reflect.ValueOf(w.metrics).NumField(); i++ {
+		collectors = append(collectors, reflect.ValueOf(w.metrics).Field(i).Interface().(prometheus.Collector))
+	}
+	return collectors
+}

internal/witness/witness.go

@@ -15,16 +15,19 @@ import (
 	"sync"
 
 	"filippo.io/torchwood"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/mod/sumdb/note"
 	"golang.org/x/mod/sumdb/tlog"
 	"zombiezen.com/go/sqlite"
 	"zombiezen.com/go/sqlite/sqlitex"
 )
 
 type Witness struct {
-	s   *torchwood.CosignatureSigner
-	mux *http.ServeMux
-	log *slog.Logger
+	s       *torchwood.CosignatureSigner
+	mux     *http.ServeMux
+	log     *slog.Logger
+	metrics metrics
 
 	dmMu sync.Mutex
 	db   *sqlite.Conn
@@ -74,12 +77,24 @@ func NewWitness(dbPath, name string, key crypto.Signer, log *slog.Logger) (*Witn
 	}
 
 	w := &Witness{
-		db:  db,
-		s:   s,
-		log: log,
-		mux: http.NewServeMux(),
+		db:      db,
+		s:       s,
+		log:     log,
+		metrics: initMetrics(),
+		mux:     http.NewServeMux(),
 	}
-	w.mux.Handle("POST /add-checkpoint", http.HandlerFunc(w.serveAddCheckpoint))
+
+	// Give initial values to metrics that are valid on startup
+	if n, err := w.countKnownLogs(); err == nil {
+		w.metrics.KnownLogs.Set(float64(n))
+	}
+
+	labels := prometheus.Labels{"endpoint": "add-checkpoint"}
+	var addCheckpoint http.Handler = http.HandlerFunc(w.serveAddCheckpoint)
+	addCheckpoint = promhttp.InstrumentHandlerCounter(w.metrics.ReqCount.MustCurryWith(labels), addCheckpoint)
+	addCheckpoint = promhttp.InstrumentHandlerDuration(w.metrics.ReqDuration.MustCurryWith(labels), addCheckpoint)
+	addCheckpoint = promhttp.InstrumentHandlerInFlight(w.metrics.ReqInFlight.With(labels), addCheckpoint)
+	w.mux.Handle("POST /add-checkpoint", addCheckpoint)
 	return w, nil
 }
 
@@ -148,6 +163,14 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
 		fmt.Fprintf(rw, "%d\n", err.known)
 		return
 	}
+
+	// Metrics sidequest: ensure known logs counter is synced with reality.
+	// (At the moment of writing, there's a dearth of more convenient time to notice this;
+	// when this number changes, it's due to an external poke of the db.)
+	if n, err := w.countKnownLogs(); err == nil {
+		w.metrics.KnownLogs.Set(float64(n))
+	}
+
 	switch err {
 	case errUnknownLog:
 		http.Error(rw, err.Error(), http.StatusNotFound)
@@ -172,12 +195,15 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
 }
 
 func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosig []byte, err error) {
+	labels := prometheus.Labels{"error": "", "origin": "", "progress": ""}
 	l := w.log.With("request", string(body))
 	defer func() {
 		if err != nil {
 			l = l.With("error", err)
+			labels["error"] = err.Error()
 		}
 		l.Debug("processed add-checkpoint request")
+		w.metrics.AddCheckpointCount.With(labels).Inc()
 	}()
 	body, noteBytes, ok := bytes.Cut(body, []byte("\n\n"))
 	if !ok {
@@ -212,6 +238,7 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi
 		return nil, errBadRequest
 	}
 	l = l.With("origin", origin)
+	labels["origin"] = origin
 	bastions, err := w.getBastions(origin)
 	if err != nil {
 		return nil, err
@@ -237,6 +264,10 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi
 		return nil, err
 	}
 	l = l.With("size", c.N)
+	labels["progress"] = "false"
+	if c.N > oldSize {
+		labels["progress"] = "true"
+	}
 	if err := w.checkConsistency(c.Origin, oldSize, c.N, c.Hash, proof); err != nil {
 		return nil, err
 	}
@@ -308,9 +339,22 @@ func (w *Witness) persistTreeHead(origin string, oldSize, newSize int64, newHash
 		}
 		return &conflictError{knownSize}
 	}
+	if err == nil {
+		w.metrics.LogSize.WithLabelValues(origin).Set(float64(newSize))
+	}
 	return err
 }
 
+func (w *Witness) countKnownLogs() (int, error) {
+	var n int
+	err := w.dbExec("SELECT COUNT(*) AS n FROM log",
+		func(stmt *sqlite.Stmt) error {
+			n = int(stmt.GetInt64("n"))
+			return nil
+		})
+	return n, err
+}
+
 func (w *Witness) getLog(origin string) (treeSize int64, treeHash tlog.Hash, err error) {
 	found := false
 	err = w.dbExec("SELECT tree_size, tree_hash FROM log WHERE origin = ?",