diff --git a/cmd/litewitness/README.md b/cmd/litewitness/README.md index 975acc0..90345bd 100644 --- a/cmd/litewitness/README.md +++ b/cmd/litewitness/README.md @@ -69,13 +69,13 @@ until one connects successfully. If the connection drops after establishing, litewitness exits. -obscurity - enable obscurity mode (disable / and /logz endpoints) + enable obscurity mode (disable / and /logz and /metrics endpoints) Note that the c2sp.org/tlog-witness protocol is not designed to keep the supported logs or their tree states secret. Moreover, litewitness has no access -to any secrets (becuase the private key is in ssh-agent) except arguably the IP +to any secrets (because the private key is in ssh-agent) except arguably the IP addresses of its clients (which are always redacted from /logz). Obscurity mode -disables the `/` and `/logz` endpoints to make it harder to enumerate the logs +disables the `/` and `/logz` and `/metrics` endpoints to make it harder to enumerate the logs known to the witness. ## witnessctl diff --git a/cmd/litewitness/litewitness.go b/cmd/litewitness/litewitness.go index 92ee697..ff05b4f 100644 --- a/cmd/litewitness/litewitness.go +++ b/cmd/litewitness/litewitness.go @@ -26,6 +26,9 @@ import ( "syscall" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" "golang.org/x/net/http2" @@ -128,11 +131,23 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() + metricsRegistry := prometheus.NewRegistry() + metricsRegistry.MustRegister(collectors.NewGoCollector()) + metricsRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + litewitnessMetrics := prometheus.WrapRegistererWithPrefix("litewitness_", metricsRegistry) + witnessMetrics := prometheus.WrapRegistererWithPrefix("witness_", litewitnessMetrics) + witnessMetrics.MustRegister(w.Metrics()...) + mux := http.NewServeMux() mux.Handle("/", w) if !*obscurityFlag { mux.Handle("/logz", console) mux.Handle("/{$}", indexHandler(w)) + mux.Handle("/metrics", promhttp.HandlerFor(metricsRegistry, promhttp.HandlerOpts{ + ErrorLog: slog.NewLogLogger(slog.Default().Handler().WithAttrs( + []slog.Attr{slog.String("source", "metrics")}, + ), slog.LevelWarn), + })) } srv := &http.Server{ diff --git a/go.mod b/go.mod index 452160d..c71437e 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,14 @@ go 1.25.0 require ( filippo.io/age v1.3.1 filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5 + github.com/prometheus/client_golang v1.23.2 github.com/rogpeppe/go-internal v1.15.0 github.com/transparency-dev/tessera v1.0.2 golang.org/x/crypto v0.53.0 golang.org/x/mod v0.37.0 golang.org/x/net v0.55.0 golang.org/x/sync v0.21.0 + golang.org/x/sys v0.46.0 lukechampine.com/blake3 v1.4.1 sigsum.org/sigsum-go v0.14.1 zombiezen.com/go/sqlite v1.4.2 @@ -20,6 +22,7 @@ require ( filippo.io/edwards25519 v1.2.0 // indirect filippo.io/hpke v0.4.0 // indirect filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -30,7 +33,11 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/mattn/go-isatty v0.0.22 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/transparency-dev/formats v0.1.1 // indirect github.com/transparency-dev/merkle v0.0.2 // indirect @@ -38,11 +45,12 @@ require ( go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect - golang.org/x/sys v0.46.0 // indirect golang.org/x/term v0.44.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/tools v0.45.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect k8s.io/klog/v2 v2.140.0 // indirect modernc.org/libc v1.73.0 // indirect modernc.org/mathutil v1.7.1 // indirect diff --git a/go.sum b/go.sum index 9c35dcd..992f69a 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4j filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY= filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5 h1:YEPDyOyfzVugubcTaInzVisW4sdP2tNMRXhn49mU+sQ= filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5/go.mod h1:ac5Gah0LmA0/YD4SHdO2M+WUjScWsc99zrAfJK4QViY= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -31,14 +33,32 @@ github.com/gtank/ristretto255 v0.2.0 h1:LeOuWr6giplWkkMizx2emfG03SRPJqKt1nfIHLVH github.com/gtank/ristretto255 v0.2.0/go.mod h1:OJ1ox/dWcp7sJ5grYDcZ+kkHYuj5nelW5aaL7ESVXBw= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= @@ -59,6 +79,10 @@ go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSY go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= @@ -77,6 +101,11 @@ golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= diff --git a/internal/witness/metrics.go b/internal/witness/metrics.go new file mode 100644 index 0000000..a604afc --- /dev/null +++ b/internal/witness/metrics.go @@ -0,0 +1,80 @@ +package witness + +import ( + "reflect" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +type metrics struct { + KnownLogs prometheus.Gauge + LogSize *prometheus.GaugeVec + AddCheckpointCount *prometheus.CounterVec + + ReqCount *prometheus.CounterVec + ReqInFlight *prometheus.GaugeVec + ReqDuration *prometheus.SummaryVec +} + +func initMetrics() metrics { + return metrics{ + KnownLogs: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "known_logs", + Help: "Number of logs known to the witness.", + }), + LogSize: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "log_tree_size", + Help: "Latest cosigned tree size for each log, by origin.", + // n.b. yes "tree size" is the term for number of entries, per language in c2sp.org/tlog-checkpoint . + }, + []string{"origin"}, + ), + AddCheckpointCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "add_checkpoint_requests_total", + Help: "Total add-checkpoint requests processed, by log origin.", + }, + // error is empty on success; + // origin is empty for requests that fail before the origin is parsed (e.g. malformed input); + // progress is true if the new checkpoint is for a larger tree, false if same, or empty if earlier errors. + []string{"error", "origin", "progress"}, + ), + + ReqInFlight: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "http_in_flight_requests", + Help: "HTTP requests currently being served, by endpoint.", + }, + []string{"endpoint"}, + ), + ReqCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "HTTP requests served, by endpoint and response code.", + }, + []string{"endpoint", "code"}, + ), + ReqDuration: prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "http_request_duration_seconds", + Help: "HTTP request latency in seconds, by endpoint.", + Objectives: map[float64]float64{0.5: 0.05, 0.75: 0.025, 0.9: 0.01, 0.99: 0.001}, + MaxAge: 1 * time.Minute, + AgeBuckets: 6, + }, + []string{"endpoint"}, + ), + } +} + +// Metrics returns all Prometheus collectors owned by this Witness. +// The caller should register them to a [prometheus.Registry]. +func (w *Witness) Metrics() []prometheus.Collector { + var collectors []prometheus.Collector + for i := 0; i < reflect.ValueOf(w.metrics).NumField(); i++ { + collectors = append(collectors, reflect.ValueOf(w.metrics).Field(i).Interface().(prometheus.Collector)) + } + return collectors +} diff --git a/internal/witness/witness.go b/internal/witness/witness.go index f6d9666..e0e8196 100644 --- a/internal/witness/witness.go +++ b/internal/witness/witness.go @@ -15,6 +15,8 @@ import ( "sync" "filippo.io/torchwood" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/mod/sumdb/note" "golang.org/x/mod/sumdb/tlog" "zombiezen.com/go/sqlite" @@ -22,9 +24,10 @@ import ( ) type Witness struct { - s *torchwood.CosignatureSigner - mux *http.ServeMux - log *slog.Logger + s *torchwood.CosignatureSigner + mux *http.ServeMux + log *slog.Logger + metrics metrics dmMu sync.Mutex db *sqlite.Conn @@ -74,12 +77,24 @@ func NewWitness(dbPath, name string, key crypto.Signer, log *slog.Logger) (*Witn } w := &Witness{ - db: db, - s: s, - log: log, - mux: http.NewServeMux(), + db: db, + s: s, + log: log, + metrics: initMetrics(), + mux: http.NewServeMux(), } - w.mux.Handle("POST /add-checkpoint", http.HandlerFunc(w.serveAddCheckpoint)) + + // Give initial values to metrics that are valid on startup + if n, err := w.countKnownLogs(); err == nil { + w.metrics.KnownLogs.Set(float64(n)) + } + + labels := prometheus.Labels{"endpoint": "add-checkpoint"} + var addCheckpoint http.Handler = http.HandlerFunc(w.serveAddCheckpoint) + addCheckpoint = promhttp.InstrumentHandlerCounter(w.metrics.ReqCount.MustCurryWith(labels), addCheckpoint) + addCheckpoint = promhttp.InstrumentHandlerDuration(w.metrics.ReqDuration.MustCurryWith(labels), addCheckpoint) + addCheckpoint = promhttp.InstrumentHandlerInFlight(w.metrics.ReqInFlight.With(labels), addCheckpoint) + w.mux.Handle("POST /add-checkpoint", addCheckpoint) return w, nil } @@ -148,6 +163,14 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) { fmt.Fprintf(rw, "%d\n", err.known) return } + + // Metrics sidequest: ensure known logs counter is synced with reality. + // (At the moment of writing, there's a dearth of more convenient time to notice this; + // when this number changes, it's due to an external poke of the db.) + if n, err := w.countKnownLogs(); err == nil { + w.metrics.KnownLogs.Set(float64(n)) + } + switch err { case errUnknownLog: http.Error(rw, err.Error(), http.StatusNotFound) @@ -172,12 +195,15 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) { } func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosig []byte, err error) { + labels := prometheus.Labels{"error": "", "origin": "", "progress": ""} l := w.log.With("request", string(body)) defer func() { if err != nil { l = l.With("error", err) + labels["error"] = err.Error() } l.Debug("processed add-checkpoint request") + w.metrics.AddCheckpointCount.With(labels).Inc() }() body, noteBytes, ok := bytes.Cut(body, []byte("\n\n")) if !ok { @@ -212,6 +238,7 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi return nil, errBadRequest } l = l.With("origin", origin) + labels["origin"] = origin bastions, err := w.getBastions(origin) if err != nil { return nil, err @@ -237,6 +264,10 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi return nil, err } l = l.With("size", c.N) + labels["progress"] = "false" + if c.N > oldSize { + labels["progress"] = "true" + } if err := w.checkConsistency(c.Origin, oldSize, c.N, c.Hash, proof); err != nil { return nil, err } @@ -308,9 +339,22 @@ func (w *Witness) persistTreeHead(origin string, oldSize, newSize int64, newHash } return &conflictError{knownSize} } + if err == nil { + w.metrics.LogSize.WithLabelValues(origin).Set(float64(newSize)) + } return err } +func (w *Witness) countKnownLogs() (int, error) { + var n int + err := w.dbExec("SELECT COUNT(*) AS n FROM log", + func(stmt *sqlite.Stmt) error { + n = int(stmt.GetInt64("n")) + return nil + }) + return n, err +} + func (w *Witness) getLog(origin string) (treeSize int64, treeHash tlog.Hash, err error) { found := false err = w.dbExec("SELECT tree_size, tree_hash FROM log WHERE origin = ?",