Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/litewitness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,13 @@ until one connects successfully. If the connection drops after establishing,
litewitness exits.

-obscurity
enable obscurity mode (disable / and /logz endpoints)
enable obscurity mode (disable / and /logz and /metrics endpoints)

Note that the c2sp.org/tlog-witness protocol is not designed to keep the
supported logs or their tree states secret. Moreover, litewitness has no access
to any secrets (becuase the private key is in ssh-agent) except arguably the IP
to any secrets (because the private key is in ssh-agent) except arguably the IP
addresses of its clients (which are always redacted from /logz). Obscurity mode
disables the `/` and `/logz` endpoints to make it harder to enumerate the logs
disables the `/` and `/logz` and `/metrics` endpoints to make it harder to enumerate the logs
known to the witness.

## witnessctl
Expand Down
15 changes: 15 additions & 0 deletions cmd/litewitness/litewitness.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import (
"syscall"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/net/http2"
Expand Down Expand Up @@ -128,11 +131,23 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()

metricsRegistry := prometheus.NewRegistry()
metricsRegistry.MustRegister(collectors.NewGoCollector())
metricsRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
litewitnessMetrics := prometheus.WrapRegistererWithPrefix("litewitness_", metricsRegistry)
witnessMetrics := prometheus.WrapRegistererWithPrefix("witness_", litewitnessMetrics)
witnessMetrics.MustRegister(w.Metrics()...)

mux := http.NewServeMux()
mux.Handle("/", w)
if !*obscurityFlag {
mux.Handle("/logz", console)
mux.Handle("/{$}", indexHandler(w))
mux.Handle("/metrics", promhttp.HandlerFor(metricsRegistry, promhttp.HandlerOpts{
ErrorLog: slog.NewLogLogger(slog.Default().Handler().WithAttrs(
[]slog.Attr{slog.String("source", "metrics")},
), slog.LevelWarn),
}))
}

srv := &http.Server{
Expand Down
10 changes: 9 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.25.0
require (
filippo.io/age v1.3.1
filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5
github.com/prometheus/client_golang v1.23.2
github.com/rogpeppe/go-internal v1.15.0
github.com/transparency-dev/tessera v1.0.2
golang.org/x/crypto v0.53.0
golang.org/x/mod v0.37.0
golang.org/x/net v0.55.0
golang.org/x/sync v0.21.0
golang.org/x/sys v0.46.0
lukechampine.com/blake3 v1.4.1
sigsum.org/sigsum-go v0.14.1
zombiezen.com/go/sqlite v1.4.2
Expand All @@ -20,6 +22,7 @@ require (
filippo.io/edwards25519 v1.2.0 // indirect
filippo.io/hpke v0.4.0 // indirect
filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
Expand All @@ -30,19 +33,24 @@ require (
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/mattn/go-isatty v0.0.22 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/transparency-dev/formats v0.1.1 // indirect
github.com/transparency-dev/merkle v0.0.2 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.44.0 // indirect
go.opentelemetry.io/otel/metric v1.44.0 // indirect
go.opentelemetry.io/otel/trace v1.44.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/term v0.44.0 // indirect
golang.org/x/text v0.38.0 // indirect
golang.org/x/tools v0.45.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
k8s.io/klog/v2 v2.140.0 // indirect
modernc.org/libc v1.73.0 // indirect
modernc.org/mathutil v1.7.1 // indirect
Expand Down
29 changes: 29 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e h1:VsUbObBMxXlc23Eb9VeeJYE4j
filippo.io/mldsa v0.0.0-20260215214346-43d0283efc3e/go.mod h1:32qQ5yj3R24Eu03iWFWchdC3OB653wPvoepWejkefbY=
filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5 h1:YEPDyOyfzVugubcTaInzVisW4sdP2tNMRXhn49mU+sQ=
filippo.io/mostly-harmless/vrf-r255 v0.0.0-20260605095812-d3620d7874c5/go.mod h1:ac5Gah0LmA0/YD4SHdO2M+WUjScWsc99zrAfJK4QViY=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
Expand All @@ -31,14 +33,32 @@ github.com/gtank/ristretto255 v0.2.0 h1:LeOuWr6giplWkkMizx2emfG03SRPJqKt1nfIHLVH
github.com/gtank/ristretto255 v0.2.0/go.mod h1:OJ1ox/dWcp7sJ5grYDcZ+kkHYuj5nelW5aaL7ESVXBw=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4=
github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
Expand All @@ -59,6 +79,10 @@ go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSY
go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q=
Expand All @@ -77,6 +101,11 @@ golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
Expand Down
80 changes: 80 additions & 0 deletions internal/witness/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package witness

import (
"reflect"
"time"

"github.com/prometheus/client_golang/prometheus"
)

type metrics struct {
KnownLogs prometheus.Gauge
LogSize *prometheus.GaugeVec
AddCheckpointCount *prometheus.CounterVec

ReqCount *prometheus.CounterVec
ReqInFlight *prometheus.GaugeVec
ReqDuration *prometheus.SummaryVec
}

func initMetrics() metrics {
return metrics{
KnownLogs: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "known_logs",
Help: "Number of logs known to the witness.",
}),
LogSize: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "log_tree_size",
Help: "Latest cosigned tree size for each log, by origin.",
// n.b. yes "tree size" is the term for number of entries, per language in c2sp.org/tlog-checkpoint .
},
[]string{"origin"},
),
AddCheckpointCount: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "add_checkpoint_requests_total",
Help: "Total add-checkpoint requests processed, by log origin.",
},
// error is empty on success;
// origin is empty for requests that fail before the origin is parsed (e.g. malformed input);
// progress is true if the new checkpoint is for a larger tree, false if same, or empty if earlier errors.
[]string{"error", "origin", "progress"},
),

ReqInFlight: prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "http_in_flight_requests",
Help: "HTTP requests currently being served, by endpoint.",
},
[]string{"endpoint"},
),
ReqCount: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "http_requests_total",
Help: "HTTP requests served, by endpoint and response code.",
},
[]string{"endpoint", "code"},
),
ReqDuration: prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "http_request_duration_seconds",
Help: "HTTP request latency in seconds, by endpoint.",
Objectives: map[float64]float64{0.5: 0.05, 0.75: 0.025, 0.9: 0.01, 0.99: 0.001},
MaxAge: 1 * time.Minute,
AgeBuckets: 6,
},
[]string{"endpoint"},
),
}
}

// Metrics returns all Prometheus collectors owned by this Witness.
// The caller should register them to a [prometheus.Registry].
func (w *Witness) Metrics() []prometheus.Collector {
var collectors []prometheus.Collector
for i := 0; i < reflect.ValueOf(w.metrics).NumField(); i++ {
collectors = append(collectors, reflect.ValueOf(w.metrics).Field(i).Interface().(prometheus.Collector))
}
return collectors
}
60 changes: 52 additions & 8 deletions internal/witness/witness.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,19 @@ import (
"sync"

"filippo.io/torchwood"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/mod/sumdb/note"
"golang.org/x/mod/sumdb/tlog"
"zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitex"
)

type Witness struct {
s *torchwood.CosignatureSigner
mux *http.ServeMux
log *slog.Logger
s *torchwood.CosignatureSigner
mux *http.ServeMux
log *slog.Logger
metrics metrics

dmMu sync.Mutex
db *sqlite.Conn
Expand Down Expand Up @@ -74,12 +77,24 @@ func NewWitness(dbPath, name string, key crypto.Signer, log *slog.Logger) (*Witn
}

w := &Witness{
db: db,
s: s,
log: log,
mux: http.NewServeMux(),
db: db,
s: s,
log: log,
metrics: initMetrics(),
mux: http.NewServeMux(),
}
w.mux.Handle("POST /add-checkpoint", http.HandlerFunc(w.serveAddCheckpoint))

// Give initial values to metrics that are valid on startup
if n, err := w.countKnownLogs(); err == nil {
w.metrics.KnownLogs.Set(float64(n))
}

labels := prometheus.Labels{"endpoint": "add-checkpoint"}
var addCheckpoint http.Handler = http.HandlerFunc(w.serveAddCheckpoint)
addCheckpoint = promhttp.InstrumentHandlerCounter(w.metrics.ReqCount.MustCurryWith(labels), addCheckpoint)
addCheckpoint = promhttp.InstrumentHandlerDuration(w.metrics.ReqDuration.MustCurryWith(labels), addCheckpoint)
addCheckpoint = promhttp.InstrumentHandlerInFlight(w.metrics.ReqInFlight.With(labels), addCheckpoint)
w.mux.Handle("POST /add-checkpoint", addCheckpoint)
return w, nil
}

Expand Down Expand Up @@ -148,6 +163,14 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
fmt.Fprintf(rw, "%d\n", err.known)
return
}

// Metrics sidequest: ensure known logs counter is synced with reality.
// (At the moment of writing, there's a dearth of more convenient time to notice this;
// when this number changes, it's due to an external poke of the db.)
if n, err := w.countKnownLogs(); err == nil {
w.metrics.KnownLogs.Set(float64(n))
}

switch err {
case errUnknownLog:
http.Error(rw, err.Error(), http.StatusNotFound)
Expand All @@ -172,12 +195,15 @@ func (w *Witness) serveAddCheckpoint(rw http.ResponseWriter, r *http.Request) {
}

func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosig []byte, err error) {
labels := prometheus.Labels{"error": "", "origin": "", "progress": ""}
l := w.log.With("request", string(body))
defer func() {
if err != nil {
l = l.With("error", err)
labels["error"] = err.Error()
}
l.Debug("processed add-checkpoint request")
w.metrics.AddCheckpointCount.With(labels).Inc()
}()
body, noteBytes, ok := bytes.Cut(body, []byte("\n\n"))
if !ok {
Expand Down Expand Up @@ -212,6 +238,7 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi
return nil, errBadRequest
}
l = l.With("origin", origin)
labels["origin"] = origin
bastions, err := w.getBastions(origin)
if err != nil {
return nil, err
Expand All @@ -237,6 +264,10 @@ func (w *Witness) processAddCheckpointRequest(body []byte, bastion string) (cosi
return nil, err
}
l = l.With("size", c.N)
labels["progress"] = "false"
if c.N > oldSize {
labels["progress"] = "true"
}
if err := w.checkConsistency(c.Origin, oldSize, c.N, c.Hash, proof); err != nil {
return nil, err
}
Expand Down Expand Up @@ -308,9 +339,22 @@ func (w *Witness) persistTreeHead(origin string, oldSize, newSize int64, newHash
}
return &conflictError{knownSize}
}
if err == nil {
w.metrics.LogSize.WithLabelValues(origin).Set(float64(newSize))
}
return err
}

func (w *Witness) countKnownLogs() (int, error) {
var n int
err := w.dbExec("SELECT COUNT(*) AS n FROM log",
func(stmt *sqlite.Stmt) error {
n = int(stmt.GetInt64("n"))
return nil
})
return n, err
}

func (w *Witness) getLog(origin string) (treeSize int64, treeHash tlog.Hash, err error) {
found := false
err = w.dbExec("SELECT tree_size, tree_hash FROM log WHERE origin = ?",
Expand Down
Loading