Add Upstreams tab

Author: alexandre-daubois

README.md

@@ -25,6 +25,7 @@ Caddy exposes rich metrics through its admin API and Prometheus endpoint, but re
 - Sorting, filtering, and full-screen ASCII graphs (CPU, RPS, RSS)
 - Config Inspector tab: browse the live Caddy JSON config as a collapsible tree
 - Certificates tab: TLS certificate monitoring with expiry tracking, color-coded warnings, and likely auto-renewal indication
+- Upstreams tab: reverse proxy upstream health monitoring with per-upstream status, auto-detected when `reverse_proxy` is configured
 - Automatic Caddy restart detection
 
 **FrankenPHP Introspection**

docs/caddy-dashboard.md

@@ -66,6 +66,26 @@ Navigation in the Caddy Config tab:
 | `E` | Collapse all nodes |
 | `r` | Refresh config from Caddy |
 
+## Upstreams
+
+The **Upstreams** tab appears automatically when Caddy exposes `caddy_reverse_proxy_upstreams_healthy` metrics, which happens when at least one `reverse_proxy` handler is configured.
+
+The table shows one row per upstream:
+
+| Column | Description |
+|--------|-------------|
+| **Upstream** | Upstream address (host:port) |
+| **Check** | Active health check URI and interval (e.g. `/health @5s`), extracted from Caddy config |
+| **LB** | Load balancing policy (e.g. `round_robin`, `least_conn`), extracted from Caddy config |
+| **Health** | Health status: `● healthy` or `○ down` |
+| **Down** | Duration since the upstream went down (e.g. `5s`, `2m30s`, `1h5m`) |
+
+A `!` suffix on the health status indicates a state change since the previous poll (e.g. an upstream just went down or recovered).
+
+The Check and LB columns are populated from the Caddy config when the tab first appears. Press `r` to refresh the config data.
+
+Press `s`/`S` to sort by address or health status. Press `/` to filter by address or handler name.
+
 ## Graphs
 
 Press `g` to toggle full-screen graphs showing:

docs/json-output.md

@@ -81,6 +81,11 @@ Each line is a JSON object with the following fields:
 | `hosts[].statusCodes` | Status code → rate (req/s) |
 | `hosts[].methodRates` | HTTP method → rate (req/s) |
 | `hosts[].avgRequestSize` | Average request body size in bytes (omitted when 0) |
+| `upstreams` | Reverse proxy upstream health (omitted when no `reverse_proxy` is configured) |
+| `upstreams[].address` | Upstream address (host:port) |
+| `upstreams[].handler` | Reverse proxy handler name (omitted when Caddy doesn't expose the label) |
+| `upstreams[].healthy` | Whether the upstream is healthy |
+| `upstreams[].healthChanged` | Whether health status changed since last poll (omitted when false) |
 
 ## Single Snapshot
 

docs/prometheus-export.md

@@ -90,6 +90,14 @@ This re-reads `--ca-cert`, `--client-cert`, and `--client-key` files and applies
 | `ember_host_status_rate` | gauge | `host`, `class` (`2xx`, `3xx`, `4xx`, `5xx`) | Request rate by status class |
 | `ember_host_error_rate` | gauge | `host` | Middleware error rate (handler-level errors, distinct from HTTP status codes) |
 
+### Caddy Reverse Proxy Upstreams
+
+| Metric | Type | Labels | Description |
+|--------|------|--------|-------------|
+| `caddy_reverse_proxy_upstreams_healthy` | gauge | `upstream`, `handler` (omitted when empty) | Whether the upstream is healthy (1) or down (0). Matches Caddy's own metric name. |
+
+These metrics are only emitted when Caddy has at least one `reverse_proxy` handler configured.
+
 ### Caddy Config Reload
 
 | Metric | Type | Description |

internal/app/json.go

@@ -20,6 +20,14 @@ type jsonOutput struct {
 	Errors    []string                `json:"errors,omitempty"`
 	Derived   *jsonDerived            `json:"derived,omitempty"`
 	Hosts     []jsonHost              `json:"hosts,omitempty"`
+	Upstreams []jsonUpstream          `json:"upstreams,omitempty"`
+}
+
+type jsonUpstream struct {
+	Address       string `json:"address"`
+	Handler       string `json:"handler,omitempty"`
+	Healthy       bool   `json:"healthy"`
+	HealthChanged bool   `json:"healthChanged,omitempty"`
 }
 
 type jsonThreadsResponse struct {
@@ -146,6 +154,15 @@ func buildJSONOutput(snap *fetcher.Snapshot, state *model.State) jsonOutput {
 		out.Hosts = append(out.Hosts, jh)
 	}
 
+	for _, ud := range state.UpstreamDerived {
+		out.Upstreams = append(out.Upstreams, jsonUpstream{
+			Address:       ud.Address,
+			Handler:       ud.Handler,
+			Healthy:       ud.Healthy,
+			HealthChanged: ud.HealthChanged,
+		})
+	}
+
 	sanitizeForJSON(&out)
 
 	return out

internal/app/status.go

@@ -174,6 +174,16 @@ func formatStatusLine(state *model.State, hasFrankenPHP bool) string {
 		parts = append(parts, fmt.Sprintf("up %s", model.FormatUptime(snap.Process.Uptime)))
 	}
 
+	if upCount := len(snap.Metrics.Upstreams); upCount > 0 {
+		healthy := 0
+		for _, u := range snap.Metrics.Upstreams {
+			if u.Healthy >= 1 {
+				healthy++
+			}
+		}
+		parts = append(parts, fmt.Sprintf("%d/%d upstreams healthy", healthy, upCount))
+	}
+
 	if hasFrankenPHP {
 		total := d.TotalBusy + d.TotalIdle
 		fpPart := fmt.Sprintf("FrankenPHP %d/%d busy", d.TotalBusy, total)

internal/exporter/exporter.go

@@ -51,6 +51,7 @@ func Handler(holder *StateHolder, prefix ...string) http.HandlerFunc {
 		writeThreadMemory(w, &s, p)
 		writeWorkerMetrics(w, &s, p)
 		writeHostMetrics(w, &s, p)
+		writeUpstreamMetrics(w, &s, p)
 		writeErrorMetrics(w, &s, p)
 		writePercentiles(w, &s, p)
 		writeProcessMetrics(w, &s, p)
@@ -245,6 +246,41 @@ func sortedHostNames(hosts []model.HostDerived) []model.HostDerived {
 	return sorted
 }
 
+func writeUpstreamMetrics(w http.ResponseWriter, s *model.State, prefix string) {
+	if len(s.UpstreamDerived) == 0 {
+		return
+	}
+
+	sorted := make([]model.UpstreamDerived, len(s.UpstreamDerived))
+	copy(sorted, s.UpstreamDerived)
+	slices.SortFunc(sorted, func(a, b model.UpstreamDerived) int {
+		if c := cmp.Compare(a.Address, b.Address); c != 0 {
+			return c
+		}
+		return cmp.Compare(a.Handler, b.Handler)
+	})
+
+	// Mirror Caddy's own metric name so consumers can scrape Ember as a drop-in
+	// source with the same series identity.
+	name := prefixed(prefix, "caddy_reverse_proxy_upstreams_healthy")
+	fmt.Fprintf(w, "# HELP %s Whether the upstream is healthy\n", name)
+	fmt.Fprintf(w, "# TYPE %s gauge\n", name)
+	for _, u := range sorted {
+		v := 0
+		if u.Healthy {
+			v = 1
+		}
+		// Omit handler label when empty to match Caddy's output shape: current
+		// Caddy versions don't set the handler label on this metric, so writing
+		// handler="" would make Ember's export cosmetically diverge.
+		if u.Handler == "" {
+			fmt.Fprintf(w, "%s{upstream=\"%s\"} %d\n", name, escapeLabelValue(u.Address), v)
+		} else {
+			fmt.Fprintf(w, "%s{upstream=\"%s\",handler=\"%s\"} %d\n", name, escapeLabelValue(u.Address), escapeLabelValue(u.Handler), v)
+		}
+	}
+}
+
 func writeErrorMetrics(w http.ResponseWriter, s *model.State, prefix string) {
 	hasErrors := false
 	for _, hd := range s.HostDerived {

internal/exporter/exporter_test.go

@@ -672,3 +672,101 @@ func TestBasicAuth_InvalidUser(t *testing.T) {
 
 	assert.Equal(t, http.StatusUnauthorized, rec.Code)
 }
+
+func stateWithUpstreams(upstreams []model.UpstreamDerived) model.State {
+	s := stateWithThreads(nil, nil)
+	s.UpstreamDerived = upstreams
+	return s
+}
+
+func TestHandler_UpstreamMetrics(t *testing.T) {
+	upstreams := []model.UpstreamDerived{
+		{Address: "10.0.0.1:8080", Handler: "rp", Healthy: true},
+		{Address: "10.0.0.2:8080", Handler: "rp", Healthy: false},
+		{Address: "api.internal:9090", Handler: "rp_1", Healthy: true},
+	}
+	holder := &StateHolder{}
+	holder.Store(stateWithUpstreams(upstreams))
+
+	body := get(holder).Body.String()
+
+	assert.Contains(t, body, "# HELP caddy_reverse_proxy_upstreams_healthy")
+	assert.Contains(t, body, "# TYPE caddy_reverse_proxy_upstreams_healthy gauge")
+	assert.Contains(t, body, `caddy_reverse_proxy_upstreams_healthy{upstream="10.0.0.1:8080",handler="rp"} 1`)
+	assert.Contains(t, body, `caddy_reverse_proxy_upstreams_healthy{upstream="10.0.0.2:8080",handler="rp"} 0`)
+	assert.Contains(t, body, `caddy_reverse_proxy_upstreams_healthy{upstream="api.internal:9090",handler="rp_1"} 1`)
+}
+
+func TestHandler_UpstreamMetrics_SkippedWhenEmpty(t *testing.T) {
+	holder := &StateHolder{}
+	holder.Store(stateWithThreads(nil, nil))
+
+	body := get(holder).Body.String()
+	assert.NotContains(t, body, "caddy_reverse_proxy_upstreams_healthy")
+}
+
+func TestHandler_UpstreamMetrics_EmptyHandlerOmitted(t *testing.T) {
+	upstreams := []model.UpstreamDerived{
+		{Address: "backend1:80", Handler: "", Healthy: true},
+		{Address: "backend2:80", Handler: "rp_0", Healthy: false},
+	}
+	holder := &StateHolder{}
+	holder.Store(stateWithUpstreams(upstreams))
+
+	body := get(holder).Body.String()
+
+	assert.Contains(t, body, `caddy_reverse_proxy_upstreams_healthy{upstream="backend1:80"} 1`,
+		"empty handler should produce no label at all, matching Caddy's output")
+	assert.NotContains(t, body, `backend1:80",handler=""`,
+		"handler=\"\" must not appear")
+	assert.Contains(t, body, `caddy_reverse_proxy_upstreams_healthy{upstream="backend2:80",handler="rp_0"} 0`,
+		"non-empty handler is still emitted")
+}
+
+func TestHandler_UpstreamMetrics_SortedDeterministic(t *testing.T) {
+	upstreams := []model.UpstreamDerived{
+		{Address: "z.internal:8080", Handler: "rp", Healthy: true},
+		{Address: "a.internal:8080", Handler: "rp", Healthy: true},
+		{Address: "m.internal:8080", Handler: "rp", Healthy: false},
+	}
+	holder := &StateHolder{}
+	holder.Store(stateWithUpstreams(upstreams))
+
+	body1 := get(holder).Body.String()
+	body2 := get(holder).Body.String()
+	require.Equal(t, body1, body2, "output should be deterministic")
+}
+
+func TestHandler_UpstreamMetrics_ValidPrometheus(t *testing.T) {
+	upstreams := []model.UpstreamDerived{
+		{Address: "10.0.0.1:8080", Handler: "rp", Healthy: true},
+		{Address: "10.0.0.2:8080", Handler: "rp", Healthy: false},
+	}
+	holder := &StateHolder{}
+	holder.Store(stateWithUpstreams(upstreams))
+
+	rec := get(holder)
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	parser := expfmt.NewTextParser(prommodel.UTF8Validation)
+	families, err := parser.TextToMetricFamilies(rec.Body)
+	require.NoError(t, err, "output must be valid Prometheus text format")
+	assert.Contains(t, families, "caddy_reverse_proxy_upstreams_healthy")
+}
+
+func TestHandler_UpstreamMetrics_WithPrefix(t *testing.T) {
+	upstreams := []model.UpstreamDerived{
+		{Address: "10.0.0.1:8080", Handler: "rp", Healthy: true},
+	}
+	holder := &StateHolder{}
+	holder.Store(stateWithUpstreams(upstreams))
+
+	rec := getWithPrefix(holder, "prod")
+	require.Equal(t, http.StatusOK, rec.Code)
+
+	parser := expfmt.NewTextParser(prommodel.UTF8Validation)
+	families, err := parser.TextToMetricFamilies(rec.Body)
+	require.NoError(t, err)
+	assert.Contains(t, families, "prod_caddy_reverse_proxy_upstreams_healthy")
+	assert.NotContains(t, families, "caddy_reverse_proxy_upstreams_healthy")
+}

internal/fetcher/fetcher.go

@@ -37,6 +37,19 @@ type WorkerMetrics struct {
 	QueueDepth   float64 `json:"queueDepth"`
 }
 
+// UpstreamMetrics represents a single Caddy reverse proxy upstream health entry.
+// Address is the dial target (e.g. "backend1:80"). It is not unique on its own
+// when Caddy exports the same address from multiple handlers: in that case the
+// parser disambiguates by combining Address and Handler, so consumers that need
+// a stable identity should use both fields together. Handler is empty when
+// Caddy omits the label (the common case for caddy_reverse_proxy_upstreams_healthy).
+// Healthy is 1.0 when healthy, 0.0 when down.
+type UpstreamMetrics struct {
+	Address string  `json:"address"`
+	Handler string  `json:"handler,omitempty"`
+	Healthy float64 `json:"healthy"`
+}
+
 type HostMetrics struct {
 	Host              string             `json:"host"`
 	RequestsTotal     float64            `json:"requestsTotal"`
@@ -75,6 +88,9 @@ type MetricsSnapshot struct {
 	// Per-host Caddy HTTP metrics
 	Hosts map[string]*HostMetrics `json:"hosts,omitempty"`
 
+	// Caddy reverse proxy upstream health
+	Upstreams map[string]*UpstreamMetrics `json:"upstreams,omitempty"`
+
 	// Go runtime process metrics (from standard Prometheus collector)
 	ProcessCPUSecondsTotal  float64 `json:"processCpuSecondsTotal,omitempty"`
 	ProcessRSSBytes         float64 `json:"processRssBytes,omitempty"`
@@ -121,6 +137,19 @@ type CertificateInfo struct {
 	AutoRenew bool
 }
 
+type ReverseProxyConfig struct {
+	Handler        string
+	LBPolicy       string
+	HealthURI      string
+	HealthInterval string
+	Upstreams      []ReverseProxyUpstreamConfig
+}
+
+type ReverseProxyUpstreamConfig struct {
+	Address     string
+	MaxRequests int
+}
+
 type Fetcher interface {
 	Fetch(ctx context.Context) (*Snapshot, error)
 }

internal/fetcher/prometheus.go

@@ -80,6 +80,7 @@ func parsePrometheusMetrics(r io.Reader) (snap MetricsSnapshot, err error) {
 	snap.ConfigLastReloadSuccessTimestamp = scalarValue(families, "caddy_config_last_reload_success_timestamp_seconds")
 
 	snap.Hosts = perHostMetrics(families)
+	snap.Upstreams = upstreamMetrics(families)
 
 	// Fallback: if HTTP metrics exist but no host labels, aggregate as a single "*" entry
 	if snap.HasHTTPMetrics && len(snap.Hosts) == 0 {
@@ -395,6 +396,36 @@ func perHostMetrics(families map[string]*dto.MetricFamily) map[string]*HostMetri
 	return hosts
 }
 
+func upstreamMetrics(families map[string]*dto.MetricFamily) map[string]*UpstreamMetrics {
+	fam, ok := families["caddy_reverse_proxy_upstreams_healthy"]
+	if !ok {
+		return nil
+	}
+
+	upstreams := make(map[string]*UpstreamMetrics)
+	for _, m := range fam.GetMetric() {
+		addr := labelValue(m, "upstream")
+		if addr == "" {
+			continue
+		}
+		handler := labelValue(m, "handler")
+		key := addr
+		if handler != "" {
+			key = addr + "/" + handler
+		}
+		upstreams[key] = &UpstreamMetrics{
+			Address: addr,
+			Handler: handler,
+			Healthy: metricValue(m),
+		}
+	}
+
+	if len(upstreams) == 0 {
+		return nil
+	}
+	return upstreams
+}
+
 func (s *MetricsSnapshot) getOrCreateWorker(name string) *WorkerMetrics {
 	wm, ok := s.Workers[name]
 	if !ok {