Add Upstreams tab (#15)

Author: alexandre-daubois

README.md

@@ -25,6 +25,7 @@ Caddy exposes rich metrics through its admin API and Prometheus endpoint, but re
 - Sorting, filtering, and full-screen ASCII graphs (CPU, RPS, RSS)
 - Config Inspector tab: browse the live Caddy JSON config as a collapsible tree
 - Certificates tab: TLS certificate monitoring with expiry tracking, color-coded warnings, and likely auto-renewal indication
+- Upstreams tab: reverse proxy upstream health monitoring with per-upstream status, auto-detected when `reverse_proxy` is configured
 - Automatic Caddy restart detection
 
 **FrankenPHP Introspection**

docs/caddy-dashboard.md

@@ -66,6 +66,26 @@ Navigation in the Caddy Config tab:
 | `E` | Collapse all nodes |
 | `r` | Refresh config from Caddy |
 
+## Upstreams
+
+The **Upstreams** tab appears automatically when Caddy exposes `caddy_reverse_proxy_upstreams_healthy` metrics, which happens when at least one `reverse_proxy` handler is configured.
+
+The table shows one row per upstream:
+
+| Column | Description |
+|--------|-------------|
+| **Upstream** | Upstream address (host:port) |
+| **Check** | Active health check URI and interval (e.g. `/health @5s`), extracted from Caddy config |
+| **LB** | Load balancing policy (e.g. `round_robin`, `least_conn`), extracted from Caddy config |
+| **Health** | Health status: `● healthy` or `○ down` |
+| **Down** | Duration since the upstream went down (e.g. `5s`, `2m30s`, `1h5m`) |
+
+A `!` suffix on the health status indicates a state change since the previous poll (e.g. an upstream just went down or recovered).
+
+The Check and LB columns are populated from the Caddy config when the tab first appears. Press `r` to refresh the config data.
+
+Press `s`/`S` to sort by address or health status. Press `/` to filter by address or handler name.
+
 ## Graphs
 
 Press `g` to toggle full-screen graphs showing:

docs/json-output.md

@@ -81,6 +81,11 @@ Each line is a JSON object with the following fields:
 | `hosts[].statusCodes` | Status code → rate (req/s) |
 | `hosts[].methodRates` | HTTP method → rate (req/s) |
 | `hosts[].avgRequestSize` | Average request body size in bytes (omitted when 0) |
+| `upstreams` | Reverse proxy upstream health (omitted when no `reverse_proxy` is configured) |
+| `upstreams[].address` | Upstream address (host:port) |
+| `upstreams[].handler` | Reverse proxy handler name (omitted when Caddy doesn't expose the label) |
+| `upstreams[].healthy` | Whether the upstream is healthy |
+| `upstreams[].healthChanged` | Whether health status changed since last poll (omitted when false) |
 
 ## Single Snapshot
 

internal/app/json.go

@@ -20,6 +20,14 @@ type jsonOutput struct {
 	Errors    []string                `json:"errors,omitempty"`
 	Derived   *jsonDerived            `json:"derived,omitempty"`
 	Hosts     []jsonHost              `json:"hosts,omitempty"`
+	Upstreams []jsonUpstream          `json:"upstreams,omitempty"`
+}
+
+type jsonUpstream struct {
+	Address       string `json:"address"`
+	Handler       string `json:"handler,omitempty"`
+	Healthy       bool   `json:"healthy"`
+	HealthChanged bool   `json:"healthChanged,omitempty"`
 }
 
 type jsonThreadsResponse struct {
@@ -146,6 +154,15 @@ func buildJSONOutput(snap *fetcher.Snapshot, state *model.State) jsonOutput {
 		out.Hosts = append(out.Hosts, jh)
 	}
 
+	for _, ud := range state.UpstreamDerived {
+		out.Upstreams = append(out.Upstreams, jsonUpstream{
+			Address:       ud.Address,
+			Handler:       ud.Handler,
+			Healthy:       ud.Healthy,
+			HealthChanged: ud.HealthChanged,
+		})
+	}
+
 	sanitizeForJSON(&out)
 
 	return out

internal/app/status.go

@@ -174,6 +174,16 @@ func formatStatusLine(state *model.State, hasFrankenPHP bool) string {
 		parts = append(parts, fmt.Sprintf("up %s", model.FormatUptime(snap.Process.Uptime)))
 	}
 
+	if upCount := len(snap.Metrics.Upstreams); upCount > 0 {
+		healthy := 0
+		for _, u := range snap.Metrics.Upstreams {
+			if u.Healthy >= 1 {
+				healthy++
+			}
+		}
+		parts = append(parts, fmt.Sprintf("%d/%d upstreams healthy", healthy, upCount))
+	}
+
 	if hasFrankenPHP {
 		total := d.TotalBusy + d.TotalIdle
 		fpPart := fmt.Sprintf("FrankenPHP %d/%d busy", d.TotalBusy, total)

internal/fetcher/fetcher.go

@@ -37,6 +37,19 @@ type WorkerMetrics struct {
 	QueueDepth   float64 `json:"queueDepth"`
 }
 
+// UpstreamMetrics represents a single Caddy reverse proxy upstream health entry.
+// Address is the dial target (e.g. "backend1:80"). It is not unique on its own
+// when Caddy exports the same address from multiple handlers: in that case the
+// parser disambiguates by combining Address and Handler, so consumers that need
+// a stable identity should use both fields together. Handler is empty when
+// Caddy omits the label (the common case for caddy_reverse_proxy_upstreams_healthy).
+// Healthy is 1.0 when healthy, 0.0 when down.
+type UpstreamMetrics struct {
+	Address string  `json:"address"`
+	Handler string  `json:"handler,omitempty"`
+	Healthy float64 `json:"healthy"`
+}
+
 type HostMetrics struct {
 	Host              string             `json:"host"`
 	RequestsTotal     float64            `json:"requestsTotal"`
@@ -75,6 +88,9 @@ type MetricsSnapshot struct {
 	// Per-host Caddy HTTP metrics
 	Hosts map[string]*HostMetrics `json:"hosts,omitempty"`
 
+	// Caddy reverse proxy upstream health
+	Upstreams map[string]*UpstreamMetrics `json:"upstreams,omitempty"`
+
 	// Go runtime process metrics (from standard Prometheus collector)
 	ProcessCPUSecondsTotal  float64 `json:"processCpuSecondsTotal,omitempty"`
 	ProcessRSSBytes         float64 `json:"processRssBytes,omitempty"`
@@ -121,6 +137,19 @@ type CertificateInfo struct {
 	AutoRenew bool
 }
 
+type ReverseProxyConfig struct {
+	Handler        string
+	LBPolicy       string
+	HealthURI      string
+	HealthInterval string
+	Upstreams      []ReverseProxyUpstreamConfig
+}
+
+type ReverseProxyUpstreamConfig struct {
+	Address     string
+	MaxRequests int
+}
+
 type Fetcher interface {
 	Fetch(ctx context.Context) (*Snapshot, error)
 }

internal/fetcher/prometheus.go

@@ -80,6 +80,7 @@ func parsePrometheusMetrics(r io.Reader) (snap MetricsSnapshot, err error) {
 	snap.ConfigLastReloadSuccessTimestamp = scalarValue(families, "caddy_config_last_reload_success_timestamp_seconds")
 
 	snap.Hosts = perHostMetrics(families)
+	snap.Upstreams = upstreamMetrics(families)
 
 	// Fallback: if HTTP metrics exist but no host labels, aggregate as a single "*" entry
 	if snap.HasHTTPMetrics && len(snap.Hosts) == 0 {
@@ -395,6 +396,36 @@ func perHostMetrics(families map[string]*dto.MetricFamily) map[string]*HostMetri
 	return hosts
 }
 
+func upstreamMetrics(families map[string]*dto.MetricFamily) map[string]*UpstreamMetrics {
+	fam, ok := families["caddy_reverse_proxy_upstreams_healthy"]
+	if !ok {
+		return nil
+	}
+
+	upstreams := make(map[string]*UpstreamMetrics)
+	for _, m := range fam.GetMetric() {
+		addr := labelValue(m, "upstream")
+		if addr == "" {
+			continue
+		}
+		handler := labelValue(m, "handler")
+		key := addr
+		if handler != "" {
+			key = addr + "/" + handler
+		}
+		upstreams[key] = &UpstreamMetrics{
+			Address: addr,
+			Handler: handler,
+			Healthy: metricValue(m),
+		}
+	}
+
+	if len(upstreams) == 0 {
+		return nil
+	}
+	return upstreams
+}
+
 func (s *MetricsSnapshot) getOrCreateWorker(name string) *WorkerMetrics {
 	wm, ok := s.Workers[name]
 	if !ok {

internal/fetcher/prometheus_test.go

@@ -734,3 +734,74 @@ frankenphp_busy_threads 5
 	assert.Equal(t, float64(0), snap.ProcessRSSBytes)
 	assert.Equal(t, float64(0), snap.ProcessStartTimeSeconds)
 }
+
+// Caddy exports caddy_reverse_proxy_upstreams_healthy with only two labels:
+// upstream and handler. The current release even omits handler entirely, so this
+// fixture exercises both shapes.
+const sampleUpstreamMetrics = `# HELP caddy_reverse_proxy_upstreams_healthy Health status of reverse proxy upstreams
+# TYPE caddy_reverse_proxy_upstreams_healthy gauge
+caddy_reverse_proxy_upstreams_healthy{upstream="10.0.0.1:8080"} 1
+caddy_reverse_proxy_upstreams_healthy{upstream="10.0.0.2:8080"} 1
+caddy_reverse_proxy_upstreams_healthy{upstream="10.0.0.3:8080"} 0
+caddy_reverse_proxy_upstreams_healthy{handler="reverse_proxy_1",upstream="api.internal:9090"} 1
+`
+
+func TestParsePrometheusMetrics_Upstreams(t *testing.T) {
+	snap, err := parsePrometheusMetrics(strings.NewReader(sampleUpstreamMetrics))
+	require.NoError(t, err)
+	require.Len(t, snap.Upstreams, 4)
+
+	u1 := snap.Upstreams["10.0.0.1:8080"]
+	require.NotNil(t, u1)
+	assert.Equal(t, "10.0.0.1:8080", u1.Address)
+	assert.Empty(t, u1.Handler, "current Caddy omits the handler label")
+	assert.Equal(t, float64(1), u1.Healthy)
+
+	u3 := snap.Upstreams["10.0.0.3:8080"]
+	require.NotNil(t, u3)
+	assert.Equal(t, float64(0), u3.Healthy)
+
+	api := snap.Upstreams["api.internal:9090/reverse_proxy_1"]
+	require.NotNil(t, api)
+	assert.Equal(t, "api.internal:9090", api.Address)
+	assert.Equal(t, "reverse_proxy_1", api.Handler)
+	assert.Equal(t, float64(1), api.Healthy)
+}
+
+func TestParsePrometheusMetrics_NoUpstreams(t *testing.T) {
+	snap, err := parsePrometheusMetrics(strings.NewReader(sampleMetrics))
+	require.NoError(t, err)
+	assert.Nil(t, snap.Upstreams)
+}
+
+func TestParsePrometheusMetrics_UpstreamsWithHandlerLabel(t *testing.T) {
+	input := `# TYPE caddy_reverse_proxy_upstreams_healthy gauge
+caddy_reverse_proxy_upstreams_healthy{handler="rp_0",upstream="a:80"} 1
+caddy_reverse_proxy_upstreams_healthy{handler="rp_1",upstream="a:80"} 0
+`
+	snap, err := parsePrometheusMetrics(strings.NewReader(input))
+	require.NoError(t, err)
+	require.Len(t, snap.Upstreams, 2, "same address with different handlers should be distinct entries")
+
+	u0 := snap.Upstreams["a:80/rp_0"]
+	require.NotNil(t, u0)
+	assert.Equal(t, "rp_0", u0.Handler)
+	assert.Equal(t, float64(1), u0.Healthy)
+
+	u1 := snap.Upstreams["a:80/rp_1"]
+	require.NotNil(t, u1)
+	assert.Equal(t, "rp_1", u1.Handler)
+	assert.Equal(t, float64(0), u1.Healthy)
+}
+
+func TestParsePrometheusMetrics_UpstreamsEmptyLabel(t *testing.T) {
+	input := `# HELP caddy_reverse_proxy_upstreams_healthy Health status
+# TYPE caddy_reverse_proxy_upstreams_healthy gauge
+caddy_reverse_proxy_upstreams_healthy{handler="rp",upstream=""} 1
+caddy_reverse_proxy_upstreams_healthy{handler="rp",upstream="valid:8080"} 1
+`
+	snap, err := parsePrometheusMetrics(strings.NewReader(input))
+	require.NoError(t, err)
+	require.Len(t, snap.Upstreams, 1)
+	assert.Contains(t, snap.Upstreams, "valid:8080/rp")
+}