feat(#91): agent — phase 4/5/7 metrics collection (nginx, Apache, Caddy, Traefik) + access-log

gearbox-agent/cmd/gearbox-agent/main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/sarg3nt/gearbox-agent/internal/framework/services/sync"
 
 	// Import plugins - blank identifier triggers init() registration
+	_ "github.com/sarg3nt/gearbox-agent/internal/gears/accesslog"
 	_ "github.com/sarg3nt/gearbox-agent/internal/gears/apache"
 	_ "github.com/sarg3nt/gearbox-agent/internal/gears/caddy"
 	_ "github.com/sarg3nt/gearbox-agent/internal/gears/certs"
@@ -397,6 +398,10 @@ func main() {
 		CaddyAdminURL:        cfg.CaddyAdminURL,
 		TraefikMetricsURL:    cfg.TraefikMetricsURL,
 		DockerSocket:         cfg.DockerSocket,
+		HAProxyAccessLog:     cfg.HAProxyAccessLog,
+		NginxAccessLog:       cfg.NginxAccessLog,
+		ApacheAccessLog:      cfg.ApacheAccessLog,
+		CaddyAccessLog:       cfg.CaddyAccessLog,
 	}
 
 	// Create plugin manager

gearbox-agent/internal/framework/config/config.go

@@ -101,6 +101,17 @@ type Config struct {
 	CaddyAdminURL     string // CADDY_ADMIN_URL     — force the admin/Prometheus URL
 	TraefikMetricsURL string // TRAEFIK_METRICS_URL — force the Prometheus endpoint URL
 	DockerSocket      string // DOCKER_SOCKET       — force a specific docker socket path
+
+	// Access-log paths per source. The /api/v1/access-log/{source}/recent
+	// endpoint reads the most recent N lines from these files and parses
+	// each with the matching profile. Empty (the default) means the
+	// endpoint falls back to a well-known path; if that doesn't exist
+	// the endpoint reports "no readable log file" rather than failing
+	// the agent. See [docs/source-detection.md] / issue #91 Phase 5.
+	HAProxyAccessLog string // HAPROXY_ACCESS_LOG
+	NginxAccessLog   string // NGINX_ACCESS_LOG
+	ApacheAccessLog  string // APACHE_ACCESS_LOG
+	CaddyAccessLog   string // CADDY_ACCESS_LOG
 }
 
 // DefaultConfig returns the default configuration.
@@ -212,6 +223,13 @@ func Load() (*Config, error) {
 	cfg.TraefikMetricsURL = strings.TrimSpace(os.Getenv("TRAEFIK_METRICS_URL"))
 	cfg.DockerSocket = strings.TrimSpace(os.Getenv("DOCKER_SOCKET"))
 
+	// Access-log path overrides — trimmed but case-preserved (paths
+	// are case-sensitive on most filesystems).
+	cfg.HAProxyAccessLog = strings.TrimSpace(os.Getenv("HAPROXY_ACCESS_LOG"))
+	cfg.NginxAccessLog = strings.TrimSpace(os.Getenv("NGINX_ACCESS_LOG"))
+	cfg.ApacheAccessLog = strings.TrimSpace(os.Getenv("APACHE_ACCESS_LOG"))
+	cfg.CaddyAccessLog = strings.TrimSpace(os.Getenv("CADDY_ACCESS_LOG"))
+
 	return cfg, nil
 }
 

gearbox-agent/internal/framework/gear/dependencies.go

@@ -66,6 +66,15 @@ type Dependencies struct {
 	CaddyAdminURL     string // CADDY_ADMIN_URL
 	TraefikMetricsURL string // TRAEFIK_METRICS_URL
 	DockerSocket      string // DOCKER_SOCKET
+
+	// Per-source access-log paths. Empty means "fall back to the
+	// gear's well-known default"; an explicit value bypasses the
+	// fallback (and a non-existent path then surfaces as
+	// "log file not readable" through the access-log endpoint).
+	HAProxyAccessLog string // HAPROXY_ACCESS_LOG
+	NginxAccessLog   string // NGINX_ACCESS_LOG
+	ApacheAccessLog  string // APACHE_ACCESS_LOG
+	CaddyAccessLog   string // CADDY_ACCESS_LOG
 }
 
 // Common event types used across plugins.

gearbox-agent/internal/framework/services/accesslog/apache.go

@@ -0,0 +1,89 @@
+package accesslog
+
+import (
+	"regexp"
+	"strconv"
+	"time"
+)
+
+// Apache CLF (common log format):
+//
+//	%h %l %u %t \"%r\" %>s %b
+//
+// Concrete example:
+//
+//	192.168.1.1 - - [28/Aug/2025:10:24:13 +0000] "GET /path HTTP/1.1" 200 1234
+//
+// Combined format adds two trailing quoted fields for Referer and
+// User-Agent — see ApacheCombinedProfile, which delegates to
+// parseCombined for that shape.
+//
+// We use a single regex for CLF: it's strict about the leading fields
+// (IP, dash, dash, bracketed date, quoted request) but tolerant about
+// what comes after the byte count, since some operators append %D
+// (duration in µs) or trace IDs.
+var reApacheCommon = regexp.MustCompile(
+	`^([^ ]+) [^ ]+ [^ ]+ \[([^\]]+)\] "([A-Z]+) ([^ "]+)[^"]*" (\d{3}) (\d+|-)`,
+)
+
+// apacheTimeLayout is identical to nginx's `%t` format — both
+// projects use the NCSA Common Log convention.
+const apacheTimeLayout = "02/Jan/2006:15:04:05 -0700"
+
+// ApacheCommonProfile parses Apache CLF lines (without Referer /
+// User-Agent). Returns nil for lines that don't carry a valid HTTP
+// status code.
+type ApacheCommonProfile struct{}
+
+// Profile satisfies Parser.
+func (ApacheCommonProfile) Profile() string { return ProfileApacheCommon }
+
+// Parse returns a Record for one Apache CLF log line, or nil on
+// shape mismatch.
+func (ApacheCommonProfile) Parse(raw string) *Record {
+	m := reApacheCommon.FindStringSubmatch(raw)
+	if len(m) < 7 {
+		return nil
+	}
+	status, err := strconv.Atoi(m[5])
+	if err != nil || !validStatusCode(status) {
+		return nil
+	}
+
+	rec := &Record{
+		Profile:      ProfileApacheCommon,
+		SourceIP:     m[1],
+		TimestampRaw: m[2],
+		Method:       m[3],
+		Path:         m[4],
+		StatusCode:   status,
+		Raw:          trimRaw(raw),
+	}
+
+	if m[6] != "-" {
+		if n, err := strconv.ParseInt(m[6], 10, 64); err == nil {
+			rec.BytesSent = n
+		}
+	}
+
+	if t, err := time.Parse(apacheTimeLayout, m[2]); err == nil {
+		rec.Timestamp = t
+	}
+
+	return rec
+}
+
+// ApacheCombinedProfile parses Apache "combined" format — CLF plus
+// Referer and User-Agent. Same byte-for-byte shape as nginx's
+// combined format (the Apache directive `combined` was inherited
+// from NCSA), so we delegate to parseCombined and only differ in the
+// Profile identifier we stamp on each Record.
+type ApacheCombinedProfile struct{}
+
+// Profile satisfies Parser.
+func (ApacheCombinedProfile) Profile() string { return ProfileApacheCombined }
+
+// Parse returns a Record for one Apache combined-format log line.
+func (ApacheCombinedProfile) Parse(raw string) *Record {
+	return parseCombined(raw, ProfileApacheCombined)
+}

gearbox-agent/internal/framework/services/accesslog/caddy_json.go

@@ -0,0 +1,85 @@
+package accesslog
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// caddyAccessLog is the shape Caddy's `http.log.access` logger emits
+// when the operator hasn't disabled fields. We only need a handful
+// of values for the Record; everything else is left in the raw line
+// for the dashboard's "see the original" affordance.
+//
+// Field-by-field (Caddy v2.x):
+//
+//	ts:       float Unix seconds with sub-second precision
+//	duration: float seconds (we convert to ms)
+//	size:     int response body bytes
+//	status:   int HTTP status code
+//	request:  embedded {remote_ip, method, uri, host, headers{User-Agent,Referer}}
+type caddyAccessLog struct {
+	TS       float64 `json:"ts"`
+	Duration float64 `json:"duration"`
+	Size     int64   `json:"size"`
+	Status   int     `json:"status"`
+	Request  struct {
+		RemoteIP string              `json:"remote_ip"`
+		Method   string              `json:"method"`
+		URI      string              `json:"uri"`
+		Host     string              `json:"host"`
+		Headers  map[string][]string `json:"headers"`
+	} `json:"request"`
+}
+
+// CaddyJSONProfile parses Caddy's structured JSON access log. Each
+// line must be one valid JSON object; multi-line / pretty-printed
+// output returns nil. Returns nil for any object that's missing a
+// recognisable HTTP status code (heartbeats, admin events, etc.
+// that Caddy may also write to the same logger if misconfigured).
+type CaddyJSONProfile struct{}
+
+// Profile satisfies Parser.
+func (CaddyJSONProfile) Profile() string { return ProfileCaddyJSON }
+
+// Parse returns a Record for one Caddy JSON access-log entry, or nil
+// when the line isn't a recognisable HTTP access event.
+func (CaddyJSONProfile) Parse(raw string) *Record {
+	var entry caddyAccessLog
+	if err := json.Unmarshal([]byte(raw), &entry); err != nil {
+		return nil
+	}
+	if !validStatusCode(entry.Status) {
+		return nil
+	}
+
+	rec := &Record{
+		Profile:    ProfileCaddyJSON,
+		StatusCode: entry.Status,
+		BytesSent:  entry.Size,
+		DurationMs: entry.Duration * 1000.0,
+		SourceIP:   entry.Request.RemoteIP,
+		Method:     entry.Request.Method,
+		Path:       entry.Request.URI,
+		Host:       entry.Request.Host,
+		Raw:        trimRaw(raw),
+	}
+
+	if entry.TS > 0 {
+		// Caddy's `ts` is float Unix seconds with sub-second
+		// precision; time.UnixMicro keeps that precision when the
+		// caller wants to render at ms granularity downstream.
+		rec.Timestamp = time.UnixMicro(int64(entry.TS * 1e6)).UTC()
+		rec.TimestampRaw = rec.Timestamp.Format(time.RFC3339Nano)
+	}
+
+	if h := entry.Request.Headers; h != nil {
+		if v := h["User-Agent"]; len(v) > 0 {
+			rec.UserAgent = v[0]
+		}
+		if v := h["Referer"]; len(v) > 0 {
+			rec.Referer = v[0]
+		}
+	}
+
+	return rec
+}

gearbox-agent/internal/framework/services/accesslog/haproxy.go

@@ -0,0 +1,82 @@
+package accesslog
+
+import (
+	"regexp"
+	"strconv"
+)
+
+// HAProxy HTTP log format (from `option httplog` / the default
+// log-format), as the field positions appear in practice:
+//
+//	<date> <host> haproxy[<pid>]: <client_ip>:<port> [<accept_date>] <frontend>~ <backend>/<server> <Tq>/<Tw>/<Tc>/<Tr>/<Tt> <status> <bytes_read> ... "<method> <path> HTTP/1.x"
+//
+// We parse defensively — fields can vary by HAProxy version and the
+// operator's custom log-format. Anything we can't pull out stays
+// zero-valued on the Record. This is a direct port of the dashboard's
+// original `parseHAProxyLogLine`, kept identical in behaviour so the
+// metrics-page Error Insights panel sees no diff when the dashboard
+// flips to consuming the agent endpoint instead of parsing locally.
+var (
+	reHAProxyStatus = regexp.MustCompile(`\s(\d{3})\s+\d+\s`)
+	reHAProxyReq    = regexp.MustCompile(`"([A-Z]+)\s+([^\s"]+)`)
+	reHAProxyBkSvr  = regexp.MustCompile(`\s([A-Za-z0-9_.\-]+)/([A-Za-z0-9_.\-]+)\s+\d+\/\-?\d+\/\-?\d+\/\-?\d+\/\-?\d+\s`)
+	reHAProxyClient = regexp.MustCompile(`(?:^|\s)(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[0-9a-fA-F:]+):\d+\s+\[`)
+	// Anchored on the date shape (day/Mon/year:hh:mm:ss) so the
+	// syslog-style `haproxy[1234]:` PID bracket doesn't claim the
+	// match. The dashboard-side original had this latent bug; the
+	// fixtures there never carried the syslog wrapper.
+	reHAProxyDate = regexp.MustCompile(`\[(\d{1,2}/\w{3}/\d{4}:[^\]]+)\]`)
+	// Tt is the total time field; it appears as the fifth slash-
+	// separated number in Tq/Tw/Tc/Tr/Tt. Negative values mean the
+	// session was aborted before that timer was set; we surface ms
+	// only when Tt is non-negative.
+	reHAProxyTimings = regexp.MustCompile(`\s\-?\d+\/\-?\d+\/\-?\d+\/\-?\d+\/(\-?\d+)\s+\d{3}\s`)
+)
+
+// HAProxyProfile parses HAProxy HTTP access log lines. Returns nil
+// for lines that don't carry a valid HTTP status code (SSL handshake
+// errors, connection diagnostics, etc.).
+type HAProxyProfile struct{}
+
+// Profile satisfies Parser.
+func (HAProxyProfile) Profile() string { return ProfileHAProxy }
+
+// Parse pulls structured fields out of one HAProxy HTTP log line.
+func (HAProxyProfile) Parse(raw string) *Record {
+	statusMatch := reHAProxyStatus.FindStringSubmatch(raw)
+	if len(statusMatch) < 2 {
+		return nil
+	}
+	status, _ := strconv.Atoi(statusMatch[1])
+	if !validStatusCode(status) {
+		return nil
+	}
+
+	rec := &Record{
+		Profile:    ProfileHAProxy,
+		StatusCode: status,
+		Raw:        trimRaw(raw),
+	}
+
+	if m := reHAProxyDate.FindStringSubmatch(raw); len(m) > 1 {
+		rec.TimestampRaw = m[1]
+	}
+	if m := reHAProxyClient.FindStringSubmatch(raw); len(m) > 1 {
+		rec.SourceIP = m[1]
+	}
+	if m := reHAProxyBkSvr.FindStringSubmatch(raw); len(m) > 2 {
+		rec.Backend = m[1]
+		rec.Server = m[2]
+	}
+	if m := reHAProxyReq.FindStringSubmatch(raw); len(m) > 2 {
+		rec.Method = m[1]
+		rec.Path = m[2]
+	}
+	if m := reHAProxyTimings.FindStringSubmatch(raw); len(m) > 1 {
+		if tt, err := strconv.Atoi(m[1]); err == nil && tt >= 0 {
+			rec.DurationMs = float64(tt)
+		}
+	}
+
+	return rec
+}