config.example.toml

# Kafka Lag Exporter Configuration
# Copy this file to config.toml and adjust as needed

[exporter]
# How often to poll Kafka for offsets (default: 30s)
poll_interval = "30s"

# HTTP server settings for Prometheus endpoint
http_port = 8000
http_host = "0.0.0.0"

# Metric granularity: "topic" (default) or "partition"
# "topic" aggregates per-topic, "partition" includes partition-level metrics
granularity = "topic"

# How long a cluster's metrics remain in /metrics output after its last
# successful collection. Past this age the cluster is filtered out so
# Prometheus sees a gap rather than a frozen snapshot. Defaults to
# poll_interval * 3 when unset — raise this on large clusters where a
# single collection cycle can take several minutes (timestamp sampling
# in 'message' mode, many consumer groups, etc.).
# staleness_threshold = "3m"

[exporter.timestamp_sampling]
# Enable time lag calculation
enabled = true

# Time-lag estimation mode:
#   "rate"    - (default) Estimate from observed high-watermark production
#               rate. No consumer pool, no FFI, scales to any cluster size.
#   "message" - Read the actual Kafka message at the committed offset and
#               use its produce timestamp. Exact, but requires a pooled
#               BaseConsumer per concurrent fetch (~5-15 MB each).
# mode = "rate"

# Message-mode settings (ignored when mode = "rate"):
cache_ttl = "60s"
max_concurrent_fetches = 5

# Rate-mode settings (ignored when mode = "message"):
# rate_history_samples = 5       # watermark observations per partition
# rate_history_max_age = "10m"   # evict samples older than this
# rate_min_msgs_per_sec = 0.01   # below this rate → time lag missing

[exporter.performance]
# Timeout for Kafka API operations (metadata, watermarks, etc.)
# kafka_timeout = "30s"

# Timeout for fetching committed offsets per consumer group
# offset_fetch_timeout = "10s"

# Maximum consumer groups to fetch offsets for in parallel
# max_concurrent_groups = 10

# Maximum partitions to fetch watermarks for in parallel
# max_concurrent_watermarks = 50

# Client recycling interval (number of collection cycles).
#
# librdkafka caches internal topic handles that are never freed until the
# client is destroyed. On large clusters with many topics (especially with
# topic churn — topics being created and deleted), this cache grows
# unboundedly and can consume gigabytes of memory.
#
# Recycling periodically destroys and recreates the internal Kafka clients,
# releasing all accumulated metadata. The trade-off is a brief allocation
# spike during the swap (~2-10 MB depending on cluster size).
#
# Guidelines:
#   0   = disabled (recommended for small/stable clusters)
#   50  = default (~25 min at 30s poll; good for large clusters)
#   100 = less frequent (lower overhead, more metadata accumulation)
#
# client_recycle_interval = 50

[exporter.otel]
# Enable OpenTelemetry export (default: false)
enabled = false

# OTLP endpoint (gRPC)
endpoint = "http://localhost:4317"

# How often to push metrics to OTLP
export_interval = "60s"

# Kafka cluster configuration
# You can define multiple [[clusters]] sections for multi-cluster monitoring

[[clusters]]
# Unique name for this cluster (used in metric labels)
name = "production"

# Kafka bootstrap servers (comma-separated)
bootstrap_servers = "kafka1:9092,kafka2:9092,kafka3:9092"

# Consumer group filters (regex patterns)
# Groups matching whitelist AND not matching blacklist will be monitored
group_whitelist = [".*"]
group_blacklist = []

# Topic filters (regex patterns)
topic_whitelist = [".*"]
topic_blacklist = ["^__.*"]  # Exclude internal topics like __consumer_offsets

# Additional Kafka consumer properties
# Use ${ENV_VAR} syntax for environment variable substitution
[clusters.consumer_properties]
# Uncomment and configure for SASL authentication:
# "security.protocol" = "SASL_SSL"
# "sasl.mechanism" = "PLAIN"
# "sasl.username" = "${KAFKA_USER}"
# "sasl.password" = "${KAFKA_PASSWORD}"

# Uncomment for SSL/TLS:
# "ssl.ca.location" = "/path/to/ca.pem"
# "ssl.certificate.location" = "/path/to/client.pem"
# "ssl.key.location" = "/path/to/client.key"

# Custom labels added to all metrics for this cluster
[clusters.labels]
environment = "production"
# datacenter = "us-east-1"

# Example: Second cluster configuration
# [[clusters]]
# name = "staging"
# bootstrap_servers = "kafka-staging:9092"
# group_whitelist = ["^staging-.*"]
# group_blacklist = []
# topic_whitelist = [".*"]
# topic_blacklist = ["^__.*"]
#
# [clusters.consumer_properties]
#
# [clusters.labels]
# environment = "staging"