diff --git a/charts/kminion/values.yaml b/charts/kminion/values.yaml index a5e210ba..d7631e2c 100644 --- a/charts/kminion/values.yaml +++ b/charts/kminion/values.yaml @@ -223,6 +223,12 @@ kminion: # retryInitConnection: false # # minion: +# clusterInfo: +# # Enabled specifies whether cluster information shall be scraped and exported or not. +# enabled: true +# brokerInfo: +# # Enabled specifies whether broker information shall be scraped and exported or not. +# enabled: true # consumerGroups: # # Enabled specifies whether consumer groups shall be scraped and exported or not. # enabled: true @@ -241,6 +247,8 @@ kminion: # # take precedence over allowed groups. # ignoredGroups: [ ] # topics: +# # Enabled specifies whether topics shall be scraped and exported or not. +# enabled: true # # Granularity can be per topic or per partition. If you want to reduce the number of exported metric series and # # you aren't interested in per partition metrics you could choose "topic". # granularity: partition diff --git a/docs/reference-config.yaml b/docs/reference-config.yaml index 402f4e1e..34cd7a92 100644 --- a/docs/reference-config.yaml +++ b/docs/reference-config.yaml @@ -66,6 +66,12 @@ kafka: scope: "" minion: + clusterInfo: + # Enabled specifies whether cluster information shall be scraped and exported or not. + enabled: true + brokerInfo: + # Enabled specifies whether broker information shall be scraped and exported or not. + enabled: true consumerGroups: # Enabled specifies whether consumer groups shall be scraped and exported or not. enabled: true @@ -90,6 +96,8 @@ minion: # take precedence over allowed groups. ignoredGroups: [ ] topics: + # Enabled specifies whether topics shall be scraped and exported or not. + enabled: true # Granularity can be per topic or per partition. If you want to reduce the number of exported metric series and # you aren't interested in per partition metrics you could choose "topic". granularity: partition diff --git a/minion/config.go b/minion/config.go index 3b26a760..17f8780d 100644 --- a/minion/config.go +++ b/minion/config.go @@ -7,6 +7,8 @@ import ( ) type Config struct { + ClusterInfo ClusterInfoConfig `koanf:"clusterInfo"` + BrokerInfo BrokerInfoConfig `koanf:"brokerInfo"` ConsumerGroups ConsumerGroupConfig `koanf:"consumerGroups"` Topics TopicConfig `koanf:"topics"` LogDirs LogDirsConfig `koanf:"logDirs"` @@ -21,7 +23,17 @@ func (c *Config) SetDefaults() { } func (c *Config) Validate() error { - err := c.ConsumerGroups.Validate() + err := c.ClusterInfo.Validate() + if err != nil { + return fmt.Errorf("failed to cluster info config: %w", err) + } + + err = c.BrokerInfo.Validate() + if err != nil { + return fmt.Errorf("failed to broker info config: %w", err) + } + + err = c.ConsumerGroups.Validate() if err != nil { return fmt.Errorf("failed to consumer group config: %w", err) } diff --git a/minion/config_broker_info.go b/minion/config_broker_info.go new file mode 100644 index 00000000..f24a86e5 --- /dev/null +++ b/minion/config_broker_info.go @@ -0,0 +1,14 @@ +package minion + +type BrokerInfoConfig struct { + // Enabled specifies whether broker information shall be scraped and exported or not. + Enabled bool `koanf:"enabled"` +} + +func (c *BrokerInfoConfig) SetDefaults() { + c.Enabled = true +} + +func (c *BrokerInfoConfig) Validate() error { + return nil +} diff --git a/minion/config_cluster_info.go b/minion/config_cluster_info.go new file mode 100644 index 00000000..cba68ba0 --- /dev/null +++ b/minion/config_cluster_info.go @@ -0,0 +1,14 @@ +package minion + +type ClusterInfoConfig struct { + // Enabled specifies whether cluster information shall be scraped and exported or not. + Enabled bool `koanf:"enabled"` +} + +func (c *ClusterInfoConfig) SetDefaults() { + c.Enabled = true +} + +func (c *ClusterInfoConfig) Validate() error { + return nil +} diff --git a/minion/config_topic_config.go b/minion/config_topic_config.go index 262300a3..f122ba1f 100644 --- a/minion/config_topic_config.go +++ b/minion/config_topic_config.go @@ -10,6 +10,9 @@ const ( ) type TopicConfig struct { + // Enabled specifies whether topics shall be scraped and exported or not. + Enabled bool `koanf:"enabled"` + // Granularity can be per topic or per partition. If you want to reduce the number of exported metric series and // you aren't interested in per partition metrics you could choose "topic". Granularity string `koanf:"granularity"` @@ -60,6 +63,7 @@ func (c *TopicConfig) Validate() error { // SetDefaults for topic config func (c *TopicConfig) SetDefaults() { + c.Enabled = true c.Granularity = TopicGranularityPartition c.AllowedTopics = []string{"/.*/"} c.InfoMetric = InfoMetricConfig{ConfigKeys: []string{"cleanup.policy"}} diff --git a/prometheus/collect_broker_info.go b/prometheus/collect_broker_info.go index 6aa0c701..75c275bf 100644 --- a/prometheus/collect_broker_info.go +++ b/prometheus/collect_broker_info.go @@ -2,12 +2,17 @@ package prometheus import ( "context" + "strconv" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "strconv" ) func (e *Exporter) collectBrokerInfo(ctx context.Context, ch chan<- prometheus.Metric) bool { + if !e.minionSvc.Cfg.BrokerInfo.Enabled { + return true + } + metadata, err := e.minionSvc.GetMetadataCached(ctx) if err != nil { e.logger.Error("failed to get kafka metadata", zap.Error(err)) diff --git a/prometheus/collect_cluster_info.go b/prometheus/collect_cluster_info.go index 10da5fd0..0414e247 100644 --- a/prometheus/collect_cluster_info.go +++ b/prometheus/collect_cluster_info.go @@ -2,12 +2,17 @@ package prometheus import ( "context" + "strconv" + "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "strconv" ) func (e *Exporter) collectClusterInfo(ctx context.Context, ch chan<- prometheus.Metric) bool { + if !e.minionSvc.Cfg.ClusterInfo.Enabled { + return true + } + version, err := e.minionSvc.GetClusterVersion(ctx) if err != nil { e.logger.Error("failed to get kafka cluster version", zap.Error(err)) diff --git a/prometheus/collect_consumer_group_lags.go b/prometheus/collect_consumer_group_lags.go index b61fd073..7d4fda1a 100644 --- a/prometheus/collect_consumer_group_lags.go +++ b/prometheus/collect_consumer_group_lags.go @@ -2,13 +2,14 @@ package prometheus import ( "context" + "math" + "strconv" + "github.com/cloudhut/kminion/v2/minion" "github.com/prometheus/client_golang/prometheus" "github.com/twmb/franz-go/pkg/kerr" "github.com/twmb/franz-go/pkg/kmsg" "go.uber.org/zap" - "math" - "strconv" ) type waterMark struct { @@ -19,6 +20,10 @@ type waterMark struct { } func (e *Exporter) collectConsumerGroupLags(ctx context.Context, ch chan<- prometheus.Metric) bool { + if !e.minionSvc.Cfg.ConsumerGroups.Enabled { + return true + } + // Low Watermarks (at the moment they are not needed at all, they could be used to calculate the lag on partitions // that don't have any active offsets) lowWaterMarks, err := e.minionSvc.ListOffsetsCached(ctx, -2) diff --git a/prometheus/collect_topic_info.go b/prometheus/collect_topic_info.go index 7474ec16..c5300ae5 100644 --- a/prometheus/collect_topic_info.go +++ b/prometheus/collect_topic_info.go @@ -10,6 +10,10 @@ import ( ) func (e *Exporter) collectTopicInfo(ctx context.Context, ch chan<- prometheus.Metric) bool { + if !e.minionSvc.Cfg.Topics.Enabled { + return true + } + metadata, err := e.minionSvc.GetMetadataCached(ctx) if err != nil { e.logger.Error("failed to get metadata", zap.Error(err)) diff --git a/prometheus/collect_topic_partition_offsets.go b/prometheus/collect_topic_partition_offsets.go index 993ab72e..e58b00dc 100644 --- a/prometheus/collect_topic_partition_offsets.go +++ b/prometheus/collect_topic_partition_offsets.go @@ -10,6 +10,10 @@ import ( ) func (e *Exporter) collectTopicPartitionOffsets(ctx context.Context, ch chan<- prometheus.Metric) bool { + if !e.minionSvc.Cfg.Topics.Enabled { + return true + } + isOk := true // Low Watermarks