Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ linters-settings:
- kubevirt.io
- sigs.k8s.io
- github.com/pkg/errors
- github.com/prometheus/client_golang
- github.com/prometheus/common

mnd:
settings:
Expand Down
15 changes: 14 additions & 1 deletion config/monitoring/prometheus-rule.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,17 @@ metadata:
prometheus.kubemacpool.io: "true"
name: prometheus-rule
namespace: kubemacpool-system
spec: {}
spec:
groups:
- name: alerts.rules
rules:
- alert: KubemacpoolMACCollisionDetected
annotations:
description: '{{ $value }} MAC address(es) have collisions. Multiple running
objects are using the same MAC address.'
summary: MAC address collisions detected.
expr: count(kmp_mac_collisions > 1) > 0
for: 30s
labels:
operator_health_impact: warning
severity: warning
15 changes: 14 additions & 1 deletion config/release/kubemacpool.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,20 @@ metadata:
prometheus.kubemacpool.io: "true"
name: kubemacpool-prometheus-rule
namespace: kubemacpool-system
spec: {}
spec:
groups:
- name: alerts.rules
rules:
- alert: KubemacpoolMACCollisionDetected
annotations:
description: '{{ $value }} MAC address(es) have collisions. Multiple running
objects are using the same MAC address.'
summary: MAC address collisions detected.
expr: count(kmp_mac_collisions > 1) > 0
for: 30s
labels:
operator_health_impact: warning
severity: warning
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
Expand Down
15 changes: 14 additions & 1 deletion config/test/kubemacpool.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,20 @@ metadata:
prometheus.kubemacpool.io: "true"
name: kubemacpool-prometheus-rule
namespace: kubemacpool-system
spec: {}
spec:
groups:
- name: alerts.rules
rules:
- alert: KubemacpoolMACCollisionDetected
annotations:
description: '{{ $value }} MAC address(es) have collisions. Multiple running
objects are using the same MAC address.'
summary: MAC address collisions detected.
expr: count(kmp_mac_collisions > 1) > 0
for: 30s
labels:
operator_health_impact: warning
severity: warning
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
Expand Down
7 changes: 5 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ require (
github.com/onsi/gomega v1.36.1
github.com/pkg/errors v0.9.1
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.88.1
github.com/prometheus/client_golang v1.22.0
github.com/prometheus/client_model v0.6.1
github.com/prometheus/common v0.62.0
github.com/qinqon/kube-admission-webhook v0.21.1
github.com/rhobs/operator-observability-toolkit v0.0.30
gomodules.xyz/jsonpatch/v2 v2.4.0
Expand Down Expand Up @@ -54,15 +56,15 @@ require (
github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/openshift/custom-resource-status v1.1.2 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.22.0 // indirect
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/spf13/pflag v1.0.6 // indirect
go.uber.org/multierr v1.11.0 // indirect
Expand All @@ -80,6 +82,7 @@ require (
google.golang.org/protobuf v1.36.5 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
kubevirt.io/containerized-data-importer-api v1.60.3-0.20241105012228-50fbed985de9 // indirect
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1947,6 +1947,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down Expand Up @@ -2080,6 +2081,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
Expand Down Expand Up @@ -3588,6 +3590,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
52 changes: 52 additions & 0 deletions pkg/monitoring/rules/alerts/alerts.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Copyright 2025 The KubeMacPool Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package alerts

import (
promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
"github.com/rhobs/operator-observability-toolkit/pkg/operatorrules"
"k8s.io/apimachinery/pkg/util/intstr"
)

const (
severityAlertLabelKey = "severity"
operatorHealthImpactLabelKey = "operator_health_impact"
)

func Register() error {
return operatorrules.RegisterAlerts(
vmiCollisionAlerts(),
)
}

func vmiCollisionAlerts() []promv1.Rule {
return []promv1.Rule{
{
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider adding a for duration to prevent alert flapping:

For: promv1.Duration("5m"),

This gives the system time to self-heal before alerting operators.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking that MAC collision is something that shouldn't wait for 5m.
Maybe something smaller like 10s?

Copy link
Member Author

@RamLavi RamLavi Jan 29, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added 30s.
We are handling with controllers and caches - we should avoid false transient spikes due to stale informers..

Alert: "KubemacpoolMACCollisionDetected",
Expr: intstr.FromString("count(kmp_mac_collisions > 1) > 0"),
For: promv1.DurationPointer("30s"),
Annotations: map[string]string{
"summary": "MAC address collisions detected.",
"description": "{{ $value }} MAC address(es) have collisions. Multiple running objects are using the same MAC address.",
},
Labels: map[string]string{
severityAlertLabelKey: "warning",
operatorHealthImpactLabelKey: "warning",
},
},
}
}
4 changes: 3 additions & 1 deletion pkg/monitoring/rules/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,16 @@ package rules
import (
promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
"github.com/rhobs/operator-observability-toolkit/pkg/operatorrules"

"github.com/k8snetworkplumbingwg/kubemacpool/pkg/monitoring/rules/alerts"
)

const (
PrometheusRuleName = "prometheus-rule"
)

func SetupRules() error {
return nil
return alerts.Register()
}

func BuildPrometheusRule(namespace string) (*promv1.PrometheusRule, error) {
Expand Down
Loading