Skip to content

Commit 8c46c28

Browse files
AndrewChubatiukAndrewChubatiuk
committed
add operator_bad_objects_count metric for vmauth and vmagent, additionally unified it
1 parent 5289f17 commit 8c46c28

23 files changed

+375
-403
lines changed

‎docs/CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ aliases:
1313

1414
## tip
1515

16-
* FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): add `object_namespace` label to the `operator_alertmanager_bad_objects_count` and `operator_vmalert_bad_objects_count` metrics.
16+
* FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): add unique `operator_bad_objects_count` metric with `object_namespace` and `crd` labels to track invalid objects managed by VMAgent, VMAuth, VMAlert and VMAlertmanager. Removed old `operator_alertmanager_bad_objects_count` and `operator_vmalert_bad_objects_count` metrics.
1717

1818
## [v0.66.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.66.1)
1919

‎internal/controller/operator/factory/build/metrics.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package build
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"sigs.k8s.io/controller-runtime/pkg/metrics"
6+
)
7+
8+
var BadObjectsTotal = prometheus.NewCounterVec(
9+
prometheus.CounterOpts{
10+
Name: "operator_bad_objects_count",
11+
Help: "Number of incorrect objects by controller",
12+
},
13+
[]string{"crd", "object_namespace"},
14+
)
15+
16+
func init() {
17+
metrics.Registry.MustRegister(BadObjectsTotal)
18+
}

‎internal/controller/operator/factory/vmagent/collect_scrapes.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger"
1414
)
1515

16-
func selectScrapeConfig(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMScrapeConfig, error) {
16+
func selectScrapeConfigs(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMScrapeConfig, error) {
1717
if cr.Spec.DaemonSetMode {
1818
return nil, nil
1919
}
@@ -72,7 +72,7 @@ func selectPodScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client
7272
return selectedConfigs, nil
7373
}
7474

75-
func selectVMProbes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMProbe, error) {
75+
func selectProbes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMProbe, error) {
7676
if cr.Spec.DaemonSetMode {
7777
return nil, nil
7878
}
@@ -102,7 +102,7 @@ func selectVMProbes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.C
102102
return selectedConfigs, nil
103103
}
104104

105-
func selectVMNodeScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMNodeScrape, error) {
105+
func selectNodeScrapes(ctx context.Context, cr *vmv1beta1.VMAgent, rclient client.Client) ([]*vmv1beta1.VMNodeScrape, error) {
106106
if cr.Spec.DaemonSetMode {
107107
return nil, nil
108108
}

‎internal/controller/operator/factory/vmagent/collect_scrapes_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,7 +341,7 @@ func TestSelectPodMonitors(t *testing.T) {
341341
})
342342
}
343343

344-
func TestSelectVMProbes(t *testing.T) {
344+
func TestSelectProbes(t *testing.T) {
345345
type opts struct {
346346
cr *vmv1beta1.VMAgent
347347
want []string
@@ -351,9 +351,9 @@ func TestSelectVMProbes(t *testing.T) {
351351
f := func(o opts) {
352352
t.Helper()
353353
fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects)
354-
got, err := selectVMProbes(context.TODO(), o.cr, fclient)
354+
got, err := selectProbes(context.TODO(), o.cr, fclient)
355355
if err != nil {
356-
t.Errorf("SelectVMProbes() error = %v", err)
356+
t.Errorf("SelectProbes() error = %v", err)
357357
return
358358
}
359359
var result []string
@@ -362,7 +362,7 @@ func TestSelectVMProbes(t *testing.T) {
362362
}
363363
sort.Strings(result)
364364
if !reflect.DeepEqual(result, o.want) {
365-
t.Errorf("SelectVMProbes(): %s", cmp.Diff(got, o.want))
365+
t.Errorf("SelectProbes(): %s", cmp.Diff(got, o.want))
366366
}
367367
}
368368

‎internal/controller/operator/factory/vmagent/nodescrape.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ func generateNodeScrapeConfig(
1414
ctx context.Context,
1515
cr *vmv1beta1.VMAgent,
1616
sc *vmv1beta1.VMNodeScrape,
17-
apiserverConfig *vmv1beta1.APIServerConfig,
1817
ac *build.AssetsCache,
19-
se vmv1beta1.VMAgentSecurityEnforcements,
2018
) (yaml.MapSlice, error) {
21-
nodeSpec := &sc.Spec
19+
spec := &sc.Spec
20+
apiserverConfig := cr.Spec.APIServerConfig
21+
se := cr.Spec.VMAgentSecurityEnforcements
2222
cfg := yaml.MapSlice{
2323
{
2424
Key: "job_name",
@@ -28,11 +28,11 @@ func generateNodeScrapeConfig(
2828

2929
scrapeClass := getScrapeClass(sc.Spec.ScrapeClassName, cr)
3030
if scrapeClass != nil {
31-
mergeEndPointAuthWithScrapeClass(&sc.Spec.EndpointAuth, scrapeClass)
31+
mergeEndpointAuthWithScrapeClass(&sc.Spec.EndpointAuth, scrapeClass)
3232
mergeEndpointRelabelingsWithScrapeClass(&sc.Spec.EndpointRelabelings, scrapeClass)
3333
}
3434

35-
setScrapeIntervalToWithLimit(ctx, &nodeSpec.EndpointScrapeParams, cr)
35+
setScrapeIntervalToWithLimit(ctx, &spec.EndpointScrapeParams, cr)
3636

3737
k8sSDOpts := generateK8SSDConfigOptions{
3838
shouldAddSelectors: cr.Spec.EnableKubernetesAPISelectors,
@@ -47,12 +47,12 @@ func generateNodeScrapeConfig(
4747
cfg = append(cfg, c...)
4848
}
4949

50-
cfg = addCommonScrapeParamsTo(cfg, nodeSpec.EndpointScrapeParams, se)
50+
cfg = addCommonScrapeParamsTo(cfg, spec.EndpointScrapeParams, se)
5151

5252
var relabelings []yaml.MapSlice
5353

5454
skipRelabelSelectors := cr.Spec.EnableKubernetesAPISelectors
55-
relabelings = addSelectorToRelabelingFor(relabelings, "node", nodeSpec.Selector, skipRelabelSelectors)
55+
relabelings = addSelectorToRelabelingFor(relabelings, "node", spec.Selector, skipRelabelSelectors)
5656
// Add __address__ as internalIP and pod and service labels into proper labels.
5757
relabelings = append(relabelings, []yaml.MapSlice{
5858
{
@@ -90,16 +90,16 @@ func generateNodeScrapeConfig(
9090
})
9191
}
9292

93-
if nodeSpec.Port != "" {
93+
if spec.Port != "" {
9494
relabelings = append(relabelings, yaml.MapSlice{
9595
{Key: "source_labels", Value: []string{"__address__"}},
9696
{Key: "target_label", Value: "__address__"},
9797
{Key: "regex", Value: "^(.*):(.*)"},
98-
{Key: "replacement", Value: fmt.Sprintf("${1}:%s", nodeSpec.Port)},
98+
{Key: "replacement", Value: fmt.Sprintf("${1}:%s", spec.Port)},
9999
})
100100
}
101101

102-
for _, c := range nodeSpec.RelabelConfigs {
102+
for _, c := range spec.RelabelConfigs {
103103
relabelings = append(relabelings, generateRelabelConfig(c))
104104
}
105105
for _, trc := range cr.Spec.NodeScrapeRelabelTemplate {
@@ -111,11 +111,11 @@ func generateNodeScrapeConfig(
111111
relabelings = enforceNamespaceLabel(relabelings, sc.Namespace, se.EnforcedNamespaceLabel)
112112

113113
cfg = append(cfg, yaml.MapItem{Key: "relabel_configs", Value: relabelings})
114-
cfg = addMetricRelabelingsTo(cfg, nodeSpec.MetricRelabelConfigs, se)
114+
cfg = addMetricRelabelingsTo(cfg, spec.MetricRelabelConfigs, se)
115115
if c, err := buildVMScrapeParams(sc.Namespace, sc.Spec.VMScrapeParams, ac); err != nil {
116116
return nil, err
117117
} else {
118118
cfg = append(cfg, c...)
119119
}
120-
return addEndpointAuthTo(cfg, &nodeSpec.EndpointAuth, sc.Namespace, ac)
120+
return addEndpointAuthTo(cfg, &spec.EndpointAuth, sc.Namespace, ac)
121121
}

‎internal/controller/operator/factory/vmagent/nodescrape_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func Test_generateNodeScrapeConfig(t *testing.T) {
2727
ctx := context.Background()
2828
fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects)
2929
ac := getAssetsCache(ctx, fclient, o.cr)
30-
got, err := generateNodeScrapeConfig(ctx, o.cr, o.sc, nil, ac, o.cr.Spec.VMAgentSecurityEnforcements)
30+
got, err := generateNodeScrapeConfig(ctx, o.cr, o.sc, ac)
3131
if err != nil {
3232
t.Errorf("cannot generate NodeScrapeConfig, err: %e", err)
3333
return

‎internal/controller/operator/factory/vmagent/podscrape.go‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,33 @@ func generatePodScrapeConfig(
1616
sc *vmv1beta1.VMPodScrape,
1717
ep vmv1beta1.PodMetricsEndpoint,
1818
i int,
19-
apiserverConfig *vmv1beta1.APIServerConfig,
2019
ac *build.AssetsCache,
21-
se vmv1beta1.VMAgentSecurityEnforcements,
2220
) (yaml.MapSlice, error) {
21+
spec := &sc.Spec
22+
apiserverConfig := cr.Spec.APIServerConfig
23+
se := cr.Spec.VMAgentSecurityEnforcements
2324
cfg := yaml.MapSlice{
2425
{
2526
Key: "job_name",
2627
Value: fmt.Sprintf("podScrape/%s/%s/%d", sc.Namespace, sc.Name, i),
2728
},
2829
}
2930

30-
scrapeClass := getScrapeClass(sc.Spec.ScrapeClassName, cr)
31+
scrapeClass := getScrapeClass(spec.ScrapeClassName, cr)
3132
if scrapeClass != nil {
32-
mergeEndPointAuthWithScrapeClass(&ep.EndpointAuth, scrapeClass)
33+
mergeEndpointAuthWithScrapeClass(&ep.EndpointAuth, scrapeClass)
3334
mergeEndpointRelabelingsWithScrapeClass(&ep.EndpointRelabelings, scrapeClass)
3435
mergeAttachMetadataWithScrapeClass(&ep.AttachMetadata, scrapeClass)
3536
}
3637

37-
selectedNamespaces := getNamespacesFromNamespaceSelector(&sc.Spec.NamespaceSelector, sc.Namespace, se.IgnoreNamespaceSelectors)
38-
if ep.AttachMetadata.Node == nil && sc.Spec.AttachMetadata.Node != nil {
39-
ep.AttachMetadata = sc.Spec.AttachMetadata
38+
selectedNamespaces := getNamespacesFromNamespaceSelector(&spec.NamespaceSelector, sc.Namespace, se.IgnoreNamespaceSelectors)
39+
if ep.AttachMetadata.Node == nil && spec.AttachMetadata.Node != nil {
40+
ep.AttachMetadata = spec.AttachMetadata
4041
}
4142
k8sSDOpts := generateK8SSDConfigOptions{
4243
namespaces: selectedNamespaces,
4344
shouldAddSelectors: cr.Spec.EnableKubernetesAPISelectors,
44-
selectors: sc.Spec.Selector,
45+
selectors: spec.Selector,
4546
apiServerConfig: apiserverConfig,
4647
role: kubernetesSDRolePod,
4748
attachMetadata: &ep.AttachMetadata,
@@ -58,10 +59,10 @@ func generatePodScrapeConfig(
5859

5960
// set defaults
6061
if ep.SampleLimit == 0 {
61-
ep.SampleLimit = sc.Spec.SampleLimit
62+
ep.SampleLimit = spec.SampleLimit
6263
}
6364
if ep.SeriesLimit == 0 {
64-
ep.SeriesLimit = sc.Spec.SeriesLimit
65+
ep.SeriesLimit = spec.SeriesLimit
6566
}
6667

6768
setScrapeIntervalToWithLimit(ctx, &ep.EndpointScrapeParams, cr)
@@ -79,7 +80,7 @@ func generatePodScrapeConfig(
7980
}
8081

8182
skipRelabelSelectors := cr.Spec.EnableKubernetesAPISelectors
82-
relabelings = addSelectorToRelabelingFor(relabelings, "pod", sc.Spec.Selector, skipRelabelSelectors)
83+
relabelings = addSelectorToRelabelingFor(relabelings, "pod", spec.Selector, skipRelabelSelectors)
8384

8485
// Filter targets based on correct port for the endpoint.
8586
switch {
@@ -129,7 +130,7 @@ func generatePodScrapeConfig(
129130
}...)
130131

131132
// Relabel targetLabels from Pod onto target.
132-
for _, l := range sc.Spec.PodTargetLabels {
133+
for _, l := range spec.PodTargetLabels {
133134
relabelings = append(relabelings, yaml.MapSlice{
134135
{Key: "source_labels", Value: []string{"__meta_kubernetes_pod_label_" + sanitizeLabelName(l)}},
135136
{Key: "target_label", Value: sanitizeLabelName(l)},
@@ -148,9 +149,9 @@ func generatePodScrapeConfig(
148149
{Key: "target_label", Value: "job"},
149150
{Key: "replacement", Value: fmt.Sprintf("%s/%s", sc.GetNamespace(), sc.GetName())},
150151
})
151-
if sc.Spec.JobLabel != "" {
152+
if spec.JobLabel != "" {
152153
relabelings = append(relabelings, yaml.MapSlice{
153-
{Key: "source_labels", Value: []string{"__meta_kubernetes_pod_label_" + sanitizeLabelName(sc.Spec.JobLabel)}},
154+
{Key: "source_labels", Value: []string{"__meta_kubernetes_pod_label_" + sanitizeLabelName(spec.JobLabel)}},
154155
{Key: "target_label", Value: "job"},
155156
{Key: "regex", Value: "(.+)"},
156157
{Key: "replacement", Value: "${1}"},

‎internal/controller/operator/factory/vmagent/podscrape_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func Test_generatePodScrapeConfig(t *testing.T) {
2626
ctx := context.Background()
2727
fclient := k8stools.GetTestClientWithObjects(nil)
2828
ac := getAssetsCache(ctx, fclient, o.cr)
29-
got, err := generatePodScrapeConfig(ctx, o.cr, o.sc, o.ep, 0, nil, ac, o.cr.Spec.VMAgentSecurityEnforcements)
29+
got, err := generatePodScrapeConfig(ctx, o.cr, o.sc, o.ep, 0, ac)
3030
if err != nil {
3131
t.Errorf("cannot generate PodScrapeConfig, err: %e", err)
3232
return

0 commit comments

Comments
 (0)