Skip to content

Commit a08c6b8

Browse files
mruegmrueg
committed
feat: Use PartialObjectMetadata for Configmaps
1 parent f8aa7d9 commit a08c6b8

File tree

9 files changed

+162
-21
lines changed

9 files changed

+162
-21
lines changed

‎internal/store/builder.go

+65-5
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ import (
3838
policyv1 "k8s.io/api/policy/v1"
3939
rbacv1 "k8s.io/api/rbac/v1"
4040
storagev1 "k8s.io/api/storage/v1"
41+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
4142
clientset "k8s.io/client-go/kubernetes"
43+
"k8s.io/client-go/metadata"
4244
"k8s.io/client-go/tools/cache"
4345
"k8s.io/klog/v2"
4446

@@ -65,9 +67,10 @@ var _ ksmtypes.BuilderInterface = &Builder{}
6567
// Builder helps to build store. It follows the builder pattern
6668
// (https://en.wikipedia.org/wiki/Builder_pattern).
6769
type Builder struct {
68-
kubeClient clientset.Interface
69-
customResourceClients map[string]interface{}
70-
namespaces options.NamespaceList
70+
kubeClient clientset.Interface
71+
metadataOnlyKubeClient metadata.Interface
72+
customResourceClients map[string]interface{}
73+
namespaces options.NamespaceList
7174
// namespaceFilter is inside fieldSelectorFilter
7275
fieldSelectorFilter string
7376
ctx context.Context
@@ -78,6 +81,7 @@ type Builder struct {
7881
shard int32
7982
totalShards int
8083
buildStoresFunc ksmtypes.BuildStoresFunc
84+
buildMetadataOnlyStoresFunc ksmtypes.BuildMetadataOnlyStoresFunc
8185
buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
8286
allowAnnotationsList map[string][]string
8387
allowLabelsList map[string][]string
@@ -157,6 +161,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) {
157161
b.kubeClient = c
158162
}
159163

164+
// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder.
165+
func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) {
166+
b.metadataOnlyKubeClient = c
167+
}
168+
160169
// WithCustomResourceClients sets the customResourceClients property of a Builder.
161170
func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
162171
b.customResourceClients = cs
@@ -178,6 +187,11 @@ func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
178187
b.buildStoresFunc = f
179188
}
180189

190+
// WithGenerateMetadataOnlyStoresFunc configures a custom generate custom resource store function
191+
func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) {
192+
b.buildMetadataOnlyStoresFunc = f
193+
}
194+
181195
// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
182196
func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
183197
b.buildCustomResourceStoresFunc = f
@@ -188,6 +202,11 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
188202
return b.buildStores
189203
}
190204

205+
// DefaultGenerateMetadataOnlyStoresFunc returns default buildStores function
206+
func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc {
207+
return b.buildMetadataOnlyStores
208+
}
209+
191210
// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function
192211
func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
193212
return b.buildCustomResourceStores
@@ -362,7 +381,7 @@ func availableResources() []string {
362381
}
363382

364383
func (b *Builder) buildConfigMapStores() []cache.Store {
365-
return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache)
384+
return b.buildMetadataOnlyStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &metav1.PartialObjectMetadata{}, createConfigMapListWatch, b.useAPIServerCache)
366385
}
367386

368387
func (b *Builder) buildCronJobStores() []cache.Store {
@@ -519,7 +538,8 @@ func (b *Builder) buildStores(
519538
if b.fieldSelectorFilter != "" {
520539
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
521540
}
522-
listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
541+
kubeClient := b.kubeClient
542+
listWatcher := listWatchFunc(kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
523543
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
524544
return []cache.Store{store}
525545
}
@@ -541,6 +561,46 @@ func (b *Builder) buildStores(
541561
return stores
542562
}
543563

564+
func (b *Builder) buildMetadataOnlyStores(
565+
metricFamilies []generator.FamilyGenerator,
566+
expectedType interface{},
567+
listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher,
568+
useAPIServerCache bool,
569+
) []cache.Store {
570+
metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
571+
composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
572+
familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)
573+
574+
if b.namespaces.IsAllNamespaces() {
575+
store := metricsstore.NewMetricsStore(
576+
familyHeaders,
577+
composedMetricGenFuncs,
578+
)
579+
if b.fieldSelectorFilter != "" {
580+
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
581+
}
582+
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
583+
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
584+
return []cache.Store{store}
585+
}
586+
587+
stores := make([]cache.Store, 0, len(b.namespaces))
588+
for _, ns := range b.namespaces {
589+
store := metricsstore.NewMetricsStore(
590+
familyHeaders,
591+
composedMetricGenFuncs,
592+
)
593+
if b.fieldSelectorFilter != "" {
594+
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
595+
}
596+
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, ns, b.fieldSelectorFilter)
597+
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
598+
stores = append(stores, store)
599+
}
600+
601+
return stores
602+
}
603+
544604
// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores`
545605
func (b *Builder) buildCustomResourceStores(resourceName string,
546606
metricFamilies []generator.FamilyGenerator,

‎internal/store/configmap.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ package store
1919
import (
2020
"context"
2121

22-
v1 "k8s.io/api/core/v1"
2322
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2423
"k8s.io/apimachinery/pkg/runtime"
24+
"k8s.io/apimachinery/pkg/runtime/schema"
2525
"k8s.io/apimachinery/pkg/watch"
26-
clientset "k8s.io/client-go/kubernetes"
26+
"k8s.io/client-go/metadata"
2727
"k8s.io/client-go/tools/cache"
2828
basemetrics "k8s.io/component-base/metrics"
2929

@@ -43,7 +43,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
4343
metric.Gauge,
4444
basemetrics.ALPHA,
4545
"",
46-
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
46+
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
4747
if len(allowAnnotationsList) == 0 {
4848
return &metric.Family{}
4949
}
@@ -65,7 +65,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
6565
metric.Gauge,
6666
basemetrics.STABLE,
6767
"",
68-
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
68+
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
6969
if len(allowLabelsList) == 0 {
7070
return &metric.Family{}
7171
}
@@ -87,7 +87,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
8787
metric.Gauge,
8888
basemetrics.STABLE,
8989
"",
90-
wrapConfigMapFunc(func(_ *v1.ConfigMap) *metric.Family {
90+
wrapConfigMapFunc(func(_ *metav1.PartialObjectMetadata) *metric.Family {
9191
return &metric.Family{
9292
Metrics: []*metric.Metric{{
9393
LabelKeys: []string{},
@@ -103,7 +103,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
103103
metric.Gauge,
104104
basemetrics.STABLE,
105105
"",
106-
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
106+
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
107107
ms := []*metric.Metric{}
108108

109109
if !c.CreationTimestamp.IsZero() {
@@ -125,7 +125,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
125125
metric.Gauge,
126126
basemetrics.ALPHA,
127127
"",
128-
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
128+
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
129129
return &metric.Family{
130130
Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion),
131131
}
@@ -134,22 +134,22 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
134134
}
135135
}
136136

137-
func createConfigMapListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher {
137+
func createConfigMapListWatch(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher {
138138
return &cache.ListWatch{
139139
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
140140
opts.FieldSelector = fieldSelector
141-
return kubeClient.CoreV1().ConfigMaps(ns).List(context.TODO(), opts)
141+
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).List(context.TODO(), opts)
142142
},
143143
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
144144
opts.FieldSelector = fieldSelector
145-
return kubeClient.CoreV1().ConfigMaps(ns).Watch(context.TODO(), opts)
145+
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).Watch(context.TODO(), opts)
146146
},
147147
}
148148
}
149149

150-
func wrapConfigMapFunc(f func(*v1.ConfigMap) *metric.Family) func(interface{}) *metric.Family {
150+
func wrapConfigMapFunc(f func(*metav1.PartialObjectMetadata) *metric.Family) func(interface{}) *metric.Family {
151151
return func(obj interface{}) *metric.Family {
152-
configMap := obj.(*v1.ConfigMap)
152+
configMap := obj.(*metav1.PartialObjectMetadata)
153153

154154
metricFamily := f(configMap)
155155

‎internal/store/configmap_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package store
1919
import (
2020
"testing"
2121

22-
v1 "k8s.io/api/core/v1"
2322
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2423

2524
generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
@@ -37,7 +36,7 @@ func TestConfigMapStore(t *testing.T) {
3736
AllowLabelsList: []string{
3837
"app",
3938
},
40-
Obj: &v1.ConfigMap{
39+
Obj: &metav1.PartialObjectMetadata{
4140
ObjectMeta: metav1.ObjectMeta{
4241
Name: "configmap1",
4342
Namespace: "ns1",
@@ -73,7 +72,7 @@ func TestConfigMapStore(t *testing.T) {
7372
},
7473
},
7574
{
76-
Obj: &v1.ConfigMap{
75+
Obj: &metav1.PartialObjectMetadata{
7776
ObjectMeta: metav1.ObjectMeta{
7877
Name: "configmap2",
7978
Namespace: "ns2",

‎pkg/app/server.go

+7
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
266266

267267
storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache)
268268
storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc())
269+
storeBuilder.WithGenerateMetadataOnlyStoresFunc(storeBuilder.DefaultGenerateMetadataOnlyStoresFunc())
269270
proc.StartReaper()
270271

271272
storeBuilder.WithUtilOptions(opts)
@@ -275,6 +276,12 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
275276
}
276277
storeBuilder.WithKubeClient(kubeClient)
277278

279+
metadataOnlyKubeClient, err := util.CreateMetadataOnlyKubeClient(opts.Apiserver, opts.Kubeconfig)
280+
if err != nil {
281+
return fmt.Errorf("failed to create metadata-only client: %v", err)
282+
}
283+
storeBuilder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
284+
278285
storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
279286
if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil {
280287
return fmt.Errorf("failed to set up annotations allowlist: %v", err)

‎pkg/app/server_test.go

+18
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ import (
3838
"k8s.io/apimachinery/pkg/types"
3939
"k8s.io/apimachinery/pkg/watch"
4040
"k8s.io/client-go/kubernetes/fake"
41+
mfake "k8s.io/client-go/metadata/fake"
4142
"k8s.io/client-go/rest"
4243
"k8s.io/client-go/tools/cache"
4344
samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
@@ -65,6 +66,7 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
6566
)
6667

6768
kubeClient := fake.NewSimpleClientset()
69+
metadataOnlyKubeClient := mfake.NewSimpleMetadataClient(mfake.NewTestScheme())
6870

6971
if err := injectFixtures(kubeClient, fixtureMultiplier); err != nil {
7072
b.Errorf("error injecting resources: %v", err)
@@ -80,10 +82,12 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
8082
b.Fatal(err)
8183
}
8284
builder.WithKubeClient(kubeClient)
85+
builder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
8386
builder.WithSharding(0, 1)
8487
builder.WithContext(ctx)
8588
builder.WithNamespaces(options.DefaultNamespaces)
8689
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
90+
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())
8791

8892
allowDenyListFilter, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
8993
if err != nil {
@@ -141,6 +145,7 @@ func TestFullScrapeCycle(t *testing.T) {
141145
t.Parallel()
142146

143147
kubeClient := fake.NewSimpleClientset()
148+
metadataOnlyKubeClient := mfake.NewSimpleMetadataClient(mfake.NewTestScheme())
144149

145150
err := pod(kubeClient, 0)
146151
if err != nil {
@@ -157,8 +162,10 @@ func TestFullScrapeCycle(t *testing.T) {
157162
t.Fatal(err)
158163
}
159164
builder.WithKubeClient(kubeClient)
165+
builder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
160166
builder.WithNamespaces(options.DefaultNamespaces)
161167
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
168+
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())
162169

163170
l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
164171
if err != nil {
@@ -436,6 +443,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
436443
t.Parallel()
437444

438445
kubeClient := fake.NewSimpleClientset()
446+
metadataOnlyKubeClient := mfake.NewSimpleMetadataClient(mfake.NewTestScheme())
439447

440448
for i := 0; i < 10; i++ {
441449
err := pod(kubeClient, i)
@@ -459,10 +467,12 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
459467
t.Fatal(err)
460468
}
461469
unshardedBuilder.WithKubeClient(kubeClient)
470+
unshardedBuilder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
462471
unshardedBuilder.WithNamespaces(options.DefaultNamespaces)
463472
unshardedBuilder.WithFamilyGeneratorFilter(l)
464473
unshardedBuilder.WithAllowLabels(map[string][]string{})
465474
unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc())
475+
unshardedBuilder.WithGenerateMetadataOnlyStoresFunc(unshardedBuilder.DefaultGenerateMetadataOnlyStoresFunc())
466476

467477
unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false)
468478
unshardedHandler.ConfigureSharding(ctx, 0, 1)
@@ -475,10 +485,13 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
475485
t.Fatal(err)
476486
}
477487
shardedBuilder1.WithKubeClient(kubeClient)
488+
shardedBuilder1.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
489+
478490
shardedBuilder1.WithNamespaces(options.DefaultNamespaces)
479491
shardedBuilder1.WithFamilyGeneratorFilter(l)
480492
shardedBuilder1.WithAllowLabels(map[string][]string{})
481493
shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc())
494+
shardedBuilder1.WithGenerateMetadataOnlyStoresFunc(shardedBuilder1.DefaultGenerateMetadataOnlyStoresFunc())
482495

483496
shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false)
484497
shardedHandler1.ConfigureSharding(ctx, 0, 2)
@@ -491,10 +504,12 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
491504
t.Fatal(err)
492505
}
493506
shardedBuilder2.WithKubeClient(kubeClient)
507+
shardedBuilder2.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
494508
shardedBuilder2.WithNamespaces(options.DefaultNamespaces)
495509
shardedBuilder2.WithFamilyGeneratorFilter(l)
496510
shardedBuilder2.WithAllowLabels(map[string][]string{})
497511
shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc())
512+
shardedBuilder2.WithGenerateMetadataOnlyStoresFunc(shardedBuilder2.DefaultGenerateMetadataOnlyStoresFunc())
498513

499514
shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false)
500515
shardedHandler2.ConfigureSharding(ctx, 1, 2)
@@ -606,6 +621,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
606621
// We use custom resource object samplev1alpha1.Foo in kubernetes/sample-controller as an example.
607622
func TestCustomResourceExtension(t *testing.T) {
608623
kubeClient := fake.NewSimpleClientset()
624+
metadataOnlyKubeClient := mfake.NewSimpleMetadataClient(mfake.NewTestScheme())
609625
factories := []customresource.RegistryFactory{new(fooFactory)}
610626
resources := options.DefaultResources.AsSlice()
611627
customResourceClients := make(map[string]interface{}, len(factories))
@@ -632,9 +648,11 @@ func TestCustomResourceExtension(t *testing.T) {
632648
}
633649

634650
builder.WithKubeClient(kubeClient)
651+
builder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)
635652
builder.WithCustomResourceClients(customResourceClients)
636653
builder.WithNamespaces(options.DefaultNamespaces)
637654
builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
655+
builder.WithGenerateMetadataOnlyStoresFunc(builder.DefaultGenerateMetadataOnlyStoresFunc())
638656
builder.WithGenerateCustomResourceStoresFunc(builder.DefaultGenerateCustomResourceStoresFunc())
639657

640658
l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})

0 commit comments

Comments
 (0)