Skip to content
This repository was archived by the owner on Nov 27, 2024. It is now read-only.

Commit 68daa44

Browse files
sadlerapsadlerap
authored
remove kube-rbac-proxy as a dependency (#288)
We need to set up a ServiceMonitor for our service. We could use kube-rbac-proxy, but that has apparently been deprecated by kubebuilder, and they recommend moving away from it. Instead, we can do the authentication and authorization checks inside our metrics server. To do this, we need to set up the ServiceMonitor with a ServiceAccount with permissions to talk to `/metrics` from our metrics service. Signed-off-by: Andy Sadler <ansadler@redhat.com>
1 parent 4c2b427 commit 68daa44

11 files changed

Lines changed: 192 additions & 53 deletions

‎operator/cmd/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
ctrl "sigs.k8s.io/controller-runtime"
3232
"sigs.k8s.io/controller-runtime/pkg/healthz"
3333
"sigs.k8s.io/controller-runtime/pkg/log/zap"
34+
"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
3435
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
3536
"sigs.k8s.io/controller-runtime/pkg/webhook"
3637

@@ -86,7 +87,8 @@ func main() {
8687
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
8788
Scheme: scheme,
8889
Metrics: metricsserver.Options{
89-
BindAddress: metricsAddr,
90+
BindAddress: metricsAddr,
91+
FilterProvider: filters.WithAuthenticationAndAuthorization,
9092
},
9193
WebhookServer: webhook.NewServer(webhook.Options{
9294
Port: 944,

‎operator/config/default/kustomization.yaml‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,41 @@ resources:
66
- ../crd
77
- ../rbac
88
- ../manager
9-
- metrics-service.yaml
9+
- ../metrics
1010
patches:
1111
- path: manager_auth_proxy_patch.yaml
12+
replacements:
13+
- source:
14+
fieldPath: metadata.name
15+
kind: Secret
16+
name: metrics-reader
17+
targets:
18+
- fieldPaths:
19+
- spec.endpoints.*.authorization.credentials.name
20+
select:
21+
group: monitoring.coreos.com
22+
kind: ServiceMonitor
23+
name: metrics-proxy
24+
- source:
25+
fieldPath: metadata.name
26+
kind: ServiceAccount
27+
name: metrics-reader
28+
targets:
29+
- fieldPaths:
30+
- metadata.annotations.[kubernetes.io/service-account.name]
31+
select:
32+
kind: Secret
33+
name: metrics-reader
34+
options:
35+
create: true
36+
37+
# We need to make the secrets containing the service account tokens before the
38+
# service monitor can start checking for metrics. To ensure this, let's make
39+
# the service account and its secrets before anything else.
40+
sortOptions:
41+
order: legacy
42+
legacySortOptions:
43+
orderFirst:
44+
- Namespace
45+
- ServiceAccount
46+
- Secret

‎operator/config/default/manager_auth_proxy_patch.yaml‎

Lines changed: 1 addition & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -23,32 +23,8 @@ spec:
2323
values:
2424
- linux
2525
containers:
26-
- name: kube-rbac-proxy
27-
securityContext:
28-
allowPrivilegeEscalation: false
29-
readOnlyRootFilesystem: true
30-
capabilities:
31-
drop:
32-
- "ALL"
33-
image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1
34-
args:
35-
- "--secure-listen-address=0.0.0.0:8443"
36-
- "--upstream=http://127.0.0.1:8080/"
37-
- "--logtostderr=true"
38-
- "--v=0"
39-
ports:
40-
- containerPort: 8443
41-
protocol: TCP
42-
name: https
43-
resources:
44-
limits:
45-
cpu: 500m
46-
memory: 128Mi
47-
requests:
48-
cpu: 5m
49-
memory: 64Mi
5026
- name: manager
5127
args:
5228
- "--health-probe-bind-address=:8081"
53-
- "--metrics-bind-address=127.0.0.1:8080"
29+
- "--metrics-bind-address=0.0.0.0:8080"
5430
- "--leader-elect"

‎operator/config/default/metrics-service.yaml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎operator/config/metrics/kustomization.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
resources:
4+
- monitor.yaml
5+
- service-account.yaml
6+
- metrics-service.yaml
7+
secretGenerator:
8+
- name: metrics-reader
9+
type: "kubernetes.io/service-account-token"
10+
generatorOptions:
11+
disableNameSuffixHash: true

operator/config/rbac/auth_proxy_service.yaml renamed to operator/config/metrics/metrics-service.yaml

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,23 @@
11
apiVersion: v1
22
kind: Service
33
metadata:
4+
name: controller-manager-metrics
5+
namespace: system
46
labels:
57
control-plane: controller-manager
6-
app.kubernetes.io/name: service
7-
app.kubernetes.io/instance: controller-manager-metrics-service
8-
app.kubernetes.io/component: kube-rbac-proxy
8+
app.kubernetes.io/name: servicemonitor
9+
app.kubernetes.io/instance: controller-manager-metrics-monitor
10+
app.kubernetes.io/component: metrics
911
app.kubernetes.io/created-by: workspaces
1012
app.kubernetes.io/part-of: workspaces
1113
app.kubernetes.io/managed-by: kustomize
12-
name: controller-manager-metrics-service
13-
namespace: system
1414
spec:
1515
ports:
16-
- name: https
17-
port: 8443
16+
- name: metrics
1817
protocol: TCP
19-
targetPort: https
18+
port: 8080
19+
targetPort: 8080
2020
selector:
2121
control-plane: controller-manager
22+
type: ClusterIP
23+
sessionAffinity: None

‎operator/config/metrics/monitor.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
name: metrics-proxy
5+
namespace: system
6+
labels:
7+
control-plane: controller-manager
8+
app.kubernetes.io/name: servicemonitor
9+
app.kubernetes.io/instance: controller-manager-metrics-monitor
10+
app.kubernetes.io/component: metrics
11+
app.kubernetes.io/created-by: workspaces
12+
app.kubernetes.io/part-of: workspaces
13+
app.kubernetes.io/managed-by: kustomize
14+
spec:
15+
endpoints:
16+
- interval: 15s
17+
scheme: http
18+
path: /metrics
19+
port: metrics
20+
authorization:
21+
credentials:
22+
key: token
23+
name: metrics-reader
24+
selector:
25+
matchLabels:
26+
control-plane: controller-manager
27+
app.kubernetes.io/component: metrics

‎operator/config/metrics/service-account.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: metrics-reader
5+
namespace: system
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: ClusterRoleBinding
9+
metadata:
10+
name: metrics-reader
11+
roleRef:
12+
apiGroup: rbac.authorization.k8s.io
13+
kind: ClusterRole
14+
name: metrics-reader
15+
subjects:
16+
- kind: ServiceAccount
17+
name: metrics-reader
18+
namespace: system

‎operator/config/rbac/kustomization.yaml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ resources:
1212
# Comment the following 4 lines if you want to disable
1313
# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
1414
# which protects your /metrics endpoint.
15-
# - auth_proxy_service.yaml
16-
# - auth_proxy_role.yaml
17-
# - auth_proxy_role_binding.yaml
18-
# - auth_proxy_client_clusterrole.yaml
15+
- auth_proxy_role.yaml
16+
- auth_proxy_role_binding.yaml
17+
- auth_proxy_client_clusterrole.yaml

‎operator/go.mod‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,21 @@ require (
1414
)
1515

1616
require (
17+
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
18+
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
1719
github.com/beorn7/perks v1.0.1 // indirect
20+
github.com/blang/semver/v4 v4.0.0 // indirect
21+
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
1822
github.com/cespare/xxhash/v2 v2.3.0 // indirect
1923
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
2024
github.com/emicklei/go-restful/v3 v3.11.2 // indirect
2125
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
2226
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
27+
github.com/felixge/httpsnoop v1.0.4 // indirect
2328
github.com/fsnotify/fsnotify v1.7.0 // indirect
2429
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
2530
github.com/go-logr/logr v1.4.2 // indirect
31+
github.com/go-logr/stdr v1.2.2 // indirect
2632
github.com/go-logr/zapr v1.3.0 // indirect
2733
github.com/go-openapi/jsonpointer v0.20.2 // indirect
2834
github.com/go-openapi/jsonreference v0.20.4 // indirect
@@ -31,12 +37,15 @@ require (
3137
github.com/gogo/protobuf v1.3.2 // indirect
3238
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3339
github.com/golang/protobuf v1.5.4 // indirect
40+
github.com/google/cel-go v0.20.1 // indirect
3441
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
3542
github.com/google/go-cmp v0.6.0 // indirect
3643
github.com/google/gofuzz v1.2.0 // indirect
3744
github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect
3845
github.com/google/uuid v1.6.0 // indirect
46+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
3947
github.com/imdario/mergo v0.3.16 // indirect
48+
github.com/inconshreveable/mousetrap v1.1.0 // indirect
4049
github.com/josharian/intern v1.0.0 // indirect
4150
github.com/json-iterator/go v1.1.12 // indirect
4251
github.com/klauspost/compress v1.17.9 // indirect
@@ -49,28 +58,45 @@ require (
4958
github.com/prometheus/client_model v0.6.1 // indirect
5059
github.com/prometheus/common v0.55.0 // indirect
5160
github.com/prometheus/procfs v0.15.1 // indirect
61+
github.com/spf13/cobra v1.8.1 // indirect
5262
github.com/spf13/pflag v1.0.5 // indirect
63+
github.com/stoewer/go-strcase v1.2.0 // indirect
5364
github.com/x448/float16 v0.8.4 // indirect
65+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
66+
go.opentelemetry.io/otel v1.28.0 // indirect
67+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
68+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
69+
go.opentelemetry.io/otel/metric v1.28.0 // indirect
70+
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
71+
go.opentelemetry.io/otel/trace v1.28.0 // indirect
72+
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
5473
go.uber.org/multierr v1.11.0 // indirect
5574
go.uber.org/zap v1.26.0 // indirect
5675
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
5776
golang.org/x/net v0.28.0 // indirect
5877
golang.org/x/oauth2 v0.21.0 // indirect
78+
golang.org/x/sync v0.8.0 // indirect
5979
golang.org/x/sys v0.24.0 // indirect
6080
golang.org/x/term v0.23.0 // indirect
6181
golang.org/x/text v0.17.0 // indirect
6282
golang.org/x/time v0.5.0 // indirect
6383
golang.org/x/tools v0.24.0 // indirect
6484
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
85+
google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
86+
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
87+
google.golang.org/grpc v1.65.0 // indirect
6588
google.golang.org/protobuf v1.34.2 // indirect
6689
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
6790
gopkg.in/inf.v0 v0.9.1 // indirect
6891
gopkg.in/yaml.v2 v2.4.0 // indirect
6992
gopkg.in/yaml.v3 v3.0.1 // indirect
7093
k8s.io/apiextensions-apiserver v0.31.0 // indirect
94+
k8s.io/apiserver v0.31.0 // indirect
95+
k8s.io/component-base v0.31.0 // indirect
7196
k8s.io/klog/v2 v2.130.1 // indirect
7297
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
7398
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
99+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
74100
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
75101
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
76102
sigs.k8s.io/yaml v1.4.0 // indirect

0 commit comments

Comments
 (0)