Skip to content

Commit 0b80cb1

Browse files
jatcod3rjatcod3r
authored
Merge pull request #91 from coder/jatcod3r/kyverno-self-managed
Migrate Self-Managed Objects into Custom Kyverno Chart
2 parents 417566e + ea74892 commit 0b80cb1

4 files changed

Lines changed: 264 additions & 42 deletions

File tree

‎charts/kyverno/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
apiVersion: v2
22
name: kyverno
3-
description: A Helm chart for Kubernetes
3+
description: A Helm chart for Kyverno and auxillary resources.
44
type: application
55
version: 0.1.0
66
appVersion: 0.1.0

‎charts/kyverno/templates/warmNodes.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{{- if .Values.warmNodes.enabled }}
2+
{{- range .Values.warmNodes.daemonSets }}
3+
---
4+
apiVersion: apps/v1
5+
kind: DaemonSet
6+
metadata:
7+
name: {{ .name }}
8+
namespace: {{ $.Values.warmNodes.namespace | default "default" }}
9+
{{- with $.Values.warmNodes.labels }}
10+
labels:
11+
{{- toYaml . | nindent 4 }}
12+
{{- end }}
13+
{{- with $.Values.warmNodes.annotations }}
14+
annotations:
15+
{{- toYaml . | nindent 4 }}
16+
{{- end }}
17+
spec:
18+
selector:
19+
matchLabels:
20+
{{- toYaml $.Values.warmNodes.selectorLabels | nindent 6 }}
21+
template:
22+
metadata:
23+
labels:
24+
{{- toYaml $.Values.warmNodes.selectorLabels | nindent 8 }}
25+
spec:
26+
{{- with .nodeSelector }}
27+
nodeSelector:
28+
{{- toYaml . | nindent 8 }}
29+
{{- end }}
30+
31+
{{- with .tolerations }}
32+
tolerations:
33+
{{- toYaml . | nindent 8 }}
34+
{{- end }}
35+
36+
terminationGracePeriodSeconds: {{ $.Values.warmNodes.terminationGracePeriodSeconds | default 5 }}
37+
38+
{{- with $.Values.warmNodes.images }}
39+
initContainers:
40+
{{- range . }}
41+
- name: {{ . | replace "/" "-" | replace ":" "-" | replace "." "-" | replace "@" "-" | trunc 63 | trimSuffix "-" }}
42+
image: {{ . }}
43+
command: []
44+
{{- end }}
45+
{{- end }}
46+
47+
containers:
48+
- name: pause
49+
image: {{ $.Values.warmNodes.pauseImage | default "registry.k8s.io/pause:3.9" }}
50+
{{- with $.Values.warmNodes.resources }}
51+
resources:
52+
{{- toYaml . | nindent 12 }}
53+
{{- end }}
54+
{{- end }}
55+
{{- end }}

‎charts/kyverno/values.yaml‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,49 @@ mutatingPolicy:
4646
# value: "123456789.dkr.ecr.us-east-2.amazonaws.com/ghcr/" + string(image(c.image).repository()) + ":" + string(image(c.image).tag())
4747
# } :
4848
# null
49-
# ).filter(p, p != null)
49+
# ).filter(p, p != null)
50+
51+
warmNodes:
52+
enabled: false
53+
namespace: "default"
54+
55+
annotations: {}
56+
# "helm.sh/hook": post-install
57+
# "helm.sh/hook-weight": "-5"
58+
# "helm.sh/hook-delete-policy": hook-succeeded
59+
60+
labels:
61+
# app.kubernetes.io/name: "img-fetch"
62+
# app.kubernetes.io/part-of: "coder-workspaces"
63+
64+
selectorLabels:
65+
app.kubernetes.io/name: "img-fetch"
66+
67+
terminationGracePeriodSeconds: 5
68+
69+
pauseImage: "registry.k8s.io/pause:3.9"
70+
71+
resources:
72+
requests:
73+
cpu: "1m"
74+
memory: "1Mi"
75+
limits:
76+
cpu: "10m"
77+
memory: "10Mi"
78+
79+
# List of images to pre-warm on nodes
80+
images:
81+
- "codercom/enterprise-java:latest"
82+
- "codercom/enterprise-golang:latest"
83+
- "codercom/enterprise-node:latest"
84+
- "codercom/enterprise-base:ubuntu"
85+
- "public.ecr.aws/f7a1d7a4/coder-aienv:1.1.4"
86+
87+
# DaemonSets to create (one per target node pool)
88+
daemonSets:
89+
- name: "imgs-for-coder-workspace"
90+
nodeSelector: {}
91+
tolerations: []
92+
- name: "imgs-for-coder-workspace-static"
93+
nodeSelector: {}
94+
tolerations: []

‎infra/aws/us-east-2/k8s/argo/kyverno/main.tf‎

Lines changed: 162 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ provider "aws" {
33
profile = var.profile
44
}
55

6+
data "aws_caller_identity" "this" {}
7+
68
data "aws_eks_cluster" "this" {
79
name = var.cluster_name
810
}
@@ -33,6 +35,14 @@ locals {
3335
}]
3436
}
3537
}
38+
reg_mirror = "${data.aws_caller_identity.this.account_id}.dkr.ecr.${var.region}.amazonaws.com"
39+
reg_suffix = {
40+
"ghcr" = "ghcr.io"
41+
"k8s" = "registry.k8s.io"
42+
"quay" = "quay.io"
43+
"docker-hub" = "index.docker.io"
44+
"ecr-public" = "public.ecr.aws"
45+
}
3646
}
3747

3848
resource "kubernetes_manifest" "kyverno" {
@@ -62,57 +72,169 @@ resource "kubernetes_manifest" "kyverno" {
6272
spec = {
6373
project = "default"
6474
source = {
65-
repoURL = "https://kyverno.github.io/kyverno/"
66-
chart = "kyverno"
67-
targetRevision = "3.7.1"
75+
repoURL = "https://github.com/coder/ai.coder.com"
76+
path = "charts/kyverno"
77+
targetRevision = "main"
6878
helm = {
6979
releaseName = "kyverno"
7080
values = yamlencode({
81+
kyverno = {
82+
enabled = true
83+
global = {
84+
tolerations = [{
85+
effect = "NoSchedule"
86+
key = "CriticalAddonsOnly"
87+
operator = "Exists"
88+
}]
89+
}
90+
config = {
91+
defaultRegistry = "docker.io"
92+
enableDefaultRegistryMutation = true
93+
webhooks = {
94+
namespacesSelector = {
95+
matchExpressions = [{
96+
key = "kubernetes.io/metadata.name"
97+
operator = "NotIn"
98+
values = [
99+
"kube-system",
100+
]
101+
}]
102+
}
103+
}
104+
}
71105

72-
global = {
73-
tolerations = [{
74-
effect = "NoSchedule"
75-
key = "CriticalAddonsOnly"
76-
operator = "Exists"
77-
}]
106+
crds = {
107+
migration = {
108+
nodeAffinity = local.nodeAffinity
109+
}
110+
}
111+
admissionController = {
112+
replicas = 3
113+
nodeAffinity = local.nodeAffinity
114+
}
115+
backgroundController = {
116+
replicas = 2
117+
nodeAffinity = local.nodeAffinity
118+
}
119+
cleanupController = {
120+
replicas = 2
121+
nodeAffinity = local.nodeAffinity
122+
}
123+
reportsController = {
124+
replicas = 2
125+
nodeAffinity = local.nodeAffinity
126+
}
78127
}
79-
80-
config = {
81-
defaultRegistry = "docker.io"
82-
enableDefaultRegistryMutation = true
83-
webhooks = {
84-
namespacesSelector = {
85-
matchExpressions = [{
86-
key = "kubernetes.io/metadata.name"
128+
mutatingPolicy = {
129+
enabled = true
130+
name = "mutate-ws-image"
131+
matchPolicy = "Equivalent"
132+
annotations = {
133+
"helm.sh/hook" = "post-install"
134+
"helm.sh/hook-weight" = "-5"
135+
"helm.sh/hook-delete-policy" = "hook-succeeded"
136+
}
137+
namespaceSelector = {
138+
matchExpressions = [{
139+
key = "kubernetes.io/metadata.name"
140+
operator = "In"
141+
values = [
142+
"default",
143+
"observability",
144+
"ebs-controller",
145+
"coder",
146+
"coder-ws-demo",
147+
"coder-ws-experiment",
148+
"coder-ws"
149+
]
150+
}]
151+
}
152+
objectSelector = {
153+
matchExpressions = [
154+
{
155+
key = "app.kubernetes.io/name"
87156
operator = "NotIn"
88157
values = [
89-
"kube-system",
158+
# "coder-provisioner",
159+
# "coder"
160+
"test"
90161
]
91-
}]
92-
}
162+
},
163+
{
164+
key = "app.kubernetes.io/managed-by"
165+
operator = "NotIn"
166+
values = [
167+
# "Helm",
168+
"test"
169+
]
170+
}
171+
]
93172
}
173+
resourceRules = [
174+
{
175+
apiGroups = [""]
176+
apiVersions = ["v1"]
177+
operations = ["CREATE", "UPDATE"]
178+
resources = ["pods"]
179+
}
180+
]
181+
mutations = [for k in ["containers", "initContainers", "ephemeralContainers"] : {
182+
patchType = "JSONPatch"
183+
jsonPatch = {
184+
expression = <<-EOT
185+
object.spec.?${k}.orValue([]).map(c,
186+
%{for suffix, reg in local.reg_suffix~}
187+
image(c.image).registry() == "${reg}" ?
188+
JSONPatch{
189+
op: "replace",
190+
path: "/spec/${k}/" + string(object.spec.?${k}.orValue([]).indexOf(c)) + "/image",
191+
value: "${local.reg_mirror}" + "/" + "${suffix}" + "/" + string(image(c.image).repository()) + ":" + string(image(c.image).tag())
192+
} :
193+
%{endfor~}
194+
null
195+
).filter(p, p != null)
196+
EOT
197+
}
198+
}]
94199
}
95-
96-
crds = {
97-
migration = {
98-
nodeAffinity = local.nodeAffinity
200+
warmNodes = {
201+
enabled = true
202+
namespace = "default"
203+
annotations = {
204+
"helm.sh/hook" = "post-install"
205+
"helm.sh/hook-weight" = "-5"
206+
"helm.sh/hook-delete-policy" = "hook-succeeded"
99207
}
100-
}
101-
admissionController = {
102-
replicas = 3
103-
nodeAffinity = local.nodeAffinity
104-
}
105-
backgroundController = {
106-
replicas = 2
107-
nodeAffinity = local.nodeAffinity
108-
}
109-
cleanupController = {
110-
replicas = 2
111-
nodeAffinity = local.nodeAffinity
112-
}
113-
reportsController = {
114-
replicas = 2
115-
nodeAffinity = local.nodeAffinity
208+
labels = {
209+
"app.kubernetes.io/name" = "img-fetch"
210+
}
211+
selectorLabels = {
212+
"app.kubernetes.io/name" = "img-fetch"
213+
}
214+
terminationGracePeriodSeconds = 5
215+
pauseImage = "registry.k8s.io/pause:3.9"
216+
images = [
217+
"codercom/enterprise-java:latest",
218+
"codercom/enterprise-golang:latest",
219+
"codercom/enterprise-node:latest",
220+
"codercom/enterprise-base:ubuntu",
221+
"public.ecr.aws/f7a1d7a4/coder-aienv:1.1.4"
222+
]
223+
daemonSets = [for k in ["coder-workspace", "coder-workspace-static"] : {
224+
name = "imgs-for-${k}"
225+
nodeSelector = {
226+
"node.coder.io/instance" = "coder-v2"
227+
"node.coder.io/managed-by" = "karpenter"
228+
"node.coder.io/name" = "coder"
229+
"node.coder.io/part-of" = "coder"
230+
"node.coder.io/used-for" = k
231+
}
232+
tolerations = [{
233+
key = "dedicated"
234+
value = k
235+
effect = "NoSchedule"
236+
}]
237+
}]
116238
}
117239
})
118240
}

0 commit comments

Comments
 (0)