Skip to content

Commit b67957b

Browse files
jatcod3rjatcod3r
authored
Merge pull request #103 from coder/jatcod3r/karpenter-migrate
feat: add NodePool + Class configs
2 parents e9fcee8 + 377cce0 commit b67957b

3 files changed

Lines changed: 287 additions & 2 deletions

File tree

‎infra/aws/us-east-2/k8s/argo/karpenter/main.tf‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,23 @@ provider "aws" {
55
profile = var.profile
66
}
77

8+
data "aws_vpc" "this" {
9+
tags = {
10+
Name = var.vpc_name
11+
}
12+
}
13+
14+
data "aws_subnets" "private" {
15+
filter {
16+
name = "vpc-id"
17+
values = [data.aws_vpc.this.id]
18+
}
19+
20+
tags = {
21+
Name = "*${var.private_subnet_suffix}*"
22+
}
23+
}
24+
825
data "aws_eks_cluster" "this" {
926
name = var.cluster_name
1027
}
@@ -36,6 +53,29 @@ locals {
3653

3754
locals {
3855
cluster_oidc_provider = trimprefix(data.aws_iam_openid_connect_provider.this.url, "https://")
56+
common_labels = {
57+
"node.coder.io/instance" = "coder-v2"
58+
"node.coder.io/managed-by" = "karpenter"
59+
"node.coder.io/name" = "coder"
60+
"node.coder.io/part-of" = "coder"
61+
}
62+
common_node_requirements = [
63+
{
64+
key = "kubernetes.io/arch"
65+
operator = "In"
66+
values = ["amd64", "arm64"]
67+
},
68+
{
69+
key = "kubernetes.io/os"
70+
operator = "In"
71+
values = ["linux"]
72+
},
73+
{
74+
key = "karpenter.sh/capacity-type"
75+
operator = "In"
76+
values = ["spot", "on-demand"]
77+
}
78+
]
3979
}
4080

4181
module "karpenter" {
@@ -197,6 +237,234 @@ resource "kubernetes_manifest" "karpenter" {
197237
interruptionQueue = module.karpenter.queue_name
198238
}
199239
}
240+
nodeClasses = [
241+
{
242+
name = "platform"
243+
apiVersion = "eks.amazonaws.com/v1"
244+
kind = "NodeClass"
245+
subnetSelectorTerms = [for subnet_id in data.aws_subnets.private.ids : { id = subnet_id }]
246+
sgSelectorTerms = [{ id = data.aws_eks_cluster.this.vpc_config[0].cluster_security_group_id }]
247+
networkPolicy = "DefaultAllow"
248+
networkPolicyEventLogs = "Disabled"
249+
snatPolicy = "Disabled"
250+
ephemeralStorage = {
251+
iops = 3000
252+
size = "80Gi"
253+
throughput = 125
254+
}
255+
role = var.cluster_node_iam_role_name
256+
tags = {
257+
Name = "platform-node"
258+
}
259+
},
260+
{
261+
name = "coder-provisioner"
262+
apiVersion = "eks.amazonaws.com/v1"
263+
kind = "NodeClass"
264+
subnetSelectorTerms = [for subnet_id in data.aws_subnets.private.ids : { id = subnet_id }]
265+
sgSelectorTerms = [{ id = data.aws_eks_cluster.this.vpc_config[0].cluster_security_group_id }]
266+
networkPolicy = "DefaultAllow"
267+
networkPolicyEventLogs = "Disabled"
268+
snatPolicy = "Disabled"
269+
ephemeralStorage = {
270+
iops = 3000
271+
size = "80Gi"
272+
throughput = 125
273+
}
274+
role = var.cluster_node_iam_role_name
275+
tags = {
276+
Name = "coder-provisioner-node"
277+
}
278+
},
279+
{
280+
name = "coder-workspace"
281+
apiVersion = "karpenter.k8s.aws/v1"
282+
kind = "EC2NodeClass"
283+
userData = <<-EOT
284+
MIME-Version: 1.0
285+
Content-Type: multipart/mixed; boundary="//"
286+
287+
--//
288+
Content-Type: application/node.eks.aws
289+
290+
apiVersion: node.eks.aws/v1alpha1
291+
kind: NodeConfig
292+
spec:
293+
kubelet:
294+
config:
295+
registryPullQPS: 30
296+
--//--
297+
EOT
298+
subnetSelectorTerms = [for subnet_id in data.aws_subnets.private.ids : { id = subnet_id }]
299+
amiSelectorTerms = [{ alias = "al2023@latest" }]
300+
sgSelector = [{ id = data.aws_eks_cluster.this.vpc_config[0].cluster_security_group_id }]
301+
blockDeviceMappings = [{
302+
deviceName = "/dev/xvda"
303+
ebs = {
304+
volumeSize = "200Gi"
305+
volumeType = "gp3"
306+
encrypted = false
307+
deleteOnTermination = true
308+
}
309+
}]
310+
role = module.karpenter.node_iam_role_name
311+
tags = {
312+
Name = "coder-workspace-node"
313+
}
314+
}
315+
]
316+
nodePools = [
317+
{
318+
name = "system"
319+
consolidationPolicy = "WhenEmptyOrUnderutilized"
320+
consolidateAfter = "72h"
321+
budgets = [{ nodes = "10%" }]
322+
labels = merge(local.common_labels, {
323+
"node.coder.io/used-for" = "system"
324+
})
325+
expireAfter = "480h"
326+
taints = [{
327+
key = "CriticalAddonsOnly"
328+
value = "true"
329+
effect = "NoSchedule"
330+
}]
331+
requirements = concat(local.common_node_requirements, [{
332+
key = "node.kubernetes.io/instance-type"
333+
operator = "In"
334+
values = ["c6g.large", "c6g.xlarge", "c6g.2xlarge"]
335+
}])
336+
nodeClassRef = {
337+
group = "eks.amazonaws.com"
338+
kind = "NodeClass"
339+
name = "platform"
340+
}
341+
},
342+
{
343+
name = "observability-platform"
344+
consolidationPolicy = "WhenEmptyOrUnderutilized"
345+
consolidateAfter = "72h"
346+
budgets = [{ nodes = "10%" }]
347+
labels = merge(local.common_labels, {
348+
"node.coder.io/used-for" = "observability-platform"
349+
})
350+
expireAfter = "480h"
351+
taints = [{
352+
key = "platform"
353+
value = "observability-platform"
354+
effect = "NoSchedule"
355+
}]
356+
requirements = concat(local.common_node_requirements, [{
357+
key = "node.kubernetes.io/instance-type"
358+
operator = "In"
359+
values = ["c6g.large", "c6g.xlarge"]
360+
}])
361+
nodeClassRef = {
362+
group = "eks.amazonaws.com"
363+
kind = "NodeClass"
364+
name = "platform"
365+
}
366+
},
367+
{
368+
name = "coder-server"
369+
consolidationPolicy = "WhenEmptyOrUnderutilized"
370+
consolidateAfter = "8h"
371+
budgets = [{ nodes = "10%" }]
372+
labels = merge(local.common_labels, {
373+
"node.coder.io/used-for" = "coder-server"
374+
})
375+
expireAfter = "480h"
376+
taints = [{
377+
key = "platform"
378+
value = "coder-server"
379+
effect = "NoSchedule"
380+
}]
381+
requirements = concat(local.common_node_requirements, [{
382+
key = "node.kubernetes.io/instance-type"
383+
operator = "In"
384+
values = ["c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge"]
385+
}])
386+
nodeClassRef = {
387+
group = "eks.amazonaws.com"
388+
kind = "NodeClass"
389+
name = "platform"
390+
}
391+
},
392+
{
393+
name = "coder-provisioner"
394+
consolidationPolicy = "WhenEmptyOrUnderutilized"
395+
consolidateAfter = "8h"
396+
budgets = [{ nodes = "100%" }]
397+
labels = merge(local.common_labels, {
398+
"node.coder.io/used-for" = "coder-provisioner"
399+
})
400+
expireAfter = "8h"
401+
taints = [{
402+
key = "coder"
403+
value = "provisioner"
404+
effect = "NoSchedule"
405+
}]
406+
requirements = concat(local.common_node_requirements, [{
407+
key = "node.kubernetes.io/instance-type"
408+
operator = "In"
409+
values = ["c6g.large", "c6g.xlarge", "c6g.2xlarge", "c6g.4xlarge"]
410+
}])
411+
nodeClassRef = {
412+
group = "eks.amazonaws.com"
413+
kind = "NodeClass"
414+
name = "platform"
415+
}
416+
},
417+
{
418+
name = "coder-workspace"
419+
consolidationPolicy = "WhenEmptyOrUnderutilized"
420+
consolidateAfter = "4h"
421+
budgets = [{ nodes = "100%" }]
422+
labels = {
423+
"node.coder.io/instance" = "coder-v2"
424+
"node.coder.io/managed-by" = "karpenter"
425+
"node.coder.io/name" = "coder"
426+
"node.coder.io/part-of" = "coder"
427+
"node.coder.io/used-for" = "coder-workspace"
428+
}
429+
expireAfter = "Never"
430+
taints = []
431+
requirements = concat(local.common_node_requirements, [{
432+
key = "node.kubernetes.io/instance-type"
433+
operator = "In"
434+
values = ["c6a.4xlarge", "c6a.8xlarge"]
435+
}])
436+
nodeClassRef = {
437+
group = "karpenter.k8s.aws"
438+
kind = "EC2NodeClass"
439+
name = "coder-workspace"
440+
}
441+
},
442+
{
443+
name = "coder-workspace-static"
444+
consolidationPolicy = "WhenEmptyOrUnderutilized"
445+
consolidateAfter = "0s"
446+
budgets = [{ nodes = "100%" }]
447+
replicas = 2
448+
limits = {
449+
nodes = 100
450+
}
451+
labels = merge(local.common_labels, {
452+
"node.coder.io/used-for" = "coder-workspace-static"
453+
})
454+
expireAfter = "Never"
455+
taints = []
456+
requirements = concat(local.common_node_requirements, [{
457+
key = "node.kubernetes.io/instance-type"
458+
operator = "In"
459+
values = ["c6a.4xlarge", "c6a.8xlarge"]
460+
}])
461+
nodeClassRef = {
462+
group = "karpenter.k8s.aws"
463+
kind = "EC2NodeClass"
464+
name = "coder-workspace"
465+
}
466+
}
467+
]
200468
})
201469
}
202470
}

‎infra/aws/us-east-2/k8s/argo/karpenter/terragrunt.hcl‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,13 @@ include "config" {
88
expose = true
99
}
1010

11+
dependency "eks" {
12+
config_path = "../../../eks"
13+
mock_outputs = {
14+
eks_node_iam_role_name = "iam-role-name"
15+
}
16+
}
17+
1118
dependencies {
1219
paths = [
1320
"../../../eks"
@@ -18,6 +25,7 @@ inputs = {
1825
profile = include.root.locals.CODER_AWS_PROFILE
1926
region = include.config.locals.AWS_REGION
2027

21-
cluster_name = include.root.locals.CODER_CLUSTER_NAME
22-
vpc_name = include.root.locals.CODER_VPC_NAME
28+
cluster_name = include.root.locals.CODER_CLUSTER_NAME
29+
cluster_node_iam_role_name = dependency.eks.outputs.eks_node_iam_role_name
30+
vpc_name = include.root.locals.CODER_VPC_NAME
2331
}

‎infra/aws/us-east-2/k8s/argo/karpenter/variables.tf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,15 @@ variable "cluster_name" {
1111
type = string
1212
}
1313

14+
variable "cluster_node_iam_role_name" {
15+
type = string
16+
}
17+
1418
variable "vpc_name" {
1519
type = string
20+
}
21+
22+
variable "private_subnet_suffix" {
23+
type = string
24+
default = "private"
1625
}

0 commit comments

Comments
 (0)