Skip to content

Commit 346042f

Browse files
andrejatcastaiandrejatcastai
committed
Reworking helm values
1 parent e3a987f commit 346042f

File tree

11 files changed

+231
-313
lines changed

11 files changed

+231
-313
lines changed

‎examples/onboarding-with-existing-gke-cluster/main.tf‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ module "castai_omni_cluster" {
4040
pod_cidr = data.google_container_cluster.gke.cluster_ipv4_cidr
4141
service_cidr = data.google_container_cluster.gke.services_ipv4_cidr
4242
reserved_subnet_cidrs = [data.google_compute_subnetwork.gke_subnet.ip_cidr_range]
43+
44+
skip_helm = var.skip_helm
4345
}
4446

4547
module "castai_omni_edge_location_gcp" {

‎examples/onboarding-with-existing-gke-cluster/providers.tf‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ terraform {
1414
source = "hashicorp/helm"
1515
version = ">= 2.0"
1616
}
17+
kubernetes = {
18+
source = "hashicorp/kubernetes"
19+
version = ">= 2.35.0"
20+
}
1721
}
1822
}
1923

@@ -29,6 +33,12 @@ provider "helm" {
2933
}
3034
}
3135

36+
provider "kubernetes" {
37+
host = "https://${data.google_container_cluster.gke.endpoint}"
38+
token = data.google_client_config.default.access_token
39+
cluster_ca_certificate = base64decode(data.google_container_cluster.gke.master_auth.0.cluster_ca_certificate)
40+
}
41+
3242
provider "castai" {
3343
api_token = var.castai_api_token
3444
api_url = var.castai_api_url

‎examples/onboarding-with-existing-gke-cluster/variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,9 @@ variable "cluster_id" {
3434
description = "Cast AI Cluster ID"
3535
type = string
3636
}
37+
38+
variable "skip_helm" {
39+
description = "Skip installing any helm release; allows managing helm releases using GitOps"
40+
type = bool
41+
default = false
42+
}

‎main.tf‎

Lines changed: 90 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,12 @@ check "reserved_cidrs_required_for_gke" {
66
}
77

88
locals {
9-
liqo_chart_repo = "https://castai.github.io/liqo"
10-
liqo_chart_name = "liqo"
11-
liqo_release_name = "omni"
12-
liqo_image_tag = var.liqo_chart_version
13-
14-
omni_namespace = "castai-omni"
15-
omni_agent_release = "omni-agent"
16-
omni_agent_chart = "omni-agent"
17-
castai_helm_repository = "https://castai.github.io/helm-charts"
18-
19-
# Common Liqo configurations as YAML
20-
common_liqo_yaml_values = <<-EOT
21-
networking:
22-
fabric:
23-
config:
24-
healthProbeBindAddressPort: '7071'
25-
metricsAddressPort: '7072'
26-
EOT
27-
28-
# Select the appropriate set_values based on k8s_provider
29-
provider_helm_values = merge(
30-
{for v in module.liqo_helm_values_gke : "gke" => v.set_values},
31-
{for v in module.liqo_helm_values_eks : "eks" => v.set_values},
32-
{for v in module.liqo_helm_values_aks : "aks" => v.set_values},
33-
)
34-
provider_specific_liqo_values = local.provider_helm_values[var.k8s_provider]
9+
liqo_image_tag = var.liqo_chart_version
3510
}
3611

3712
# GKE-specific Liqo Helm chart configuration
3813
module "liqo_helm_values_gke" {
39-
count = !var.skip_helm && var.k8s_provider == "gke" ? 1 : 0
14+
count = var.k8s_provider == "gke" ? 1 : 0
4015
source = "./modules/gke"
4116

4217
image_tag = local.liqo_image_tag
@@ -51,7 +26,7 @@ module "liqo_helm_values_gke" {
5126

5227
# EKS-specific Liqo Helm chart configuration
5328
module "liqo_helm_values_eks" {
54-
count = !var.skip_helm && var.k8s_provider == "eks" ? 1 : 0
29+
count = var.k8s_provider == "eks" ? 1 : 0
5530
source = "./modules/eks"
5631

5732
image_tag = local.liqo_image_tag
@@ -76,79 +51,51 @@ module "liqo_helm_values_aks" {
7651
service_cidr = var.service_cidr
7752
}
7853

79-
resource "helm_release" "liqo" {
80-
count = var.skip_helm ? 0 : 1
81-
82-
name = local.liqo_release_name
83-
repository = local.liqo_chart_repo
84-
chart = local.liqo_chart_name
85-
version = var.liqo_chart_version
86-
namespace = local.omni_namespace
87-
create_namespace = true
88-
cleanup_on_fail = true
89-
wait = true
90-
91-
values = [local.common_liqo_yaml_values]
92-
set = local.provider_specific_liqo_values
93-
}
94-
95-
# Wait for Liqo network resources to be ready before proceeding
96-
resource "null_resource" "wait_for_liqo_network" {
97-
count = var.skip_helm ? 0 : 1
98-
99-
provisioner "local-exec" {
100-
command = <<-EOT
101-
set -e
102-
103-
echo "Waiting for Liqo networks.ipam.liqo.io CRD to be established..."
104-
kubectl wait --for condition=established --timeout=300s crd/networks.ipam.liqo.io
105-
106-
echo "Waiting for external CIDR network resource to be created..."
107-
timeout=300
108-
elapsed=0
109-
interval=5
110-
111-
while [ $elapsed -lt $timeout ]; do
112-
CIDR=$(kubectl get networks.ipam.liqo.io -n ${local.omni_namespace} \
113-
-l ipam.liqo.io/network-type=external-cidr \
114-
-o jsonpath='{.items[0].status.cidr}' 2>/dev/null || echo "")
115-
116-
if [ -n "$CIDR" ]; then
117-
echo "External CIDR network resource is ready: $CIDR"
118-
exit 0
119-
fi
120-
121-
echo "Waiting for external CIDR to be populated... ($elapsed/$timeout seconds)"
122-
sleep $interval
123-
elapsed=$((elapsed + interval))
124-
done
125-
126-
echo "Timeout waiting for external CIDR network resource"
127-
exit 1
128-
EOT
129-
}
130-
131-
depends_on = [helm_release.liqo]
132-
}
54+
locals {
55+
liqo_chart_repo = "https://castai.github.io/liqo"
56+
liqo_chart_name = "liqo"
57+
liqo_release_name = "omni"
13358

134-
# Extract the external CIDR value from Liqo network resource
135-
data "external" "liqo_external_cidr" {
136-
count = var.skip_helm ? 0 : 1
59+
omni_namespace = "castai-omni"
60+
omni_agent_release = "castai-omni-agent"
61+
omni_agent_chart = "omni-agent"
62+
castai_helm_repository = "https://castai.github.io/helm-charts"
13763

138-
program = ["bash", "-c", <<-EOT
139-
CIDR=$(kubectl get networks.ipam.liqo.io -n ${local.omni_namespace} \
140-
-l ipam.liqo.io/network-type=external-cidr \
141-
-o jsonpath='{.items[0].status.cidr}' 2>/dev/null)
64+
# Omni agent configuration as YAML
65+
omni_agent_yaml_values = <<-EOT
66+
castai:
67+
apiUrl: ${var.api_url}
68+
organizationID: ${var.organization_id}
69+
clusterID: ${var.cluster_id}
70+
clusterName: ${var.cluster_name}
71+
EOT
14272

143-
if [ -z "$CIDR" ]; then
144-
echo '{"cidr":""}'
145-
else
146-
echo "{\"cidr\":\"$CIDR\"}"
147-
fi
73+
# Common Liqo configuration as YAML
74+
common_liqo_yaml_values = <<-EOT
75+
networking:
76+
fabric:
77+
config:
78+
healthProbeBindAddressPort: '7071'
79+
metricsAddressPort: '7072'
14880
EOT
149-
]
15081

151-
depends_on = [null_resource.wait_for_liqo_network]
82+
# Select the appropriate yaml_values based on k8s_provider
83+
provider_yaml_values = merge(
84+
{ for v in module.liqo_helm_values_gke : "gke" => v.liqo_yaml_values },
85+
{ for v in module.liqo_helm_values_eks : "eks" => v.liqo_yaml_values },
86+
{ for v in module.liqo_helm_values_aks : "aks" => v.liqo_yaml_values },
87+
)
88+
provider_specific_yaml_values = local.provider_yaml_values[var.k8s_provider]
89+
90+
helm_yaml_values = {
91+
castai = {
92+
apiUrl = var.api_url
93+
organizationID = var.organization_id
94+
clusterID = var.cluster_id
95+
clusterName = var.cluster_name
96+
}
97+
liqo = local.provider_specific_yaml_values.liqo
98+
}
15299
}
153100

154101
# CAST AI Omni Agent Helm Release
@@ -159,45 +106,11 @@ resource "helm_release" "omni_agent" {
159106
repository = local.castai_helm_repository
160107
chart = local.omni_agent_chart
161108
namespace = local.omni_namespace
162-
create_namespace = true
109+
create_namespace = false
163110
cleanup_on_fail = true
164111
wait = true
165112

166-
set = [
167-
{
168-
name = "network.externalCIDR"
169-
value = data.external.liqo_external_cidr[0].result.cidr
170-
},
171-
{
172-
name = "network.podCIDR"
173-
value = var.pod_cidr
174-
},
175-
{
176-
name = "castai.apiUrl"
177-
value = var.api_url
178-
},
179-
{
180-
name = "castai.organizationID"
181-
value = var.organization_id
182-
},
183-
{
184-
name = "castai.clusterID"
185-
value = var.cluster_id
186-
},
187-
{
188-
name = "castai.clusterName"
189-
value = var.cluster_name
190-
}
191-
]
192-
193-
set_sensitive = [
194-
{
195-
name = "castai.apiKey"
196-
value = var.api_token
197-
}
198-
]
199-
200-
depends_on = [null_resource.wait_for_liqo_network]
113+
values = [yamlencode(local.helm_yaml_values)]
201114
}
202115

203116
# Enabling CAST AI Omni functionality for a given cluster
@@ -207,3 +120,48 @@ resource "castai_omni_cluster" "this" {
207120

208121
depends_on = [helm_release.omni_agent]
209122
}
123+
124+
resource "kubernetes_namespace_v1" "omni" {
125+
count = var.skip_helm ? 1 : 0
126+
127+
metadata {
128+
name = local.omni_namespace
129+
}
130+
}
131+
132+
# Secret with API token for GitOps (when skip_helm = true)
133+
resource "kubernetes_secret_v1" "api_token" {
134+
count = var.skip_helm ? 1 : 0
135+
136+
metadata {
137+
name = "castai-omni-agent-token"
138+
namespace = local.omni_namespace
139+
}
140+
141+
data = {
142+
"CASTAI_AGENT_TOKEN" = var.api_token
143+
}
144+
145+
depends_on = [kubernetes_namespace_v1.omni]
146+
}
147+
148+
# ConfigMap with helm values for GitOps (when skip_helm = true)
149+
resource "kubernetes_config_map_v1" "helm_values" {
150+
count = var.skip_helm ? 1 : 0
151+
152+
metadata {
153+
name = "castai-omni-helm-values"
154+
namespace = local.omni_namespace
155+
}
156+
157+
data = {
158+
"liqo.repository" = local.liqo_chart_repo
159+
"liqo.chart" = local.liqo_chart_name
160+
"liqo.version" = var.liqo_chart_version
161+
"omni-agent.repository" = local.castai_helm_repository
162+
"omni-agent.chart" = local.omni_agent_chart
163+
"values.yaml" = yamlencode(local.helm_yaml_values)
164+
}
165+
166+
depends_on = [kubernetes_namespace_v1.omni]
167+
}

‎modules/aks/main.tf‎

Lines changed: 31 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,36 @@
11
locals {
22
pools_cidrs = ["10.0.0.0/8", "192.168.0.0/16", "172.16.0.0/12", var.service_cidr]
3+
provider = "aks"
34

4-
basic_set_values = [
5-
{
6-
name = "tag"
7-
value = var.image_tag
8-
},
9-
{
10-
name = "apiServer.address"
11-
value = var.api_server_address
12-
},
13-
{
14-
name = "discovery.config.clusterID"
15-
value = var.cluster_name
16-
},
17-
{
18-
name = "discovery.config.clusterLabels.liqo\\.io/provider"
19-
value = "aks"
20-
},
21-
{
22-
name = "discovery.config.clusterLabels.topology\\.kubernetes\\.io/region"
23-
value = var.cluster_region
24-
},
25-
{
26-
name = "ipam.podCIDR"
27-
value = var.pod_cidr
28-
},
29-
{
30-
name = "ipam.serviceCIDR"
31-
value = var.service_cidr
32-
},
33-
{
34-
name = "ipam.serviceCIDR"
35-
value = var.service_cidr
36-
},
37-
{
38-
name = "telemetry.enabled"
39-
value = "false"
5+
liqo_yaml_values = {
6+
liqo = {
7+
enabled = true
8+
tag = var.image_tag
9+
apiServer = {
10+
address = var.api_server_address
11+
}
12+
discovery = {
13+
config = {
14+
clusterID = var.cluster_name
15+
clusterLabels = merge(
16+
{
17+
"liqo.io/provider" = local.provider
18+
"topology.kubernetes.io/region" = var.cluster_region
19+
},
20+
var.cluster_zone != "" ? {
21+
"topology.kubernetes.io/zone" = var.cluster_zone
22+
} : {}
23+
)
24+
}
25+
}
26+
ipam = {
27+
podCIDR = var.pod_cidr
28+
serviceCIDR = var.service_cidr
29+
pools = local.pools_cidrs
30+
}
31+
telemetry = {
32+
enabled = false
33+
}
4034
}
41-
]
42-
43-
pools_set_values = [
44-
for idx, cidr in local.pools_cidrs : {
45-
name = "ipam.pools[${idx}]"
46-
value = cidr
47-
}
48-
]
49-
50-
all_set_values = concat(
51-
local.basic_set_values,
52-
local.pools_set_values,
53-
)
35+
}
5436
}

0 commit comments

Comments
 (0)