Add tags on resources, update ipv6 func (rancher#233)

Author: mdrahman-suse

docs/testing.md

@@ -64,6 +64,12 @@ Note/TODO: k3s external db fails working with etcd only node. Refer: https://doc
 
 ## Validating Dual-Stack
 
+- Required vars for `.env` file
+
+```
+TEST_DIR=dualstack
+```
+
 - Required vars for `*.tfvars` file
 - `kubelet-arg: \n - node-ip=0.0.0.0` is required to be added to both server and worker flags if the public and private IPv6 IPs are same
 
@@ -82,6 +88,13 @@ bastion_subnets      = "<dual-stack-subnet>"
 
 ## Validating IPv6 Only
 
+- Required vars for `.env` file
+
+```
+ENV_MODULE=ipv6only
+TEST_DIR=ipv6only
+```
+
 - Required vars for `*.tfvars` file
 
 ```

modules/airgap/setup/get_artifacts.sh

@@ -11,8 +11,9 @@ product=${1}
 version=${2}
 platform=${3}
 arch=${4}
-server_flags=${5}
-tarball_type=${6}
+registry_url=${5}
+server_flags=${6}
+tarball_type=${7}
 k3s_binary=$product
 
 validate_args() {
@@ -52,7 +53,7 @@ download_retry() {
   max_attempts=3
   attempt_num=1
 
-  while [ $attempt_num -le $max_attempts ]; do
+  while [[ $attempt_num -le $max_attempts ]]; do
     if eval "$cmd"; then
       echo "Command succeeded after $attempt_num attempts."
       break
@@ -63,7 +64,7 @@ download_retry() {
     fi
   done
 
-  if [ $attempt_num -gt $max_attempts ]; then
+  if [[ $attempt_num -ge $max_attempts ]]; then
     echo "Command failed after $max_attempts attempts."
   fi
 }
@@ -79,7 +80,12 @@ get_assets() {
       download_retry "wget $url/k3s-airgap-images-$arch.$tarball_type"
     fi
   elif [[ "$product" == "rke2" ]]; then
-    url="https://github.com/rancher/rke2/releases/download/$version"
+    if [[ -n "$registry_url" ]]; then
+      url=$registry_url/rke2/$version
+    else
+      url="https://github.com/rancher/rke2/releases/download/$version"
+    fi
+    echo "Download assets using url: $url"
     download_retry "wget $url/sha256sum-$arch.txt"
     # Ref: https://docs.rke2.io/install/airgap
     if [[ -n "$server_flags" ]] && [[ "$server_flags" =~ "cni" ]]; then
@@ -102,7 +108,12 @@ get_assets() {
 
 get_cni_assets() {
   if [[ -n "$server_flags" ]] && [[ "$server_flags" =~ "cni" ]] && [[ "$server_flags" != *"cni: none"* ]]; then
-    url="https://github.com/rancher/rke2/releases/download/$version"
+    if [[ -n "$registry_url" ]]; then
+      url=$registry_url/rke2/$version
+    else
+      url="https://github.com/rancher/rke2/releases/download/$version"
+    fi
+    echo "Download cni assets using url: $url"
     cnis=("calico" "canal" "cilium" "flannel")
     for cni in "${cnis[@]}"; do
       if [[ "$server_flags" =~ $cni ]]; then

modules/airgap/setup/podman_cmds.sh

@@ -30,6 +30,9 @@ for image_file in $image_files; do
   echo "Reading from file: $image_file"
   while read -r image_url_tag; do
     if [[ -n "$registry_url" ]]; then
+      if [[ $registry_url =~ "http" ]]; then
+        registry_url=$(echo $registry_url | cut -d '/' -f 3)
+      fi
       image_url_tag="${image_url_tag/docker.io/$registry_url}"
     fi
     echo "Pulling image: $image_url_tag"

modules/bastion/instance_server.tf

@@ -21,6 +21,7 @@ resource "aws_instance" "bastion" {
   key_name               = var.key_name
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-bastion-server"
+    Team                 = local.resource_tag
   }
 
   provisioner "file" {

modules/install/k3s_master.sh

@@ -257,7 +257,7 @@ main() {
   fi
   if [[ "${install_or_enable}" == "enable" ]] || [[ "${install_or_enable}" == "both" ]]; then
     check_service
-    if [ "$etcd_only_node" -eq 0 ]; then
+    if [[ "$etcd_only_node" -eq 0 ]]; then
       # If etcd only node count is 0, then wait for nodes/pods to come up.
       # etcd only node needs api server to come up fully, which is in control plane node.
       # and hence we cannot wait for node/pod status in this case.

modules/ipv6only/instance/instance_server.tf

@@ -17,10 +17,11 @@ resource "aws_instance" "master" {
   key_name               = var.key_name
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-server${count.index + 1}"
+    Team                 = local.resource_tag
   }
 
   provisioner "local-exec" { 
-    command = "aws ec2 wait instance-status-ok --region ${var.region} --instance-ids ${aws_instance.master[count.index].id}" 
+    command = "aws ec2 wait instance-status-ok --region ${var.region} --instance-ids ${self.id}" 
   }
 }
 
@@ -43,10 +44,11 @@ resource "aws_instance" "worker" {
   key_name               = var.key_name
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-worker${count.index + 1}"
+    Team                 = local.resource_tag
   }
 
   provisioner "local-exec" { 
-    command = "aws ec2 wait instance-status-ok --region ${var.region} --instance-ids ${aws_instance.worker[count.index].id}" 
+    command = "aws ec2 wait instance-status-ok --region ${var.region} --instance-ids ${self.id}" 
   }
 }
 
@@ -70,8 +72,11 @@ resource "aws_instance" "bastion" {
   availability_zone      = var.availability_zone
   vpc_security_group_ids = [var.sg_id]
   key_name               = var.key_name
+  user_data              = file("scripts/prepare.sh")
+
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-bastion"
+    Team                 = local.resource_tag
   }
 
   provisioner "local-exec" { 

modules/ipv6only/scripts/configure.sh

@@ -3,18 +3,45 @@
 set +x
 
 instance_id=${1}
+product=${2}
+flags=${3}
 
 ipv6_config() {
-    echo "Stopping systemd-resolved"
-    systemctl stop systemd-resolved.service
-    echo "Updating /etc/hosts"
-    sed -i -e 's/127.0.0.1/::1/g' -e "s/ip6-loopback/ip6-loopback $instance_id/g" /etc/hosts
-    echo "Updating /etc/resolv.conf"
-    sed -i 's/127.0.0.53/2a00:1098:2c::1/g' /etc/resolv.conf
+  echo "Stopping systemd-resolved"
+  systemctl stop systemd-resolved.service
+  echo "Updating /etc/hosts"
+  sed -i -e 's/127.0.0.1/::1/g' -e "s/ip6-loopback/ip6-loopback $instance_id/g" /etc/hosts
+  echo "Updating /etc/resolv.conf"
+  sed -i 's/127.0.0.53/2a00:1098:2c::1/g' /etc/resolv.conf
+}
+
+# Ref: https://github.com/rancher/rke2/issues/8033
+cilium_config() {
+  echo "Setting helmchartconfig for cilium with ipv6only"
+  if [[ "$flags" =~ "cilium" ]]; then
+    mkdir -p /var/lib/rancher/rke2/server/manifests
+    cat <<EOF >>/var/lib/rancher/rke2/server/manifests/rke2-cilium-ipv6config.yaml
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-cilium
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    bgp:
+      enabled: true
+      announce:
+        podCIDR: true
+EOF
+  fi
 }
 
 main() {
   ipv6_config
+  if [[ "$product" == "rke2" ]] && [ -n "$flags" ]; then
+    cilium_config
+  fi
 }
 
 main "$@"

modules/ipv6only/scripts/prepare.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# This script prepares bastion node with installing kubectl
+# set +x
+
+arch=$(uname -m)
+if [ "$arch" = "aarch64" ]; then
+    KUBE_ARCH="arm64"
+else
+    KUBE_ARCH="amd64"
+fi
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/$KUBE_ARCH/kubectl" && \
+chmod +x ./kubectl && \
+mv ./kubectl /usr/local/bin

modules/k3s/master/instances_server.tf

@@ -12,7 +12,8 @@ resource "aws_db_instance" "db" {
   password               = var.db_password
   availability_zone      = var.availability_zone
   tags = {
-    Environment = var.environment
+    Environment          = var.environment
+    Team                 = local.resource_tag
   }
   skip_final_snapshot    = true
 }
@@ -29,6 +30,7 @@ resource "aws_rds_cluster" "db" {
   engine_mode            = var.engine_mode
   tags = {
     Environment          = var.environment
+    Team                 = local.resource_tag
   }
   skip_final_snapshot    = true
 }
@@ -45,8 +47,9 @@ resource "aws_rds_cluster_instance" "db" {
 resource "aws_eip" "master_with_eip" {
   count                   = var.create_eip ? 1 : 0
   domain                  = "vpc"
-  tags                    = {
-    Name ="${var.resource_name}-${local.resource_tag}-server1"
+  tags = {
+    Name                  = "${var.resource_name}-${local.resource_tag}-server1"
+    Team                  = local.resource_tag
   }
 }
 
@@ -77,8 +80,9 @@ resource "aws_instance" "master" {
   availability_zone      = var.availability_zone
   vpc_security_group_ids = [var.sg_id]
   key_name               = var.key_name
-  tags                   = {
+  tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-server1"
+    Team                 = local.resource_tag
   }
 
   provisioner "remote-exec" {
@@ -206,8 +210,9 @@ data "local_file" "token" {
 resource "aws_eip" "master2_with_eip" {
   count         = var.create_eip ? local.secondary_masters : 0
   domain        = "vpc"
-  tags       = {
-    Name ="${var.resource_name}-${local.resource_tag}-server${count.index + 2}"
+  tags = {
+    Name        = "${var.resource_name}-${local.resource_tag}-server${count.index + 2}"
+    Team        = local.resource_tag
   }
   depends_on = [aws_eip.master_with_eip ]
 }
@@ -243,6 +248,7 @@ resource "aws_instance" "master2-ha" {
   depends_on             = [aws_instance.master]
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-server${count.index + 2}"
+    Team                 = local.resource_tag
   }
   provisioner "remote-exec" {
     inline = [

modules/k3s/worker/instances_worker.tf

@@ -23,6 +23,7 @@ resource "aws_instance" "worker" {
   key_name               = var.key_name
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-worker${count.index + 1}"
+    Team                 = local.resource_tag
   }
   provisioner "remote-exec" {
     inline = [
@@ -69,6 +70,7 @@ resource "aws_eip" "worker_with_eip" {
   domain             = "vpc"
   tags = {
     Name                 = "${var.resource_name}-${local.resource_tag}-worker${count.index + 1}"
+    Team                 = local.resource_tag
   }
 }