add airgap env support

Author: zugwan

01_prepare_assets.sh

@@ -17,37 +17,60 @@ check_if_supported_os
 # TODO: check if the bootstrap cluster already exist.
 rm -rf ~/.kube/config
 
-check_if_supported_os
+GUM_ASSETS_DIR="$ASSETS_DIR/gum/$(ls $ASSETS_DIR/gum | grep v)"
+sudo cp $GUM_ASSETS_DIR/gum /usr/local/bin
 
-log_info "Installing Docker-ce"
-# TODO: install only when not installed
+log_info "Installing container engine"
 case $OS_ID in
 	"rocky" | "centos" | "rhel")
-		sudo dnf localinstall $ASSETS_DIR/docker-ce/*.rpm
+		sudo dnf install -y podman
 		;;
 
 	"ubuntu" )
-		sudo dpkg -i $ASSETS_DIR/docker-ce/*.deb
+		sudo apt-get -y install podman
 		;;
 esac
-sudo systemctl start docker
-sudo docker load -i $ASSETS_DIR/kind-node-image.tar.gz
 
-log_info "Creating bootstrap cluster"
+for img in $(ls $ASSETS_DIR/images); do
+	sudo podman load -i $ASSETS_DIR/images/$img
+done
+
 
 sudo cp $ASSETS_DIR/kubectl /usr/local/bin
 sudo chmod +x /usr/local/bin/kubectl
 sudo cp $KIND_ASSETS_DIR/kind-linux-amd64 /usr/local/bin/kind
 sudo chmod +x /usr/local/bin/kind
 sudo cp $1/helm /usr/local/bin
 
+log_info "Running local container registry"
+reg_name='registry'
+reg_port='5000'
+if [ "$(sudo podman inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  sudo podman run \
+    -d --restart=always -p "0.0.0.0:${reg_port}:5000" --privileged --name "${reg_name}" \
+    --net podman -v $(realpath $ASSETS_DIR)/registry:/var/lib/registry \
+    localhost:5000/registry:2
+fi
+cat <<EOF | sudo tee /etc/containers/registries.conf.d/localregistry.conf
+[[registry]]
+location = "${BOOTSTRAP_CLUSTER_SERVER_IP}:5000"
+insecure = true
+EOF
+
+log_info "Running nginx for downloading byoh scripts"
+sudo dnf install nginx -y
+sudo systemctl enable --now nginx
+nginx -v
+# /usr/share/nginx/html
+
+log_info "Creating bootstrap cluster"
 export BOOTSTRAP_CLUSTER_SERVER_IP
 envsubst '${BOOTSTRAP_CLUSTER_SERVER_IP}' < ./templates/kind-config.yaml.template >output/kind-config.yaml
-sudo /usr/local/bin/kind create cluster --config=output/kind-config.yaml --image kindest/node:$KIND_NODE_IMAGE_TAG
+sudo /usr/local/bin/kind create cluster --config=output/kind-config.yaml --image ${BOOTSTRAP_CLUSTER_SERVER_IP}:5000/kind-node:${KIND_NODE_IMAGE_TAG%%@*}
 mkdir -p ~/.kube
 sudo cp /root/.kube/config ~/.kube
 sudo chown $USER:$USER ~/.kube/config
-sed -i "s/0.0.0.0:6443/$BOOTSTRAP_CLUSTER_SERVER_IP:6443/g" ~/.kube/config
+sed -i "s/:6443/$BOOTSTRAP_CLUSTER_SERVER_IP:6443/g" ~/.kube/config
 
 while true
 do
@@ -60,4 +83,27 @@ do
 	sleep 10
 done
 
+# Add the registry config to the nodes
+REGISTRY_DIR="/etc/containerd/certs.d/${BOOTSTRAP_CLUSTER_SERVER_IP}:5000"
+for node in $(sudo /usr/local/bin/kind get nodes); do
+          sudo podman exec "${node}" mkdir -p "${REGISTRY_DIR}"
+            cat <<EOF | sudo podman exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${BOOTSTRAP_CLUSTER_SERVER_IP}:5000"]
+EOF
+done
+
+# Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | /usr/local/bin/kubectl --kubeconfig /home/rocky/.kube/config apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "${BOOTSTRAP_CLUSTER_SERVER_IP}:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
 log_info "Bootstrap cluster created successfully. You can access bootstrap cluster using ~/.kube/config as a kubeconfig file"

02_create_bootstrap_cluster.sh

@@ -3,79 +3,23 @@
 set -e
 
 source lib/common.sh
+source lib/capi.sh
 
 if [ -z "$1" ]
   then
     echo "usage: $0 <assets dir>"
     exit 1
 fi
 
-provision_aws_iam_resources () {
-	cat templates/bootstrap-manager-account.yaml.template | envsubst > bootstrap-manager-account.yaml
-	clusterawsadm bootstrap iam update-cloudformation-stack --config bootstrap-manager-account.yaml
-}
+ASSET_DIR=${1%%/}
 
-aws_cli() {
-	sudo docker run --rm -it -v ~/.aws:/root/.aws public.ecr.aws/aws-cli/aws-cli $1
-}
-log_info "Preparing Cluster API providers initilizaiton"
-
-sudo cp $1/cluster-api/$CAPI_VERSION/clusterctl-linux-amd64 /usr/local/bin/clusterctl
-sudo chmod +x /usr/local/bin/clusterctl
-
-cat > clusterctl.yaml <<EOF
-providers:
-  - name: "cluster-api"
-    url: "file://localhost$(realpath $1)/cluster-api/$CAPI_VERSION/core-components.yaml"
-    type: "CoreProvider"
-  - name: "kubeadm"
-    url: "file://localhost$(realpath $1)/bootstrap-kubeadm/$CAPI_VERSION/bootstrap-components.yaml"
-    type: "BootstrapProvider"
-  - name: "kubeadm"
-    url: "file://localhost$(realpath $1)/control-plane-kubeadm/$CAPI_VERSION/control-plane-components.yaml"
-    type: "ControlPlaneProvider"
-  - name: "aws"
-    url: "file://localhost$(realpath $1)/infrastructure-aws/$CAPA_VERSION/infrastructure-components.yaml"
-    type: "InfrastructureProvider"
-  - name: "byoh"
-    url: "file://localhost$(realpath $1)/infrastructure-byoh/$BYOH_VERSION/infrastructure-components.yaml"
-    type: "InfrastructureProvider"
-EOF
+YQ_ASSETS_DIR="$ASSET_DIR/yq/$(ls $ASSET_DIR/yq | grep v)"
+YQ_PATH="$YQ_ASSETS_DIR/yq_linux_amd64"
+chmod +x $YQ_PATH
 
 log_info "Installing Cluster API providers"
-
-mkdir -p ~/.cluster-api
-cp clusterctl.yaml ~/.cluster-api/
-
-CAPI_NAMESPACE="cert-manager capi-system capi-kubeadm-bootstrap-system capi-kubeadm-control-plane-system"
-
-for provider in ${CAPI_INFRA_PROVIDERS[@]}
-do
-	case $provider in
-		"aws")
-			sudo cp $1/cluster-api-provider-aws/$CAPA_VERSION/clusterawsadm-linux-amd64 /usr/local/bin/clusterawsadm
-			sudo chmod +x /usr/local/bin/clusterawsadm
-
-			export AWS_REGION
-			export AWS_ACCESS_KEY_ID
-			export AWS_SECRET_ACCESS_KEY
-			export AWS_ACCOUNT_ID
-
-			gum confirm "Do you want to create IAM resources?" && provision_aws_iam_resources
-
-			export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile)
-			export EXP_MACHINE_POOL=true
-			export CAPA_EKS_IAM=true
-			export CAPA_EKS_ADD_ROLES=true
-			
-			CAPI_NAMESPACE+=" $provider-system"
-			;;
-		"byoh")
-			CAPI_NAMESPACE+=" $provider-system"
-			;;
-	esac
-done
-
-gum spin --spinner dot --title "Waiting for providers to be installed..." -- clusterctl init --infrastructure $(printf -v joined '%s,' "${CAPI_INFRA_PROVIDERS[@]}"; echo "${joined%,}") --wait-providers
+install_clusterctl
+prepare_capi_providers kind ${BOOTSTRAP_CLUSTER_SERVER_IP}:5000
+install_capi_providers kind ~/.kube/config
 
 log_info "...Done"

03_initialize_capi_providers.sh

@@ -3,6 +3,7 @@
 set -e
 
 source lib/common.sh
+source lib/capi.sh
 
 if [ -z "$1" ] || [ -z "$2" ]
   then
@@ -17,6 +18,10 @@ HELM_VALUE_K8S_ADDONS="--set cni.calico.enabled=true"
 IS_MANAGED_CLUSTER="false"
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+YQ_ASSETS_DIR="$ASSET_DIR/yq/$(ls $ASSET_DIR/yq | grep v)"
+YQ_PATH="$YQ_ASSETS_DIR/yq_linux_amd64"
+chmod +x $YQ_PATH
+
 export KUBECONFIG=~/.kube/config
 
 log_info "Creating TKS Admin Cluster via Cluster API"
@@ -83,6 +88,7 @@ EOF
 		else
 			clusterctl get kubeconfig $CLUSTER_NAME > output/kubeconfig_$CLUSTER_NAME
 			chmod 600 output/kubeconfig_$CLUSTER_NAME
+			export KUBECONFIG=output/kubeconfig_$CLUSTER_NAME
 			helm upgrade -i k8s-addons $ASSET_DIR/taco-helm/kubernetes-addons $HELM_VALUE_K8S_ADDONS
 			helm upgrade -i aws-ebs-csi-driver --namespace kube-system $ASSET_DIR/aws-ebs-csi-driver-helm/aws-ebs-csi-driver
 		fi
@@ -93,7 +99,7 @@ EOF
 		chmod 600 output/kubeconfig_$CLUSTER_NAME
 		export KUBECONFIG=output/kubeconfig_$CLUSTER_NAME
 		kubectl apply -f $ASSET_DIR/calico/calico.yaml
-		helm upgrade -i local-path-provisioner --namespace kube-system --set storageClass.name=taco-storage $ASSET_DIR/local-path-provisioner/deploy/chart/local-path-provisioner
+		helm upgrade -i local-path-provisioner --namespace kube-system --set storageClass.name=taco-storage --set image.repository=${BOOTSTRAP_CLUSTER_SERVER_IP}:5000/local-path-provisioner  $ASSET_DIR/local-path-provisioner/deploy/chart/local-path-provisioner
 		;;
 esac
 
@@ -102,66 +108,48 @@ for node in $(kubectl get no -o jsonpath='{.items[*].metadata.name}');do
 	kubectl wait --for=condition=Ready no/$node
 done
 echo "-----"
+case $TKS_ADMIN_CLUSTER_INFRA_PROVIDER in
+	"byoh")
+		log_info "remove taint from the interim node in BYOH"
+		for no in $(kubectl get no -o name); do
+			BYOH_TMP_NODE=${no#*/}
+			kubectl taint nodes $no node-role.kubernetes.io/control-plane:NoSchedule- || true
+		done
+		;;
+esac
+
 kubectl get no
 log_info  "Make sure all node status are ready"
 
-install_capi_to_admin_cluster() {
-	export KUBECONFIG=output/kubeconfig_$CLUSTER_NAME
-	log_info "Initializing cluster API provider components in TKS admin cluster"
-	for provider in ${CAPI_INFRA_PROVIDERS[@]}; do
-		case $provider in
-			"aws")
-				export AWS_REGION
-				export AWS_ACCESS_KEY_ID
-				export AWS_SECRET_ACCESS_KEY
-
-				export AWS_B64ENCODED_CREDENTIALS=$(clusterawsadm bootstrap credentials encode-as-profile)
-				export EXP_MACHINE_POOL=true
-				export CAPA_EKS_IAM=true
-				export CAPA_EKS_ADD_ROLES=true
-
-				CAPI_PROVIDER_NS=capa-system
-				;;
-			"byoh")
-				CAPI_PROVIDER_NS=byoh-system
-				;;
-		esac
-	done
-
-	case $TKS_ADMIN_CLUSTER_INFRA_PROVIDER in
-		"byoh")
-			for no in $(kubectl get no -o name); do
-				BYOH_TMP_NODE=${no#*/}
-				kubectl taint nodes $no node-role.kubernetes.io/control-plane:NoSchedule- || true
-			done
-			;;
-	esac
-	gum spin --spinner dot --title "Waiting for providers to be installed..." -- clusterctl init --infrastructure $(printf -v joined '%s,' "${CAPI_INFRA_PROVIDERS[@]}"; echo "${joined%,}") --wait-providers
-}
-
-install_capi_to_admin_cluster
+log_info "Initializing cluster API provider components in TKS admin cluster"
+prepare_capi_providers admin ${BOOTSTRAP_CLUSTER_SERVER_IP}:5000
+install_capi_providers admin output/kubeconfig_$CLUSTER_NAME
 
 move_byoh_resources() {
 	rc_kind=$1
+	src_k8s_kubeconfig=$2
+	dst_k8s_kubeconfig=$3
 
 	log_info "Move $rc_kind BYOH resources to the admin cluster"
 
-	export KUBECONFIG=~/.kube/config
-	for rc in $(kubectl get $rc_kind -o name); do
+	for rc in $(kubectl get --kubeconfig $src_k8s_kubeconfig $rc_kind -o name); do
 		rc_file=${rc#*/}
-		kubectl get $rc -o yaml | egrep -v 'uid|resourceVersion|creationTimestamp|generation' > output/$rc_kind-"$rc_file".yaml
-		kubectl apply --kubeconfig output/kubeconfig_$CLUSTER_NAME -f output/$rc_kind-"$rc_file".yaml
+		kubectl get --kubeconfig $src_k8s_kubeconfig $rc -o yaml | egrep -v 'uid|resourceVersion|creationTimestamp|generation' > output/$rc_kind-"$rc_file".yaml
+		kubectl apply --kubeconfig $dst_k8s_kubeconfig -f output/$rc_kind-"$rc_file".yaml
 	done
 }
 
 if [ $TKS_ADMIN_CLUSTER_INFRA_PROVIDER == "byoh" ]; then
-       gum spin --spinner dot --title "Deleting BYOH infra provider for a moment..." -- clusterctl delete --infrastructure byoh
+       log_info "Deleting BYOH infra provider for a moment..."
+       kubectl delete -f output/admin-byoh-infra.yaml
+       ## XXX: check no pod in byoh-system
 
-       move_byoh_resources byoh
-       move_byoh_resources k8sinstallerconfigtemplates
+       move_byoh_resources byoh ~/.kube/config output/kubeconfig_$CLUSTER_NAME
+       move_byoh_resources k8sinstallerconfigtemplates ~/.kube/config output/kubeconfig_$CLUSTER_NAME
 
-       export KUBECONFIG=output/kubeconfig_$CLUSTER_NAME
-       gum spin --spinner dot --title "Reinstalling BYOH infra provider..." -- clusterctl init --infrastructure $(printf -v joined '%s,' "${CAPI_INFRA_PROVIDERS[@]}"; echo "${joined%,}") --wait-providers
+       kubectl apply -f output/admin-byoh-infra.yaml
+       sleep 10
+       ./util/wait_for_all_pods_in_ns.sh byoh-system
 
        kubectl patch deploy -n byoh-system  byoh-controller-manager --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/resources/limits/memory", "value":"1000Mi"}]'
        kubectl patch deploy -n byoh-system  byoh-controller-manager --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/resources/limits/cpu", "value":"1000m"}]'

04_create_tks-admin_cluster.sh

@@ -321,6 +321,8 @@ case $TKS_ADMIN_CLUSTER_INFRA_PROVIDER in
 	# BYOH
 	curl -v --user tks_admin:$GIT_SVC_TOKEN -X DELETE $GIT_SVC_HTTP://${GIT_SVC_BASE_URL}/api/packages/decapod10/generic/byoh_hostagent/$BYOH_TKS_VERSION/byoh-hostagent-linux-amd64 || true
 	curl -v --user tks_admin:$GIT_SVC_TOKEN --upload-file output/byoh-hostagent $GIT_SVC_HTTP://${GIT_SVC_BASE_URL}/api/packages/decapod10/generic/byoh_hostagent/$BYOH_TKS_VERSION/byoh-hostagent-linux-amd64
+	curl -v --user tks_admin:$GIT_SVC_TOKEN -X DELETE $GIT_SVC_HTTP://${GIT_SVC_BASE_URL}/api/packages/decapod10/generic/imgpkg/$BYOH_TKS_VERSION/imgpkg || true
+	curl -v --user tks_admin:$GIT_SVC_TOKEN --upload-file output/imgpkg $GIT_SVC_HTTP://${GIT_SVC_BASE_URL}/api/packages/decapod10/generic/imgpkg/$BYOH_TKS_VERSION/imgpkg
 	;;
 esac
 
@@ -338,8 +340,9 @@ kubectl create secret generic tks-admin-kubeconfig-secret -n argo --from-file=va
 kubectl delete secret byoh-hostagent-install-template -n argo || true
 cp templates/install_byoh_hostagent.sh.template templates/install_byoh_hostagent.sh.template.orig
 HOSTAGENT_CHECKSUM=$(sha1sum output/byoh-hostagent | awk '{print $1}')
-export HOSTAGENT_CHECKSUM BYOH_TKS_VERSION GIT_SVC_USERNAME GITEA_NODE_PORT GITEA_NODE_IP
-envsubst '$HOSTAGENT_CHECKSUM $BYOH_TKS_VERSION $GIT_SVC_USERNAME $GITEA_NODE_IP $GITEA_NODE_PORT' <templates/install_byoh_hostagent.sh.template.orig >templates/install_byoh_hostagent.sh.template
+IMGPKG_CHECKSUM=$(sha1sum output/imgpkg | awk '{print $1}')
+export HOSTAGENT_CHECKSUM IMGPKG_CHECKSUM BYOH_TKS_VERSION GIT_SVC_USERNAME GITEA_NODE_PORT GITEA_NODE_IP
+envsubst '$HOSTAGENT_CHECKSUM $IMGPKG_CHECKSUM $BYOH_TKS_VERSION $GIT_SVC_USERNAME $GITEA_NODE_IP $GITEA_NODE_PORT' <templates/install_byoh_hostagent.sh.template.orig >templates/install_byoh_hostagent.sh.template
 kubectl create secret generic byoh-hostagent-install-template -n argo --from-file=agent-install-template=templates/install_byoh_hostagent.sh.template
 
 log_info "...Done"

05_install_decapod.sh

@@ -5,18 +5,19 @@ set -e
 source lib/common.sh
 
 function usage {
-        echo -e "\nUsage: $0 <kubeconfig> <hostname> [--output OUTPUT_SCRIPT_PATH]"
+        echo -e "\nUsage: $0 <kubeconfig> <container registry> <hostname> [--output OUTPUT_SCRIPT_PATH]"
 	echo -e "\n\tOUTPUT_SCRIPT_PATH (default): output/install_byoh_hostagent-<hostname>.sh"
         exit 1
 }
 
-[[ $# -ge 2 ]] || usage 
+[[ $# -ge 3 ]] || usage
 
 inarray=$(echo ${CAPI_INFRA_PROVIDERS[@]} | grep -ow "byoh" | wc -w)
 [[ $inarray -ne 0 ]] || log_error "byoh infra provider is not configured"
 
 export KUBECONFIG=$1
-HOSTNAME=$2
+export TARGET_REGISTRY=$2
+HOSTNAME=$3
 shift
 OUTPUT_SCRIPT_PATH=output/install_byoh_hostagent-$HOSTNAME.sh
 
@@ -78,18 +79,17 @@ sleep 3
 kubectl get bootstrapkubeconfig bootstrap-kubeconfig-$HOSTNAME -n default -o=jsonpath='{.status.bootstrapKubeconfigData}' > output/bootstrap-kubeconfig-$HOSTNAME.conf
 
 if kubectl get no | grep kind; then
-	export KIND_PORT=$(sudo docker inspect kind-control-plane -f '{{ $published := index .NetworkSettings.Ports "6443/tcp" }}{{ range $published }}{{ .HostPort }}{{ end }}')
+	export KIND_PORT=$(sudo podman inspect kind-control-plane -f '{{ $published := index .NetworkSettings.Ports "6443/tcp" }}{{ range $published }}{{ .HostPort }}{{ end }}')
 	sed -i 's/    server\:.*/    server\: https\:\/\/'"$BOOTSTRAP_CLUSTER_SERVER_IP:$KIND_PORT"'/g' output/bootstrap-kubeconfig-$HOSTNAME.conf
 fi
 
 # the variable to be substituded for BYOH host agent install script
 BOOTSTRAP_KUBECONFIG=$(cat output/bootstrap-kubeconfig-$HOSTNAME.conf | base64 -w 0)
 GITEA_NODE_PORT=$(kubectl get -n gitea -o jsonpath="{.spec.ports[0].nodePort}" services gitea-http 2>/dev/null) || true
 GITEA_NODE_IP=$(kubectl get no -ojsonpath='{.items[0].status.addresses[0].address}') || true
-export BOOTSTRAP_KUBECONFIG GITEA_NODE_IP GITEA_NODE_PORT GIT_SVC_USERNAME
-
-envsubst '$BOOTSTRAP_KUBECONFIG $GITEA_NODE_IP $GITEA_NODE_PORT $GIT_SVC_USERNAME' < ./templates/install_byoh_hostagent.sh.template >$OUTPUT_SCRIPT_PATH
+export BOOTSTRAP_CLUSTER_SERVER_IP BOOTSTRAP_KUBECONFIG GITEA_NODE_IP GITEA_NODE_PORT GIT_SVC_USERNAME
+envsubst '$BOOTSTRAP_CLUSTER_SERVER_IP $BOOTSTRAP_KUBECONFIG $GITEA_NODE_IP $GITEA_NODE_PORT $GIT_SVC_USERNAME $TARGET_REGISTRY' < ./templates/install_byoh_hostagent.sh.template >$OUTPUT_SCRIPT_PATH
 chmod +x $OUTPUT_SCRIPT_PATH
 
 log_info "Copy below files to each host and run a script!"
-echo "$OUTPUT_SCRIPT_PATH output/byoh-hostagent"
+echo "$OUTPUT_SCRIPT_PATH output/byoh-hostagent output/imgpkg"

byoh_generate_script_for_host.sh

@@ -37,3 +37,5 @@ DATABASE_PASSWORD=secretPassword
 GITEA_ADMIN_USER=tks-admin
 GITEA_ADMIN_PASSWORD=secretPassword
 
+# Admin Cluster
+ADMIN_KUBE_VERSION=1.26.14