From 0a374aa5b1ed98464b62c875940a89de2089ed14 Mon Sep 17 00:00:00 2001 From: JSpon <115185500+JSpon@users.noreply.github.com> Date: Thu, 23 May 2024 14:43:08 -0400 Subject: [PATCH 1/3] Allow for scripts in CKS ISO to fully control CKS deployment --- .../resources/conf/k8s-control-node-add.yml | 16 ++ .../main/resources/conf/k8s-control-node.yml | 16 ++ .../src/main/resources/conf/k8s-node.yml | 16 ++ scripts/util/cks-samples/README.md | 24 +++ .../control-node-add/deploy-kube-system.tmpl | 27 +++ .../control-node-add/setup-containerd.tmpl | 13 ++ .../control-node-add/setup-kube-system | 138 ++++++++++++++ .../control-node/deploy-kube-system.tmpl | 62 +++++++ .../control-node/setup-containerd.tmpl | 13 ++ .../scripts/control-node/setup-kube-system | 140 ++++++++++++++ .../scripts/node/deploy-kube-system.tmpl | 22 +++ .../basic/scripts/node/setup-containerd.tmpl | 13 ++ .../basic/scripts/node/setup-kube-system | 137 ++++++++++++++ .../cilium/create-kubernetes-binaries-iso.sh | 174 ++++++++++++++++++ .../helm-overrides/cilium-overrides.yaml | 6 + .../control-node-add/deploy-kube-system.tmpl | 27 +++ .../control-node-add/setup-containerd.tmpl | 13 ++ .../control-node-add/setup-kube-system | 138 ++++++++++++++ .../control-node/deploy-kube-system.tmpl | 64 +++++++ .../control-node/setup-containerd.tmpl | 13 ++ .../scripts/control-node/setup-kube-system | 163 ++++++++++++++++ .../scripts/node/deploy-kube-system.tmpl | 22 +++ .../cilium/scripts/node/setup-containerd.tmpl | 13 ++ .../cilium/scripts/node/setup-kube-system | 137 ++++++++++++++ .../util/create-kubernetes-binaries-iso.sh | 4 + 25 files changed, 1411 insertions(+) create mode 100644 scripts/util/cks-samples/README.md create mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system create mode 100644 scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system create mode 100644 scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/basic/scripts/node/setup-kube-system create mode 100755 scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh create mode 100644 scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system create mode 100644 scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl create mode 100644 scripts/util/cks-samples/cilium/scripts/node/setup-kube-system diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 2c18efa01891..3e25ce433a9b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -89,6 +89,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/control-node-add/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system for the second stage + if /bin/bash ${BINARIES_DIR}/scripts/control-node-add/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.ha.certificate.key }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on a control plane node add from ISO." + else + rc=$? + echo "Failed running setup-kube-system on a control plane node add from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index aa7eec97ac82..a896c6028e84 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -109,6 +109,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/control-node/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc + if /bin/bash ${BINARIES_DIR}/scripts/control-node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.initargs }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on control plane node from ISO." + else + rc=$? + echo "Failed running setup-kube-system on control plane node from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index de1f4c9ffc70..ed525b8f602f 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -89,6 +89,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/node/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc + if /bin/bash ${BINARIES_DIR}/scripts/node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on worker node from ISO." + else + rc=$? + echo "Failed running setup-kube-system on worker node from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/scripts/util/cks-samples/README.md b/scripts/util/cks-samples/README.md new file mode 100644 index 000000000000..c9210e69c771 --- /dev/null +++ b/scripts/util/cks-samples/README.md @@ -0,0 +1,24 @@ +# CloudStack Kubernetes Service Cloud-init Overrides - Examples + +Example files that can be used when creating the CloudStack Kubernetes Service ISO file to override the normal behavior of the Kubernetes cluster creation + + +## Usage + +### The basic directory + +In the basic directory, there is a scripts directory that can be used as a template to start the customization. It is basically the original cloud-init scripts, but they can then be modified as needed for the CKS cluster. To start, just copy the scripts directory into the util directory, peer with the create-kubernetes-binaries-iso.sh script. + +### The cilium directory + +In the cilium directory, there is a modified version of create-kubernetes-binaries-iso.sh which contains what would be needed to install Helm and install Cilium instead of Weave. Helm needs to be installed and in the path on the OS where the ISO build will be run. There is no need to customize anything under the cilium directory if you are only looking to run Cilium instead of Weave, but just like the basic directory, other aspects can be customized. + + +## References in CloudStack + +The following cloud-init scripts will call the setup-kube-system scripts found here when the cluster is created: + +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml + diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl new file mode 100644 index 000000000000..77c97786ca04 --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl @@ -0,0 +1,27 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 +fi + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +modprobe ip_vs +modprobe ip_vs_wrr +modprobe ip_vs_sh +modprobe nf_conntrack +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + +mkdir -p /root/.kube +cp -i /etc/kubernetes/admin.conf /root/.kube/config +chown $(id -u):$(id -g) /root/.kube/config + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success + diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system new file mode 100644 index 000000000000..a2e3773940cf --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system @@ -0,0 +1,138 @@ +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl new file mode 100644 index 000000000000..194e84c5b516 --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl @@ -0,0 +1,62 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then +echo "Already provisioned!" +exit 0 +fi + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 +while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm init failed!" + exit 1 + fi + retval=0 + set +e + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] +done + +K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +export KUBECONFIG=/etc/kubernetes/admin.conf + +mkdir -p /root/.kube +cp -i /etc/kubernetes/admin.conf /root/.kube/config +chown $(id -u):$(id -g) /root/.kube/config +echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + +if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then + ### Network, dashboard configs available offline ### + echo "Offline configs are available!" + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml + rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" +else + /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" + /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml +fi + +/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true +/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true +/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system new file mode 100644 index 000000000000..510334192cc4 --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system @@ -0,0 +1,140 @@ +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl new file mode 100644 index 000000000000..d2e37f04c588 --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl @@ -0,0 +1,22 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then +echo "Already provisioned!" +exit 0 +fi + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +modprobe ip_vs +modprobe ip_vs_wrr +modprobe ip_vs_sh +modprobe nf_conntrack +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/node/setup-kube-system new file mode 100644 index 000000000000..8e48e23bae99 --- /dev/null +++ b/scripts/util/cks-samples/basic/scripts/node/setup-kube-system @@ -0,0 +1,137 @@ +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh b/scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh new file mode 100755 index 000000000000..cbb0588befe0 --- /dev/null +++ b/scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh @@ -0,0 +1,174 @@ +#!/bin/bash -x +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +if [ $# -lt 8 ]; then + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION CILIUM_VERSION HELM_VERSION DASHBOARD_YAML_CONFIG BUILD_NAME" + echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.27.14 1.4.0 1.29.0 1.15.1 3.14.2 https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml setup-v1.27.14" + exit 1 +fi + +RELEASE="v${2}" +VAL="1.18.0" +output_dir="${1}" +start_dir="$PWD" +iso_dir="/tmp/iso" +working_dir="${iso_dir}/" +mkdir -p "${working_dir}" +build_name="${8}.iso" +[ -z "${build_name}" ] && build_name="setup-${RELEASE}.iso" + +CNI_VERSION="v${3}" +echo "Downloading CNI ${CNI_VERSION}..." +cni_dir="${working_dir}/cni/" +mkdir -p "${cni_dir}" +cni_status_code=$(curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz") +if [[ ${cni_status_code} -eq 404 ]] ; then + curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz" +fi + +CRICTL_VERSION="v${4}" +echo "Downloading CRI tools ${CRICTL_VERSION}..." +crictl_dir="${working_dir}/cri-tools/" +mkdir -p "${crictl_dir}" +curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-linux-amd64.tar.gz" + +echo "Downloading Kubernetes tools ${RELEASE}..." +k8s_dir="${working_dir}/k8s" +mkdir -p "${k8s_dir}" +cd "${k8s_dir}" +curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +kubeadm_file_permissions=`stat --format '%a' kubeadm` +chmod +x kubeadm + +echo "Downloading kubelet.service ${RELEASE}..." +cd "${start_dir}" +kubelet_service_file="${working_dir}/kubelet.service" +touch "${kubelet_service_file}" +if [[ `echo "${2} $VAL" | awk '{print ($1 < $2)}'` == 1 ]]; then + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} +else + curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} +fi + +echo "Downloading 10-kubeadm.conf ${RELEASE}..." +kubeadm_conf_file="${working_dir}/10-kubeadm.conf" +touch "${kubeadm_conf_file}" +if [[ `echo "${2} $val" | awk '{print ($1 < $2)}'` == 1 ]]; then + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} +else + curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} +fi + +HELM_VERSION="${6}" +mkdir -p ${iso_dir}/installs +curl -L -o ${iso_dir}/installs/helm-v${HELM_VERSION}-linux-amd64.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz + +CILIUM_VERSION="${5}" +echo "Downloading Cilium Helm chart version ${CILIUM_VERSION} and cilium cli" +mkdir -p ${iso_dir}/installs/charts +curl -Lo ${iso_dir}/installs/charts/cilium-${CILIUM_VERSION}.tgz https://helm.cilium.io/cilium-${CILIUM_VERSION}.tgz + +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) +CLI_ARCH=amd64 +curl -Lo ${iso_dir}/installs/cilium-linux-${CLI_ARCH}-cli-${CILIUM_CLI_VERSION}.tar.gz --fail https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz + +DASHBOARD_CONFIG_URL="${7}" +echo "Downloading dashboard config ${DASHBOARD_CONFIG_URL}" +dashboard_conf_file="${working_dir}/dashboard.yaml" +curl -sSL ${DASHBOARD_CONFIG_URL} -o ${dashboard_conf_file} + +# TODO : Change the url once merged +AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" +echo "Downloading kubernetes cluster autoscaler ${AUTOSCALER_URL}" +autoscaler_conf_file="${working_dir}/autoscaler.yaml" +curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} + +PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml" +echo "Downloading kubernetes cluster provider ${PROVIDER_URL}" +provider_conf_file="${working_dir}/provider.yaml" +curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} + +echo "Fetching k8s docker images..." +ctr -v +if [ $? -ne 0 ]; then + echo "Installing containerd..." + if [ -f /etc/redhat-release ]; then + sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine + sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs + sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm + sudo yum install -y containerd.io + elif [ -f /etc/lsb-release ]; then + sudo apt update && sudo apt install containerd.io -y + fi + sudo systemctl enable containerd && sudo systemctl start containerd +fi +mkdir -p "${working_dir}/docker" +output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` + +# Don't forget about the yaml images ! +for i in ${network_conf_file} ${dashboard_conf_file} +do + images=`grep "image:" $i | cut -d ':' -f2- | tr -d ' ' | tr -d "'"` + output=`printf "%s\n" ${output} ${images}` +done + +# Don't forget about the other image ! +autoscaler_image=`grep "image:" ${autoscaler_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${autoscaler_image}` + +provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${provider_image}` + +cp -r helm-overrides/ ${iso_dir}/installs + +chart_images=`for chart in ${iso_dir}/installs/charts/*; do chartfile=${chart##*/}; chartbase=${chartfile%-*}; if [ -f ${iso_dir}/installs/helm-overrides/${chartbase}-overrides.yaml ]; then helm template -f ${iso_dir}/installs/helm-overrides/${chartbase}-overrides.yaml ${chart}; else helm template ${chart}; fi; done | grep "[[:space:]]image:" | cut -d ':' -f2- | tr -d ' ' | tr -d '\r' | tr -d "'" | tr -d '"' | sort -u` +output=`printf "%s\n" ${output} ${chart_images}` + +while read -r line; do + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + if [ ! -z "${https_proxy}" ]; then + sudo https_proxy=${https_proxy} ctr image pull "$line" + else + sudo ctr image pull "$line" + fi + image_name=`echo "$line" | grep -oE "[^/]+$"` + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" +done <<< "$output" + +if [ -d scripts ]; then + /bin/cp -r scripts ${iso_dir} +fi + +echo "Restore kubeadm permissions..." +if [ -z "${kubeadm_file_permissions}" ]; then + kubeadm_file_permissions=644 +fi +chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" + +echo "Updating imagePullPolicy to IfNotPresent in yaml files..." +sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml + +mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}" + +rm -rf "${iso_dir}" diff --git a/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml b/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml new file mode 100644 index 000000000000..355ed0d8b063 --- /dev/null +++ b/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml @@ -0,0 +1,6 @@ +image: + useDigest: false + +operator: + image: + useDigest: false diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl new file mode 100644 index 000000000000..77c97786ca04 --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl @@ -0,0 +1,27 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 +fi + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +modprobe ip_vs +modprobe ip_vs_wrr +modprobe ip_vs_sh +modprobe nf_conntrack +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + +mkdir -p /root/.kube +cp -i /etc/kubernetes/admin.conf /root/.kube/config +chown $(id -u):$(id -g) /root/.kube/config + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success + diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system new file mode 100644 index 000000000000..a2e3773940cf --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system @@ -0,0 +1,138 @@ +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl new file mode 100644 index 000000000000..9ce17720bd94 --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl @@ -0,0 +1,64 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then +echo "Already provisioned!" +exit 0 +fi + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 +while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm init failed!" + exit 1 + fi + retval=0 + set +e + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] +done + +K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ +K8S_HELM=/tmp/k8shelm/ + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +export KUBECONFIG=/etc/kubernetes/admin.conf + +mkdir -p /root/.kube +cp -i /etc/kubernetes/admin.conf /root/.kube/config +chown $(id -u):$(id -g) /root/.kube/config +echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + +if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then + ### Network, dashboard configs available offline ### + echo "Offline configs are available!" + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml + rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" +else + /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml +fi + +helm upgrade --install cilium ${K8S_HELM}/charts/cilium-*.tgz -f ${K8S_HELM}/overrides/cilium-overrides.yaml -n kube-system +/bin/rm -rf ${K8S_HELM} + +/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true +/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true +/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system new file mode 100644 index 000000000000..c4d9ece1c1d6 --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system @@ -0,0 +1,163 @@ +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ +K8S_HELM=/tmp/k8shelm/ +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + if [ -f ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz ] + then + TMPDIR=$(mktemp -d) + tar -xzf ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz -C ${TMPDIR} + /bin/mv ${TMPDIR}/linux-amd64/helm /usr/local/bin/helm + /bin/rm -rf ${TMPDIR} + fi + + if [ -f ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz ] + then + tar -xzf ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz -C /usr/local/bin + chmod 755 /usr/local/bin/cilium + fi + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + mkdir -p "${K8S_HELM}/charts" + if [ -d "${BINARIES_DIR}/installs/charts" ]; then + cp ${BINARIES_DIR}/installs/charts/*.*gz "${K8S_HELM}/charts" + fi + mkdir -p "${K8S_HELM}/overrides" + if [ -d "${BINARIES_DIR}/installs/helm-overrides" ]; then + cp ${BINARIES_DIR}/installs/helm-overrides/*.yaml "${K8S_HELM}/overrides" + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl new file mode 100644 index 000000000000..d2e37f04c588 --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl @@ -0,0 +1,22 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then +echo "Already provisioned!" +exit 0 +fi + +if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 +fi +modprobe ip_vs +modprobe ip_vs_wrr +modprobe ip_vs_sh +modprobe nf_conntrack +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi +kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl new file mode 100644 index 000000000000..29955cb138fb --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl @@ -0,0 +1,13 @@ +#!/bin/bash -e + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl daemon-reload +systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system new file mode 100644 index 000000000000..8e48e23bae99 --- /dev/null +++ b/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system @@ -0,0 +1,137 @@ +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +crucial_cmd_attempts=1 + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + +fi +if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + +if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done +fi diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index d5fb014f220b..cc72ec240221 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -128,6 +128,10 @@ output=`printf "%s\n" ${output} ${autoscaler_image}` provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '` output=`printf "%s\n" ${output} ${provider_image}` +if [ -d scripts ]; then + /bin/cp -r scripts ${iso_dir} +fi + while read -r line; do echo "Downloading image $line ---" if [[ $line == kubernetesui* ]] || [[ $line == apache* ]] || [[ $line == weaveworks* ]]; then From 0d7cc4f47d88afe093c4aebaf5b6d295b6bdf2aa Mon Sep 17 00:00:00 2001 From: JSpon <115185500+JSpon@users.noreply.github.com> Date: Tue, 4 Jun 2024 10:01:15 -0400 Subject: [PATCH 2/3] Add licenses --- scripts/util/cks-samples/README.md | 19 +++++++++++++++++++ .../control-node-add/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../control-node-add/setup-containerd.tmpl | 16 ++++++++++++++++ .../control-node-add/setup-kube-system | 16 ++++++++++++++++ .../control-node/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../control-node/setup-containerd.tmpl | 16 ++++++++++++++++ .../scripts/control-node/setup-kube-system | 16 ++++++++++++++++ .../scripts/node/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../basic/scripts/node/setup-containerd.tmpl | 16 ++++++++++++++++ .../basic/scripts/node/setup-kube-system | 16 ++++++++++++++++ .../helm-overrides/cilium-overrides.yaml | 17 +++++++++++++++++ .../control-node-add/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../control-node-add/setup-containerd.tmpl | 16 ++++++++++++++++ .../control-node-add/setup-kube-system | 16 ++++++++++++++++ .../control-node/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../control-node/setup-containerd.tmpl | 16 ++++++++++++++++ .../scripts/control-node/setup-kube-system | 16 ++++++++++++++++ .../scripts/node/deploy-kube-system.tmpl | 16 ++++++++++++++++ .../cilium/scripts/node/setup-containerd.tmpl | 16 ++++++++++++++++ .../cilium/scripts/node/setup-kube-system | 16 ++++++++++++++++ 20 files changed, 324 insertions(+) diff --git a/scripts/util/cks-samples/README.md b/scripts/util/cks-samples/README.md index c9210e69c771..50717e4ed1fc 100644 --- a/scripts/util/cks-samples/README.md +++ b/scripts/util/cks-samples/README.md @@ -22,3 +22,22 @@ The following cloud-init scripts will call the setup-kube-system scripts found h * plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml * plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +## License + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl index 77c97786ca04..67be0e79bf6b 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system index a2e3773940cf..f9e77fdeb20b 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system +++ b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 9 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 diff --git a/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl index 194e84c5b516..16a6a9f9d5dc 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl +++ b/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system index 510334192cc4..09462832fe40 100644 --- a/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system +++ b/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 8 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 diff --git a/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl index d2e37f04c588..266bdfb0fe6c 100644 --- a/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl +++ b/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/node/setup-kube-system index 8e48e23bae99..ddd4f8b4c13a 100644 --- a/scripts/util/cks-samples/basic/scripts/node/setup-kube-system +++ b/scripts/util/cks-samples/basic/scripts/node/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 8 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 diff --git a/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml b/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml index 355ed0d8b063..7fe557862c97 100644 --- a/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml +++ b/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + image: useDigest: false diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl index 77c97786ca04..67be0e79bf6b 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system index a2e3773940cf..f9e77fdeb20b 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system +++ b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 9 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl index 9ce17720bd94..7e6c61c30663 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system index c4d9ece1c1d6..1180e4c6a9fe 100644 --- a/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system +++ b/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 8 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 diff --git a/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl index d2e37f04c588..266bdfb0fe6c 100644 --- a/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl index 29955cb138fb..98f2e45b37a0 100644 --- a/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl +++ b/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system index 8e48e23bae99..ddd4f8b4c13a 100644 --- a/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system +++ b/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system @@ -1,4 +1,20 @@ #!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. if [ $# -lt 8 ]; then echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 From d1f2c6c06b47302e047d9ec16e03ce16891c6ab7 Mon Sep 17 00:00:00 2001 From: JSpon <115185500+JSpon@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:56:32 -0400 Subject: [PATCH 3/3] Create scripts to extract scripts, removing duplication --- scripts/util/cks-samples/README.md | 22 +- .../control-node-add/deploy-kube-system.tmpl | 43 ---- .../control-node-add/setup-containerd.tmpl | 29 --- .../control-node-add/setup-kube-system | 154 ------------- .../control-node/deploy-kube-system.tmpl | 78 ------- .../control-node/setup-containerd.tmpl | 29 --- .../scripts/control-node/setup-kube-system | 156 ------------- .../scripts/node/deploy-kube-system.tmpl | 38 ---- .../basic/scripts/node/setup-containerd.tmpl | 29 --- .../basic/scripts/node/setup-kube-system | 153 ------------- .../control-node-add/deploy-kube-system.tmpl | 43 ---- .../control-node-add/setup-containerd.tmpl | 29 --- .../control-node-add/setup-kube-system | 154 ------------- .../control-node/deploy-kube-system.tmpl | 80 ------- .../control-node/setup-containerd.tmpl | 29 --- .../scripts/control-node/setup-kube-system | 179 --------------- .../scripts/node/deploy-kube-system.tmpl | 38 ---- .../cilium/scripts/node/setup-containerd.tmpl | 29 --- .../cilium/scripts/node/setup-kube-system | 153 ------------- ... create-kubernetes-binaries-iso-cilium.sh} | 4 +- .../util/cks-samples/create-scripts-cilium.sh | 211 ++++++++++++++++++ scripts/util/cks-samples/create-scripts.sh | 148 ++++++++++++ .../helm-overrides/cilium-overrides.yaml | 0 23 files changed, 379 insertions(+), 1449 deletions(-) delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system delete mode 100644 scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/basic/scripts/node/setup-kube-system delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system delete mode 100644 scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl delete mode 100644 scripts/util/cks-samples/cilium/scripts/node/setup-kube-system rename scripts/util/cks-samples/{cilium/create-kubernetes-binaries-iso.sh => create-kubernetes-binaries-iso-cilium.sh} (95%) create mode 100644 scripts/util/cks-samples/create-scripts-cilium.sh create mode 100644 scripts/util/cks-samples/create-scripts.sh rename scripts/util/cks-samples/{cilium => }/helm-overrides/cilium-overrides.yaml (100%) diff --git a/scripts/util/cks-samples/README.md b/scripts/util/cks-samples/README.md index 50717e4ed1fc..4ebad8986ac6 100644 --- a/scripts/util/cks-samples/README.md +++ b/scripts/util/cks-samples/README.md @@ -5,13 +5,27 @@ Example files that can be used when creating the CloudStack Kubernetes Service I ## Usage -### The basic directory +### The basic template - create-scripts.sh -In the basic directory, there is a scripts directory that can be used as a template to start the customization. It is basically the original cloud-init scripts, but they can then be modified as needed for the CKS cluster. To start, just copy the scripts directory into the util directory, peer with the create-kubernetes-binaries-iso.sh script. +If you want to override the CKS installation starting with the basic CKS installation scripts, just run: -### The cilium directory +``` +bash create-scripts.sh ${CLOUDSTACK_REPO_HOME}/plugins/integrations/kubernetes-service/src/main/resources/conf ${CLOUDSTACK_REPO_HOME}/scripts/util/scripts +``` -In the cilium directory, there is a modified version of create-kubernetes-binaries-iso.sh which contains what would be needed to install Helm and install Cilium instead of Weave. Helm needs to be installed and in the path on the OS where the ISO build will be run. There is no need to customize anything under the cilium directory if you are only looking to run Cilium instead of Weave, but just like the basic directory, other aspects can be customized. +You can make any adjustments to the scripts under ${CLOUDSTACK_REPO_HOME}/scripts/util/scripts as necessary. Then you can run create-kubernetes-binaries-iso.sh in the parent directory as you would normally to create your ISO. + +### Cilium - create-scripts-cilium.sh + +If you want a CKS installation that installs both Cilium (instead of Weave) and Helm, you can run the Cilium create script like so: + +``` +bash create-scripts-cilium.sh ${CLOUDSTACK_REPO_HOME}/plugins/integrations/kubernetes-service/src/main/resources/conf ${CLOUDSTACK_REPO_HOME}/scripts/util/cks-samples/scripts +``` + +You can make any adjustments to the scripts under ${CLOUDSTACK_REPO_HOME}/scripts/util/cks-samples/scripts as necessary, although by default, the scripts should install what is necessary to install Cilium. There are helm-overrides for Cilium under the helm-overrides directory that can be modified as well. + +Once any adjustments are made (not required), you can run create-kubernetes-binaries-iso-cilium.sh in a similar fashion to how create-kubernetes-binaries-iso.sh would be run, replacing the Weave URL with the Cilium version and Helm version (see examples in the usage). ## References in CloudStack diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl deleted file mode 100644 index 67be0e79bf6b..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 -fi - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -modprobe ip_vs -modprobe ip_vs_wrr -modprobe ip_vs_sh -modprobe nf_conntrack -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - -mkdir -p /root/.kube -cp -i /etc/kubernetes/admin.conf /root/.kube/config -chown $(id -u):$(id -g) /root/.kube/config - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success - diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system deleted file mode 100644 index f9e77fdeb20b..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 9 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_JOIN_IP="$2" -CLUSTER_TOKEN="$3" -CERTIFICATE_KEY="$4" -REGISTRY_URL="$5" -REGISTRY_URL_ENDPOINT="$6" -REGISTRY_USERNAME="$7" -REGISTRY_PASSWORD="$8" -REGISTRY_TOKEN="$9" - -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl deleted file mode 100644 index 16a6a9f9d5dc..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then -echo "Already provisioned!" -exit 0 -fi - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 -while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm init failed!" - exit 1 - fi - retval=0 - set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] -done - -K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -export KUBECONFIG=/etc/kubernetes/admin.conf - -mkdir -p /root/.kube -cp -i /etc/kubernetes/admin.conf /root/.kube/config -chown $(id -u):$(id -g) /root/.kube/config -echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - -if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then - ### Network, dashboard configs available offline ### - echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml - rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" -else - /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" - /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml -fi - -/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true -/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true -/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system deleted file mode 100644 index 09462832fe40..000000000000 --- a/scripts/util/cks-samples/basic/scripts/control-node/setup-kube-system +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 8 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_TOKEN="$2" -CLUSTER_INITARGS="$3" -REGISTRY_URL="$4" -REGISTRY_URL_ENDPOINT="$5" -REGISTRY_USERNAME="$6" -REGISTRY_PASSWORD="$7" -REGISTRY_TOKEN="$8" - -K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl deleted file mode 100644 index 266bdfb0fe6c..000000000000 --- a/scripts/util/cks-samples/basic/scripts/node/deploy-kube-system.tmpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then -echo "Already provisioned!" -exit 0 -fi - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -modprobe ip_vs -modprobe ip_vs_wrr -modprobe ip_vs_sh -modprobe nf_conntrack -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/basic/scripts/node/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/basic/scripts/node/setup-kube-system b/scripts/util/cks-samples/basic/scripts/node/setup-kube-system deleted file mode 100644 index ddd4f8b4c13a..000000000000 --- a/scripts/util/cks-samples/basic/scripts/node/setup-kube-system +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 8 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_JOIN_IP="$2" -CLUSTER_TOKEN="$3" -REGISTRY_URL="$4" -REGISTRY_URL_ENDPOINT="$5" -REGISTRY_USERNAME="$6" -REGISTRY_PASSWORD="$7" -REGISTRY_TOKEN="$8" - -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl deleted file mode 100644 index 67be0e79bf6b..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/deploy-kube-system.tmpl +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 -fi - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -modprobe ip_vs -modprobe ip_vs_wrr -modprobe ip_vs_sh -modprobe nf_conntrack -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - -mkdir -p /root/.kube -cp -i /etc/kubernetes/admin.conf /root/.kube/config -chown $(id -u):$(id -g) /root/.kube/config - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success - diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system deleted file mode 100644 index f9e77fdeb20b..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node-add/setup-kube-system +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 9 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_JOIN_IP="$2" -CLUSTER_TOKEN="$3" -CERTIFICATE_KEY="$4" -REGISTRY_URL="$5" -REGISTRY_URL_ENDPOINT="$6" -REGISTRY_USERNAME="$7" -REGISTRY_PASSWORD="$8" -REGISTRY_TOKEN="$9" - -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl deleted file mode 100644 index 7e6c61c30663..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node/deploy-kube-system.tmpl +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then -echo "Already provisioned!" -exit 0 -fi - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 -while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm init failed!" - exit 1 - fi - retval=0 - set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] -done - -K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ -K8S_HELM=/tmp/k8shelm/ - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -export KUBECONFIG=/etc/kubernetes/admin.conf - -mkdir -p /root/.kube -cp -i /etc/kubernetes/admin.conf /root/.kube/config -chown $(id -u):$(id -g) /root/.kube/config -echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - -if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then - ### Network, dashboard configs available offline ### - echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml - rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" -else - /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml -fi - -helm upgrade --install cilium ${K8S_HELM}/charts/cilium-*.tgz -f ${K8S_HELM}/overrides/cilium-overrides.yaml -n kube-system -/bin/rm -rf ${K8S_HELM} - -/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true -/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true -/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system deleted file mode 100644 index 1180e4c6a9fe..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/control-node/setup-kube-system +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 8 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_TOKEN="$2" -CLUSTER_INITARGS="$3" -REGISTRY_URL="$4" -REGISTRY_URL_ENDPOINT="$5" -REGISTRY_USERNAME="$6" -REGISTRY_PASSWORD="$7" -REGISTRY_TOKEN="$8" - -K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ -K8S_HELM=/tmp/k8shelm/ -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - if [ -f ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz ] - then - TMPDIR=$(mktemp -d) - tar -xzf ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz -C ${TMPDIR} - /bin/mv ${TMPDIR}/linux-amd64/helm /usr/local/bin/helm - /bin/rm -rf ${TMPDIR} - fi - - if [ -f ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz ] - then - tar -xzf ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz -C /usr/local/bin - chmod 755 /usr/local/bin/cilium - fi - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - mkdir -p "${K8S_HELM}/charts" - if [ -d "${BINARIES_DIR}/installs/charts" ]; then - cp ${BINARIES_DIR}/installs/charts/*.*gz "${K8S_HELM}/charts" - fi - mkdir -p "${K8S_HELM}/overrides" - if [ -d "${BINARIES_DIR}/installs/helm-overrides" ]; then - cp ${BINARIES_DIR}/installs/helm-overrides/*.yaml "${K8S_HELM}/overrides" - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl b/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl deleted file mode 100644 index 266bdfb0fe6c..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/node/deploy-kube-system.tmpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [[ -f "/home/cloud/success" ]]; then -echo "Already provisioned!" -exit 0 -fi - -if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 -fi -modprobe ip_vs -modprobe ip_vs_wrr -modprobe ip_vs_sh -modprobe nf_conntrack -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi -kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl b/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl deleted file mode 100644 index 98f2e45b37a0..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/node/setup-containerd.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl daemon-reload -systemctl restart containerd diff --git a/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system b/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system deleted file mode 100644 index ddd4f8b4c13a..000000000000 --- a/scripts/util/cks-samples/cilium/scripts/node/setup-kube-system +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -if [ $# -lt 8 ]; then - echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 - exit 1 -fi - -BINARIES_DIR="$1" -CLUSTER_JOIN_IP="$2" -CLUSTER_TOKEN="$3" -REGISTRY_URL="$4" -REGISTRY_URL_ENDPOINT="$5" -REGISTRY_USERNAME="$6" -REGISTRY_PASSWORD="$7" -REGISTRY_TOKEN="$8" - -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -crucial_cmd_attempts=1 - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then - sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system -fi - -if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then - sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` - echo $PAUSE_IMAGE - if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml - fi - systemctl daemon-reload - systemctl restart containerd - -fi -if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - -if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done -fi diff --git a/scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh b/scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh similarity index 95% rename from scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh rename to scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh index cbb0588befe0..6a4bb46fa363 100755 --- a/scripts/util/cks-samples/cilium/create-kubernetes-binaries-iso.sh +++ b/scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh @@ -19,8 +19,8 @@ set -e if [ $# -lt 8 ]; then - echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION CILIUM_VERSION HELM_VERSION DASHBOARD_YAML_CONFIG BUILD_NAME" - echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.27.14 1.4.0 1.29.0 1.15.1 3.14.2 https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml setup-v1.27.14" + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso-cilium.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION CILIUM_VERSION HELM_VERSION DASHBOARD_YAML_CONFIG BUILD_NAME" + echo "eg: ./create-kubernetes-binaries-iso-cilium.sh ./ 1.27.14 1.4.0 1.29.0 1.15.1 3.14.2 https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml setup-v1.27.14" exit 1 fi diff --git a/scripts/util/cks-samples/create-scripts-cilium.sh b/scripts/util/cks-samples/create-scripts-cilium.sh new file mode 100644 index 000000000000..fec9a6841395 --- /dev/null +++ b/scripts/util/cks-samples/create-scripts-cilium.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 2 ]; then + echo "Usage: /bin/bash $0 PLUGIN_CONFDIR TARGETSCRIPTDIR" >&2 + echo " eg: /bin/bash $0 ./plugins/integrations/kubernetes-service/src/main/resources/conf ./scripts/util/scripts" >&2 + exit 1 +fi + +PLUGIN_CONFDIR="${1}" +SCRIPTS_DIR="${2}" + +# Control node +## setup-kube-system + +mkdir -p ${SCRIPTS_DIR}/control-node + +cat > "${SCRIPTS_DIR}/control-node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node/setup-kube-system" + +# Helm and Cilium-specific updates + +BINARIES_LINE=$(grep -n 'Installing binaries from' "${SCRIPTS_DIR}/control-node/setup-kube-system" | cut -f1 -d: | head -1) + +if [ -z "${BINARIES_LINE}" ]; then + echo "Unable to find the Installing binaries from... line in the setup script (setup-kube-system). Unable to generate scripts." >&2 + exit 99 +fi + +echo "$(expr ${BINARIES_LINE} - 1)a${BINARIES_LINE},$(expr ${BINARIES_LINE} + 23)" > /tmp/setup.patch + +cat <<\! >> /tmp/setup.patch +> if [ -f ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz ] +> then +> TMPDIR=$(mktemp -d) +> tar -xzf ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz -C ${TMPDIR} +> /bin/mv ${TMPDIR}/linux-amd64/helm /usr/local/bin/helm +> /bin/rm -rf ${TMPDIR} +> fi +> # +> if [ -f ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz ] +> then +> tar -xzf ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz -C /usr/local/bin +> chmod 755 /usr/local/bin/cilium +> fi +> # +> K8S_HELM=/tmp/k8shelm/ +> if [ -d "${BINARIES_DIR}/installs/charts" ]; then +> mkdir -p "${K8S_HELM}/charts" +> cp ${BINARIES_DIR}/installs/charts/*.*gz "${K8S_HELM}/charts" +> fi +> if [ -d "${BINARIES_DIR}/installs/helm-overrides" ]; then +> mkdir -p "${K8S_HELM}/overrides" +> cp ${BINARIES_DIR}/installs/helm-overrides/*.yaml "${K8S_HELM}/overrides" +> fi +> # +! + +patch ${SCRIPTS_DIR}/control-node/setup-kube-system < /tmp/setup.patch + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +# Helm and Cilium-specific updates + +sed -i -e '/weave.works/d' -e '/network.yaml/d' "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +FIRST_SUCCESS=$(grep -n '/home/cloud/success' "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" | grep -v ':if' | cut -f1 -d: | head -1) + +if [ -z "${FIRST_SUCCESS}" ]; then + echo "Unable to find the /home/cloud/success line in the deployment script (deploy-kube-system.tmpl). Unable to generate scripts." >&2 + exit 99 +fi + +echo "$(expr ${FIRST_SUCCESS} - 1)a${FIRST_SUCCESS},$(expr ${FIRST_SUCCESS} + 3)" > /tmp/deploy.patch + +cat <<\! >> /tmp/deploy.patch +> K8S_HELM=/tmp/k8shelm/ +> helm upgrade --install cilium ${K8S_HELM}/charts/cilium-*.tgz -f ${K8S_HELM}/overrides/cilium-overrides.yaml -n kube-system +> /bin/rm -rf ${K8S_HELM} +> # +! + +patch "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" < /tmp/deploy.patch + + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/setup-containerd.tmpl" + +# Control node add + +mkdir -p ${SCRIPTS_DIR}/control-node-add + +cat > "${SCRIPTS_DIR}/control-node-add/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node-add\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node-add/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/setup-containerd.tmpl" + +# Node + +mkdir -p ${SCRIPTS_DIR}/node + +cat > "${SCRIPTS_DIR}/node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/setup-containerd.tmpl" diff --git a/scripts/util/cks-samples/create-scripts.sh b/scripts/util/cks-samples/create-scripts.sh new file mode 100644 index 000000000000..c979b5b6d8b5 --- /dev/null +++ b/scripts/util/cks-samples/create-scripts.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 2 ]; then + echo "Usage: /bin/bash $0 PLUGIN_CONFDIR TARGETSCRIPTDIR" >&2 + echo " eg: /bin/bash $0 ./plugins/integrations/kubernetes-service/src/main/resources/conf ./scripts/util/scripts" >&2 + exit 1 +fi + +PLUGIN_CONFDIR="${1}" +SCRIPTS_DIR="${2}" + +# Control node +## setup-kube-system + +mkdir -p ${SCRIPTS_DIR}/control-node + +cat > "${SCRIPTS_DIR}/control-node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/setup-containerd.tmpl" + +# Control node add + +mkdir -p ${SCRIPTS_DIR}/control-node-add + +cat > "${SCRIPTS_DIR}/control-node-add/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node-add\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node-add/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/setup-containerd.tmpl" + +# Node + +mkdir -p ${SCRIPTS_DIR}/node + +cat > "${SCRIPTS_DIR}/node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/setup-containerd.tmpl" diff --git a/scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml b/scripts/util/cks-samples/helm-overrides/cilium-overrides.yaml similarity index 100% rename from scripts/util/cks-samples/cilium/helm-overrides/cilium-overrides.yaml rename to scripts/util/cks-samples/helm-overrides/cilium-overrides.yaml