diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 2c18efa01891..3e25ce433a9b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -89,6 +89,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/control-node-add/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system for the second stage + if /bin/bash ${BINARIES_DIR}/scripts/control-node-add/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.ha.certificate.key }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on a control plane node add from ISO." + else + rc=$? + echo "Failed running setup-kube-system on a control plane node add from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index aa7eec97ac82..a896c6028e84 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -109,6 +109,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/control-node/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc + if /bin/bash ${BINARIES_DIR}/scripts/control-node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.initargs }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on control plane node from ISO." + else + rc=$? + echo "Failed running setup-kube-system on control plane node from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index de1f4c9ffc70..ed525b8f602f 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -89,6 +89,22 @@ write_files: export PATH=$PATH:/opt/bin fi + if [ -f "${BINARIES_DIR}/scripts/node/setup-kube-system" ]; then + # This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc + if /bin/bash ${BINARIES_DIR}/scripts/node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then + rc=0 + echo "Success running setup-kube-system on worker node from ISO." + else + rc=$? + echo "Failed running setup-kube-system on worker node from ISO." + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + exit $rc + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" diff --git a/scripts/util/cks-samples/README.md b/scripts/util/cks-samples/README.md new file mode 100644 index 000000000000..4ebad8986ac6 --- /dev/null +++ b/scripts/util/cks-samples/README.md @@ -0,0 +1,57 @@ +# CloudStack Kubernetes Service Cloud-init Overrides - Examples + +Example files that can be used when creating the CloudStack Kubernetes Service ISO file to override the normal behavior of the Kubernetes cluster creation + + +## Usage + +### The basic template - create-scripts.sh + +If you want to override the CKS installation starting with the basic CKS installation scripts, just run: + +``` +bash create-scripts.sh ${CLOUDSTACK_REPO_HOME}/plugins/integrations/kubernetes-service/src/main/resources/conf ${CLOUDSTACK_REPO_HOME}/scripts/util/scripts +``` + +You can make any adjustments to the scripts under ${CLOUDSTACK_REPO_HOME}/scripts/util/scripts as necessary. Then you can run create-kubernetes-binaries-iso.sh in the parent directory as you would normally to create your ISO. + +### Cilium - create-scripts-cilium.sh + +If you want a CKS installation that installs both Cilium (instead of Weave) and Helm, you can run the Cilium create script like so: + +``` +bash create-scripts-cilium.sh ${CLOUDSTACK_REPO_HOME}/plugins/integrations/kubernetes-service/src/main/resources/conf ${CLOUDSTACK_REPO_HOME}/scripts/util/cks-samples/scripts +``` + +You can make any adjustments to the scripts under ${CLOUDSTACK_REPO_HOME}/scripts/util/cks-samples/scripts as necessary, although by default, the scripts should install what is necessary to install Cilium. There are helm-overrides for Cilium under the helm-overrides directory that can be modified as well. + +Once any adjustments are made (not required), you can run create-kubernetes-binaries-iso-cilium.sh in a similar fashion to how create-kubernetes-binaries-iso.sh would be run, replacing the Weave URL with the Cilium version and Helm version (see examples in the usage). + + +## References in CloudStack + +The following cloud-init scripts will call the setup-kube-system scripts found here when the cluster is created: + +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml + +## License + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + diff --git a/scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh b/scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh new file mode 100755 index 000000000000..6a4bb46fa363 --- /dev/null +++ b/scripts/util/cks-samples/create-kubernetes-binaries-iso-cilium.sh @@ -0,0 +1,174 @@ +#!/bin/bash -x +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +if [ $# -lt 8 ]; then + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso-cilium.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION CILIUM_VERSION HELM_VERSION DASHBOARD_YAML_CONFIG BUILD_NAME" + echo "eg: ./create-kubernetes-binaries-iso-cilium.sh ./ 1.27.14 1.4.0 1.29.0 1.15.1 3.14.2 https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml setup-v1.27.14" + exit 1 +fi + +RELEASE="v${2}" +VAL="1.18.0" +output_dir="${1}" +start_dir="$PWD" +iso_dir="/tmp/iso" +working_dir="${iso_dir}/" +mkdir -p "${working_dir}" +build_name="${8}.iso" +[ -z "${build_name}" ] && build_name="setup-${RELEASE}.iso" + +CNI_VERSION="v${3}" +echo "Downloading CNI ${CNI_VERSION}..." +cni_dir="${working_dir}/cni/" +mkdir -p "${cni_dir}" +cni_status_code=$(curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz") +if [[ ${cni_status_code} -eq 404 ]] ; then + curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz" +fi + +CRICTL_VERSION="v${4}" +echo "Downloading CRI tools ${CRICTL_VERSION}..." +crictl_dir="${working_dir}/cri-tools/" +mkdir -p "${crictl_dir}" +curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-linux-amd64.tar.gz" + +echo "Downloading Kubernetes tools ${RELEASE}..." +k8s_dir="${working_dir}/k8s" +mkdir -p "${k8s_dir}" +cd "${k8s_dir}" +curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +kubeadm_file_permissions=`stat --format '%a' kubeadm` +chmod +x kubeadm + +echo "Downloading kubelet.service ${RELEASE}..." +cd "${start_dir}" +kubelet_service_file="${working_dir}/kubelet.service" +touch "${kubelet_service_file}" +if [[ `echo "${2} $VAL" | awk '{print ($1 < $2)}'` == 1 ]]; then + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} +else + curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} +fi + +echo "Downloading 10-kubeadm.conf ${RELEASE}..." +kubeadm_conf_file="${working_dir}/10-kubeadm.conf" +touch "${kubeadm_conf_file}" +if [[ `echo "${2} $val" | awk '{print ($1 < $2)}'` == 1 ]]; then + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} +else + curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} +fi + +HELM_VERSION="${6}" +mkdir -p ${iso_dir}/installs +curl -L -o ${iso_dir}/installs/helm-v${HELM_VERSION}-linux-amd64.tar.gz https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz + +CILIUM_VERSION="${5}" +echo "Downloading Cilium Helm chart version ${CILIUM_VERSION} and cilium cli" +mkdir -p ${iso_dir}/installs/charts +curl -Lo ${iso_dir}/installs/charts/cilium-${CILIUM_VERSION}.tgz https://helm.cilium.io/cilium-${CILIUM_VERSION}.tgz + +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) +CLI_ARCH=amd64 +curl -Lo ${iso_dir}/installs/cilium-linux-${CLI_ARCH}-cli-${CILIUM_CLI_VERSION}.tar.gz --fail https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz + +DASHBOARD_CONFIG_URL="${7}" +echo "Downloading dashboard config ${DASHBOARD_CONFIG_URL}" +dashboard_conf_file="${working_dir}/dashboard.yaml" +curl -sSL ${DASHBOARD_CONFIG_URL} -o ${dashboard_conf_file} + +# TODO : Change the url once merged +AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" +echo "Downloading kubernetes cluster autoscaler ${AUTOSCALER_URL}" +autoscaler_conf_file="${working_dir}/autoscaler.yaml" +curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} + +PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml" +echo "Downloading kubernetes cluster provider ${PROVIDER_URL}" +provider_conf_file="${working_dir}/provider.yaml" +curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} + +echo "Fetching k8s docker images..." +ctr -v +if [ $? -ne 0 ]; then + echo "Installing containerd..." + if [ -f /etc/redhat-release ]; then + sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine + sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs + sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm + sudo yum install -y containerd.io + elif [ -f /etc/lsb-release ]; then + sudo apt update && sudo apt install containerd.io -y + fi + sudo systemctl enable containerd && sudo systemctl start containerd +fi +mkdir -p "${working_dir}/docker" +output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` + +# Don't forget about the yaml images ! +for i in ${network_conf_file} ${dashboard_conf_file} +do + images=`grep "image:" $i | cut -d ':' -f2- | tr -d ' ' | tr -d "'"` + output=`printf "%s\n" ${output} ${images}` +done + +# Don't forget about the other image ! +autoscaler_image=`grep "image:" ${autoscaler_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${autoscaler_image}` + +provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '` +output=`printf "%s\n" ${output} ${provider_image}` + +cp -r helm-overrides/ ${iso_dir}/installs + +chart_images=`for chart in ${iso_dir}/installs/charts/*; do chartfile=${chart##*/}; chartbase=${chartfile%-*}; if [ -f ${iso_dir}/installs/helm-overrides/${chartbase}-overrides.yaml ]; then helm template -f ${iso_dir}/installs/helm-overrides/${chartbase}-overrides.yaml ${chart}; else helm template ${chart}; fi; done | grep "[[:space:]]image:" | cut -d ':' -f2- | tr -d ' ' | tr -d '\r' | tr -d "'" | tr -d '"' | sort -u` +output=`printf "%s\n" ${output} ${chart_images}` + +while read -r line; do + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + if [ ! -z "${https_proxy}" ]; then + sudo https_proxy=${https_proxy} ctr image pull "$line" + else + sudo ctr image pull "$line" + fi + image_name=`echo "$line" | grep -oE "[^/]+$"` + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" +done <<< "$output" + +if [ -d scripts ]; then + /bin/cp -r scripts ${iso_dir} +fi + +echo "Restore kubeadm permissions..." +if [ -z "${kubeadm_file_permissions}" ]; then + kubeadm_file_permissions=644 +fi +chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" + +echo "Updating imagePullPolicy to IfNotPresent in yaml files..." +sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml + +mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}" + +rm -rf "${iso_dir}" diff --git a/scripts/util/cks-samples/create-scripts-cilium.sh b/scripts/util/cks-samples/create-scripts-cilium.sh new file mode 100644 index 000000000000..fec9a6841395 --- /dev/null +++ b/scripts/util/cks-samples/create-scripts-cilium.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 2 ]; then + echo "Usage: /bin/bash $0 PLUGIN_CONFDIR TARGETSCRIPTDIR" >&2 + echo " eg: /bin/bash $0 ./plugins/integrations/kubernetes-service/src/main/resources/conf ./scripts/util/scripts" >&2 + exit 1 +fi + +PLUGIN_CONFDIR="${1}" +SCRIPTS_DIR="${2}" + +# Control node +## setup-kube-system + +mkdir -p ${SCRIPTS_DIR}/control-node + +cat > "${SCRIPTS_DIR}/control-node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node/setup-kube-system" + +# Helm and Cilium-specific updates + +BINARIES_LINE=$(grep -n 'Installing binaries from' "${SCRIPTS_DIR}/control-node/setup-kube-system" | cut -f1 -d: | head -1) + +if [ -z "${BINARIES_LINE}" ]; then + echo "Unable to find the Installing binaries from... line in the setup script (setup-kube-system). Unable to generate scripts." >&2 + exit 99 +fi + +echo "$(expr ${BINARIES_LINE} - 1)a${BINARIES_LINE},$(expr ${BINARIES_LINE} + 23)" > /tmp/setup.patch + +cat <<\! >> /tmp/setup.patch +> if [ -f ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz ] +> then +> TMPDIR=$(mktemp -d) +> tar -xzf ${BINARIES_DIR}/installs/helm-*-linux-amd64.tar.gz -C ${TMPDIR} +> /bin/mv ${TMPDIR}/linux-amd64/helm /usr/local/bin/helm +> /bin/rm -rf ${TMPDIR} +> fi +> # +> if [ -f ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz ] +> then +> tar -xzf ${BINARIES_DIR}/installs/cilium-*-cli-*.tar.gz -C /usr/local/bin +> chmod 755 /usr/local/bin/cilium +> fi +> # +> K8S_HELM=/tmp/k8shelm/ +> if [ -d "${BINARIES_DIR}/installs/charts" ]; then +> mkdir -p "${K8S_HELM}/charts" +> cp ${BINARIES_DIR}/installs/charts/*.*gz "${K8S_HELM}/charts" +> fi +> if [ -d "${BINARIES_DIR}/installs/helm-overrides" ]; then +> mkdir -p "${K8S_HELM}/overrides" +> cp ${BINARIES_DIR}/installs/helm-overrides/*.yaml "${K8S_HELM}/overrides" +> fi +> # +! + +patch ${SCRIPTS_DIR}/control-node/setup-kube-system < /tmp/setup.patch + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +# Helm and Cilium-specific updates + +sed -i -e '/weave.works/d' -e '/network.yaml/d' "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +FIRST_SUCCESS=$(grep -n '/home/cloud/success' "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" | grep -v ':if' | cut -f1 -d: | head -1) + +if [ -z "${FIRST_SUCCESS}" ]; then + echo "Unable to find the /home/cloud/success line in the deployment script (deploy-kube-system.tmpl). Unable to generate scripts." >&2 + exit 99 +fi + +echo "$(expr ${FIRST_SUCCESS} - 1)a${FIRST_SUCCESS},$(expr ${FIRST_SUCCESS} + 3)" > /tmp/deploy.patch + +cat <<\! >> /tmp/deploy.patch +> K8S_HELM=/tmp/k8shelm/ +> helm upgrade --install cilium ${K8S_HELM}/charts/cilium-*.tgz -f ${K8S_HELM}/overrides/cilium-overrides.yaml -n kube-system +> /bin/rm -rf ${K8S_HELM} +> # +! + +patch "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" < /tmp/deploy.patch + + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/setup-containerd.tmpl" + +# Control node add + +mkdir -p ${SCRIPTS_DIR}/control-node-add + +cat > "${SCRIPTS_DIR}/control-node-add/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node-add\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node-add/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/setup-containerd.tmpl" + +# Node + +mkdir -p ${SCRIPTS_DIR}/node + +cat > "${SCRIPTS_DIR}/node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/setup-containerd.tmpl" diff --git a/scripts/util/cks-samples/create-scripts.sh b/scripts/util/cks-samples/create-scripts.sh new file mode 100644 index 000000000000..c979b5b6d8b5 --- /dev/null +++ b/scripts/util/cks-samples/create-scripts.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ $# -lt 2 ]; then + echo "Usage: /bin/bash $0 PLUGIN_CONFDIR TARGETSCRIPTDIR" >&2 + echo " eg: /bin/bash $0 ./plugins/integrations/kubernetes-service/src/main/resources/conf ./scripts/util/scripts" >&2 + exit 1 +fi + +PLUGIN_CONFDIR="${1}" +SCRIPTS_DIR="${2}" + +# Control node +## setup-kube-system + +mkdir -p ${SCRIPTS_DIR}/control-node + +cat > "${SCRIPTS_DIR}/control-node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_TOKEN CLUSTER_INITARGS REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_TOKEN="$2" +CLUSTER_INITARGS="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.initargs }}/${CLUSTER_INITARGS}/g" ${BINARIES_DIR}/scripts/control-node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node/setup-containerd.tmpl" + +# Control node add + +mkdir -p ${SCRIPTS_DIR}/control-node-add + +cat > "${SCRIPTS_DIR}/control-node-add/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 9 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +CERTIFICATE_KEY="$4" +REGISTRY_URL="$5" +REGISTRY_URL_ENDPOINT="$6" +REGISTRY_USERNAME="$7" +REGISTRY_PASSWORD="$8" +REGISTRY_TOKEN="$9" + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/control-node-add\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/control-node-add/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-control-node-add.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/control-node-add/setup-containerd.tmpl" + +# Node + +mkdir -p ${SCRIPTS_DIR}/node + +cat > "${SCRIPTS_DIR}/node/setup-kube-system" <<\! +#!/bin/bash -e + +if [ $# -lt 8 ]; then + echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2 + exit 1 +fi + +BINARIES_DIR="$1" +CLUSTER_JOIN_IP="$2" +CLUSTER_TOKEN="$3" +REGISTRY_URL="$4" +REGISTRY_URL_ENDPOINT="$5" +REGISTRY_USERNAME="$6" +REGISTRY_PASSWORD="$7" +REGISTRY_TOKEN="$8" + +if [ -f ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl ]; then + sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/node/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system +fi + +if [ -f ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl ]; then + sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/node/setup-containerd.tmpl > /opt/bin/setup-containerd +fi + +! + +sed -e "/- path: \/opt\/bin\/setup-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" | sed -e '/if \[ -f "${BINARIES_DIR}\/scripts\/node\/setup-kube-system" \]; then/,/^fi/d' -e '/^while true; do/,/^done/d' -e '/if \[ "$EJECT_ISO_FROM_OS" = true \]/,/fi/d' | egrep -v "^#!|^BINARIES_DIR|^EJECT_ISO_FROM_OS|umount" >> "${SCRIPTS_DIR}/node/setup-kube-system" + +## deploy-kube-system + +sed -e "/- path: \/opt\/bin\/deploy-kube-system/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/deploy-kube-system.tmpl" + +## setup-containerd +sed -e "/- path: \/opt\/bin\/setup-containerd/,/- path: /p" -e d "${PLUGIN_CONFDIR}/k8s-node.yml" | sed -e "0,/content: /d" -e "/- path: /d" -e "s/^ //" > "${SCRIPTS_DIR}/node/setup-containerd.tmpl" diff --git a/scripts/util/cks-samples/helm-overrides/cilium-overrides.yaml b/scripts/util/cks-samples/helm-overrides/cilium-overrides.yaml new file mode 100644 index 000000000000..7fe557862c97 --- /dev/null +++ b/scripts/util/cks-samples/helm-overrides/cilium-overrides.yaml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +image: + useDigest: false + +operator: + image: + useDigest: false diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index d5fb014f220b..cc72ec240221 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -128,6 +128,10 @@ output=`printf "%s\n" ${output} ${autoscaler_image}` provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' '` output=`printf "%s\n" ${output} ${provider_image}` +if [ -d scripts ]; then + /bin/cp -r scripts ${iso_dir} +fi + while read -r line; do echo "Downloading image $line ---" if [[ $line == kubernetesui* ]] || [[ $line == apache* ]] || [[ $line == weaveworks* ]]; then