add systemd services for configuration after start

.gitignore

@@ -11,3 +11,4 @@ podman-remote/
 .sw[a-p]
 crc-cluster-kube-apiserver-operator
 crc-cluster-kube-controller-manager-operator
+systemd/crc-dnsmasq.sh

createdisk-library.sh

@@ -216,6 +216,7 @@ function prepare_hyperV() {
             echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules
 EOF
 }
+
 function prepare_qemu_guest_agent() {
     local vm_ip=$1
 
@@ -385,3 +386,36 @@ function remove_pull_secret_from_disk() {
     esac
 }
 
+function copy_systemd_units() {
+    case "${BUNDLE_TYPE}" in
+        "snc"|"okd")
+            export APPS_DOMAIN="apps-crc.testing"
+            envsubst '${APPS_DOMAIN}' < systemd/dnsmasq.sh.template > systemd/crc-dnsmasq.sh
+            unset APPS_DOMAIN
+            ;;
+        "microshift")
+            export APPS_DOMAIN="apps.crc.testing"
+            envsubst '${APPS_DOMAIN}' < systemd/dnsmasq.sh.template > systemd/crc-dnsmasq.sh
+            unset APPS_DOMAIN
+            ;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts'
+    ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+
+    case "${BUNDLE_TYPE}" in
+        "snc"|"okd")
+            ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/
+            ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/
+            ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+            ;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%'
+    ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable'
+    ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts'
+}

createdisk.sh

@@ -137,6 +137,8 @@ EOF
    ${SSH} core@${VM_IP} -- "sudo rpm-ostree install qemu-user-static-x86"
 fi
 
+copy_systemd_units
+
 cleanup_vm_image ${VM_NAME} ${VM_IP}
 
 # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state.

docs/self-sufficient-bundle.md

@@ -0,0 +1,34 @@
+# Self sufficient bundles
+
+Since release 4.19.0 of OpenShift Local, the bundles generated by `snc` contain additional systemd services to provision the cluster and remove the need for
+an outside entity to provision the cluster, although an outside process needs to create some files on pre-defined locations inside the VM for the  systemd
+services to do their work.
+
+## The following table lists the systemd services and the location of files they need to provision the cluster, users of SNC need to create those files
+
+|     Systemd unit               | Runs for (ocp, MicroShift, both) |         Input files location         | Marker env variables |
+| :----------------------------: | :------------------------------: | :----------------------------------: | :------------------: |
+|  `crc-cluster-status.service`  |               both               |                 none                 |         none         |
+|    `crc-pullsecret.service`    |               both               |         /opt/crc/pull-secret         |         none         |
+|      `crc-dnsmasq.service`     |               both               |                 none                 |         none         |
+| `crc-routes-controller.service`|               both               |                 none                 |         none         |
+|    `ocp-cluster-ca.service`    |               ocp                |        /opt/crc/custom-ca.crt        |     CRC_CLOUD=1      |
+|     `ocp-clusterid.service`    |               ocp                |                 none                 |         none         |
+|   `ocp-custom-domain.service`  |               ocp                |                 none                 |     CRC_CLOUD=1      |
+|      `ocp-growfs.service`      |               ocp                |                 none                 |         none         |
+|   `ocp-userpasswords.service`  |               ocp                | /opt/crc/pass_{kubeadmin, developer} |         none         |
+
+In addition to the above services we have `ocp-cluster-ca.path`, `crc-pullsecret.path` and `ocp-userpasswords.path` that monitors the filesystem paths
+related to their `*.service` counterparts and starts the service when the paths become available.
+
+> [!NOTE]
+> "Marker env variable" is set using an env file, if the required env variable is not set then unit is skipped
+> some units are run only when CRC_CLOUD=1 is set, these are only needed when using the bundles with crc-cloud
+
+The systemd services are heavily based on the [`clustersetup.sh`](https://github.com/crc-org/crc-cloud/blob/main/pkg/bundle/setup/clustersetup.sh) script found in the `crc-cloud` project.
+
+## Naming convention for the systemd unit files
+
+Systemd units that are needed for both 'OpenShift' and 'MicroShift' are named as `crc-*.service`, units that are needed only for 'OpenShift' are named
+as `ocp-*.service` and when we add units that are only needed for 'MicroShift' they should be named as `ucp-*.service`
+

machine.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo -n "Machine lord: ${AVAR}"

systemd/crc-cluster-status.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit checking if cluster is ready
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-cluster-status.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-cluster-status.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+function check_cluster_healthy() {
+    WAIT="authentication|console|etcd|ingress|openshift-apiserver"
+
+    until `oc get co > /dev/null 2>&1`
+    do
+        sleep 2
+    done
+
+    for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }')
+    do
+        if [[ $i == "False" ]]
+        then
+            return 1
+        fi
+    done
+    return 0
+}
+
+# rm -rf /tmp/.crc-cluster-ready
+
+COUNTER=0
+CLUSTER_HEALTH_SLEEP=8
+CLUSTER_HEALTH_RETRIES=500
+
+while ! check_cluster_healthy
+do
+    sleep $CLUSTER_HEALTH_SLEEP
+    if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]]
+    then
+        return 1
+    fi
+    ((COUNTER++))
+done
+
+# need to set a marker to let `crc` know the cluster is ready
+# touch /tmp/.crc-cluster-ready
+

systemd/crc-dnsmasq.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=CRC Unit for configuring dnsmasq
+Wants=network-online.target
+After=network-online.target ssh-access.target ovs-configuration.service
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+Restart=on-failure
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/crc-dnsmasq.sh
+ExecStartPost=/usr/bin/systemctl start dnsmasq.service
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.path

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit for monitoring the pull secret path
+After=kubelet.service
+
+[Path]
+PathExists=/opt/crc/pull-secret
+TriggerLimitIntervalSec=1min
+TriggerLimitBurst=0
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=CRC Unit for adding pull secret to cluster
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-pullsecret.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+
+wait_for_resource secret
+
+# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret
+existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}")
+existingPs=$(echo "${existingPsB64}" | base64 -d)
+
+echo "${existingPs}" | jq -e '.auths'
+
+if [[ $? != 0 ]]; then
+    pullSecretB64=$(cat /opt/crc/pull-secret | base64 -w0)
+    oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}"
+    rm -f /opt/crc/pull-secret
+fi
+

systemd/crc-routes-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit starting routes controller
+Wants=network-online.target gvisor-tap-vsock.service sys-class-net-tap0.device
+After=sys-class-net-tap0.device network-online.target kubelet.service gvisor-tap-vsock.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/crc-routes-controller.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-routes-controller.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -x
+
+if [[ ${CRC_NETWORK_MODE_USER} -eq 0 ]]; then
+    echo -n "network-mode 'system' detected: skipping routes-controller pod deployment"
+    exit 0
+fi
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG=/opt/kubeconfig
+
+wait_for_resource pods
+
+oc apply -f /opt/crc/routes-controller.yaml
+

systemd/crc-systemd-common.sh

@@ -0,0 +1,12 @@
+# $1 is the resource to check
+# $2 is an optional maximum retry count; default 20
+function wait_for_resource() {
+    local retry=0
+    local max_retry=${2:-20}
+    until `oc get "$1" > /dev/null 2>&1`
+    do
+        [ $retry == $max_retry ] && exit 1
+        sleep 5
+        ((retry++))
+    done
+}

systemd/dnsmasq.sh.template

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -x
+
+if [[ ${CRC_NETWORK_MODE_USER} -eq 1 ]]; then
+    echo -n "network-mode 'user' detected: skipping dnsmasq configuration"
+    exit 0
+fi
+
+hostName=$(hostname)
+hostIp=$(hostname --all-ip-addresses | awk '{print $1}')
+
+cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf
+listen-address=192.168.130.11
+expand-hosts
+log-queries
+local=/crc.testing/
+domain=crc.testing
+address=/${APPS_DOMAIN}/192.168.130.11
+address=/api.crc.testing/192.168.130.11
+address=/api-int.crc.testing/192.168.130.11
+address=/$hostName/$hostIp
+EOF
+

systemd/ocp-cluster-ca.path

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit monitoring custom-ca.crt file path
+After=kubelet.service
+
+[Path]
+PathExists=/opt/crc/custom-ca.crt
+TriggerLimitIntervalSec=1min
+TriggerLimitBurst=0
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-ca.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=CRC Unit setting custom cluster ca
+After=kubelet.service ocp-clusterid.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-cluster-ca.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-ca.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+
+wait_for_resource configmap
+
+custom_ca_path=/opt/crc/custom-ca.crt
+
+if [[ ! -f ${custom_ca_path} ]]; then
+    echo "Cert bundle /opt/crc/custom-ca.crt not found"
+    exit 0
+fi
+
+oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path}
+oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}'
+oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \
+--dry-run -o yaml | oc replace -f -
+
+rm -f /opt/crc/custom-ca.crt

systemd/ocp-clusterid.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit setting random cluster ID
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-clusterid.sh
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-clusterid.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+uuid=$(uuidgen)
+
+wait_for_resource clusterversion
+
+oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge

systemd/ocp-custom-domain.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit setting nip.io domain for cluster
+After=kubelet.service ocp-clusterid.service ocp-cluster-ca.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/ocp-custom-domain.sh
+
+[Install]
+WantedBy=multi-user.target