add systemd services for configuration after start

the services does the various needed tasks to setup
the ocp or microshift cluster, these systemd  units
runs small shell scripts which are based on:
https://github.com/crc-org/crc-cloud/blob/main/pkg/bundle/setup/clustersetup.sh

and does the following tasks:
- creates crc specific configurations for dnsmasq
- sets a new uuid as cluster id
- creates the pod for routes-controller
- tries to grow the disk and filesystem
- checks if the cluster operators are ready
- adds the pull secret to the cluster
- sets kubeadmin and developer user passwords
- sets a custom ca for authentication
- sets custom nip.io cluster domain

.gitignore

@@ -11,3 +11,4 @@ podman-remote/
 .sw[a-p]
 crc-cluster-kube-apiserver-operator
 crc-cluster-kube-controller-manager-operator
+systemd/crc-dnsmasq.sh

createdisk-library.sh

@@ -216,6 +216,7 @@ function prepare_hyperV() {
             echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules
 EOF
 }
+
 function prepare_qemu_guest_agent() {
     local vm_ip=$1
 
@@ -385,3 +386,36 @@ function remove_pull_secret_from_disk() {
     esac
 }
 
+function copy_systemd_units() {
+    case "${BUNDLE_TYPE}" in
+        "snc"|"okd")
+            export APPS_DOMAIN="apps-crc.testing"
+            envsubst "${APPS_DOMAIN}" < systemd/dnsmasq.sh.template > systemd/crc-dnsmasq.sh
+            unset APPS_DOMAIN
+            ;;
+        "microshift")
+            export APPS_DOMAIN="apps.crc.testing"
+            envsubst "${APPS_DOMAIN}" < systemd/dnsmasq.sh.template > systemd/crc-dnsmasq.sh
+            unset APPS_DOMAIN
+            ;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts'
+    ${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/
+    ${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+
+    case "${BUNDLE_TYPE}" in
+        "snc"|"okd")
+            ${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/
+            ${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/
+            ${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/
+            ;;
+    esac
+
+    ${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%'
+    ${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin'
+    ${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable'
+    ${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts'
+}

createdisk.sh

@@ -137,6 +137,8 @@ EOF
    ${SSH} core@${VM_IP} -- "sudo rpm-ostree install qemu-user-static-x86"
 fi
 
+copy_systemd_units
+
 cleanup_vm_image ${VM_NAME} ${VM_IP}
 
 # Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state.

docs/self-sufficient-bundle.md

@@ -0,0 +1,34 @@
+# Self sufficient bundles
+
+Since release 4.19.0 of OpenShift Local, the bundles generated by `snc` contain additional systemd services to provision the cluster and remove the need for
+an outside entity to provision the cluster, although an outside process needs to create some files on pre-defined locations inside the VM for the  systemd
+services to do their work.
+
+## The following table lists the systemd services and the location of files they need to provision the cluster, users of SNC need to create those files
+
+|     Systemd unit               | Runs for (ocp, MicroShift, both) |         Input files location         | Marker env variables |
+| :----------------------------: | :------------------------------: | :----------------------------------: | :------------------: |
+|  `crc-cluster-status.service`  |               both               |                 none                 |         none         |
+|    `crc-pullsecret.service`    |               both               |         /opt/crc/pull-secret         |         none         |
+|      `crc-dnsmasq.service`     |               both               |                 none                 |         none         |
+| `crc-routes-controller.service`|               both               |                 none                 |         none         |
+|    `ocp-cluster-ca.service`    |               ocp                |        /opt/crc/custom-ca.crt        |     CRC_CLOUD=1      |
+|     `ocp-clusterid.service`    |               ocp                |                 none                 |         none         |
+|   `ocp-custom-domain.service`  |               ocp                |                 none                 |     CRC_CLOUD=1      |
+|      `ocp-growfs.service`      |               ocp                |                 none                 |         none         |
+|   `ocp-userpasswords.service`  |               ocp                | /opt/crc/pass_{kubeadmin, developer} |         none         |
+
+In addition to the above services we have `ocp-cluster-ca.path`, `crc-pullsecret.path` and `ocp-userpasswords.path` that monitors the filesystem paths
+related to their `*.service` counterparts and starts the service when the paths become available.
+
+> [!NOTE]
+> "Marker env variable" is set using an env file, if the required env variable is not set then unit is skipped
+> some units are run only when CRC_CLOUD=1 is set, these are only needed when using the bundles with crc-cloud
+
+The systemd services are heavily based on the [`clustersetup.sh`](https://github.com/crc-org/crc-cloud/blob/main/pkg/bundle/setup/clustersetup.sh) script found in the `crc-cloud` project.
+
+## Naming convention for the systemd unit files
+
+Systemd units that are needed for both 'OpenShift' and 'MicroShift' are named as `crc-*.service`, units that are needed only for 'OpenShift' are named
+as `ocp-*.service` and when we add units that are only needed for 'MicroShift' they should be named as `ucp-*.service`
+

machine.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo -n "Machine lord: ${AVAR}"

systemd/crc-cluster-status.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit checking if cluster is ready
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-cluster-status.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-cluster-status.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -x
+
+export KUBECONFIG=/opt/kubeconfig
+
+function check_cluster_healthy() {
+    WAIT="authentication|console|etcd|ingress|openshift-apiserver"
+
+    until `oc get co > /dev/null 2>&1`
+    do
+        sleep 2
+    done
+
+    for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }')
+    do
+        if [[ $i == "False" ]]
+        then
+            return 1
+        fi
+    done
+    return 0
+}
+
+# rm -rf /tmp/.crc-cluster-ready
+
+COUNTER=0
+CLUSTER_HEALTH_SLEEP=8
+CLUSTER_HEALTH_RETRIES=500
+
+while ! check_cluster_healthy
+do
+    sleep $CLUSTER_HEALTH_SLEEP
+    if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]]
+    then
+        return 1
+    fi
+    ((COUNTER++))
+done
+
+# need to set a marker to let `crc` know the cluster is ready
+# touch /tmp/.crc-cluster-ready
+

systemd/crc-dnsmasq.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=CRC Unit for configuring dnsmasq
+Wants=network-online.target
+After=network-online.target ssh-access.target ovs-configuration.service
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+Restart=on-failure
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/crc-dnsmasq.sh
+ExecStartPost=/usr/bin/systemctl start dnsmasq.service
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.path

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit for monitoring the pull secret path
+After=kubelet.service
+
+[Path]
+PathExists=/opt/crc/pull-secret
+TriggerLimitIntervalSec=1min
+TriggerLimitBurst=0
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=CRC Unit for adding pull secret to cluster
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/crc-pullsecret.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-pullsecret.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+
+wait_for_resource secret
+
+# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret
+existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}")
+existingPs=$(echo "${existingPsB64}" | base64 -d)
+
+echo "${existingPs}" | jq -e '.auths'
+
+if [[ $? != 0 ]]; then
+    pullSecretB64=$(cat /opt/crc/pull-secret | base64 -w0)
+    oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}"
+    rm -f /opt/crc/pull-secret
+fi
+

systemd/crc-routes-controller.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRC Unit starting routes controller
+Wants=network-online.target gvisor-tap-vsock.service sys-class-net-tap0.device
+After=sys-class-net-tap0.device network-online.target kubelet.service gvisor-tap-vsock.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/crc-routes-controller.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/crc-routes-controller.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -x
+
+if [[ ${CRC_NETWORK_MODE_USER} -eq 0 ]]; then
+    echo -n "network-mode 'system' detected: skipping routes-controller pod deployment"
+    exit 0
+fi
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG=/opt/kubeconfig
+
+wait_for_resource pods
+
+oc apply -f /opt/crc/routes-controller.yaml
+

systemd/crc-systemd-common.sh

@@ -0,0 +1,12 @@
+# $1 is the resource to check
+# $2 is an optional maximum retry count; default 20
+function wait_for_resource() {
+    local retry=0
+    local max_retry=${2:-20}
+    until `oc get "$1" > /dev/null 2>&1`
+    do
+        [ $retry == $max_retry ] && exit 1
+        sleep 5
+        ((retry++))
+    done
+}

systemd/dnsmasq.sh.template

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -x
+
+if [[ ${CRC_NETWORK_MODE_USER} -eq 1 ]]; then
+    echo -n "network-mode 'user' detected: skipping dnsmasq configuration"
+    exit 0
+fi
+
+hostName=$(hostname)
+hostIp=$(hostname --all-ip-addresses | awk '{print $1}')
+
+cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf
+listen-address=192.168.130.11
+expand-hosts
+log-queries
+local=/crc.testing/
+domain=crc.testing
+address=/${APPS_DOMAIN}/192.168.130.11
+address=/api.crc.testing/192.168.130.11
+address=/api-int.crc.testing/192.168.130.11
+address=/$hostName/$hostIp
+EOF
+

systemd/ocp-cluster-ca.path

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit monitoring custom-ca.crt file path
+After=kubelet.service
+
+[Path]
+PathExists=/opt/crc/custom-ca.crt
+TriggerLimitIntervalSec=1min
+TriggerLimitBurst=0
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-ca.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=CRC Unit setting custom cluster ca
+After=kubelet.service ocp-clusterid.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-cluster-ca.sh
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-cluster-ca.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+
+wait_for_resource configmap
+
+custom_ca_path=/opt/crc/custom-ca.crt
+
+if [[ ! -f ${custom_ca_path} ]]; then
+    echo "Cert bundle /opt/crc/custom-ca.crt not found"
+    exit 0
+fi
+
+oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path}
+oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}'
+oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \
+--dry-run -o yaml | oc replace -f -
+
+rm -f /opt/crc/custom-ca.crt

systemd/ocp-clusterid.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit setting random cluster ID
+After=kubelet.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/ocp-clusterid.sh
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

systemd/ocp-clusterid.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -x
+
+source /usr/local/bin/crc-systemd-common.sh
+export KUBECONFIG="/opt/kubeconfig"
+uuid=$(uuidgen)
+
+wait_for_resource clusterversion
+
+oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge

systemd/ocp-custom-domain.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=CRC Unit setting nip.io domain for cluster
+After=kubelet.service ocp-clusterid.service ocp-cluster-ca.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/systemd/system/crc-env
+ExecStart=/usr/local/bin/ocp-custom-domain.sh
+
+[Install]
+WantedBy=multi-user.target