RFC (post 0.18): libvirt and azure helm provisioning support with cleaner API (conflicts with #2809)

.github/workflows/azure-e2e-test.yml

@@ -19,6 +19,11 @@ on:
       podvm-image-id:
         type: string
         description: prebuilt podvm image
+      install_method:
+        default: 'kustomize'
+        description: Installation method. Either kustomize or helm.
+        required: false
+        type: string
     secrets:
       AZURE_CLIENT_ID:
         required: true
@@ -32,6 +37,11 @@ on:
       podvm-image-id:
         type: string
         description: prebuilt podvm image
+      install_method:
+        default: 'kustomize'
+        description: Installation method. Either kustomize or helm.
+        required: false
+        type: string
 
 jobs:
   build-caa-container-image:
@@ -211,11 +221,24 @@ jobs:
         sudo apt-get install -y sipcalc
 
     - name: Install kustomize
+      if: ${{ inputs.install_method == 'kustomize' }}
       run: |
         command -v kustomize >/dev/null || \
         curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \
           sudo bash -s /usr/local/bin
 
+    - name: Install Helm
+      if: ${{ inputs.install_method == 'helm' }}
+      run: |
+        HELM_VERSION="$(yq -e '.tools.helm.version' versions.yaml)"
+        HELM_CHECKSUM="$(yq -e '.tools.helm.sha256' versions.yaml)"
+        curl -fsSL -o helm.tar.gz "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz"
+        echo "${HELM_CHECKSUM}  helm.tar.gz" | sha256sum --check --strict
+        tar -xzf helm.tar.gz
+        sudo mv linux-amd64/helm /usr/local/bin/helm
+        rm -rf helm.tar.gz linux-amd64
+        helm version
+
     - name: Restore the configuration created before
       uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
       with:
@@ -246,6 +269,48 @@ jobs:
         AZURE_SUBNET_ID="$subnet_id"
         EOF
 
+        # Export for helm values step
+        echo "AZURE_SUBNET_ID=$subnet_id" >> "$GITHUB_ENV"
+
+    - name: Create helm values file
+      if: ${{ inputs.install_method == 'helm' }}
+      env:
+        CAA_IMAGE: "${{ needs.build-caa-container-image.outputs.caa-image }}"
+        AZURE_IMAGE_ID: "${{ inputs.podvm-image-id }}"
+        AZURE_INSTANCE_SIZE: "${{ matrix.parameters.machine_type }}"
+        AZURE_SUBSCRIPTION_ID: "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+        AZURE_CLIENT_ID: "${{ secrets.AZURE_CLIENT_ID }}"
+      run: |
+        CAA_IMAGE_TAG="${CAA_IMAGE##*:}"
+        CAA_IMAGE_NAME="${CAA_IMAGE%:*}"
+
+        cat <<EOF > helm-values.yaml
+        image:
+          name: "${CAA_IMAGE_NAME}"
+          tag: "${CAA_IMAGE_TAG}"
+        providerConfigs:
+          azure:
+            AZURE_SUBSCRIPTION_ID: "${AZURE_SUBSCRIPTION_ID}"
+            AZURE_REGION: "${LOCATION}"
+            AZURE_RESOURCE_GROUP: "${RG_NAME}"
+            AZURE_IMAGE_ID: "${AZURE_IMAGE_ID}"
+            AZURE_INSTANCE_SIZE: "${AZURE_INSTANCE_SIZE}"
+            AZURE_SUBNET_ID: "${AZURE_SUBNET_ID}"
+        providerSecrets:
+          azure:
+            AZURE_CLIENT_ID: "${AZURE_CLIENT_ID}"
+        EOF
+
+        echo "HELM_VALUES_FILES=$PWD/install/charts/peerpods/providers/azure.yaml,$PWD/helm-values.yaml" >> "$GITHUB_ENV"
+
+        # For debugging (without secrets)
+        echo "::group::helm-values.yaml (structure only)"
+        echo "image.name: ${CAA_IMAGE_NAME}"
+        echo "image.tag: ${CAA_IMAGE_TAG}"
+        echo "providerConfigs.azure: (subscription, region, resource group, image, instance size, subnet)"
+        echo "providerSecrets.azure: (client id)"
+        echo "::endgroup::"
+
     - name: Checkout KBS Repository
       run: test/utils/checkout_kbs.sh
 
@@ -254,6 +319,7 @@ jobs:
       env:
         TEST_PROVISION: "no"
         DEPLOY_KBS: "yes"
+        INSTALL_METHOD: "${{ inputs.install_method }}"
         CUSTOM_PCCS_URL: "https://global.acccache.azure.net/sgx/certification/v4"
         CLUSTER_NAME: "${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}"
       run: |

.github/workflows/e2e_aws.yaml

@@ -191,6 +191,31 @@ jobs:
           cat aws.properties
           echo "::endgroup::"
 
+      - name: Create helm values file
+        if: ${{ inputs.install_method == 'helm' }}
+        run: |
+          CAA_IMAGE_TAG="${CAA_IMAGE##*:}"
+          CAA_IMAGE_NAME="${CAA_IMAGE%:*}"
+          cat <<EOF > helm-values.yaml
+          image:
+            name: "${CAA_IMAGE_NAME}"
+            tag: "${CAA_IMAGE_TAG}"
+          providerConfigs:
+            aws:
+              DISABLECVM: "true"
+              SSH_KP_NAME: "caa-e2e-test"
+              AWS_REGION: "us-east-1"
+              PODVM_INSTANCE_TYPE: "t2.medium"
+              USE_PUBLIC_IP: "true"
+          EOF
+
+          echo "HELM_VALUES_FILES=install/charts/peerpods/providers/aws.yaml,$PWD/helm-values.yaml" >> "$GITHUB_ENV"
+
+          # For debugging
+          echo "::group::helm-values.yaml"
+          cat helm-values.yaml
+          echo "::endgroup::"
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1
         with:

.github/workflows/e2e_docker.yaml

@@ -144,16 +144,36 @@ jobs:
           DOCKER_API_VERSION="1.44"
           EOF
 
-          # If using helm, split CAA_IMAGE into CAA_IMAGE_NAME and CAA_IMAGE_TAG
-          if [ "${INSTALL_METHOD}" = "helm" ]; then
-            CAA_IMAGE_TAG="${CAA_IMAGE##*:}"
-            CAA_IMAGE_NAME="${CAA_IMAGE%:*}"
-            echo "CAA_IMAGE=\"${CAA_IMAGE_NAME}\"" >> docker.properties
-            echo "CAA_IMAGE_TAG=\"${CAA_IMAGE_TAG}\"" >> docker.properties
-          fi
-
           # For debugging
+          echo "::group::docker.properties"
           cat docker.properties
+          echo "::endgroup::"
+
+      - name: Create helm values file
+        if: ${{ inputs.install_method == 'helm' }}
+        env:
+          DOCKER_PODVM_IMAGE: ${{ inputs.podvm_image }}
+        run: |
+          CAA_IMAGE_TAG="${CAA_IMAGE##*:}"
+          CAA_IMAGE_NAME="${CAA_IMAGE%:*}"
+          cat <<EOF > helm-values.yaml
+          image:
+            name: "${CAA_IMAGE_NAME}"
+            tag: "${CAA_IMAGE_TAG}"
+          providerConfigs:
+            docker:
+              DOCKER_HOST: "unix:///var/run/docker.sock"
+              DOCKER_NETWORK_NAME: "kind"
+              DOCKER_API_VERSION: "1.44"
+              DOCKER_PODVM_IMAGE: "${DOCKER_PODVM_IMAGE}"
+          EOF
+
+          echo "HELM_VALUES_FILES=install/charts/peerpods/providers/docker.yaml,$PWD/helm-values.yaml" >> "$GITHUB_ENV"
+
+          # For debugging
+          echo "::group::helm-values.yaml"
+          cat helm-values.yaml
+          echo "::endgroup::"
 
       - name: run tests
         id: runTests

.github/workflows/e2e_libvirt.yaml

@@ -196,113 +196,38 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Provision cluster (helm only)
-        if: ${{ inputs.install_method == 'helm' }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CLOUD_PROVIDER: libvirt
-          CONTAINER_RUNTIME: ${{ inputs.container_runtime }}
-          TEST_PROVISION_FILE: ${{ github.workspace }}/src/cloud-api-adaptor/libvirt.properties
-          TEST_PODVM_IMAGE: ${{ env.PODVM_QCOW2 }}
-          TEST_E2E_TIMEOUT: "30m"
-        run: make provision-cluster
-
-      - name: Deploy CAA using Helm chart
+      - name: Create helm values file
         if: ${{ inputs.install_method == 'helm' }}
         env:
           CAA_IMAGE: ${{ inputs.caa_image }}
         run: |
-          export KUBECONFIG="${HOME}/.kcli/clusters/peer-pods/auth/kubeconfig"
-
-          # Install cert-manager (required for webhook)
-          echo "Installing cert-manager..."
-          make -C ../webhook deploy-cert-manager
-
-          kubectl create namespace confidential-containers-system --dry-run=client -o yaml | kubectl apply -f -
-
-          # Extract image name and tag from caa_image (format: registry/image:tag)
-          IMAGE_FULL="${CAA_IMAGE}"
-          IMAGE_TAG="${IMAGE_FULL##*:}"
-          IMAGE_NAME="${IMAGE_FULL%:*}"
-
-          echo "Deploying with Helm:"
-          echo "  Image: ${IMAGE_NAME}"
-          echo "  Tag: ${IMAGE_TAG}"
-
-          # Build chart dependencies (downloads kata-deploy from OCI registry)
-          echo "Building Helm chart dependencies..."
-          helm dependency build ./install/charts/peerpods
-
-          # TODO: Temporary solution. Providers lack Helm Apply() in test framework.
-          #       Long term: implement HelmInstallOverlay.Apply() similar to other ProviderInstallOverlay implementations.
-          # Read libvirt config from properties file (created by config_libvirt.sh)
           source libvirt.properties
 
-          # Create SSH key secret for libvirt (keys created by config_libvirt.sh)
-          # TODO: This should use --set-file providerSecrets.libvirt.id_rsa once
-          #       providerSecrets.libvirt is defined in providers/libvirt.yaml
-          kubectl create secret generic ssh-key-secret \
-            --from-file=id_rsa="${HOME}/.ssh/id_rsa" \
-            --from-file=id_rsa.pub="${HOME}/.ssh/id_rsa.pub" \
-            -n confidential-containers-system
-
-          # Install chart with libvirt configuration
-          helm install peerpods ./install/charts/peerpods \
-            -f install/charts/peerpods/providers/libvirt.yaml \
-            --set "image.name=${IMAGE_NAME}" \
-            --set "image.tag=${IMAGE_TAG}" \
-            --set-string "providerConfigs.libvirt.LIBVIRT_URI=${libvirt_uri}" \
-            -n confidential-containers-system \
-            --wait --timeout=10m || {
-              echo "::error::Helm install failed. Collecting debug info..."
-              echo "::group::Pre-install job pods"
-              kubectl get pods -n confidential-containers-system -l app.kubernetes.io/component=installer -o wide
-              echo "::endgroup::"
-              echo "::group::Pre-install job pod describe (shows OOMKilled, exit code, etc)"
-              kubectl describe pods -n confidential-containers-system -l app.kubernetes.io/component=installer || true
-              echo "::endgroup::"
-              echo "::group::Pre-install job describe"
-              kubectl describe job -n confidential-containers-system -l app.kubernetes.io/component=installer || true
-              echo "::endgroup::"
-              echo "::group::Pre-install job logs (current)"
-              kubectl logs -n confidential-containers-system -l app.kubernetes.io/component=installer --tail=500 || true
-              echo "::endgroup::"
-              echo "::group::Pre-install job logs (previous - shows failed run)"
-              kubectl logs -n confidential-containers-system -l app.kubernetes.io/component=installer --previous --tail=500 || true
-              echo "::endgroup::"
-              echo "::group::Pre-install job events"
-              kubectl get events -n confidential-containers-system --sort-by='.lastTimestamp' | tail -50
-              echo "::endgroup::"
-              echo "::group::Webhook namespace check (peer-pods-webhook-system)"
-              kubectl get namespace peer-pods-webhook-system 2>&1 || echo "Namespace does not exist"
-              kubectl get events -n peer-pods-webhook-system --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
-              echo "::endgroup::"
-              exit 1
-            }
-
-          echo "Waiting for CAA daemonset to be ready..."
-          kubectl rollout status daemonset/cloud-api-adaptor-daemonset -n confidential-containers-system --timeout=5m
-
-          echo "Waiting for kata-deploy daemonset to be ready..."
-          kubectl rollout status daemonset/kata-deploy -n confidential-containers-system --timeout=5m
-
-          # kata-deploy labels nodes with katacontainers.io/kata-runtime=true after installing kata.
-          # The kata-remote RuntimeClass has a nodeSelector requiring this label.
-          # We must wait for at least one node to have this label before tests can run.
-          echo "Waiting for node to be labeled with katacontainers.io/kata-runtime=true..."
-          timeout 120 bash -c '
-            until kubectl get nodes -l katacontainers.io/kata-runtime=true --no-headers 2>/dev/null | grep -q .; do
-              echo "  Waiting for kata-runtime node label..."
-              sleep 5
-            done
-          '
-          echo "Node labeled successfully:"
-          kubectl get nodes -l katacontainers.io/kata-runtime=true
-
-          echo "::group::Helm deployment info"
-          helm list -n confidential-containers-system
-          kubectl get daemonset -n confidential-containers-system
-          kubectl get pods -n confidential-containers-system -l app=cloud-api-adaptor
+          CAA_IMAGE_TAG="${CAA_IMAGE##*:}"
+          CAA_IMAGE_NAME="${CAA_IMAGE%:*}"
+
+          # Create base values file
+          cat <<EOF > helm-values.yaml
+          image:
+            name: "${CAA_IMAGE_NAME}"
+            tag: "${CAA_IMAGE_TAG}"
+          providerConfigs:
+            libvirt:
+              LIBVIRT_URI: "${libvirt_uri}"
+          EOF
+
+          # Append SSH keys using yq for proper YAML formatting
+          yq -i '.providerSecrets.libvirt.id_rsa = load_str("'"${HOME}"'/.ssh/id_rsa")' helm-values.yaml
+          yq -i '.providerSecrets.libvirt."id_rsa.pub" = load_str("'"${HOME}"'/.ssh/id_rsa.pub")' helm-values.yaml
+
+          echo "HELM_VALUES_FILES=install/charts/peerpods/providers/libvirt.yaml,$PWD/helm-values.yaml" >> "$GITHUB_ENV"
+
+          # For debugging (without secrets)
+          echo "::group::helm-values.yaml (structure only)"
+          echo "image.name: ${CAA_IMAGE_NAME}"
+          echo "image.tag: ${CAA_IMAGE_TAG}"
+          echo "providerConfigs.libvirt.LIBVIRT_URI: ${libvirt_uri}"
+          echo "providerSecrets.libvirt: (ssh keys present)"
           echo "::endgroup::"
 
       - name: run tests
@@ -320,21 +245,12 @@ jobs:
           TEST_PODVM_IMAGE: ${{ env.PODVM_QCOW2 }}
           TEST_E2E_TIMEOUT: "75m"
         run: |
-          # Default: provision cluster and install CAA
           export TEST_PROVISION="yes"
           export TEST_TEARDOWN="no"
           export TEST_PROVISION_FILE="$PWD/libvirt.properties"
           export TEST_PODVM_IMAGE="${{ env.PODVM_QCOW2 }}"
           export TEST_E2E_TIMEOUT="75m"
 
-          # Skip provisioning and CAA installation if using helm (already done above)
-          # KBS is deployed here (not in provision step) so keys match
-          if [ "${INSTALL_METHOD}" = "helm" ]; then
-            export TEST_PROVISION="no"
-            export TEST_INSTALL_CAA="no"
-            export KUBECONFIG="${HOME}/.kcli/clusters/peer-pods/auth/kubeconfig"
-          fi
-
           make test-e2e
 
       - name: Debug tests failure

src/cloud-api-adaptor/go.mod

@@ -73,6 +73,7 @@ require (
 	github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.0.0-00010101000000-000000000000
 	github.com/distribution/reference v0.6.0
 	github.com/fenglyu/go-dmidecode v0.0.0-20220417074508-03f52eb45fe9
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -238,7 +239,6 @@ require (
 	gopkg.in/evanphx/json-patch.v5 v5.6.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gotest.tools/v3 v3.5.2 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect