ci: improve fetch Core manifest workflow

Signed-off-by: Andrea Mazzotti <andrea.mazzotti@suse.com>

Author: anmazzotti

.github/scripts/fetch-core-capi.sh

@@ -1,65 +1,75 @@
-name: Fetch core CAPI components manifest and embed in Turtles chart for air-gapped installations.
+name: Update Core CAPI components manifest
 on:
-  schedule:
-    - cron: "0 0 * * *" # Run every day at midnight (UTC)
-  # allow running manually on demand
   workflow_dispatch:
+    inputs:
+      capi_version:
+        type: string
+        description: CAPI Core version (ex. 'v1.2.3' or empty for latest)
+        required: false
+
 
 env:
   TURTLES_REF: "${{ github.ref_name }}"
-  GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+  GH_TOKEN: "${{ github.token }}"
 
 jobs:
   create-core-capi-turtles-pr:
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      pull-requests: write
+      contents: write
     steps:
       - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v5
         with:
-          ref: "${{ env.TURTLES_REF }}"
-          token: ${{ env.GH_TOKEN }}
-          # Allow making git push request later on
-          persist-credentials: true
+          fetch-depth: 0
 
       - name: Configure the committer
         run: |
           user_id=$(gh api "/users/$APP_USER" --jq .id)
           git config --global user.name "$APP_USER"
           git config --global user.email "${user_id}+${APP_USER}@users.noreply.github.com"
         env:
-          GH_TOKEN: "${{ env.GH_TOKEN }}"
           APP_USER: "${{ github.actor }}"
 
       - name: Run script to fetch components manifest
         run: |
-          CAPI_VERSION=$(curl -s "https://api.github.com/repos/rancher-sandbox/cluster-api/releases/latest" | jq -r ".tag_name")
-          echo "CAPI_VERSION=${CAPI_VERSION}" >> $GITHUB_ENV
+          CAPI_MANIFEST_UPDATE_VERSION=${{ inputs.capi_version }}
+          if [ -z "${CAPI_MANIFEST_UPDATE_VERSION}" ]; then
+            echo "Fetching latest CAPI Core version"
+            CAPI_MANIFEST_UPDATE_VERSION=$(curl -s "https://api.github.com/repos/rancher-sandbox/cluster-api/releases/latest" | jq -r ".tag_name")
+            echo "Found version ${CAPI_MANIFEST_UPDATE_VERSION}"
+          fi
+          echo "CAPI_MANIFEST_UPDATE_VERSION=${CAPI_MANIFEST_UPDATE_VERSION}" >> $GITHUB_ENV
           BRANCH="fetch-core-capi-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
           echo "BRANCH=${BRANCH}" >> $GITHUB_ENV
           git checkout -b "$BRANCH" "$TURTLES_REF"
-          CAPI_VERSION=$CAPI_VERSION ./.github/scripts/fetch-core-capi.sh
+          CAPI_MANIFEST_UPDATE_VERSION=${CAPI_MANIFEST_UPDATE_VERSION} make update-core-capi-manifest
           git add charts/rancher-turtles
           if git diff --cached --quiet; then
             echo "No changes detected"
             echo "SKIP_PUSH=true" >> $GITHUB_ENV
           else
-            git commit -m "chore: embed core CAPI ${CAPI_VERSION} in Turtles chart"
+            git commit -m "chore: embed core CAPI ${CAPI_MANIFEST_UPDATE_VERSION} in Turtles chart"
             echo "SKIP_PUSH=false" >> $GITHUB_ENV
           fi
 
       - name: Push and create pull request
         if: env.SKIP_PUSH == 'false'
-        env:
-          GH_TOKEN: "${{ env.GH_TOKEN }}"
         run: |
           git push origin "$BRANCH"
-          body="This PR fetches core CAPI $CAPI_VERSION components manifest from release and embeds the template in the Turtles chart for a simplified air-gapped installation."
+          body="This PR fetches core CAPI ${CAPI_MANIFEST_UPDATE_VERSION} components manifest from release and embeds the template in the Turtles chart for a simplified air-gapped installation."
 
           gh pr create \
-            --title "chore: embed core CAPI provider $CAPI_VERSION manifest in chart" \
+            --title "chore: embed core CAPI provider ${CAPI_MANIFEST_UPDATE_VERSION} manifest in chart" \
             --body "$body" \
             --head "${{ github.repository_owner }}:$BRANCH" \
             --base "$TURTLES_REF" \
             --label "area/installation" \
             --label "kind/ci"
+
+      - name: Collect run artifacts
+        if: env.SKIP_PUSH == 'false'
+        uses: actions/upload-artifact@v7
+        with:
+          name: update-core-capi-artifacts
+          path: _artifacts

.github/workflows/fetch-core-capi-airgapped.yml

@@ -39,4 +39,7 @@ hack/crd/bases/*
 # helm
 **/Chart.lock
 
-.buildx-cache/
+.buildx-cache/
+
+# krew
+hack/tools/krew/*

.gitignore

@@ -39,6 +39,8 @@ REPO ?= rancher/turtles
 CAPI_VERSION ?= $(shell grep "sigs.k8s.io/cluster-api" go.mod | head -1 |awk '{print $$NF}')
 CAPI_UPSTREAM_REPO ?= https://github.com/kubernetes-sigs/cluster-api
 CAPI_UPSTREAM_RELEASES ?= $(CAPI_UPSTREAM_REPO)/releases
+CAPI_MANIFEST_UPDATE_VERSION ?= $(shell curl -s "https://api.github.com/repos/rancher-sandbox/cluster-api/releases/latest" | jq -r ".tag_name")
+CAPI_MANIFEST_OUTPUT_FILE ?= $(CHART_DIR)/templates/core-provider-configmap.yaml
 
 # Use GOPROXY environment variable if set
 GOPROXY := $(shell go env GOPROXY)
@@ -66,7 +68,9 @@ $(TOOLS_BIN_DIR):
 	mkdir -p $@
 
 export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
-export KREW_ROOT := $(abspath $(TOOLS_BIN_DIR))
+
+# Configure krew root directory to tools dir 
+export KREW_ROOT := $(abspath $(TOOLS_DIR))/krew
 export PATH := $(KREW_ROOT)/bin:$(PATH)
 
 # Set --output-base for conversion-gen if we are not within GOPATH
@@ -76,7 +80,6 @@ else
 	export GOPATH := $(shell go env GOPATH)
 endif
 
-
 #
 # Ginkgo configuration.
 #
@@ -168,9 +171,6 @@ GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/v2/cmd/golangci-lint
 NOTES_BIN := notes
 NOTES := $(abspath $(TOOLS_BIN_DIR)/$(NOTES_BIN))
 
-CRUST_GATHER_BIN := crust-gather
-CRUST_GATHER := $(abspath $(TOOLS_BIN_DIR)/$(CRUST_GATHER_BIN))
-
 CHART_TESTING_VER := v3.14.0
 
 # Registry / images
@@ -495,9 +495,6 @@ $(CONVERSION_GEN): # Build conversion-gen from tools folder.
 .PHONY: $(GINKGO_BIN)
 $(GINKGO_BIN): $(GINKGO) ## Build a local copy of ginkgo.
 
-.PHONY: $(CRUST_GATHER_BIN)
-$(CRUST_GATHER_BIN): $(CRUST_GATHER) ## Download crust-gather.
-
 $(GO_APIDIFF): # Build go-apidiff from tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GO_APIDIFF_PKG) $(GO_APIDIFF_BIN) $(GO_APIDIFF_VER)
 
@@ -530,9 +527,6 @@ $(GH): # Download GitHub cli into the tools bin folder
 		-b $(TOOLS_BIN_DIR) \
 		$(GH_VERSION)
 
-$(CRUST_GATHER): # Downloads and install crust-gather
-	curl -sSfL https://github.com/crust-gather/crust-gather/raw/main/install.sh | sh -s - -f -b $(TOOLS_BIN_DIR)
-
 kubectl: # Download kubectl cli into tools bin folder
 	hack/ensure-kubectl.sh \
 		-b $(TOOLS_BIN_DIR) \
@@ -727,5 +721,16 @@ clean-rancher-charts: ## Remove the local rancher charts folder
 ## --------------------------------------
 
 .PHONY: collect-artifacts
-collect-artifacts: $(CRUST_GATHER_BIN)
-	$(CRUST_GATHER) collect -f $(ARTIFACTS_FOLDER)/gather
+collect-artifacts: kubectl
+	mkdir -p $(ARTIFACTS_FOLDER)
+	kubectl crust-gather collect -f $(ARTIFACTS_FOLDER)/gather
+
+## --------------------------------------
+## Update the CAPI Core Provider manifest
+## --------------------------------------
+
+.PHONY: update-core-capi-manifest
+update-core-capi-manifest: kubectl
+	mkdir -p $(ARTIFACTS_FOLDER)
+	ARTIFACTS_FOLDER=$(ARTIFACTS_FOLDER) CAPI_VERSION=$(CAPI_MANIFEST_UPDATE_VERSION) OUTPUT_FILE=$(CAPI_MANIFEST_OUTPUT_FILE) hack/fetch-core-capi.sh
+

Makefile

@@ -55,9 +55,7 @@ kind: ConfigMap
 metadata:
   annotations:
     provider.cluster.x-k8s.io/compressed: "true"
-  creationTimestamp: null
   labels:
-    managed-by.operator.cluster.x-k8s.io: "true"
     provider.cluster.x-k8s.io/name: cluster-api
     provider.cluster.x-k8s.io/type: core
     provider.cluster.x-k8s.io/version: v1.12.2

charts/rancher-turtles/templates/core-provider-configmap.yaml

@@ -2,6 +2,11 @@ module github.com/rancher/turtles
 
 go 1.25.8
 
+ignore (
+	./hack/tools/krew
+	./out
+)
+
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/go-logr/logr v1.4.3

go.mod

@@ -66,7 +66,15 @@ install_plugins() {
     tar zxvf "${KREW}.tar.gz" &&
     ./"${KREW}" install krew
   )
+  kubectl krew version
+
   kubectl krew install crust-gather
+  kubectl crust-gather --version
+
+  rm -rf ${KREW_ROOT}/index/operator # Clear the index to prevent errors and ensure update on next add
+  kubectl krew index add operator https://github.com/kubernetes-sigs/cluster-api-operator.git
+  kubectl krew install operator/clusterctl-operator
+  kubectl operator version
 }
 
 verify_kubectl_version

hack/ensure-kubectl.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# script-specific variables
+CAPI_VERSION="${CAPI_VERSION:-latest}"
+CAPI_RELEASE_URL="${CAPI_RELEASE_URL:-https://github.com/rancher-sandbox/cluster-api/releases/${CAPI_VERSION}/core-components.yaml}"
+CORE_CAPI_NAMESPACE="${CORE_CAPI_NAMESPACE:-cattle-capi-system}"
+OUTPUT_FILE="${OUTPUT_FILE:-/tmp/core-provider-configmap.yaml}"
+ARTIFACTS_FOLDER="${ARTIFACTS_FOLDER:-_artifacts}"
+
+# parameters that must be substituted in CAPI manifest
+export CAPI_DIAGNOSTICS_ADDRESS=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}
+export CAPI_INSECURE_DIAGNOSTICS=${CAPI_INSECURE_DIAGNOSTICS:=false}
+export EXP_MACHINE_POOL=${EXP_MACHINE_POOL:=true}
+export EXP_CLUSTER_RESOURCE_SET=${EXP_CLUSTER_RESOURCE_SET:=true}
+export CLUSTER_TOPOLOGY=${CLUSTER_TOPOLOGY:=true}
+export EXP_RUNTIME_SDK=${EXP_RUNTIME_SDK:=false}
+export EXP_MACHINE_SET_PREFLIGHT_CHECKS=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=true}
+export EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS=${EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS:=true}
+export EXP_PRIORITY_QUEUE=${EXP_PRIORITY_QUEUE:=false}
+
+# use CAPI Operator plugin to generate ConfigMap with core CAPI components
+kubectl operator preload --core cluster-api --target-namespace ${CORE_CAPI_NAMESPACE} -u ${CAPI_RELEASE_URL} > ${OUTPUT_FILE}
+# replace cluster-api-operator managed label with turtles
+yq -i 'del(.metadata.labels["managed-by.operator.cluster.x-k8s.io"])'  ${OUTPUT_FILE}
+yq -i '.metadata.labels["managed-by.turtles.cattle.io"]="true"' ${OUTPUT_FILE}
+
+# Dump the manifest in the artifacts directory
+if [ $(yq '.metadata.annotations["provider.cluster.x-k8s.io/compressed"]' ${OUTPUT_FILE}) = "true" ]; then
+    echo "Dumping uncompressed manifest in ${ARTIFACTS_FOLDER}/core-manifest-update-dump.yaml"
+    yq '.binaryData.components' ${OUTPUT_FILE} | base64 -d | gzip -d > ${ARTIFACTS_FOLDER}/core-manifest-update-dump.yaml
+else
+    echo "Dumping manifest in ${ARTIFACTS_FOLDER}/core-manifest-update-dump.yaml"
+    yq '.data.components' ${OUTPUT_FILE} > ${ARTIFACTS_FOLDER}/core-manifest-update-dump.yaml
+fi