Common automatic update

common/.github/workflows/pattern-sh-ci.yml

@@ -0,0 +1,48 @@
+name: Run Bash Script on Multiple Distributions
+
+on:
+  push:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+
+jobs:
+  run-script:
+    name: Run Bash Script
+    strategy:
+      matrix:
+        # Fedora is not an option yet
+        os: [ubuntu-latest, ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Podman on Ubuntu
+        if: contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y podman
+
+      # Currently we do not do MacOSX as it is not free, maybe in the future
+      # - name: Install Podman on macOS
+      #   if: contains(matrix.os, 'macos')
+      #   run: |
+      #     brew install podman
+      #     podman machine init
+      #     podman machine start
+
+      - name: Verify Podman Installation
+        run: podman --version
+
+      - name: Run pattern.sh script
+        run: |
+          export TARGET_BRANCH=main
+          ./scripts/pattern-util.sh make validate-origin

common/Makefile

@@ -8,6 +8,10 @@ endif
 # the command line. I.e. we can set things without having to tweak values files
 EXTRA_HELM_OPTS ?=
 
+# This variable can be set in order to pass additional ansible-playbook arguments from the
+# the command line. I.e. we can set -vvv for more verbose logging
+EXTRA_PLAYBOOK_OPTS ?=
+
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248
 # or
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248,registry-proxy.engineering.redhat.com/rh-osbs/iib:394249
@@ -18,7 +22,7 @@ TARGET_ORIGIN ?= origin
 # This is because we expect to use tokens for repo authentication as opposed to SSH keys
 TARGET_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN) | sed -e 's/.*URL:[[:space:]]*//' -e 's%^git@%%' -e 's%^https://%%' -e 's%:%/%' -e 's%^%https://%')
 # git branch --show-current is also available as of git 2.22, but we will use this for compatibility
-TARGET_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
+TARGET_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 UUID_FILE ?= ~/.config/validated-patterns/pattern-uuid
 UUID_HELM_OPTS ?=
@@ -68,11 +72,7 @@ preview-%:
 
 .PHONY: operator-deploy
 operator-deploy operator-upgrade: validate-prereq validate-origin validate-cluster ## runs helm install
-	@set -e -o pipefail
-	# Retry five times because the CRD might not be fully installed yet
-	for i in {1..5}; do \
-		helm template --include-crds --name-template $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) | oc apply -f- && break || sleep 10; \
-	done
+	@common/scripts/deploy-pattern.sh $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS)
 
 .PHONY: uninstall
 uninstall: ## runs helm uninstall
@@ -115,7 +115,7 @@ secrets-backend-none: ## Edits values files to remove secrets manager + ESO
 .PHONY: load-iib
 load-iib: ## CI target to install Index Image Bundles
 	@set -e; if [ x$(INDEX_IMAGES) != x ]; then \
-		ansible-playbook rhvp.cluster_utils.iib_ci; \
+		ansible-playbook $(EXTRA_PLAYBOOK_OPTS) rhvp.cluster_utils.iib_ci; \
 	else \
 		echo "No INDEX_IMAGES defined. Bailing out"; \
 		exit 1; \
@@ -129,12 +129,22 @@ token-kubeconfig: ## Create a local ~/.kube/config with password (not usually ne
 
 # We only check the remote ssh git branch's existance if we're not running inside a container
 # as getting ssh auth working inside a container seems a bit brittle
+# If the main repoUpstreamURL field is set, then we need to check against
+# that and not target_repo
 .PHONY: validate-origin
 validate-origin: ## verify the git origin is available
 	@echo "Checking repository:"
-	@echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': "
-	@git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
-		echo "OK" || (echo "NOT FOUND"; exit 1)
+	$(eval UPSTREAMURL := $(shell yq -r '.main.git.repoUpstreamURL // (.main.git.repoUpstreamURL = "")' values-global.yaml))
+	@if [ -z "$(UPSTREAMURL)" ]; then\
+		echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	else\
+		echo "Upstream URL set to: $(UPSTREAMURL)";\
+		echo -n "  $(UPSTREAMURL) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(UPSTREAMURL) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	fi
 
 .PHONY: validate-cluster
 validate-cluster: ## Do some cluster validations before installing
@@ -153,15 +163,20 @@ validate-cluster: ## Do some cluster validations before installing
 validate-schema: ## validates values files against schema in common/clustergroup
 	$(eval VAL_PARAMS := $(shell for i in ./values-*.yaml; do echo -n "$${i} "; done))
 	@echo -n "Validating clustergroup schema of: "
-	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template common/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
+	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template oci://quay.io/hybridcloudpatterns/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
 	@echo
 
 .PHONY: validate-prereq
 validate-prereq: ## verify pre-requisites
+	$(eval GLOBAL_PATTERN := $(shell yq -r .global.pattern values-global.yaml))
+	@if [ $(NAME) != $(GLOBAL_PATTERN) ]; then\
+		echo "";\
+		echo "WARNING: folder directory is \"$(NAME)\" and global.pattern is set to \"$(GLOBAL_PATTERN)\"";\
+		echo "this can create problems. Please make sure they are the same!";\
+		echo "";\
+	fi
 	@if [ ! -f /run/.containerenv ]; then\
 	  echo "Checking prerequisites:";\
-	  for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\
-	  echo "  Check for '$(EXECUTABLES)': OK";\
 	  echo -n "  Check for python-kubernetes: ";\
 	  if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
 	  echo "OK";\
@@ -182,16 +197,16 @@ validate-prereq: ## verify pre-requisites
 .PHONY: argo-healthcheck
 argo-healthcheck: ## Checks if all argo applications are synced
 	@echo "Checking argo applications"
-	$(eval APPS := $(shell oc get applications -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
+	$(eval APPS := $(shell oc get applications.argoproj.io -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
 	@NOTOK=0; \
 	for i in $(APPS); do\
 		n=`echo "$${i}" | cut -f1 -d,`;\
 		a=`echo "$${i}" | cut -f2 -d,`;\
-		STATUS=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.sync.status}'`;\
+		STATUS=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.sync.status}'`;\
 		if [[ $$STATUS != "Synced" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
-		HEALTH=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.health.status}'`;\
+		HEALTH=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.health.status}'`;\
 		if [[ $$HEALTH != "Healthy" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
@@ -208,7 +223,7 @@ argo-healthcheck: ## Checks if all argo applications are synced
 .PHONY: qe-tests
 qe-tests: ## Runs the tests that QE runs
 	@set -e; if [ -f ./tests/interop/run_tests.sh ]; then \
-		./tests/interop/run_tests.sh; \
+		pushd ./tests/interop; ./run_tests.sh; popd; \
 	else \
 		echo "No ./tests/interop/run_tests.sh found skipping"; \
 	fi

common/README.md

@@ -34,16 +34,16 @@ main:
 ## Start Here
 
 This repository is never used as standalone. It is usually imported in each pattern as a subtree.
-In order to import the common/ the very first time you can use
-`https://github.com/validatedpatterns/multicloud-gitops/blob/main/common/scripts/make_common_subtree.sh`
+In order to import the common subtree the very first time you can use the script
+[make_common_subtree.sh](scripts/make-common-subtree.sh).
 
 In order to update your common subtree inside your pattern repository you can either use
 `https://github.com/validatedpatterns/utilities/blob/main/scripts/update-common-everywhere.sh` or
-do it manually by doing the following:
+do it manually with the following commands:
 
 ```sh
-git remote add -f upstream-common https://github.com/validatedpatterns/common.git
-git merge -s subtree -Xtheirs -Xsubtree=common upstream-common/main
+git remote add -f common-upstream https://github.com/validatedpatterns/common.git
+git merge -s subtree -Xtheirs -Xsubtree=common common-upstream/main
 ```
 
 ## Secrets

common/scripts/deploy-pattern.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -o pipefail
+
+RUNS=10
+WAIT=15
+# Retry five times because the CRD might not be fully installed yet
+echo -n "Installing pattern: "
+for i in $(seq 1 ${RUNS}); do \
+    exec 3>&1 4>&2
+    OUT=$( { helm template --include-crds --name-template $* 2>&4 | oc apply -f- 2>&4 1>&3; } 4>&1 3>&1)
+    ret=$?
+    exec 3>&- 4>&-
+    if [ ${ret} -eq 0 ]; then
+        break;
+    else
+        echo -n "."
+        sleep "${WAIT}"
+    fi
+done
+
+# All the runs failed
+if [ ${i} -eq ${RUNS} ]; then
+    echo "Installation failed [${i}/${RUNS}]. Error:"
+    echo "${OUT}"
+    exit 1
+fi
+echo "Done"

common/scripts/display-secrets-info.sh

@@ -23,4 +23,6 @@ fi
 
 PATTERN_NAME=$(basename "`pwd`")
 
-ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e override_no_log=false "rhvp.cluster_utils.display_secrets_info"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e hide_sensitive_output=false ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.display_secrets_info"

common/scripts/load-k8s-secrets.sh

@@ -13,4 +13,6 @@ PATTERNPATH=$(dirname "${COMMONPATH}")
 
 PATTERN_NAME=${1:-$(basename "`pwd`")}
 
-ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" "rhvp.cluster_utils.k8s_secrets"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.k8s_secrets"

common/scripts/make_common_subtree.sh → common/scripts/make-common-subtree.sh

@@ -7,7 +7,7 @@ if [ "$1" = "-h" ]; then
 	echo "$0 <subtree_repo> <subtree_branch> <subtree_remote_name>"
 	echo
 	echo "Run without arguments, the script would run as if these arguments had been passed:"
-	echo "$0 https://github.com/hybrid-cloud-patterns/common.git main common-subtree"
+	echo "$0 https://github.com/validatedpatterns/common.git main common-upstream"
 	echo
 	echo "Please ensure the git subtree command is available. On RHEL/Fedora, the git subtree command"
 	echo "is in a separate package called git-subtree"
@@ -41,7 +41,7 @@ fi
 if [ "$1" ]; then
 	subtree_repo=$1
 else
-	subtree_repo=https://github.com/hybrid-cloud-patterns/common.git
+	subtree_repo=https://github.com/validatedpatterns/common.git
 fi
 
 if [ "$2" ]; then
@@ -53,7 +53,7 @@ fi
 if [ "$3" ]; then
 	subtree_remote=$3
 else
-	subtree_remote=common-subtree
+	subtree_remote=common-upstream
 fi
 
 git diff --quiet || (echo "This script must be run on a clean working tree" && exit 1)

common/scripts/pattern-util.sh

@@ -64,8 +64,10 @@ fi
 # if we are using podman machine then we do not bind mount anything (for now!)
 REMOTE_PODMAN=$(podman system connection list -q | wc -l)
 if [ $REMOTE_PODMAN -eq 0 ]; then # If we are not using podman machine we check the hosts folders
-    # Use /etc/pki by default and try a couple of fallbacks if it does not exist
-    if [ -d /etc/pki ]; then
+    # We check /etc/pki/tls because on ubuntu /etc/pki/fwupd sometimes
+    # exists but not /etc/pki/tls and we do not want to bind mount in such a case
+    # as it would find no certificates at all.
+    if [ -d /etc/pki/tls ]; then
         PKI_HOST_MOUNT_ARGS="-v /etc/pki:/etc/pki:ro"
     elif [ -d /etc/ssl ]; then
         PKI_HOST_MOUNT_ARGS="-v /etc/ssl:/etc/ssl:ro"
@@ -85,6 +87,8 @@ podman run -it --rm --pull=newer \
     -e EXTRA_HELM_OPTS \
     -e EXTRA_PLAYBOOK_OPTS \
     -e TARGET_ORIGIN \
+    -e TARGET_SITE \
+    -e TARGET_BRANCH \
     -e NAME \
     -e TOKEN_SECRET \
     -e TOKEN_NAMESPACE \

common/scripts/process-secrets.sh

@@ -14,4 +14,6 @@ PATTERNPATH=$(dirname "${COMMONPATH}")
 PATTERN_NAME=${1:-$(basename "`pwd`")}
 SECRETS_BACKING_STORE="$($SCRIPTPATH/determine-secretstore-backend.sh)"
 
-ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" "rhvp.cluster_utils.process_secrets"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.process_secrets"

common/scripts/vault-utils.sh

@@ -25,4 +25,6 @@ if [ -z ${TASK} ]; then
 	exit 1
 fi
 
-ansible-playbook -t "${TASK}" -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" "rhvp.cluster_utils.vault"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -t "${TASK}" -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.vault"

common/scripts/write-token-kubeconfig.sh

@@ -13,4 +13,6 @@ SCRIPTPATH=$(dirname "${SCRIPT}")
 COMMONPATH=$(dirname "${SCRIPTPATH}")
 PATTERNPATH=$(dirname "${COMMONPATH}")
 
-ansible-playbook -e pattern_dir="${PATTERNPATH}" -e kubeconfig_file="${OUTPUTFILE}" "rhvp.cluster_utils.write-token-kubeconfig"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_dir="${PATTERNPATH}" -e kubeconfig_file="${OUTPUTFILE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.write-token-kubeconfig"