Merge pull request #43 from dminnear-rh/common-automatic-update

Common automatic update

Author: mbaldessari

common/.github/workflows/pattern-sh-ci.yml

@@ -0,0 +1,48 @@
+name: Run Bash Script on Multiple Distributions
+
+on:
+  push:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+    branches:
+      - main
+  pull_request:
+    paths:
+      - "scripts/**"
+      - "Makefile"
+
+jobs:
+  run-script:
+    name: Run Bash Script
+    strategy:
+      matrix:
+        # Fedora is not an option yet
+        os: [ubuntu-latest, ubuntu-22.04]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install Podman on Ubuntu
+        if: contains(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y podman
+
+      # Currently we do not do MacOSX as it is not free, maybe in the future
+      # - name: Install Podman on macOS
+      #   if: contains(matrix.os, 'macos')
+      #   run: |
+      #     brew install podman
+      #     podman machine init
+      #     podman machine start
+
+      - name: Verify Podman Installation
+        run: podman --version
+
+      - name: Run pattern.sh script
+        run: |
+          export TARGET_BRANCH=main
+          ./scripts/pattern-util.sh make validate-origin

common/Makefile

@@ -8,6 +8,10 @@ endif
 # the command line. I.e. we can set things without having to tweak values files
 EXTRA_HELM_OPTS ?=
 
+# This variable can be set in order to pass additional ansible-playbook arguments from the
+# the command line. I.e. we can set -vvv for more verbose logging
+EXTRA_PLAYBOOK_OPTS ?=
+
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248
 # or
 # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248,registry-proxy.engineering.redhat.com/rh-osbs/iib:394249
@@ -18,7 +22,7 @@ TARGET_ORIGIN ?= origin
 # This is because we expect to use tokens for repo authentication as opposed to SSH keys
 TARGET_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN) | sed -e 's/.*URL:[[:space:]]*//' -e 's%^git@%%' -e 's%^https://%%' -e 's%:%/%' -e 's%^%https://%')
 # git branch --show-current is also available as of git 2.22, but we will use this for compatibility
-TARGET_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
+TARGET_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 UUID_FILE ?= ~/.config/validated-patterns/pattern-uuid
 UUID_HELM_OPTS ?=
@@ -68,11 +72,7 @@ preview-%:
 
 .PHONY: operator-deploy
 operator-deploy operator-upgrade: validate-prereq validate-origin validate-cluster ## runs helm install
-	@set -e -o pipefail
-	# Retry five times because the CRD might not be fully installed yet
-	for i in {1..5}; do \
-		helm template --include-crds --name-template $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) | oc apply -f- && break || sleep 10; \
-	done
+	@common/scripts/deploy-pattern.sh $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS)
 
 .PHONY: uninstall
 uninstall: ## runs helm uninstall
@@ -115,7 +115,7 @@ secrets-backend-none: ## Edits values files to remove secrets manager + ESO
 .PHONY: load-iib
 load-iib: ## CI target to install Index Image Bundles
 	@set -e; if [ x$(INDEX_IMAGES) != x ]; then \
-		ansible-playbook rhvp.cluster_utils.iib-ci; \
+		ansible-playbook $(EXTRA_PLAYBOOK_OPTS) rhvp.cluster_utils.iib_ci; \
 	else \
 		echo "No INDEX_IMAGES defined. Bailing out"; \
 		exit 1; \
@@ -129,12 +129,22 @@ token-kubeconfig: ## Create a local ~/.kube/config with password (not usually ne
 
 # We only check the remote ssh git branch's existance if we're not running inside a container
 # as getting ssh auth working inside a container seems a bit brittle
+# If the main repoUpstreamURL field is set, then we need to check against
+# that and not target_repo
 .PHONY: validate-origin
 validate-origin: ## verify the git origin is available
 	@echo "Checking repository:"
-	@echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': "
-	@git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
-		echo "OK" || (echo "NOT FOUND"; exit 1)
+	$(eval UPSTREAMURL := $(shell yq -r '.main.git.repoUpstreamURL // (.main.git.repoUpstreamURL = "")' values-global.yaml))
+	@if [ -z "$(UPSTREAMURL)" ]; then\
+		echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	else\
+		echo "Upstream URL set to: $(UPSTREAMURL)";\
+		echo -n "  $(UPSTREAMURL) - branch '$(TARGET_BRANCH)': ";\
+		git ls-remote --exit-code --heads $(UPSTREAMURL) $(TARGET_BRANCH) >/dev/null &&\
+			echo "OK" || (echo "NOT FOUND"; exit 1);\
+	fi
 
 .PHONY: validate-cluster
 validate-cluster: ## Do some cluster validations before installing
@@ -143,7 +153,7 @@ validate-cluster: ## Do some cluster validations before installing
 	@oc cluster-info >/dev/null && echo "OK" || (echo "Error"; exit 1)
 	@echo -n "  storageclass: "
 	@if [ `oc get storageclass -o go-template='{{printf "%d\n" (len .items)}}'` -eq 0 ]; then\
-		echo "None Found"; exit 1;\
+		echo "WARNING: No storageclass found";\
 	else\
 		echo "OK";\
 	fi
@@ -153,15 +163,20 @@ validate-cluster: ## Do some cluster validations before installing
 validate-schema: ## validates values files against schema in common/clustergroup
 	$(eval VAL_PARAMS := $(shell for i in ./values-*.yaml; do echo -n "$${i} "; done))
 	@echo -n "Validating clustergroup schema of: "
-	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template common/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
+	@set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template oci://quay.io/hybridcloudpatterns/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done
 	@echo
 
 .PHONY: validate-prereq
 validate-prereq: ## verify pre-requisites
+	$(eval GLOBAL_PATTERN := $(shell yq -r .global.pattern values-global.yaml))
+	@if [ $(NAME) != $(GLOBAL_PATTERN) ]; then\
+		echo "";\
+		echo "WARNING: folder directory is \"$(NAME)\" and global.pattern is set to \"$(GLOBAL_PATTERN)\"";\
+		echo "this can create problems. Please make sure they are the same!";\
+		echo "";\
+	fi
 	@if [ ! -f /run/.containerenv ]; then\
 	  echo "Checking prerequisites:";\
-	  for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\
-	  echo "  Check for '$(EXECUTABLES)': OK";\
 	  echo -n "  Check for python-kubernetes: ";\
 	  if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
 	  echo "OK";\
@@ -182,16 +197,16 @@ validate-prereq: ## verify pre-requisites
 .PHONY: argo-healthcheck
 argo-healthcheck: ## Checks if all argo applications are synced
 	@echo "Checking argo applications"
-	$(eval APPS := $(shell oc get applications -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
+	$(eval APPS := $(shell oc get applications.argoproj.io -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}'))
 	@NOTOK=0; \
 	for i in $(APPS); do\
 		n=`echo "$${i}" | cut -f1 -d,`;\
 		a=`echo "$${i}" | cut -f2 -d,`;\
-		STATUS=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.sync.status}'`;\
+		STATUS=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.sync.status}'`;\
 		if [[ $$STATUS != "Synced" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
-		HEALTH=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.health.status}'`;\
+		HEALTH=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.health.status}'`;\
 		if [[ $$HEALTH != "Healthy" ]]; then\
 			NOTOK=$$(( $${NOTOK} + 1));\
 		fi;\
@@ -205,6 +220,14 @@ argo-healthcheck: ## Checks if all argo applications are synced
 
 ##@ Test and Linters Tasks
 
+.PHONY: qe-tests
+qe-tests: ## Runs the tests that QE runs
+	@set -e; if [ -f ./tests/interop/run_tests.sh ]; then \
+		pushd ./tests/interop; ./run_tests.sh; popd; \
+	else \
+		echo "No ./tests/interop/run_tests.sh found skipping"; \
+	fi
+
 .PHONY: super-linter
 super-linter: ## Runs super linter locally
 	rm -rf .mypy_cache

common/README.md

@@ -34,16 +34,16 @@ main:
 ## Start Here
 
 This repository is never used as standalone. It is usually imported in each pattern as a subtree.
-In order to import the common/ the very first time you can use
-`https://github.com/validatedpatterns/multicloud-gitops/blob/main/common/scripts/make_common_subtree.sh`
+In order to import the common subtree the very first time you can use the script
+[make_common_subtree.sh](scripts/make-common-subtree.sh).
 
 In order to update your common subtree inside your pattern repository you can either use
 `https://github.com/validatedpatterns/utilities/blob/main/scripts/update-common-everywhere.sh` or
-do it manually by doing the following:
+do it manually with the following commands:
 
 ```sh
-git remote add -f upstream-common https://github.com/validatedpatterns/common.git
-git merge -s subtree -Xtheirs -Xsubtree=common upstream-common/main
+git remote add -f common-upstream https://github.com/validatedpatterns/common.git
+git merge -s subtree -Xtheirs -Xsubtree=common common-upstream/main
 ```
 
 ## Secrets

common/scripts/deploy-pattern.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -o pipefail
+
+RUNS=10
+WAIT=15
+# Retry five times because the CRD might not be fully installed yet
+echo -n "Installing pattern: "
+for i in $(seq 1 ${RUNS}); do \
+    exec 3>&1 4>&2
+    OUT=$( { helm template --include-crds --name-template $* 2>&4 | oc apply -f- 2>&4 1>&3; } 4>&1 3>&1)
+    ret=$?
+    exec 3>&- 4>&-
+    if [ ${ret} -eq 0 ]; then
+        break;
+    else
+        echo -n "."
+        sleep "${WAIT}"
+    fi
+done
+
+# All the runs failed
+if [ ${i} -eq ${RUNS} ]; then
+    echo "Installation failed [${i}/${RUNS}]. Error:"
+    echo "${OUT}"
+    exit 1
+fi
+echo "Done"

common/scripts/display-secrets-info.sh

@@ -23,4 +23,6 @@ fi
 
 PATTERN_NAME=$(basename "`pwd`")
 
-ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e override_no_log=false "rhvp.cluster_utils.display_secrets_info"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e hide_sensitive_output=false ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.display_secrets_info"

common/scripts/load-k8s-secrets.sh

@@ -13,4 +13,6 @@ PATTERNPATH=$(dirname "${COMMONPATH}")
 
 PATTERN_NAME=${1:-$(basename "`pwd`")}
 
-ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" "rhvp.cluster_utils.k8s_secrets"
+EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}"
+
+ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.k8s_secrets"

common/scripts/make_common_subtree.sh renamed to common/scripts/make-common-subtree.sh

@@ -7,7 +7,7 @@ if [ "$1" = "-h" ]; then
 	echo "$0 <subtree_repo> <subtree_branch> <subtree_remote_name>"
 	echo
 	echo "Run without arguments, the script would run as if these arguments had been passed:"
-	echo "$0 https://github.com/hybrid-cloud-patterns/common.git main common-subtree"
+	echo "$0 https://github.com/validatedpatterns/common.git main common-upstream"
 	echo
 	echo "Please ensure the git subtree command is available. On RHEL/Fedora, the git subtree command"
 	echo "is in a separate package called git-subtree"
@@ -41,7 +41,7 @@ fi
 if [ "$1" ]; then
 	subtree_repo=$1
 else
-	subtree_repo=https://github.com/hybrid-cloud-patterns/common.git
+	subtree_repo=https://github.com/validatedpatterns/common.git
 fi
 
 if [ "$2" ]; then
@@ -53,7 +53,7 @@ fi
 if [ "$3" ]; then
 	subtree_remote=$3
 else
-	subtree_remote=common-subtree
+	subtree_remote=common-upstream
 fi
 
 git diff --quiet || (echo "This script must be run on a clean working tree" && exit 1)

common/scripts/manage-secret-app.sh

@@ -11,13 +11,17 @@ case "$APP" in
         APP_NAME="vault"
         NAMESPACE="vault"
         PROJECT="$MAIN_CLUSTERGROUP_PROJECT"
-        CHART_LOCATION="common/hashicorp-vault"
+        CHART_NAME="hashicorp-vault"
+        CHART_VERSION=0.1.*
+
     ;;
     "golang-external-secrets")
         APP_NAME="golang-external-secrets"
         NAMESPACE="golang-external-secrets"
         PROJECT="$MAIN_CLUSTERGROUP_PROJECT"
-        CHART_LOCATION="common/golang-external-secrets"
+        CHART_NAME="golang-external-secrets"
+        CHART_VERSION=0.1.*
+
     ;;
     *)
         echo "Error - cannot manage $APP can only manage vault and golang-external-secrets"
@@ -32,13 +36,13 @@ case "$STATE" in
         RES=$(yq ".clusterGroup.applications[] | select(.path == \"$CHART_LOCATION\")" "$MAIN_CLUSTERGROUP_FILE" 2>/dev/null)
         if [ -z "$RES" ]; then
             echo "Application with chart location $CHART_LOCATION not found, adding"
-            yq -i ".clusterGroup.applications.$APP_NAME = { \"name\": \"$APP_NAME\", \"namespace\": \"$NAMESPACE\", \"project\": \"$PROJECT\", \"path\": \"$CHART_LOCATION\" }" "$MAIN_CLUSTERGROUP_FILE"
+            yq -i ".clusterGroup.applications.$APP_NAME = { \"name\": \"$APP_NAME\", \"namespace\": \"$NAMESPACE\", \"project\": \"$PROJECT\", \"chart\": \"$CHART_NAME\",  \"chartVersion\": \"$CHART_VERSION\"}" "$MAIN_CLUSTERGROUP_FILE"
         fi
     ;;
     "absent")
         common/scripts/manage-secret-namespace.sh "$NAMESPACE" "$STATE"
         echo "Removing application wth chart location $CHART_LOCATION"
-        yq -i "del(.clusterGroup.applications[] | select(.path == \"$CHART_LOCATION\"))" "$MAIN_CLUSTERGROUP_FILE"
+        yq -i "del(.clusterGroup.applications[] | select(.chart == \"$CHART_NAME\"))" "$MAIN_CLUSTERGROUP_FILE"
     ;;
     *)
         echo "$STATE not supported"

common/scripts/pattern-util.sh

@@ -11,6 +11,16 @@ function version {
 if [ -z "$PATTERN_UTILITY_CONTAINER" ]; then
 	PATTERN_UTILITY_CONTAINER="quay.io/hybridcloudpatterns/utility-container"
 fi
+# If PATTERN_DISCONNECTED_HOME is set it will be used to populate both PATTERN_UTILITY_CONTAINER
+# and PATTERN_INSTALL_CHART automatically
+if [ -n "${PATTERN_DISCONNECTED_HOME}" ]; then
+    PATTERN_UTILITY_CONTAINER="${PATTERN_DISCONNECTED_HOME}/utility-container"
+    PATTERN_INSTALL_CHART="oci://${PATTERN_DISCONNECTED_HOME}/pattern-install"
+    echo "PATTERN_DISCONNECTED_HOME is set to ${PATTERN_DISCONNECTED_HOME}"
+    echo "Setting the following variables:"
+    echo "  PATTERN_UTILITY_CONTAINER: ${PATTERN_UTILITY_CONTAINER}"
+    echo "  PATTERN_INSTALL_CHART: ${PATTERN_INSTALL_CHART}"
+fi
 
 readonly commands=(podman)
 for cmd in ${commands[@]}; do is_available "$cmd"; done
@@ -54,8 +64,10 @@ fi
 # if we are using podman machine then we do not bind mount anything (for now!)
 REMOTE_PODMAN=$(podman system connection list -q | wc -l)
 if [ $REMOTE_PODMAN -eq 0 ]; then # If we are not using podman machine we check the hosts folders
-    # Use /etc/pki by default and try a couple of fallbacks if it does not exist
-    if [ -d /etc/pki ]; then
+    # We check /etc/pki/tls because on ubuntu /etc/pki/fwupd sometimes
+    # exists but not /etc/pki/tls and we do not want to bind mount in such a case
+    # as it would find no certificates at all.
+    if [ -d /etc/pki/tls ]; then
         PKI_HOST_MOUNT_ARGS="-v /etc/pki:/etc/pki:ro"
     elif [ -d /etc/ssl ]; then
         PKI_HOST_MOUNT_ARGS="-v /etc/ssl:/etc/ssl:ro"
@@ -75,11 +87,15 @@ podman run -it --rm --pull=newer \
     -e EXTRA_HELM_OPTS \
     -e EXTRA_PLAYBOOK_OPTS \
     -e TARGET_ORIGIN \
+    -e TARGET_SITE \
+    -e TARGET_BRANCH \
     -e NAME \
     -e TOKEN_SECRET \
     -e TOKEN_NAMESPACE \
     -e VALUES_SECRET \
     -e KUBECONFIG \
+    -e PATTERN_INSTALL_CHART \
+    -e PATTERN_DISCONNECTED_HOME \
     -e K8S_AUTH_HOST \
     -e K8S_AUTH_VERIFY_SSL \
     -e K8S_AUTH_SSL_CA_CERT \

common/scripts/preview.sh

@@ -2,8 +2,7 @@
 
 # DISCLAIMER
 # 
-# - Parsing of applications needs to be more clever. Currently the code assumes that all 
-# targets will be local charts. This is not true, for example, in industrial-edge.
+# - Parsing of applications needs to be more clever.
 # - There is currently not a mechanism to actually preview against multiple clusters 
 # (i.e. a hub and a remote). All previews will be done against the current.
 # - Make output can be included in the YAML.
@@ -22,11 +21,22 @@ if [ "${APPNAME}" != "clustergroup" ]; then
   #   path: charts/all/foo
   # So we retrieve the actual index ("foobar") given the name attribute of the application
   APP=$(yq ".clusterGroup.applications | with_entries(select(.value.name == \"$APPNAME\")) | keys | .[0]" values-$SITE.yaml)
-  chart=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml)
+  isLocalHelmChart=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml)
+  if [ $isLocalHelmChart != "null" ]; then
+    chart=$(yq ".clusterGroup.applications.$APP.path" values-$SITE.yaml)
+  else
+    helmrepo=$(yq ".clusterGroup.applications.$APP.repoURL" values-$SITE.yaml)
+    helmrepo="${helmrepo:+oci://quay.io/hybridcloudpatterns}"
+    chartversion=$(yq ".clusterGroup.applications.$APP.chartVersion" values-$SITE.yaml)
+    chartname=$(yq ".clusterGroup.applications.$APP.chart" values-$SITE.yaml)
+    chart="${helmrepo}/${chartname} --version ${chartversion}"
+  fi
   namespace=$(yq ".clusterGroup.applications.$APP.namespace" values-$SITE.yaml)
 else
   APP=$APPNAME
-  chart="common/clustergroup"
+  clusterGroupChartVersion=$(yq ".main.multiSourceConfig.clusterGroupChartVersion" values-global.yaml)
+  helmrepo="oci://quay.io/hybridcloudpatterns"
+  chart="${helmrepo}/clustergroup --version ${clusterGroupChartVersion}"
   namespace="openshift-operators"
 fi
 pattern=$(yq ".global.pattern" values-global.yaml)