diff --git a/common/.github/workflows/pattern-sh-ci.yml b/common/.github/workflows/pattern-sh-ci.yml new file mode 100644 index 00000000..ed0e6a07 --- /dev/null +++ b/common/.github/workflows/pattern-sh-ci.yml @@ -0,0 +1,48 @@ +name: Run Bash Script on Multiple Distributions + +on: + push: + paths: + - "scripts/**" + - "Makefile" + branches: + - main + pull_request: + paths: + - "scripts/**" + - "Makefile" + +jobs: + run-script: + name: Run Bash Script + strategy: + matrix: + # Fedora is not an option yet + os: [ubuntu-latest, ubuntu-22.04] + runs-on: ${{ matrix.os }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install Podman on Ubuntu + if: contains(matrix.os, 'ubuntu') + run: | + sudo apt-get update + sudo apt-get install -y podman + + # Currently we do not do MacOSX as it is not free, maybe in the future + # - name: Install Podman on macOS + # if: contains(matrix.os, 'macos') + # run: | + # brew install podman + # podman machine init + # podman machine start + + - name: Verify Podman Installation + run: podman --version + + - name: Run pattern.sh script + run: | + export TARGET_BRANCH=main + ./scripts/pattern-util.sh make validate-origin diff --git a/common/Makefile b/common/Makefile index a5c102b8..84f6afc4 100644 --- a/common/Makefile +++ b/common/Makefile @@ -8,6 +8,10 @@ endif # the command line. I.e. we can set things without having to tweak values files EXTRA_HELM_OPTS ?= +# This variable can be set in order to pass additional ansible-playbook arguments from the +# the command line. I.e. we can set -vvv for more verbose logging +EXTRA_PLAYBOOK_OPTS ?= + # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248 # or # INDEX_IMAGES=registry-proxy.engineering.redhat.com/rh-osbs/iib:394248,registry-proxy.engineering.redhat.com/rh-osbs/iib:394249 @@ -18,7 +22,7 @@ TARGET_ORIGIN ?= origin # This is because we expect to use tokens for repo authentication as opposed to SSH keys TARGET_REPO=$(shell git ls-remote --get-url --symref $(TARGET_ORIGIN) | sed -e 's/.*URL:[[:space:]]*//' -e 's%^git@%%' -e 's%^https://%%' -e 's%:%/%' -e 's%^%https://%') # git branch --show-current is also available as of git 2.22, but we will use this for compatibility -TARGET_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) +TARGET_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) UUID_FILE ?= ~/.config/validated-patterns/pattern-uuid UUID_HELM_OPTS ?= @@ -68,11 +72,7 @@ preview-%: .PHONY: operator-deploy operator-deploy operator-upgrade: validate-prereq validate-origin validate-cluster ## runs helm install - @set -e -o pipefail - # Retry five times because the CRD might not be fully installed yet - for i in {1..5}; do \ - helm template --include-crds --name-template $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) | oc apply -f- && break || sleep 10; \ - done + @common/scripts/deploy-pattern.sh $(NAME) $(PATTERN_INSTALL_CHART) $(HELM_OPTS) .PHONY: uninstall uninstall: ## runs helm uninstall @@ -115,7 +115,7 @@ secrets-backend-none: ## Edits values files to remove secrets manager + ESO .PHONY: load-iib load-iib: ## CI target to install Index Image Bundles @set -e; if [ x$(INDEX_IMAGES) != x ]; then \ - ansible-playbook rhvp.cluster_utils.iib_ci; \ + ansible-playbook $(EXTRA_PLAYBOOK_OPTS) rhvp.cluster_utils.iib_ci; \ else \ echo "No INDEX_IMAGES defined. Bailing out"; \ exit 1; \ @@ -129,12 +129,22 @@ token-kubeconfig: ## Create a local ~/.kube/config with password (not usually ne # We only check the remote ssh git branch's existance if we're not running inside a container # as getting ssh auth working inside a container seems a bit brittle +# If the main repoUpstreamURL field is set, then we need to check against +# that and not target_repo .PHONY: validate-origin validate-origin: ## verify the git origin is available @echo "Checking repository:" - @echo -n " $(TARGET_REPO) - branch '$(TARGET_BRANCH)': " - @git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\ - echo "OK" || (echo "NOT FOUND"; exit 1) + $(eval UPSTREAMURL := $(shell yq -r '.main.git.repoUpstreamURL // (.main.git.repoUpstreamURL = "")' values-global.yaml)) + @if [ -z "$(UPSTREAMURL)" ]; then\ + echo -n " $(TARGET_REPO) - branch '$(TARGET_BRANCH)': ";\ + git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\ + echo "OK" || (echo "NOT FOUND"; exit 1);\ + else\ + echo "Upstream URL set to: $(UPSTREAMURL)";\ + echo -n " $(UPSTREAMURL) - branch '$(TARGET_BRANCH)': ";\ + git ls-remote --exit-code --heads $(UPSTREAMURL) $(TARGET_BRANCH) >/dev/null &&\ + echo "OK" || (echo "NOT FOUND"; exit 1);\ + fi .PHONY: validate-cluster validate-cluster: ## Do some cluster validations before installing @@ -153,15 +163,20 @@ validate-cluster: ## Do some cluster validations before installing validate-schema: ## validates values files against schema in common/clustergroup $(eval VAL_PARAMS := $(shell for i in ./values-*.yaml; do echo -n "$${i} "; done)) @echo -n "Validating clustergroup schema of: " - @set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template common/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done + @set -e; for i in $(VAL_PARAMS); do echo -n " $$i"; helm template oci://quay.io/hybridcloudpatterns/clustergroup $(HELM_OPTS) -f "$${i}" >/dev/null; done @echo .PHONY: validate-prereq validate-prereq: ## verify pre-requisites + $(eval GLOBAL_PATTERN := $(shell yq -r .global.pattern values-global.yaml)) + @if [ $(NAME) != $(GLOBAL_PATTERN) ]; then\ + echo "";\ + echo "WARNING: folder directory is \"$(NAME)\" and global.pattern is set to \"$(GLOBAL_PATTERN)\"";\ + echo "this can create problems. Please make sure they are the same!";\ + echo "";\ + fi @if [ ! -f /run/.containerenv ]; then\ echo "Checking prerequisites:";\ - for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\ - echo " Check for '$(EXECUTABLES)': OK";\ echo -n " Check for python-kubernetes: ";\ if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\ echo "OK";\ @@ -182,16 +197,16 @@ validate-prereq: ## verify pre-requisites .PHONY: argo-healthcheck argo-healthcheck: ## Checks if all argo applications are synced @echo "Checking argo applications" - $(eval APPS := $(shell oc get applications -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}')) + $(eval APPS := $(shell oc get applications.argoproj.io -A -o jsonpath='{range .items[*]}{@.metadata.namespace}{","}{@.metadata.name}{"\n"}{end}')) @NOTOK=0; \ for i in $(APPS); do\ n=`echo "$${i}" | cut -f1 -d,`;\ a=`echo "$${i}" | cut -f2 -d,`;\ - STATUS=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.sync.status}'`;\ + STATUS=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.sync.status}'`;\ if [[ $$STATUS != "Synced" ]]; then\ NOTOK=$$(( $${NOTOK} + 1));\ fi;\ - HEALTH=`oc get -n "$${n}" application/"$${a}" -o jsonpath='{.status.health.status}'`;\ + HEALTH=`oc get -n "$${n}" applications.argoproj.io/"$${a}" -o jsonpath='{.status.health.status}'`;\ if [[ $$HEALTH != "Healthy" ]]; then\ NOTOK=$$(( $${NOTOK} + 1));\ fi;\ @@ -208,7 +223,7 @@ argo-healthcheck: ## Checks if all argo applications are synced .PHONY: qe-tests qe-tests: ## Runs the tests that QE runs @set -e; if [ -f ./tests/interop/run_tests.sh ]; then \ - ./tests/interop/run_tests.sh; \ + pushd ./tests/interop; ./run_tests.sh; popd; \ else \ echo "No ./tests/interop/run_tests.sh found skipping"; \ fi diff --git a/common/README.md b/common/README.md index 41223529..b36bc1a4 100644 --- a/common/README.md +++ b/common/README.md @@ -34,16 +34,16 @@ main: ## Start Here This repository is never used as standalone. It is usually imported in each pattern as a subtree. -In order to import the common/ the very first time you can use -`https://github.com/validatedpatterns/multicloud-gitops/blob/main/common/scripts/make_common_subtree.sh` +In order to import the common subtree the very first time you can use the script +[make_common_subtree.sh](scripts/make-common-subtree.sh). In order to update your common subtree inside your pattern repository you can either use `https://github.com/validatedpatterns/utilities/blob/main/scripts/update-common-everywhere.sh` or -do it manually by doing the following: +do it manually with the following commands: ```sh -git remote add -f upstream-common https://github.com/validatedpatterns/common.git -git merge -s subtree -Xtheirs -Xsubtree=common upstream-common/main +git remote add -f common-upstream https://github.com/validatedpatterns/common.git +git merge -s subtree -Xtheirs -Xsubtree=common common-upstream/main ``` ## Secrets diff --git a/common/scripts/deploy-pattern.sh b/common/scripts/deploy-pattern.sh new file mode 100755 index 00000000..61074fe1 --- /dev/null +++ b/common/scripts/deploy-pattern.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -o pipefail + +RUNS=10 +WAIT=15 +# Retry five times because the CRD might not be fully installed yet +echo -n "Installing pattern: " +for i in $(seq 1 ${RUNS}); do \ + exec 3>&1 4>&2 + OUT=$( { helm template --include-crds --name-template $* 2>&4 | oc apply -f- 2>&4 1>&3; } 4>&1 3>&1) + ret=$? + exec 3>&- 4>&- + if [ ${ret} -eq 0 ]; then + break; + else + echo -n "." + sleep "${WAIT}" + fi +done + +# All the runs failed +if [ ${i} -eq ${RUNS} ]; then + echo "Installation failed [${i}/${RUNS}]. Error:" + echo "${OUT}" + exit 1 +fi +echo "Done" diff --git a/common/scripts/display-secrets-info.sh b/common/scripts/display-secrets-info.sh index d9915855..ca0069e2 100755 --- a/common/scripts/display-secrets-info.sh +++ b/common/scripts/display-secrets-info.sh @@ -23,4 +23,6 @@ fi PATTERN_NAME=$(basename "`pwd`") -ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e override_no_log=false "rhvp.cluster_utils.display_secrets_info" +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" -e hide_sensitive_output=false ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.display_secrets_info" diff --git a/common/scripts/load-k8s-secrets.sh b/common/scripts/load-k8s-secrets.sh index 9219f92f..707e51a7 100755 --- a/common/scripts/load-k8s-secrets.sh +++ b/common/scripts/load-k8s-secrets.sh @@ -13,4 +13,6 @@ PATTERNPATH=$(dirname "${COMMONPATH}") PATTERN_NAME=${1:-$(basename "`pwd`")} -ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" "rhvp.cluster_utils.k8s_secrets" +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.k8s_secrets" diff --git a/common/scripts/make_common_subtree.sh b/common/scripts/make-common-subtree.sh similarity index 91% rename from common/scripts/make_common_subtree.sh rename to common/scripts/make-common-subtree.sh index a5e406d8..196a4c8d 100755 --- a/common/scripts/make_common_subtree.sh +++ b/common/scripts/make-common-subtree.sh @@ -7,7 +7,7 @@ if [ "$1" = "-h" ]; then echo "$0 " echo echo "Run without arguments, the script would run as if these arguments had been passed:" - echo "$0 https://github.com/hybrid-cloud-patterns/common.git main common-subtree" + echo "$0 https://github.com/validatedpatterns/common.git main common-upstream" echo echo "Please ensure the git subtree command is available. On RHEL/Fedora, the git subtree command" echo "is in a separate package called git-subtree" @@ -41,7 +41,7 @@ fi if [ "$1" ]; then subtree_repo=$1 else - subtree_repo=https://github.com/hybrid-cloud-patterns/common.git + subtree_repo=https://github.com/validatedpatterns/common.git fi if [ "$2" ]; then @@ -53,7 +53,7 @@ fi if [ "$3" ]; then subtree_remote=$3 else - subtree_remote=common-subtree + subtree_remote=common-upstream fi git diff --quiet || (echo "This script must be run on a clean working tree" && exit 1) diff --git a/common/scripts/pattern-util.sh b/common/scripts/pattern-util.sh index 1fcaee76..8258d469 100755 --- a/common/scripts/pattern-util.sh +++ b/common/scripts/pattern-util.sh @@ -64,8 +64,10 @@ fi # if we are using podman machine then we do not bind mount anything (for now!) REMOTE_PODMAN=$(podman system connection list -q | wc -l) if [ $REMOTE_PODMAN -eq 0 ]; then # If we are not using podman machine we check the hosts folders - # Use /etc/pki by default and try a couple of fallbacks if it does not exist - if [ -d /etc/pki ]; then + # We check /etc/pki/tls because on ubuntu /etc/pki/fwupd sometimes + # exists but not /etc/pki/tls and we do not want to bind mount in such a case + # as it would find no certificates at all. + if [ -d /etc/pki/tls ]; then PKI_HOST_MOUNT_ARGS="-v /etc/pki:/etc/pki:ro" elif [ -d /etc/ssl ]; then PKI_HOST_MOUNT_ARGS="-v /etc/ssl:/etc/ssl:ro" @@ -85,6 +87,8 @@ podman run -it --rm --pull=newer \ -e EXTRA_HELM_OPTS \ -e EXTRA_PLAYBOOK_OPTS \ -e TARGET_ORIGIN \ + -e TARGET_SITE \ + -e TARGET_BRANCH \ -e NAME \ -e TOKEN_SECRET \ -e TOKEN_NAMESPACE \ diff --git a/common/scripts/process-secrets.sh b/common/scripts/process-secrets.sh index 47eff7fa..a0d34f88 100755 --- a/common/scripts/process-secrets.sh +++ b/common/scripts/process-secrets.sh @@ -14,4 +14,6 @@ PATTERNPATH=$(dirname "${COMMONPATH}") PATTERN_NAME=${1:-$(basename "`pwd`")} SECRETS_BACKING_STORE="$($SCRIPTPATH/determine-secretstore-backend.sh)" -ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" "rhvp.cluster_utils.process_secrets" +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" -e secrets_backing_store="${SECRETS_BACKING_STORE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.process_secrets" diff --git a/common/scripts/vault-utils.sh b/common/scripts/vault-utils.sh index b014e5a4..2f766499 100755 --- a/common/scripts/vault-utils.sh +++ b/common/scripts/vault-utils.sh @@ -25,4 +25,6 @@ if [ -z ${TASK} ]; then exit 1 fi -ansible-playbook -t "${TASK}" -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" "rhvp.cluster_utils.vault" +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -t "${TASK}" -e pattern_name="${PATTERN_NAME}" -e pattern_dir="${PATTERNPATH}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.vault" diff --git a/common/scripts/write-token-kubeconfig.sh b/common/scripts/write-token-kubeconfig.sh index 7544fac2..e7913e53 100755 --- a/common/scripts/write-token-kubeconfig.sh +++ b/common/scripts/write-token-kubeconfig.sh @@ -13,4 +13,6 @@ SCRIPTPATH=$(dirname "${SCRIPT}") COMMONPATH=$(dirname "${SCRIPTPATH}") PATTERNPATH=$(dirname "${COMMONPATH}") -ansible-playbook -e pattern_dir="${PATTERNPATH}" -e kubeconfig_file="${OUTPUTFILE}" "rhvp.cluster_utils.write-token-kubeconfig" +EXTRA_PLAYBOOK_OPTS="${EXTRA_PLAYBOOK_OPTS:-}" + +ansible-playbook -e pattern_dir="${PATTERNPATH}" -e kubeconfig_file="${OUTPUTFILE}" ${EXTRA_PLAYBOOK_OPTS} "rhvp.cluster_utils.write-token-kubeconfig"