Makefile.maintainer

# pull image by default
CITOOL_EXTRA_PODMAN_ARGS ?= --pull newer

# default worker image
WORKER_IMAGE ?= quay.io/testing-farm/worker-public:latest

# skip nomadstart if asked
SKIP_NOMAD_START := $(if $(SKIP_NOMAD_START),-e skip_nomad_start=true,)

# default development cluster name
DEV_CLUSTER_NAME ?= $(or $(TF_VAR_cluster_name),testing-farm-dev-$$USER)

# run pytest tests in parallel
PYTEST_PARALLEL_RUNS ?= 16
PYTEST_PARALLEL_OPTIONS ?= -d --tx $(PYTEST_PARALLEL_RUNS)*popen//python=python3.12

# default Terraform lock timeout
DEFAULT_LOCK_TIMEOUT ?= 10m

# Testing Farm production API
TESTING_FARM_PRODUCTION_API_URL ?= $(TESTING_FARM_API_URL)
TESTING_FARM_PRODUCTION_API_TOKEN_PUBLIC ?= $(TESTING_FARM_API_TOKEN_PUBLIC)

# Staging CI exclude directories
STAGING_CI_EXCLUDE_DIR = $(PROJECT_ROOT)/terragrunt/environments/staging/ci/**/*

# For scale-redhat-worker target, make all additional targets parameters
# https://stackoverflow.com/questions/2214575/passing-arguments-to-make-run
ifneq (,$(filter $(firstword $(MAKECMDGOALS)),\
	create/public/worker \
	create/redhat/worker \
	dev/create/public/worker \
	staging/create/public/worker \
        production/public/request/shell \
        production/redhat/request/shell))
  # use the rest as arguments for "SCALE_TARGET"
  SCALE_TARGET := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
  SCALE_TARGET := $(if $(SCALE_TARGET),$(SCALE_TARGET),1)
  # sanitize, eval breaks if it is an URL for example
  SCALE_TARGET := $(patsubst https://%,%,$(SCALE_TARGET))
  SCALE_TARGET := $(patsubst http://%,%,$(SCALE_TARGET))
  # ...and turn them into do-nothing targets
  $(eval $(SCALE_TARGET):;@:)
endif

# output helpers
ccred=$(shell tput setaf 1)
ccgreen=$(shell tput setaf 2)
ccend=$(shell tput sgr0)
ccbold=$(shell tput bold)

# Help prelude
define PRELUDE
Testing Farm Infrastructure

$(ccbold)Usage$(ccend)

  [variables] make [target] [parameter]

$(ccbold)Environment Variables$(ccend)

  ⚙️  $(ccgreen)test/*$(ccend) targets

     WORKER_IMAGE                 ... Adjust worker image when running tests [$(WORKER_IMAGE)]

  ⚙️  $(ccgreen)create/*/worker$(ccend) targets

     SKIP_NOMAD_START             ... Skip registering the worker to Nomad
     SKIP_MAIN_CHECK              ... Skip the main branch check for production targets (dangerous)

  ⚙️  $(ccgreen)staging/*/ci$(ccend) targets

     STAGING_CI_SUFFIX            ... Suffix added to staging subdomains, $(ccbold)required$(ccend)
     STAGING_CI_API_TAG           ... Override API image tag [latest]
     STAGING_CI_ARTEMIS_TAG       ... Override Artemis image tag [latest]
     STAGING_CI_DISPATCHER_TAG    ... Override Dispatcher image tag [latest]
     STAGING_CI_RANCH_PUBLIC_TAG  ... Override Public Ranch Configuration image tag [latest]
     STAGING_CI_UI_TAG            ... Override UI image tag [latest]

endef


##@ Infrastructure | Infra (Gitlab, etc.)

define run_terragrunt
	TERRAGRUNT_WORKING_DIR=terragrunt/environments/$1 terragrunt run-all $2 -lock-timeout $(DEFAULT_LOCK_TIMEOUT)
endef

define run_terragrunt_single
	TERRAGRUNT_WORKING_DIR=terragrunt/environments/$1 terragrunt $2 -auto-approve -lock-timeout $(DEFAULT_LOCK_TIMEOUT)
endef

infra/init: .FORCE  ## Initialize | infra
	$(call run_terragrunt,infra,init)

infra/plan: .FORCE  ## Plan deployment | infra
	$(call run_terragrunt,infra,plan)

infra/plan/eks: .FORCE  ## Plan deployment | infra | eks
	$(call run_terragrunt,infra/eks,plan)

infra/apply: .FORCE  ## Deploy | infra
	$(call run_terragrunt,infra,apply)

infra/apply/eks: .FORCE  ## Deploy | infra | eks
	$(call run_terragrunt,infra/eks,apply)

infra/destroy: .FORCE  ## Destroy | infra
	$(call run_terragrunt,infra,destroy)

infra/kubeconfig: .FORCE  ## Update kubeconfig for infra environment
	aws eks --profile fedora_us_east_2 update-kubeconfig --name testing-farm-infra


##@ Infrastructure | Dev

dev/generate: .FORCE  ## Generate deployment files | dev
	ansible-playbook terragrunt/environments/dev/generate.yaml
	env -C terragrunt/environments/dev/server/ec2 butane -psd $(PROJECT_ROOT) -o server.ign.json server.bu
	jq -c < $$PROJECT_ROOT/terragrunt/environments/dev/server/ec2/server.ign.json > $$PROJECT_ROOT/terragrunt/environments/dev/server/ec2/server.ign

dev/init: .FORCE  ## Initialize | dev | all
	$(call run_terragrunt,dev,init)

dev/plan: dev/generate .FORCE  ## Plan deployment | dev | all
	$(call run_terragrunt,dev,plan)

dev/plan/eks: .FORCE  ## Plan deployment | dev | eks
	$(call run_terragrunt,dev/eks,plan)

dev/plan/artemis: .FORCE  ## Plan deployment | dev | artemis
	$(call run_terragrunt,dev/artemis,plan)

dev/plan/server: dev/generate .FORCE  ## Plan deployment | dev | server
	$(call run_terragrunt,dev/server,plan)

dev/apply: dev/generate .FORCE  ## Deploy | dev | all
	# workaround for terragrunt bug having issues with dependencies
	$(call run_terragrunt,dev/server/security-group,apply)
	$(call run_terragrunt,dev,apply)

dev/apply/eks: .FORCE  ## Deploy | dev | eks
	$(call run_terragrunt,dev/eks,apply)

dev/apply/artemis: .FORCE  ## Deploy | dev | artemis
	$(call run_terragrunt,dev/artemis,apply)

dev/apply/server: dev/generate .FORCE  ## Deploy | dev | server
	$(call run_terragrunt,dev/server,apply)

dev/destroy: .FORCE terminate/artemis/guests/dev dev/destroy/workers  ## Destroy | dev | all
	$(call run_terragrunt,dev,destroy)

dev/destroy/force: .FORCE  ## Force destroy od dev environment, set USER env var to force a specific user
	# NOTE: this destroy method is special to mitigate issues with frequently changing dev environment
	TERRAGRUNT_WORKING_DIR=terragrunt/environments/dev terragrunt run-all init -lock-timeout $(DEFAULT_LOCK_TIMEOUT) || true
	TERRAGRUNT_WORKING_DIR=terragrunt/environments/dev terragrunt run-all init -reconfigure -lock-timeout $(DEFAULT_LOCK_TIMEOUT) || true

	# Remove Testing Farm workers
	make dev/destroy/workers || true

	# Terminate leftover Artemis guests
	make terminate/artemis/guests/dev || true

	# Force destroy all components
	find terragrunt/environments/dev/ -iname terragrunt.hcl | grep -v \\.terragrunt-cache | sed 's/terragrunt.hcl//' | \
		parallel -I{} env -C {} terragrunt destroy -auto-approve || true

	# Remove security groups (revoke cross-references first, then delete)
	sg_ids=$$(aws --profile fedora_us_east_2 ec2 describe-security-groups | \
		jq -r ".SecurityGroups[] | select(.GroupName | contains(\"dev-$$USER\")) | .GroupId"); \
	for sg_id in $$sg_ids; do \
		aws --profile fedora_us_east_2 ec2 revoke-security-group-ingress \
			--group-id $$sg_id --ip-permissions "$$(aws --profile fedora_us_east_2 ec2 describe-security-groups \
			--group-ids $$sg_id --query 'SecurityGroups[0].IpPermissions' --output json)" 2>/dev/null || true; \
	done; \
	for sg_id in $$sg_ids; do \
		aws --profile fedora_us_east_2 ec2 delete-security-group --group-id $$sg_id || true; \
	done

	# Remove leftover DNS records
	aws --profile fedora_us_east_2 \
		route53 list-resource-record-sets --hosted-zone-id Z0711647188EWQGM661TT \
			--query "ResourceRecordSets[?contains(Name, 'dev-$$USER')]" --output json | \
		jq -c '.[]' | while read -r record; do \
			aws --profile fedora_us_east_2 route53 change-resource-record-sets --hosted-zone-id Z0711647188EWQGM661TT \
				--change-batch "$$(jq -n --argjson rec "$$record" '{Changes: [{Action: "DELETE", ResourceRecordSet: $$rec}]}')"; \
		done

	# Remove terraform cloud workspaces
	terraform-cloud list-workspaces | grep dev-$$USER | parallel -n1 terraform-cloud delete-workspace --confirm || true

dev/destroy/force/all: .FORCE  ## Force destroy all dev environments for all users
	@echo "$(ccgreen)Finding all dev server users...$(ccend)"
	@users=$$(aws --profile fedora_us_east_2 ec2 describe-instances \
		--filters "Name=tag:ServicePhase,Values=Dev" "Name=tag:ServiceComponent,Values=Server" "Name=instance-state-name,Values=running" \
		--query 'Reservations[].Instances[].Tags[?Key==`Name`].Value[]' --output text | \
		sed 's/testing_farm_dev_server_//g' | tr '\t' '\n' | sort -u); \
	if [ -z "$$users" ]; then \
		echo "No dev environments found."; \
	else \
		echo "Found dev environments for users: $$users"; \
		for user in $$users; do \
			echo "$(ccgreen)Destroying dev environment for user: $$user$(ccend)"; \
			USER=$$user make dev/destroy/force || true; \
		done; \
	fi

dev/destroy/workers: .FORCE  ## Destroy | dev | all
	$(call run_terragrunt,dev/worker-public,apply)
	worker_instance_ids=$$( \
			env -C $(PROJECT_ROOT)/terragrunt/environments/dev/worker-public \
			terragrunt output -json | jq -r '.workers_instance_ids.value | join(" ")' \
	); \
	if [ -n "$$worker_instance_ids" ]; then \
		echo "Worker instance ids: $$worker_instance_ids"; \
		aws --profile fedora_us_east_2 ec2 terminate-instances --instance-ids $$worker_instance_ids; \
	fi

dev/kubeconfig: .FORCE  ## Update kubeconfig for development environemnt
	aws eks --profile fedora_us_east_2 update-kubeconfig --name $(DEV_CLUSTER_NAME)

dev/server/connect: .FORCE  ## Connect to dev server
	ssh -o IdentitiesOnly=yes -i $$PROJECT_ROOT/ansible/secrets/ssh/id_rsa_public_server.decrypted \
		core@$$(env -C terragrunt/environments/dev/server/ec2 terragrunt output -json | jq -r .public_ip.value)

.dev/create/public/worker%:
	ansible-playbook -e deployment=dev ${SKIP_NOMAD_START} -v ansible/playbooks/create-public-worker.yaml

dev/create/public/worker: generate/dev/citool-config/public .FORCE  ## Create a Public ranch worker, use `make create/public/worker N` to create N workers
	@echo -e "🚀 \033[0;32mcreating $(SCALE_TARGET) dev public worker(s)\033[0m"
	@$(MAKE) -j $(addprefix .dev/create/public/worker, $(shell seq 1 $(SCALE_TARGET)))

dev/update/public/pipeline: .FORCE  ## Update pipeline container image on Public ranch | dev
	$(call run_pipeline_update,💿️,updating Public ranch pipeline image,testing_farm_dev_public_workers,update_image,dev)
	$(call run_pipeline_update,🛠,updating Public ranch pipeline configuration,testing_farm_dev_public_workers,update_config,dev)
	$(call run_pipeline_update,⛴️,updating Public ranch pipeline jobs,testing_farm_dev_public_workers,update_jobs,dev)

dev/update/public/pipeline/image: .FORCE  ## Update pipeline container image on Public ranch | dev
	$(call run_pipeline_update,💿️,updating Public ranch pipeline image,testing_farm_dev_public_workers,update_image,dev)

dev/update/public/pipeline/config: generate/dev/citool-config/public .FORCE  ## Update pipeline configuration on Public ranch | dev
	$(call run_pipeline_update,🛠,updating Public ranch pipeline configuration,testing_farm_dev_public_workers,update_config,dev)

dev/update/public/pipeline/jobs: .FORCE  ## Update pipeline jobs on Public ranch | dev
	$(call run_pipeline_update,⛴️,updating Public ranch pipeline jobs,testing_farm_dev_public_workers,update_jobs,dev)


##@ Infrastructure | Staging

define check_staging_ci_env
	@if [ -z "$$STAGING_CI_SUFFIX" ]; then \
		echo -e "⛔️ \033[0;31mEnvironment variable 'STAGING_CI_SUFFIX' is required.\033[0m"; \
		exit 1; \
	fi
endef

staging/generate: .FORCE  ## Generate deployment files | staging
	ansible-playbook terragrunt/environments/staging/generate.yaml
	env -C $$PROJECT_ROOT/terragrunt/environments/staging/server/ec2 butane -psd $(PROJECT_ROOT) -o server.ign.json server.bu
	# Compact the json, because AWS user data is limited to 16k characters
	jq -c < $$PROJECT_ROOT/terragrunt/environments/staging/server/ec2/server.ign.json > $$PROJECT_ROOT/terragrunt/environments/staging/server/ec2/server.ign
	rm -f $$PROJECT_ROOT/terragrunt/environments/staging/server/ec2/server.ign.json

staging/generate/ci: .FORCE  ## Generate deployment files | staging
	$(call check_staging_ci_env)
	ansible-playbook terragrunt/environments/staging/generate-ci.yaml
	env -C $$PROJECT_ROOT/terragrunt/environments/staging/ci/server/ec2 butane -psd $(PROJECT_ROOT) -o server.ign.json server.bu
	# Compact the json, because AWS user data is limited to 16k characters
	jq -c < $$PROJECT_ROOT/terragrunt/environments/staging/ci/server/ec2/server.ign.json > $$PROJECT_ROOT/terragrunt/environments/staging/ci/server/ec2/server.ign
	rm -f $$PROJECT_ROOT/terragrunt/environments/staging/ci/server/ec2/server.ign.json

staging/init: .FORCE  ## Initialize | staging | all
	TERRAGRUNT_EXCLUDE_DIR=$(STAGING_CI_EXCLUDE_DIR) \
		$(call run_terragrunt,staging,init)

staging/init/reconfigure: .FORCE  ## Initialize and reconfigure | staging | all
	TERRAGRUNT_EXCLUDE_DIR=$(STAGING_CI_EXCLUDE_DIR) \
		$(call run_terragrunt,staging,init -reconfigure)

staging/init/ci: .FORCE  ## Initialize | staging | artemis | CI
	$(call run_terragrunt,staging/ci,init)

staging/plan: staging/generate .FORCE  ## Plan deployment | staging
	TERRAGRUNT_EXCLUDE_DIR=$(STAGING_CI_EXCLUDE_DIR) \
		$(call run_terragrunt,staging,plan)

staging/plan/eks: .FORCE  ## Plan deployment | staging | eks
	$(call run_terragrunt,staging/eks,plan)

staging/plan/artemis: .FORCE  ## Plan deployment | staging | artemis
	$(call run_terragrunt,staging/artemis,plan)

staging/plan/server: staging/generate .FORCE  ## Plan deployment | staging | server
	$(call run_terragrunt,staging/server,plan)

staging/plan/ci: staging/generate/ci .FORCE  ## Plan deployment | staging | artemis | CI
	$(call check_staging_ci_env)
	$(call run_terragrunt,staging/ci,plan)

staging/apply: staging/generate .FORCE  ## Deploy | staging | all
	TERRAGRUNT_EXCLUDE_DIR=$(STAGING_CI_EXCLUDE_DIR) \
		$(call run_terragrunt,staging,apply)

staging/apply/eks: .FORCE  ## Deploy | staging | eks
	$(call run_terragrunt,staging/eks,apply)

staging/apply/artemis: .FORCE  ## Deploy | staging | artemis
	$(call run_terragrunt,staging/artemis,apply)

staging/apply/ci: staging/generate/ci .FORCE  ## Deploy | staging | artemis | CI
	$(call check_staging_ci_env)
	$(call run_terragrunt,staging/ci,apply)

staging/apply/server: staging/generate .FORCE  ## Deploy | staging | server
	$(call run_terragrunt,staging/server,apply)

staging/destroy: .FORCE terminate/artemis/guests/staging  ## Destroy | staging
	make staging/destroy/workers || true
	TERRAGRUNT_EXCLUDE_DIR=$(STAGING_CI_EXCLUDE_DIR) \
		$(call run_terragrunt,staging,destroy)

staging/destroy/ci: .FORCE terminate/artemis/guests/staging/ci  ## Destroy | staging | CI
	$(call check_staging_ci_env)
	# NOTE: do not use run-all here, as it would also destroy the EKS cluster
	$(call run_terragrunt_single,staging/ci/artemis,destroy)
	$(call run_terragrunt_single,staging/ci/localhost,destroy)
	$(call run_terragrunt_single,staging/ci/worker,destroy)
	$(call run_terragrunt,staging/ci/server,destroy)

staging/destroy/workers: .FORCE  ## Destroy | staging | all
	$(call run_terragrunt,staging/worker-public,apply)
	worker_instance_ids=$$( \
			env -C $(PROJECT_ROOT)/terragrunt/environments/staging/worker-public \
			terragrunt output -json | jq -r '.workers_instance_ids.value | join(" ")' \
	); \
	if [ -n "$$worker_instance_ids" ]; then \
		echo "Worker instance ids: $$worker_instance_ids"; \
		aws --profile fedora_us_east_2 ec2 terminate-instances --instance-ids $$worker_instance_ids; \
	fi

staging/kubeconfig: .FORCE  ## Update kubeconfig for staging environment
	aws eks --profile fedora_us_east_1 update-kubeconfig --name testing-farm-staging

staging/server/connect: .FORCE  ## Connect to staging server
	ssh -o IdentitiesOnly=yes -i $$PROJECT_ROOT/ansible/secrets/ssh/id_rsa_public_server.decrypted \
		core@$$(env -C terragrunt/environments/staging/server/ec2 terragrunt output -json | jq -r .public_ip.value)

staging/server/connect/ci: .FORCE  ## Connect to staging server
	$(call check_staging_ci_env)
	ssh -o IdentitiesOnly=yes -i $$PROJECT_ROOT/ansible/secrets/ssh/id_rsa_public_server.decrypted \
		core@$$(env -C terragrunt/environments/staging/ci/server/ec2 terragrunt output -json | jq -r .public_ip.value)

.staging/create/public/worker%:
	ansible-playbook -e deployment=staging ${SKIP_NOMAD_START} -v ansible/playbooks/create-public-worker.yaml

staging/create/public/worker: generate/staging/citool-config .FORCE  ## Create a Public ranch worker, use `make create/public/worker N` to create N workers
	@echo -e "🚀 \033[0;32mcreating $(SCALE_TARGET) staging public worker(s)\033[0m"
	@$(MAKE) -j $(addprefix .staging/create/public/worker, $(shell seq 1 $(SCALE_TARGET)))

staging/update/public/pipeline: .FORCE  ## Update pipeline container image on Public ranch | staging
	$(call run_pipeline_update,💿️,updating Public ranch pipeline image,testing_farm_staging_public_workers,update_image,staging)
	$(call run_pipeline_update,🛠,updating Public ranch pipeline configuration,testing_farm_staging_public_workers,update_config,staging)
	$(call run_pipeline_update,⛴️,updating Public ranch pipeline jobs,testing_farm_staging_public_workers,update_jobs,staging)

staging/update/public/pipeline/image: .FORCE  ## Update pipeline container image on Public ranch | staging
	$(call run_pipeline_update,💿️,updating Public ranch pipeline image,testing_farm_staging_public_workers,update_image,staging)

staging/update/public/pipeline/config: generate/staging/citool-config .FORCE  ## Update pipeline configuration on Public ranch | staging
	$(call run_pipeline_update,🛠,updating Public ranch pipeline configuration,testing_farm_staging_public_workers,update_config,staging)

staging/update/public/pipeline/jobs: .FORCE  ## Update pipeline jobs on Public ranch | staging
	$(call run_pipeline_update,⛴️,updating Public ranch pipeline jobs,testing_farm_staging_public_workers,update_jobs,staging)


##@ Infrastructure | Production

# Check if the code being deployed is `main` and it is the latest one pushed to the repository
define check_deployment_branch
  $(if $(SKIP_MAIN_CHECK),@echo -e "\033[0;33m⚠️  Skipping main branch check (SKIP_MAIN_CHECK is set)\033[0m",\
  @branch=$$(git symbolic-ref --short HEAD); \
  if [ "$$branch" != "main" ]; then \
		echo -e "⛔️ \033[0;31mProduction must be deployed from 'main' branch, detected '$$branch'.\033[0m"; \
		exit 1; \
  fi; \
	git fetch origin; \
	if ! git status -uno | grep -q "up to date"; then \
		echo -e "⛔️ \033[0;31mThe branch '$$branch' is not up-to-date. Please run 'git pull --rebase --autostash'.\033[0m"; \
		exit 1; \
	fi)
	@echo -e "\033[0;32m✅ branch 'main' is up-to-date with 'origin'\033[0m"
endef

production/init: .FORCE  ## Initialize | production | all
	$(call run_terragrunt,production,init)

production/plan: .FORCE  ## Plan deployment | production
	$(call run_terragrunt,production,plan)

production/plan/eks: .FORCE  ## Plan deployment | production | eks
	$(call run_terragrunt,production/eks,plan)

production/plan/artemis: .FORCE  ## Plan deployment | production | artemis
	$(call run_terragrunt,production/artemis,plan)

production/apply: .FORCE  ## Deploy | production | all
	$(call run_terragrunt,production,apply)

production/apply/eks: .FORCE  ## Deploy | production | eks
	$(call run_terragrunt,production/eks,apply)

production/apply/artemis: .FORCE  ## Deploy | production | artemis
	$(call run_terragrunt,production/artemis,apply)

update/api/settings: .FORCE  ## Deploy API settings to production (graceful reload)
	$(call check_deployment_branch)
	@echo -e "⚙️  \033[0;32mdeploying API settings to production\033[0m"
	ansible-playbook -v ansible/playbooks/deploy-api-settings.yaml

production/destroy: .FORCE terminate/artemis/guests/production  ## Destroy | production
	$(call run_terragrunt,production,destroy)

production/kubeconfig: .FORCE  ## Update kubeconfig for production environment
	aws eks --profile fedora_us_east_1 update-kubeconfig --name testing-farm-production

production/public/proxy/nomad:  ## Setup ssh proxy for Public ranch nomad
	@if ! curl -s http://localhost:4646/v1/health/client; then \
		echo -e "🚀 \033[0;32mSetting up ssh proxy for public ranch nomad: http://localhost:4646\033[0m"; \
		ssh -fNL 4646:localhost:4646 testing_farm_public_server_01; \
	else \
	        echo -e "🚀 \033[0;32mProxy for public ranch nomad already available: http://localhost:4646\033[0m"; \
	fi

production/public/request/shell: production/public/proxy/nomad  ## Connect to shell of a running Public ranch request
	@setup/request_shell.sh $(SCALE_TARGET)

production/redhat/request/shell:  ## Connect to shell of a running Red Hat ranch request
	@NOMAD_SSH="ssh fedora@testing-farm-server-01.osci.redhat.com" \
		NOMAD_ADDR=http://testing-farm-server-01.osci.redhat.com:4646 \
		bash setup/request_shell.sh "$(SCALE_TARGET)"

.create/redhat/worker%:
	ansible-playbook -e deployment=production ${SKIP_NOMAD_START} -v ansible/playbooks/create-redhat-worker.yaml

create/redhat/worker: .FORCE  ## Create a Red Hat ranch worker, use `make create/redhat/worker N` to create N workers
	$(call check_deployment_branch)
	@echo -e "🚀 \033[0;32mcreating $(SCALE_TARGET) redhat worker(s)\033[0m"
	@$(MAKE) -j $(addprefix .create/redhat/worker, $(shell seq 1 $(SCALE_TARGET)))

.create/public/worker%:
	ansible-playbook -e deployment=production ${SKIP_NOMAD_START} -v ansible/playbooks/create-public-worker.yaml

create/public/worker: .FORCE  ## Create a Public ranch worker, use `make create/public/worker N` to create N workers
	$(call check_deployment_branch)
	@echo -e "🚀 \033[0;32mcreating $(SCALE_TARGET) public worker(s)\033[0m"
	@$(MAKE) -j $(addprefix .create/public/worker, $(shell seq 1 $(SCALE_TARGET)))

define run_pipeline_update
	@echo -e "$(1) \033[0;32m$(2)\033[0m"
	ansible-playbook -v -e deployment=$(5) -l $(3) -t $(4) ansible/playbooks/testing-farm.yaml
endef

# To pass a comma in the call command arguments
# https://www.gnu.org/software/make/manual/html_node/Syntax-of-Functions.html
comma := ,

update/pipeline/image: .FORCE  ## Update pipeline container image on all ranches
	$(call check_deployment_branch)
	$(call run_pipeline_update,💿️,updating pipeline image on all ranches,testing_farm_public_workers$(comma)testing_farm_redhat_workers,update_image,production)

update/public/pipeline/image: .FORCE  ## Update pipeline container image on Public ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,💿️,updating Public ranch pipeline image,testing_farm_public_workers,update_image,production)

update/redhat/pipeline/image: .FORCE  ## Update pipeline container image on Red Hat ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,💿️,updating Red Hat ranch pipeline image,testing_farm_redhat_workers,update_image,production)

update/pipeline/config: .FORCE  ## Update pipeline configuration on all ranches
	$(call check_deployment_branch)
	$(call run_pipeline_update,🛠,updating pipeline configuration on all ranches,testing_farm_public_workers$(comma)testing_farm_redhat_workers,update_config,production)

update/public/pipeline/config: .FORCE  ## Update pipeline configuration on Public ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,🛠,updating Public ranch pipeline configuration,testing_farm_public_workers,update_config,production)

update/redhat/pipeline/config: .FORCE  ## Update pipeline configuration on Red Hat ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,🛠,updating Red Hat ranch pipeline configuration,testing_farm_redhat_workers,update_config,production)

update/pipeline/jobs: .FORCE  ## Update pipeline jobs on all ranches
	$(call check_deployment_branch)
	$(call run_pipeline_update,⛴️,updating pipeline jobs on all ranches,testing_farm_public_workers$(comma)testing_farm_redhat_workers,update_jobs,production)

update/public/pipeline/jobs: .FORCE  ## Update pipeline jobs on Public ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,⛴️,updating Public ranch pipeline jobs,testing_farm_public_workers,update_jobs,production)

update/redhat/pipeline/jobs: .FORCE  ## Update pipeline jobs on Red Hat ranch
	$(call check_deployment_branch)
	$(call run_pipeline_update,⛴️,updating Red Hat ranch pipeline jobs,testing_farm_redhat_workers,update_jobs,production)


##@ infrastructure | Fedora CI

osci/kubeconfig: .FORCE  ## Update kubeconfig for OSCI access to Fedora CI
	aws eks --profile fedora_us_east_2_osci update-kubeconfig --name fedora-ci-osci


##@ Tests | maintainer

define run_pytest_gluetool
	poetry run pytest $(PYTEST_OPTIONS) $(PYTEST_PARALLEL_OPTIONS) $3 -vvv --basetemp $$PROJECT_ROOT/.pytest \
	--color=yes \
	--citool-extra-podman-args "$(CITOOL_EXTRA_PODMAN_ARGS)" \
	--citool-config terragrunt/environments/$1/$2/citool-config --citool-image $(WORKER_IMAGE) \
	--test-assets tests/worker \
	--variables terragrunt/environments/$1/$2/citool-config/variables.yaml \
	--variables terragrunt/environments/$1/$2/citool-config/environment.yaml \
	--variables tests/common.yaml \
	--html report.html tests/worker/test_pipeline.py
endef

define download_oculus_to_pytest
	curl -s --fail --retry 5 --show-error https://gitlab.com/api/v4/projects/testing-farm%2Foculus/repository/files/results.html/raw?ref=main | tee > /dev/null $$(find .pytest -mindepth 2 -maxdepth 2 -type d -exec echo {}/results.html \;)
endef

test/dev/pipeline: .FORCE wait/artemis/dev generate/dev/citool-config/local  ## Run worker tests | dev
	$(call run_pytest_gluetool,dev,worker-local,-m "public and pipeline and not container")
	$(call download_oculus_to_pytest)

test/dev/pipeline/container: .FORCE wait/artemis/dev generate/dev/citool-config/local  ## Run worker tests | dev
	$(call run_pytest_gluetool,dev,worker-local,-m "public and pipeline and container")
	$(call download_oculus_to_pytest)

test/dev/compose: .FORCE wait/artemis/dev generate/dev/citool-config/local  ## Run compose tests | dev
	$(call run_pytest_gluetool,dev,worker-local,-m "public and compose")
	$(call download_oculus_to_pytest)

test/staging/pipeline: .FORCE wait/artemis/staging generate/staging/citool-config  ## Run worker tests, except container | staging
	$(call run_pytest_gluetool,staging,worker-local,-m "public and pipeline and not container")
	$(call download_oculus_to_pytest)

test/staging/pipeline/container: .FORCE wait/artemis/staging generate/staging/citool-config  ## Run worker tests, container only | staging
	$(call run_pytest_gluetool,staging,worker-local,-m "public and pipeline and container")
	$(call download_oculus_to_pytest)

test/staging/pipeline/ci: .FORCE wait/artemis/staging/ci generate/staging/citool-config/ci  ## Run worker tests, except container | staging | CI
	$(call run_pytest_gluetool,staging,worker-local,-m "public and pipeline and not container")

test/staging/pipeline/container/ci: .FORCE generate/staging/citool-config/container/ci  ## Run worker tests, container only | staging | CI
	$(call run_pytest_gluetool,staging,worker-local,-m "public and pipeline and container")

test/staging/compose: .FORCE wait/artemis/staging generate/staging/citool-config  ## Run compose tests | staging
	$(call run_pytest_gluetool,staging,worker-local,-m "public and compose")
	$(call download_oculus_to_pytest)

test/staging/compose/ci: .FORCE wait/artemis/staging/ci generate/staging/citool-config/ci  ## Run compose tests | staging | CI
	$(call run_pytest_gluetool,staging,worker-local,-m "public and compose")

test/pytest/oculus: .FORCE  ## Download oculus to .pytest directory
	$(call download_oculus_to_pytest)

test/terraform/modules:  ## Run terratest tests for Teraform modules
	env -C terragrunt/modules/vault/test go test -v
	env -C terragrunt/modules/url_wait/test go test -v
	env -C terragrunt/modules/gitlab/group/test go test -v
	env -C terragrunt/modules/gitlab/project/test go test -v
	env -C terragrunt/modules/gitlab/schedules/test go test -v

generate/dev/citool-config/local: .FORCE  ## Generate citool-config | dev
	WORKER=worker-local poetry run python setup/generate_environment.py dev

generate/dev/citool-config/public: .FORCE  ## Generate citool-config | dev
	WORKER=worker-public poetry run python setup/generate_environment.py dev

generate/staging/citool-config: .FORCE  ## Generate citool-config for public workers | staging
	poetry run python setup/generate_environment.py staging

generate/staging/citool-config/ci: .FORCE  ## Generate citool-config | staging | CI
	$(call check_staging_ci_env)
	WORKER=worker-local ARTEMIS_DEPLOYMENT=ci/artemis STAGING_CI_SUFFIX=$$STAGING_CI_SUFFIX poetry run python setup/generate_environment.py staging

generate/staging/citool-config/container/ci: .FORCE  ## Generate citool-config for container testing only (no Artemis) | staging | CI
	$(call check_staging_ci_env)
	WORKER=worker-local ARTEMIS_DEPLOYMENT=none poetry run python setup/generate_environment.py staging

generate/public/tests/compose: .FORCE  ## Generate or update compose tests for Public Ranch
	@TESTING_FARM_API_TOKEN=${TESTING_FARM_PRODUCTION_API_TOKEN_PUBLIC} TESTING_FARM_API_URL=${TESTING_FARM_PRODUCTION_API_URL} poetry run python setup/generate_compose_tests.py

list/integration/tests: .FORCE  ## List available integration tests
	poetry run pytest $(PYTEST_OPTIONS) -v --basetemp $$PROJECT_ROOT/.pytest --collect-only \
	--citool-config ranch/redhat/citool-config --citool-image $(WORKER_IMAGE) \
	--test-assets tests/worker \
	--html report.html tests/worker/test_pipeline.py


##@ Compose

compose/update/public: .FORCE  ## Update composes in the Public ranch
	poetry run python setup/compose_update_public.py


##@ Utility

edit/secrets: .FORCE  ## Edit secrets software
	ansible-vault edit ansible/secrets/credentials.yaml

$(ENVIRONMENT_FILES):
	# Generate `environment.yaml` variable files
	poetry run python setup/generate_environment.py

	# Decrypt ssh keys
	for key in $(ENVIRONMENT_KEYS); do \
		echo "Decrypting $${key%.decrypted}..."; \
		ansible-vault decrypt --vault-password-file .vault_pass --output $${key} $${key%.decrypted}; \
	done

wait/artemis/dev: .FORCE  ## Wait until Artemis is available | dev
	@bash setup/wait_artemis_available.sh dev

wait/artemis/staging: .FORCE  ## Wait until Artemis is available | staging
	@bash setup/wait_artemis_available.sh staging

wait/artemis/staging/ci: .FORCE  ## Wait until Artemis is available | staging | CI
	$(call check_staging_ci_env)
	@ARTEMIS_DEPLOYMENT=ci/artemis bash setup/wait_artemis_available.sh staging


##@ Cleanup | maintainer

terminate/artemis/guests/dev: .FORCE  ## Terminate all EC2 instances created by Artemis | dev
	@bash $$PROJECT_ROOT/setup/terminate_artemis_guests.sh dev || true

terminate/artemis/guests/staging: .FORCE  ## Terminate all EC2 instances created by Artemis | staging
	@bash $$PROJECT_ROOT/setup/terminate_artemis_guests.sh staging || true

terminate/artemis/guests/staging/ci: .FORCE  ## Terminate all EC2 instances created by Artemis | staging | CI
	$(call check_staging_ci_env)
	@ARTEMIS_DEPLOYMENT=ci/artemis bash $$PROJECT_ROOT/setup/terminate_artemis_guests.sh staging || true

terminate/artemis/guests/production: .FORCE  ## Terminate all EC2 instances created by Artemis | staging
	@bash $$PROJECT_ROOT/setup/terminate_artemis_guests.sh production || true

cleanup/ci: .FORCE staging/kubeconfig  ## Cleanup CI leftovers
	@echo "$(ccgreen)Deleting leftover EKS CI clusters$(ccend)"
	@bash $$PROJECT_ROOT/setup/terminate_eks_ci_clusters.sh
	@echo "$(ccgreen)Terminating leftover namespaces from staging EKS$(ccend)"
	@bash $$PROJECT_ROOT/setup/terminate_eks_ci_namespaces.sh
	@echo "$(ccgreen)Removing unused load balancers$(ccend)"
	@bash $$PROJECT_ROOT/setup/terminate_unused_loadbalancers.sh
	@echo "$(ccgreen)Cleaning up Route53 records$(ccend)"
	@bash $$PROJECT_ROOT/setup/cleanup_route53_records.sh
	@echo "$(ccgreen)Removing leftover security groups$(ccend)"
	@bash $$PROJECT_ROOT/setup/terminate_security_groups.sh
	@echo "$(ccgreen)Removing orphaned EBS volumes$(ccend)"
	@bash $$PROJECT_ROOT/setup/terminate_ebs_volumes.sh