Makefile

# Import development related environment variables from dev.env
ifneq ("$(wildcard dev.env)","")
    include dev.env
endif

# Default version for the project.
PROJECT_VERSION ?= v0.0.1

####################################
# GPU Operator Image Build variables
# Note: when using images from DockerHub, please make sure to input the full DockerHub registry URL (docker.io) into DOCKER_REGISTRY
# user's container runtime may not set DockerHub as default registry and auto-search on DockerHub
TOP_DIR = $(PWD)
GOFLAGS := "-mod=mod"
GIT_COMMIT ?= $(shell git rev-parse --short HEAD)
CONTAINER_TOOL ?= docker
DOCKER_REGISTRY ?= docker.io/rocm
IMAGE_NAME ?= amd-gpu-operator
IMAGE_TAG_BASE ?= $(DOCKER_REGISTRY)/$(IMAGE_NAME)
IMAGE_TAG ?= dev
IMG ?= $(IMAGE_TAG_BASE):$(IMAGE_TAG)
HOURLY_TAG_LABEL ?= latest

# KMM related images
KMM_IMAGE_TAG ?= latest
KMM_SIGNER_IMG ?= $(DOCKER_REGISTRY)/kernel-module-management-signimage:$(KMM_IMAGE_TAG)
KMM_WORKER_IMG ?= $(DOCKER_REGISTRY)/kernel-module-management-worker:$(KMM_IMAGE_TAG)
KMM_BUILDER_IMG ?= gcr.io/kaniko-project/executor:v1.23.2
KMM_WEBHOOK_IMG_NAME ?= $(DOCKER_REGISTRY)/kernel-module-management-webhook-server
KMM_OPERATOR_IMG_NAME ?= $(DOCKER_REGISTRY)/kernel-module-management-operator

# Operand related images
EXPORTER_IMAGE_TAG ?= latest
METRICS_EXPORTER_IMG = $(DOCKER_REGISTRY)/device-metrics-exporter:$(EXPORTER_IMAGE_TAG)
DEVICE_CONFIG_MANAGER_IMAGE_TAG ?= latest
DEVICE_CONFIG_MANAGER_IMG = $(DOCKER_REGISTRY)/device-config-manager:$(DEVICE_CONFIG_MANAGER_IMAGE_TAG)
TEST_RUNNER_IMAGE_TAG ?= latest
TEST_RUNNER_IMG = $(DOCKER_REGISTRY)/test-runner:$(TEST_RUNNER_IMAGE_TAG)
UTILS_IMAGE_TAG ?= latest
UTILS_IMAGE_NAME ?= $(IMAGE_NAME)-utils
UTILS_IMG ?= $(DOCKER_REGISTRY)/$(UTILS_IMAGE_NAME):$(UTILS_IMAGE_TAG)
DRA_DRIVER_IMAGE_TAG ?= latest
DRA_DRIVER_IMG = $(DOCKER_REGISTRY)/k8s-gpu-dra-driver:$(DRA_DRIVER_IMAGE_TAG)

#######################
# Helm Charts variables
YAML_FILES=bundle/manifests/amd-gpu-operator-node-metrics_rbac.authorization.k8s.io_v1_rolebinding.yaml bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml bundle/manifests/amd-gpu-operator-node-labeller_rbac.authorization.k8s.io_v1_clusterrolebinding.yaml bundle/manifests/amd-gpu-operator-node-metrics_monitoring.coreos.com_v1_servicemonitor.yaml config/samples/amd.com_deviceconfigs.yaml config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml example/deviceconfig_example.yaml config/default/kustomization.yaml
CRD_YAML_FILES = deviceconfig-crd.yaml remediationworkflowstatus-crd.yaml
K8S_KMM_CRD_YAML_FILES=module-crd.yaml nodemodulesconfig-crd.yaml
DEFAULT_VALUES_FILES=helm-charts-k8s/values.yaml hack/k8s-patch/metadata-patch/values.yaml
REMEDIATION_CRD_YAML_FILES=clusterworkflowtemplate-crd.yaml cronworkflow-crd.yaml workflowartifactgctask-crd.yaml workflow-crd.yaml workfloweventbinding-crd.yaml workflowtaskresult-crd.yaml workflowtaskset-crd.yaml workflowtemplate-crd.yaml
HELM_CHART_VERSION ?= $(PROJECT_VERSION)
HELM_APP_VERSION ?= $(IMAGE_TAG)

HELM_OUTPUT_FILE_NAME ?= gpu-operator-helm-k8s-$(HELM_CHART_VERSION).tgz
GPU_OPERATOR_CHART ?= $(shell pwd)/helm-charts-k8s/$(HELM_OUTPUT_FILE_NAME)
KUBECTL_CMD ?= kubectl

ifdef SKIP_NFD
	SKIP_NFD_CMD=--set node-feature-discovery.enabled=false
endif

ifdef SKIP_KMM
	SKIP_KMM_CMD=--set kmm.enabled=false
endif

ifdef SIM_ENABLE
	SIM_ENABLE_CMD=--set controllerManager.env.simEnable=true
endif

ifdef SKIP_INSTALL_DEFAULT_CR
	SKIP_INSTALL_DEFAULT_CR_CMD=--set crds.defaultCR.install=false
endif

ifdef SKIP_REMEDIATION
	SKIP_REMEDIATION_CMD=--set remediation.enabled=false
endif

ifdef SKIP_REMEDIATION_CRDS
	SKIP_REMEDIATION_CRDS_CMD=--set remediation.installCRDs=false
endif

#################################
# OpenShift OLM Bundle varaiables
# BUNDLE_IMG defines the image:tag used for the bundle.
# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:$(IMAGE_TAG)
INDEX_IMG := $(IMAGE_TAG_BASE)-index:$(IMAGE_TAG)
BUNDLE_NAMESPACE ?= default # the namespace to deploy the OLM bundle 

# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
# To re-generate a bundle for other specific channels without changing the standard setup, you can:
# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable)
# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable")
ifneq ($(origin CHANNELS), undefined)
	BUNDLE_CHANNELS := --channels=$(CHANNELS)
endif

# DEFAULT_CHANNEL defines the default channel used in the bundle.
# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable")
# To re-generate a bundle for any other default channel without changing the default setup, you can:
# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable)
# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable")
ifneq ($(origin DEFAULT_CHANNEL), undefined)
	BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
endif

BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command
BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(shell echo $(PROJECT_VERSION) | sed 's/^v//') $(BUNDLE_METADATA_OPTS)
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
ENVTEST_K8S_VERSION = 1.23

##################################
# Docker shell container variables
# Setting SHELL to bash allows bash commands to be executed by recipes.
# This is a requirement for 'setup-envtest.sh' in the test target.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec
HOST_DOCKER_SOCKET ?= /var/run/docker.sock
CONTAINER_DOCKER_SOCKET ?= /var/run/docker.sock
DOCKER_GID := $(shell stat -c '%g' $(HOST_DOCKER_SOCKET))
USER_UID := $(shell id -u)
USER_GID := $(shell id -g)
DOCKER_BUILDER_TAG := v1.4
DOCKER_BUILDER_IMAGE := $(DOCKER_REGISTRY)/gpu-operator-build:$(DOCKER_BUILDER_TAG)
CONTAINER_WORKDIR := /gpu-operator
BUILD_BASE_IMG ?= ubuntu:22.04
GOLANG_BASE_IMG ?= golang:1.23

##################
# Documentation website build variables
DOCS_DIR := ${TOP_DIR}/docs
BUILD_DIR := $(DOCS_DIR)/_build
HTML_DIR := $(BUILD_DIR)/html

##################
# Makefile targets

##@ QuickStart
.PHONY: default
default: docker-build-env ## Quick start to build everything from docker shell container.
	@echo "Starting a shell in the Docker build container..."
	@$(CONTAINER_TOOL) run --rm -it --privileged \
	    --network host \
		--name gpu-operator-build \
		-e "USER_NAME=$(shell whoami)" \
		-e "USER_UID=$(shell id -u)" \
		-e "USER_GID=$(shell id -g)" \
		-e "DOCKER_GID=$(DOCKER_GID)" \
		-v $(CURDIR):/gpu-operator \
		-v $(CURDIR):/home/$(shell whoami)/go/src/github.com/ROCm/gpu-operator \
		-v $(HOME)/.ssh:/home/$(shell whoami)/.ssh \
		-v $(HOST_DOCKER_SOCKET):$(CONTAINER_DOCKER_SOCKET) \
		-w $(CONTAINER_WORKDIR) \
		$(DOCKER_BUILDER_IMAGE) \
		'bash -ic "source ~/.bashrc && cd /gpu-operator && git config --global --add safe.directory /gpu-operator && make all && GOFLAGS=-mod=mod go run tools/build/copyright/main.go && make fmt"'

.PHONY: docker/shell
docker/shell: docker-build-env ## Bring up and attach to a container that has dev environment configured.
	@echo "Starting a shell in the Docker build container..."
	@$(CONTAINER_TOOL) run --rm -it --privileged \
		--name gpu-operator-build \
		-e "USER_NAME=$(shell whoami)" \
		-e "USER_UID=$(shell id -u)" \
		-e "USER_GID=$(shell id -g)" \
		-e "DOCKER_GID=$(DOCKER_GID)" \
		-v $(CURDIR):/gpu-operator \
		-v $(CURDIR):/home/$(shell whoami)/go/src/github.com/ROCm/gpu-operator \
		-v $(HOME)/.ssh:/home/$(shell whoami)/.ssh \
		-v $(HOST_DOCKER_SOCKET):$(CONTAINER_DOCKER_SOCKET) \
		-w $(CONTAINER_WORKDIR) \
		$(DOCKER_BUILDER_IMAGE) \
		"cd /gpu-operator && git config --global --add safe.directory /gpu-operator && bash"

.PHONY: distrobox/shell
distrobox/shell: distrobox-build-env ## Bring up and attach to a container that has dev environment configured.
	@echo "Starting a shell in the Distrobox build container..."
	@distrobox enter --name gpu-operator-build

.PHONY: all
all: generate manager manifests helm-k8s bundle-build docker-build

##@ General

# The help target prints out all targets with their descriptions organized
# beneath their categories. The categories are represented by '##@' and the
# target descriptions by '##'. The awk commands is responsible for reading the
# entire set of makefiles included in this invocation, looking for lines of the
# file as xyz: ## something, and then pretty-format the target and help. Then,
# if there's a line with ##@ something, that gets pretty-printed as a category.
# More info on the usage of ANSI control characters for terminal formatting:
# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
# More info on the awk command:
# http://linuxcommand.org/lc3_adv_awk.php

.PHONY: help
help: ## Display this help.
	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9\/\-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

##@ Development

.PHONY: update-registry
update-registry:
	# updating registry information in yaml files
	sed -i -e 's|image:.*$$|image: ${IMG}|' bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml
	sed -i -e 's|containerImage:.*$$|containerImage: ${IMG}|' config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml
	sed -i -e 's|metricsExporterImage:.*$$|metricsExporterImage: ${METRICS_EXPORTER_IMG}|' \
	bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml \
	config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml
	sed -i -e 's|repository:.*$$|repository: ${IMAGE_TAG_BASE}|' \
	hack/k8s-patch/metadata-patch/values.yaml
	sed -i -e "s/newTag:.*$$/newTag: ${IMAGE_TAG}/" -e "s/tag:.*$$/tag: ${IMAGE_TAG}/" \
	-e 's|newName:.*$$|newName: ${IMAGE_TAG_BASE}|' \
	config/manager-base/kustomization.yaml config/manager/kustomization.yaml \
	hack/k8s-patch/metadata-patch/values.yaml helm-charts-k8s/values.yaml \
	example/deviceconfig_example.yaml
	# update operands image tags
	@for file in $(DEFAULT_VALUES_FILES); do \
		yq eval -i '.deviceConfig.spec.metricsExporter.image = "$(METRICS_EXPORTER_IMG)"' $$file; \
		yq eval -i '.deviceConfig.spec.configManager.image = "$(DEVICE_CONFIG_MANAGER_IMG)"' $$file; \
		yq eval -i '.deviceConfig.spec.testRunner.image = "$(TEST_RUNNER_IMG)"' $$file; \
		yq eval -i '.deviceConfig.spec.commonConfig.utilsContainer.image = "$(UTILS_IMG)"' $$file; \
		yq eval -i '.deviceConfig.spec.draDriver.image = "$(DRA_DRIVER_IMG)"' $$file; \
		yq eval -i '.deviceConfig.spec.remediationWorkflow.testerImage = "$(TEST_RUNNER_IMG)"' $$file; \
	done
	sed -i -e 's|tag:.*$$|tag: ${KMM_IMAGE_TAG}|' \
	-e 's|repository:.*operator.*$$|repository: ${KMM_OPERATOR_IMG_NAME}|' \
	-e 's|repository:.*webhook.*$$|repository: ${KMM_WEBHOOK_IMG_NAME}|' \
	-e 's|relatedImageBuild:.*$$|relatedImageBuild: ${KMM_BUILDER_IMG}|' \
	-e 's|relatedImageSign:.*$$|relatedImageSign: ${KMM_SIGNER_IMG}|' \
	-e 's|relatedImageWorker:.*$$|relatedImageWorker: ${KMM_WORKER_IMG}|' \
	hack/k8s-patch/k8s-kmm-patch/metadata-patch/values.yaml

.PHONY: update-helm-metadata
update-helm-metadata: ## Update helm Chart.yaml version and appVersion based on HELM_CHART_VERSION
	sed -i -e 's|appVersion:.*$$|appVersion: "$(HELM_APP_VERSION)"|' hack/k8s-patch/metadata-patch/Chart.yaml
	sed -i '0,/version:/s|version:.*|version: $(HELM_CHART_VERSION)|' hack/k8s-patch/metadata-patch/Chart.yaml

.PHONY: update-version
update-version: update-helm-metadata ## Update the Project version in helm charts based on ${PROJECT_VERSION}
	# updating project version in Dockerfile metadata
	sed -i 's/release="[^"]*"/release="${IMAGE_TAG}"/g' Dockerfile internal/utils_container/Dockerfile
	sed -i 's/version="[^"]*"/version="${IMAGE_TAG}"/g' Dockerfile internal/utils_container/Dockerfile
	# updating default image tags in Go source files
	sed -i 's|defaultConfigManagerImage.*=.*"docker.io/rocm/device-config-manager:[^"]*"|defaultConfigManagerImage = "docker.io/rocm/device-config-manager:${PROJECT_VERSION}"|' internal/configmanager/configmanager.go
	sed -i 's|defaultMetricsExporterImage.*=.*"docker.io/rocm/device-metrics-exporter:[^"]*"|defaultMetricsExporterImage = "docker.io/rocm/device-metrics-exporter:${PROJECT_VERSION}"|' internal/metricsexporter/metricsexporter.go
	sed -i 's|defaultTestRunnerImage.*=.*"docker.io/rocm/test-runner:[^"]*"|defaultTestRunnerImage = "docker.io/rocm/test-runner:${PROJECT_VERSION}"|' internal/testrunner/testrunner.go
	${MAKE} fmt

.PHONY: update-version-in-ci
update-version-in-ci: ## Update project version and helm chart references in CI job config (.job.yml) and asset-push script
	# updating project version in CI job config
	sed -i -e 's|PROJECT_VERSION=[^ ]*|PROJECT_VERSION=${PROJECT_VERSION}|' .job.yml
	sed -i '0,/HELM_CHARTS_VERSION=/s|HELM_CHARTS_VERSION=[^ ]*|HELM_CHARTS_VERSION=${PROJECT_VERSION}-$${RELEASE:-dev}|' .job.yml
	sed -i 's|PROJECT_VERSION:-.*$$|PROJECT_VERSION:-${PROJECT_VERSION}\}|' asset-build/gpuoperator-asset-push.sh

.PHONY: manifests
manifests: controller-gen update-registry update-version ## Generate ClusterRole and CustomResourceDefinition objects.
	$(CONTROLLER_GEN) crd paths="./api/..." output:crd:artifacts:config=config/crd/bases
	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="./internal/controllers" output:rbac:artifacts:config=config/rbac

.PHONY: generate
generate: controller-gen mockgen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
	go generate ./...

.PHONY: fmt
fmt: ## Run go fmt against code.
	go fmt ./...

.PHONY: vet
vet: ## Run go vet against code.
	go vet ./...

UNIT_TEST ?= ./internal ./internal/controllers ./internal/kmmmodule

.PHONY: unit-test
unit-test: vet ## Run the unit tests.
	go test $(UNIT_TEST) -v -coverprofile cover.out

.PHONY: e2e
e2e: ## Run the e2e tests. Make sure you have ~/.kube/config configured for your test cluster.
	$(info deploying ${GPU_OPERATOR_CHART})
	SKIP_INSTALL_DEFAULT_CR=1 ${MAKE} helm-install
	export OPENSHIFT
	export SIM_ENABLE
	${MAKE} -C tests/e2e/nodeapp
	${MAKE} -C tests/e2e
	${MAKE} helm-uninstall
	${MAKE} helm-e2e

.PHONY: helm-e2e
helm-e2e: ## Run the helm chart e2e tests.
	GPU_OPERATOR_CHART=${GPU_OPERATOR_CHART} ${MAKE} -C tests/helm-e2e

.PHONY: dcm_e2e
dcm_e2e:
	$(info deploying ${GPU_OPERATOR_CHART})
	SKIP_INSTALL_DEFAULT_CR=1 ${MAKE} helm-install
	export OPENSHIFT
	export SIM_ENABLE
	${MAKE} -C tests/e2e/nodeapp
	${MAKE} -C tests/e2e dcm_e2e
	${MAKE} helm-uninstall

GOFILES_NO_VENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
.PHONY: lint
lint: golangci-lint ## Run golangci-lint against code.
	@if [ `gofmt -l $(GOFILES_NO_VENDOR) | wc -l` -ne 0 ]; then \
		echo There are some malformed files, please make sure to run \'make fmt\'; \
		gofmt -l $(GOFILES_NO_VENDOR); \
		exit 1; \
	fi
	$(GOLANGCI_LINT) run -v --timeout 5m0s

DOCS_MARKDOWNLINTCONFIG ?= docs/.markdownlint.yaml
DOCS_MD_GLOB ?= "**/*.md"
DOCS_SPELLCHECK_CONFIG ?= .spellcheck.yaml

.PHONY: docs-lint-markdown
docs-lint-markdown:
	markdownlint-cli2 $(DOCS_MD_GLOB) --config $(DOCS_MARKDOWNLINTCONFIG)

.PHONY: docs-lint-spelling
docs-lint-spelling:
	pyspelling -c $(DOCS_SPELLCHECK_CONFIG)

.PHONY: docs-lint
docs-lint: ## Run docs Markdown lint + spelling (full ROCm-style docs lint).
	${MAKE} docs-lint-markdown
	${MAKE} docs-lint-spelling

##@ Build

manager: $(shell find -name "*.go") go.mod go.sum  ## Build manager binary.
	go build -ldflags="-X main.Version=$(PROJECT_VERSION) -X main.GitCommit=$(GIT_COMMIT) -X main.BuildTag=$(HOURLY_TAG_LABEL)" -o $@ ./cmd

.PHONY: docker-build
docker-build: ## Build docker image with the manager.
	DOCKER_BUILDKIT=1 $(CONTAINER_TOOL) build -t $(IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) --build-arg TARGET=manager --build-arg GOLANG_BASE_IMG=$(GOLANG_BASE_IMG) .

.PHONY: docker-push
docker-push: ## Push docker image with the manager.
	$(CONTAINER_TOOL) push $(IMG)

.PHONY: docker-save
docker-save: ## Save the container image with the manager.
	$(CONTAINER_TOOL) save $(IMG) | gzip > $(IMAGE_NAME).tar.gz

.PHONY: docker-build-utils
docker-build-utils: ## Build docker image for utils container.
	DOCKER_BUILDKIT=1 docker build -t $(UTILS_IMG) --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f internal/utils_container/Dockerfile .

.PHONY: docker-push-utils
docker-push-utils: ## Push docker image for utils container.
	$(CONTAINER_TOOL) push $(UTILS_IMG)

.PHONY: docker-save-utils
docker-save-utils: ## Save the utils container image as tar.gz.
	$(CONTAINER_TOOL) save $(UTILS_IMG) | gzip > $(IMAGE_NAME)-utils.tar.gz

.PHONY: docker-build-env
docker-build-env: ## Build the docker shell container.
	@echo "Building the Docker environment..."
	@if [ -n $(INSECURE_REGISTRY) ]; then \
    $(CONTAINER_TOOL) build \
        -t $(DOCKER_BUILDER_IMAGE) \
        --build-arg BUILD_BASE_IMG=$(BUILD_BASE_IMG) \
        --build-arg INSECURE_REGISTRY=$(INSECURE_REGISTRY) \
        -f Dockerfile.build .; \
	else \
		$(CONTAINER_TOOL) build \
			-t $(DOCKER_BUILDER_IMAGE) \
			--build-arg BUILD_BASE_IMG=$(BUILD_BASE_IMG) \
			-f Dockerfile.build .; \
	fi

.PHONY: distrobox-build-env
distrobox-build-env: docker-build-env
	@echo "Creating the distrobox environment..."
	@if [ -f "/usr/bin/distrobox" ]; then \
		distrobox create --image $(DOCKER_BUILDER_IMAGE) gpu-operator-build; \
	else \
		echo "ERROR: distrobox not found in PATH" >&2; \
		exit 1; \
	fi

.PHONY: helm
helm: ## Build helm charts for Kubernetes.
	$(MAKE) helm-k8s

.PHONY: helm-k8s
helm-k8s: helmify manifests kustomize clean-helm gen-kmm-charts gen-remediation-charts
	$(KUSTOMIZE) build config/default | $(HELMIFY) helm-charts-k8s
	# Patching k8s helm chart metadata
	cp $(shell pwd)/hack/k8s-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/
	# Patching k8s helm chart template
	cp $(shell pwd)/hack/k8s-patch/template-patch/* $(shell pwd)/helm-charts-k8s/templates/
	# Removing OpenShift related rbac from vanilla k8s helm charts
	rm $(shell pwd)/helm-charts-k8s/templates/kmm-device-plugin-rbac.yaml
	rm $(shell pwd)/helm-charts-k8s/templates/kmm-module-loader-rbac.yaml
	rm $(shell pwd)/helm-charts-k8s/templates/prometheus-k8s-rbac.yaml
	# Patching k8s helm chart kmm subchart
	cp $(shell pwd)/hack/k8s-patch/k8s-kmm-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/kmm/
	cp $(shell pwd)/hack/k8s-patch/k8s-kmm-patch/template-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/kmm/templates/
	# Patching k8s helm chart remediation subchart
	cp $(shell pwd)/hack/k8s-patch/k8s-remediation-patch/metadata-patch/*.yaml $(shell pwd)/helm-charts-k8s/charts/remediation-crds/
	cd $(shell pwd)/helm-charts-k8s; helm dependency update; helm lint .; cd ..;
	mkdir $(shell pwd)/helm-charts-k8s/crds
	echo "moving crd yaml files to crds folder"
	@for file in $(CRD_YAML_FILES); do \
		helm template amd-gpu helm-charts-k8s -s templates/$$file > $(shell pwd)/helm-charts-k8s/crds/$$file; \
	done
	rm $(shell pwd)/helm-charts-k8s/templates/*crd.yaml
	$(MAKE) helm-docs
	echo "dependency update, lint and pack charts"
	cd $(shell pwd)/helm-charts-k8s; helm dependency update; helm lint .; cd ..; helm package helm-charts-k8s/ --destination ./helm-charts-k8s
	mv $(shell pwd)/helm-charts-k8s/gpu-operator-charts-$(HELM_CHART_VERSION).tgz $(GPU_OPERATOR_CHART)

.PHONY: bundle-build
bundle-build: operator-sdk manifests kustomize ## OpenShift Build OLM bundle.
	rm -fr ./bundle
	VERSION=$(shell echo $(PROJECT_VERSION) | sed 's/^v//') ${OPERATOR_SDK} generate kustomize manifests --apis-dir api
	cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG)
	cd config/manager-base && $(KUSTOMIZE) edit set image controller=$(IMG)
	OPERATOR_SDK="${OPERATOR_SDK}" \
		     BUNDLE_GEN_FLAGS="${BUNDLE_GEN_FLAGS} --extra-service-accounts amd-gpu-operator-kmm-device-plugin,amd-gpu-operator-kmm-module-loader,amd-gpu-operator-node-labeller,amd-gpu-operator-metrics-exporter,amd-gpu-operator-metrics-exporter-rbac-proxy,amd-gpu-operator-test-runner,amd-gpu-operator-config-manager,amd-gpu-operator-utils-container" \
		     PKG=amd-gpu-operator \
		     SOURCE_DIR=$(dir $(realpath $(lastword $(MAKEFILE_LIST)))) \
		     KUBECTL_CMD=${KUBECTL_CMD} ./hack/generate-bundle
	cp $(shell pwd)/hack/openshift-patch/olm-bundle-patch/*.yaml $(shell pwd)/bundle/manifests/
	${OPERATOR_SDK} bundle validate ./bundle
	$(CONTAINER_TOOL) build --label HOURLY_TAG=$(HOURLY_TAG_LABEL) -f bundle.Dockerfile -t $(BUNDLE_IMG) .

.PHONY: dep-docs
dep-docs:
	pip install -r $(DOCS_DIR)/sphinx/requirements.txt

.PHONY: docs
docs: dep-docs ## Build documentation website
	sphinx-build -b html $(DOCS_DIR) $(HTML_DIR)
	@echo "Docs built at $(HTML_DIR)/index.html"

.PHONY: clean-docs
clean-docs: ## Clean up documentation website build cache
	rm -rf $(BUILD_DIR)

##@ Deployment

ifndef ignore-not-found
  ignore-not-found = false
endif

KUSTOMIZE_CONFIG_CRD ?= config/crd
.PHONY: install
install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config.
	${KUBECTL_CMD} apply -k $(KUSTOMIZE_CONFIG_CRD)

.PHONY: uninstall
uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
	${KUBECTL_CMD} delete -k $(KUSTOMIZE_CONFIG_CRD) --ignore-not-found=$(ignore-not-found)

KUSTOMIZE_CONFIG_DEFAULT ?= config/default
KUSTOMIZE_CONFIG_HUB_DEFAULT ?= config/default-hub

.PHONY: deploy
deploy: helm-install ## Deploy Helm Charts.

.PHONY: undeploy
undeploy: helm-uninstall ## Undeploy Helm Charts.

CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
CONTROLLER_GEN_VERSION=v0.17.0
.PHONY: controller-gen
controller-gen:
	$(call remove-wrong-version-tool,$(CONTROLLER_GEN),--version,${CONTROLLER_GEN_VERSION})
	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION})

GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
GOLANGCI_LINT_VERSION=v1.63.4
.PHONY: golangci-lint
golangci-lint:
	$(call remove-wrong-version-tool,$(GOLANGCI_LINT),--version,${GOLANGCI_LINT_VERSION})
	$(call go-get-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION})

HELMDOCS = $(shell pwd)/bin/helm-docs
.PHONY: helm-docs
helm-docs:
	$(call go-get-tool,$(HELMDOCS),github.com/norwoodj/helm-docs/cmd/helm-docs@v1.12.0)
	$(HELMDOCS) -c $(shell pwd)/helm-charts-k8s/ -g $(shell pwd)/helm-charts-k8s -u --ignore-non-descriptions
	cat $(shell pwd)/README.md $(shell pwd)/helm-charts-k8s/README.md | sed 's/# gpu-operator-charts/\n## gpu-operator-charts/' > /tmp/README.md
	sed -i -e :a -e '/^\n*$$/{$$d;N;};/\n$$/ba' /tmp/README.md
	mv /tmp/README.md $(shell pwd)/helm-charts-k8s/README.md

.PHONY: mockgen
mockgen:
	go install go.uber.org/mock/mockgen@v0.3.0

KUSTOMIZE = $(shell pwd)/bin/kustomize
.PHONY: kustomize
kustomize:
	@if [ ! -f ${KUSTOMIZE} ]; then \
		BINDIR=$(shell pwd)/bin ./hack/download-kustomize; \
	fi

# go-get-tool will 'go install' any package $2 and install it to $1.
PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
define go-get-tool
    if [ -z "$$GOBIN" ]; then \
        GOBIN=$$(go env GOPATH)/bin; \
    else \
        GOBIN=$$GOBIN; \
    fi; \
    [ -f $(1) ] || { \
    set -e; \
    echo "Downloading $(2)"; \
    echo "Running: GOBIN=$(PROJECT_DIR)/bin go install $(2)"; \
    GOBIN=$(PROJECT_DIR)/bin go install $(2); \
    }
endef

# remove-wrong-version-tool will use $1 $2 to check binary version
# any binary with mismatched version compared to $3 will be removed
# 1 - Path to the binary
# 2 - Version argument (e.g., --version)
# 3 - Expected version string (e.g., v0.17.0)
define remove-wrong-version-tool
@if [ -f $(1) ]; then \
version_output=`$(1) $(2) 2>/dev/null || echo "not found"`; \
echo "$$version_output" | grep -q $(3) || { \
echo "Incorrect version ($$version_output), removing $(1)"; \
rm -f $(1); \
}; \
fi
endef

OPERATOR_SDK = $(shell pwd)/bin/operator-sdk
OPERATOR_SDK_VERSION=v1.32.0
.PHONY: operator-sdk
operator-sdk:
	$(call remove-wrong-version-tool,$(OPERATOR_SDK),version,${OPERATOR_SDK_VERSION})
	@if [ ! -f ${OPERATOR_SDK} ]; then \
		set -e ;\
		echo "Downloading ${OPERATOR_SDK} ${OPERATOR_SDK_VERSION}"; \
		mkdir -p $(dir ${OPERATOR_SDK}) ;\
		curl -Lo ${OPERATOR_SDK} 'https://github.com/operator-framework/operator-sdk/releases/download/${OPERATOR_SDK_VERSION}/operator-sdk_linux_amd64'; \
		chmod +x ${OPERATOR_SDK}; \
	fi

.PHONY: bundle-push
bundle-push:
	$(CONTAINER_TOOL) push $(BUNDLE_IMG)

.PHONY: bundle-save
bundle-save:
	$(CONTAINER_TOOL) save $(BUNDLE_IMG) | gzip > $(IMAGE_NAME)-olm-bundle.tar.gz

.PHONY: bundle-scorecard-test
bundle-scorecard-test:
	${OPERATOR_SDK} scorecard --config bundle/tests/scorecard/config.yaml --kubeconfig ~/.kube/config $(BUNDLE_IMG)

.PHONY: bundle-deploy
bundle-deploy: ## OpenShift deploy OLM bundle.
	${OPERATOR_SDK} run bundle $(BUNDLE_IMG) --namespace=${BUNDLE_NAMESPACE}

.PHONY: bundle-deploy-upgrade
bundle-deploy-upgrade:
	${OPERATOR_SDK} run bundle-upgrade $(BUNDLE_IMG)

.PHONY: bundle-cleanup
bundle-cleanup: ## OpenShift undeploy OLM bundle.
	${OPERATOR_SDK} cleanup amd-gpu-operator --namespace=${BUNDLE_NAMESPACE}

.PHONY: opm
OPM = ./bin/opm
opm:
	@if [ ! -f ${OPM} ]; then \
                set -e ;\
                echo "Downloading ${OPM}"; \
                mkdir -p $(dir ${OPM}) ;\
                curl -Lo ${OPM}.tar 'https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest-4.8/opm-linux.tar.gz'; \
		tar -C $(dir ${OPM}) -xzf ${OPM}.tar; \
                chmod +x ${OPM}; \
		rm -f ${OPM}.tar; \
        fi

.PHONY: index
index: opm
	${OPM} index add --bundles ${BUNDLE_IMG} --tag ${INDEX_IMG}

HELMIFY = $(shell pwd)/bin/helmify
.PHONY: helmify
helmify:
	@if [ ! -f "$(shell pwd)/bin/helmify" ]; then \
		echo "helmify not found. Downloading..."; \
		mkdir -p $(shell pwd)/bin/; \
		curl -Lo $(shell pwd)/bin/helmify.tar.gz https://github.com/arttor/helmify/releases/download/v0.4.13/helmify_Linux_x86_64.tar.gz; \
		tar -xzf $(shell pwd)/bin/helmify.tar.gz -C $(shell pwd)/bin; \
		chmod +x $(shell pwd)/bin/helmify; \
		rm $(shell pwd)/bin/helmify.tar.gz; \
	else \
		echo "helmify already exists."; \
	fi

.PHONY: helm-install
helm-install: ## Deploy Helm Charts.
	helm install -f helm-charts-k8s/values.yaml amd-gpu-operator ${GPU_OPERATOR_CHART} -n kube-amd-gpu --create-namespace ${SKIP_NFD_CMD} ${SKIP_KMM_CMD} ${SKIP_REMEDIATION_CONTROLLER_CMD} ${HELM_OC_CMD} ${SIM_ENABLE_CMD} ${SKIP_INSTALL_DEFAULT_CR_CMD}

.PHONY: helm-uninstall
helm-uninstall-k8s: ## Undeploy Helm Charts.
	echo "Deleting all device configs before uninstalling operator..."
	${KUBECTL_CMD} delete deviceconfigs.amd.com -n kube-amd-gpu --all
	${KUBECTL_CMD} delete remediationworkflowstatuses.amd.com -n kube-amd-gpu --all
	echo "Uninstalling operator..."
	helm uninstall amd-gpu-operator -n kube-amd-gpu

gen-kmm-charts:
ifdef JOB_ID
	@echo "Running in CI"
	$(KUSTOMIZE) build /ws/builder/kernel-module-management/config/default | $(HELMIFY) helm-charts-k8s/charts/kmm
else
	$(KUSTOMIZE) build $(shell pwd)/hack/kmmConfig/default | $(HELMIFY) helm-charts-k8s/charts/kmm
endif
	cp $(shell pwd)/hack/k8s-patch/k8s-kmm-patch/metadata-patch/Chart.yaml $(shell pwd)/helm-charts-k8s/charts/kmm/
	mkdir helm-charts-k8s/charts/kmm/crds
	@for file in $(K8S_KMM_CRD_YAML_FILES); do \
		helm template amd-gpu helm-charts-k8s/charts/kmm -s templates/$$file > helm-charts-k8s/charts/kmm/crds/$$file; \
		rm helm-charts-k8s/charts/kmm/templates/$$file; \
	done

gen-remediation-charts:
	$(KUSTOMIZE) build $(shell pwd)/hack/k8s-patch/k8s-remediation-patch | $(HELMIFY) helm-charts-k8s/charts/remediation-crds
	mkdir -p helm-charts-k8s/charts/remediation-crds/crds
	@for file in $(REMEDIATION_CRD_YAML_FILES); do \
		helm template amd-gpu helm-charts-k8s/charts/remediation-crds -s templates/$$file > helm-charts-k8s/charts/remediation-crds/crds/$$file; \
		rm helm-charts-k8s/charts/remediation-crds/templates/$$file; \
	done

cert-manager-install:
	helm repo add jetstack https://charts.jetstack.io --force-update
	helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.15.1 --set crds.enabled=true

cert-manager-uninstall: ## Undeploy cert-manager.
	helm uninstall cert-manager -n cert-manager
	${KUBECTL_CMD} delete crd issuers.cert-manager.io clusterissuers.cert-manager.io certificates.cert-manager.io certificaterequests.cert-manager.io orders.acme.cert-manager.io challenges.acme.cert-manager.io

.PHONY: clean-helm
clean-helm: ## Clean up generated helm chart folder ./helm-charts-k8s.
	rm -rf $(shell pwd)/helm-charts-k8s

copyrights:
	GOFLAGS=-mod=mod go run tools/build/copyright/main.go && ${MAKE} fmt && ./tools/build/check-local-files.sh