refactor makefiles and usage examples

Makefile

@@ -1,67 +1,21 @@
 SHELL := /bin/bash
 
 include .env
+include includes/python.mk
+include includes/terraform.mk
 
 PROJECT_ID=$(PROJECT)-$(ENV)
 GOOGLE_CLOUD_PROJECT=$(PROJECT_ID)
 
 .EXPORT_ALL_VARIABLES:
+.DEFAULT_GOAL := help
+
 .PHONY: all test check quality run docker
 
 
 # -- bucket definitions
 DEPLOY_BUCKET   := $(PROJECT_ID)-gcs-deploy
 
-check: poetry-lock clean-code clean-coverage quality test poetry-build docker-build
-test: prepare-test poetry-test
-
-quality: prepare-quality poetry-quality
-run: prepare-run poetry-run
-docker: docker-build docker-run
-
-prepare-quality:
-	@poetry install --only nox,fmt,lint,type_check,docs
-
-poetry-quality:
-	@poetry run nox -s fmt_check
-	@poetry run nox -s lint
-	@poetry run nox -s type_check
-	@poetry run nox -s docs
-
-poetry-lock:
-	@poetry env use python3.9
-	@poetry install
-	@poetry lock --check
-
-clean-coverage:
-	@rm -f .coverage*
-	@rm -f coverage.xml
-
-prepare-test:
-	@poetry install --only nox
-
-poetry-test:
-	@poetry run nox -s test-3.9
-	@rm -f .coverage*
-	@rm -f coverage.xml
-	@poetry run nox -s test-3.10
-
-clean-code:
-	@poetry run isort .
-	@poetry run black .
-
-poetry-build:
-	@poetry build
-
-prepare-run:
-	@poetry install --only main
-
-poetry-run:
-	@poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload
-
-poetry-pulumi:
-	@poetry run python -m iac-pulumi.main
-
 docker-build:
 	@docker build --tag dbt-serverless .
 
@@ -87,6 +41,9 @@ docker-run:
 # This target will perform the complete setup of the current repository.
 # ---------------------------------------------------------------------------------------- #
 
+help: ## Displays the current message
+	@awk -F ':.*?##' '/^[^\t].+?:.*?.*?##/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF}' $(MAKEFILE_LIST)
+
 all: create-project create-bucket create-artifactregistry build deploy-app
 
 build: check build-app
@@ -157,117 +114,3 @@ deploy-app:
 	@echo "[$@] :: Pushing docker image"
 	@docker push $(REGION)-docker.pkg.dev/$(PROJECT_ID)/$(REPOSITORY_ID)/dbt-serverless:latest;
 	@echo "[$@] :: docker push is over."
-
-# ---------------------------------------------------------------------------------------- #
-# -- < IaC > --
-# ---------------------------------------------------------------------------------------- #
-# -- terraform variables declaration
-IAC_DIR = iac/
-DBT_DIR = $(DBT_PROJECT)/
-TF_DIR = $(IAC_DIR).terraform/
-TF_INIT  = $(TF_DIR)terraform.tfstate
-TF_VARS  = $(IAC_DIR)terraform.tfvars
-TF_PLAN  = $(IAC_DIR)tfplan
-TF_STATE = $(wildcard $(IAC_DIR)*.tfstate $(TF_DIR)*.tfstate)
-TF_FILES = $(wildcard $(IAC_DIR)*.tf)
-
-# -- this target will clean the local terraform infrastructure
-.PHONY: iac-clean
-iac-clean:
-	@echo "[$@] :: cleaning the infrastructure intermediary files"
-	@rm -fr $(TF_PLAN) $(TF_VARS);
-	@if [ ! -f $(IAC_DIR).iac-env ] || [ $$(cat $(IAC_DIR).iac-env || echo -n) != $(PROJECT_ID) ]; then \
-		echo "[$@] :: env has changed, removing also $(TF_DIR) and $(IAC_DIR).terraform.lock.hcl"; \
-		rm -rf $(TF_DIR) $(IAC_DIR).terraform.lock.hcl; \
-	fi;
-
-	@echo "[$@] :: infrastructure cleaning DONE"
-
-# -- this target will initialize the terraform initialization
-.PHONY: iac-init
-iac-init: $(TF_INIT) # provided for convenience
-$(TF_INIT):
-	@set -euo pipefail; \
-	if [ ! -d $(TF_DIR) ]; then \
-		function remove_me() { if (( $$? != 0 )); then rm -fr $(TF_DIR); fi; }; \
-		trap remove_me EXIT; \
-		echo "[iac-init] :: initializing terraform"; \
-		echo "$(PROJECT_ID)" > $(IAC_DIR).iac-env; \
-		cd $(IAC_DIR) && terraform init \
-			-backend-config=bucket=$(DEPLOY_BUCKET) \
-			-backend-config=prefix=terraform-state/$(ENV) \
-			-input=false; \
-	else \
-		echo "[iac-init] :: terraform already initialized"; \
-	fi;
-
-# -- internal definition for easing changes
-define HERE_TF_VARS
-project             = "$(PROJECT_ID)"
-zone                = "$(ZONE)"
-region              = "$(REGION)"
-env       		    = "$(ENV)"
-repository_id		= "$(REPOSITORY_ID)"
-github_owner        = "$(GITHUB_OWNER)"
-github_repo         = "$(GITHUB_REPO)"
-github_token        = "$(GITHUB_TOKEN)"
-pypi_token          = "$(PYPI_TOKEN)"
-codecov_token       = "$(CODECOV_TOKEN)"
-endef
-export HERE_TF_VARS
-
-# -- this target will create the terraform.tfvars file
-.PHONY: iac-prepare
-iac-prepare: $(TF_VARS) # provided for convenience
-$(TF_VARS): $(TF_INIT)
-	@echo "[iac-prepare] :: generation of $(TF_VARS) file";
-	@echo "$$HERE_TF_VARS" > $(TF_VARS);
-	@echo "[iac-prepare] :: generation of $(TF_VARS) file DONE.";
-
-# -- this target will create the tfplan file whenever the variables file and any *.tf
-# file have changed
-.PHONY: iac-plan iac-plan-clean
-iac-plan-clean:
-	@rm -f $(TF_PLAN)
-
-iac-plan: iac-plan-clean $(TF_PLAN) # provided for convenience
-$(TF_PLAN): $(TF_VARS) $(TF_FILES)
-	@echo "[iac-plan] :: planning the iac in $(PROJECT_ID)";
-	@set -euo pipefail; \
-	cd $(IAC_DIR) && terraform plan \
-		-var-file $(shell basename $(TF_VARS)) \
-		-out=$(shell basename $(TF_PLAN));
-	@echo "[iac-plan] :: planning the iac for $(PROJECT_ID) DONE.";
-
-# -- this target will only trigger the iac of the current parent
-.PHONY: iac-validate
-iac-validate:
-	@echo "[$@] :: validating the infrastructure for $(PROJECT_ID)"
-	@set -euo pipefail; \
-	cd $(IAC_DIR) && terraform validate;
-	@echo "[$@] :: infrastructure validated on $(PROJECT_ID)"
-
-# -- this target will only trigger the iac of the current parent
-.PHONY: iac-sec
-iac-sec:
-	@echo "[$@] :: checking the infrastructure security for $(PROJECT_ID)"
-	@tfsec .
-	@echo "[$@] :: security checked on $(PROJECT_ID)"
-
-# -- this target will only trigger the iac of the current parent
-.PHONY: iac-version
-iac-version:
-	@cd $(IAC_DIR) && terraform -version
-
-# -- this target will only trigger the iac of the current parent
-.PHONY: iac-deploy
-iac-deploy: iac-clean $(TF_PLAN)
-	@echo "[$@] :: applying the infrastructure for $(PROJECT_ID)"
-	@set -euo pipefail; \
-	cd $(IAC_DIR) && terraform apply -auto-approve -input=false $(shell basename $(TF_PLAN));
-	@echo "[$@] :: infrastructure applied on $(PROJECT_ID)"
-
-# -- this target re-initializes the git working tree removing untracked and ignored files
-.PHONY: reinit
-reinit:
-	@rm -rf $(IAC_DIR).terraform* $(IAC_DIR)terraform.tfstate* $(IAC_DIR)tfplan

README.md

@@ -52,25 +52,7 @@ poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload
 
 
 ### Docker deployment
-Simple docker image to build dbt-serverless for local or cloud run testing (for example).
-
-```docker
-ARG build_for=linux/amd64
-
-FROM --platform=$build_for python:3.10-slim-bullseye
-
-ARG DBT_PROJECT
-ARG PROFILES_DIR
-
-WORKDIR /usr/app
-
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir dbt-serverless
-
-COPY ${DBT_PROJECT}/ ${PROFILES_DIR}/profiles.yml ${DBT_PROJECT}/
-
-ENTRYPOINT ["uvicorn", "dbt_serverless.main:app", "--host", "0.0.0.0", "--port", "8080"]
-```
+Simple docker image to build dbt-serverless for local or cloud run testing: [example Dockerfile](examples/Dockerfile.dbt).
 
 If you're not on a Google product (like Cloud Run), you will need to specify google creds at docker runtime.
 
@@ -80,6 +62,6 @@ For example you can add these cli parameters at runtime, if you're testing and d
     -v /gcp/config/logs \
     --env CLOUDSDK_CONFIG=/gcp/config \
     --env GOOGLE_APPLICATION_CREDENTIALS=/gcp/config/application_default_credentials.json \
-    --env GOOGLE_CLOUD_PROJECT=$(PROJECT_ID) \
+    --env GOOGLE_CLOUD_PROJECT=$(YOUR_PROJECT_ID) \
 ```
 

dbt_serverless/main.py

@@ -14,6 +14,7 @@
 
 app = FastAPI()
 
+PROJECT = environ.get("GOOGLE_CLOUD_PROJECT")
 DBT_PROJECT = environ.get("DBT_PROJECT")
 DBT_PROFILES_DIR = environ.get("DBT_PROFILES_DIR")
 
@@ -79,6 +80,6 @@ async def docs() -> str:
 
     logging.info("Uploading the file to GCS for static website serving")
 
-    upload_blob("dbt-static-docs-bucket", merged_content_path, "index_merged.html")
+    upload_blob(f"{PROJECT}-dbt-docs", merged_content_path, "index_merged.html")
 
     return "https://storage.cloud.google.com/dbt-static-docs-bucket/index_merged.html"

examples/.env.template

@@ -0,0 +1,7 @@
+GOOGLE_CLOUD_PROJECT=dbt-serverless
+REGION=europe-west9
+REPOSITORY_ID=gcr_artifact_repo_name
+
+DBT_PROJECT=dbt_project
+DBT_DATASET=warehouse
+DBT_PROFILES_DIR=.

examples/Dockerfile.dbt

@@ -0,0 +1,18 @@
+FROM ghcr.io/dbt-labs/dbt-bigquery:1.3.0
+
+ARG DBT_PROJECT
+ARG DBT_DATASET
+ARG DBT_PROFILES_DIR
+
+ENV DBT_PROJECT=${DBT_PROJECT}
+ENV DBT_DATASET=${DBT_DATASET}
+ENV DBT_PROFILES_DIR=${DBT_PROFILES_DIR}
+
+WORKDIR / usr/app
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir dbt-serverless
+
+COPY ${DBT_PROJECT}/ ${DBT_PROFILES_DIR}/profiles.yml ${DBT_PROJECT}/
+
+ENTRYPOINT ["uvicorn", "dbt_serverless.main:app", "--host", "0.0.0.0", "--port", "8080"]

examples/README.md

@@ -0,0 +1,60 @@
+Dockerfiles, Terraform and Pulumi files to test and deploy your dbt-serverless in no time.
+
+## Docker commands to test your own image and dbt project locally
+
+In your project, you could use this `Dockerfile.dbt` file and with few environment variables set (see the `.env.template` file to fill your own `.env` file), you would only need to do the following :
+
+```bash
+export $(grep -v '^#' .env | xargs)
+```
+
+```bash
+docker build . \
+    -t $(DBT_PROJECT) \
+    -f dbt.Dockerfile \
+    --build-arg DBT_PROJECT=$(DBT_PROJECT) \
+    --build-arg DBT_DATASET=$(DBT_DATASET) \
+    --build-arg DBT_PROFILES_DIR=$(DBT_PROJECT)
+```
+
+```bash
+docker run \
+		--rm \
+		--interactive \
+		--tty \
+		-p 8080:8080 \
+		-v "$(HOME)/.config/gcloud:/gcp/config:ro" \
+		-v /gcp/config/logs \
+		--env CLOUDSDK_CONFIG=/gcp/config \
+		--env GOOGLE_APPLICATION_CREDENTIALS=/gcp/config/application_default_credentials.json \
+		--env GOOGLE_CLOUD_PROJECT=$(GOOGLE_CLOUD_PROJECT) \
+		$(DBT_PROJECT)
+```
+
+## Deploy it on GCP
+
+Here we provide a `main.tf` example to deploy your dbt-serverless in an already setup GCP Project with APIs enabled and a repository in Artifact Registry. You're just gonna need 3 terraform variables (but you might already use them in your project):
+- var.project
+- var.region
+- var.repository_id
+
+If it's not setup, you could just git clone the entire repository and use make commands to deploy the iac from scratch. In this case, look for the `.env.template` at the root directory and `make all`.
+
+Anyway!
+
+Build the image and tag it:
+
+```bash
+docker build . \
+    -f Dockerfile.dbt \
+    --tag $REGION-docker.pkg.dev/$GOOGLE_CLOUD_PROJECT/$REPOSITORY_ID/dbt-serverless:latest \
+    --build-arg DBT_PROJECT=$DBT_PROJECT \
+    --build-arg DBT_DATASET=$DBT_DATASET \
+    --build-arg DBT_PROFILES_DIR=$DBT_PROJECT
+```
+
+Push it to Google Cloud Artifact Registry repository:
+
+```bash
+docker push $REGION-docker.pkg.dev/$GOOGLE_CLOUD_PROJECT/$REPOSITORY_ID/dbt-serverless:latest
+```

examples/main.tf

@@ -0,0 +1,56 @@
+locals {
+  dbt_roles = toset(["bigquery.dataEditor", "bigquery.user"])
+}
+resource "google_service_account" "dbt_sa" {
+  account_id   = "dbt-runner"
+  project      = var.project
+  display_name = "dbt Service Account"
+  description  = "dbt service account"
+}
+
+resource "google_project_iam_member" "sa_iam_dbt" {
+  for_each = local.dbt_roles
+  project  = var.project
+  role     = "roles/${each.key}"
+  member   = "serviceAccount:${google_service_account.dbt_sa.email}"
+}
+
+resource "google_cloud_run_service" "dbt_serverless" {
+  provider = google-beta
+  project  = var.project
+  location = var.region
+  name     = "dbt-serverless"
+
+  template {
+    metadata {
+      annotations = {
+        "run.googleapis.com/execution-environment" : "gen2"
+      }
+    }
+    spec {
+      containers {
+        image = "${var.region}-docker.pkg.dev/${var.project}/${var.repository_id}/dbt-serverless:latest"
+        env {
+          name = "GOOGLE_CLOUD_PROJECT"
+          value = var.project
+        }
+        resources {
+          limits = {
+            "cpu"  = "1000m"
+            memory = "2048Mi"
+          }
+        }
+      }
+      service_account_name = google_service_account.dbt_sa.email
+    }
+  }
+
+  traffic {
+    percent         = 100
+    latest_revision = true
+  }
+}
+
+output "dbt_serverless_url" {
+  value = google_cloud_run_service.dbt_serverless.status[0].url
+}