diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 038614b..35dbc7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,12 +7,14 @@ on: - "tests/**" - "schemas/**" - ".github/workflows/ci.yml" + - "!examples/**" push: branches: [main] paths: - "**.tf" - "tests/**" - "schemas/**" + - "!examples/**" jobs: validate: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53ab8d5..9e9c525 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,21 @@ on: push: branches: - main + paths: + - "**.tf" + - "modules/**" + - "tests/**" + - "schemas/**" + - ".github/workflows/test.yml" + - "!examples/**" pull_request: + paths: + - "**.tf" + - "modules/**" + - "tests/**" + - "schemas/**" + - ".github/workflows/test.yml" + - "!examples/**" permissions: contents: read diff --git a/examples/basic/.github/workflows/cd-starter.yml b/examples/basic/.github/workflows/cd-starter.yml new file mode 100644 index 0000000..ff04015 --- /dev/null +++ b/examples/basic/.github/workflows/cd-starter.yml @@ -0,0 +1,92 @@ +name: Deploy Terraform Changes + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + actions: write # require to upload artifacts + +concurrency: + group: terraform-deploy + cancel-in-progress: false + +jobs: + changes: + runs-on: ubuntu-latest + # Required permissions + permissions: + pull-requests: read + outputs: + # Expose matched filters as job 'packages' output variable + modules: ${{ steps.filter.outputs.changes }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + account: + - 'dbt_cloud/account/**' + initial_project: + - 'dbt_cloud/projects/initial/**' + terraform: + runs-on: ubuntu-latest + strategy: + matrix: + # TODO: set dynamically based on ${{ fromJSON(needs.changes.outputs.projects) }} + # but I'm not sure how to dynamically set the other variables in a way that feels this clean + include: + - module: account + - module: initial_project + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + + - name: Download Encrypted Artifact & Decrypt + uses: badgerhobbs/terraform-state@v2 + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: download + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + with: + terraform_version: 1.5.0 # Specify your Terraform version + + - name: Terraform Init + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + run: terraform init + + - name: Terraform Plan + id: plan + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + run: terraform plan -out=tfplan_${{ matrix.module }} -target=module.${{ matrix.module }} + + - name: Apply Terraform Changes + # if: github.ref == 'refs/heads/main' && contains(fromJSON(needs.changes.outputs.modules), matrix.module) + if: false + run: terraform apply -auto-approve tfplan_${{ matrix.module }} + env: + TF_VAR_dbt_account_id: ${{ secrets.TF_VAR_DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.TF_VAR_DBT_TOKEN }} + TF_VAR_dbt_host_url: ${{ secrets.TF_VAR_DBT_HOST_URL }} + + - name: Encrypt Artifact & Upload Encrypted Artifact + uses: badgerhobbs/terraform-state@v2 + if: ${{ contains(fromJSON(needs.changes.outputs.modules), matrix.module) }} + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: upload + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/examples/basic/.github/workflows/cd.yml b/examples/basic/.github/workflows/cd.yml index b37e063..9198aa5 100644 --- a/examples/basic/.github/workflows/cd.yml +++ b/examples/basic/.github/workflows/cd.yml @@ -5,7 +5,7 @@ # Settings > Environments > production > Required reviewers # # Uses the same secrets as ci.yml — see that file for the full list -# and setup instructions. +# and setup instructions, including AES_256_ENCRYPTION_KEY for state storage. name: CD — Terraform Apply @@ -18,6 +18,7 @@ on: permissions: contents: read + actions: write # required to upload state artifact jobs: apply: @@ -39,6 +40,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download Terraform State + uses: badgerhobbs/terraform-state@v2 + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: download + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true # OK to fail on first run — no artifact exists yet + - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: @@ -52,3 +62,12 @@ jobs: - name: Terraform Apply run: terraform apply -auto-approve tfplan + + - name: Upload Terraform State + uses: badgerhobbs/terraform-state@v2 + if: always() # upload even if apply partially succeeded, to preserve any changes + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: upload + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/examples/basic/.github/workflows/ci-starter.yml b/examples/basic/.github/workflows/ci-starter.yml new file mode 100644 index 0000000..d596497 --- /dev/null +++ b/examples/basic/.github/workflows/ci-starter.yml @@ -0,0 +1,45 @@ +name: Validate Terraform Changes + +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + terraform-validate: + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Download Encrypted Artifact & Decrypt + uses: badgerhobbs/terraform-state@v2 + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: download + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.5.0 # Specify your Terraform version + + - name: Check Terraform Format + run: terraform fmt -check -recursive + + - name: Terraform Init + run: terraform init + + - name: Validate Terraform + run: terraform validate + + - name: Terraform Plan + run: terraform plan + env: + TF_VAR_dbt_account_id: ${{ secrets.TF_VAR_DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.TF_VAR_DBT_TOKEN }} + TF_VAR_dbt_host_url: ${{ secrets.TF_VAR_DBT_HOST_URL }} \ No newline at end of file diff --git a/examples/basic/.github/workflows/ci.yml b/examples/basic/.github/workflows/ci.yml index 7f9d379..546a8e6 100644 --- a/examples/basic/.github/workflows/ci.yml +++ b/examples/basic/.github/workflows/ci.yml @@ -8,15 +8,18 @@ # DBT_PAT — personal access token (GitHub App integration only; # can be the same value as DBT_TOKEN otherwise) # ENVIRONMENT_CREDENTIALS — JSON blob, see .env.example for shape +# AES_256_ENCRYPTION_KEY — random 32-char string used to encrypt the state artifact +# generate with: openssl rand -hex 16 # # Optional secrets (omit if not used): # CONNECTION_CREDENTIALS — JSON blob for OAuth/service principal connections # LINEAGE_TOKENS — JSON blob for Tableau/Looker integrations # OAUTH_CLIENT_SECRETS — JSON blob for OAuth configurations # -# Remote state: configure a backend in main.tf (S3, GCS, Terraform Cloud, etc.) -# before using these workflows in production. Without it, state is local and -# lost between runs. +# State storage: these workflows store Terraform state as an encrypted GitHub Actions +# artifact — no backend configuration required to get started. For team use or +# production, migrate to a real backend (S3, GCS, Terraform Cloud, etc.) in main.tf +# and remove the badgerhobbs/terraform-state steps. name: CI — Terraform Plan @@ -50,11 +53,23 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Download Terraform State + uses: badgerhobbs/terraform-state@v2 + with: + encryption_key: ${{ secrets.AES_256_ENCRYPTION_KEY }} + operation: download + location: artifact + github_token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true # OK to fail on first run — no artifact exists yet + - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: "~1" + - name: Terraform Format Check + run: terraform fmt -check -recursive + - name: Terraform Init run: terraform init diff --git a/examples/basic/README.md b/examples/basic/README.md index ba08194..7992c81 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -57,8 +57,10 @@ terraform apply The `.github/workflows/` directory has ready-to-use GitHub Actions workflows: -- **`ci.yml`** — runs `terraform plan` on every PR and posts the plan as a comment -- **`cd.yml`** — runs `terraform apply` when changes merge to main +- **`ci.yml`** — formats check, validates, plans on every PR, and posts the plan as a comment +- **`cd.yml`** — applies on merge to main, with an optional approval gate via GitHub Environments + +Terraform state is stored as an encrypted artifact in GitHub Actions — no remote backend required to get started. Set these GitHub repository secrets (Settings > Secrets and variables > Actions): @@ -66,11 +68,12 @@ Set these GitHub repository secrets (Settings > Secrets and variables > Actions) DBT_ACCOUNT_ID numeric account ID DBT_TOKEN dbt Cloud service token ENVIRONMENT_CREDENTIALS JSON, e.g. {"analytics_prod":{"credential_type":"databricks","token":"dapi...","catalog":"main","schema":"analytics"}} +AES_256_ENCRYPTION_KEY random key used to encrypt the state artifact — generate with: openssl rand -hex 16 ``` Optional secrets (omit if not used): `DBT_PAT`, `CONNECTION_CREDENTIALS`, `LINEAGE_TOKENS`, `OAUTH_CLIENT_SECRETS`. -Add a [Terraform backend](https://developer.hashicorp.com/terraform/language/backend) to `main.tf` before enabling CD so state is stored remotely. +> **When to graduate off artifact state:** artifact state works well for a single user or small team. When you need concurrent runs, state locking, or a more durable audit trail, add a [Terraform backend](https://developer.hashicorp.com/terraform/language/backend) to `main.tf` and remove the `badgerhobbs/terraform-state` steps from both workflow files. ## Going further