diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..d6abdc3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +## Summary + + + +## Type of change + +- [ ] Bug fix +- [ ] New feature / resource support +- [ ] Refactoring (no behavior change) +- [ ] Documentation +- [ ] CI / tooling + +## Schema changes + +- [ ] This PR modifies the YAML schema — `schemas/v1.json` and `docs/configuration/yaml-schema.md` updated +- [ ] No schema changes + +## Checklist + +- [ ] `make fmt` passes +- [ ] `make test` passes +- [ ] `make lint` passes +- [ ] `CHANGELOG.md` updated under `[Unreleased]` +- [ ] Docs updated if behavior changed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..038614b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + pull_request: + paths: + - "**.tf" + - "tests/**" + - "schemas/**" + - ".github/workflows/ci.yml" + push: + branches: [main] + paths: + - "**.tf" + - "tests/**" + - "schemas/**" + +jobs: + validate: + name: Validate + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version_file: .terraform-version + + - name: Cache Terraform providers + uses: actions/cache@v4 + with: + path: .terraform + key: terraform-${{ hashFiles('.terraform.lock.hcl') }} + restore-keys: terraform- + + - name: Format check + run: terraform fmt -check -recursive + + - name: Init + run: terraform init -backend=false + + - name: Set up tflint + uses: terraform-linters/setup-tflint@v4 + with: + tflint_version: latest + + - name: Lint + run: | + tflint --init + tflint --recursive + + - name: Test (mock providers) + run: terraform test diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..689a080 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,50 @@ +# integration.yml — runs Go/Terratest integration tests against a real dbt Cloud account. +# Triggered manually only to avoid consuming sandbox API quota on every PR. +# Requires secrets: DBT_CLOUD_ACCOUNT_ID, DBT_CLOUD_TOKEN + +name: Integration Tests + +on: + workflow_dispatch: + inputs: + host_url: + description: "dbt Cloud host URL (leave blank for https://cloud.getdbt.com)" + required: false + default: "" + +permissions: + contents: read + +jobs: + integration: + name: Integration Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: "1.20" + cache-dependency-path: test/go.sum + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version_file: .terraform-version + terraform_wrapper: false + + - name: Go mod download + working-directory: test + run: go mod download + + - name: Run integration tests + working-directory: test + env: + RUN_INTEGRATION_TESTS: "1" + DBT_CLOUD_ACCOUNT_ID: ${{ secrets.DBT_CLOUD_ACCOUNT_ID }} + DBT_CLOUD_TOKEN: ${{ secrets.DBT_CLOUD_TOKEN }} + DBT_CLOUD_HOST_URL: ${{ github.event.inputs.host_url }} + run: go test -v -timeout 30m -run Integration ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c3a8cf0 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,35 @@ +# release.yml — triggered by a version tag (e.g. git tag v1.2.3 && git push --tags). +# +# Produces two release assets: +# starter.tar.gz — the examples/basic/ directory, ready to extract +# install.sh — bootstrap script that downloads starter.tar.gz + +name: Release + +on: + push: + tags: + - 'v*' + +permissions: + contents: write + +jobs: + release: + name: Create GitHub Release + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Package starter + run: tar -czf starter.tar.gz -C examples/basic . + + - name: Create release with assets + uses: softprops/action-gh-release@v2 + with: + files: | + install.sh + starter.tar.gz + generate_release_notes: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..53ab8d5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,61 @@ +# test.yml — runs Terraform unit tests on every pull request and push to main. +# Uses mock providers so no dbt Cloud credentials are required. + +name: Test + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +jobs: + root-tests: + name: Root Module Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version_file: .terraform-version + + - name: Terraform Init + run: terraform init -backend=false + + - name: Terraform Test + run: terraform test -verbose + + module-tests: + name: Module Tests (${{ matrix.module }}) + runs-on: ubuntu-latest + strategy: + matrix: + module: + - project + - environments + - jobs + - credentials + - repository + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version_file: .terraform-version + + - name: Terraform Init + working-directory: modules/${{ matrix.module }} + run: terraform init -backend=false + + - name: Terraform Test + working-directory: modules/${{ matrix.module }} + run: terraform test -verbose diff --git a/.gitignore b/.gitignore index 6349e36..a3bc1fd 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc + +*.DS_Store \ No newline at end of file diff --git a/.terraform-version b/.terraform-version new file mode 100644 index 0000000..9be7846 --- /dev/null +++ b/.terraform-version @@ -0,0 +1 @@ +1.14.8 diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index c9a7a35..8dc8cba 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,42 +2,26 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/dbt-labs/dbtcloud" { - version = "0.3.26" - constraints = "~> 0.3" + version = "1.8.2" + constraints = "~> 1.8" hashes = [ - "h1:ZR4O8v7khn5upq1M6LM6i8XDkD02Jhrq23deYFCmMW0=", - "zh:06990d8a1372f9dc8a50b0b99f9464e6b344b393198dbc0333c3af2177b35e09", - "zh:238ec8e30e68029e4c3283305b7a038103399a710f8c2572d3fc93dcf78e40e5", - "zh:32b7f9bf8b451472d0b5388aba775622e17c32acc186dc2c529aa1f3047452e4", - "zh:676b78547a25fa3b9705c6705309d07962681bb1c638c120850529341050850f", - "zh:7d676bdd50e781a037ed76032d1ea21f5acd2d88b6a734a5c05c644ad8896f5e", - "zh:809e5adc57b19cde1b2bc05ef65fa98d61c0a1f2e7ac4a4367a8fb4fb37e1143", - "zh:9e96c575aa878e5fa2d1165c1a35025329bf98a18453a19d35e5c453a0c27b35", - "zh:a5b26815d7789fa66ec1fe60a0d0f1cf3a4026398f9a83a7be2cd608980fe5c1", - "zh:b1994d0b1a41a4c5b4740be2e8e621bda17c36fbc68bd3872b96bbeee21fcb48", - "zh:b89f73a1c5250b2d947862541f65f6f7f5cd27e1298b85e58a3a2a90cc716b9b", - "zh:bc763478462899b11a3b7f1f5033340476d4c7be5f32c461d213f441c8f2b0ae", - "zh:c0bef1bd362e70819d87afda4660234e2f3f2f5aa6094fc9188e739c6f6e0f35", - "zh:e1dd61914a1977f94cad8bb1677b0f0ac2a238d08ac7056a21bb3e3fdb32e504", - "zh:fb94f72296707dcac9e9c2507ad46c2da71e6282f736ade10b2906b00fcdda7f", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.4" - hashes = [ - "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", - "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", - "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", - "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", - "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", - "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", - "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", - "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", - "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", - "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", - "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", ] } diff --git a/.terraformrc.test b/.terraformrc.test new file mode 100644 index 0000000..677f5cc --- /dev/null +++ b/.terraformrc.test @@ -0,0 +1,17 @@ +# Terraform CLI config for running tests locally. +# Removes the dev_overrides that point to a local provider build, +# allowing terraform test to use the provider from .terraform/providers/. +# +# Usage: +# TF_CLI_CONFIG_FILE=.terraformrc.test terraform test +# TF_CLI_CONFIG_FILE=/path/to/repo/.terraformrc.test terraform -chdir=modules/project test + +provider_installation { + filesystem_mirror { + path = "/Users/tylerrouze/dev/terraform-dbtcloud-yaml/.terraform/providers" + include = ["registry.terraform.io/dbt-labs/dbtcloud"] + } + direct { + exclude = ["registry.terraform.io/dbt-labs/dbtcloud"] + } +} diff --git a/.tflint.hcl b/.tflint.hcl index 3135be5..921f222 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -1,7 +1,3 @@ -plugin "aws" { - enabled = false -} - rule "terraform_required_version" { enabled = true } @@ -21,6 +17,12 @@ rule "terraform_documented_outputs" { rule "terraform_naming_convention" { enabled = true format = "snake_case" + + # Allow a leading underscore for locals — used as a convention for + # "private" intermediate locals in validation.tf. + locals { + custom = "^_?[a-z][a-z0-9_]*$" + } } rule "terraform_comment_syntax" { @@ -38,3 +40,7 @@ rule "terraform_unused_required_providers" { rule "terraform_standard_module_structure" { enabled = true } + +rule "terraform_module_pinned_source" { + enabled = false +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6557e0e..4e031d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,9 +42,9 @@ Feature requests are welcome! Please include: 3. **Test your changes** ```bash - terraform init - terraform validate - terraform plan + make fmt # auto-format + make test # run tests with mock providers (no credentials needed) + make lint # run tflint ``` 4. **Commit with clear messages** @@ -61,22 +61,31 @@ Feature requests are welcome! Please include: ### Prerequisites -- Terraform >= 1.0 +- Terraform >= 1.7 (use [tfenv](https://github.com/tfutils/tfenv) or [asdf](https://asdf-vm.com/) — `.terraform-version` is provided) +- [tflint](https://github.com/terraform-linters/tflint) - Git -- A dbt Cloud account for testing ### Local Development ```bash -git clone https://github.com/your-username/dbt-terraform-modules-yaml.git -cd dbt-terraform-modules-yaml +git clone https://github.com/your-username/terraform-dbtcloud-yaml.git +cd terraform-dbtcloud-yaml -# Set up your test environment -cp examples/basic/terraform.tfvars.example terraform.tfvars -# Edit terraform.tfvars with your credentials +terraform init -backend=false +``` + +### Available Make Targets + +Run `make help` to see all targets: -terraform init -terraform plan +``` +make fmt # Auto-format all Terraform files +make fmt-check # Check formatting without modifying (used in CI) +make lint # Run tflint on all modules +make validate # Run terraform validate +make test # Run terraform test with mock providers (no credentials needed) +make docs # Regenerate terraform-docs for all modules +make pre-commit # Run all pre-commit hooks on staged files ``` ## Testing @@ -84,14 +93,9 @@ terraform plan Before submitting a PR: ```bash -# Validate syntax -terraform validate - -# Format check -terraform fmt -check -recursive - -# Plan to check for errors -terraform plan +make fmt-check # verify formatting +make test # runs all tests against mock providers — no dbt Cloud credentials needed +make lint # check for linting issues ``` ## Documentation diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0aa0783 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +.DEFAULT_GOAL := help + +.PHONY: fmt fmt-check lint validate test test-integration docs pre-commit pre-commit-all help + +fmt: ## Auto-format all Terraform files + terraform fmt -recursive + +fmt-check: ## Check formatting without modifying (used in CI) + terraform fmt -check -recursive + +lint: ## Run tflint on all modules + tflint --recursive + +validate: ## Run terraform validate (requires terraform init) + terraform init -backend=false -reconfigure + terraform validate + +test: ## Run terraform test with mock providers (no credentials needed) + terraform test + +test-integration: ## Run Go integration tests against a real dbt Cloud account (requires DBT_CLOUD_ACCOUNT_ID and DBT_CLOUD_TOKEN) + cd test && RUN_INTEGRATION_TESTS=1 go test -v -timeout 30m -run Integration ./... + +docs: ## Regenerate terraform-docs for all modules + pre-commit run terraform-docs-go --all-files + +pre-commit: ## Run all pre-commit hooks on staged files + pre-commit run + +pre-commit-all: ## Run all pre-commit hooks on all files + pre-commit run --all-files + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index d0d167b..1ff6f98 100644 --- a/README.md +++ b/README.md @@ -1,162 +1,290 @@ # terraform-dbtcloud-yaml [![Terraform Version](https://img.shields.io/badge/terraform-%3E%3D%201.0-blue?logo=terraform)](https://www.terraform.io) -[![dbt Cloud Provider](https://img.shields.io/badge/dbt--cloud--provider-%3E%3D%201.3-blue)](https://registry.terraform.io/providers/dbt-labs/dbtcloud/latest) -[![License](https://img.shields.io/badge/license-MIT-green)](./LICENSE) +[![dbt Cloud Provider](https://img.shields.io/badge/dbt--cloud--provider-%3E%3D%201.8-blue)](https://registry.terraform.io/providers/dbt-labs/dbtcloud/latest) +[![License](https://img.shields.io/badge/license-Apache%202.0-green)](./LICENSE) -Manage your entire dbt Cloud setup with infrastructure-as-code using Terraform and YAML. Define projects, repositories, environments, credentials, and jobs in a single, human-readable YAML file. +## Get started in 60 seconds -## Why This Project Exists - -Setting up dbt Cloud via Terraform requires writing complex HCL for every resource. This module simplifies that by letting you define your infrastructure in YAML - the language data teams already know. No need to learn Terraform syntax just to configure dbt. +```bash +curl -fsSL https://github.com/trouze/terraform-dbtcloud-yaml/releases/latest/download/install.sh | bash +``` -**Benefits:** -- ✅ **YAML-based configuration** - intuitive for data engineers -- ✅ **Infrastructure as Code** - version control your dbt setup -- ✅ **Multi-provider Git support** - GitHub, GitLab, Azure DevOps, SSH -- ✅ **Complete resource management** - projects, repos, environments, credentials, jobs -- ✅ **Environment variable management** - set dbt variables alongside infrastructure -- ✅ **Reusable modules** - standardize your dbt deployments +This downloads the [examples/basic/](examples/basic/) starter into `./my-dbt-cloud`. No npm, no git magic — just curl and tar. To use a different directory name: `curl -fsSL ... | bash -s -- my-project`. -## Quick Start +Then: -See [examples/README.md](examples/README.md) for a complete walkthrough. Here's the 30-second version: +```bash +cd my-dbt-platform +cp .env.example .env # fill in your dbt Cloud credentials +# edit dbt-config.yml # replace YOUR_ placeholders with your warehouse details +source .env && terraform init && terraform apply +``` -1. **Copy the basic example** - ```bash - cp -r examples/basic my-dbt-setup - cd my-dbt-setup - ``` +See [examples/basic/README.md](examples/basic/README.md) for a full walkthrough. -2. **Create your .env file** - ```bash - cp .env.example .env - # Edit with your dbt Cloud credentials - ``` +--- -3. **Deploy** - ```bash - source .env - terraform init && terraform plan && terraform apply - ``` +## Why this exists -## Features +dbt developers already speak YAML — `dbt_project.yml`, `schema.yml`, `sources.yml`. But managing the infrastructure that runs dbt (projects, environments, jobs, credentials) typically means writing Terraform HCL, a different language with a steep learning curve that most data teams don't have time to acquire. -### YAML Configuration +This module flips that. You write one YAML file that describes your entire dbt Cloud setup. Terraform reads it and manages everything. Your data team owns the config. Your platform team (or a CI pipeline) owns the apply. No HCL required on day one. -Define everything in one file: +--- -```yaml -project: - name: "my-dbt-project" - repository: - remote_url: "https://github.com/myorg/myrepo.git" - git_clone_strategy: "github_app" - github_installation_id: 123456 - - environments: - - name: "dev" - type: "development" - connection_id: 1 - jobs: - - name: "daily_run" - execute_steps: - - "dbt run" - triggers: - schedule: true - schedule_hours: [6] +## Quickstart + +**1. Create `main.tf`** + +```hcl +terraform { + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +provider "dbtcloud" { + account_id = var.dbt_account_id + token = var.dbt_token + host_url = var.dbt_host_url +} + +module "dbt_cloud" { + source = "github.com/trouze/terraform-dbtcloud-yaml" + + dbt_account_id = var.dbt_account_id + dbt_token = var.dbt_token + dbt_host_url = var.dbt_host_url + yaml_file = "${path.module}/dbt-config.yml" + + # Sensitive credentials passed separately (never in YAML) + token_map = var.token_map + environment_credentials = var.environment_credentials +} ``` -### Multi-Provider Git Support - -Automatically configures your Git provider: -- **GitHub** with GitHub App -- **GitLab** with Deploy Token -- **Azure DevOps** with Azure AD -- **SSH** Deploy Key (universal) +**2. Create `dbt-config.yml`** -### Secure Credentials +```yaml +projects: + - name: Analytics + key: analytics + + repository: + remote_url: "your-org/your-repo" + github_installation_id: 1234567 # GitHub App installation ID + + environments: + - name: Production + key: prod + connection_key: databricks_prod # references global_connections key below + deployment_type: production + type: deployment + custom_branch: main + protected: true + credential: + credential_type: databricks + catalog: main + schema: analytics + + - name: Development + key: dev + connection_key: databricks_prod + type: development + + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] + schedule_hours: [6] + +global_connections: + - name: Databricks Production + key: databricks_prod + type: databricks + host: adb-1234567890.1.azuredatabricks.net + http_path: /sql/1.0/warehouses/abc123 + catalog: main +``` -- Keep secrets in `.env` or GitHub Secrets -- Never commit sensitive values -- Support for database credentials as environment variables +**3. Create `terraform.tfvars`** + +```hcl +dbt_account_id = 12345 +dbt_token = "dbt_your_api_token" +dbt_host_url = "https://cloud.getdbt.com" + +# Databricks token for the Production environment credential +# Key format: "{project_key}_{env_key}" +environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi..." + catalog = "main" + schema = "analytics" + } +} +``` -## Use Cases +**4. Deploy** -### Single dbt Project ```bash +terraform init +terraform plan terraform apply ``` -### Multiple dbt Projects (parallel CI/CD) -```bash -# Run each project separately -for config in configs/*.yml; do - terraform plan -var="yaml_file_path=$config" -done +That's it. Or skip the manual setup entirely — use the [60-second clone](#get-started-in-60-seconds) at the top. + +--- + +## How credentials work + +Sensitive values are never in the YAML file. They're passed as Terraform variables and matched to YAML resources by key. + +| Variable | Key format | Matches | +|---|---|---| +| `token_map` | `"my_token_name"` | `credential.token_name` in YAML (Databricks legacy) | +| `environment_credentials` | `"project_key_env_key"` | Environment credential by composite key | +| `connection_credentials` | `"connection_key"` | `global_connections[].key` in YAML | +| `lineage_tokens` | `"project_key_integration_key"` | `lineage_integrations[].key` composite | +| `oauth_client_secrets` | `"oauth_config_key"` | `oauth_configurations[].key` in YAML | + +The composite key for `environment_credentials` uses underscores: a project with `key: analytics` and an environment with `key: prod` → `analytics_prod`. + +--- + +## What you can manage + +**Account-level** +- `account_features` — advanced CI, partial parsing, repo caching flags +- `global_connections` — shared warehouse connections (Databricks, Snowflake, BigQuery, Postgres, Redshift) +- `service_tokens` — API tokens with scoped permissions +- `groups` — user groups with project/account permissions +- `user_groups` — user-to-group assignments +- `notifications` — email, Slack, PagerDuty, webhook alerts +- `oauth_configurations` — OAuth provider configs +- `ip_restrictions` — IP allowlist/denylist rules + +**Per-project** +- `repository` — Git integration (GitHub App, GitLab deploy token, Azure DevOps, SSH) +- `environments` — deployment and development environments +- `credentials` — warehouse credentials (14 types: Databricks, Snowflake password/keypair, BigQuery, Postgres, Redshift, Athena, Fabric, Synapse, Starburst, Spark, Teradata) +- `jobs` — scheduled, CI, merge, and on-demand jobs +- `environment_variables` — project and environment-level dbt vars +- `extended_attributes` — connection-level overrides per environment +- `profiles` — links connection + credential + extended attributes +- `lineage_integrations` — Tableau/Looker lineage config +- `artefacts` — docs job and freshness job links +- `semantic_layer` — semantic layer configuration + +--- + +## Resource protection + +Set `protected: true` on any resource to prevent accidental deletion: + +```yaml +global_connections: + - name: Databricks Production + key: databricks_prod + protected: true # terraform destroy will be blocked for this resource + ... + +projects: + - name: Analytics + key: analytics + protected: true + environments: + - name: Production + key: prod + protected: true + ... ``` -### CI/CD Pipeline +Protection uses `lifecycle { prevent_destroy = true }` under the hood, which means the resource appears in Terraform state with a `protected_` prefix in the resource name (e.g., `dbtcloud_project.protected_projects["analytics"]`). + +--- + +## Multi-project support + +Put all your dbt Cloud projects in a single YAML file under `projects:`: + ```yaml -# GitHub Actions example -- env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - run: terraform apply +projects: + - name: Finance Analytics + key: finance + ... + + - name: Marketing Analytics + key: marketing + ... ``` -## Requirements +Or keep separate YAML files per team and apply them independently: -- Terraform >= 1.0 -- dbt Cloud account -- dbt Cloud API token -- Git repository for your dbt models +```bash +terraform apply -var="yaml_file=./configs/finance.yml" +terraform apply -var="yaml_file=./configs/marketing.yml" +``` -## Documentation +**Backward compatibility:** If your existing YAML uses the singular `project:` key, it still works — the module automatically wraps it in a list. -- [Examples](examples/README.md) - Get started in 5 minutes -- [Module Details](docs/) - Complete configuration reference -- [Best Practices](docs/BEST_PRACTICES.md) - Recommended patterns +--- -## Best Practices +## Job scheduling -### Security -- ✅ **Never commit credentials** - use `.env` (local) or GitHub Secrets (CI/CD) -- ✅ **Use service principals** - create dedicated dbt Cloud API tokens -- ✅ **Limit token scope** - only grant necessary permissions -- ✅ **Rotate regularly** - refresh tokens periodically +Three mutually exclusive schedule modes (cron takes precedence if multiple are set): -### Organization -- ✅ **Version control your YAML** - track all infrastructure changes -- ✅ **Use environments** - separate dev/staging/prod configs -- ✅ **Document custom settings** - add comments in your YAML -- ✅ **Test before deploying** - use `terraform plan` first +```yaml +# Cron expression +schedule_cron: "0 6 * * 1-5" -### CI/CD -- ✅ **Run in parallel** - use matrix builds for multiple projects -- ✅ **Store secrets** - use platform-native secret management -- ✅ **Automate deployments** - trigger on config changes -- ✅ **Require approvals** - review plans before applying +# Hours-based (every N hours) +schedule_interval: 2 -## Contributing +# Specific days and hours +schedule_type: days_of_week +schedule_days: [1, 2, 3, 4, 5] # 0=Sun, 6=Sat +schedule_hours: [6, 18] # UTC +``` -Contributions are welcome! Please: +--- -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Submit a pull request +## Requirements -## Support +- Terraform >= 1.0 +- dbt Cloud account with API token +- dbt Labs Terraform provider >= 1.8 -- 📖 **Documentation** - Check [docs/](docs/) for detailed guides -- 🐛 **Issues** - Report bugs or request features on GitHub -- 💬 **Discussions** - Share ideas and best practices +--- -## License +## Documentation -This project is licensed under Apache License 2.0. See [LICENSE](LICENSE) for details. +- [examples/basic/](examples/basic/) — clone-and-go starter with step-by-step README +- [docs/configuration/yaml-schema.md](docs/configuration/yaml-schema.md) — full YAML field reference +- [docs/guides/cicd.md](docs/guides/cicd.md) — CI/CD setup (GitHub Actions, etc.) +- [docs/guides/best-practices.md](docs/guides/best-practices.md) — patterns and recommendations --- -**Ready to manage your dbt Cloud with code?** Start with the [examples](examples/README.md)! +## Contributing + +Contributions welcome. Please open an issue before large changes to align on approach. + +## License + +Apache License 2.0. See [LICENSE](LICENSE). diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md index 6aa5e82..fd546ee 100644 --- a/docs/configuration/environment-variables.md +++ b/docs/configuration/environment-variables.md @@ -4,26 +4,27 @@ Learn how to manage credentials and configuration using environment variables. ## Overview -This module uses Terraform's `TF_VAR_` prefix pattern to pass sensitive values without hardcoding them in `.tf` files. This approach works seamlessly in both local development (`.env` files) and CI/CD (GitHub Secrets, etc.). +This module uses Terraform's `TF_VAR_` prefix pattern to pass sensitive values without hardcoding them in `.tf` files. This approach works seamlessly in both local development (`.env` files) and CI/CD (GitHub Secrets, GitLab masked variables, Azure key vault, etc.). ## Required Variables -These variables must be set for the module to function: - | Variable | Description | Example | |----------|-------------|---------| -| `TF_VAR_dbt_account_id` | Your dbt Cloud account ID | `12345` | -| `TF_VAR_dbt_api_token` | dbt Cloud API token | `dbtc_xxxxx...` | -| `TF_VAR_dbt_pat` | dbt Cloud Personal Access Token | `dbtc_xxxxx...` | -| `TF_VAR_dbt_host_url` | dbt Cloud API endpoint | `https://cloud.getdbt.com/api` | -| `TF_VAR_yaml_file_path` | Path to your YAML config | `./dbt-config.yml` | +| `TF_VAR_dbt_account_id` | Numeric dbt Cloud account ID | `12345` | +| `TF_VAR_dbt_token` | dbt Cloud API token | `dbtc_xxxxx...` | +| `TF_VAR_dbt_host_url` | dbt Cloud host URL | `https://cloud.getdbt.com` | ## Optional Variables | Variable | Description | Default | |----------|-------------|---------| -| `TF_VAR_token_map` | Map of database credential tokens | `{}` | -| `TF_VAR_target_name` | dbt target name | `null` | +| `TF_VAR_dbt_pat` | Personal access token (for GitHub App integration; can equal `dbt_token`) | `null` | +| `TF_VAR_environment_credentials` | JSON map of environment credential objects | `{}` | +| `TF_VAR_connection_credentials` | JSON map of global connection OAuth/key credentials | `{}` | +| `TF_VAR_token_map` | Map of Databricks token names to values (legacy) | `{}` | +| `TF_VAR_lineage_tokens` | Map of lineage integration tokens | `{}` | +| `TF_VAR_oauth_client_secrets` | Map of OAuth configuration client secrets | `{}` | +| `TF_VAR_target_name` | Default dbt target name | `""` | --- @@ -33,63 +34,66 @@ These variables must be set for the module to function: The recommended approach for local development: -#### Step 1: Create .env File - ```bash # Create from example cp .env.example .env ``` -#### Step 2: Edit with Your Values +Edit `.env` with your actual values (never commit this file): ```bash title=".env" -# dbt Cloud Configuration +# --- Required --- export TF_VAR_dbt_account_id=12345 -export TF_VAR_dbt_api_token=dbtc_xxxxxxxxxxxxx -export TF_VAR_dbt_pat=dbtc_xxxxxxxxxxxxx -export TF_VAR_dbt_host_url=https://cloud.getdbt.com/api - -# YAML Configuration Path -export TF_VAR_yaml_file_path=./dbt-config.yml - -# Database Credentials (JSON format, single line) -export TF_VAR_token_map='{"prod_databricks":"dapi123","dev_snowflake":"abc456"}' -``` +export TF_VAR_dbt_token=dbtc_your_api_token +export TF_VAR_dbt_host_url=https://cloud.getdbt.com + +# --- Environment credentials (keyed by "{project_key}_{env_key}") --- +export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics" + } +}' -!!! warning "JSON Format for token_map" - The `token_map` must be valid single-line JSON. Use single quotes around the entire value. +# --- Optional: Global connection OAuth credentials --- +# export TF_VAR_connection_credentials='{"databricks_prod": {"client_id": "...", "client_secret": "..."}}' -#### Step 3: Load Variables +# --- Optional: Lineage integration tokens --- +# export TF_VAR_lineage_tokens='{"analytics_tableau_prod": "..."}' -```bash -# Load into current shell -source .env - -# Verify they're set -echo $TF_VAR_dbt_account_id +# --- Optional: OAuth config client secrets --- +# export TF_VAR_oauth_client_secrets='{"snowflake_oauth": "..."}' ``` -#### Step 4: Run Terraform +Then load and run: ```bash +source .env terraform plan terraform apply ``` +!!! warning "JSON Format" + JSON blob variables must be valid single-line JSON when set via `export`. In `terraform.tfvars` you can use HCL map syntax instead (see below). + ### Using terraform.tfvars (Alternative) -If you prefer HCL syntax: +If you prefer HCL syntax instead of JSON: ```hcl title="terraform.tfvars" -dbt_account_id = 12345 -dbt_api_token = "dbtc_xxxxxxxxxxxxx" -dbt_pat = "dbtc_xxxxxxxxxxxxx" -dbt_host_url = "https://cloud.getdbt.com/api" -yaml_file_path = "./dbt-config.yml" - -token_map = { - prod_databricks = "dapi123" - dev_snowflake = "abc456" +dbt_account_id = 12345 +dbt_token = "dbtc_your_api_token" +dbt_host_url = "https://cloud.getdbt.com" + +environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi..." + catalog = "main" + schema = "analytics" + } } ``` @@ -105,146 +109,136 @@ token_map = { ## CI/CD Setup +In CI/CD, set credentials as platform secrets — never in the workflow file itself. + ### GitHub Actions -Store secrets in GitHub Settings > Secrets and variables > Actions: - -```yaml title=".github/workflows/deploy.yml" -name: Deploy dbt Cloud - -on: - push: - branches: [main] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - - - name: Terraform Init - run: terraform init - - - name: Terraform Apply - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - run: terraform apply -auto-approve +Store in **Settings > Secrets and variables > Actions**: + +```yaml title=".github/workflows/cd.yml" +env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + TF_VAR_connection_credentials: ${{ secrets.CONNECTION_CREDENTIALS }} + TF_VAR_lineage_tokens: ${{ secrets.LINEAGE_TOKENS }} + TF_VAR_oauth_client_secrets: ${{ secrets.OAUTH_CLIENT_SECRETS }} ``` -**Required GitHub Secrets:** -- `DBT_ACCOUNT_ID` -- `DBT_API_TOKEN` -- `DBT_PAT` -- `TOKEN_MAP` (JSON string: `{"key":"value"}`) +**Required secrets:** `DBT_ACCOUNT_ID`, `DBT_TOKEN` + +**Credential secrets (add as needed):** `ENVIRONMENT_CREDENTIALS`, `CONNECTION_CREDENTIALS`, `LINEAGE_TOKENS`, `OAUTH_CLIENT_SECRETS` + +Store each JSON blob as a single-line string in the secret value: +``` +{"analytics_prod": {"credential_type": "databricks", "token": "dapi...", "catalog": "main", "schema": "analytics"}} +``` + +See the [CI/CD Guide](../guides/cicd.md) for complete workflow files. ### GitLab CI/CD -Store in Settings > CI/CD > Variables (masked & protected): +Store in **Settings > CI/CD > Variables** (mark as **Masked** and **Protected**): ```yaml title=".gitlab-ci.yml" -deploy: - image: hashicorp/terraform:latest - stage: deploy - variables: - TF_VAR_dbt_account_id: $DBT_ACCOUNT_ID - TF_VAR_dbt_api_token: $DBT_API_TOKEN - TF_VAR_dbt_pat: $DBT_PAT - TF_VAR_dbt_host_url: "https://cloud.getdbt.com/api" - TF_VAR_yaml_file_path: "./dbt-config.yml" - TF_VAR_token_map: $TOKEN_MAP - script: - - terraform init - - terraform apply -auto-approve - only: - - main +variables: + TF_VAR_dbt_account_id: $DBT_ACCOUNT_ID + TF_VAR_dbt_token: $DBT_TOKEN + TF_VAR_dbt_pat: $DBT_PAT + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: $ENVIRONMENT_CREDENTIALS + TF_VAR_connection_credentials: $CONNECTION_CREDENTIALS ``` ### Azure DevOps -Store in Pipelines > Library > Variable groups: +Store in **Pipelines > Library > Variable groups** (mark as secret): ```yaml title="azure-pipelines.yml" -trigger: - - main - -pool: - vmImage: 'ubuntu-latest' - -variables: - - group: dbt-cloud-credentials - -steps: - - task: TerraformInstaller@0 - inputs: - terraformVersion: 'latest' - - - task: TerraformTaskV2@2 - inputs: - command: 'init' - - - task: TerraformTaskV2@2 - inputs: - command: 'apply' - environmentServiceNameAzureRM: 'terraform' - env: - TF_VAR_dbt_account_id: $(DBT_ACCOUNT_ID) - TF_VAR_dbt_api_token: $(DBT_API_TOKEN) - TF_VAR_dbt_pat: $(DBT_PAT) - TF_VAR_dbt_host_url: 'https://cloud.getdbt.com/api' - TF_VAR_yaml_file_path: './dbt-config.yml' - TF_VAR_token_map: $(TOKEN_MAP) +env: + TF_VAR_dbt_account_id: $(DBT_ACCOUNT_ID) + TF_VAR_dbt_token: $(DBT_TOKEN) + TF_VAR_dbt_pat: $(DBT_PAT) + TF_VAR_dbt_host_url: 'https://cloud.getdbt.com' + TF_VAR_environment_credentials: $(ENVIRONMENT_CREDENTIALS) ``` --- -## Token Map Configuration +## Credential Variable Reference + +### `environment_credentials` + +Map of environment credential objects, keyed by `"{project_key}_{env_key}"`. + +Each object must include `credential_type` and the type-specific fields: + +```bash +export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics" + }, + "analytics_dev": { + "credential_type": "snowflake", + "auth_type": "password", + "user": "DBT_USER", + "password": "...", + "schema": "DEV_ANALYTICS", + "database": "ANALYTICS", + "warehouse": "TRANSFORMING" + } +}' +``` -The `token_map` variable maps credential names in your YAML to actual database tokens. +The key `"analytics_prod"` maps to a project with `key: analytics` and an environment with `key: prod`. -### How It Works +### `connection_credentials` -**In your YAML:** +Map of connection credential objects for global connections, keyed by `global_connections[].key`: -```yaml -environments: - - name: "Production" - credential: - token_name: "prod_databricks_token" # ← This is the key - schema: "analytics" +```bash +export TF_VAR_connection_credentials='{ + "databricks_prod": { + "client_id": "...", + "client_secret": "..." + }, + "snowflake_prod": { + "oauth_client_id": "...", + "oauth_client_secret": "..." + } +}' ``` -**In your environment variables:** +### `token_map` + +Legacy Databricks token map, keyed by `credential.token_name` in YAML: ```bash -export TF_VAR_token_map='{"prod_databricks_token":"dapi_abc123xyz"}' -# ↑ Must match ↑ Actual token +export TF_VAR_token_map='{"my_databricks_token": "dapi_abc123"}' ``` -### Multiple Credentials Example +This is the older pattern. Prefer `environment_credentials` for new setups. + +### `lineage_tokens` + +Tokens for Tableau/Looker lineage integrations, keyed by `"{project_key}_{integration_key}"`: ```bash -# Multiple database credentials -export TF_VAR_token_map='{ - "prod_databricks": "dapi_prod123", - "dev_databricks": "dapi_dev456", - "staging_snowflake": "sf_stg789", - "prod_snowflake": "sf_prd012" -}' +export TF_VAR_lineage_tokens='{"analytics_tableau_prod": "..."}' ``` -!!! tip "Token Security" - - **Never** commit actual tokens to Git - - Use environment-specific tokens (dev, staging, prod) - - Rotate tokens periodically - - Use service principals, not personal tokens +### `oauth_client_secrets` + +Client secrets for OAuth configurations, keyed by `oauth_configurations[].key`: + +```bash +export TF_VAR_oauth_client_secrets='{"snowflake_oauth": "..."}' +``` --- @@ -256,68 +250,42 @@ export TF_VAR_token_map='{ 2. Look at the URL: `https://cloud.getdbt.com/accounts/{account_id}/` 3. The number after `/accounts/` is your account ID -### API Token & PAT +### API Token 1. Go to [https://cloud.getdbt.com/settings/profile](https://cloud.getdbt.com/settings/profile) -2. Scroll to "API Access" -3. Click "Create Token" or "Create Service Account Token" -4. Copy the token (starts with `dbtc_`) +2. Scroll to **API Access** +3. Click **Create Token** or use an existing service account token +4. Copy the token — it starts with `dbtc_` !!! info "Token vs PAT" - For this module, use the same token for both `dbt_api_token` and `dbt_pat`. + For most setups, use the same token for both `dbt_token` and `dbt_pat`. The PAT is only required separately if you're using GitHub App integration with a different auth token. ### Host URL | Region | Host URL | |--------|----------| -| US (Multi-tenant) | `https://cloud.getdbt.com/api` | -| EMEA (Multi-tenant) | `https://emea.dbt.com/api` | -| AU (Multi-tenant) | `https://au.dbt.com/api` | -| Single-tenant | `https://{your-account}.getdbt.com/api` | - -### Connection IDs - -1. In dbt Cloud: Admin > Connections -2. Click on your connection -3. Look at the URL: `/connections/{connection_id}` -4. Or check the connection details page +| US (Multi-tenant) | `https://cloud.getdbt.com` | +| EMEA (Multi-tenant) | `https://emea.dbt.com` | +| AU (Multi-tenant) | `https://au.dbt.com` | +| Single-tenant | `https://{your-account}.getdbt.com` | --- ## Best Practices -### Security - ✅ **DO:** -- Use `.env` for local development -- Use CI/CD secrets for automation +- Use CI/CD platform secrets for all automated workflows +- Use `.env` for local development only — never in production - Add `.env` and `terraform.tfvars` to `.gitignore` - Use service account tokens, not personal tokens -- Rotate credentials regularly -- Use different tokens for dev/staging/prod +- Rotate tokens regularly ❌ **DON'T:** - Commit credentials to Git -- Share tokens in chat/email +- Echo secret values in scripts or logs - Use production tokens in development - Hardcode credentials in `.tf` files -### Organization - -``` -my-dbt-project/ -├── .env # Local credentials (gitignored) -├── .env.example # Template (committed) -├── .gitignore # Includes .env, *.tfvars -├── main.tf # No credentials here! -├── variables.tf # Variable definitions only -├── dbt-config.yml # References token_map keys -└── configs/ # Multiple project configs - ├── finance.yml - ├── marketing.yml - └── operations.yml -``` - ### Debugging Check if variables are loaded: @@ -326,7 +294,7 @@ Check if variables are loaded: # After source .env env | grep TF_VAR -# Or for a specific variable +# Specific variable echo $TF_VAR_dbt_account_id ``` @@ -336,49 +304,35 @@ echo $TF_VAR_dbt_account_id ### "Error: No value for required variable" -**Problem:** Terraform can't find the variable. - -**Solutions:** ```bash -# Make sure to source .env +# Make sure to source .env first source .env - -# Verify it's set -echo $TF_VAR_dbt_account_id +echo $TF_VAR_dbt_account_id # Should print your account ID # Or pass directly terraform plan -var="dbt_account_id=12345" ``` -### "Invalid JSON for token_map" +### "Invalid JSON for environment_credentials" -**Problem:** `token_map` isn't valid JSON. - -**Solutions:** ```bash -# ❌ Multi-line won't work -export TF_VAR_token_map='{ +# ❌ Multi-line won't work in export +export TF_VAR_environment_credentials='{ "key": "value" }' -# ✅ Single line -export TF_VAR_token_map='{"key":"value"}' +# ✅ Single line with single quotes +export TF_VAR_environment_credentials='{"analytics_prod": {"credential_type": "databricks", "token": "dapi..."}}' -# ✅ Or use terraform.tfvars -token_map = { - key = "value" -} +# ✅ Or use terraform.tfvars with HCL syntax ``` -### "401 Unauthorized" from dbt Cloud - -**Problem:** Invalid or expired API token. +### "401 Unauthorized" -**Solutions:** -- Regenerate token in dbt Cloud settings +- Regenerate token at [dbt Cloud Profile](https://cloud.getdbt.com/settings/profile) - Verify token starts with `dbtc_` -- Check you're using the right account -- Ensure token has necessary permissions +- Confirm you're using the correct account ID +- Ensure the token has account-level permissions --- @@ -386,23 +340,23 @@ token_map = {
-- :material-file-yaml:{ .lg .middle } __YAML Schema__ +- :material-file-yaml:{ .lg .middle } **YAML Schema** --- - Configure your dbt projects + Configure your dbt projects in YAML [:octicons-arrow-right-24: YAML Schema](yaml-schema.md) -- :material-github-box:{ .lg .middle } __CI/CD Guide__ +- :material-github-box:{ .lg .middle } **CI/CD Guide** --- - Automate deployments + Complete GitHub Actions workflow examples [:octicons-arrow-right-24: CI/CD Integration](../guides/cicd.md) -- :material-lifebuoy:{ .lg .middle } __Troubleshooting__ +- :material-lifebuoy:{ .lg .middle } **Troubleshooting** --- diff --git a/docs/configuration/multi-project.md b/docs/configuration/multi-project.md index f800d2d..3aa1a16 100644 --- a/docs/configuration/multi-project.md +++ b/docs/configuration/multi-project.md @@ -152,24 +152,23 @@ jobs: fail-fast: false # Continue even if one fails steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.6.0 - + terraform_version: "~1" + - name: Terraform Init run: terraform init - + - name: Deploy ${{ matrix.project }} env: TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./configs/${{ matrix.project }}.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} run: | terraform plan -out=tfplan terraform apply tfplan @@ -186,10 +185,10 @@ stages: stage: deploy variables: TF_VAR_dbt_account_id: $DBT_ACCOUNT_ID - TF_VAR_dbt_api_token: $DBT_API_TOKEN + TF_VAR_dbt_token: $DBT_TOKEN TF_VAR_dbt_pat: $DBT_PAT - TF_VAR_dbt_host_url: "https://cloud.getdbt.com/api" - TF_VAR_token_map: $TOKEN_MAP + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: $ENVIRONMENT_CREDENTIALS script: - terraform init - terraform plan -out=tfplan @@ -286,16 +285,19 @@ Use consistent naming across projects: ```yaml # finance.yml -project: - name: "finance-analytics" +projects: + - name: finance-analytics + key: finance -# marketing.yml -project: - name: "marketing-analytics" +# marketing.yml +projects: + - name: marketing-analytics + key: marketing # operations.yml -project: - name: "operations-analytics" +projects: + - name: operations-analytics + key: operations ``` ### 2. Shared Variables @@ -400,15 +402,15 @@ provider "dbtcloud" { } module "dbt_cloud" { - source = "git::https://github.com/trouze/terraform-dbtcloud-yaml.git" - - dbt_account_id = var.dbt_account_id - dbt_token = var.dbt_api_token - dbt_pat = var.dbt_pat - dbt_host_url = var.dbt_host_url - yaml_file = var.yaml_file_path - token_map = var.token_map - target_name = var.target_name + source = "github.com/trouze/terraform-dbtcloud-yaml" + + dbt_account_id = var.dbt_account_id + dbt_token = var.dbt_token + dbt_pat = var.dbt_pat + dbt_host_url = var.dbt_host_url + yaml_file = var.yaml_file + environment_credentials = var.environment_credentials + target_name = var.target_name } ``` @@ -430,7 +432,7 @@ jobs: outputs: projects: ${{ steps.filter.outputs.changes }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dorny/paths-filter@v2 id: filter @@ -453,10 +455,10 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 - name: Terraform Init working-directory: terraform @@ -466,11 +468,10 @@ jobs: working-directory: terraform env: TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ../configs/${{ matrix.project }}.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP_${{ matrix.project | upper }} }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} run: | terraform workspace select -or-create ${{ matrix.project }} terraform plan -out=tfplan diff --git a/docs/configuration/yaml-schema.md b/docs/configuration/yaml-schema.md index 8d048f7..ce06f90 100644 --- a/docs/configuration/yaml-schema.md +++ b/docs/configuration/yaml-schema.md @@ -1,464 +1,1055 @@ # YAML Schema Reference -Complete reference for the dbt Cloud YAML configuration format. +Complete field reference for `dbt-config.yml`. Every key the module reads is documented here with its type, whether it is required, valid values, and an example. -## Schema Overview - -The YAML configuration is validated against a JSON Schema to ensure correctness. Your IDE can use this schema for auto-completion and validation. +--- -### IDE Setup +## Full skeleton -Add this to the top of your `dbt-config.yml`: +The top-level keys available in any `dbt-config.yml`: ```yaml -# yaml-language-server: $schema=https://raw.githubusercontent.com/trouze/terraform-dbtcloud-yaml/main/schemas/v1.json - -project: - name: "my-dbt-project" - ... +# ── Account-level (all optional — omit any section you don't need) ──────────── +account_features: { ... } +global_connections: [ ... ] +service_tokens: [ ... ] +groups: [ ... ] +user_groups: [ ... ] +notifications: [ ... ] +oauth_configurations: [ ... ] +ip_restrictions: [ ... ] + +# ── Projects (required) ─────────────────────────────────────────────────────── +projects: + - name: Analytics + key: analytics + protected: false + repository: { ... } + environments: [ ... ] + jobs: [ ... ] + environment_variables: [ ... ] + extended_attributes: [ ... ] + profiles: [ ... ] + lineage_integrations: [ ... ] + artefacts: { ... } + semantic_layer: { ... } ``` -!!! tip "VS Code Users" - Install the [YAML extension by Red Hat](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) for auto-completion and validation. +!!! note "Backward compatibility" + The singular `project:` key is accepted and automatically wrapped into a one-element list. Existing single-project configs work without change. --- -## Root Structure +## `account_features` + +Singleton object. All fields are optional and default to `null` (dbt Cloud account default applies). + +| Field | Type | Default | Description | +|---|---|---|---| +| `advanced_ci` | bool | null | Enable Advanced CI comparison features | +| `partial_parsing` | bool | null | Enable incremental manifest parsing | +| `repo_caching` | bool | null | Enable repository-level caching | ```yaml -project: # Required: Root configuration object - name: # Required: Project name - repository: # Required: Git repository configuration - ... - environments: # Required: List of environments - - ... - environment_variables: # Optional: Project-level env vars - - ... +account_features: + advanced_ci: true + partial_parsing: true + repo_caching: false ``` --- -## Project Configuration +## `global_connections` -### `project.name` +Account-level warehouse connections shared across projects. Reference them from environments using `connection_key`. -**Type:** `string` (required) +### Common fields -**Description:** Name of your dbt Cloud project. +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name in dbt Cloud | +| `key` | string | no | `name` | Unique identifier used in `connection_key` references | +| `type` | string | **yes** | — | Adapter type — see valid values below | +| `private_link_endpoint_id` | string | no | null | PrivateLink endpoint ID | +| `protected` | bool | no | false | Prevents `terraform destroy` | -**Constraints:** -- Must be alphanumeric with underscores or hyphens -- 1-128 characters -- Pattern: `^[a-zA-Z0-9_-]+$` +**Valid `type` values:** `databricks` · `snowflake` · `bigquery` · `postgres` · `redshift` -**Examples:** +### Databricks ```yaml -project: - name: "analytics" # ✅ Valid - name: "finance-reporting" # ✅ Valid - name: "data_warehouse_2" # ✅ Valid - name: "Finance Analytics!" # ❌ Invalid (special char) - name: "" # ❌ Invalid (empty) +global_connections: + - name: Databricks Production + key: databricks_prod + type: databricks + host: adb-1234567890123456.7.azuredatabricks.net + http_path: /sql/1.0/warehouses/abc1234def567890 + catalog: main # optional — Unity Catalog catalog name + private_link_endpoint_id: null # optional + protected: false + # OAuth credentials via connection_credentials["databricks_prod"] ``` ---- +| Field | Type | Required | Default | +|---|---|---|---| +| `host` | string | **yes** | "" | +| `http_path` | string | **yes** | "" | +| `catalog` | string | no | null | -## Repository Configuration +### Snowflake -### `project.repository` +```yaml +global_connections: + - name: Snowflake Production + key: snowflake_prod + type: snowflake + account: xy12345.us-east-1 + database: ANALYTICS + warehouse: TRANSFORMING + role: TRANSFORMER # optional + allow_sso: false # optional + client_session_keep_alive: false # optional + # OAuth credentials via connection_credentials["snowflake_prod"] +``` -**Type:** `object` (required) +| Field | Type | Required | Default | +|---|---|---|---| +| `account` | string | **yes** | "" | +| `database` | string | **yes** | "" | +| `warehouse` | string | **yes** | "" | +| `role` | string | no | null | +| `allow_sso` | bool | no | false | +| `client_session_keep_alive` | bool | no | false | -Configure Git integration for your dbt project. +### BigQuery -#### Fields +```yaml +global_connections: + - name: BigQuery Production + key: bigquery_prod + type: bigquery + gcp_project_id: my-gcp-project-id + client_email: dbt-sa@my-project.iam.gserviceaccount.com # optional + client_id: "123456789012345678901" # optional + auth_uri: https://accounts.google.com/o/oauth2/auth # optional + token_uri: https://oauth2.googleapis.com/token # optional + auth_provider_x509_cert_url: https://www.googleapis.com/oauth2/v1/certs # optional + client_x509_cert_url: https://www.googleapis.com/... # optional + timeout_seconds: 300 # optional + location: US # optional + # private_key / private_key_id via connection_credentials["bigquery_prod"] +``` -##### `remote_url` +| Field | Type | Required | Default | +|---|---|---|---| +| `gcp_project_id` | string | **yes** | "" | +| `client_email` | string | no | null | +| `client_id` | string | no | null | +| `auth_uri` | string | no | null | +| `token_uri` | string | no | null | +| `auth_provider_x509_cert_url` | string | no | null | +| `client_x509_cert_url` | string | no | null | +| `timeout_seconds` | number | no | null | +| `location` | string | no | null | -**Type:** `string` (required) +### Postgres -**Description:** Git repository URL (HTTPS or SSH format). +```yaml +global_connections: + - name: Postgres Production + key: postgres_prod + type: postgres + hostname: my-host.rds.amazonaws.com + dbname: analytics + port: 5432 # optional — default 5432 +``` -**Supported Providers:** -- GitHub: `https://github.com/org/repo.git` or `git@github.com:org/repo.git` -- GitLab: `https://gitlab.com/group/repo.git` or `git@gitlab.com:group/repo.git` -- Azure DevOps: `https://dev.azure.com/org/project/_git/repo` -- Bitbucket: `https://bitbucket.org/user/repo.git` +| Field | Type | Required | Default | +|---|---|---|---| +| `hostname` | string | **yes** | "" | +| `dbname` | string | **yes** | "" | +| `port` | number | no | 5432 | -**Examples:** +### Redshift ```yaml -repository: - remote_url: "https://github.com/myorg/dbt-analytics.git" # GitHub HTTPS - remote_url: "git@github.com:myorg/dbt-analytics.git" # GitHub SSH - remote_url: "https://gitlab.com/myorg/data/dbt.git" # GitLab - remote_url: "https://dev.azure.com/myorg/data/_git/dbt" # Azure DevOps +global_connections: + - name: Redshift Production + key: redshift_prod + type: redshift + hostname: my-cluster.abc123.us-east-1.redshift.amazonaws.com + dbname: analytics + port: 5439 # optional — default 5439 ``` -##### `git_clone_strategy` +| Field | Type | Required | Default | +|---|---|---|---| +| `hostname` | string | **yes** | "" | +| `dbname` | string | **yes** | "" | +| `port` | number | no | 5439 | -**Type:** `string` (optional) +--- + +## `service_tokens` + +Account-level API service tokens. -**Description:** How dbt Cloud authenticates with your Git provider. +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `permissions` | list | no | [] | List of permission objects | +| `protected` | bool | no | false | Prevents `terraform destroy` | -**Options:** -- `deploy_key` (default) - SSH deploy key (universal, works with all providers) -- `github_app` - GitHub App integration (GitHub only, recommended) -- `deploy_token` - GitLab deploy token (GitLab only, recommended) -- `azure_active_directory_app` - Azure AD App (Azure DevOps only) +### `permissions[]` -**Auto-Detection:** If omitted, defaults to `deploy_key` for all providers. +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `permission_set` | string | **yes** | — | Permission level — see values below | +| `all_projects` | bool | no | true | Apply to all projects | +| `project_id` | number | no | null | Numeric project ID when `all_projects: false` | -**Examples:** +**Valid `permission_set` values:** `account_admin` · `git_admin` · `job_admin` · `job_runner` · `job_viewer` · `member` · `metadata_only` · `owner` · `readonly` · `seeker_user` · `webhook_admin` ```yaml -# GitHub with GitHub App (recommended for GitHub) -repository: - remote_url: "https://github.com/myorg/repo.git" - git_clone_strategy: "github_app" - github_installation_id: 12345678 - -# GitLab with Deploy Token (recommended for GitLab) -repository: - remote_url: "https://gitlab.com/myorg/repo.git" - git_clone_strategy: "deploy_token" - gitlab_project_id: 9876543 - -# Universal SSH Deploy Key (works everywhere) -repository: - remote_url: "git@github.com:myorg/repo.git" - git_clone_strategy: "deploy_key" +service_tokens: + - name: CI Service Token + key: ci_token + protected: false + permissions: + - permission_set: job_runner + all_projects: true + - permission_set: git_admin + all_projects: false + project_id: 12345 ``` -##### Provider-Specific Fields +--- + +## `groups` + +Account-level user groups. -**GitHub:** +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `assign_by_default` | bool | no | false | Auto-assign new users to this group | +| `sso_mapping_groups` | list(string) | no | null | SSO/IdP group names to sync | +| `permissions` | list | no | [] | Project-level permission grants | +| `protected` | bool | no | false | Prevents `terraform destroy` | -- `github_installation_id` (integer, required with `github_app`) - - Your GitHub App installation ID - - Find in: GitHub Settings > Applications > dbt Cloud +### `permissions[]` -**GitLab:** +Same structure as `service_tokens[].permissions[]` above. -- `gitlab_project_id` (integer, required with `deploy_token`) - - Numeric project ID - - Find in: GitLab Project > Settings > General +```yaml +groups: + - name: Developers + key: developers + assign_by_default: false + sso_mapping_groups: + - "data-team-eng" + permissions: + - permission_set: job_runner + all_projects: true +``` + +--- -**Azure DevOps:** +## `user_groups` -- `azure_active_directory_project_id` (string, required with `azure_active_directory_app`) - - Project UUID -- `azure_active_directory_repository_id` (string, required with `azure_active_directory_app`) - - Repository UUID -- `azure_bypass_webhook_registration_failure` (boolean, optional) - - Set `true` if user can't register webhooks - - Default: `false` +Assigns existing dbt Cloud users to groups. `group_keys` references `groups[].key`. -##### Other Fields +| Field | Type | Required | Default | +|---|---|---|---| +| `user_id` | number | **yes** | — | +| `group_keys` | list(string) | no | [] | -- `is_active` (boolean, optional) - Default: `true` -- `private_link_endpoint_id` (string, optional) - For VPC/Private Link -- `pull_request_url_template` (string, optional) - Custom PR URL template +```yaml +user_groups: + - user_id: 12345 + group_keys: + - developers + - analysts +``` --- -## Environment Configuration +## `notifications` -### `project.environments` +Job notification rules. -**Type:** `array` (required, min 1 item) +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `notification_type` | number | no | 1 | See valid values below | +| `user_id` | number | no | null | dbt Cloud user ID (required for type 1) | +| `slack_channel_id` | string | no | null | Slack channel ID (required for type 2) | +| `slack_channel_name` | string | no | null | Slack channel display name (type 2) | +| `external_email` | string | no | null | Email address (required for type 3) | +| `on_failure` | list(number) | no | [] | Job IDs to notify on failure | +| `on_success` | list(number) | no | [] | Job IDs to notify on success | +| `on_cancel` | list(number) | no | [] | Job IDs to notify on cancel | +| `on_warning` | list(number) | no | [] | Job IDs to notify on warning | -Define your dbt Cloud environments (dev, staging, prod, etc.). +**Valid `notification_type` values:** -#### Environment Object +| Value | Destination | +|---|---| +| `1` | dbt Cloud user (email) | +| `2` | Slack channel | +| `3` | External email address | ```yaml -environments: - - name: # Required - type: # Required: "development" or "deployment" - connection_id: # Required - credential: # Required - token_name: # Required - schema: # Required - catalog: # Optional - dbt_version: # Optional - custom_branch: # Optional - enable_model_query_history: # Optional - jobs: # Optional - - ... +notifications: + # dbt Cloud user notification + - name: prod-failures-user + key: prod_failures_user + notification_type: 1 + user_id: 12345 + on_failure: [1001, 1002] + on_success: [] + + # Slack channel notification + - name: prod-failures-slack + key: prod_failures_slack + notification_type: 2 + slack_channel_id: C0123456789 + slack_channel_name: "#dbt-alerts" + on_failure: [1001, 1002] + on_cancel: [1001] + + # External email notification + - name: prod-failures-email + key: prod_failures_email + notification_type: 3 + external_email: oncall@example.com + on_failure: [1001] ``` -#### Fields +--- + +## `oauth_configurations` -##### `name` +Account-level OAuth configurations (e.g., Snowflake OAuth, BigQuery WIF). -**Type:** `string` (required) +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `type` | string | **yes** | — | OAuth provider type | +| `authorize_url` | string | **yes** | — | OAuth authorization endpoint | +| `token_url` | string | **yes** | — | OAuth token endpoint | +| `redirect_uri` | string | **yes** | — | Redirect URI registered with the provider | +| `client_id` | string | **yes** | — | OAuth client ID | -Environment name (e.g., "Production", "Development", "Staging"). +!!! note "Client secret" + The `client_secret` is supplied via the `oauth_client_secrets` Terraform variable keyed by this entry's `key` — never hard-code it in YAML. -##### `type` +```yaml +oauth_configurations: + - name: Snowflake OAuth + key: snowflake_oauth + type: snowflake + authorize_url: https://xy12345.snowflakecomputing.com/oauth/authorize + token_url: https://xy12345.snowflakecomputing.com/oauth/token-request + redirect_uri: https://cloud.getdbt.com/complete/oauth + client_id: my-client-id + # client_secret via: TF_VAR_oauth_client_secrets='{"snowflake_oauth":"..."}' +``` -**Type:** `string` (required) +--- -**Options:** -- `development` - For development work -- `deployment` - For production/staging deployments +## `ip_restrictions` -##### `connection_id` +Account-level IP allowlist / denylist rules. -**Type:** `integer` (required) +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `type` | string | no | `allow` | `allow` or `deny` | +| `description` | string | no | null | Human-readable description | +| `rule_set_enabled` | bool | no | true | Whether this rule is active | +| `cidrs` | list | no | [] | List of CIDR objects | -dbt Cloud connection ID for your data warehouse. +### `cidrs[]` -!!! info "Where to find this" - In dbt Cloud: Admin > Connections > Copy the connection ID +| Field | Type | Required | +|---|---|---| +| `cidr` | string | **yes** | -##### `credential` +```yaml +ip_restrictions: + - name: Corporate VPN + key: corp_vpn + type: allow + description: "Allow traffic from corporate network" + rule_set_enabled: true + cidrs: + - cidr: 203.0.113.0/24 + - cidr: 198.51.100.0/24 + + - name: Block public ranges + key: block_public + type: deny + rule_set_enabled: true + cidrs: + - cidr: 0.0.0.0/0 +``` -**Type:** `object` (required) +--- -Database credentials for this environment. +## `projects[]` -**Fields:** -- `token_name` (string, required) - Key in `token_map` variable containing the database token -- `schema` (string, required) - Default schema/dataset name -- `catalog` (string, optional) - Catalog name (for Unity Catalog, etc.) +### Core fields -**Example:** +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | dbt Cloud project display name | +| `key` | string | no | `name` | Unique identifier used in cross-references | +| `protected` | bool | no | false | Prevents `terraform destroy` | ```yaml -credential: - token_name: "prod_databricks_token" # References token_map["prod_databricks_token"] - schema: "analytics_prod" - catalog: "production" # Optional: for Databricks Unity Catalog +projects: + - name: Analytics + key: analytics + protected: false ``` -##### `dbt_version` +--- + +### `repository` + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `remote_url` | string | **yes** | — | `"org/repo"` slug or full HTTPS URL | +| `git_clone_strategy` | string | no | auto-detected | Override clone strategy — see values below | +| `is_active` | bool | no | true | Whether the repository integration is active | +| `github_installation_id` | number | no | null | GitHub App installation ID | +| `gitlab_project_id` | number | no | null | GitLab numeric project ID | +| `pull_request_url_template` | string | no | null | Custom PR URL template (GitLab) | +| `azure_active_directory_project_id` | string | no | null | Azure DevOps project UUID | +| `azure_active_directory_repository_id` | string | no | null | Azure DevOps repository UUID | +| `azure_bypass_webhook_registration_failure` | bool | no | false | Skip webhook registration errors | +| `private_link_endpoint_id` | string | no | null | PrivateLink endpoint | +| `protected` | bool | no | false | Prevents `terraform destroy` | + +**Valid `git_clone_strategy` values:** `github_app` · `deploy_key` · `deploy_token` · `azure_active_directory_app` + +The strategy is auto-detected from the presence of `github_installation_id`, `gitlab_project_id`, or Azure fields — you normally don't need to set it manually. + +=== "GitHub App" + ```yaml + repository: + remote_url: "your-org/your-repo" + github_installation_id: 12345678 + ``` + +=== "GitLab" + ```yaml + repository: + remote_url: "https://gitlab.com/your-org/your-repo.git" + gitlab_project_id: 9876543 + pull_request_url_template: "https://gitlab.com/your-org/your-repo/-/merge_requests/{{prNumber}}" + ``` + +=== "Azure DevOps" + ```yaml + repository: + remote_url: "https://dev.azure.com/org/project/_git/repo" + git_clone_strategy: azure_active_directory_app + azure_active_directory_project_id: "00000000-0000-0000-0000-000000000001" + azure_active_directory_repository_id: "00000000-0000-0000-0000-000000000002" + azure_bypass_webhook_registration_failure: false + ``` + +=== "Deploy Key (public repos)" + ```yaml + repository: + remote_url: "https://github.com/your-org/your-repo.git" + git_clone_strategy: deploy_key + ``` -**Type:** `string` (optional) +--- -dbt version to use (e.g., `"1.6.0"`, `"1.7.1"`). Defaults to latest. +### `environments[]` + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier used in `deferring_environment_key`, etc. | +| `type` | string | **yes** | — | `deployment` or `development` | +| `deployment_type` | string | conditional | — | `production` or `staging` — required when `type: deployment` | +| `connection_key` | string | no | null | References `global_connections[].key` | +| `dbt_version` | string | no | null | Pin dbt Core version (e.g., `"1.9.0"`) | +| `custom_branch` | string | no | null | Custom git branch (development envs) | +| `enable_model_query_history` | bool | no | null | Enable query history tracking | +| `extended_attributes_key` | string | no | null | References `extended_attributes[].key` | +| `protected` | bool | no | false | Prevents `terraform destroy` | +| `credential` | object | no | — | Warehouse credential block — see below | -##### `custom_branch` +```yaml +environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod + dbt_version: "1.9.0" + custom_branch: main + enable_model_query_history: false + extended_attributes_key: databricks_overrides + protected: true + credential: + credential_type: databricks + catalog: main + schema: analytics +``` -**Type:** `string` (optional) +#### `credential` — per warehouse type + +The `credential_type` field selects which credential resource is created. Sensitive values (passwords, tokens, keys) must be supplied via the `environment_credentials` Terraform variable keyed by `"{project_key}_{env_key}"`. + +=== "Databricks" + | Field | Type | Required | Default | Notes | + |---|---|---|---|---| + | `credential_type` | string | **yes** | — | `"databricks"` | + | `catalog` | string | no | null | Unity Catalog catalog | + | `schema` | string | no | `""` | Target schema | + | `token_name` | string | no | — | Legacy: key in `token_map` variable | + + ```yaml + credential: + credential_type: databricks + catalog: main + schema: analytics + # token via environment_credentials["analytics_prod"]["token"] + ``` + +=== "Snowflake (password)" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `auth_type` | string | **yes** | `"password"` | + | `user` | string | **yes** | — | + | `schema` | string | no | `""` | + | `database` | string | no | null | + | `role` | string | no | null | + | `warehouse` | string | no | null | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: snowflake + auth_type: password + user: DBT_USER + schema: ANALYTICS + database: ANALYTICS + warehouse: TRANSFORMING + role: TRANSFORMER + num_threads: 8 + # password via environment_credentials["analytics_prod"]["password"] + ``` + +=== "Snowflake (keypair)" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `auth_type` | string | **yes** | `"keypair"` | + | `user` | string | **yes** | — | + | `schema` | string | no | `""` | + | `database` | string | no | null | + | `role` | string | no | null | + | `warehouse` | string | no | null | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: snowflake + auth_type: keypair + user: DBT_USER + schema: ANALYTICS + # private_key + private_key_passphrase via environment_credentials["analytics_prod"] + ``` + +=== "BigQuery" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `dataset` | string | no | `""` | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: bigquery + dataset: analytics + num_threads: 8 + ``` + +=== "Postgres" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `username` | string | **yes** | — | + | `default_schema` | string | no | `""` | + | `num_threads` | number | no | null | + | `target_name` | string | no | null | + + ```yaml + credential: + credential_type: postgres + username: dbt_user + default_schema: analytics + num_threads: 4 + target_name: prod + # password via environment_credentials["analytics_prod"]["password"] + ``` + +=== "Redshift" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `username` | string | **yes** | — | + | `default_schema` | string | no | `""` | + | `num_threads` | number | no | 4 | + + ```yaml + credential: + credential_type: redshift + username: dbt_user + default_schema: analytics + num_threads: 4 + # password via environment_credentials["analytics_prod"]["password"] + ``` + +=== "Athena" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `schema` | string | no | `""` | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: athena + schema: analytics + num_threads: 4 + # aws_access_key_id + aws_secret_access_key via environment_credentials["analytics_prod"] + ``` + +=== "Fabric / Synapse" + **SQL auth** (`credential_type: fabric` or `credential_type: synapse`): + + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `schema` | string | no | `""` | + | `user` | string | **yes** | — | + | `schema_authorization` | string | no | null | + | `authentication` | string | no | `"sql"` (Synapse only) | + + ```yaml + credential: + credential_type: fabric + schema: analytics + user: DBT_USER + schema_authorization: dbo + # password via environment_credentials["analytics_prod"]["password"] + ``` + + **Service Principal auth:** + + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `schema` | string | no | `""` | + | `tenant_id` | string | **yes** | — | + | `client_id` | string | **yes** | — | + | `schema_authorization` | string | no | null | + | `authentication` | string | no | `"ServicePrincipal"` (Synapse only) | + + ```yaml + credential: + credential_type: synapse + schema: analytics + tenant_id: "00000000-0000-0000-0000-000000000001" + client_id: "my-app-client-id" + authentication: ServicePrincipal + # client_secret via environment_credentials["analytics_prod"]["client_secret"] + ``` + +=== "Starburst / Trino" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | `"starburst"` or `"trino"` | + | `schema` | string | no | `""` | + | `catalog` | string | no | `""` | + | `user` | string | **yes** | — | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: starburst + schema: analytics + catalog: iceberg + user: DBT_USER + num_threads: 4 + # password via environment_credentials["analytics_prod"]["password"] + ``` + +=== "Spark" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `schema` | string | no | `""` | + + ```yaml + credential: + credential_type: spark + schema: analytics + # token via environment_credentials["analytics_prod"]["token"] + ``` + +=== "Teradata" + | Field | Type | Required | Default | + |---|---|---|---| + | `credential_type` | string | **yes** | — | + | `user` | string | **yes** | — | + | `schema` | string | no | `""` | + | `num_threads` | number | no | null | + + ```yaml + credential: + credential_type: teradata + user: DBT_USER + schema: analytics + num_threads: 4 + # password via environment_credentials["analytics_prod"]["password"] + ``` -Git branch to use for this environment. Defaults to repository default branch. +--- -**Example:** +### `jobs[]` + +Jobs are defined at project level and reference environments by `key`. They are **not** nested inside environments. + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier — used in `artefacts` references | +| `environment_key` | string | **yes** | — | References `environments[].key` | +| `execute_steps` | list(string) | **yes** | — | Ordered list of dbt CLI commands | +| `triggers` | object | **yes** | — | At least one trigger must be set — see below | +| `description` | string | no | null | Job description shown in dbt Cloud | +| `dbt_version` | string | no | null | Pin dbt Core version (overrides environment) | +| `num_threads` | number | no | 4 | Thread count | +| `target_name` | string | no | null | dbt target name (e.g., `"prod"`) | +| `timeout_seconds` | number | no | 0 | Job timeout — `0` means no timeout | +| `is_active` | bool | no | true | Whether the job is enabled | +| `job_type` | string | no | — | `"scheduled"` · `"ci"` · `"merge"` | +| `generate_docs` | bool | no | false | Regenerate docs on each run | +| `run_generate_sources` | bool | no | false | Run `dbt source freshness` | +| `run_lint` | bool | no | false | Run SQLFluff lint step | +| `errors_on_lint_failure` | bool | no | true | Treat lint failures as errors | +| `run_compare_changes` | bool | no | false | Advanced CI — requires deployment env with deferral | +| `triggers_on_draft_pr` | bool | no | false | Trigger on draft pull requests | +| `deferring_environment_key` | string | no | null | References `environments[].key` for state deferral | +| `self_deferring` | bool | no | null | Defer to the job's own previous run | +| `force_node_selection` | bool | no | auto | SAO — set automatically; null for CI/merge jobs | +| `protected` | bool | no | false | Prevents `terraform destroy` | +| `env_var_overrides` | map(string) | no | {} | Job-level env var overrides | + +#### `triggers` + +| Field | Type | Default | Description | +|---|---|---|---| +| `schedule` | bool | false | Run on a schedule | +| `github_webhook` | bool | false | Trigger on GitHub PR events | +| `git_provider_webhook` | bool | false | Trigger on generic git provider webhooks | +| `on_merge` | bool | false | Trigger when PR is merged | + +At least one of these must be `true`. + +#### Schedule fields + +Only one schedule mode is applied — precedence order: `schedule_cron` > `schedule_interval` > `schedule_hours`. + +| Field | Type | Default | Description | +|---|---|---|---| +| `schedule_type` | string | null | `"every_day"` · `"days_of_week"` · `"days_of_month"` | +| `schedule_days` | list(number) | null | Days to run — 0–6 (Sun–Sat) for week; 1–31 for month | +| `schedule_hours` | list(number) | null | UTC hours to run — e.g., `[6, 18]` | +| `schedule_cron` | string | null | Cron expression — overrides other schedule fields | +| `schedule_interval` | number | null | Run every N hours — overrides `schedule_hours` | ```yaml -- name: "Staging" - custom_branch: "staging" # Use staging branch instead of main +jobs: + # Scheduled job — weekdays at 6 AM UTC + - name: Production Daily + key: prod_daily + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] + schedule_hours: [6] + num_threads: 8 + target_name: prod + timeout_seconds: 3600 + generate_docs: true + deferring_environment_key: prod + self_deferring: true + protected: true + env_var_overrides: + DBT_TARGET: prod + + # CI job — triggered on PR + - name: Staging CI + key: staging_ci + environment_key: staging + execute_steps: + - dbt build --select state:modified+ + triggers: + github_webhook: true + git_provider_webhook: true + job_type: ci + run_compare_changes: true + deferring_environment_key: prod + + # Merge job — triggered on PR merge + - name: Staging Merge + key: staging_merge + environment_key: staging + execute_steps: + - dbt build --select state:modified+ + triggers: + on_merge: true + job_type: merge + deferring_environment_key: prod + + # Cron schedule + - name: Hourly Refresh + key: hourly_refresh + environment_key: prod + execute_steps: + - dbt run --select marts.finance + triggers: + schedule: true + schedule_cron: "0 * * * *" ``` --- -## Jobs Configuration +### `environment_variables[]` -### `environments[].jobs` +Project-level dbt environment variables with per-environment value overrides. -**Type:** `array` (optional) +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | string | **yes** | Variable name — must use `DBT_` prefix convention | +| `environment_values` | list | **yes** | Per-environment value list | -Define dbt jobs that run in this environment. +#### `environment_values[]` -#### Job Object +| Field | Type | Required | Description | +|---|---|---|---| +| `env` | string | **yes** | `"project"` for project default, or the environment `name` (not key) | +| `value` | string | **yes** | Variable value | ```yaml -jobs: - - name: # Required - description: # Optional - is_active: # Optional, default: true - execute_steps: # Required - - - triggers: # Required - schedule: - github_webhook: - git_provider_webhook: - on_merge: - # Scheduling - schedule_type: # Optional: "every_day", "every_week", "every_month" - schedule_hours: [] # Optional: Hours (0-23, UTC) - schedule_days: [] # Optional: Days (0=Sun, 6=Sat) - schedule_cron: # Optional: Custom cron expression - # Execution - num_threads: # Optional: 1-16, default: 4 - timeout_seconds: # Optional: 300-86400 - target_name: # Optional: dbt target name - dbt_version: # Optional: Job-specific dbt version - # Features - generate_docs: # Optional: Generate docs - run_lint: # Optional: Run linters - run_generate_sources: # Optional: Source freshness - # Deferral - deferring_environment: # Optional: Defer to environment name +environment_variables: + - name: DBT_WAREHOUSE + environment_values: + - env: project # project-level default + value: "prod_warehouse" + - env: Production # matches environment name + value: "prod_warehouse" + - env: Development + value: "dev_warehouse" + + - name: DBT_TARGET + environment_values: + - env: project + value: "prod" + - env: Staging + value: "staging" ``` -### Examples +--- -#### Simple Daily Job +### `extended_attributes[]` -```yaml -jobs: - - name: "Daily Production Run" - execute_steps: - - "dbt run" - - "dbt test" - triggers: - schedule: true - schedule_hours: [6] # 6 AM UTC - github_webhook: false - git_provider_webhook: false - on_merge: false -``` +Per-project connection-level overrides applied at the environment level. Linked to environments via `extended_attributes_key`. -#### CI Job (On Merge) +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | string | **yes** | Display name | +| `key` | string | no | Unique identifier — referenced by `environments[].extended_attributes_key` | +| `content` | map | no | Nested YAML object of connection overrides | ```yaml -jobs: - - name: "Production CI" - description: "Run on merge to main" - execute_steps: - - "dbt build --select state:modified+" - triggers: - schedule: false - github_webhook: false - git_provider_webhook: false - on_merge: true - deferring_environment: "Production" # Defer to prod for state comparison +extended_attributes: + - name: Databricks HTTP Override + key: databricks_overrides + content: + databricks: + http_path: /sql/1.0/warehouses/override-warehouse-id + catalog: overridden_catalog + + - name: Snowflake Warehouse Override + key: snowflake_overrides + content: + snowflake: + warehouse: HIGH_MEMORY_WH ``` -#### Advanced Scheduled Job +--- + +### `profiles[]` + +Links a global connection + environment credential + extended attributes into a named profile for an environment. + +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `connection_key` | string | no | null | References `global_connections[].key` | +| `connection_id` | number | no | null | Numeric connection ID (alternative to `connection_key`) | +| `credential_key` | string | no | null | Composite key `"{project_key}_{env_key}"` | +| `credentials_id` | number | no | null | Numeric credential ID (alternative to `credential_key`) | +| `extended_attributes_key` | string | no | null | References `extended_attributes[].key` | ```yaml -jobs: - - name: "Weekly Full Refresh" - execute_steps: - - "dbt run --full-refresh" - - "dbt test" - triggers: - schedule: true - github_webhook: false - git_provider_webhook: false - on_merge: false - schedule_type: "every_week" - schedule_days: [0] # Sunday - schedule_hours: [2] # 2 AM UTC - num_threads: 8 - timeout_seconds: 7200 # 2 hours - generate_docs: true + profiles: + - name: prod-profile + key: prod_profile + connection_key: databricks_prod + credential_key: analytics_prod + extended_attributes_key: databricks_overrides ``` --- -## Environment Variables - -### `project.environment_variables` +### `lineage_integrations[]` -**Type:** `array` (optional) +Per-project lineage integrations (e.g., Tableau, Looker). -Define project-level environment variables accessible to all jobs. +| Field | Type | Required | Default | Description | +|---|---|---|---|---| +| `name` | string | **yes** | — | Display name | +| `key` | string | no | `name` | Unique identifier | +| `host` | string | **yes** | — | Lineage tool host URL | +| `site_id` | string | **yes** | — | Site/workspace identifier | +| `token_name` | string | **yes** | — | Token name label | -#### Environment Variable Object +!!! note "Token" + The actual token value is supplied via `lineage_tokens` Terraform variable keyed by `"{project_key}_{integration_key}"`. ```yaml -environment_variables: - - name: # Required: Must use UPPER_SNAKE_CASE - environment_values: # Required: Values by environment name - : + lineage_integrations: + - name: Tableau Production + key: tableau_prod + host: https://tableau.example.com + site_id: my-site + token_name: dbt-cloud-tableau-token + # token via: TF_VAR_lineage_tokens='{"analytics_tableau_prod":"token..."}' ``` -### Examples +--- + +### `artefacts` + +Links the project's documentation and source freshness jobs. Both fields reference `jobs[].key`. + +| Field | Type | Required | Description | +|---|---|---|---| +| `docs_job` | string | no | Job key for the documentation artifact | +| `freshness_job` | string | no | Job key for the source freshness artifact | ```yaml -environment_variables: - - name: "DBT_WAREHOUSE_NAME" - environment_values: - Development: "dev_warehouse" - Staging: "staging_warehouse" - Production: "prod_warehouse" - - - name: "DBT_MAX_RETRIES" - environment_values: - Development: "1" - Production: "3" + artefacts: + docs_job: prod_daily + freshness_job: prod_daily ``` --- -## Complete Example +### `semantic_layer` + +Configures the dbt Semantic Layer for a project. -Here's a full example combining all elements: +| Field | Type | Required | Description | +|---|---|---|---| +| `environment` | string | **yes** | References `environments[].key` | + +!!! warning "Create-only" + The semantic layer configuration cannot be imported. It is created once and Terraform will not attempt to update it on subsequent runs if it already exists. ```yaml -# yaml-language-server: $schema=https://raw.githubusercontent.com/trouze/terraform-dbtcloud-yaml/main/schemas/v1.json - -project: - name: "analytics" - - repository: - remote_url: "https://github.com/myorg/dbt-analytics.git" - git_clone_strategy: "github_app" - github_installation_id: 12345678 - - environments: - - name: "Development" - type: "development" - connection_id: 1001 - credential: - token_name: "dev_snowflake_token" - schema: "dev_analytics" - custom_branch: "develop" - enable_model_query_history: true - - - name: "Production" - type: "deployment" - connection_id: 1002 - credential: - token_name: "prod_snowflake_token" - schema: "analytics" - dbt_version: "1.7.1" - jobs: - - name: "Daily Run" - execute_steps: - - "dbt run" - - "dbt test" - triggers: - schedule: true - schedule_hours: [6] - github_webhook: false - git_provider_webhook: false - on_merge: false - num_threads: 8 - generate_docs: true - - - name: "CI Check" - execute_steps: - - "dbt build --select state:modified+" - triggers: - schedule: false - github_webhook: false - git_provider_webhook: false - on_merge: true - deferring_environment: "Production" - - environment_variables: - - name: "DBT_CLOUD_ENV" - environment_values: - Development: "dev" - Production: "prod" + semantic_layer: + environment: prod +``` + +--- + +## Sensitive credential reference + +Sensitive values are never written directly in YAML. Instead, they are passed as Terraform variables and matched by key at apply time. + +| Terraform variable | Key format | Matched to | +|---|---|---| +| `token_map` | `"my_token_name"` | `credential.token_name` in YAML (legacy Databricks) | +| `environment_credentials` | `"project_key_env_key"` | Environment `credential` block | +| `connection_credentials` | `"connection_key"` | `global_connections[].key` | +| `lineage_tokens` | `"project_key_integration_key"` | `lineage_integrations[].key` composite | +| `oauth_client_secrets` | `"oauth_config_key"` | `oauth_configurations[].key` | + +Keys use underscores and must exactly match the `key:` values in your YAML. For example, a project with `key: analytics` and an environment with `key: prod` uses the `environment_credentials` key `"analytics_prod"`. + +```bash +export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics" + }, + "analytics_staging": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics_staging" + } +}' + +export TF_VAR_connection_credentials='{ + "snowflake_prod": { + "oauth_client_id": "...", + "oauth_client_secret": "..." + } +}' + +export TF_VAR_lineage_tokens='{ + "analytics_tableau_prod": "tableau-pat-token..." +}' + +export TF_VAR_oauth_client_secrets='{ + "snowflake_oauth": "client-secret-value..." +}' ``` --- -## Validation +## `protected: true` behavior -The module validates your YAML against the schema during `terraform plan`. Common errors: +Any resource that supports a `protected` field uses a duplicate Terraform resource block pattern: -- **Missing required fields**: Add the required field -- **Invalid enum value**: Check allowed values in this reference -- **Type mismatch**: Ensure strings are quoted, numbers are not -- **Pattern mismatch**: Check format constraints (e.g., project name pattern) +- `protected: false` (default) — resource created normally; `terraform destroy` works +- `protected: true` — resource created with `lifecycle { prevent_destroy = true }`; `terraform destroy` is blocked with an error -For IDE validation, add the schema URL to your YAML file header as shown above. +This is the only reliable way to prevent accidental destruction in Terraform. Use it for production environments, production jobs, and any resource that would be costly to recreate. diff --git a/docs/getting-started/examples.md b/docs/getting-started/examples.md index eea64fb..0b75583 100644 --- a/docs/getting-started/examples.md +++ b/docs/getting-started/examples.md @@ -10,17 +10,22 @@ The simplest possible setup to get started. - Single dbt Cloud project - GitHub repository integration -- One production environment +- One production environment with Databricks credentials - One scheduled job +- GitHub Actions workflows for CI (plan on PR) and CD (apply on merge) ### Directory Structure ``` examples/basic/ -├── main.tf # Terraform module call -├── variables.tf # Input variables -├── dbt-config.yml # dbt Cloud configuration -└── .env.example # Credential template +├── main.tf # Terraform module call +├── variables.tf # Input variables +├── dbt-config.yml # dbt Cloud configuration +├── .env.example # Credential template +└── .github/ + └── workflows/ + ├── ci.yml # Plan on PR, post as comment + └── cd.yml # Apply on merge to main ``` ### Try It Out @@ -39,265 +44,290 @@ terraform apply --- -## Managing Multiple Projects +## Multi-Project in One YAML -Store multiple YAML configurations and manage them independently or in parallel. +Store multiple projects in a single YAML file. All share one Terraform state. -### Scenario: Multiple Teams +```yaml title="dbt-config.yml" +projects: + - name: Finance Analytics + key: finance + repository: + remote_url: "your-org/finance-dbt" + github_installation_id: 1234567 + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod + credential: + credential_type: databricks + catalog: main + schema: finance_analytics + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: every_day + schedule_hours: [5] + + - name: Marketing Analytics + key: marketing + repository: + remote_url: "your-org/marketing-dbt" + github_installation_id: 1234567 + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: snowflake_prod + credential: + credential_type: snowflake + auth_type: password + user: DBT_USER + schema: MARKETING + database: ANALYTICS + warehouse: TRANSFORMING + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: every_day + schedule_hours: [6] +``` -You have separate dbt projects for different teams (Finance, Marketing, Operations) and want to manage them with the same Terraform workflow. +Environment credentials are keyed by `"{project_key}_{env_key}"`: -### Directory Structure +```bash +export TF_VAR_environment_credentials='{ + "finance_prod": { + "credential_type": "databricks", + "token": "dapi_finance...", + "catalog": "main", + "schema": "finance_analytics" + }, + "marketing_prod": { + "credential_type": "snowflake", + "auth_type": "password", + "user": "DBT_USER", + "password": "...", + "schema": "MARKETING", + "database": "ANALYTICS", + "warehouse": "TRANSFORMING" + } +}' +``` + +--- + +## Multiple Teams — One YAML File Per Team + +For larger teams, use separate YAML files and deploy with the `yaml_file` variable: ``` my-dbt-infrastructure/ ├── main.tf ├── variables.tf -├── .env └── configs/ ├── finance.yml ├── marketing.yml └── operations.yml ``` -### Deploy Specific Project - ```bash -# Load credentials once source .env # Deploy Finance project -terraform plan -var="yaml_file_path=./configs/finance.yml" -terraform apply -var="yaml_file_path=./configs/finance.yml" +terraform apply -var="yaml_file=./configs/finance.yml" # Deploy Marketing project -terraform plan -var="yaml_file_path=./configs/marketing.yml" -terraform apply -var="yaml_file_path=./configs/marketing.yml" +terraform apply -var="yaml_file=./configs/marketing.yml" ``` -### Deploy All Projects (Sequential) +--- -```bash -source .env +## Multi-Environment Project + +Development, staging, and production environments in one project, with job deferral: -for config in configs/*.yml; do - echo "Deploying $config..." - terraform apply -var="yaml_file_path=$config" -auto-approve -done +```yaml title="dbt-config.yml" +projects: + - name: Analytics + key: analytics + repository: + remote_url: "your-org/analytics-dbt" + github_installation_id: 1234567 + + environments: + - name: Development + key: dev + type: development + connection_key: databricks_prod + custom_branch: develop + credential: + credential_type: databricks + catalog: main + schema: dev_analytics + + - name: Staging + key: staging + type: deployment + deployment_type: staging + connection_key: databricks_prod + credential: + credential_type: databricks + catalog: main + schema: staging_analytics + + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod + protected: true + credential: + credential_type: databricks + catalog: main + schema: analytics + + jobs: + - name: Production Daily + key: prod_daily + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] # Weekdays + schedule_hours: [6] + generate_docs: true + + - name: Staging CI + key: staging_ci + environment_key: staging + execute_steps: + - dbt build --select state:modified+ + triggers: + on_merge: true + deferring_environment_key: prod # Defer to prod for state comparison ``` --- ## CI/CD with GitHub Actions -Automate dbt Cloud infrastructure deployment on configuration changes. - -### Single Project Workflow - -```yaml title=".github/workflows/dbt-infrastructure.yml" -name: Deploy dbt Cloud Infrastructure - -on: - push: - branches: [main] - paths: - - 'dbt-config.yml' - - '**.tf' - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.6.0 - - - name: Terraform Init - run: terraform init - - - name: Terraform Plan - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - run: terraform plan - - - name: Terraform Apply - if: github.ref == 'refs/heads/main' - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - run: terraform apply -auto-approve -``` - -### Multi-Project Parallel Deployment - -```yaml title=".github/workflows/multi-project.yml" -name: Deploy Multiple dbt Projects - -on: - push: - branches: [main] - paths: - - 'configs/**.yml' - -jobs: - deploy: - runs-on: ubuntu-latest - strategy: - matrix: - project: [finance, marketing, operations] - steps: - - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - - - name: Terraform Init - run: terraform init - - - name: Deploy ${{ matrix.project }} - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./configs/${{ matrix.project }}.yml - run: | - terraform plan - terraform apply -auto-approve -``` - -!!! tip "Required Secrets" - Add these to your GitHub repository secrets: - - - `DBT_ACCOUNT_ID` - - `DBT_API_TOKEN` - - `DBT_PAT` - ---- +Use the two-workflow pattern from `examples/basic/.github/workflows/`: -## Advanced: Multi-Environment Configuration +- `ci.yml` — validates and plans on every PR, posts plan as a comment (updates existing comment on re-push) +- `cd.yml` — applies on merge to main, with optional approval gate via GitHub Environments -Manage development, staging, and production environments in one YAML file. +```yaml title=".github/workflows/ci.yml (key steps)" +- name: Terraform Plan + id: plan + run: | + terraform plan -no-color -out=tfplan + terraform show -no-color tfplan > plan.txt + continue-on-error: true -```yaml title="dbt-config.yml" -project: - name: "analytics" - repository: - remote_url: "https://github.com/myorg/dbt-analytics.git" - git_clone_strategy: "github_app" - github_installation_id: 123456 - - environments: - # Development environment - - name: "Development" - type: "development" - connection_id: 1 - credential: - token_name: "dev_databricks_token" - schema: "dev" - custom_branch: "develop" - - # Staging environment - - name: "Staging" - type: "deployment" - connection_id: 2 - credential: - token_name: "staging_databricks_token" - schema: "staging" - jobs: - - name: "Staging CI" - execute_steps: - - "dbt build" - triggers: - on_merge: true - - # Production environment - - name: "Production" - type: "deployment" - connection_id: 3 - credential: - token_name: "prod_databricks_token" - schema: "prod" - jobs: - - name: "Production Daily" - execute_steps: - - "dbt run" - - "dbt test" - triggers: - schedule: true - schedule_hours: [6] - schedule_days: [0, 1, 2, 3, 4] - - - name: "Production CI" - execute_steps: - - "dbt build --select state:modified+" - triggers: - on_merge: true - deferring_environment: "Production" +- name: Post plan as PR comment + uses: actions/github-script@v7 + # ... posts/updates comment with plan output ``` +See the [CI/CD Guide](../guides/cicd.md) for the full workflow files and GitLab/Azure DevOps equivalents. + --- -## Complete YAML Schema Reference +## Full Schema Reference -For the full specification of all available configuration options, see the [YAML Schema documentation](../configuration/yaml-schema.md). +See [YAML Schema](../configuration/yaml-schema.md) for every field with types, defaults, and examples. ### Quick Reference ```yaml -project: - name: # Required - repository: - remote_url: # Required - git_clone_strategy: # Required - gitlab_project_id: # Optional - github_installation_id: # Optional - - environments: - - name: # Required - type: # Required: "development" or "deployment" - connection_id: # Required - credential: - token_name: # Optional - schema: # Optional - catalog: # Optional - dbt_version: # Optional: default "latest" - custom_branch: # Optional - jobs: - - name: # Required - execute_steps: # Required - - - triggers: # Required - schedule: - github_webhook: - on_merge: - # ... many more optional fields - - environment_variables: # Optional - - name: # Must start with DBT_ - environment_values: - - env: - value: +# Account-level (optional) +account_features: + advanced_ci: true + partial_parsing: true + +global_connections: + - name: Databricks Production + key: databricks_prod + type: databricks + host: adb-1234.azuredatabricks.net + http_path: /sql/1.0/warehouses/abc123 + +service_tokens: + - name: CI Service Token + key: ci_token + permissions: + - permission_set: job_runner + all_projects: true + +groups: + - name: Developers + key: developers + assign_by_default: false + +notifications: + - name: prod-failures + key: prod_failures + notification_type: 2 # 2 = Slack + slack_channel_id: C0123456789 + slack_channel_name: "#dbt-alerts" + on_failure: [] + +# Projects (required) +projects: + - name: Analytics + key: analytics + protected: false + + repository: + remote_url: "your-org/your-repo" + github_installation_id: 1234567 + + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod + protected: true + credential: + credential_type: databricks + catalog: main + schema: analytics + + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: every_day + schedule_hours: [6] + generate_docs: true + deferring_environment_key: prod + + environment_variables: + - name: DBT_WAREHOUSE + environment_values: + - env: project + value: "prod_warehouse" + - env: Production + value: "prod_warehouse" ``` - ---- - -## More Examples Coming Soon - -- **Snowflake Integration** -- **Databricks Unity Catalog** -- **BigQuery with Service Accounts** -- **GitLab CI/CD Pipeline** -- **Azure DevOps Integration** - -Want to contribute an example? [Open a PR](https://github.com/trouze/terraform-dbtcloud-yaml/pulls)! diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d210364..a52afe8 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -8,10 +8,10 @@ Before you begin, make sure you have: - [x] Terraform >= 1.0 installed - [x] dbt Cloud account with admin access -- [x] dbt Cloud API token ([Get one here](https://cloud.getdbt.com/settings/profile)) +- [x] dbt Cloud API token ([generate at Profile > API Access](https://cloud.getdbt.com/settings/profile)) - [x] Git repository with your dbt project -## Step 1: Clone the Example +## Step 1: Copy the Example Start with the basic example as a template: @@ -28,102 +28,109 @@ The basic example includes: ``` basic/ -├── main.tf # Terraform configuration -├── variables.tf # Input variable definitions -├── dbt-config.yml # Your dbt Cloud configuration -└── .env.example # Credential template +├── main.tf # Terraform module call +├── variables.tf # Input variable definitions +├── dbt-config.yml # Your dbt Cloud configuration +├── .env.example # Credential template +└── .github/ + └── workflows/ + ├── ci.yml # Plan on PR + └── cd.yml # Apply on merge ``` -## Step 2: Configure Your Credentials +## Step 2: Set Your Credentials -Create a `.env` file for your dbt Cloud credentials: +Credentials are passed via environment variables — never hardcoded. In CI/CD, set these as secrets in your platform (see [CI/CD Guide](../guides/cicd.md)). For local use: ```bash cp .env.example .env +# Edit .env with your actual values, then: +source .env ``` -Edit `.env` with your actual values: +Your `.env` should look like: ```bash -# Required: dbt Cloud credentials +# Required export TF_VAR_dbt_account_id=12345 -export TF_VAR_dbt_api_token=dbtc_xxxxxxxxxxxxx -export TF_VAR_dbt_pat=dbtc_xxxxxxxxxxxxx -export TF_VAR_dbt_host_url=https://cloud.getdbt.com/api - -# Optional: Path to your YAML config (you can pass this via -var flag to switch between projects easily) -export TF_VAR_yaml_file_path=./dbt-config.yml +export TF_VAR_dbt_token=dbtc_your_api_token +export TF_VAR_dbt_host_url=https://cloud.getdbt.com -# Optional: Database credential tokens (if using Databricks, Snowflake, etc.) -export TF_VAR_token_map='{"my_credential":"abc123"}' +# Environment credentials — keyed by "{project_key}_{env_key}" +export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics" + } +}' ``` !!! tip "Where to find these values" - **Account ID**: Found in your dbt Cloud URL: `https://cloud.getdbt.com/accounts/{account_id}/` - - **API Token**: Generate at [https://cloud.getdbt.com/settings/profile](https://cloud.getdbt.com/settings/profile) - - **PAT**: Same as API Token (Personal Access Token) - - **Host URL**: `https://cloud.getdbt.com/api` for US, check [docs](https://docs.getdbt.com/docs/dbt-cloud/api-v2) for other regions + - **API Token**: Generate at [https://cloud.getdbt.com/settings/profile](https://cloud.getdbt.com/settings/profile) — starts with `dbtc_` + - **Host URL**: `https://cloud.getdbt.com` for US multi-tenant; see [Environment Variables](../configuration/environment-variables.md) for other regions !!! warning "Security" - Never commit `.env` to version control! It's already in `.gitignore`. + Never commit `.env` or `terraform.tfvars` to version control. Both are in `.gitignore` by default. ## Step 3: Configure Your dbt Project Edit `dbt-config.yml` with your project details: ```yaml -project: - name: "my-dbt-project" - repository: - remote_url: "https://github.com/myorg/my-dbt-repo.git" - git_clone_strategy: "github_app" # or "deploy_key", "gitlab_deploy_token" - github_installation_id: 123456 # Your GitHub App installation ID - - environments: - - name: "Production" - type: "deployment" - connection_id: 1 # Your dbt Cloud connection ID - credential: - token_name: "databricks_token" # Maps to token_map - schema: "prod" - jobs: - - name: "Daily Production Run" - execute_steps: - - "dbt run" - - "dbt test" - triggers: - schedule: true - schedule_hours: [6] # 6 AM daily - schedule_days: [0, 1, 2, 3, 4] # Weekdays -``` - -!!! info "Git Clone Strategies" - Choose based on your Git provider: - - - **GitHub**: `github_app` (recommended) or `deploy_key` - - **GitLab**: `gitlab_deploy_token` or `deploy_key` - - **Azure DevOps**: `azure_active_directory_app` - - **Other**: `deploy_key` (universal SSH) +projects: + - name: Analytics + key: analytics + repository: + remote_url: "your-org/your-repo" # GitHub: "org/repo", or full URL + github_installation_id: 1234567 # From GitHub App integration + + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod # References global_connections[].key + credential: + credential_type: databricks + catalog: main + schema: analytics + + jobs: + - name: Daily Build + key: daily_build + environment_key: prod # References environments[].key + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: every_day + schedule_hours: [6] # 6 AM UTC +``` + +!!! info "connection_key" + Instead of a raw numeric `connection_id`, environments reference connections by `connection_key` — matching `global_connections[].key` in the YAML. This keeps your config portable and readable. + +!!! info "Jobs at project level" + Jobs are defined at the project level with an `environment_key` field, not nested inside environments. This makes them easier to read and reference by key for deferral. + +See the [YAML Schema](../configuration/yaml-schema.md) for all available fields including global connections, service tokens, Snowflake credentials, scheduled jobs, and more. ## Step 4: Initialize Terraform -Load your credentials and initialize: - ```bash -# Load credentials from .env source .env -# Initialize Terraform terraform init ``` You should see: -``` ``` Initializing modules... Downloading git::https://github.com/trouze/terraform-dbtcloud-yaml.git... -``` Terraform has been successfully initialized! ``` @@ -136,87 +143,79 @@ Always review what Terraform will create: terraform plan ``` -You'll see an output like: +You'll see output like: ``` -Plan: 8 to add, 0 to change, 0 to destroy. +Plan: 5 to add, 0 to change, 0 to destroy. -Changes to Outputs: - + project_id = (known after apply) - + repository_id = (known after apply) - + environment_ids = { - + "Production" = (known after apply) - } + + dbtcloud_project.projects["analytics"] + + dbtcloud_repository.repositories["analytics"] + + dbtcloud_environment.environments["analytics_prod"] + + dbtcloud_databricks_credential.credentials["analytics_prod"] + + dbtcloud_job.jobs["analytics_daily_build"] ``` -!!! tip "Understanding the Plan" - - **Resources being created**: Project, repository, environments, credentials, jobs - - **Nothing exists yet**: Resources are only created when you run `apply` - - **Review carefully**: Make sure connection IDs, URLs, and names are correct - ## Step 6: Apply Configuration -Create your dbt Cloud infrastructure: - ```bash terraform apply ``` -Type `yes` when prompted. Terraform will create: - -1. ✅ dbt Cloud project -2. ✅ Repository connection -3. ✅ Environments (Production, Development, etc.) -4. ✅ Credentials (database connections) -5. ✅ Jobs (scheduled runs, CI checks) +Type `yes` when prompted. Terraform will create your dbt Cloud project, environments, credentials, and jobs. ``` -Apply complete! Resources: 8 added, 0 changed, 0 destroyed. - -Outputs: -project_id = 12345 -repository_id = 67890 -environment_ids = { - "Production" = 11111 -} +Apply complete! Resources: 5 added, 0 changed, 0 destroyed. ``` ## Step 7: Verify in dbt Cloud 1. Log into [dbt Cloud](https://cloud.getdbt.com) 2. Navigate to your account -3. You should see your new project! -4. Check that environments and jobs are configured correctly +3. Confirm your project, environments, and jobs are configured correctly + +## Making Changes + +After initial setup, all changes go through the same loop: + +1. Edit `dbt-config.yml` +2. Run `terraform plan` to preview +3. Run `terraform apply` to apply + +```bash +source .env +terraform plan +terraform apply +``` ## What's Next?
-- :material-rocket:{ .lg .middle } __Customize Your Setup__ +- :material-file-document:{ .lg .middle } **YAML Schema** --- - Learn about all configuration options + All configuration options with types, defaults, and examples [:octicons-arrow-right-24: YAML Schema](../configuration/yaml-schema.md) -- :material-github-box:{ .lg .middle } __CI/CD Integration__ +- :material-github-box:{ .lg .middle } **CI/CD Integration** --- - Automate deployments with GitHub Actions + Automate plan on PR and apply on merge [:octicons-arrow-right-24: CI/CD Guide](../guides/cicd.md) -- :material-book-multiple:{ .lg .middle } __More Examples__ +- :material-key:{ .lg .middle } **Environment Variables** --- - See real-world use cases + Full credential variable reference - [:octicons-arrow-right-24: Examples](examples.md) + [:octicons-arrow-right-24: Environment Variables](../configuration/environment-variables.md) -- :material-lifebuoy:{ .lg .middle } __Need Help?__ +- :material-lifebuoy:{ .lg .middle } **Troubleshooting** --- @@ -226,21 +225,5 @@ environment_ids = {
-## Making Changes - -After initial setup, you can modify your configuration: - -1. Edit `dbt-config.yml` -2. Run `terraform plan` to preview changes -3. Run `terraform apply` to apply them - -```bash -# Example: Add a new environment -nano dbt-config.yml # Add new environment -source .env -terraform plan # Review changes -terraform apply # Apply changes -``` - -!!! success "You're All Set!" +!!! success "You're all set!" Your dbt Cloud project is now managed as code. All changes are tracked in Git and deployed via Terraform. diff --git a/docs/guides/best-practices.md b/docs/guides/best-practices.md index a55e3f8..d201890 100644 --- a/docs/guides/best-practices.md +++ b/docs/guides/best-practices.md @@ -10,8 +10,8 @@ Recommended patterns for managing dbt Cloud infrastructure with Terraform. ```bash # .env file (never commit) -export TF_VAR_dbt_api_token=dbtc_xxxxx -export TF_VAR_token_map='{"key":"value"}' +export TF_VAR_dbt_token=dbtc_xxxxx +export TF_VAR_environment_credentials='{"analytics_prod": {"credential_type": "databricks", "token": "dapi..."}}' ``` ❌ **Never hardcode** credentials: @@ -19,7 +19,7 @@ export TF_VAR_token_map='{"key":"value"}' ```hcl # DON'T DO THIS! variable "dbt_token" { - default = "dbtc_xxxxx" # ❌ Exposed in version control + default = "dbtc_xxxxx" # ❌ Never hardcode — exposed in version control } ``` @@ -340,7 +340,7 @@ Cache Terraform plugins: ```yaml - name: Cache Terraform Plugins - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.terraform.d/plugin-cache key: ${{ runner.os }}-terraform-${{ hashFiles('**/.terraform.lock.hcl') }} @@ -500,24 +500,19 @@ environments: enable_model_query_history: true # Staging: Production-like, limited access - - name: "Staging" - type: "deployment" - dbt_version: "1.7.1" # Pin to same as prod - jobs: - - name: "Staging CI" - triggers: - on_merge: true - deferring_environment: "Production" - + - name: Staging + key: staging + type: deployment + deployment_type: staging + dbt_version: "1.9.0" # Pin to same as prod + # Production: Locked down, scheduled - - name: "Production" - type: "deployment" - dbt_version: "1.7.1" - jobs: - - name: "Daily Run" - triggers: - schedule: true - schedule_hours: [6] + - name: Production + key: prod + type: deployment + deployment_type: production + dbt_version: "1.9.0" + protected: true ``` ### Job Naming @@ -539,15 +534,13 @@ jobs: ### Credential Mapping -Organize by environment and warehouse: +Use `environment_credentials` keyed by `"{project_key}_{env_key}"`: ```bash -export TF_VAR_token_map='{ - "dev_databricks_uc": "dapi_dev123", - "staging_databricks_uc": "dapi_stg456", - "prod_databricks_uc": "dapi_prd789", - "dev_snowflake": "sf_dev_abc", - "prod_snowflake": "sf_prd_xyz" +export TF_VAR_environment_credentials='{ + "analytics_dev": {"credential_type": "databricks", "token": "dapi_dev123"}, + "analytics_staging": {"credential_type": "databricks", "token": "dapi_stg456"}, + "analytics_prod": {"credential_type": "databricks", "token": "dapi_prd789"} }' ``` diff --git a/docs/guides/cicd.md b/docs/guides/cicd.md index bff1ce4..a749137 100644 --- a/docs/guides/cicd.md +++ b/docs/guides/cicd.md @@ -4,83 +4,44 @@ Automate your dbt Cloud infrastructure deployments using CI/CD pipelines. ## Overview -This module integrates seamlessly with popular CI/CD platforms: +The recommended pattern is two separate workflows: -- **GitHub Actions** - Native GitHub integration -- **GitLab CI/CD** - Built into GitLab -- **Azure DevOps Pipelines** - Microsoft Azure -- **Jenkins** - Self-hosted automation -- **CircleCI** - Cloud-based CI/CD +- **CI** (`ci.yml`) — runs on every PR, validates config and posts the Terraform plan as a comment +- **CD** (`cd.yml`) — runs on merge to main, applies the plan with an optional approval gate -All examples use environment variables for credentials, making them portable across platforms. +The `examples/basic/.github/workflows/` directory contains ready-to-use versions of both. --- ## GitHub Actions -### Basic Workflow +### Required Secrets -Deploy on push to main: +Set these in your repository: **Settings > Secrets and variables > Actions** -```yaml title=".github/workflows/dbt-cloud.yml" -name: Deploy dbt Cloud Infrastructure +| Secret | Description | Required | +|--------|-------------|----------| +| `DBT_ACCOUNT_ID` | Numeric dbt Cloud account ID | Yes | +| `DBT_TOKEN` | dbt Cloud API token | Yes | +| `DBT_PAT` | Personal access token (GitHub App integration only; can equal `DBT_TOKEN`) | Conditional | +| `ENVIRONMENT_CREDENTIALS` | JSON blob — see [Environment Variables](../configuration/environment-variables.md) | Yes (if using env credentials) | +| `CONNECTION_CREDENTIALS` | JSON blob for global connection OAuth/keys | If using global connections | +| `LINEAGE_TOKENS` | JSON blob for Tableau/Looker tokens | If using lineage integrations | +| `OAUTH_CLIENT_SECRETS` | JSON blob for OAuth configurations | If using OAuth | -on: - push: - branches: [main] - paths: - - 'dbt-config.yml' - - '**.tf' - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.6.0 - - - name: Terraform Init - run: terraform init - - - name: Terraform Plan - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - run: terraform plan -out=tfplan - - - name: Terraform Apply - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - run: terraform apply tfplan -``` +### CI — Plan on PR -### With Pull Request Preview +Runs on every pull request that touches `dbt-config.yml` or any `.tf` file. Posts the Terraform plan as a PR comment (updates the existing comment on re-push rather than stacking new ones). -Show plan in PR comments: - -```yaml title=".github/workflows/terraform-pr.yml" -name: Terraform PR Check +```yaml title=".github/workflows/ci.yml" +name: CI — Terraform Plan on: pull_request: + branches: [main] paths: - - 'dbt-config.yml' - - '**.tf' + - "dbt-config.yml" + - "**.tf" permissions: contents: read @@ -88,96 +49,171 @@ permissions: jobs: plan: + name: Validate and Plan runs-on: ubuntu-latest - + + env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + TF_VAR_connection_credentials: ${{ secrets.CONNECTION_CREDENTIALS }} + TF_VAR_lineage_tokens: ${{ secrets.LINEAGE_TOKENS }} + TF_VAR_oauth_client_secrets: ${{ secrets.OAUTH_CLIENT_SECRETS }} + steps: - - uses: actions/checkout@v3 - + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "~1" + - name: Terraform Init run: terraform init - + + - name: Terraform Validate + run: terraform validate + - name: Terraform Plan id: plan - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - run: terraform plan -no-color - continue-on-error: true - - - name: Comment Plan on PR - uses: actions/github-script@v6 + run: | + terraform plan -no-color -out=tfplan + terraform show -no-color tfplan > plan.txt + continue-on-error: true # Post comment even if plan fails + + - name: Post plan as PR comment + uses: actions/github-script@v7 with: script: | - const output = `#### Terraform Plan 📋 - - \`\`\` - ${{ steps.plan.outputs.stdout }} + const fs = require('fs'); + const raw = fs.readFileSync('plan.txt', 'utf8'); + + // Truncate if the plan is too large for a GitHub comment + const maxLen = 60000; + const plan = raw.length > maxLen + ? raw.slice(0, maxLen) + '\n\n... output truncated (full plan in Actions log)' + : raw; + + const status = '${{ steps.plan.outcome }}' === 'success' ? '✅' : '❌'; + const body = `### ${status} Terraform Plan + +
Show plan + + \`\`\`hcl + ${plan} \`\`\` - - *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; - - github.rest.issues.createComment({ - issue_number: context.issue.number, + +
+ + > Triggered by @${{ github.actor }} on \`${{ github.head_ref }}\``; + + // Replace any previous plan comment instead of stacking new ones + const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, - body: output - }) + issue_number: context.issue.number, + }); + const prev = comments.find(c => + c.user.type === 'Bot' && c.body.includes('Terraform Plan') + ); + if (prev) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: prev.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Fail if plan errored + if: steps.plan.outcome == 'failure' + run: exit 1 ``` -### Multi-Project Matrix +### CD — Apply on Merge -Deploy multiple projects in parallel: +Runs on push to main (i.e., after a PR merges). Uses a GitHub Environment (`production`) which can be configured with required reviewers for an approval gate before apply. -```yaml title=".github/workflows/multi-project.yml" -name: Deploy Multiple Projects +```yaml title=".github/workflows/cd.yml" +name: CD — Terraform Apply on: push: branches: [main] + paths: + - "dbt-config.yml" + - "**.tf" + +permissions: + contents: read jobs: - deploy: + apply: + name: Apply runs-on: ubuntu-latest - strategy: - matrix: - project: [finance, marketing, operations] - fail-fast: false - + environment: production # remove this line if you don't need an approval gate + + env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + TF_VAR_connection_credentials: ${{ secrets.CONNECTION_CREDENTIALS }} + TF_VAR_lineage_tokens: ${{ secrets.LINEAGE_TOKENS }} + TF_VAR_oauth_client_secrets: ${{ secrets.OAUTH_CLIENT_SECRETS }} + steps: - - uses: actions/checkout@v3 - + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "~1" + - name: Terraform Init run: terraform init - - - name: Deploy ${{ matrix.project }} - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./configs/${{ matrix.project }}.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - run: | - terraform workspace select -or-create ${{ matrix.project }} - terraform plan -out=tfplan - terraform apply tfplan + + - name: Terraform Plan + run: terraform plan -no-color -out=tfplan + + - name: Terraform Apply + run: terraform apply -auto-approve tfplan ``` +### Setting Up the Approval Gate + +To require a reviewer before applying to production: + +1. Go to **Settings > Environments** in your GitHub repository +2. Create an environment named `production` +3. Add **Required reviewers** +4. Optionally add branch protection rules (e.g., only allow deploys from `main`) + +Remove the `environment: production` line from `cd.yml` if you don't need this gate. + +### Remote State + +Before using these workflows in production, configure a [Terraform backend](https://developer.hashicorp.com/terraform/language/settings/backends/configuration) in `main.tf` (S3, GCS, Terraform Cloud, etc.). Without it, state is local and lost between CI runs. + --- ## GitLab CI/CD -### Basic Pipeline +### Masked Variables + +Store credentials in **Settings > CI/CD > Variables**. Mark all credential variables as **Masked** and **Protected**. ```yaml title=".gitlab-ci.yml" stages: @@ -186,59 +222,59 @@ stages: - apply variables: - TF_ROOT: ${CI_PROJECT_DIR} - TF_VAR_dbt_host_url: "https://cloud.getdbt.com/api" - TF_VAR_yaml_file_path: "./dbt-config.yml" + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" .terraform-base: image: hashicorp/terraform:latest before_script: - - cd ${TF_ROOT} - terraform init validate: extends: .terraform-base stage: validate script: - - terraform fmt -check - terraform validate plan: extends: .terraform-base stage: plan variables: - TF_VAR_dbt_account_id: ${DBT_ACCOUNT_ID} - TF_VAR_dbt_api_token: ${DBT_API_TOKEN} - TF_VAR_dbt_pat: ${DBT_PAT} - TF_VAR_token_map: ${TOKEN_MAP} + TF_VAR_dbt_account_id: $DBT_ACCOUNT_ID + TF_VAR_dbt_token: $DBT_TOKEN + TF_VAR_dbt_pat: $DBT_PAT + TF_VAR_environment_credentials: $ENVIRONMENT_CREDENTIALS + TF_VAR_connection_credentials: $CONNECTION_CREDENTIALS script: - terraform plan -out=tfplan artifacts: paths: - - ${TF_ROOT}/tfplan + - tfplan expire_in: 1 day apply: extends: .terraform-base stage: apply variables: - TF_VAR_dbt_account_id: ${DBT_ACCOUNT_ID} - TF_VAR_dbt_api_token: ${DBT_API_TOKEN} - TF_VAR_dbt_pat: ${DBT_PAT} - TF_VAR_token_map: ${TOKEN_MAP} + TF_VAR_dbt_account_id: $DBT_ACCOUNT_ID + TF_VAR_dbt_token: $DBT_TOKEN + TF_VAR_dbt_pat: $DBT_PAT + TF_VAR_environment_credentials: $ENVIRONMENT_CREDENTIALS + TF_VAR_connection_credentials: $CONNECTION_CREDENTIALS script: - terraform apply tfplan dependencies: - plan - only: - - main - when: manual + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: manual # Approval gate ``` --- ## Azure DevOps +Store credentials in **Pipelines > Library > Variable groups** (mark as secret). + ```yaml title="azure-pipelines.yml" trigger: branches: @@ -255,37 +291,32 @@ pool: variables: - group: dbt-cloud-credentials - name: TF_VAR_dbt_host_url - value: 'https://cloud.getdbt.com/api' - - name: TF_VAR_yaml_file_path - value: './dbt-config.yml' + value: 'https://cloud.getdbt.com' stages: - stage: Plan jobs: - job: TerraformPlan steps: - - task: TerraformInstaller@0 + - task: TerraformInstaller@1 inputs: terraformVersion: 'latest' - - - task: TerraformTaskV2@2 + + - task: TerraformTaskV4@4 displayName: 'Terraform Init' inputs: command: 'init' - workingDirectory: '$(System.DefaultWorkingDirectory)' - - - task: TerraformTaskV2@2 + + - task: TerraformTaskV4@4 displayName: 'Terraform Plan' inputs: command: 'plan' - workingDirectory: '$(System.DefaultWorkingDirectory)' - environmentServiceNameAzureRM: 'terraform-sp' env: TF_VAR_dbt_account_id: $(DBT_ACCOUNT_ID) - TF_VAR_dbt_api_token: $(DBT_API_TOKEN) + TF_VAR_dbt_token: $(DBT_TOKEN) TF_VAR_dbt_pat: $(DBT_PAT) - TF_VAR_token_map: $(TOKEN_MAP) - + TF_VAR_environment_credentials: $(ENVIRONMENT_CREDENTIALS) + - stage: Apply dependsOn: Plan condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) @@ -296,244 +327,86 @@ stages: runOnce: deploy: steps: - - task: TerraformTaskV2@2 + - task: TerraformTaskV4@4 displayName: 'Terraform Apply' inputs: command: 'apply' - workingDirectory: '$(System.DefaultWorkingDirectory)' env: TF_VAR_dbt_account_id: $(DBT_ACCOUNT_ID) - TF_VAR_dbt_api_token: $(DBT_API_TOKEN) + TF_VAR_dbt_token: $(DBT_TOKEN) TF_VAR_dbt_pat: $(DBT_PAT) - TF_VAR_token_map: $(TOKEN_MAP) + TF_VAR_environment_credentials: $(ENVIRONMENT_CREDENTIALS) ``` --- ## Best Practices -### 1. Secret Management +### Secret Management ✅ **DO:** -- Use platform-native secrets (GitHub Secrets, GitLab Variables, etc.) -- Mark secrets as "masked" or "protected" -- Use different secrets for dev/staging/prod -- Rotate secrets regularly +- Use platform-native secrets (GitHub Secrets, GitLab masked variables, Azure Library, key vault) +- Mark secrets as "masked" or "protected" so they never appear in logs +- Use the same secret names across environments for consistency +- Rotate tokens regularly ❌ **DON'T:** - Hardcode credentials in workflow files -- Echo secrets in logs -- Share secrets across unrelated projects +- Echo secrets in scripts +- Use personal tokens for automated workflows (use service account tokens) -### 2. Environment Separation +### Credential JSON Format -Use different workflows for environments: +JSON blob variables (`ENVIRONMENT_CREDENTIALS`, `CONNECTION_CREDENTIALS`, etc.) must be single-line JSON strings in CI/CD secrets: -```yaml -# Production -on: - push: - branches: [main] - -# Staging -on: - push: - branches: [staging] - -# Development -on: - push: - branches: [develop] -``` - -### 3. Approval Gates - -Require manual approval for production: - -**GitHub Actions:** -```yaml -environment: - name: production - url: https://cloud.getdbt.com ``` - -**GitLab CI/CD:** -```yaml -apply: - when: manual - only: - - main +{"analytics_prod": {"credential_type": "databricks", "token": "dapi...", "catalog": "main", "schema": "analytics"}} ``` -### 4. Plan Artifact - -Save plan output for review: - -```yaml -- name: Save Plan - run: terraform show -no-color tfplan > plan.txt - -- name: Upload Plan - uses: actions/upload-artifact@v3 - with: - name: terraform-plan - path: plan.txt +In `terraform.tfvars` (local use only), you can use HCL map syntax instead: + +```hcl +environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi..." + catalog = "main" + schema = "analytics" + } +} ``` -### 5. Parallel Execution +### Approval Gates -Use matrix strategy for multiple projects: +Require manual approval before production apply: -```yaml -strategy: - matrix: - project: [a, b, c] - fail-fast: false # Continue even if one fails - max-parallel: 3 # Limit concurrent jobs -``` +- **GitHub Actions**: `environment: production` with Required reviewers configured +- **GitLab CI**: `when: manual` on the apply job +- **Azure DevOps**: deployment environment with approval policies --- -## Troubleshooting - -### "No value for required variable" - -**Problem:** Secrets not loaded. - -**Solution:** -- Verify secrets are defined in CI/CD platform -- Check variable names match exactly -- Ensure workflow has access to secrets - -### "State Lock Timeout" - -**Problem:** Previous run didn't release state lock. - -**Solution:** -```yaml -- name: Force Unlock (emergency only) - run: terraform force-unlock -force -``` - -### "Plan Changes Unexpectedly" - -**Problem:** State drift or external changes. - -**Solution:** -- Run `terraform refresh` to sync state -- Review changes carefully -- Consider using `terraform import` for existing resources - ---- - -## Complete Example - -Putting it all together: - -```yaml title=".github/workflows/complete.yml" -name: dbt Cloud Infrastructure - -on: - pull_request: - paths: ['**.yml', '**.tf'] - push: - branches: [main] - paths: ['**.yml', '**.tf'] +## Next Steps -permissions: - contents: read - pull-requests: write +
-jobs: - terraform: - name: Terraform ${{ github.event_name == 'pull_request' && 'Plan' || 'Apply' }} - runs-on: ubuntu-latest - - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} - TF_VAR_dbt_host_url: https://cloud.getdbt.com/api - TF_VAR_yaml_file_path: ./dbt-config.yml - TF_VAR_token_map: ${{ secrets.TOKEN_MAP }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: 1.6.0 - - - name: Terraform Format Check - run: terraform fmt -check -recursive - - - name: Terraform Init - run: terraform init - - - name: Terraform Validate - run: terraform validate - - - name: Terraform Plan - id: plan - run: | - terraform plan -no-color -out=tfplan - terraform show -no-color tfplan > plan.txt - - - name: Comment Plan on PR - if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const plan = fs.readFileSync('plan.txt', 'utf8'); - const output = `#### Terraform Plan 📋 -
Show Plan - - \`\`\` - ${plan} - \`\`\` - -
- - *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }) - - - name: Terraform Apply - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - run: terraform apply -auto-approve tfplan - - - name: Upload Plan Artifact - if: always() - uses: actions/upload-artifact@v3 - with: - name: terraform-plan - path: plan.txt - retention-days: 30 -``` +- :material-key:{ .lg .middle } **Environment Variables** ---- + --- -## Next Steps + Full credential variable reference and setup instructions -
+ [:octicons-arrow-right-24: Environment Variables](../configuration/environment-variables.md) -- :material-folder-multiple:{ .lg .middle } __Multi-Project Setup__ +- :material-folder-multiple:{ .lg .middle } **Multi-Project Setup** --- - Manage multiple dbt projects + Manage multiple dbt projects in one repo [:octicons-arrow-right-24: Multi-Project Guide](../configuration/multi-project.md) -- :material-security:{ .lg .middle } __Best Practices__ +- :material-security:{ .lg .middle } **Best Practices** --- diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index e034873..f1a36d4 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -183,26 +183,29 @@ Error: dbt Cloud API error: 403 Forbidden --- -### "404 Not Found - Connection ID" +### "Connection Not Found" **Error Message:** ``` -Error: Cannot find connection with ID: 1234 +Error: Cannot find connection with key: "my_connection" ``` -**Cause:** Connection ID doesn't exist or is in different account. +**Cause:** The `connection_key` in your environment doesn't match any `global_connections[].key`. **Solutions:** -1. **Find correct connection ID:** - - In dbt Cloud: Admin > Connections - - Click connection, check URL: `/connections/{CONNECTION_ID}` +1. **Check your global_connections keys:** +```yaml +global_connections: + - name: Databricks Production + key: databricks_prod # ← this is the key +``` -2. **Update YAML:** +2. **Make sure your environment references it correctly:** ```yaml environments: - - name: "Production" - connection_id: 1234 # Use the correct ID + - name: Production + connection_key: databricks_prod # ← must match exactly ``` --- @@ -259,7 +262,7 @@ name: "My Project: Production" **Error Message:** ``` -Error: Missing required field: connection_id +Error: Missing required field ``` **Cause:** Required field not provided in YAML. @@ -270,12 +273,14 @@ Check the [YAML Schema](../configuration/yaml-schema.md) for required fields: ```yaml environments: - - name: "Production" # Required - type: "deployment" # Required - connection_id: 1001 # Required ← Add this! - credential: # Required - token_name: "token" # Required - schema: "analytics" # Required + - name: Production # Required + key: prod # Required + type: deployment # Required + deployment_type: production # Required for deployment envs + connection_key: my_conn # References global_connections[].key + credential: + credential_type: databricks + schema: analytics ``` --- @@ -374,25 +379,30 @@ Error: Token key 'prod_databricks_token' not found in token_map **Solutions:** -1. **Check YAML credential name:** +**Option A — Using `environment_credentials` (recommended):** ```yaml -credential: - token_name: "prod_databricks_token" # This key... +# In YAML: environment with key: prod in project with key: analytics +environments: + - name: Production + key: prod + credential: + credential_type: databricks + catalog: main + schema: analytics ``` - -2. **Must match token_map:** ```bash -export TF_VAR_token_map='{ - "prod_databricks_token": "dapi_abc123" - # ↑ Must match exactly -}' +# Key is "{project_key}_{env_key}" +export TF_VAR_environment_credentials='{"analytics_prod": {"credential_type": "databricks", "token": "dapi..."}}' ``` -3. **Or add to terraform.tfvars:** -```hcl -token_map = { - prod_databricks_token = "dapi_abc123" -} +**Option B — Using `token_map` (legacy Databricks):** +```yaml +credential: + token_name: "prod_databricks_token" # This key... +``` +```bash +export TF_VAR_token_map='{"prod_databricks_token": "dapi_abc123"}' +# ↑ Must match exactly ``` --- @@ -439,7 +449,9 @@ token_map = { ```yaml env: TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - # Secret name must match exactly ↑ + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + # Secret names must match exactly ↑ ``` 3. **Ensure workflow has access:** diff --git a/docs/index.md b/docs/index.md index 17a0461..30dcf04 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,211 +1,224 @@ # terraform-dbtcloud-yaml -[![Terraform Version](https://img.shields.io/badge/terraform-%3E%3D%201.0-blue?logo=terraform)](https://www.terraform.io) [![dbt Cloud Provider](https://img.shields.io/badge/dbt--cloud--provider-v1.3-blue)](https://registry.terraform.io/providers/dbt-labs/dbtcloud/latest) [![License](https://img.shields.io/badge/license-Apache%202.0-green)](https://github.com/trouze/terraform-dbtcloud-yaml/blob/main/LICENSE) +[![Terraform Version](https://img.shields.io/badge/terraform-%3E%3D%201.0-blue?logo=terraform)](https://www.terraform.io) [![dbt Cloud Provider](https://img.shields.io/badge/dbt--cloud--provider-v1.8-blue)](https://registry.terraform.io/providers/dbt-labs/dbtcloud/latest) [![License](https://img.shields.io/badge/license-Apache%202.0-green)](https://github.com/trouze/terraform-dbtcloud-yaml/blob/main/LICENSE) Manage your entire dbt Cloud setup with infrastructure-as-code using Terraform and YAML. Define projects, repositories, environments, credentials, and jobs in a single, human-readable YAML file. ## Why This Project Exists -Setting up dbt Cloud via Terraform requires writing complex HCL for every resource. This module simplifies that by letting you define your infrastructure in YAML - the language data teams already know. No need to learn Terraform syntax just to configure dbt. +dbt engineers already know YAML. They write it every day for models, sources, and tests. But managing dbt Cloud infrastructure — projects, environments, jobs, credentials — means writing Terraform HCL, a completely different language with its own mental model. -**Benefits:** - -- ✅ **YAML-based configuration** - intuitive for data engineers -- ✅ **Infrastructure as Code** - version control your dbt setup -- ✅ **Multi-provider Git support** - GitHub, GitLab, Azure DevOps, SSH -- ✅ **Complete resource management** - projects, repos, environments, credentials, jobs -- ✅ **Environment variable management** - set dbt variables alongside infrastructure -- ✅ **Reusable modules** - standardize your dbt deployments +This module bridges that gap: you describe your dbt Cloud setup in YAML, and Terraform handles the rest. No HCL required for day-to-day changes. -## Quick Start +**Benefits:** -Get started in 3 simple steps: +- **YAML-based configuration** — intuitive for data engineers +- **Infrastructure as Code** — version control your dbt Cloud setup +- **Multi-project support** — manage multiple projects from one YAML or one per team +- **Complete resource coverage** — projects, environments, jobs, global connections, service tokens, groups, notifications, IP restrictions, and more +- **Safe by default** — `protected: true` prevents accidental `terraform destroy` on critical resources +- **CI/CD ready** — GitHub Actions workflows included in the example ## Quick Start -Get started in 3 simple steps: +=== "Step 1: Create main.tf" + + ```hcl + terraform { + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } + } + + module "dbt_cloud" { + source = "github.com/trouze/terraform-dbtcloud-yaml" + + dbt_account_id = var.dbt_account_id + dbt_token = var.dbt_token + dbt_host_url = var.dbt_host_url + yaml_file = "${path.module}/dbt-config.yml" + environment_credentials = var.environment_credentials + } + ``` -=== "Step 1: Clone the Example" +=== "Step 2: Create dbt-config.yml" - ```bash - # Clone or copy the basic example - git clone https://github.com/trouze/terraform-dbtcloud-yaml.git - cd terraform-dbtcloud-yaml/examples/basic - - # Or copy to your own directory - cp -r examples/basic my-dbt-setup - cd my-dbt-setup + ```yaml + projects: + - name: Analytics + key: analytics + repository: + remote_url: "your-org/your-repo" + github_installation_id: 1234567 + + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + connection_key: databricks_prod # references global_connections[].key + credential: + credential_type: databricks + catalog: main + schema: analytics + + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + schedule_type: every_day + schedule_hours: [6] ``` -=== "Step 2: Configure Credentials" +=== "Step 3: Set credentials" ```bash - # Create .env file from example - cp .env.example .env + # In CI/CD, set these as GitHub Secrets (never hardcode values here) + # For local dev, export them before running terraform - # Edit with your dbt Cloud credentials export TF_VAR_dbt_account_id=12345 - export TF_VAR_dbt_api_token=dbtc_xxxxx - export TF_VAR_dbt_pat=dbtc_xxxxx - export TF_VAR_dbt_host_url=https://cloud.getdbt.com/api - export TF_VAR_yaml_file_path=./dbt-config.yml + export TF_VAR_dbt_token=dbtc_your_api_token + export TF_VAR_dbt_host_url=https://cloud.getdbt.com + + # Environment credentials — JSON blob keyed by "{project_key}_{env_key}" + export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi...", + "catalog": "main", + "schema": "analytics" + } + }' ``` -=== "Step 3: Deploy" +=== "Step 4: Deploy" ```bash - # Load credentials - source .env - - # Initialize and deploy terraform init terraform plan terraform apply ``` !!! success "That's it!" - Your dbt Cloud project is now managed with infrastructure-as-code! + Your dbt Cloud project is now managed as code. ## Features -### YAML Configuration - -Define everything in one file: - -```yaml -project: - name: "my-dbt-project" - repository: - remote_url: "https://github.com/myorg/myrepo.git" - git_clone_strategy: "github_app" - github_installation_id: 123456 - - environments: - - name: "Production" - type: "deployment" - connection_id: 1 - jobs: - - name: "daily_run" - execute_steps: - - "dbt run" - triggers: - schedule: true - schedule_hours: [6] -``` - -### Multi-Provider Git Support - -Automatically configures your Git provider: - -- **GitHub** with GitHub App -- **GitLab** with Deploy Token -- **Azure DevOps** with Azure AD -- **SSH** Deploy Key (universal) - -### Secure Credentials - -- Keep secrets in `.env` or GitHub Secrets -- Never commit sensitive values -- Support for database credentials as environment variables - -## Use Cases +### Supported Resources -
+| Scope | Resources | +|-------|-----------| +| Account | Projects, global connections, service tokens, groups, user groups, notifications, OAuth configurations, IP restrictions, account features | +| Project | Repository, environments, credentials (14 warehouse types), jobs, environment variables, extended attributes, profiles, lineage integrations, project artefacts, semantic layer | -- :material-rocket-launch:{ .lg .middle } **Single dbt Project** +### Credential Types - --- +Databricks, Snowflake (password + keypair), BigQuery, Postgres, Redshift, Athena, Fabric, Synapse, Starburst, Trino, Spark, Teradata — all managed from YAML. - Get started quickly with a single project +### Multi-Project - ```bash - terraform apply - ``` +Manage multiple dbt Cloud projects from one YAML file: -- :material-git:{ .lg .middle } **Multiple Projects** +```yaml +projects: + - name: Finance Analytics + key: finance + ... + - name: Marketing Analytics + key: marketing + ... +``` - --- +Or one YAML file per team using `yaml_file` variable: - Run multiple dbt projects in parallel CI/CD +```bash +terraform apply -var="yaml_file=./configs/finance.yml" +``` - ```bash - # Run each project separately - for config in configs/*.yml; do - terraform plan -var="yaml_file_path=$config" - done - ``` +### Credential Keys -- :material-github:{ .lg .middle } **CI/CD Pipeline** +Sensitive values are never in the YAML. They're passed as Terraform variables and matched by key: - --- +| Variable | Key format | Matches in YAML | +|---|---|---| +| `environment_credentials` | `"project_key_env_key"` | Environment `credential:` block | +| `connection_credentials` | `"connection_key"` | `global_connections[].key` | +| `token_map` | `"token_name"` | `credential.token_name` (legacy Databricks) | +| `lineage_tokens` | `"project_key_integration_key"` | `lineage_integrations[].key` composite | +| `oauth_client_secrets` | `"oauth_config_key"` | `oauth_configurations[].key` | - Automate deployments with GitHub Actions +### Protection Lifecycle - ```yaml - # GitHub Actions example - - env: - TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} - TF_VAR_dbt_api_token: ${{ secrets.DBT_API_TOKEN }} - run: terraform apply - ``` +Set `protected: true` on any resource to prevent accidental deletion: -
+```yaml +environments: + - name: Production + key: prod + protected: true # terraform destroy will fail for this resource +``` ## Requirements - Terraform >= 1.0 -- dbt Cloud account +- dbt Cloud account with admin access - dbt Cloud API token -- Git repository for your dbt models ## What's Next?
-- :material-rocket-launch:{ .lg .middle } **Getting Started** +- :material-rocket-launch:{ .lg .middle } **Quick Start** --- - Follow the quick start guide to deploy your first dbt Cloud project + Deploy your first dbt Cloud project in minutes [:octicons-arrow-right-24: Quick Start](getting-started/quickstart.md) -- :material-file-document:{ .lg .middle } **Configuration** +- :material-file-document:{ .lg .middle } **YAML Schema** --- - Learn about YAML schema and configuration options + Full reference for every field in `dbt-config.yml` - [:octicons-arrow-right-24: Configuration](configuration/yaml-schema.md) + [:octicons-arrow-right-24: YAML Schema](configuration/yaml-schema.md) -- :material-book-open-variant:{ .lg .middle } **Examples** +- :material-github-box:{ .lg .middle } **CI/CD Guide** --- - Explore real-world examples and use cases + GitHub Actions workflows for plan on PR and apply on merge - [:octicons-arrow-right-24: Examples](getting-started/examples.md) + [:octicons-arrow-right-24: CI/CD Guide](guides/cicd.md) -- :material-github:{ .lg .middle } **Reference** +- :material-book-open-variant:{ .lg .middle } **Examples** --- - Complete Terraform module API documentation + Real-world configuration examples - [:octicons-arrow-right-24: Module API](reference/terraform.md) + [:octicons-arrow-right-24: Examples](getting-started/examples.md)
## Community & Support -- 📖 **Documentation** - You're reading it! -- 🐛 **Issues** - [Report bugs or request features](https://github.com/trouze/terraform-dbtcloud-yaml/issues) -- 💬 **Discussions** - [Share ideas and best practices](https://github.com/trouze/terraform-dbtcloud-yaml/discussions) +- 📖 **Documentation** — You're reading it! +- 🐛 **Issues** — [Report bugs or request features](https://github.com/trouze/terraform-dbtcloud-yaml/issues) +- 💬 **Discussions** — [Share ideas and best practices](https://github.com/trouze/terraform-dbtcloud-yaml/discussions) ## License -This project is licensed under Apache License 2.0. See [LICENSE](https://github.com/trouze/terraform-dbtcloud-yaml/blob/main/LICENSE) for details. +Apache License 2.0. See [LICENSE](https://github.com/trouze/terraform-dbtcloud-yaml/blob/main/LICENSE) for details. --- -**Ready to manage your dbt Cloud with code?** Start with the [Quick Start Guide](getting-started/quickstart.md)! +**Ready to manage your dbt Cloud with code?** Start with the [Quick Start Guide](getting-started/quickstart.md). diff --git a/docs/reference/terraform.md b/docs/reference/terraform.md index d4b3219..035bab1 100644 --- a/docs/reference/terraform.md +++ b/docs/reference/terraform.md @@ -3,7 +3,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [dbtcloud](#requirement\_dbtcloud) | ~> 1.3 | +| [dbtcloud](#requirement\_dbtcloud) | ~> 1.8 | ## Providers @@ -13,14 +13,27 @@ No providers. | Name | Source | Version | |------|--------|---------| +| [account\_features](#module\_account\_features) | ./modules/account_features | n/a | | [credentials](#module\_credentials) | ./modules/credentials | n/a | | [environment\_variable\_job\_overrides](#module\_environment\_variable\_job\_overrides) | ./modules/environment_variable_job_overrides | n/a | | [environment\_variables](#module\_environment\_variables) | ./modules/environment_variables | n/a | | [environments](#module\_environments) | ./modules/environments | n/a | +| [extended\_attributes](#module\_extended\_attributes) | ./modules/extended_attributes | n/a | +| [global\_connections](#module\_global\_connections) | ./modules/global_connections | n/a | +| [groups](#module\_groups) | ./modules/groups | n/a | +| [ip\_restrictions](#module\_ip\_restrictions) | ./modules/ip_restrictions | n/a | | [jobs](#module\_jobs) | ./modules/jobs | n/a | +| [lineage\_integrations](#module\_lineage\_integrations) | ./modules/lineage_integrations | n/a | +| [notifications](#module\_notifications) | ./modules/notifications | n/a | +| [oauth\_configurations](#module\_oauth\_configurations) | ./modules/oauth_configurations | n/a | +| [profiles](#module\_profiles) | ./modules/profiles | n/a | | [project](#module\_project) | ./modules/project | n/a | +| [project\_artefacts](#module\_project\_artefacts) | ./modules/project_artefacts | n/a | | [project\_repository](#module\_project\_repository) | ./modules/project_repository | n/a | | [repository](#module\_repository) | ./modules/repository | n/a | +| [semantic\_layer](#module\_semantic\_layer) | ./modules/semantic_layer | n/a | +| [service\_tokens](#module\_service\_tokens) | ./modules/service_tokens | n/a | +| [user\_groups](#module\_user\_groups) | ./modules/user_groups | n/a | ## Resources @@ -31,19 +44,26 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [dbt\_account\_id](#input\_dbt\_account\_id) | dbt Cloud account ID | `number` | n/a | yes | -| [dbt\_host\_url](#input\_dbt\_host\_url) | dbt Cloud host URL (e.g., https://cloud.getdbt.com or custom domain) | `string` | n/a | yes | -| [dbt\_pat](#input\_dbt\_pat) | n/a | `string` | `""` | no | +| [dbt\_host\_url](#input\_dbt\_host\_url) | dbt Cloud host URL (e.g., https://cloud.getdbt.com) | `string` | n/a | yes | +| [dbt\_pat](#input\_dbt\_pat) | dbt Cloud personal access token (for GitHub App integration; can equal dbt\_token) | `string` | `null` | no | | [dbt\_token](#input\_dbt\_token) | dbt Cloud API token for authentication | `string` | n/a | yes | -| [target\_name](#input\_target\_name) | Default target name for the dbt project (e.g., 'dev', 'prod') | `string` | `""` | no | -| [token\_map](#input\_token\_map) | Map of credential token names to their actual values (e.g., Databricks tokens). Token names should correspond to credential.token\_name in YAML. | `map(string)` | `{}` | no | -| [yaml\_file](#input\_yaml\_file) | Path to the YAML file defining dbt Cloud resources (projects, environments, jobs, etc.) | `string` | n/a | yes | +| [yaml\_file](#input\_yaml\_file) | Path to the YAML file defining dbt Cloud resources | `string` | n/a | yes | +| [target\_name](#input\_target\_name) | Default target name for dbt jobs (e.g., 'prod') | `string` | `""` | no | +| [token\_map](#input\_token\_map) | Map of Databricks token names to values. Key corresponds to `credential.token_name` in YAML. | `map(string)` | `{}` | no | +| [environment\_credentials](#input\_environment\_credentials) | Map of environment credential objects keyed by `"{project_key}_{env_key}"`. Each object must include `credential_type` and type-specific fields. Supports 14 warehouse types. | `map(any)` | `{}` | no | +| [connection\_credentials](#input\_connection\_credentials) | Map of global connection keys to OAuth/auth credential objects. Key corresponds to `global_connections[].key` in YAML. | `map(any)` | `{}` | no | +| [lineage\_tokens](#input\_lineage\_tokens) | Map of lineage integration tokens keyed by `"{project_key}_{integration_key}"`. | `map(string)` | `{}` | no | +| [oauth\_client\_secrets](#input\_oauth\_client\_secrets) | Map of OAuth configuration keys to their client secrets. Key corresponds to `oauth_configurations[].key` in YAML. | `map(string)` | `{}` | no | ## Outputs | Name | Description | |------|-------------| -| [credential\_ids](#output\_credential\_ids) | Map of credential names to their dbt Cloud IDs | -| [environment\_ids](#output\_environment\_ids) | Map of environment names to their dbt Cloud IDs | -| [job\_ids](#output\_job\_ids) | Map of job names to their dbt Cloud IDs | -| [project\_id](#output\_project\_id) | The dbt Cloud project ID | -| [repository\_id](#output\_repository\_id) | The dbt Cloud repository ID | +| [project\_ids](#output\_project\_ids) | Map of project keys to their dbt Cloud IDs | +| [environment\_ids](#output\_environment\_ids) | Map of environment keys (project\_key\_env\_key) to their dbt Cloud IDs | +| [job\_ids](#output\_job\_ids) | Map of job keys to their dbt Cloud IDs | +| [credential\_ids](#output\_credential\_ids) | Map of credential keys to their dbt Cloud IDs | +| [repository\_ids](#output\_repository\_ids) | Map of project keys to their dbt Cloud repository IDs | +| [connection\_ids](#output\_connection\_ids) | Map of global connection keys to their dbt Cloud IDs | +| [service\_token\_ids](#output\_service\_token\_ids) | Map of service token keys to their dbt Cloud IDs | +| [group\_ids](#output\_group\_ids) | Map of group keys to their dbt Cloud IDs | diff --git a/examples/README.md b/examples/README.md index cb26a90..196bbcd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,122 +1,105 @@ # Examples -## Quick Start +## basic/ -1. **Copy the example** - ```bash - cp -r examples/basic my-dbt-project - cd my-dbt-project - ``` +Minimal working example. Copy it, fill in your values, and deploy. -2. **Set up credentials** - ```bash - cp .env.example .env - # Edit .env with your dbt Cloud credentials - ``` +``` +examples/basic/ +├── main.tf # Root module — provider + module block +├── variables.tf # All input variable definitions with descriptions +└── dbt-config.yml # Full annotated YAML schema example +``` + +### Steps + +```bash +cp -r examples/basic my-dbt-setup +cd my-dbt-setup +``` -3. **Deploy** - ```bash - source .env - terraform init - terraform plan - terraform apply - ``` +Create `terraform.tfvars`: -## Managing Multiple Projects +```hcl +dbt_account_id = 12345 +dbt_token = "dbt_your_api_token" +dbt_host_url = "https://cloud.getdbt.com" -Store multiple YAML configs and switch between them: +# Key format: "{project_key}_{env_key}" +environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi..." + catalog = "main" + schema = "analytics" + } +} +``` + +Edit `dbt-config.yml` with your project details, then: ```bash -# Directory structure -configs/ - ├── finance.yml - ├── marketing.yml - └── operations.yml - -# Deploy specific project -source .env -terraform plan -var="yaml_file_path=./configs/finance.yml" -terraform apply -var="yaml_file_path=./configs/finance.yml" - -# Or in GitHub Actions (parallel execution) -terraform plan -var="yaml_file_path=./configs/${{ matrix.project }}.yml" +terraform init +terraform plan +terraform apply ``` -## Available Examples +--- + +## Credential keys + +Sensitive values are never in the YAML. They're passed as Terraform variables and matched to YAML resources by key. + +| Variable | Key format | Matches in YAML | +|---|---|---| +| `token_map` | `"token_name"` | `credential.token_name` | +| `environment_credentials` | `"project_key_env_key"` | Environment `credential:` block | +| `connection_credentials` | `"connection_key"` | `global_connections[].key` | +| `lineage_tokens` | `"project_key_integration_key"` | `lineage_integrations[].key` composite | +| `oauth_client_secrets` | `"oauth_config_key"` | `oauth_configurations[].key` | + +Keys use underscores and must exactly match the `key:` values in your YAML. -- **basic/** - Minimal working example with environment variable setup +--- -## YAML Configuration Spec +## Multiple teams / projects -The YAML file defines all dbt Cloud resources. Below is the complete specification: +**Option A — one YAML, multiple projects in a list:** +```yaml +projects: + - name: Finance Analytics + key: finance + ... + - name: Marketing Analytics + key: marketing + ... +``` + +**Option B — one YAML file per team, one Terraform workspace per team:** +```bash +terraform apply -var="yaml_file=./configs/finance.yml" +terraform apply -var="yaml_file=./configs/marketing.yml" +``` + +--- + +## CI/CD (GitHub Actions) ```yaml -project: - name: # Required. Name of the dbt project. - repository: - remote_url: # Required. URL of the remote Git repository. - gitlab_project_id: # Optional. GitLab project ID if using GitLab integration. - environments: - - name: # Required. Name of the environment. - credential: - token_name: # Optional. Name of the token to use. - schema: # Optional. Schema to be used. - catalog: # Optional. Catalog to be used. - connection_id: # Required. Connection ID for the environment. - type: # Required. Type of environment. Allowed values: 'development', 'deployment'. - dbt_version: # Optional. dbt version to use. Defaults to "latest". - enable_model_query_history: # Optional. Enable model query history. Defaults to false. - custom_branch: # Optional. Custom branch for dbt. Defaults to null. - deployment_type: # Optional. Deployment type (e.g., 'production'). Defaults to null. - jobs: - - name: # Required. Name of the job. - execute_steps: - - # Required. Steps to execute in the job. - triggers: - github_webhook: # Required. Trigger job on GitHub webhook. - git_provider_webhook: # Required. Trigger job on Git provider webhook. - schedule: # Required. Trigger job on a schedule. - on_merge: # Required. Trigger job on merge. - dbt_version: # Optional. dbt version for the job. Defaults to "latest". - deferring_environment: # Optional. Enable deferral of job to environment. Defaults to no deferral. - description: # Optional. Description of the job. Defaults to null. - errors_on_lint_failure: # Optional. Fail job on lint errors. Defaults to true. - generate_docs: # Optional. Generate docs. Defaults to false. - is_active: # Optional. Whether the job is active. Defaults to true. - num_threads: # Optional. Number of threads for the job. Defaults to 4. - run_compare_changes: # Optional. Compare changes before running. Defaults to false. - run_generate_sources: # Optional. Generate sources before running. Defaults to false. - run_lint: # Optional. Run lint before running. Defaults to false. - schedule_cron: # Optional. Cron schedule for the job. Defaults to null. - schedule_days: of # Optional. Days for schedule. Defaults to null. e.g. [0, 1, 2] - schedule_hours: of # Optional. Hours for schedule. Defaults to null. e.g. [0, 1, 2] - schedule_interval: # Optional. Interval for schedule. Defaults to null. - schedule_type: # Optional. Type of schedule. Defaults to null. - self_deferring: # Optional. Whether the job is self-deferring. Defaults to false. - target_name: # Optional. Target name for the job. Defaults to null. - timeout_seconds: # Optional. Job timeout in seconds. Defaults to 0. - triggers_on_draft_pr: # Optional. Trigger job on draft PRs. Defaults to false. - env_var_overrides: - : # Optional. Specify a job env var override - environment_variables: - - name: DBT_ # Required. Name of the environment variable. Starts with DBT_ - environment_values: - - env: project - value: # Optional. Environment value - - env: Production - value: # Optional. Environment value - - env: UAT - value: # Optional. Environment value - - env: Development - value: # Optional. Environment value - - name: DBT_SECRET_ # Required. Name of the secret environment variable. Starts with DBT_SECRET_ - environment_values: - - env: project - value: secret_ # Optional. Environment value - - env: Production - value: secret_ # Optional. Environment value - - env: UAT - value: secret_ # Optional. Environment value - - env: Development - value: secret_ # Optional. Environment value -``` \ No newline at end of file +- name: Deploy dbt Cloud config + env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_API_TOKEN }} + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS_JSON }} + run: | + terraform init + terraform apply -auto-approve +``` + +Store `environment_credentials` as a JSON-encoded secret. + +--- + +## Full schema reference + +See [docs/configuration/yaml-schema.md](../docs/configuration/yaml-schema.md) for every field with types, defaults, and examples. diff --git a/examples/basic/.env.example b/examples/basic/.env.example index 2325cfc..f7db947 100644 --- a/examples/basic/.env.example +++ b/examples/basic/.env.example @@ -1,21 +1,99 @@ -# dbt-terraform-modules-yaml Test Repository Environment Variables# dbt-terraform-modules-yaml Test Repository Environment Variables +# This file documents the environment variables consumed by Terraform. +# +# In CI/CD, set these as secrets — never hardcode values in this file: +# GitHub Actions: Settings > Secrets and variables > Actions +# GitLab CI: Settings > CI/CD > Variables (mask all of them) +# Azure Pipelines: Library > Variable groups (mark as secret) +# Key vault: Retrieve at pipeline start and export before terraform runs +# +# For local development only, copy to .env, fill in values, and run: +# source .env && terraform plan +# Add .env to .gitignore — never commit it. -# Source this file before running terraform: source .env# Copy this to .env.local or source it before running terraform +# --------------------------------------------------------------- +# dbt Cloud connection +# --------------------------------------------------------------- +export TF_VAR_dbt_account_id=12345 +export TF_VAR_dbt_token="dbt_your_api_token" +export TF_VAR_dbt_host_url="https://cloud.getdbt.com" +# Personal Access Token — only needed for GitHub App repository integration. +# Can be the same value as dbt_token for most setups. +export TF_VAR_dbt_pat="dbt_your_api_token" -# dbt Cloud API credentials (REQUIRED)# dbt Cloud API credentials +# --------------------------------------------------------------- +# Environment credentials +# +# Key format: "{project_key}_{env_key}" — must match the key: +# values in dbt-config.yml exactly. +# +# In CI/CD, store the entire JSON blob as a single masked secret +# (e.g. DBT_ENVIRONMENT_CREDENTIALS in GitHub/GitLab), then export: +# export TF_VAR_environment_credentials="$DBT_ENVIRONMENT_CREDENTIALS" +# --------------------------------------------------------------- -export TF_VAR_dbt_account_id=51798 -export TF_VAR_dbt_api_token=dbtc_-abcdefghijklmnopqrstuvwxyz -export TF_VAR_dbt_pat=dbtc_-abcdefghijklmnopqrstuvwxyz -# dbt Cloud host URL# Optional: dbt Cloud host URL (defaults to https://cloud.getdbt.com) +# Databricks +export TF_VAR_environment_credentials='{ + "analytics_prod": { + "credential_type": "databricks", + "token": "dapi_your_databricks_token", + "catalog": "main", + "schema": "analytics" + }, + "analytics_dev": { + "credential_type": "databricks", + "token": "dapi_your_dev_token", + "catalog": "main", + "schema": "analytics_dev" + } +}' -export TF_VAR_dbt_host_url=https://cloud.getdbt.com/api +# Snowflake (replace the block above if using Snowflake) +# export TF_VAR_environment_credentials='{ +# "analytics_prod": { +# "credential_type": "snowflake", +# "auth_type": "password", +# "user": "DBT_SERVICE_USER", +# "password": "your_password", +# "schema": "ANALYTICS", +# "database": "ANALYTICS", +# "warehouse": "TRANSFORMING", +# "role": "TRANSFORMER", +# "num_threads": 8 +# } +# }' +# --------------------------------------------------------------- +# Connection credentials +# Only needed when global_connections use OAuth or service principal +# auth. Most token-based setups can omit this entirely. +# +# In CI/CD, store as a single masked secret (e.g. DBT_CONNECTION_CREDENTIALS). +# Key corresponds to global_connections[].key in dbt-config.yml. +# --------------------------------------------------------------- -# Databricks credentials for the environments# Optional: database credentials as key-value pairs (defaults to {}) +# export TF_VAR_connection_credentials='{ +# "databricks_prod": { +# "client_id": "your_client_id", +# "client_secret": "your_client_secret" +# } +# }' -# NOTE: These are placeholder values - replace with actual tokens before apply# Format: use JSON or HCL map syntax +# --------------------------------------------------------------- +# Lineage integration tokens (Tableau, Looker, etc.) +# Key format: "{project_key}_{integration_key}" +# --------------------------------------------------------------- -export TF_VAR_token_map='{"dbtcloud_databricks_token_dbt_svcprincipal_ucpr":"dapi1234567890abcdefghijklmnopqrstuvwxyz","dbtcloud_databricks_token_dbt_svcprincipal_ucnp":"dapi0987654321zyxwvutsrqponmlkjihgfedcba"}' +# export TF_VAR_lineage_tokens='{ +# "analytics_tableau_prod": "your_tableau_token" +# }' + +# --------------------------------------------------------------- +# OAuth configuration client secrets +# Key corresponds to oauth_configurations[].key in dbt-config.yml. +# --------------------------------------------------------------- + +# export TF_VAR_oauth_client_secrets='{ +# "snowflake_oauth": "your_oauth_client_secret" +# }' diff --git a/examples/basic/.github/workflows/cd.yml b/examples/basic/.github/workflows/cd.yml new file mode 100644 index 0000000..b37e063 --- /dev/null +++ b/examples/basic/.github/workflows/cd.yml @@ -0,0 +1,54 @@ +# CD — runs on merge to main and applies the Terraform plan. +# +# Uses a GitHub Environment ("production") which can be configured with +# required reviewers for an approval gate before apply: +# Settings > Environments > production > Required reviewers +# +# Uses the same secrets as ci.yml — see that file for the full list +# and setup instructions. + +name: CD — Terraform Apply + +on: + push: + branches: [main] + paths: + - "dbt-config.yml" + - "**.tf" + +permissions: + contents: read + +jobs: + apply: + name: Apply + runs-on: ubuntu-latest + environment: production # remove this line if you don't need an approval gate + + env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + TF_VAR_connection_credentials: ${{ secrets.CONNECTION_CREDENTIALS }} + TF_VAR_lineage_tokens: ${{ secrets.LINEAGE_TOKENS }} + TF_VAR_oauth_client_secrets: ${{ secrets.OAUTH_CLIENT_SECRETS }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "~1" + + - name: Terraform Init + run: terraform init + + - name: Terraform Plan + run: terraform plan -no-color -out=tfplan + + - name: Terraform Apply + run: terraform apply -auto-approve tfplan diff --git a/examples/basic/.github/workflows/ci.yml b/examples/basic/.github/workflows/ci.yml new file mode 100644 index 0000000..7f9d379 --- /dev/null +++ b/examples/basic/.github/workflows/ci.yml @@ -0,0 +1,124 @@ +# CI — runs on every PR that touches config or Terraform files. +# Validates the configuration and posts the plan as a PR comment +# so reviewers can see exactly what will change before merging. +# +# Required GitHub secrets (Settings > Secrets and variables > Actions): +# DBT_ACCOUNT_ID — numeric account ID +# DBT_TOKEN — dbt Cloud API token +# DBT_PAT — personal access token (GitHub App integration only; +# can be the same value as DBT_TOKEN otherwise) +# ENVIRONMENT_CREDENTIALS — JSON blob, see .env.example for shape +# +# Optional secrets (omit if not used): +# CONNECTION_CREDENTIALS — JSON blob for OAuth/service principal connections +# LINEAGE_TOKENS — JSON blob for Tableau/Looker integrations +# OAUTH_CLIENT_SECRETS — JSON blob for OAuth configurations +# +# Remote state: configure a backend in main.tf (S3, GCS, Terraform Cloud, etc.) +# before using these workflows in production. Without it, state is local and +# lost between runs. + +name: CI — Terraform Plan + +on: + pull_request: + branches: [main] + paths: + - "dbt-config.yml" + - "**.tf" + +permissions: + contents: read + pull-requests: write + +jobs: + plan: + name: Validate and Plan + runs-on: ubuntu-latest + + env: + TF_VAR_dbt_account_id: ${{ secrets.DBT_ACCOUNT_ID }} + TF_VAR_dbt_token: ${{ secrets.DBT_TOKEN }} + TF_VAR_dbt_pat: ${{ secrets.DBT_PAT }} + TF_VAR_dbt_host_url: "https://cloud.getdbt.com" + TF_VAR_environment_credentials: ${{ secrets.ENVIRONMENT_CREDENTIALS }} + TF_VAR_connection_credentials: ${{ secrets.CONNECTION_CREDENTIALS }} + TF_VAR_lineage_tokens: ${{ secrets.LINEAGE_TOKENS }} + TF_VAR_oauth_client_secrets: ${{ secrets.OAUTH_CLIENT_SECRETS }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "~1" + + - name: Terraform Init + run: terraform init + + - name: Terraform Validate + run: terraform validate + + - name: Terraform Plan + id: plan + run: | + terraform plan -no-color -out=tfplan + terraform show -no-color tfplan > plan.txt + continue-on-error: true # Post comment even if plan fails + + - name: Post plan as PR comment + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const raw = fs.readFileSync('plan.txt', 'utf8'); + + // Truncate if the plan is too large for a GitHub comment + const maxLen = 60000; + const plan = raw.length > maxLen + ? raw.slice(0, maxLen) + '\n\n... output truncated (full plan in Actions log)' + : raw; + + const status = '${{ steps.plan.outcome }}' === 'success' ? '✅' : '❌'; + const body = `### ${status} Terraform Plan + +
Show plan + + \`\`\`hcl + ${plan} + \`\`\` + +
+ + > Triggered by @${{ github.actor }} on \`${{ github.head_ref }}\``; + + // Replace any previous plan comment instead of stacking new ones + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const prev = comments.find(c => + c.user.type === 'Bot' && c.body.includes('Terraform Plan') + ); + if (prev) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: prev.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Fail if plan errored + if: steps.plan.outcome == 'failure' + run: exit 1 diff --git a/examples/basic/.tflint.hcl b/examples/basic/.tflint.hcl new file mode 100644 index 0000000..3a1f5ff --- /dev/null +++ b/examples/basic/.tflint.hcl @@ -0,0 +1,11 @@ +# Examples are end-user starter code, not module source — skip all linting. +rule "terraform_required_version" { enabled = false } +rule "terraform_required_providers" { enabled = false } +rule "terraform_documented_variables" { enabled = false } +rule "terraform_documented_outputs" { enabled = false } +rule "terraform_naming_convention" { enabled = false } +rule "terraform_comment_syntax" { enabled = false } +rule "terraform_unused_declarations" { enabled = false } +rule "terraform_unused_required_providers" { enabled = false } +rule "terraform_standard_module_structure" { enabled = false } +rule "terraform_module_pinned_source" { enabled = false } diff --git a/examples/basic/README.md b/examples/basic/README.md new file mode 100644 index 0000000..e4bb3be --- /dev/null +++ b/examples/basic/README.md @@ -0,0 +1,81 @@ +# Basic Example + +The fastest way to get a dbt Cloud project, environments, and a scheduled job under Terraform management — using only YAML. + +## What's here + +| File | Purpose | +|---|---| +| `dbt-config.yml` | Your dbt Cloud configuration (edit this) | +| `main.tf` | Wires the YAML file into the module (no edits needed) | +| `variables.tf` | Input variable declarations | +| `terraform.tfvars.example` | Credential template — copy to `terraform.tfvars` | +| `.env.example` | Environment variable template for CI/CD secrets | +| `.github/workflows/` | GitHub Actions CI (plan on PR) and CD (apply on merge) | + +## Get started + +**1. Get these files** + +```bash +curl -fsSL https://github.com/trouze/terraform-dbtcloud-yaml/releases/latest/download/install.sh | bash +cd my-dbt-cloud +``` + +The script downloads a pre-packaged tarball from the latest release — no npm or git required. It falls back to `degit` or `git sparse-checkout` automatically if the release asset is unavailable. To use a different directory name: `curl -fsSL ... | bash -s -- my-project`. + +**2. Set your credentials** + +```bash +cp .env.example .env +``` + +Open `.env` and fill in: +- `DBT_ACCOUNT_ID` — from dbt Cloud Settings > Account +- `DBT_TOKEN` — a service token from Settings > API Tokens +- `ENVIRONMENT_CREDENTIALS` — warehouse token/password for your prod environment + +**3. Configure your dbt Cloud setup** + +Open `dbt-config.yml` and replace all `YOUR_` placeholders: +- `global_connections` — your warehouse host, http_path, catalog +- `projects[].name` and `key` +- `repository.remote_url` and `github_installation_id` (or GitLab equivalent) +- Environment `catalog` and `schema` + +The credential key in `terraform.tfvars` must match `{project_key}_{env_key}`. With the defaults (`key: analytics`, env `key: prod`), the credential key is `analytics_prod`. + +**4. Deploy** + +```bash +source .env +terraform init +terraform plan # review what will be created +terraform apply +``` + +## CI/CD (optional) + +The `.github/workflows/` directory has ready-to-use GitHub Actions workflows: + +- **`ci.yml`** — runs `terraform plan` on every PR and posts the plan as a comment +- **`cd.yml`** — runs `terraform apply` when changes merge to main + +Set these GitHub repository secrets (Settings > Secrets and variables > Actions): + +``` +DBT_ACCOUNT_ID numeric account ID +DBT_TOKEN dbt Cloud service token +ENVIRONMENT_CREDENTIALS JSON, e.g. {"analytics_prod":{"credential_type":"databricks","token":"dapi...","catalog":"main","schema":"analytics"}} +``` + +Optional secrets (omit if not used): `DBT_PAT`, `CONNECTION_CREDENTIALS`, `LINEAGE_TOKENS`, `OAUTH_CLIENT_SECRETS`. + +Add a [Terraform backend](https://developer.hashicorp.com/terraform/language/backend) to `main.tf` before enabling CD so state is stored remotely. + +## Going further + +- [Full YAML schema reference](../../docs/configuration/yaml-schema.md) — every supported field +- [Multi-project setup](../../docs/configuration/multi-project.md) +- [CI/CD guide](../../docs/guides/cicd.md) +- [Best practices](../../docs/guides/best-practices.md) diff --git a/examples/basic/dbt-config.yml b/examples/basic/dbt-config.yml index 3e17070..a576932 100644 --- a/examples/basic/dbt-config.yml +++ b/examples/basic/dbt-config.yml @@ -1,77 +1,84 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/trouze/dbt-terraform-modules-yaml/refs/heads/main/schemas/v1.json -project: - name: Training Sandbox - repository: - remote_url: "your-org/group/repo" - gitlab_project_id: 1234 - pull_request_url_template: "https://gitlab.com/your-org/group/repo/-/merge_requests/new?merge_request%5Bsource_branch%5D={{source}}&merge_request%5Btarget_branch%5D={{destination}}" - environments: - - name: Prod - credential: - token_name: dbtcloud_databricks_token_dbt_svcprincipal_ucpr - catalog: training - schema: default - connection_id: 12345678 - deployment_type: production - type: deployment - custom_branch: main - jobs: - - name: Production Job - execute_steps: - - dbt compile - triggers: - github_webhook: false - git_provider_webhook: false - schedule: false - on_merge: false - description: Production deployment job - num_threads: 4 - schedule_days: [0, 1, 2, 3, 4, 5, 6] - schedule_hours: [0] - schedule_type: days_of_week - - name: Prod CI Job - execute_steps: - - dbt clone --select state:modified+,config.materialized:incremental,state:old - - dbt build --select state:modified+ - triggers: - github_webhook: false - git_provider_webhook: true - schedule: false - on_merge: false - description: Prod CI job to test changed models.. - num_threads: 4 - deferring_environment: Prod - - name: QA - credential: - token_name: dbtcloud_databricks_token_dbt_svcprincipal_ucnp - catalog: training - schema: default - connection_id: 12345678 - deployment_type: staging - type: deployment - custom_branch: staging - jobs: - - name: Staging Job - execute_steps: - - dbt clone --full-refresh - - dbt build - triggers: - github_webhook: false - git_provider_webhook: false - schedule: false - on_merge: false - description: Staging deployment job - num_threads: 4 - schedule_days: [0, 1, 2, 3, 4, 5, 6] - schedule_hours: [0] - schedule_type: days_of_week - deferring_environment: Prod - - name: Development - connection_id: 12345678 - type: development - custom_branch: staging - environment_variables: - - name: DBT_DATABRICKS_WORKSPACE - environment_values: - - env: project - value: ucnp +# yaml-language-server: $schema=https://raw.githubusercontent.com/trouze/terraform-dbtcloud-yaml/refs/heads/main/schemas/v1.json + +############################################################################## +# dbt Cloud Terraform YAML — Minimal Starter +# +# Fill in the placeholder values (search for YOUR_) and run: +# terraform init && terraform apply +# +# For a full reference of every supported field see: +# https://github.com/trouze/terraform-dbtcloud-yaml/blob/main/docs/configuration/yaml-schema.md +############################################################################## + +########################### +# Warehouse connection +# Shared across environments in this project. +# Supported types: databricks | snowflake | bigquery | postgres | redshift +########################### + +global_connections: + - name: YOUR_CONNECTION_NAME # e.g. "Databricks Production" + key: prod_connection # referenced by environments below + type: databricks # change to your warehouse type + host: YOUR_HOST # e.g. adb-1234567890.1.azuredatabricks.net + http_path: YOUR_HTTP_PATH # e.g. /sql/1.0/warehouses/abc123 + catalog: YOUR_CATALOG # e.g. main + protected: true # prevents accidental deletion + + # --- Snowflake alternative --- + # - name: YOUR_CONNECTION_NAME + # key: prod_connection + # type: snowflake + # account: xy12345.us-east-1 + # database: ANALYTICS + # warehouse: TRANSFORMING + # role: TRANSFORMER + +########################### +# Projects +########################### + +projects: + - name: YOUR_PROJECT_NAME # e.g. "Analytics" + key: analytics # short identifier used in credential keys + + repository: + remote_url: "YOUR_ORG/YOUR_REPO" # e.g. "acme/analytics" + github_installation_id: YOUR_GITHUB_APP_ID # GitHub App installation ID + # For GitLab, replace the two lines above with: + # remote_url: "your-org/group/repo" + # gitlab_project_id: 1234 + + environments: + - name: Production + key: prod + connection_key: prod_connection # matches global_connections[].key + deployment_type: production + type: deployment + custom_branch: main + protected: true + credential: + credential_type: databricks + catalog: YOUR_CATALOG # e.g. main + schema: YOUR_SCHEMA # e.g. analytics + # token supplied via var.environment_credentials["analytics_prod"] + + - name: Development + key: dev + connection_key: prod_connection + type: development + + jobs: + - name: Daily Build + key: daily_build + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] # Mon–Fri (0=Sun, 6=Sat) + schedule_hours: [6] # 6 AM UTC diff --git a/examples/basic/main.tf b/examples/basic/main.tf index f4c1eff..12fc3ec 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -4,25 +4,32 @@ terraform { required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" - version = "~> 1.3" + version = "~> 1.8" } } } provider "dbtcloud" { account_id = var.dbt_account_id - token = var.dbt_api_token + token = var.dbt_token + host_url = var.dbt_host_url } -# Test the terraform-dbtcloud-yaml module from GitHub -module "dbt_cloud_test" { - source = "git::https://github.com/trouze/terraform-dbtcloud-yaml.git?ref=v0.1.0-alpha" +module "dbt_cloud" { + source = "trouze/yaml/dbtcloud" + version = "0.1.0-alpha" dbt_account_id = var.dbt_account_id dbt_token = var.dbt_token - dbt_pat = var.dbt_pat != "" ? var.dbt_pat : var.dbt_token dbt_host_url = var.dbt_host_url - yaml_file = var.yaml_file_path - token_map = var.token_map + dbt_pat = var.dbt_pat + yaml_file = "${path.module}/dbt-config.yml" target_name = var.target_name + + # Sensitive credentials — never put these in the YAML file + token_map = var.token_map + environment_credentials = var.environment_credentials + connection_credentials = var.connection_credentials + lineage_tokens = var.lineage_tokens + oauth_client_secrets = var.oauth_client_secrets } diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf new file mode 100644 index 0000000..9d2a072 --- /dev/null +++ b/examples/basic/outputs.tf @@ -0,0 +1,3 @@ +# Outputs from the dbt Cloud module are available via module.dbt_cloud.* +# Expose any outputs here that are relevant to your deployment. +# See the root module outputs for the full list of available values. diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf index f6e4f70..f6ac7ac 100644 --- a/examples/basic/variables.tf +++ b/examples/basic/variables.tf @@ -1,7 +1,6 @@ variable "dbt_account_id" { description = "dbt Cloud account ID" type = number - sensitive = true } variable "dbt_token" { @@ -11,33 +10,117 @@ variable "dbt_token" { } variable "dbt_pat" { - description = "dbt Cloud Personal Access Token (optional, defaults to dbt_api_token)" + description = "dbt Cloud personal access token (required for GitHub App integration; can be the same as dbt_token)" type = string sensitive = true - default = "" + default = null } variable "dbt_host_url" { - description = "dbt Cloud host URL (e.g., https://cloud.getdbt.com)" + description = "dbt Cloud host URL" type = string default = "https://cloud.getdbt.com" } -variable "yaml_file_path" { - description = "Path to the dbt configuration YAML file" +variable "target_name" { + description = "Default target name (e.g., 'prod')" type = string - default = "./dbt-config.yml" + default = "" } +# +# Credential variables — sensitive values that are referenced by key from the YAML. +# Never put actual secrets in the YAML file. +# + variable "token_map" { - description = "Map of database credentials (warehouse tokens, API keys, etc.)" + description = <<-EOT + Map of Databricks token names to their values. + Key corresponds to credential.token_name in YAML. + Example: { "my_databricks_token" = "dapi..." } + EOT type = map(string) sensitive = true default = {} } -variable "target_name" { - description = "Override the default target name from dbt_project.yml" - type = string - default = "" +variable "environment_credentials" { + description = <<-EOT + Map of environment credential objects keyed by "{project_key}_{env_key}". + Each object must include credential_type and type-specific fields. + + Example: + environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi..." + catalog = "main" + schema = "analytics" + } + analytics_prod_snowflake = { + credential_type = "snowflake" + auth_type = "password" + user = "DBT_USER" + password = "..." + schema = "ANALYTICS" + database = "ANALYTICS" + warehouse = "TRANSFORMING" + } + } + EOT + type = map(any) + sensitive = true + default = {} +} + +variable "connection_credentials" { + description = <<-EOT + Map of global connection keys to their OAuth/auth credential objects. + Key corresponds to global_connections[].key in YAML. + + Example: + connection_credentials = { + databricks_prod = { + client_id = "..." + client_secret = "..." + } + snowflake_prod = { + oauth_client_id = "..." + oauth_client_secret = "..." + } + } + EOT + type = map(any) + sensitive = true + default = {} +} + +variable "lineage_tokens" { + description = <<-EOT + Map of lineage integration tokens keyed by "{project_key}_{integration_key}". + Key corresponds to the composite of project key + lineage_integrations[].key in YAML. + + Example: + lineage_tokens = { + analytics_tableau_prod = "..." + } + EOT + type = map(string) + sensitive = true + default = {} +} + +variable "oauth_client_secrets" { + description = <<-EOT + Map of OAuth configuration keys to their client secrets. + Key corresponds to oauth_configurations[].key in YAML. + + Example: + oauth_client_secrets = { + snowflake_oauth = "..." + } + EOT + type = map(string) + sensitive = true + default = {} } diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..de1cfe7 --- /dev/null +++ b/install.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# install.sh — bootstrap a dbt Cloud Terraform starter from terraform-dbtcloud-yaml +# +# Usage: +# curl -fsSL https://github.com/trouze/terraform-dbtcloud-yaml/releases/latest/download/install.sh | bash +# curl -fsSL https://github.com/trouze/terraform-dbtcloud-yaml/releases/latest/download/install.sh | bash -s -- my-project +# +set -euo pipefail + +TARGET=${1:-my-dbt-platform} +REPO="trouze/terraform-dbtcloud-yaml" +RELEASE_URL="https://github.com/$REPO/releases/latest/download/starter.tar.gz" + +echo "Setting up dbt Platform Terraform starter in ./$TARGET ..." +echo "" + +if [[ -e "$TARGET" ]]; then + echo "Error: '$TARGET' already exists. Pass a different directory name:" >&2 + echo " bash <(curl -fsSL ...) my-other-name" >&2 + exit 1 +fi + +mkdir -p "$TARGET" + +# Strategy 1: curl + tar (no extra tools required — fastest) +if command -v curl &>/dev/null && command -v tar &>/dev/null; then + if curl -fsSL "$RELEASE_URL" | tar -xz --strip-components=1 -C "$TARGET" 2>/dev/null; then + : + else + # Fall through to strategy 2 if the release asset doesn't exist yet + rmdir "$TARGET" 2>/dev/null || true + _fallback=1 + fi +fi + +# Strategy 2: degit (needs npm/npx) +if [[ ${_fallback:-0} -eq 1 ]] || [[ ! -d "$TARGET" ]]; then + if command -v npx &>/dev/null; then + echo "(release asset not found, falling back to degit)" + npx --yes degit "$REPO/examples/basic" "$TARGET" + else + # Strategy 3: git sparse-checkout + echo "(falling back to git sparse-checkout)" + TMP=$(mktemp -d) + trap 'rm -rf "$TMP"' EXIT + git clone --no-checkout --depth=1 "https://github.com/$REPO" "$TMP/repo" --quiet + git -C "$TMP/repo" sparse-checkout set examples/basic + git -C "$TMP/repo" checkout --quiet + cp -r "$TMP/repo/examples/basic/." "$TARGET/" + fi +fi + +echo "Done. Starter created in ./$TARGET" +echo "" +echo "Next steps:" +echo " 1. cd $TARGET" +echo " 2. cp .env.example .env" +echo " # fill in DBT_ACCOUNT_ID, DBT_TOKEN, and warehouse credentials" +echo " 3. Edit dbt-config.yml" +echo " # replace YOUR_ placeholders with your warehouse and repo details" +echo " 4. source .env && terraform init && terraform apply" +echo "" +echo "Full walkthrough: https://github.com/$REPO/blob/main/examples/basic/README.md" diff --git a/main.tf b/main.tf index ef13a42..bfe5ff0 100644 --- a/main.tf +++ b/main.tf @@ -1,25 +1,131 @@ ############################################# # Root Module - dbt Cloud Resources -# -# This module orchestrates the creation of -# dbt Cloud projects, environments, jobs, -# credentials, and environment variables -# from a YAML configuration file. +# +# Orchestrates all dbt Cloud resources from a YAML file. +# Supports both single-project (project:) and multi-project (projects:) YAML. +# Account-scoped resources are created first, then project-scoped resources. ############################################# ############################################# -# 1. Project Setup +# YAML Configuration Validation +# All validation logic lives in validation.tf. +# This resource fails at plan time if any errors are found, +# reporting all issues together in one message. +############################################# + +resource "terraform_data" "validate_yaml_config" { + lifecycle { + precondition { + condition = length(local._all_validation_errors) == 0 + error_message = "dbt Cloud YAML configuration has ${length(local._all_validation_errors)} error(s):\n\n${join("\n\n", [for i, e in local._all_validation_errors : " ${i + 1}. ${e}"])}" + } + } +} + +############################################# +# Account-Level Features +############################################# + +module "account_features" { + count = try(local.yaml_content.account_features, null) != null ? 1 : 0 + source = "./modules/account_features" + + features = try(local.yaml_content.account_features, null) +} + +############################################# +# Account-Level: Global Connections +############################################# + +module "global_connections" { + count = length(try(local.yaml_content.global_connections, [])) > 0 ? 1 : 0 + source = "./modules/global_connections" + + connections_data = try(local.yaml_content.global_connections, []) + connection_credentials = var.connection_credentials +} + +############################################# +# Account-Level: Service Tokens +############################################# + +module "service_tokens" { + count = length(try(local.yaml_content.service_tokens, [])) > 0 ? 1 : 0 + source = "./modules/service_tokens" + + service_tokens_data = try(local.yaml_content.service_tokens, []) +} + +############################################# +# Account-Level: Groups +############################################# + +module "groups" { + count = length(try(local.yaml_content.groups, [])) > 0 ? 1 : 0 + source = "./modules/groups" + + groups_data = try(local.yaml_content.groups, []) +} + +############################################# +# Account-Level: User Groups (user ↔ group assignment) +############################################# + +module "user_groups" { + count = length(try(local.yaml_content.user_groups, [])) > 0 ? 1 : 0 + source = "./modules/user_groups" + + user_groups_data = try(local.yaml_content.user_groups, []) + group_ids = length(try(local.yaml_content.groups, [])) > 0 ? module.groups[0].group_ids : {} +} + +############################################# +# Account-Level: Notifications +############################################# + +module "notifications" { + count = length(try(local.yaml_content.notifications, [])) > 0 ? 1 : 0 + source = "./modules/notifications" + + notifications_data = try(local.yaml_content.notifications, []) +} + +############################################# +# Account-Level: OAuth Configurations +############################################# + +module "oauth_configurations" { + count = length(try(local.yaml_content.oauth_configurations, [])) > 0 ? 1 : 0 + source = "./modules/oauth_configurations" + + oauth_data = try(local.yaml_content.oauth_configurations, []) + oauth_client_secrets = var.oauth_client_secrets +} + +############################################# +# Account-Level: IP Restrictions +############################################# + +module "ip_restrictions" { + count = length(try(local.yaml_content.ip_restrictions, [])) > 0 ? 1 : 0 + source = "./modules/ip_restrictions" + + ip_rules_data = try(local.yaml_content.ip_restrictions, []) +} + +############################################# +# Project Setup ############################################# module "project" { source = "./modules/project" - project_name = local.project_config.project.name - target_name = var.target_name + projects = local.projects + target_name = var.target_name } ############################################# -# 2. Repository Configuration +# Repository Configuration ############################################# module "repository" { @@ -28,78 +134,150 @@ module "repository" { dbtcloud = dbtcloud.pat_provider } - repository_data = local.project_config.project.repository - project_id = module.project.project_id + projects = local.projects + project_ids = module.project.project_ids + dbt_pat = var.dbt_pat + enable_gitlab_deploy_token = var.enable_gitlab_deploy_token } module "project_repository" { source = "./modules/project_repository" - repository_id = module.repository.project_repository_id - project_id = module.project.project_id + project_ids = module.project.project_ids + repository_ids = module.repository.repository_ids } ############################################# -# 3. Credentials +# Extended Attributes (must precede environments) +############################################# + +module "extended_attributes" { + count = length(flatten([for p in local.projects : try(p.extended_attributes, [])])) > 0 ? 1 : 0 + source = "./modules/extended_attributes" + + projects = local.projects + project_ids = module.project.project_ids +} + +############################################# +# Credentials ############################################# module "credentials" { source = "./modules/credentials" - environments_data = local.project_config.project.environments - project_id = module.project.project_id - token_map = var.token_map + projects = local.projects + project_ids = module.project.project_ids + token_map = var.token_map + environment_credentials = var.environment_credentials } ############################################# -# 4. Environments +# Environments ############################################# module "environments" { source = "./modules/environments" - project_id = module.project.project_id - environments_data = local.project_config.project.environments - credential_ids = module.credentials.credential_ids + projects = local.projects + project_ids = module.project.project_ids + credential_ids = module.credentials.credential_ids + global_connection_ids = length(try(local.yaml_content.global_connections, [])) > 0 ? module.global_connections[0].connection_ids : {} + extended_attribute_ids = length(flatten([for p in local.projects : try(p.extended_attributes, [])])) > 0 ? module.extended_attributes[0].extended_attribute_ids : {} } ############################################# -# 5. Jobs +# Jobs ############################################# module "jobs" { source = "./modules/jobs" - project_id = module.project.project_id - environments_data = local.project_config.project.environments - environment_ids = module.environments.environment_ids + projects = local.projects + project_ids = module.project.project_ids + environment_ids = module.environments.environment_ids } ############################################# -# 6. Environment Variables +# Environment Variables ############################################# module "environment_variables" { source = "./modules/environment_variables" - project_id = module.project.project_id - environment_variables = lookup(local.project_config.project, "environment_variables", {}) - environment_ids = module.environments.environment_ids - token_map = var.token_map + projects = local.projects + project_ids = module.project.project_ids + token_map = var.token_map depends_on = [module.environments] } ############################################# -# 7. Environment Variable Job Overrides +# Environment Variable Job Overrides ############################################# module "environment_variable_job_overrides" { source = "./modules/environment_variable_job_overrides" - project_id = module.project.project_id - environments_data = local.project_config.project.environments - job_ids = module.jobs.job_ids + projects = local.projects + project_ids = module.project.project_ids + job_ids = module.jobs.job_ids depends_on = [module.environment_variables] } + +############################################# +# Profiles (links connection + credential + extended_attributes) +############################################# + +module "profiles" { + count = length(flatten([for p in local.projects : try(p.profiles, [])])) > 0 ? 1 : 0 + source = "./modules/profiles" + + projects = local.projects + project_ids = module.project.project_ids + global_connection_ids = length(try(local.yaml_content.global_connections, [])) > 0 ? module.global_connections[0].connection_ids : {} + credential_ids = module.credentials.credential_ids + extended_attribute_ids = length(flatten([for p in local.projects : try(p.extended_attributes, [])])) > 0 ? module.extended_attributes[0].extended_attribute_ids : {} +} + +############################################# +# Lineage Integrations +############################################# + +module "lineage_integrations" { + count = length(flatten([for p in local.projects : try(p.lineage_integrations, [])])) > 0 ? 1 : 0 + source = "./modules/lineage_integrations" + + projects = local.projects + project_ids = module.project.project_ids + lineage_tokens = var.lineage_tokens +} + +############################################# +# Project Artefacts (docs job + freshness job links) +############################################# + +module "project_artefacts" { + count = length([for p in local.projects : p if try(p.artefacts, null) != null]) > 0 ? 1 : 0 + source = "./modules/project_artefacts" + + projects = local.projects + project_ids = module.project.project_ids + job_ids = module.jobs.job_ids + + depends_on = [module.jobs] +} + +############################################# +# Semantic Layer +############################################# + +module "semantic_layer" { + count = length([for p in local.projects : p if try(p.semantic_layer, null) != null]) > 0 ? 1 : 0 + source = "./modules/semantic_layer" + + projects = local.projects + project_ids = module.project.project_ids + environment_ids = module.environments.environment_ids +} diff --git a/modules/account_features/main.tf b/modules/account_features/main.tf new file mode 100644 index 0000000..5315db1 --- /dev/null +++ b/modules/account_features/main.tf @@ -0,0 +1,17 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +resource "dbtcloud_account_features" "features" { + count = var.features != null ? 1 : 0 + + advanced_ci = try(var.features.advanced_ci, null) + partial_parsing = try(var.features.partial_parsing, null) + repo_caching = try(var.features.repo_caching, null) +} diff --git a/modules/account_features/outputs.tf b/modules/account_features/outputs.tf new file mode 100644 index 0000000..fb3aa57 --- /dev/null +++ b/modules/account_features/outputs.tf @@ -0,0 +1,4 @@ +output "account_features_id" { + description = "The dbt Cloud account_features resource ID (if created)" + value = length(dbtcloud_account_features.features) > 0 ? dbtcloud_account_features.features[0].id : null +} diff --git a/modules/account_features/variables.tf b/modules/account_features/variables.tf new file mode 100644 index 0000000..7bfff18 --- /dev/null +++ b/modules/account_features/variables.tf @@ -0,0 +1,5 @@ +variable "features" { + description = "Account feature flags from YAML account_features. Set to null to skip (no resource created)." + type = any + default = null +} diff --git a/modules/credentials/.terraform.lock.hcl b/modules/credentials/.terraform.lock.hcl new file mode 100644 index 0000000..8dc8cba --- /dev/null +++ b/modules/credentials/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dbt-labs/dbtcloud" { + version = "1.8.2" + constraints = "~> 1.8" + hashes = [ + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", + ] +} diff --git a/modules/credentials/main.tf b/modules/credentials/main.tf index 1b3f4ce..78bb992 100644 --- a/modules/credentials/main.tf +++ b/modules/credentials/main.tf @@ -1,19 +1,335 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } -resource "dbtcloud_databricks_credential" "databricks_credential" { +locals { + # Flatten all environments across all projects that have credentials defined + all_credential_owners = flatten([ + for p in var.projects : [ + for env in try(p.environments, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + env_key = try(env.key, env.name) + composite_key = "${try(p.key, p.name)}_${try(env.key, env.name)}" + cred_data = try(env.credential, null) + } + if try(env.credential, null) != null + ] + ]) + + credential_owners_map = { + for item in local.all_credential_owners : + item.composite_key => item + } + + # Non-sensitive helpers for for_each conditions + available_env_cred_keys = toset(nonsensitive(keys(var.environment_credentials))) + + env_cred_types = nonsensitive({ + for k, v in var.environment_credentials : + k => try(v.credential_type, "") + }) + + # Fabric/Synapse: service principal auth uses tenant_id + env_cred_has_tenant = nonsensitive({ + for k, v in var.environment_credentials : + k => try(v.tenant_id, null) != null + }) +} + +############################################# +# Databricks Credentials +############################################# + +resource "dbtcloud_databricks_credential" "credentials" { for_each = { - for env in var.environments_data : env.name => env - if try(env.credential, null) != null + for k, item in local.credential_owners_map : + k => item + if( + contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "databricks" + ) || ( + !contains(local.available_env_cred_keys, k) && + try(item.cred_data.credential_type, try(item.cred_data.type, "databricks")) == "databricks" + ) } - project_id = var.project_id - token = lookup(var.token_map, each.value.credential.token_name, null) - schema = each.value.credential.schema - catalog = each.value.credential.catalog + + project_id = each.value.project_id adapter_type = "databricks" + token = try(var.environment_credentials[each.key].token, lookup(var.token_map, try(each.value.cred_data.token_name, ""), null)) + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + catalog = try(var.environment_credentials[each.key].catalog, try(each.value.cred_data.catalog, null)) +} + +############################################# +# Snowflake Credentials — Password Auth +############################################# + +resource "dbtcloud_snowflake_credential" "credentials_password" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "snowflake" && + try(nonsensitive(var.environment_credentials[k].auth_type), "password") == "password" + } + + project_id = each.value.project_id + auth_type = "password" + user = try(var.environment_credentials[each.key].user, "") + password = try(var.environment_credentials[each.key].password, null) + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + num_threads = try(var.environment_credentials[each.key].num_threads, null) + database = try(var.environment_credentials[each.key].database, null) + role = try(var.environment_credentials[each.key].role, null) + warehouse = try(var.environment_credentials[each.key].warehouse, null) +} + +############################################# +# Snowflake Credentials — Key Pair Auth +############################################# + +resource "dbtcloud_snowflake_credential" "credentials_keypair" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "snowflake" && + try(nonsensitive(var.environment_credentials[k].auth_type), "password") == "keypair" + } + + project_id = each.value.project_id + auth_type = "keypair" + user = try(var.environment_credentials[each.key].user, "") + private_key = try(var.environment_credentials[each.key].private_key, null) + private_key_passphrase = try(var.environment_credentials[each.key].private_key_passphrase, null) + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + num_threads = try(var.environment_credentials[each.key].num_threads, null) + database = try(var.environment_credentials[each.key].database, null) + role = try(var.environment_credentials[each.key].role, null) + warehouse = try(var.environment_credentials[each.key].warehouse, null) +} + +############################################# +# BigQuery Credentials +############################################# + +resource "dbtcloud_bigquery_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "bigquery" + } + + project_id = each.value.project_id + dataset = try(var.environment_credentials[each.key].dataset, try(each.value.cred_data.schema, "")) + num_threads = try(var.environment_credentials[each.key].num_threads, null) +} + +############################################# +# Postgres Credentials +############################################# + +resource "dbtcloud_postgres_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "postgres" + } + + project_id = each.value.project_id + type = "postgres" + default_schema = try(var.environment_credentials[each.key].default_schema, try(each.value.cred_data.schema, "")) + username = try(var.environment_credentials[each.key].username, "") + password = try(var.environment_credentials[each.key].password, null) + target_name = try(var.environment_credentials[each.key].target_name, null) + num_threads = try(var.environment_credentials[each.key].num_threads, null) +} + +############################################# +# Redshift Credentials +############################################# + +resource "dbtcloud_redshift_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "redshift" + } + + project_id = each.value.project_id + default_schema = try(var.environment_credentials[each.key].default_schema, try(each.value.cred_data.schema, "")) + username = try(var.environment_credentials[each.key].username, "") + password = try(var.environment_credentials[each.key].password, null) + num_threads = try(var.environment_credentials[each.key].num_threads, 4) +} + +############################################# +# Athena Credentials +############################################# + +resource "dbtcloud_athena_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "athena" + } + + project_id = each.value.project_id + aws_access_key_id = try(var.environment_credentials[each.key].aws_access_key_id, "") + aws_secret_access_key = try(var.environment_credentials[each.key].aws_secret_access_key, "") + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) +} + +############################################# +# Fabric Credentials — SQL Auth (no tenant_id) +############################################# + +resource "dbtcloud_fabric_credential" "credentials_sql" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "fabric" && + !try(local.env_cred_has_tenant[k], false) + } + + project_id = each.value.project_id + adapter_type = "fabric" + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + user = try(var.environment_credentials[each.key].user, "") + password = try(var.environment_credentials[each.key].password, "") + schema_authorization = try(var.environment_credentials[each.key].schema_authorization, null) +} + +############################################# +# Fabric Credentials — Service Principal Auth (tenant_id present) +############################################# + +resource "dbtcloud_fabric_credential" "credentials_sp" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "fabric" && + try(local.env_cred_has_tenant[k], false) + } + + project_id = each.value.project_id + adapter_type = "fabric" + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + tenant_id = try(var.environment_credentials[each.key].tenant_id, "") + client_id = try(var.environment_credentials[each.key].client_id, "") + client_secret = try(var.environment_credentials[each.key].client_secret, "") + schema_authorization = try(var.environment_credentials[each.key].schema_authorization, null) +} + +############################################# +# Synapse Credentials — SQL Auth (no tenant_id) +############################################# + +resource "dbtcloud_synapse_credential" "credentials_sql" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "synapse" && + !try(local.env_cred_has_tenant[k], false) + } + + project_id = each.value.project_id + adapter_type = "synapse" + authentication = try(var.environment_credentials[each.key].authentication, "sql") + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + user = try(var.environment_credentials[each.key].user, "") + password = try(var.environment_credentials[each.key].password, "") + schema_authorization = try(var.environment_credentials[each.key].schema_authorization, null) +} + +############################################# +# Synapse Credentials — Service Principal Auth +############################################# + +resource "dbtcloud_synapse_credential" "credentials_sp" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "synapse" && + try(local.env_cred_has_tenant[k], false) + } + + project_id = each.value.project_id + adapter_type = "synapse" + authentication = try(var.environment_credentials[each.key].authentication, "ServicePrincipal") + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + tenant_id = try(var.environment_credentials[each.key].tenant_id, "") + client_id = try(var.environment_credentials[each.key].client_id, "") + client_secret = try(var.environment_credentials[each.key].client_secret, "") + schema_authorization = try(var.environment_credentials[each.key].schema_authorization, null) +} + +############################################# +# Starburst / Trino Credentials +############################################# + +resource "dbtcloud_starburst_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + contains(["starburst", "trino"], try(local.env_cred_types[k], "")) + } + + project_id = each.value.project_id + database = try(var.environment_credentials[each.key].catalog, try(each.value.cred_data.catalog, "")) + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + user = try(var.environment_credentials[each.key].user, "") + password = try(var.environment_credentials[each.key].password, "") +} + +############################################# +# Spark Credentials +############################################# + +resource "dbtcloud_spark_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + contains(["spark", "apache_spark"], try(local.env_cred_types[k], "")) + } + + project_id = each.value.project_id + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + token = try(var.environment_credentials[each.key].token, "") +} + +############################################# +# Teradata Credentials +############################################# + +resource "dbtcloud_teradata_credential" "credentials" { + for_each = { + for k, item in local.credential_owners_map : + k => item + if contains(local.available_env_cred_keys, k) && + try(local.env_cred_types[k], "") == "teradata" + } + + project_id = each.value.project_id + schema = try(var.environment_credentials[each.key].schema, try(each.value.cred_data.schema, "")) + user = try(var.environment_credentials[each.key].user, "") + password = try(var.environment_credentials[each.key].password, "") + threads = try(var.environment_credentials[each.key].num_threads, null) } diff --git a/modules/credentials/outputs.tf b/modules/credentials/outputs.tf index b71124e..3a89d05 100644 --- a/modules/credentials/outputs.tf +++ b/modules/credentials/outputs.tf @@ -1,4 +1,19 @@ output "credential_ids" { - description = "Map of environment names to their credential IDs" - value = { for env, cred in dbtcloud_databricks_credential.databricks_credential : env => cred.credential_id } + description = "Map of composite key (project_key_env_key) to credential ID. Merges all warehouse types." + value = merge( + { for k, c in dbtcloud_databricks_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_snowflake_credential.credentials_password : k => c.credential_id }, + { for k, c in dbtcloud_snowflake_credential.credentials_keypair : k => c.credential_id }, + { for k, c in dbtcloud_bigquery_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_postgres_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_redshift_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_athena_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_fabric_credential.credentials_sql : k => c.credential_id }, + { for k, c in dbtcloud_fabric_credential.credentials_sp : k => c.credential_id }, + { for k, c in dbtcloud_synapse_credential.credentials_sql : k => c.credential_id }, + { for k, c in dbtcloud_synapse_credential.credentials_sp : k => c.credential_id }, + { for k, c in dbtcloud_starburst_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_spark_credential.credentials : k => c.credential_id }, + { for k, c in dbtcloud_teradata_credential.credentials : k => c.credential_id }, + ) } diff --git a/modules/credentials/tests/unit.tftest.hcl b/modules/credentials/tests/unit.tftest.hcl new file mode 100644 index 0000000..622e7b7 --- /dev/null +++ b/modules/credentials/tests/unit.tftest.hcl @@ -0,0 +1,335 @@ +# Unit tests for modules/credentials +# Validates that the correct credential resource type is created for each +# warehouse adapter, and that Snowflake auth_type routing works correctly. +# Run from modules/credentials/: terraform test + +mock_provider "dbtcloud" {} + +# ── Databricks credentials ──────────────────────────────────────────────────── + +run "databricks_credential_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "databricks" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "databricks" + token = "dapi-fake-token" + schema = "analytics_prod" + } + } + } + + assert { + condition = length(dbtcloud_databricks_credential.credentials) == 1 + error_message = "Expected one Databricks credential to be created" + } + + assert { + condition = dbtcloud_databricks_credential.credentials["analytics_prod"].schema == "analytics_prod" + error_message = "Databricks credential schema should match" + } +} + +# ── Snowflake credentials — password auth ───────────────────────────────────── + +run "snowflake_password_credential_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "snowflake" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "snowflake" + auth_type = "password" + user = "dbt_user" + password = "secret" + schema = "analytics" + num_threads = 4 + } + } + } + + assert { + condition = length(dbtcloud_snowflake_credential.credentials_password) == 1 + error_message = "Expected one Snowflake password credential" + } + + assert { + condition = length(dbtcloud_snowflake_credential.credentials_keypair) == 0 + error_message = "No keypair credential should be created for password auth" + } +} + +# ── Snowflake credentials — keypair auth ────────────────────────────────────── + +run "snowflake_keypair_credential_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "snowflake" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "snowflake" + auth_type = "keypair" + user = "dbt_user" + private_key = "-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----" + schema = "analytics" + num_threads = 4 + } + } + } + + assert { + condition = length(dbtcloud_snowflake_credential.credentials_keypair) == 1 + error_message = "Expected one Snowflake keypair credential" + } + + assert { + condition = length(dbtcloud_snowflake_credential.credentials_password) == 0 + error_message = "No password credential should be created for keypair auth" + } +} + +# ── BigQuery credentials ────────────────────────────────────────────────────── + +run "bigquery_credential_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "bigquery" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "bigquery" + dataset = "dbt_prod" + num_threads = 4 + } + } + } + + assert { + condition = length(dbtcloud_bigquery_credential.credentials) == 1 + error_message = "Expected one BigQuery credential" + } +} + +# ── Postgres credentials ────────────────────────────────────────────────────── + +run "postgres_credential_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "postgres" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "postgres" + username = "dbt_user" + password = "secret" + default_schema = "dbt_prod" + } + } + } + + assert { + condition = length(dbtcloud_postgres_credential.credentials) == 1 + error_message = "Expected one Postgres credential" + } +} + +# ── Fabric credentials — SQL vs service principal ───────────────────────────── + +run "fabric_sql_credential_created_without_tenant_id" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "fabric" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "fabric" + user = "dbt_user" + password = "secret" + schema = "dbt_prod" + } + } + } + + assert { + condition = length(dbtcloud_fabric_credential.credentials_sql) == 1 + error_message = "Expected Fabric SQL credential when no tenant_id" + } + + assert { + condition = length(dbtcloud_fabric_credential.credentials_sp) == 0 + error_message = "No Fabric SP credential should be created without tenant_id" + } +} + +run "fabric_service_principal_credential_created_with_tenant_id" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "fabric" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "fabric" + tenant_id = "tenant-abc" + client_id = "client-abc" + client_secret = "secret" + schema = "dbt_prod" + } + } + } + + assert { + condition = length(dbtcloud_fabric_credential.credentials_sp) == 1 + error_message = "Expected Fabric SP credential when tenant_id is present" + } + + assert { + condition = length(dbtcloud_fabric_credential.credentials_sql) == 0 + error_message = "No Fabric SQL credential should be created with tenant_id" + } +} + +# ── Only one credential resource type created per entry ─────────────────────── + +run "only_matching_resource_type_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + credential = { credential_type = "redshift" } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_credentials = { + analytics_prod = { + credential_type = "redshift" + username = "dbt_user" + password = "secret" + default_schema = "dbt_prod" + } + } + } + + assert { + condition = length(dbtcloud_redshift_credential.credentials) == 1 + error_message = "Expected one Redshift credential" + } + + assert { + condition = length(dbtcloud_databricks_credential.credentials) == 0 + error_message = "No Databricks credential should be created for redshift type" + } + + assert { + condition = length(dbtcloud_snowflake_credential.credentials_password) == 0 + error_message = "No Snowflake credential should be created for redshift type" + } + + assert { + condition = length(dbtcloud_bigquery_credential.credentials) == 0 + error_message = "No BigQuery credential should be created for redshift type" + } +} diff --git a/modules/credentials/variables.tf b/modules/credentials/variables.tf index c51a04a..a64dcef 100644 --- a/modules/credentials/variables.tf +++ b/modules/credentials/variables.tf @@ -1,15 +1,23 @@ -variable "environments_data" { - description = "List of environment configurations, including credentials" - type = any +variable "projects" { + description = "List of project configurations. Each project's environments may have a 'credential' sub-object." + type = any } -variable "project_id" { - description = "The ID of the project these credentials belong to" - type = string +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) } variable "token_map" { - type = map(string) - description = "Mapping of token names to credential" - sensitive = true + description = "Map of token names to their values (used for legacy Databricks token_name references)" + type = map(string) + default = {} + sensitive = true +} + +variable "environment_credentials" { + description = "Map of composite key (project_key_env_key) to credential objects. Each object must include 'credential_type' to select the warehouse adapter." + type = map(any) + default = {} + sensitive = true } diff --git a/modules/environment_variable_job_overrides/main.tf b/modules/environment_variable_job_overrides/main.tf index a2a4e12..137fced 100644 --- a/modules/environment_variable_job_overrides/main.tf +++ b/modules/environment_variable_job_overrides/main.tf @@ -1,38 +1,59 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } locals { - flattened_overrides = flatten([ - for env in var.environments_data : [ - for job in env.jobs : [ + # Flatten env var job overrides from environment-nested jobs (legacy layout) + overrides_from_env_jobs = flatten([ + for p in var.projects : [ + for env in try(p.environments, []) : [ + for job in try(env.jobs, []) : [ + for var_name, var_value in try(job.env_var_overrides, {}) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + job_key = "${try(p.key, p.name)}_${env.name}_${job.name}" + var_name = var_name + var_value = var_value + composite_key = "${try(p.key, p.name)}_${env.name}_${job.name}_${var_name}" + } + ] + ] if try(env.jobs, null) != null + ] + ]) + + # Flatten env var job overrides from project-level jobs (new layout) + overrides_from_project_jobs = flatten([ + for p in var.projects : [ + for job in try(p.jobs, []) : [ for var_name, var_value in try(job.env_var_overrides, {}) : { - job_key = "${env.name}_${job.name}" - var_name = var_name - var_value = var_value + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + job_key = "${try(p.key, p.name)}_${try(job.key, job.name)}" + var_name = var_name + var_value = var_value + composite_key = "${try(p.key, p.name)}_${try(job.key, job.name)}_${var_name}" } - ] if try(job.env_var_overrides, null) != null - ] if try(env.jobs, null) != null + ] + ] ]) - env_var_map = { - for item in local.flattened_overrides : "${item.job_key}_${item.var_name}" => { - job_key = item.job_key - var_name = item.var_name - var_value = item.var_value - } + all_overrides_map = { + for item in concat(local.overrides_from_env_jobs, local.overrides_from_project_jobs) : + item.composite_key => item } } resource "dbtcloud_environment_variable_job_override" "environment_variable_job_overrides" { - for_each = local.env_var_map - - name = each.value.var_name - project_id = var.project_id + for_each = local.all_overrides_map + + name = each.value.var_name + project_id = each.value.project_id job_definition_id = lookup(var.job_ids, each.value.job_key, null) - raw_value = each.value.var_value + raw_value = tostring(each.value.var_value) } diff --git a/modules/environment_variable_job_overrides/variables.tf b/modules/environment_variable_job_overrides/variables.tf index dc73008..abbd408 100644 --- a/modules/environment_variable_job_overrides/variables.tf +++ b/modules/environment_variable_job_overrides/variables.tf @@ -1,14 +1,14 @@ -variable "project_id" { - description = "The ID of the project to which jobs belong" - type = string +variable "projects" { + description = "List of project configurations. Env var job overrides are read from project.environments[].jobs[].env_var_overrides or project.jobs[].env_var_overrides." + type = any } -variable "job_ids" { - description = "Map of Env Name _ Job Name as key : Job ID" - type = any +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) } -variable "environments_data" { - description = "List of environment configurations, including credentials, overrides" - type = any +variable "job_ids" { + description = "Map of composite key (project_key_job_key) to dbt Cloud job ID (from jobs module)" + type = map(string) } diff --git a/modules/environment_variables/main.tf b/modules/environment_variables/main.tf index 6d8a3af..1d75eec 100644 --- a/modules/environment_variables/main.tf +++ b/modules/environment_variables/main.tf @@ -1,21 +1,44 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } -resource "dbtcloud_environment_variable" "environment_variables" { - for_each = { - for env_var in var.environment_variables : - env_var.name => env_var +locals { + # Flatten all env vars across all projects + all_env_vars = flatten([ + for p in var.projects : [ + for ev in try(p.environment_variables, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + var_name = ev.name + composite_key = "${try(p.key, p.name)}_${ev.name}" + ev_data = ev + } + ] + ]) + + env_vars_map = { + for item in local.all_env_vars : + item.composite_key => item } +} + +resource "dbtcloud_environment_variable" "environment_variables" { + for_each = local.env_vars_map - name = each.value.name - project_id = var.project_id + name = each.value.var_name + project_id = each.value.project_id environment_values = { - for item in each.value.environment_values : - item.env => startswith(item.value, "secret_") ? lookup(var.token_map, join("_", slice(split("_", item.value), 1, length(split("_", item.value)))), null) : item.value + for item in each.value.ev_data.environment_values : + item.env => ( + startswith(tostring(item.value), "secret_") + ? lookup(var.token_map, join("_", slice(split("_", tostring(item.value)), 1, length(split("_", tostring(item.value))))), null) + : tostring(item.value) + ) } } diff --git a/modules/environment_variables/variables.tf b/modules/environment_variables/variables.tf index cd9062e..738662f 100644 --- a/modules/environment_variables/variables.tf +++ b/modules/environment_variables/variables.tf @@ -1,20 +1,16 @@ -variable "project_id" { - description = "The ID of the project to which jobs belong" - type = string +variable "projects" { + description = "List of project configurations. Each project may have an 'environment_variables' list." + type = any } -variable "environment_ids" { - description = "The ID of the project this repository is associated with" +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" type = map(string) } -variable "environment_variables" { - description = "A list of environment variable configurations" - type = any -} - variable "token_map" { - type = map(string) - description = "Mapping of token names to credential" - sensitive = true + description = "Map of token names to values (used for secret_ prefixed env var values)" + type = map(string) + default = {} + sensitive = true } diff --git a/modules/environments/.terraform.lock.hcl b/modules/environments/.terraform.lock.hcl new file mode 100644 index 0000000..8dc8cba --- /dev/null +++ b/modules/environments/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dbt-labs/dbtcloud" { + version = "1.8.2" + constraints = "~> 1.8" + hashes = [ + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", + ] +} diff --git a/modules/environments/main.tf b/modules/environments/main.tf index 399bb90..e05af13 100644 --- a/modules/environments/main.tf +++ b/modules/environments/main.tf @@ -1,27 +1,116 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } +locals { + # Flatten all environments across all projects + all_environments = flatten([ + for p in var.projects : [ + for env in try(p.environments, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + env_key = try(env.key, env.name) + composite_key = "${try(p.key, p.name)}_${try(env.key, env.name)}" + env_data = env + } + ] + ]) + + envs_map = { + for item in local.all_environments : + item.composite_key => item + } + + protected_envs_map = { + for k, item in local.envs_map : + k => item + if try(item.env_data.protected, false) == true + } + + unprotected_envs_map = { + for k, item in local.envs_map : + k => item + if try(item.env_data.protected, false) != true + } + + # Resolve credential_id: look up composite key in credential_ids map + resolve_credential_id = { + for k, item in local.envs_map : + k => lookup(var.credential_ids, k, null) + } + + # Resolve connection_id: prefer connection key lookup (global connections), + # fall back to direct numeric ID from YAML + resolve_connection_id = { + for k, item in local.envs_map : + k => ( + try(item.env_data.connection, null) != null ? + lookup(var.global_connection_ids, tostring(item.env_data.connection), null) != null ? + lookup(var.global_connection_ids, tostring(item.env_data.connection), null) : + try(tonumber(item.env_data.connection), null) : + try(item.env_data.connection_id, null) + ) + } + + # Resolve extended_attributes_id via key lookup + resolve_extended_attributes_id = { + for k, item in local.envs_map : + k => ( + try(item.env_data.extended_attributes_key, null) != null ? + lookup(var.extended_attribute_ids, "${item.project_key}_${item.env_data.extended_attributes_key}", null) : + null + ) + } +} + +############################################# +# Unprotected Environments +############################################# + resource "dbtcloud_environment" "environments" { - for_each = { - for env in var.environments_data : - env.name => env - } - - project_id = var.project_id - credential_id = lookup(var.credential_ids, each.key, null) # Using the map of credential_ids output from the credentials module - connection_id = each.value.connection_id - name = each.value.name - type = each.value.type - - # Optional fields with defaults for missing values - dbt_version = lookup(each.value, "dbt_version", null) - enable_model_query_history = lookup(each.value, "enable_model_query_history", null) - custom_branch = lookup(each.value, "custom_branch", null) - deployment_type = lookup(each.value, "deployment_type", null) - use_custom_branch = lookup(each.value, "custom_branch", null) != null + for_each = local.unprotected_envs_map + + project_id = each.value.project_id + name = each.value.env_data.name + type = each.value.env_data.type + connection_id = local.resolve_connection_id[each.key] + credential_id = local.resolve_credential_id[each.key] + + dbt_version = try(each.value.env_data.dbt_version, null) + enable_model_query_history = try(each.value.env_data.enable_model_query_history, null) + custom_branch = try(each.value.env_data.custom_branch, null) + deployment_type = try(each.value.env_data.deployment_type, null) + use_custom_branch = try(each.value.env_data.custom_branch, null) != null + extended_attributes_id = local.resolve_extended_attributes_id[each.key] +} + +############################################# +# Protected Environments — lifecycle.prevent_destroy +############################################# + +resource "dbtcloud_environment" "protected_environments" { + for_each = local.protected_envs_map + + project_id = each.value.project_id + name = each.value.env_data.name + type = each.value.env_data.type + connection_id = local.resolve_connection_id[each.key] + credential_id = local.resolve_credential_id[each.key] + + dbt_version = try(each.value.env_data.dbt_version, null) + enable_model_query_history = try(each.value.env_data.enable_model_query_history, null) + custom_branch = try(each.value.env_data.custom_branch, null) + deployment_type = try(each.value.env_data.deployment_type, null) + use_custom_branch = try(each.value.env_data.custom_branch, null) != null + extended_attributes_id = local.resolve_extended_attributes_id[each.key] + + lifecycle { + prevent_destroy = true + } } diff --git a/modules/environments/outputs.tf b/modules/environments/outputs.tf index 28ef805..68fdf7b 100644 --- a/modules/environments/outputs.tf +++ b/modules/environments/outputs.tf @@ -1,3 +1,15 @@ output "environment_ids" { - value = { for env, environment in dbtcloud_environment.environments : env => environment.environment_id } + description = "Map of composite key (project_key_env_key) to dbt Cloud environment ID" + value = merge( + { for k, e in dbtcloud_environment.environments : k => e.environment_id }, + { for k, e in dbtcloud_environment.protected_environments : k => e.environment_id } + ) +} + +output "deployment_types" { + description = "Map of composite key (project_key_env_key) to environment deployment_type (for job SAO validation)" + value = merge( + { for k, e in dbtcloud_environment.environments : k => e.deployment_type }, + { for k, e in dbtcloud_environment.protected_environments : k => e.deployment_type } + ) } diff --git a/modules/environments/tests/unit.tftest.hcl b/modules/environments/tests/unit.tftest.hcl new file mode 100644 index 0000000..fbfdb82 --- /dev/null +++ b/modules/environments/tests/unit.tftest.hcl @@ -0,0 +1,345 @@ +# Unit tests for modules/environments +# Validates composite key construction, credential/connection ID resolution, +# custom branch handling, and protected environment routing. +# Run from modules/environments/: terraform test + +mock_provider "dbtcloud" {} + +# ── Basic environment creation ──────────────────────────────────────────────── + +run "deployment_environment_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Production" + key = "prod" + type = "deployment" + deployment_type = "production" + } + ] + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = length(dbtcloud_environment.environments) == 1 + error_message = "Expected one environment to be created" + } + + assert { + condition = dbtcloud_environment.environments["analytics_prod"].name == "Production" + error_message = "Environment name should match YAML" + } + + assert { + condition = dbtcloud_environment.environments["analytics_prod"].type == "deployment" + error_message = "Environment type should match YAML" + } +} + +run "development_environment_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Development" + key = "dev" + type = "development" + } + ] + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = length(dbtcloud_environment.environments) == 1 + error_message = "Expected one development environment" + } + + assert { + condition = dbtcloud_environment.environments["analytics_dev"].type == "development" + error_message = "Environment type should be development" + } +} + +run "multiple_environments_across_projects" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Dev" + key = "dev" + type = "development" + }, + { + name = "Prod" + key = "prod" + type = "deployment" + deployment_type = "production" + } + ] + }, + { + key = "finance" + name = "Finance" + environments = [ + { + name = "Prod" + key = "prod" + type = "deployment" + deployment_type = "production" + } + ] + } + ] + project_ids = { + analytics = "1001" + finance = "1002" + } + } + + assert { + condition = length(dbtcloud_environment.environments) == 3 + error_message = "Expected three environments total" + } + + assert { + condition = contains(keys(dbtcloud_environment.environments), "analytics_dev") + error_message = "Expected composite key 'analytics_dev'" + } + + assert { + condition = contains(keys(dbtcloud_environment.environments), "finance_prod") + error_message = "Expected composite key 'finance_prod'" + } +} + +# ── Composite key construction ──────────────────────────────────────────────── + +run "composite_key_uses_env_key_field" { + command = plan + + variables { + projects = [ + { + key = "my_project" + name = "My Project" + environments = [ + { + name = "My Env Name" + key = "my_env_key" + type = "development" + } + ] + } + ] + project_ids = { my_project = "1001" } + } + + assert { + condition = contains(keys(dbtcloud_environment.environments), "my_project_my_env_key") + error_message = "Composite key should use env.key when both key and name are present" + } +} + +# ── Credential ID resolution ────────────────────────────────────────────────── + +run "credential_id_resolved_from_map" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + type = "deployment" + deployment_type = "production" + } + ] + } + ] + project_ids = { analytics = "1001" } + credential_ids = { analytics_prod = "999" } + } + + assert { + condition = dbtcloud_environment.environments["analytics_prod"].credential_id != null + error_message = "credential_id should be looked up from credential_ids map (not null)" + } +} + +run "credential_id_null_when_not_in_map" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Dev" + key = "dev" + type = "development" + } + ] + } + ] + project_ids = { analytics = "1001" } + credential_ids = {} + } + + assert { + # Mock provider returns 0 for unset numeric attributes; either value means "not set from map" + condition = dbtcloud_environment.environments["analytics_dev"].credential_id == null || dbtcloud_environment.environments["analytics_dev"].credential_id == 0 + error_message = "credential_id should be null/0 when not in credential_ids map" + } +} + +# ── Connection ID resolution ────────────────────────────────────────────────── + +run "connection_id_resolved_from_global_connections_map" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + type = "deployment" + deployment_type = "production" + connection = "my_snowflake" + } + ] + } + ] + project_ids = { analytics = "1001" } + global_connection_ids = { my_snowflake = "42" } + } + + assert { + condition = dbtcloud_environment.environments["analytics_prod"].connection_id != null + error_message = "connection_id should be resolved from global_connection_ids when connection key matches" + } +} + +# ── Custom branch handling ──────────────────────────────────────────────────── + +run "use_custom_branch_true_when_custom_branch_set" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Dev" + key = "dev" + type = "development" + custom_branch = "feature/my-branch" + } + ] + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = dbtcloud_environment.environments["analytics_dev"].use_custom_branch == true + error_message = "use_custom_branch should be true when custom_branch is set" + } + + assert { + condition = dbtcloud_environment.environments["analytics_dev"].custom_branch == "feature/my-branch" + error_message = "custom_branch value should be passed through" + } +} + +run "use_custom_branch_false_when_custom_branch_absent" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Dev" + key = "dev" + type = "development" + } + ] + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = dbtcloud_environment.environments["analytics_dev"].use_custom_branch == false + error_message = "use_custom_branch should be false when custom_branch is not set" + } +} + +# ── Protected environments ──────────────────────────────────────────────────── + +run "protected_environment_in_protected_resource" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Prod" + key = "prod" + type = "deployment" + deployment_type = "production" + protected = true + } + ] + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = length(dbtcloud_environment.protected_environments) == 1 + error_message = "Protected environment should be in protected_environments resource" + } + + assert { + condition = length(dbtcloud_environment.environments) == 0 + error_message = "Protected environment should NOT be in unprotected environments" + } +} diff --git a/modules/environments/variables.tf b/modules/environments/variables.tf index bf0e290..ce697b9 100644 --- a/modules/environments/variables.tf +++ b/modules/environments/variables.tf @@ -1,15 +1,27 @@ -variable "project_id" { - description = "The ID of the project to which environments belong" - type = string +variable "projects" { + description = "List of project configurations. Each project may have an 'environments' list." + type = any } -variable "environments_data" { - description = "List of environment configurations, including credentials" - type = any +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) } variable "credential_ids" { - description = "A map of environment names to their corresponding credential IDs" - type = map(string) - default = {} + description = "Map of composite key (project_key_env_key) to credential ID (from credentials module)" + type = map(string) + default = {} +} + +variable "global_connection_ids" { + description = "Map of global connection key to connection ID (from global_connections module). Used when YAML environments reference connections by key." + type = map(string) + default = {} +} + +variable "extended_attribute_ids" { + description = "Map of composite key (project_key_ea_key) to extended_attributes resource ID (from extended_attributes module)" + type = map(string) + default = {} } diff --git a/modules/extended_attributes/main.tf b/modules/extended_attributes/main.tf new file mode 100644 index 0000000..68c90a5 --- /dev/null +++ b/modules/extended_attributes/main.tf @@ -0,0 +1,35 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + all_extended_attributes = flatten([ + for p in var.projects : [ + for ea in try(p.extended_attributes, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + ea_key = try(ea.key, ea.name) + composite_key = "${try(p.key, p.name)}_${try(ea.key, ea.name)}" + ea_data = ea + } + ] + ]) + + ea_map = { + for item in local.all_extended_attributes : + item.composite_key => item + } +} + +resource "dbtcloud_extended_attributes" "extended_attributes" { + for_each = local.ea_map + + project_id = each.value.project_id + extended_attributes = jsonencode(try(each.value.ea_data.content, each.value.ea_data)) +} diff --git a/modules/extended_attributes/outputs.tf b/modules/extended_attributes/outputs.tf new file mode 100644 index 0000000..640faeb --- /dev/null +++ b/modules/extended_attributes/outputs.tf @@ -0,0 +1,4 @@ +output "extended_attribute_ids" { + description = "Map of composite key (project_key_ea_key) to extended_attributes resource ID" + value = { for k, ea in dbtcloud_extended_attributes.extended_attributes : k => ea.id } +} diff --git a/modules/extended_attributes/variables.tf b/modules/extended_attributes/variables.tf new file mode 100644 index 0000000..b3c5f28 --- /dev/null +++ b/modules/extended_attributes/variables.tf @@ -0,0 +1,9 @@ +variable "projects" { + description = "List of project configurations. Each project may have an 'extended_attributes' list." + type = any +} + +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) +} diff --git a/modules/global_connections/main.tf b/modules/global_connections/main.tf new file mode 100644 index 0000000..bc4e1fc --- /dev/null +++ b/modules/global_connections/main.tf @@ -0,0 +1,146 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + connections_map = { + for conn in var.connections_data : + try(conn.key, conn.name) => conn + } + + protected_connections_map = { + for k, conn in local.connections_map : + k => conn + if try(conn.protected, false) == true + } + + unprotected_connections_map = { + for k, conn in local.connections_map : + k => conn + if try(conn.protected, false) != true + } +} + +############################################# +# Unprotected Global Connections +############################################# + +resource "dbtcloud_global_connection" "connections" { + for_each = local.unprotected_connections_map + + name = each.value.name + + private_link_endpoint_id = try(each.value.private_link_endpoint_id, null) + + databricks = try(each.value.type, "") == "databricks" ? { + host = try(each.value.host, "") + http_path = try(each.value.http_path, "") + catalog = try(each.value.catalog, null) + client_id = try(var.connection_credentials[each.key].client_id, null) + client_secret = try(var.connection_credentials[each.key].client_secret, null) + } : null + + snowflake = try(each.value.type, "") == "snowflake" ? { + account = try(each.value.account, "") + database = try(each.value.database, "") + warehouse = try(each.value.warehouse, "") + role = try(each.value.role, null) + client_session_keep_alive = try(each.value.client_session_keep_alive, false) + allow_sso = try(each.value.allow_sso, false) + oauth_client_id = try(var.connection_credentials[each.key].oauth_client_id, null) + oauth_client_secret = try(var.connection_credentials[each.key].oauth_client_secret, null) + } : null + + bigquery = try(each.value.type, "") == "bigquery" ? { + gcp_project_id = try(each.value.gcp_project_id, "") + private_key_id = try(var.connection_credentials[each.key].private_key_id, null) + private_key = try(var.connection_credentials[each.key].private_key, null) + client_email = try(each.value.client_email, null) + client_id = try(each.value.client_id, null) + auth_uri = try(each.value.auth_uri, null) + token_uri = try(each.value.token_uri, null) + auth_provider_x509_cert_url = try(each.value.auth_provider_x509_cert_url, null) + client_x509_cert_url = try(each.value.client_x509_cert_url, null) + timeout_seconds = try(each.value.timeout_seconds, null) + location = try(each.value.location, null) + } : null + + postgres = try(each.value.type, "") == "postgres" ? { + hostname = try(each.value.hostname, "") + dbname = try(each.value.dbname, "") + port = try(each.value.port, 5432) + } : null + + redshift = try(each.value.type, "") == "redshift" ? { + hostname = try(each.value.hostname, "") + dbname = try(each.value.dbname, "") + port = try(each.value.port, 5439) + } : null +} + +############################################# +# Protected Global Connections — lifecycle.prevent_destroy +############################################# + +resource "dbtcloud_global_connection" "protected_connections" { + for_each = local.protected_connections_map + + name = each.value.name + + private_link_endpoint_id = try(each.value.private_link_endpoint_id, null) + + databricks = try(each.value.type, "") == "databricks" ? { + host = try(each.value.host, "") + http_path = try(each.value.http_path, "") + catalog = try(each.value.catalog, null) + client_id = try(var.connection_credentials[each.key].client_id, null) + client_secret = try(var.connection_credentials[each.key].client_secret, null) + } : null + + snowflake = try(each.value.type, "") == "snowflake" ? { + account = try(each.value.account, "") + database = try(each.value.database, "") + warehouse = try(each.value.warehouse, "") + role = try(each.value.role, null) + client_session_keep_alive = try(each.value.client_session_keep_alive, false) + allow_sso = try(each.value.allow_sso, false) + oauth_client_id = try(var.connection_credentials[each.key].oauth_client_id, null) + oauth_client_secret = try(var.connection_credentials[each.key].oauth_client_secret, null) + } : null + + bigquery = try(each.value.type, "") == "bigquery" ? { + gcp_project_id = try(each.value.gcp_project_id, "") + private_key_id = try(var.connection_credentials[each.key].private_key_id, null) + private_key = try(var.connection_credentials[each.key].private_key, null) + client_email = try(each.value.client_email, null) + client_id = try(each.value.client_id, null) + auth_uri = try(each.value.auth_uri, null) + token_uri = try(each.value.token_uri, null) + auth_provider_x509_cert_url = try(each.value.auth_provider_x509_cert_url, null) + client_x509_cert_url = try(each.value.client_x509_cert_url, null) + timeout_seconds = try(each.value.timeout_seconds, null) + location = try(each.value.location, null) + } : null + + postgres = try(each.value.type, "") == "postgres" ? { + hostname = try(each.value.hostname, "") + dbname = try(each.value.dbname, "") + port = try(each.value.port, 5432) + } : null + + redshift = try(each.value.type, "") == "redshift" ? { + hostname = try(each.value.hostname, "") + dbname = try(each.value.dbname, "") + port = try(each.value.port, 5439) + } : null + + lifecycle { + prevent_destroy = true + } +} diff --git a/modules/global_connections/outputs.tf b/modules/global_connections/outputs.tf new file mode 100644 index 0000000..408b364 --- /dev/null +++ b/modules/global_connections/outputs.tf @@ -0,0 +1,7 @@ +output "connection_ids" { + description = "Map of connection key to dbt Cloud global connection ID" + value = merge( + { for k, c in dbtcloud_global_connection.connections : k => tostring(c.id) }, + { for k, c in dbtcloud_global_connection.protected_connections : k => tostring(c.id) } + ) +} diff --git a/modules/global_connections/variables.tf b/modules/global_connections/variables.tf new file mode 100644 index 0000000..66f4e22 --- /dev/null +++ b/modules/global_connections/variables.tf @@ -0,0 +1,12 @@ +variable "connections_data" { + description = "List of global connection configurations from YAML global_connections[]" + type = any + default = [] +} + +variable "connection_credentials" { + description = "Map of connection key to OAuth/auth credential objects (sensitive fields like private_key, client_secret)" + type = map(any) + default = {} + sensitive = true +} diff --git a/modules/groups/main.tf b/modules/groups/main.tf new file mode 100644 index 0000000..a2e190f --- /dev/null +++ b/modules/groups/main.tf @@ -0,0 +1,66 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + groups_map = { + for g in var.groups_data : + try(g.key, g.name) => g + } + + protected_groups_map = { + for k, g in local.groups_map : + k => g + if try(g.protected, false) == true + } + + unprotected_groups_map = { + for k, g in local.groups_map : + k => g + if try(g.protected, false) != true + } +} + +resource "dbtcloud_group" "groups" { + for_each = local.unprotected_groups_map + + name = each.value.name + assign_by_default = try(each.value.assign_by_default, false) + sso_mapping_groups = try(each.value.sso_mapping_groups, null) + + dynamic "group_permissions" { + for_each = try(each.value.permissions, []) + content { + permission_set = group_permissions.value.permission_set + project_id = try(group_permissions.value.project_id, null) + all_projects = try(group_permissions.value.all_projects, group_permissions.value.project_id == null) + } + } +} + +resource "dbtcloud_group" "protected_groups" { + for_each = local.protected_groups_map + + name = each.value.name + assign_by_default = try(each.value.assign_by_default, false) + sso_mapping_groups = try(each.value.sso_mapping_groups, null) + + dynamic "group_permissions" { + for_each = try(each.value.permissions, []) + content { + permission_set = group_permissions.value.permission_set + project_id = try(group_permissions.value.project_id, null) + all_projects = try(group_permissions.value.all_projects, group_permissions.value.project_id == null) + } + } + + lifecycle { + prevent_destroy = true + } +} diff --git a/modules/groups/outputs.tf b/modules/groups/outputs.tf new file mode 100644 index 0000000..fb2d2d2 --- /dev/null +++ b/modules/groups/outputs.tf @@ -0,0 +1,7 @@ +output "group_ids" { + description = "Map of group key to dbt Cloud group ID" + value = merge( + { for k, g in dbtcloud_group.groups : k => g.id }, + { for k, g in dbtcloud_group.protected_groups : k => g.id } + ) +} diff --git a/modules/groups/variables.tf b/modules/groups/variables.tf new file mode 100644 index 0000000..e655ba3 --- /dev/null +++ b/modules/groups/variables.tf @@ -0,0 +1,5 @@ +variable "groups_data" { + description = "List of group configurations from YAML groups[]" + type = any + default = [] +} diff --git a/modules/ip_restrictions/main.tf b/modules/ip_restrictions/main.tf new file mode 100644 index 0000000..687eae0 --- /dev/null +++ b/modules/ip_restrictions/main.tf @@ -0,0 +1,28 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + ip_rules_map = { + for rule in var.ip_rules_data : + try(rule.key, rule.name) => rule + } +} + +resource "dbtcloud_ip_restrictions_rule" "ip_rules" { + for_each = local.ip_rules_map + + name = each.value.name + type = try(each.value.type, "allow") + description = try(each.value.description, null) + rule_set_enabled = try(each.value.rule_set_enabled, true) + cidrs = [ + for c in try(each.value.cidrs, []) : { cidr = c.cidr } + ] +} diff --git a/modules/ip_restrictions/outputs.tf b/modules/ip_restrictions/outputs.tf new file mode 100644 index 0000000..7b1f3e2 --- /dev/null +++ b/modules/ip_restrictions/outputs.tf @@ -0,0 +1,4 @@ +output "ip_rule_ids" { + description = "Map of IP rule key to dbt Cloud IP restriction rule ID" + value = { for k, r in dbtcloud_ip_restrictions_rule.ip_rules : k => r.id } +} diff --git a/modules/ip_restrictions/variables.tf b/modules/ip_restrictions/variables.tf new file mode 100644 index 0000000..668c845 --- /dev/null +++ b/modules/ip_restrictions/variables.tf @@ -0,0 +1,5 @@ +variable "ip_rules_data" { + description = "List of IP restriction rule configurations from YAML ip_restrictions[]" + type = any + default = [] +} diff --git a/modules/jobs/.terraform.lock.hcl b/modules/jobs/.terraform.lock.hcl new file mode 100644 index 0000000..8dc8cba --- /dev/null +++ b/modules/jobs/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dbt-labs/dbtcloud" { + version = "1.8.2" + constraints = "~> 1.8" + hashes = [ + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", + ] +} diff --git a/modules/jobs/main.tf b/modules/jobs/main.tf index 7ae7a7e..70cb5de 100644 --- a/modules/jobs/main.tf +++ b/modules/jobs/main.tf @@ -1,55 +1,204 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } locals { - flattened_jobs = flatten([ - for env in var.environments_data : [ - for job in env.jobs : { - environment_name = env.name - job_name = job.name - job_data = job - } - ] if try(env.jobs, null) != null - ]) + # Flatten all jobs across all projects. + # Jobs can be at project level (job.environment_key) or nested under environments. + # We support both layouts for backward compatibility. + all_jobs_flat = flatten(concat( + # Project-level jobs (new layout: project.jobs[].environment_key) + [ + for p in var.projects : [ + for job in try(p.jobs, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + env_key = try(job.environment_key, job.environment) + composite_key = "${try(p.key, p.name)}_${try(job.key, job.name)}" + job_data = job + } + ] + ], + # Environment-nested jobs (legacy v1 layout: project.environments[].jobs[]) + [ + for p in var.projects : [ + for env in try(p.environments, []) : [ + for job in try(env.jobs, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + env_key = try(env.key, env.name) + composite_key = "${try(p.key, p.name)}_${try(env.name, env.key)}_${job.name}" + job_data = job + } + ] if try(env.jobs, null) != null + ] + ] + )) jobs_map = { - for item in local.flattened_jobs : - "${item.environment_name}_${item.job_name}" => item.job_data + for item in local.all_jobs_flat : + item.composite_key => item + } + + # Resolve environment ID per job + resolve_environment_id = { + for k, item in local.jobs_map : + k => lookup( + var.environment_ids, + "${item.project_key}_${item.env_key}", + null + ) + } + + # Resolve deferring environment ID by key + resolve_deferring_environment_id = { + for k, item in local.jobs_map : + k => ( + try(item.job_data.deferring_environment_key, null) != null ? + lookup( + var.environment_ids, + "${item.project_key}_${item.job_data.deferring_environment_key}", + null + ) : null + ) + } + + # Detect CI/Merge jobs — force_node_selection must be null for these + is_ci_or_merge_job = { + for k, item in local.jobs_map : + k => ( + try(item.job_data.triggers.github_webhook, false) == true || + try(item.job_data.triggers.git_provider_webhook, false) == true || + try(item.job_data.triggers.on_merge, false) == true || + contains(["ci", "merge"], try(item.job_data.job_type, "scheduled")) + ) + } + + # force_node_selection: null for CI/Merge, otherwise from YAML + force_node_selection_effective = { + for k, item in local.jobs_map : + k => local.is_ci_or_merge_job[k] ? null : try(item.job_data.force_node_selection, null) + } + + # Schedule mutual exclusivity: cron takes precedence, then interval, then hours + schedule_cron_effective = { + for k, item in local.jobs_map : + k => ( + try(item.job_data.schedule_cron, null) != null && try(item.job_data.schedule_cron, "") != "" + ? item.job_data.schedule_cron : null + ) + } + + schedule_interval_effective = { + for k, item in local.jobs_map : + k => ( + local.schedule_cron_effective[k] == null + ? try(item.job_data.schedule_interval, null) + : null + ) + } + + schedule_hours_effective = { + for k, item in local.jobs_map : + k => ( + local.schedule_cron_effective[k] == null && local.schedule_interval_effective[k] == null + ? try(item.job_data.schedule_hours, null) + : null + ) + } + + # Protected/unprotected split + protected_jobs_map = { + for k, item in local.jobs_map : + k => item + if try(item.job_data.protected, false) == true + } + + unprotected_jobs_map = { + for k, item in local.jobs_map : + k => item + if try(item.job_data.protected, false) != true } } -resource "dbtcloud_job" "job" { - for_each = local.jobs_map - - project_id = var.project_id - name = split("_", each.key)[1] - environment_id = lookup(var.environment_ids, split("_", each.key)[0], null) # Look up the environment ID - execute_steps = each.value.execute_steps - triggers = each.value.triggers - - # Optional fields with lookup to default to null if not provided - dbt_version = lookup(each.value, "dbt_version", null) - deferring_environment_id = try(lookup(var.environment_ids, each.value.deferring_environment, null), null) - deferring_job_id = null # this is legacy anyway - description = lookup(each.value, "description", null) - errors_on_lint_failure = lookup(each.value, "errors_on_lint_failure", true) - generate_docs = lookup(each.value, "generate_docs", false) - is_active = lookup(each.value, "is_active", true) - num_threads = lookup(each.value, "num_threads", 4) - run_compare_changes = lookup(each.value, "run_compare_changes", false) - run_generate_sources = lookup(each.value, "run_generate_sources", false) - run_lint = lookup(each.value, "run_lint", false) - schedule_cron = lookup(each.value, "schedule_cron", null) - schedule_days = lookup(each.value, "schedule_days", null) - schedule_hours = lookup(each.value, "schedule_hours", null) - schedule_interval = lookup(each.value, "schedule_interval", null) - schedule_type = lookup(each.value, "schedule_type", null) - target_name = lookup(each.value, "target_name", null) - timeout_seconds = lookup(each.value, "timeout_seconds", 0) - triggers_on_draft_pr = lookup(each.value, "triggers_on_draft_pr", false) +############################################# +# Unprotected Jobs +############################################# + +resource "dbtcloud_job" "jobs" { + for_each = local.unprotected_jobs_map + + project_id = each.value.project_id + environment_id = local.resolve_environment_id[each.key] + name = each.value.job_data.name + execute_steps = each.value.job_data.execute_steps + triggers = each.value.job_data.triggers + + dbt_version = try(each.value.job_data.dbt_version, null) + description = try(each.value.job_data.description, null) + errors_on_lint_failure = try(each.value.job_data.errors_on_lint_failure, true) + generate_docs = try(each.value.job_data.generate_docs, false) + is_active = try(each.value.job_data.is_active, true) + num_threads = try(each.value.job_data.num_threads, 4) + run_compare_changes = try(each.value.job_data.run_compare_changes, false) + run_generate_sources = try(each.value.job_data.run_generate_sources, false) + run_lint = try(each.value.job_data.run_lint, false) + self_deferring = try(each.value.job_data.self_deferring, null) + target_name = try(each.value.job_data.target_name, null) + timeout_seconds = try(each.value.job_data.timeout_seconds, 0) + triggers_on_draft_pr = try(each.value.job_data.triggers_on_draft_pr, false) + force_node_selection = local.force_node_selection_effective[each.key] + deferring_environment_id = local.resolve_deferring_environment_id[each.key] + + schedule_cron = local.schedule_cron_effective[each.key] + schedule_days = try(each.value.job_data.schedule_days, null) + schedule_hours = local.schedule_hours_effective[each.key] + schedule_interval = local.schedule_interval_effective[each.key] + schedule_type = try(each.value.job_data.schedule_type, null) +} + +############################################# +# Protected Jobs — lifecycle.prevent_destroy +############################################# + +resource "dbtcloud_job" "protected_jobs" { + for_each = local.protected_jobs_map + + project_id = each.value.project_id + environment_id = local.resolve_environment_id[each.key] + name = each.value.job_data.name + execute_steps = each.value.job_data.execute_steps + triggers = each.value.job_data.triggers + + dbt_version = try(each.value.job_data.dbt_version, null) + description = try(each.value.job_data.description, null) + errors_on_lint_failure = try(each.value.job_data.errors_on_lint_failure, true) + generate_docs = try(each.value.job_data.generate_docs, false) + is_active = try(each.value.job_data.is_active, true) + num_threads = try(each.value.job_data.num_threads, 4) + run_compare_changes = try(each.value.job_data.run_compare_changes, false) + run_generate_sources = try(each.value.job_data.run_generate_sources, false) + run_lint = try(each.value.job_data.run_lint, false) + self_deferring = try(each.value.job_data.self_deferring, null) + target_name = try(each.value.job_data.target_name, null) + timeout_seconds = try(each.value.job_data.timeout_seconds, 0) + triggers_on_draft_pr = try(each.value.job_data.triggers_on_draft_pr, false) + force_node_selection = local.force_node_selection_effective[each.key] + deferring_environment_id = local.resolve_deferring_environment_id[each.key] + + schedule_cron = local.schedule_cron_effective[each.key] + schedule_days = try(each.value.job_data.schedule_days, null) + schedule_hours = local.schedule_hours_effective[each.key] + schedule_interval = local.schedule_interval_effective[each.key] + schedule_type = try(each.value.job_data.schedule_type, null) + + lifecycle { + prevent_destroy = true + } } diff --git a/modules/jobs/outputs.tf b/modules/jobs/outputs.tf index 241091c..c9b4881 100644 --- a/modules/jobs/outputs.tf +++ b/modules/jobs/outputs.tf @@ -1,5 +1,7 @@ output "job_ids" { - value = { for key, job in dbtcloud_job.job : key => job.id } + description = "Map of composite key (project_key_job_key) to dbt Cloud job ID" + value = merge( + { for k, j in dbtcloud_job.jobs : k => j.id }, + { for k, j in dbtcloud_job.protected_jobs : k => j.id } + ) } - -// {"QA_CI_job": "1234"} diff --git a/modules/jobs/tests/unit.tftest.hcl b/modules/jobs/tests/unit.tftest.hcl new file mode 100644 index 0000000..ecdf349 --- /dev/null +++ b/modules/jobs/tests/unit.tftest.hcl @@ -0,0 +1,328 @@ +# Unit tests for modules/jobs +# Validates dual layout support, composite key construction, environment ID +# resolution, CI job detection, schedule mutual exclusivity, and protected jobs. +# Run from modules/jobs/: terraform test + +mock_provider "dbtcloud" {} + +# ── Project-level job layout (new style) ───────────────────────────────────── + +run "project_level_job_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "Daily Run" + key = "daily_run" + environment_key = "prod" + execute_steps = ["dbt build"] + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = length(dbtcloud_job.jobs) == 1 + error_message = "Expected one job from project-level layout" + } + + assert { + condition = contains(keys(dbtcloud_job.jobs), "analytics_daily_run") + error_message = "Expected composite key 'analytics_daily_run'" + } + + assert { + condition = dbtcloud_job.jobs["analytics_daily_run"].name == "Daily Run" + error_message = "Job name should match YAML" + } +} + +run "project_level_job_environment_id_resolved" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "Daily Run" + key = "daily_run" + environment_key = "prod" + execute_steps = ["dbt build"] + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = dbtcloud_job.jobs["analytics_daily_run"].environment_id != null + error_message = "Job environment_id should be looked up from environment_ids map (not null)" + } +} + +# ── Legacy nested job layout ────────────────────────────────────────────────── + +run "legacy_nested_job_created" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + environments = [ + { + name = "Production" + key = "prod" + jobs = [ + { + name = "Legacy Job" + execute_steps = ["dbt run"] + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = length(dbtcloud_job.jobs) == 1 + error_message = "Expected one job from legacy nested layout" + } + + assert { + # Legacy layout uses env.name (not env.key): "${project_key}_${env.name}_${job.name}" + condition = contains(keys(dbtcloud_job.jobs), "analytics_Production_Legacy Job") + error_message = "Legacy layout composite key should be project_envname_jobname" + } +} + +# ── CI job detection and force_node_selection ───────────────────────────────── + +run "ci_job_github_webhook_clears_force_node_selection" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "CI Check" + key = "ci_check" + environment_key = "prod" + execute_steps = ["dbt build --select state:modified+"] + force_node_selection = "state:modified+" + triggers = { + schedule = false + github_webhook = true + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + # Module sets force_node_selection = null for CI jobs; mock provider returns false for unset bools + condition = dbtcloud_job.jobs["analytics_ci_check"].force_node_selection == null || dbtcloud_job.jobs["analytics_ci_check"].force_node_selection == false + error_message = "CI jobs triggered by github_webhook should have force_node_selection cleared (null/false)" + } +} + +run "scheduled_job_retains_force_node_selection" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "Daily Run" + key = "daily_run" + environment_key = "prod" + execute_steps = ["dbt build"] + force_node_selection = true + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = dbtcloud_job.jobs["analytics_daily_run"].force_node_selection == true + error_message = "Non-CI jobs should retain force_node_selection value" + } +} + +# ── Schedule mutual exclusivity ─────────────────────────────────────────────── + +run "cron_takes_precedence_over_interval" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "Daily Run" + key = "daily_run" + environment_key = "prod" + execute_steps = ["dbt build"] + schedule_cron = "0 6 * * 1-5" + schedule_interval = 2 + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = dbtcloud_job.jobs["analytics_daily_run"].schedule_cron == "0 6 * * 1-5" + error_message = "schedule_cron should be set when provided" + } + + assert { + # Mock provider returns 0 for unset numeric attributes; either value means "not scheduled by interval" + condition = dbtcloud_job.jobs["analytics_daily_run"].schedule_interval == null || dbtcloud_job.jobs["analytics_daily_run"].schedule_interval == 0 + error_message = "schedule_interval should be null/0 when schedule_cron is set" + } +} + +# ── Protected jobs ──────────────────────────────────────────────────────────── + +run "protected_job_routed_to_protected_resource" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "Production Run" + key = "prod_run" + environment_key = "prod" + execute_steps = ["dbt build"] + protected = true + triggers = { + schedule = false + github_webhook = false + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = length(dbtcloud_job.protected_jobs) == 1 + error_message = "Protected job should be in protected_jobs resource" + } + + assert { + condition = length(dbtcloud_job.jobs) == 0 + error_message = "Protected job should NOT be in unprotected jobs resource" + } +} + +# ── Deferring environment resolution ───────────────────────────────────────── + +run "deferring_environment_id_resolved" { + command = apply + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + jobs = [ + { + name = "CI Check" + key = "ci_check" + environment_key = "prod" + deferring_environment_key = "prod" + execute_steps = ["dbt build --select state:modified+"] + triggers = { + schedule = false + github_webhook = true + git_provider_webhook = false + on_merge = false + } + } + ] + } + ] + project_ids = { analytics = "1001" } + environment_ids = { analytics_prod = "2001" } + } + + assert { + condition = dbtcloud_job.jobs["analytics_ci_check"].deferring_environment_id != null + error_message = "deferring_environment_id should be resolved from environment_ids map (not null)" + } +} diff --git a/modules/jobs/variables.tf b/modules/jobs/variables.tf index cc74477..f17774c 100644 --- a/modules/jobs/variables.tf +++ b/modules/jobs/variables.tf @@ -1,14 +1,14 @@ -variable "project_id" { - description = "The ID of the project to which jobs belong" - type = string +variable "projects" { + description = "List of project configurations. Jobs may be at project.jobs[] (with environment_key) or project.environments[].jobs[] (legacy)." + type = any } -variable "environment_ids" { - description = "The ID of the project this repository is associated with" +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" type = map(string) } -variable "environments_data" { - description = "List of environment configurations, including credentials" - type = any +variable "environment_ids" { + description = "Map of composite key (project_key_env_key) to dbt Cloud environment ID (from environments module)" + type = map(string) } diff --git a/modules/lineage_integrations/main.tf b/modules/lineage_integrations/main.tf new file mode 100644 index 0000000..3a76c24 --- /dev/null +++ b/modules/lineage_integrations/main.tf @@ -0,0 +1,41 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + all_integrations = flatten([ + for p in var.projects : [ + for li in try(p.lineage_integrations, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + li_key = try(li.key, li.name) + composite_key = "${try(p.key, p.name)}_${try(li.key, li.name)}" + li_data = li + } + ] + ]) + + integrations_map = { + for item in local.all_integrations : + item.composite_key => item + } +} + +resource "dbtcloud_lineage_integration" "integrations" { + for_each = local.integrations_map + + project_id = each.value.project_id + host = each.value.li_data.host + site_id = each.value.li_data.site_id + token_name = each.value.li_data.token_name + token = try( + lookup(var.lineage_tokens, each.key, null), + each.value.li_data.token + ) +} diff --git a/modules/lineage_integrations/outputs.tf b/modules/lineage_integrations/outputs.tf new file mode 100644 index 0000000..0982b41 --- /dev/null +++ b/modules/lineage_integrations/outputs.tf @@ -0,0 +1,4 @@ +output "lineage_integration_ids" { + description = "Map of composite key (project_key_integration_key) to lineage integration ID" + value = { for k, li in dbtcloud_lineage_integration.integrations : k => li.id } +} diff --git a/modules/lineage_integrations/variables.tf b/modules/lineage_integrations/variables.tf new file mode 100644 index 0000000..fae2bab --- /dev/null +++ b/modules/lineage_integrations/variables.tf @@ -0,0 +1,16 @@ +variable "projects" { + description = "List of project configurations. Each project may have a 'lineage_integrations' list." + type = any +} + +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) +} + +variable "lineage_tokens" { + description = "Map of composite key (project_key_integration_key) to authentication token" + type = map(string) + default = {} + sensitive = true +} diff --git a/modules/notifications/main.tf b/modules/notifications/main.tf new file mode 100644 index 0000000..6ea8e5f --- /dev/null +++ b/modules/notifications/main.tf @@ -0,0 +1,30 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + notifications_map = { + for n in var.notifications_data : + try(n.key, n.name) => n + } +} + +resource "dbtcloud_notification" "notifications" { + for_each = local.notifications_map + + user_id = try(each.value.user_id, null) + on_cancel = try(each.value.on_cancel, []) + on_failure = try(each.value.on_failure, []) + on_success = try(each.value.on_success, []) + on_warning = try(each.value.on_warning, []) + notification_type = try(each.value.notification_type, 1) + slack_channel_id = try(each.value.slack_channel_id, null) + slack_channel_name = try(each.value.slack_channel_name, null) + external_email = try(each.value.external_email, null) +} diff --git a/modules/notifications/outputs.tf b/modules/notifications/outputs.tf new file mode 100644 index 0000000..2077bef --- /dev/null +++ b/modules/notifications/outputs.tf @@ -0,0 +1,4 @@ +output "notification_ids" { + description = "Map of notification key to dbt Cloud notification ID" + value = { for k, n in dbtcloud_notification.notifications : k => n.id } +} diff --git a/modules/notifications/variables.tf b/modules/notifications/variables.tf new file mode 100644 index 0000000..fafd7ee --- /dev/null +++ b/modules/notifications/variables.tf @@ -0,0 +1,5 @@ +variable "notifications_data" { + description = "List of notification configurations from YAML notifications[]" + type = any + default = [] +} diff --git a/modules/oauth_configurations/main.tf b/modules/oauth_configurations/main.tf new file mode 100644 index 0000000..a33403e --- /dev/null +++ b/modules/oauth_configurations/main.tf @@ -0,0 +1,31 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + oauth_map = { + for o in var.oauth_data : + try(o.key, o.name) => o + } +} + +resource "dbtcloud_oauth_configuration" "oauth_configurations" { + for_each = local.oauth_map + + name = each.value.name + type = each.value.type + authorize_url = each.value.authorize_url + token_url = each.value.token_url + redirect_uri = each.value.redirect_uri + client_id = each.value.client_id + client_secret = try( + lookup(var.oauth_client_secrets, each.key, null), + each.value.client_secret + ) +} diff --git a/modules/oauth_configurations/outputs.tf b/modules/oauth_configurations/outputs.tf new file mode 100644 index 0000000..8f9122b --- /dev/null +++ b/modules/oauth_configurations/outputs.tf @@ -0,0 +1,4 @@ +output "oauth_configuration_ids" { + description = "Map of OAuth configuration key to dbt Cloud OAuth configuration ID" + value = { for k, o in dbtcloud_oauth_configuration.oauth_configurations : k => o.id } +} diff --git a/modules/oauth_configurations/variables.tf b/modules/oauth_configurations/variables.tf new file mode 100644 index 0000000..a590940 --- /dev/null +++ b/modules/oauth_configurations/variables.tf @@ -0,0 +1,12 @@ +variable "oauth_data" { + description = "List of OAuth configuration entries from YAML oauth_configurations[]" + type = any + default = [] +} + +variable "oauth_client_secrets" { + description = "Map of OAuth config key to client secret value" + type = map(string) + default = {} + sensitive = true +} diff --git a/modules/profiles/main.tf b/modules/profiles/main.tf new file mode 100644 index 0000000..63490ef --- /dev/null +++ b/modules/profiles/main.tf @@ -0,0 +1,47 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + all_profiles = flatten([ + for p in var.projects : [ + for profile in try(p.profiles, []) : { + project_key = try(p.key, p.name) + project_id = var.project_ids[try(p.key, p.name)] + profile_key = try(profile.key, profile.name) + composite_key = "${try(p.key, p.name)}_${try(profile.key, profile.name)}" + profile_data = profile + } + ] + ]) + + profiles_map = { + for item in local.all_profiles : + item.composite_key => item + } +} + +resource "dbtcloud_profile" "profiles" { + for_each = local.profiles_map + + project_id = each.value.project_id + key = each.value.profile_key + connection_id = try( + lookup(var.global_connection_ids, tostring(each.value.profile_data.connection_key), null), + try(tonumber(each.value.profile_data.connection_id), null) + ) + credentials_id = try( + lookup(var.credential_ids, "${each.value.project_key}_${each.value.profile_data.credential_key}", null), + try(each.value.profile_data.credentials_id, null) + ) + extended_attributes_id = try( + lookup(var.extended_attribute_ids, "${each.value.project_key}_${each.value.profile_data.extended_attributes_key}", null), + null + ) +} diff --git a/modules/profiles/outputs.tf b/modules/profiles/outputs.tf new file mode 100644 index 0000000..d12b063 --- /dev/null +++ b/modules/profiles/outputs.tf @@ -0,0 +1,4 @@ +output "profile_ids" { + description = "Map of composite key (project_key_profile_key) to dbt Cloud profile ID" + value = { for k, p in dbtcloud_profile.profiles : k => p.id } +} diff --git a/modules/profiles/variables.tf b/modules/profiles/variables.tf new file mode 100644 index 0000000..0ff65c2 --- /dev/null +++ b/modules/profiles/variables.tf @@ -0,0 +1,27 @@ +variable "projects" { + description = "List of project configurations. Each project may have a 'profiles' list." + type = any +} + +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) +} + +variable "global_connection_ids" { + description = "Map of global connection key to connection ID (from global_connections module)" + type = map(string) + default = {} +} + +variable "credential_ids" { + description = "Map of composite key (project_key_env_key) to credential ID (from credentials module)" + type = map(string) + default = {} +} + +variable "extended_attribute_ids" { + description = "Map of composite key (project_key_ea_key) to extended_attributes ID (from extended_attributes module)" + type = map(string) + default = {} +} diff --git a/modules/project/.terraform.lock.hcl b/modules/project/.terraform.lock.hcl new file mode 100644 index 0000000..8dc8cba --- /dev/null +++ b/modules/project/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dbt-labs/dbtcloud" { + version = "1.8.2" + constraints = "~> 1.8" + hashes = [ + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", + ] +} diff --git a/modules/project/main.tf b/modules/project/main.tf index baa459c..22d89fd 100644 --- a/modules/project/main.tf +++ b/modules/project/main.tf @@ -1,11 +1,54 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } -resource "dbtcloud_project" "project" { - name = "${var.target_name}${var.project_name}" +locals { + # Normalize projects: ensure each entry has a key field. + # For single-project YAML without a key:, fall back to the project name. + projects_map = { + for p in var.projects : + try(p.key, p.name) => p + } + + protected_projects_map = { + for k, p in local.projects_map : + k => p + if try(p.protected, false) == true + } + + unprotected_projects_map = { + for k, p in local.projects_map : + k => p + if try(p.protected, false) != true + } +} + +############################################# +# Unprotected Projects +############################################# + +resource "dbtcloud_project" "projects" { + for_each = local.unprotected_projects_map + + name = "${var.target_name}${each.value.name}" +} + +############################################# +# Protected Projects — lifecycle.prevent_destroy +############################################# + +resource "dbtcloud_project" "protected_projects" { + for_each = local.protected_projects_map + + name = "${var.target_name}${each.value.name}" + + lifecycle { + prevent_destroy = true + } } diff --git a/modules/project/outputs.tf b/modules/project/outputs.tf index 7109535..afc0b66 100644 --- a/modules/project/outputs.tf +++ b/modules/project/outputs.tf @@ -1,3 +1,7 @@ -output "project_id" { - value = dbtcloud_project.project.id +output "project_ids" { + description = "Map of project key to dbt Cloud project ID" + value = merge( + { for k, p in dbtcloud_project.projects : k => p.id }, + { for k, p in dbtcloud_project.protected_projects : k => p.id } + ) } diff --git a/modules/project/tests/unit.tftest.hcl b/modules/project/tests/unit.tftest.hcl new file mode 100644 index 0000000..ba78b6b --- /dev/null +++ b/modules/project/tests/unit.tftest.hcl @@ -0,0 +1,200 @@ +# Unit tests for modules/project +# Validates project naming, target_name prefix, and protected resource routing. +# Run from modules/project/: terraform test + +mock_provider "dbtcloud" {} + +# ── Basic project creation ──────────────────────────────────────────────────── + +run "single_project_created" { + command = plan + + variables { + projects = [ + { + name = "My Project" + key = "my_project" + } + ] + target_name = "" + } + + assert { + condition = length(dbtcloud_project.projects) == 1 + error_message = "Expected one unprotected project to be created" + } + + assert { + condition = dbtcloud_project.projects["my_project"].name == "My Project" + error_message = "Project name should match YAML name" + } +} + +run "project_key_falls_back_to_name" { + command = plan + + variables { + projects = [ + { + name = "No Key Project" + } + ] + target_name = "" + } + + assert { + condition = length(dbtcloud_project.projects) == 1 + error_message = "Expected one project even without a key field" + } + + assert { + condition = contains(keys(dbtcloud_project.projects), "No Key Project") + error_message = "Key should fall back to project name when key is absent" + } +} + +# ── target_name prefix ──────────────────────────────────────────────────────── + +run "target_name_prepended_to_project_name" { + command = plan + + variables { + projects = [ + { + name = "Analytics" + key = "analytics" + } + ] + target_name = "dev-" + } + + assert { + condition = dbtcloud_project.projects["analytics"].name == "dev-Analytics" + error_message = "target_name prefix should be prepended to the project name" + } +} + +run "empty_target_name_leaves_name_unchanged" { + command = plan + + variables { + projects = [ + { + name = "Analytics" + key = "analytics" + } + ] + target_name = "" + } + + assert { + condition = dbtcloud_project.projects["analytics"].name == "Analytics" + error_message = "Empty target_name should not alter the project name" + } +} + +# ── Protected resources ─────────────────────────────────────────────────────── + +run "protected_project_routed_to_protected_resource" { + command = plan + + variables { + projects = [ + { + name = "Finance" + key = "finance" + protected = true + } + ] + target_name = "" + } + + assert { + condition = length(dbtcloud_project.protected_projects) == 1 + error_message = "Protected project should be in protected_projects resource" + } + + assert { + condition = length(dbtcloud_project.projects) == 0 + error_message = "Protected project should NOT be in unprotected projects resource" + } +} + +run "unprotected_project_not_in_protected_resource" { + command = plan + + variables { + projects = [ + { + name = "Analytics" + key = "analytics" + } + ] + target_name = "" + } + + assert { + condition = length(dbtcloud_project.protected_projects) == 0 + error_message = "Unprotected project should not appear in protected_projects" + } +} + +run "mixed_protected_and_unprotected_projects" { + command = plan + + variables { + projects = [ + { + name = "Analytics" + key = "analytics" + }, + { + name = "Finance" + key = "finance" + protected = true + } + ] + target_name = "" + } + + assert { + condition = length(dbtcloud_project.projects) == 1 + error_message = "Expected one unprotected project" + } + + assert { + condition = length(dbtcloud_project.protected_projects) == 1 + error_message = "Expected one protected project" + } +} + +# ── Output structure ────────────────────────────────────────────────────────── + +run "output_project_ids_merges_both_resource_sets" { + command = plan + + variables { + projects = [ + { + name = "Analytics" + key = "analytics" + }, + { + name = "Finance" + key = "finance" + protected = true + } + ] + target_name = "" + } + + assert { + condition = contains(keys(output.project_ids), "analytics") + error_message = "project_ids output should contain unprotected project key" + } + + assert { + condition = contains(keys(output.project_ids), "finance") + error_message = "project_ids output should contain protected project key" + } +} diff --git a/modules/project/variables.tf b/modules/project/variables.tf index 5e24ec7..4fa5dad 100644 --- a/modules/project/variables.tf +++ b/modules/project/variables.tf @@ -1,9 +1,10 @@ -variable "project_name" { - description = "Project name" - type = string +variable "projects" { + description = "List of project configurations. Each entry must have at minimum a 'name' field, and optionally a 'key' field (defaults to name) and 'protected' boolean." + type = any } variable "target_name" { - description = "Target CI or Production" - type = string + description = "Optional prefix prepended to all project names (e.g., 'dev-' or 'prod-')" + type = string + default = "" } diff --git a/modules/project_artefacts/main.tf b/modules/project_artefacts/main.tf new file mode 100644 index 0000000..bf38692 --- /dev/null +++ b/modules/project_artefacts/main.tf @@ -0,0 +1,34 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + # Only create artefacts for projects that have an artefacts block + artefacts_map = { + for p in var.projects : + try(p.key, p.name) => p + if try(p.artefacts, null) != null + } +} + +resource "dbtcloud_project_artefacts" "artefacts" { + for_each = local.artefacts_map + + project_id = var.project_ids[each.key] + + docs_job_id = try( + lookup(var.job_ids, "${each.key}_${each.value.artefacts.docs_job}", null), + null + ) + + freshness_job_id = try( + lookup(var.job_ids, "${each.key}_${each.value.artefacts.freshness_job}", null), + null + ) +} diff --git a/modules/project_artefacts/outputs.tf b/modules/project_artefacts/outputs.tf new file mode 100644 index 0000000..f107439 --- /dev/null +++ b/modules/project_artefacts/outputs.tf @@ -0,0 +1,4 @@ +output "project_artefact_ids" { + description = "Map of project key to project_artefacts resource ID" + value = { for k, a in dbtcloud_project_artefacts.artefacts : k => a.id } +} diff --git a/modules/project_artefacts/variables.tf b/modules/project_artefacts/variables.tf new file mode 100644 index 0000000..268eeb6 --- /dev/null +++ b/modules/project_artefacts/variables.tf @@ -0,0 +1,15 @@ +variable "projects" { + description = "List of project configurations. Each project may have an 'artefacts' block with docs_job and freshness_job keys." + type = any +} + +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) +} + +variable "job_ids" { + description = "Map of composite key (project_key_job_key) to dbt Cloud job ID (from jobs module)" + type = map(string) + default = {} +} diff --git a/modules/project_repository/main.tf b/modules/project_repository/main.tf index 35defd4..d8d58cd 100644 --- a/modules/project_repository/main.tf +++ b/modules/project_repository/main.tf @@ -1,12 +1,16 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" + version = "~> 1.8" } } } -resource "dbtcloud_project_repository" "project_repository" { - project_id = var.project_id - repository_id = var.repository_id +resource "dbtcloud_project_repository" "project_repositories" { + for_each = var.repository_ids + + project_id = var.project_ids[each.key] + repository_id = each.value } diff --git a/modules/project_repository/outputs.tf b/modules/project_repository/outputs.tf index 9086a1c..c34cf5f 100644 --- a/modules/project_repository/outputs.tf +++ b/modules/project_repository/outputs.tf @@ -1,3 +1,4 @@ -output "project_repository_id" { - value = dbtcloud_project_repository.project_repository.id +output "project_repository_ids" { + description = "Map of project key to project_repository resource ID" + value = { for k, pr in dbtcloud_project_repository.project_repositories : k => pr.id } } diff --git a/modules/project_repository/variables.tf b/modules/project_repository/variables.tf index 088fd37..5a33508 100644 --- a/modules/project_repository/variables.tf +++ b/modules/project_repository/variables.tf @@ -1,9 +1,9 @@ -variable "project_id" { - description = "The ID of the project this repository is associated with" - type = string +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) } -variable "repository_id" { - description = "The ID of the repository this project is associated with" - type = string +variable "repository_ids" { + description = "Map of project key to repository_id (the integer ID used for project_repository links, not the resource ID)" + type = map(string) } diff --git a/modules/repository/.terraform.lock.hcl b/modules/repository/.terraform.lock.hcl index 739e63e..8dc8cba 100644 --- a/modules/repository/.terraform.lock.hcl +++ b/modules/repository/.terraform.lock.hcl @@ -2,41 +2,26 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/dbt-labs/dbtcloud" { - version = "1.3.0" + version = "1.8.2" + constraints = "~> 1.8" hashes = [ - "h1:t1pNp1exAjFXf8ovebsD3GwZ3vCC+tdwPSyy5li+ueA=", - "zh:084c9b84685d8e1d9d46eb7e1e9d82b45b4cceb8ae233a0a24ff97b545fdd540", - "zh:2ad7ccc1587323cf31f8cbc999d2993310e35c5bca88d75ceeaef6e032b7dc17", - "zh:3275342d2b6128b8f50fc6abeb3e267bbc35d0c0ef56065fec34483771726442", - "zh:369d467b451d848710f9c2b5059afd2e860243d475f4423db8e3cb39d2bed7ab", - "zh:41d595ee75946bf009f997a141c53e2fdaa24dc1d451aeebe3acc1a403629d76", - "zh:5dccf653e3c7d0185ae96ab77f292e3e6ee7a6facd19de9e14bdc910f6d0ec63", - "zh:5ee3e4cc028f4b51bcf2eddecde020f364862929fe773f31e7b8758a2f1d789b", - "zh:7085f53ed947aec4ff66f2c93b89bf5246e752a86f8272c070deb5acf64d8965", - "zh:8c215d812f449d227e9b8adb2a511124afd5f0a0c4ff428492e44c855345e549", - "zh:8da0a85d527f7d44141e72cb52e637a50117a623545efea93b70ba70b6fd9d35", - "zh:9b7c7a6deab3115f4f0b4f9a2fe4a3f2e4732c19d094d24c1104b5447f850349", - "zh:b096366cc30c4d006f2f8caedefa64d693eba7a33d6cf25e07d0829cfc839961", - "zh:c848fa337e22b1953050784b01f56bfc76043516b107ee8cfba51aebca1174a8", - "zh:ce6a12d19dac844312505855522d9aa641e595297deece939076df0be398222b", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.4" - hashes = [ - "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", - "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", - "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", - "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", - "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", - "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", - "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", - "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", - "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", - "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", - "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", + "h1:3Hw1in/qcHdAp2BRhEaFRr/ez9lFjBv3V5WIUxHXcSs=", + "h1:ASwPE1I3U7cufOLFvGV+yHmV4/24y2Jrmt8V3Pn9lEY=", + "h1:BIVjUC8FxluKSvwm2jvySD0AvSkFPKwGn/SHez082Oc=", + "h1:IpC3g4srpeI3iJ5GL3GcxSMO/+XHBktS/zumEm/8xbc=", + "zh:01df4a2592c7d66152aa18bae2fb6abcca205f2c0d75486a67e3c8307ea70af2", + "zh:020e8c80eb8973ac0571b8cb2c990d76c82d2032f590ffaec418fc6728a48594", + "zh:0c30dd1d7ac516efe921362143c0372b2dedab59950662c08513c8c1e0430ee4", + "zh:192bc343bfe193d79b4f2e5d5340a44230e169670d190c7034c7db67fd97b3e7", + "zh:24b8175f22a97844000facea8742bd2f0ca5eedb214c900e6b50153308874e86", + "zh:26ef5f40358401f7fadae72b16782d0a8a8761af325aac17b5507234148c19f9", + "zh:7b9cd9637bf607854dfdeceab06dd361686f54864050cfb770d4214a2d0265db", + "zh:8df586c709b83ca62056fd3d9ff5e9d73833acdcb3194d7e5bbc0cde98f87ed9", + "zh:8e561719c50ae83caefa0dd8ebf3cf4907a512d1be90ecb80eb5db6593f9e9b2", + "zh:9071202c4b459dd28fa792a62e229815bae41d637428fa66dca5eb7fa0fc5801", + "zh:926cca7812b272ab976ce49c0858de78bd5c4c6fcffe16cb2d33cdce63522bc2", + "zh:ae927fc0b059c800b0342b6c6f7217dd191349334a6d6ed1cad578f13f64722c", + "zh:b0e484d5c2dc79b1f573aa3bdac4150a2b4094b6fd9c92033b855e7b8f07045c", + "zh:f96b0ce96fe679653e5dd3820f15e7916855ce8deb832f6b92befedc2387f1f6", ] } diff --git a/modules/repository/main.tf b/modules/repository/main.tf index 7657b68..086a6a5 100644 --- a/modules/repository/main.tf +++ b/modules/repository/main.tf @@ -1,202 +1,190 @@ terraform { + required_version = ">= 1.7" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" - } - null = { - source = "hashicorp/null" + version = "~> 1.8" } } } -################################################################################ -# Auto-Detection & Validation Logic -# Detects git provider from remote_url and validates configuration -################################################################################ - locals { - # Auto-detect git provider from remote_url - # This determines which native integration fields are valid - detected_provider = ( - can(regex("github\\.com", var.repository_data.remote_url)) ? "github" : - can(regex("gitlab\\.com", var.repository_data.remote_url)) ? "gitlab" : - can(regex("dev\\.azure\\.com|ssh\\.dev\\.azure\\.com", var.repository_data.remote_url)) ? "azure_devops" : - can(regex("bitbucket\\.org", var.repository_data.remote_url)) ? "bitbucket" : - "unknown" - ) + # Build a map of project_key => repository object (skip projects with no repository) + repos_map = { + for p in var.projects : + try(p.key, p.name) => p.repository + if try(p.repository, null) != null + } - # Extract git_clone_strategy from input, or auto-detect based on provider - git_clone_strategy_explicit = try(var.repository_data.git_clone_strategy, null) - - git_clone_strategy = local.git_clone_strategy_explicit != null ? local.git_clone_strategy_explicit : ( - local.detected_provider == "github" ? "github_app" : - local.detected_provider == "gitlab" ? "deploy_token" : - local.detected_provider == "azure_devops" ? "azure_active_directory_app" : - "deploy_key" - ) + # All projects as map for protection lookups + all_projects_map = { + for p in var.projects : + try(p.key, p.name) => p + } - # Validation: Ensure provider matches clone strategy - provider_strategy_mismatch = ( - (local.detected_provider == "github" && local.git_clone_strategy == "deploy_token") || - (local.detected_provider == "github" && local.git_clone_strategy == "azure_active_directory_app") || - (local.detected_provider == "gitlab" && local.git_clone_strategy == "github_app") || - (local.detected_provider == "gitlab" && local.git_clone_strategy == "azure_active_directory_app") || - (local.detected_provider == "azure_devops" && local.git_clone_strategy == "github_app") || - (local.detected_provider == "azure_devops" && local.git_clone_strategy == "deploy_token") - ) + # Auto-detect provider from remote_url + detected_provider = { + for k, repo in local.repos_map : + k => ( + can(regex("github\\.com", try(repo.remote_url, ""))) ? "github" : + can(regex("gitlab\\.com|gitlab\\.", try(repo.remote_url, ""))) ? "gitlab" : + can(regex("dev\\.azure\\.com|ssh\\.dev\\.azure\\.com", try(repo.remote_url, ""))) ? "azure_devops" : + can(regex("bitbucket\\.org", try(repo.remote_url, ""))) ? "bitbucket" : + "unknown" + ) + } - # Validation: Check for required fields based on clone strategy - github_app_missing_id = ( - local.git_clone_strategy == "github_app" && - (try(var.repository_data.github_installation_id, null) == null) - ) + # Effective git clone strategy per repo (with fallbacks) + effective_git_clone_strategy = { + for k, repo in local.repos_map : + k => ( + # Azure DevOps: downgrade if required IDs are missing + try(repo.git_clone_strategy, "") == "azure_active_directory_app" && ( + trimspace(tostring(try(repo.azure_active_directory_project_id, ""))) == "" || + trimspace(tostring(try(repo.azure_active_directory_repository_id, ""))) == "" + ) ? "deploy_key" : + # GitLab deploy_token: downgrade to deploy_key unless explicitly enabled + try(repo.git_clone_strategy, "") == "deploy_token" ? ( + var.enable_gitlab_deploy_token ? "deploy_token" : "deploy_key" + ) : + # GitHub App: use if installation ID available, else deploy_key + try(repo.git_clone_strategy, "") == "github_app" ? ( + try(repo.github_installation_id, null) != null || var.dbt_pat != null ? "github_app" : "deploy_key" + ) : + # Explicit strategy set to something other than the above + try(repo.git_clone_strategy, null) != null ? try(repo.git_clone_strategy, "deploy_key") : + # Auto-detect defaults + local.detected_provider[k] == "github" ? "github_app" : + local.detected_provider[k] == "gitlab" ? "deploy_token" : + local.detected_provider[k] == "azure_devops" ? "azure_active_directory_app" : + "deploy_key" + ) + } - gitlab_deploy_token_missing_id = ( - local.git_clone_strategy == "deploy_token" && - (try(var.repository_data.gitlab_project_id, null) == null) - ) + # Downgraded GitLab repos (deploy_token → deploy_key) need SSH URL format + gitlab_deploy_token_downgraded = { + for k, repo in local.repos_map : + k => try(repo.git_clone_strategy, "") == "deploy_token" && local.effective_git_clone_strategy[k] == "deploy_key" + } - azure_missing_project_id = ( - local.git_clone_strategy == "azure_active_directory_app" && - (try(var.repository_data.azure_active_directory_project_id, null) == null) - ) + # Extract GitLab hostname from pull_request_url_template for SSH URL construction + gitlab_ssh_host = { + for k, repo in local.repos_map : + k => try(regex("https?://([^/]+)/", try(repo.pull_request_url_template, ""))[0], "gitlab.com") + } - azure_missing_repo_id = ( - local.git_clone_strategy == "azure_active_directory_app" && - (try(var.repository_data.azure_active_directory_repository_id, null) == null) - ) + # Effective remote URL + effective_remote_url = { + for k, repo in local.repos_map : + k => ( + local.gitlab_deploy_token_downgraded[k] ? ( + "git@${local.gitlab_ssh_host[k]}:${trimspace(try(repo.remote_url, ""))}.git" + ) : + contains(["github_app", "azure_active_directory_app"], local.effective_git_clone_strategy[k]) ? ( + trimspace(try(repo.remote_url, "")) + ) : + can(regex("^(git@|ssh:)", trimspace(try(repo.remote_url, "")))) ? + trimspace(try(repo.remote_url, "")) : + "git@github.com:dbt-labs/jaffle-shop.git" + ) + } - # Validation: Check for provider-specific fields that don't match the provider - github_id_on_non_github = ( - local.detected_provider != "github" && - (try(var.repository_data.github_installation_id, null) != null) - ) + # Effective protection status per repo + # Uses repository_protected if explicitly set, else falls back to project.protected + repo_protected = { + for k, repo in local.repos_map : + k => ( + try(local.all_projects_map[k].repository_protected, null) != null + ? local.all_projects_map[k].repository_protected + : try(local.all_projects_map[k].protected, false) + ) + } - gitlab_id_on_non_gitlab = ( - local.detected_provider != "gitlab" && - (try(var.repository_data.gitlab_project_id, null) != null) - ) + protected_repos_map = { + for k, repo in local.repos_map : k => repo + if local.repo_protected[k] == true + } - azure_project_id_on_non_azure = ( - local.detected_provider != "azure_devops" && - (try(var.repository_data.azure_active_directory_project_id, null) != null) - ) + unprotected_repos_map = { + for k, repo in local.repos_map : k => repo + if local.repo_protected[k] != true + } +} + +############################################# +# Unprotected Repositories +############################################# - azure_repo_id_on_non_azure = ( - local.detected_provider != "azure_devops" && - (try(var.repository_data.azure_active_directory_repository_id, null) != null) +resource "dbtcloud_repository" "repositories" { + for_each = local.unprotected_repos_map + + project_id = var.project_ids[each.key] + remote_url = local.effective_remote_url[each.key] + is_active = try(each.value.is_active, true) + git_clone_strategy = local.effective_git_clone_strategy[each.key] + + github_installation_id = ( + local.effective_git_clone_strategy[each.key] == "github_app" + ? try(each.value.github_installation_id, null) + : null ) - azure_webhook_bypass_on_non_azure = ( - local.detected_provider != "azure_devops" && - (try(var.repository_data.azure_bypass_webhook_registration_failure, false) != false) + gitlab_project_id = ( + local.gitlab_deploy_token_downgraded[each.key] ? null + : try(each.value.gitlab_project_id, null) ) - # Compile all validation errors - validation_errors = concat( - local.provider_strategy_mismatch ? [ - "❌ CONFIGURATION ERROR: git_clone_strategy '${local.git_clone_strategy}' does not match detected provider '${local.detected_provider}'. Check remote_url and git_clone_strategy." - ] : [], - local.github_app_missing_id ? [ - "❌ CONFIGURATION ERROR: git_clone_strategy 'github_app' requires 'github_installation_id'. See documentation for how to find your GitHub App installation ID." - ] : [], - local.gitlab_deploy_token_missing_id ? [ - "❌ CONFIGURATION ERROR: git_clone_strategy 'deploy_token' requires 'gitlab_project_id'. See documentation for how to find your GitLab project ID." - ] : [], - local.azure_missing_project_id ? [ - "❌ CONFIGURATION ERROR: git_clone_strategy 'azure_active_directory_app' requires 'azure_active_directory_project_id'. See documentation for how to find your Azure DevOps project ID." - ] : [], - local.azure_missing_repo_id ? [ - "❌ CONFIGURATION ERROR: git_clone_strategy 'azure_active_directory_app' requires 'azure_active_directory_repository_id'. See documentation for how to find your Azure DevOps repository ID." - ] : [], - local.github_id_on_non_github ? [ - "⚠️ WARNING: 'github_installation_id' is set but remote_url is not a GitHub URL. This field will be ignored. Did you mean to use a GitHub repository?" - ] : [], - local.gitlab_id_on_non_gitlab ? [ - "⚠️ WARNING: 'gitlab_project_id' is set but remote_url is not a GitLab URL. This field will be ignored. Did you mean to use a GitLab repository?" - ] : [], - local.azure_project_id_on_non_azure ? [ - "⚠️ WARNING: 'azure_active_directory_project_id' is set but remote_url is not an Azure DevOps URL. This field will be ignored. Did you mean to use an Azure DevOps repository?" - ] : [], - local.azure_repo_id_on_non_azure ? [ - "⚠️ WARNING: 'azure_active_directory_repository_id' is set but remote_url is not an Azure DevOps URL. This field will be ignored. Did you mean to use an Azure DevOps repository?" - ] : [], - local.azure_webhook_bypass_on_non_azure ? [ - "⚠️ WARNING: 'azure_bypass_webhook_registration_failure' is set but remote_url is not an Azure DevOps URL. This field will be ignored. Did you mean to use an Azure DevOps repository?" - ] : [] + azure_active_directory_project_id = try( + each.value.azure_active_directory_project_id, null + ) + azure_active_directory_repository_id = try( + each.value.azure_active_directory_repository_id, null + ) + azure_bypass_webhook_registration_failure = try( + each.value.azure_bypass_webhook_registration_failure, false ) - # Fail if there are critical errors (not just warnings) - has_critical_errors = anytrue([ - local.provider_strategy_mismatch, - local.github_app_missing_id, - local.gitlab_deploy_token_missing_id, - local.azure_missing_project_id, - local.azure_missing_repo_id - ]) + private_link_endpoint_id = try(each.value.private_link_endpoint_id, null) + pull_request_url_template = try(each.value.pull_request_url_template, null) } -# Validation check: Fail if critical errors detected -resource "null_resource" "validation" { - lifecycle { - precondition { - condition = !local.has_critical_errors - error_message = join("\n", concat( - local.validation_errors, - [ - "", - "--- CONFIGURATION HELP ---", - "Detected Provider: ${local.detected_provider}", - "Auto-Selected Strategy: ${local.git_clone_strategy}", - "Remote URL: ${var.repository_data.remote_url}", - "", - "Supported Strategies:", - " - deploy_key (default for all providers): No additional configuration needed", - " - github_app (GitHub only): Requires github_installation_id", - " - deploy_token (GitLab only): Requires gitlab_project_id", - " - azure_active_directory_app (Azure DevOps only): Requires azure_active_directory_project_id and azure_active_directory_repository_id", - "", - "See docs/REPOSITORY_CONFIGURATION.md for detailed setup instructions." - ] - )) - } - } - - triggers = { - repository = var.repository_data.remote_url - } -} +############################################# +# Protected Repositories — lifecycle.prevent_destroy +############################################# -# Create the repository -resource "dbtcloud_repository" "repository" { - depends_on = [null_resource.validation] +resource "dbtcloud_repository" "protected_repositories" { + for_each = local.protected_repos_map - project_id = var.project_id - remote_url = var.repository_data.remote_url - is_active = try(var.repository_data.is_active, true) - git_clone_strategy = local.git_clone_strategy + project_id = var.project_ids[each.key] + remote_url = local.effective_remote_url[each.key] + is_active = try(each.value.is_active, true) + git_clone_strategy = local.effective_git_clone_strategy[each.key] - # GitHub native integration - github_installation_id = try(var.repository_data.github_installation_id, null) + github_installation_id = ( + local.effective_git_clone_strategy[each.key] == "github_app" + ? try(each.value.github_installation_id, null) + : null + ) - # GitLab native integration - gitlab_project_id = try(var.repository_data.gitlab_project_id, null) + gitlab_project_id = ( + local.gitlab_deploy_token_downgraded[each.key] ? null + : try(each.value.gitlab_project_id, null) + ) - # Azure DevOps native integration azure_active_directory_project_id = try( - var.repository_data.azure_active_directory_project_id, - null + each.value.azure_active_directory_project_id, null ) azure_active_directory_repository_id = try( - var.repository_data.azure_active_directory_repository_id, - null + each.value.azure_active_directory_repository_id, null ) azure_bypass_webhook_registration_failure = try( - var.repository_data.azure_bypass_webhook_registration_failure, - false + each.value.azure_bypass_webhook_registration_failure, false ) - # Optional common fields - private_link_endpoint_id = try(var.repository_data.private_link_endpoint_id, null) - pull_request_url_template = try(var.repository_data.pull_request_url_template, null) + private_link_endpoint_id = try(each.value.private_link_endpoint_id, null) + pull_request_url_template = try(each.value.pull_request_url_template, null) + + lifecycle { + prevent_destroy = true + } } diff --git a/modules/repository/outputs.tf b/modules/repository/outputs.tf index 8f946e4..444507a 100644 --- a/modules/repository/outputs.tf +++ b/modules/repository/outputs.tf @@ -1,7 +1,7 @@ -output "repository_id" { - value = dbtcloud_repository.repository.id -} - -output "project_repository_id" { - value = dbtcloud_repository.repository.repository_id +output "repository_ids" { + description = "Map of project key to repository_id (integer ID used for project_repository links)" + value = merge( + { for k, r in dbtcloud_repository.repositories : k => tostring(r.repository_id) }, + { for k, r in dbtcloud_repository.protected_repositories : k => tostring(r.repository_id) } + ) } diff --git a/modules/repository/tests/unit.tftest.hcl b/modules/repository/tests/unit.tftest.hcl new file mode 100644 index 0000000..f0a955d --- /dev/null +++ b/modules/repository/tests/unit.tftest.hcl @@ -0,0 +1,252 @@ +# Unit tests for modules/repository +# Validates git provider auto-detection, clone strategy selection, and fallback +# behavior for GitHub (no installation ID), GitLab (deploy_token gate), and +# Azure DevOps (missing IDs fallback). +# Run from modules/repository/: terraform test + +mock_provider "dbtcloud" {} + +# ── GitHub URL auto-detection ───────────────────────────────────────────────── + +run "github_url_without_explicit_strategy_auto_detects_github_app" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "git@github.com:my-org/analytics.git" + } + } + ] + project_ids = { analytics = "1001" } + dbt_pat = null + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "github_app" + error_message = "GitHub URL without explicit strategy auto-detects to github_app (no installation_id check in auto-detect path)" + } +} + +run "github_url_with_installation_id_uses_github_app" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://github.com/my-org/analytics" + github_installation_id = 12345678 + } + } + ] + project_ids = { analytics = "1001" } + dbt_pat = null + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "github_app" + error_message = "GitHub URL with installation_id should use github_app strategy" + } +} + +run "github_url_with_pat_uses_github_app" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://github.com/my-org/analytics" + } + } + ] + project_ids = { analytics = "1001" } + dbt_pat = "ghp_fake_pat_token" + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "github_app" + error_message = "GitHub URL with PAT set should use github_app strategy" + } +} + +# ── GitLab URL auto-detection ───────────────────────────────────────────────── + +run "gitlab_url_without_explicit_strategy_auto_detects_deploy_token" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://gitlab.com/my-org/analytics" + } + } + ] + project_ids = { analytics = "1001" } + enable_gitlab_deploy_token = false + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "deploy_token" + error_message = "GitLab URL without explicit strategy auto-detects to deploy_token (enable_gitlab_deploy_token only gates explicit deploy_token strategy)" + } +} + +run "gitlab_deploy_token_enabled_uses_deploy_token" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://gitlab.com/my-org/analytics" + git_clone_strategy = "deploy_token" + gitlab_project_id = 999 + } + } + ] + project_ids = { analytics = "1001" } + enable_gitlab_deploy_token = true + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "deploy_token" + error_message = "GitLab with enable_gitlab_deploy_token=true and explicit deploy_token strategy should use deploy_token" + } +} + +# ── Azure DevOps URL auto-detection ────────────────────────────────────────── + +run "azure_devops_url_without_ids_falls_back_to_deploy_key" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://dev.azure.com/my-org/my-project/_git/analytics" + git_clone_strategy = "azure_active_directory_app" + } + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "deploy_key" + error_message = "Azure DevOps without required IDs should fall back to deploy_key" + } +} + +run "azure_devops_with_required_ids_uses_aad_strategy" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "https://dev.azure.com/my-org/my-project/_git/analytics" + git_clone_strategy = "azure_active_directory_app" + azure_active_directory_project_id = "proj-uuid" + azure_active_directory_repository_id = "repo-uuid" + } + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "azure_active_directory_app" + error_message = "Azure DevOps with both IDs should use azure_active_directory_app" + } +} + +# ── Generic / unknown URL ───────────────────────────────────────────────────── + +run "unknown_url_uses_deploy_key" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + repository = { + remote_url = "git@bitbucket.org:my-org/analytics.git" + } + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = dbtcloud_repository.repositories["analytics"].git_clone_strategy == "deploy_key" + error_message = "Bitbucket URL without explicit strategy should use deploy_key" + } +} + +# ── Protected repositories ──────────────────────────────────────────────────── + +run "protected_repository_routed_to_protected_resource" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + protected = true + repository = { + remote_url = "git@github.com:my-org/analytics.git" + } + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = length(dbtcloud_repository.protected_repositories) == 1 + error_message = "Protected project should route repository to protected_repositories" + } + + assert { + condition = length(dbtcloud_repository.repositories) == 0 + error_message = "Protected project should not put repository in unprotected resource" + } +} + +run "project_with_no_repository_creates_no_resource" { + command = plan + + variables { + projects = [ + { + key = "analytics" + name = "Analytics" + } + ] + project_ids = { analytics = "1001" } + } + + assert { + condition = length(dbtcloud_repository.repositories) == 0 + error_message = "Project without repository block should create no repository resource" + } +} diff --git a/modules/repository/variables.tf b/modules/repository/variables.tf index 18d431f..1771144 100644 --- a/modules/repository/variables.tf +++ b/modules/repository/variables.tf @@ -1,70 +1,22 @@ -variable "repository_data" { - description = <<-EOT - Repository configuration object with auto-detection and validation. - - Supported providers (auto-detected from remote_url): - - GitHub: https://github.com/* or git@github.com:* (supports github_app strategy) - - GitLab: https://gitlab.com/* or git@gitlab.com:* (supports deploy_token strategy) - - Azure DevOps: https://dev.azure.com/* or git@ssh.dev.azure.com:* (supports azure_active_directory_app strategy) - - Bitbucket: https://bitbucket.org/* (uses deploy_key strategy) - - Generic: Any other URL (uses deploy_key strategy) - - Required fields: - - remote_url: Git repository URL (HTTPS or SSH) - - Optional fields depend on git_clone_strategy: - - git_clone_strategy: Auto-detected, but can be explicitly set to: - * deploy_key (default for all providers) - * github_app (GitHub only, requires github_installation_id) - * deploy_token (GitLab only, requires gitlab_project_id) - * azure_active_directory_app (Azure DevOps only, requires azure_active_directory_project_id and azure_active_directory_repository_id) - - - github_installation_id: (GitHub app integration only) Integer ID of GitHub App installation - - gitlab_project_id: (GitLab integration only) Integer ID of GitLab project - - azure_active_directory_project_id: (Azure DevOps only) UUID of Azure DevOps project - - azure_active_directory_repository_id: (Azure DevOps only) UUID of Azure DevOps repository - - azure_bypass_webhook_registration_failure: (Azure DevOps only) Boolean, default false - - - is_active: Boolean, default true - - private_link_endpoint_id: Optional private link endpoint ID (all providers) - - pull_request_url_template: Optional custom PR URL template (all providers) - - Example GitHub with GitHub App: - repository = { - remote_url = "https://github.com/myorg/myrepo.git" - git_clone_strategy = "github_app" - github_installation_id = 12345678 - } - - Example GitLab with Deploy Token: - repository = { - remote_url = "https://gitlab.com/mygroup/myproject.git" - git_clone_strategy = "deploy_token" - gitlab_project_id = 9876543 - } - - Example Azure DevOps: - repository = { - remote_url = "https://dev.azure.com/myorg/myproject/_git/myrepo" - git_clone_strategy = "azure_active_directory_app" - azure_active_directory_project_id = "550e8400-e29b-41d4-a716-446655440000" - azure_active_directory_repository_id = "550e8400-e29b-41d4-a716-446655440001" - } - - Example Generic (SSH Deploy Key): - repository = { - remote_url = "git@github.com:myorg/myrepo.git" - } - EOT +variable "projects" { + description = "List of project configurations. Each project may have a 'repository' sub-object with git configuration." type = any - - validation { - condition = can(var.repository_data.remote_url) && var.repository_data.remote_url != "" - error_message = "repository_data must contain a non-empty 'remote_url' field." - } } -variable "project_id" { - description = "The ID of the dbt Cloud project this repository is associated with" +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID (from modules/project output)" + type = map(string) +} + +variable "dbt_pat" { + description = "Personal access token for GitHub App integration discovery. If set, github_app strategy is enabled even without an explicit installation ID." type = string + sensitive = true + default = null +} + +variable "enable_gitlab_deploy_token" { + description = "Preserve native GitLab deploy_token strategy. Defaults to false due to known API limitations. Set to true only when GitLab OAuth access is confirmed." + type = bool + default = false } diff --git a/modules/semantic_layer/main.tf b/modules/semantic_layer/main.tf new file mode 100644 index 0000000..6720150 --- /dev/null +++ b/modules/semantic_layer/main.tf @@ -0,0 +1,29 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + # Only create semantic layer config for projects that have a semantic_layer block + semantic_layer_map = { + for p in var.projects : + try(p.key, p.name) => p + if try(p.semantic_layer, null) != null + } +} + +resource "dbtcloud_semantic_layer_configuration" "semantic_layer" { + for_each = local.semantic_layer_map + + project_id = var.project_ids[each.key] + environment_id = lookup( + var.environment_ids, + "${each.key}_${each.value.semantic_layer.environment}", + null + ) +} diff --git a/modules/semantic_layer/outputs.tf b/modules/semantic_layer/outputs.tf new file mode 100644 index 0000000..5dfbacc --- /dev/null +++ b/modules/semantic_layer/outputs.tf @@ -0,0 +1,4 @@ +output "semantic_layer_ids" { + description = "Map of project key to semantic_layer_configuration resource ID" + value = { for k, sl in dbtcloud_semantic_layer_configuration.semantic_layer : k => sl.id } +} diff --git a/modules/semantic_layer/variables.tf b/modules/semantic_layer/variables.tf new file mode 100644 index 0000000..88b0718 --- /dev/null +++ b/modules/semantic_layer/variables.tf @@ -0,0 +1,15 @@ +variable "projects" { + description = "List of project configurations. Each project may have a 'semantic_layer' block with an 'environment' key." + type = any +} + +variable "project_ids" { + description = "Map of project key to dbt Cloud project ID" + type = map(string) +} + +variable "environment_ids" { + description = "Map of composite key (project_key_env_key) to dbt Cloud environment ID (from environments module)" + type = map(string) + default = {} +} diff --git a/modules/service_tokens/main.tf b/modules/service_tokens/main.tf new file mode 100644 index 0000000..c95ec81 --- /dev/null +++ b/modules/service_tokens/main.tf @@ -0,0 +1,62 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + tokens_map = { + for t in var.service_tokens_data : + try(t.key, t.name) => t + } + + protected_tokens_map = { + for k, t in local.tokens_map : + k => t + if try(t.protected, false) == true + } + + unprotected_tokens_map = { + for k, t in local.tokens_map : + k => t + if try(t.protected, false) != true + } +} + +resource "dbtcloud_service_token" "service_tokens" { + for_each = local.unprotected_tokens_map + + name = each.value.name + + dynamic "service_token_permissions" { + for_each = try(each.value.permissions, []) + content { + permission_set = service_token_permissions.value.permission_set + project_id = try(service_token_permissions.value.project_id, null) + all_projects = try(service_token_permissions.value.all_projects, service_token_permissions.value.project_id == null) + } + } +} + +resource "dbtcloud_service_token" "protected_service_tokens" { + for_each = local.protected_tokens_map + + name = each.value.name + + dynamic "service_token_permissions" { + for_each = try(each.value.permissions, []) + content { + permission_set = service_token_permissions.value.permission_set + project_id = try(service_token_permissions.value.project_id, null) + all_projects = try(service_token_permissions.value.all_projects, service_token_permissions.value.project_id == null) + } + } + + lifecycle { + prevent_destroy = true + } +} diff --git a/modules/service_tokens/outputs.tf b/modules/service_tokens/outputs.tf new file mode 100644 index 0000000..189ebe2 --- /dev/null +++ b/modules/service_tokens/outputs.tf @@ -0,0 +1,7 @@ +output "service_token_ids" { + description = "Map of service token key to dbt Cloud service token ID" + value = merge( + { for k, t in dbtcloud_service_token.service_tokens : k => t.id }, + { for k, t in dbtcloud_service_token.protected_service_tokens : k => t.id } + ) +} diff --git a/modules/service_tokens/variables.tf b/modules/service_tokens/variables.tf new file mode 100644 index 0000000..18a26ac --- /dev/null +++ b/modules/service_tokens/variables.tf @@ -0,0 +1,5 @@ +variable "service_tokens_data" { + description = "List of service token configurations from YAML service_tokens[]" + type = any + default = [] +} diff --git a/modules/user_groups/main.tf b/modules/user_groups/main.tf new file mode 100644 index 0000000..c2ef7e1 --- /dev/null +++ b/modules/user_groups/main.tf @@ -0,0 +1,27 @@ +terraform { + required_version = ">= 1.7" + required_providers { + dbtcloud = { + source = "dbt-labs/dbtcloud" + version = "~> 1.8" + } + } +} + +locals { + user_groups_map = { + for ug in var.user_groups_data : + tostring(ug.user_id) => ug + } +} + +resource "dbtcloud_user_groups" "user_groups" { + for_each = local.user_groups_map + + user_id = each.value.user_id + group_ids = [ + for gk in try(each.value.group_keys, []) : + tonumber(lookup(var.group_ids, gk, null)) + if lookup(var.group_ids, gk, null) != null + ] +} diff --git a/modules/user_groups/outputs.tf b/modules/user_groups/outputs.tf new file mode 100644 index 0000000..8fa9b08 --- /dev/null +++ b/modules/user_groups/outputs.tf @@ -0,0 +1,4 @@ +output "user_group_ids" { + description = "Map of user_id (string) to user_groups resource ID" + value = { for k, ug in dbtcloud_user_groups.user_groups : k => ug.id } +} diff --git a/modules/user_groups/variables.tf b/modules/user_groups/variables.tf new file mode 100644 index 0000000..c77be8b --- /dev/null +++ b/modules/user_groups/variables.tf @@ -0,0 +1,11 @@ +variable "user_groups_data" { + description = "List of user-to-group assignments from YAML user_groups[]. Each entry has user_id and group_keys list." + type = any + default = [] +} + +variable "group_ids" { + description = "Map of group key to dbt Cloud group ID (from groups module)" + type = map(string) + default = {} +} diff --git a/outputs.tf b/outputs.tf index b7c90a7..f1b90df 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,24 +1,92 @@ -output "project_id" { - description = "The dbt Cloud project ID" - value = module.project.project_id +############################################# +# Project Outputs +############################################# + +output "project_ids" { + description = "Map of project key to dbt Cloud project ID" + value = module.project.project_ids } -output "repository_id" { - description = "The dbt Cloud repository ID" - value = module.repository.repository_id +############################################# +# Repository Outputs +############################################# + +output "repository_ids" { + description = "Map of project key to repository ID" + value = module.repository.repository_ids } +############################################# +# Environment Outputs +############################################# + output "environment_ids" { - description = "Map of environment names to their dbt Cloud IDs" + description = "Map of composite key (project_key_env_key) to dbt Cloud environment ID" value = module.environments.environment_ids } +############################################# +# Credential Outputs +############################################# + output "credential_ids" { - description = "Map of credential names to their dbt Cloud IDs" + description = "Map of composite key (project_key_env_key) to credential ID" value = module.credentials.credential_ids } +############################################# +# Job Outputs +############################################# + output "job_ids" { - description = "Map of job names to their dbt Cloud IDs" + description = "Map of composite key (project_key_job_key) to dbt Cloud job ID" value = module.jobs.job_ids } + +############################################# +# Account-Level Outputs +############################################# + +output "connection_ids" { + description = "Map of global connection key to dbt Cloud connection ID" + value = length(try(local.yaml_content.global_connections, [])) > 0 ? module.global_connections[0].connection_ids : {} +} + +output "service_token_ids" { + description = "Map of service token key to dbt Cloud service token ID" + value = length(try(local.yaml_content.service_tokens, [])) > 0 ? module.service_tokens[0].service_token_ids : {} +} + +output "group_ids" { + description = "Map of group key to dbt Cloud group ID" + value = length(try(local.yaml_content.groups, [])) > 0 ? module.groups[0].group_ids : {} +} + +output "oauth_configuration_ids" { + description = "Map of OAuth configuration key to dbt Cloud OAuth configuration ID" + value = length(try(local.yaml_content.oauth_configurations, [])) > 0 ? module.oauth_configurations[0].oauth_configuration_ids : {} +} + +output "ip_rule_ids" { + description = "Map of IP rule key to dbt Cloud IP restriction rule ID" + value = length(try(local.yaml_content.ip_restrictions, [])) > 0 ? module.ip_restrictions[0].ip_rule_ids : {} +} + +############################################# +# Project-Scoped Outputs +############################################# + +output "extended_attribute_ids" { + description = "Map of composite key (project_key_ea_key) to extended_attributes resource ID" + value = length(flatten([for p in local.projects : try(p.extended_attributes, [])])) > 0 ? module.extended_attributes[0].extended_attribute_ids : {} +} + +output "profile_ids" { + description = "Map of composite key (project_key_profile_key) to dbt Cloud profile ID" + value = length(flatten([for p in local.projects : try(p.profiles, [])])) > 0 ? module.profiles[0].profile_ids : {} +} + +output "lineage_integration_ids" { + description = "Map of composite key (project_key_integration_key) to lineage integration ID" + value = length(flatten([for p in local.projects : try(p.lineage_integrations, [])])) > 0 ? module.lineage_integrations[0].lineage_integration_ids : {} +} diff --git a/providers.tf b/providers.tf index 7c75860..ba0bab7 100644 --- a/providers.tf +++ b/providers.tf @@ -1,9 +1,9 @@ terraform { - required_version = ">= 1.0" + required_version = ">= 1.14" required_providers { dbtcloud = { source = "dbt-labs/dbtcloud" - version = "~> 1.3" + version = "~> 1.8" } } } diff --git a/schemas/v1.json b/schemas/v1.json index 1251f0c..dd89cb5 100644 --- a/schemas/v1.json +++ b/schemas/v1.json @@ -1,383 +1,507 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/trouze/dbt-cloud-terraform-starter/refs/heads/main/schemas/project/v1.json", - "title": "dbt Cloud Terraform Configuration Schema v1", - "description": "JSON Schema for validating dbt Cloud YAML configuration files. Use this with your IDE's YAML validation (VS Code: yaml extension, JetBrains: built-in support)", + "$id": "https://raw.githubusercontent.com/trouze/terraform-dbtcloud-yaml/refs/heads/main/schemas/v1.json", + "title": "dbt Cloud Terraform YAML Configuration Schema v1", + "description": "JSON Schema for validating dbt Cloud YAML configuration files (v1). Supports both singular project: and list-based projects: shapes. Use with VS Code YAML extension or JetBrains built-in YAML validation.", "type": "object", "properties": { - "project": { + + "account_features": { "type": "object", - "description": "Root project configuration object", + "description": "Account-level feature flags", "properties": { - "name": { - "type": "string", - "description": "Project name (alphanumeric, underscores, hyphens only)", - "pattern": "^[a-zA-Z0-9_-]+$", - "minLength": 1, - "maxLength": 128 + "advanced_ci": { "type": "boolean", "description": "Enable Advanced CI (compare changes) feature" }, + "partial_parsing": { "type": "boolean", "description": "Enable partial parsing for faster runs" }, + "repo_caching": { "type": "boolean", "description": "Enable repository caching" } + }, + "additionalProperties": false + }, + + "global_connections": { + "type": "array", + "description": "Account-level global connections referenced by environments via connection_key", + "items": { + "type": "object", + "description": "Global connection configuration", + "properties": { + "name": { "type": "string", "description": "Display name for the connection", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key used to reference this connection in environments", "pattern": "^[a-z0-9_]+$" }, + "type": { + "type": "string", + "description": "Warehouse type", + "enum": ["databricks", "snowflake", "bigquery", "redshift", "postgres", "spark", "starburst_trino", "apache_spark", "athena", "fabric", "synapse"] + }, + "protected": { "type": "boolean", "description": "Protect from accidental deletion", "default": false }, + "host": { "type": "string", "description": "[Databricks] Workspace hostname (e.g. adb-1234.azuredatabricks.net)" }, + "http_path": { "type": "string", "description": "[Databricks] HTTP path for the warehouse or cluster" }, + "catalog": { "type": "string", "description": "[Databricks] Unity Catalog catalog name" }, + "account": { "type": "string", "description": "[Snowflake] Account identifier (e.g. xy12345.us-east-1)" }, + "database": { "type": "string", "description": "[Snowflake] Default database" }, + "warehouse": { "type": "string", "description": "[Snowflake] Virtual warehouse name" }, + "role": { "type": "string", "description": "[Snowflake] Default role" }, + "allow_sso": { "type": "boolean", "description": "[Snowflake] Allow SSO authentication" } + }, + "required": ["name", "key", "type"], + "additionalProperties": true + } + }, + + "service_tokens": { + "type": "array", + "description": "Account-level service tokens", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Service token display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for this service token", "pattern": "^[a-z0-9_]+$" }, + "permissions": { + "type": "array", + "description": "Permission sets granted to this token", + "items": { + "type": "object", + "properties": { + "permission_set": { + "type": "string", + "description": "Permission set name", + "enum": ["owner", "member", "developer", "analyst", "stakeholder", "readonly", "job_runner", "git_admin", "account_admin", "admin", "database_admin", "write_artifacts", "view_environment", "webhook_admin", "billing_admin", "semantic_layer_only"] + }, + "all_projects": { "type": "boolean", "description": "Apply to all projects", "default": false }, + "project_key": { "type": "string", "description": "Restrict to a specific project key" } + }, + "required": ["permission_set"], + "additionalProperties": false + } + } + }, + "required": ["name", "key", "permissions"], + "additionalProperties": false + } + }, + + "groups": { + "type": "array", + "description": "Account-level groups for RBAC", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Group display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for this group", "pattern": "^[a-z0-9_]+$" }, + "assign_by_default": { "type": "boolean", "description": "Automatically assign new users to this group", "default": false }, + "sso_mapping_groups": { "type": "array", "description": "SSO group names to map to this group", "items": { "type": "string" } } + }, + "required": ["name", "key"], + "additionalProperties": false + } + }, + + "user_groups": { + "type": "array", + "description": "Assign users to groups", + "items": { + "type": "object", + "properties": { + "user_id": { "type": "integer", "description": "dbt Cloud user ID", "minimum": 1 }, + "group_keys": { "type": "array", "description": "List of group keys to assign the user to", "items": { "type": "string" }, "minItems": 1 } + }, + "required": ["user_id", "group_keys"], + "additionalProperties": false + } + }, + + "notifications": { + "type": "array", + "description": "Account-level job notifications", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Notification display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for this notification", "pattern": "^[a-z0-9_]+$" }, + "notification_type": { + "type": "integer", + "description": "Destination type: 1=email, 2=slack, 4=pagerduty, 5=webhook", + "enum": [1, 2, 4, 5] + }, + "user_id": { "type": ["integer", "null"], "description": "dbt Cloud user ID to notify (email notifications)", "minimum": 1 }, + "slack_channel_id": { "type": ["string", "null"], "description": "[Slack] Slack channel ID" }, + "slack_channel_name": { "type": ["string", "null"], "description": "[Slack] Slack channel display name" }, + "webhook_id": { "type": ["string", "null"], "description": "[Webhook] Webhook ID" }, + "on_failure": { "type": "array", "description": "Job IDs to notify on failure", "items": { "type": "integer", "minimum": 1 }, "default": [] }, + "on_success": { "type": "array", "description": "Job IDs to notify on success", "items": { "type": "integer", "minimum": 1 }, "default": [] }, + "on_cancel": { "type": "array", "description": "Job IDs to notify on cancel", "items": { "type": "integer", "minimum": 1 }, "default": [] }, + "on_warning": { "type": "array", "description": "Job IDs to notify on warning", "items": { "type": "integer", "minimum": 1 }, "default": [] } + }, + "required": ["name", "key", "notification_type"], + "additionalProperties": false + } + }, + + "oauth_configurations": { + "type": "array", + "description": "Account-level OAuth configurations. Client secrets are supplied via var.oauth_client_secrets[key].", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "OAuth config display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key (also used as lookup key in oauth_client_secrets)", "pattern": "^[a-z0-9_]+$" }, + "type": { "type": "string", "description": "OAuth provider type", "enum": ["snowflake"] }, + "authorize_url": { "type": "string", "description": "OAuth authorization endpoint URL" }, + "token_url": { "type": "string", "description": "OAuth token exchange endpoint URL" }, + "redirect_uri": { "type": "string", "description": "OAuth redirect URI registered with the provider" }, + "client_id": { "type": "string", "description": "OAuth client ID" } + }, + "required": ["name", "key", "type", "authorize_url", "token_url", "redirect_uri", "client_id"], + "additionalProperties": false + } + }, + + "ip_restrictions": { + "type": "array", + "description": "Account-level IP restriction rules", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Rule display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for this rule", "pattern": "^[a-z0-9_]+$" }, + "type": { "type": "string", "description": "Rule type: allow or deny", "enum": ["allow", "deny"] }, + "cidrs": { + "type": "array", + "description": "CIDR blocks for this rule", + "items": { + "type": "object", + "properties": { + "cidr": { "type": "string", "description": "CIDR block (e.g. 203.0.113.0/24)", "pattern": "^([0-9]{1,3}\\.){3}[0-9]{1,3}/[0-9]{1,2}$" } + }, + "required": ["cidr"], + "additionalProperties": false + }, + "minItems": 1 + }, + "rule_set_enabled": { "type": "boolean", "description": "Whether the IP restriction rule set is active", "default": false } }, + "required": ["name", "key", "type", "cidrs"], + "additionalProperties": false + } + }, + + "projects": { + "type": "array", + "description": "List of dbt Cloud projects. Use this (plural) for multi-project configs. See also: project (singular).", + "minItems": 1, + "items": { "$ref": "#/$defs/project" } + }, + + "project": { + "$ref": "#/$defs/project", + "description": "Single dbt Cloud project. Legacy singular form — prefer projects: (list)." + } + }, + + "anyOf": [ + { "required": ["projects"] }, + { "required": ["project"] } + ], + + "additionalProperties": false, + + "$defs": { + + "project": { + "type": "object", + "description": "dbt Cloud project configuration", + "properties": { + "name": { "type": "string", "description": "Project display name", "minLength": 1, "maxLength": 128 }, + "key": { "type": "string", "description": "Unique key for cross-referencing this project", "pattern": "^[a-z0-9_]+$" }, + "protected": { "type": "boolean", "description": "Protect from accidental deletion via lifecycle ignore", "default": false }, + "repository": { "type": "object", - "description": "Git repository configuration with support for GitHub, GitLab, Azure DevOps, and Bitbucket. Provider-specific fields are auto-detected and validated.", + "description": "Git repository configuration", "properties": { "remote_url": { "type": "string", - "description": "Git repository URL (HTTPS or SSH format). Supports GitHub, GitLab, Azure DevOps, and Bitbucket.", - "oneOf": [ - { - "pattern": "^https://github\\.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+(\\.git)?$", - "title": "GitHub HTTPS" - }, - { - "pattern": "^git@github\\.com:[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+(\\.git)?$", - "title": "GitHub SSH" - }, - { - "pattern": "^https://gitlab\\.com/[a-zA-Z0-9_/-]+(\\.git)?$", - "title": "GitLab HTTPS" - }, - { - "pattern": "^git@gitlab\\.com:[a-zA-Z0-9_/-]+(\\.git)?$", - "title": "GitLab SSH" - }, - { - "pattern": "^https://dev\\.azure\\.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/_git/[a-zA-Z0-9_.-]+$", - "title": "Azure DevOps HTTPS" - }, - { - "pattern": "^git@ssh\\.dev\\.azure\\.com:v3/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+$", - "title": "Azure DevOps SSH" - }, - { - "pattern": "^https://bitbucket\\.org/[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+(\\.git)?$", - "title": "Bitbucket HTTPS" - }, - { - "pattern": "^git@bitbucket\\.org:[a-zA-Z0-9_-]+/[a-zA-Z0-9_.-]+(\\.git)?$", - "title": "Bitbucket SSH" - } - ] + "description": "Repository path or URL. For GitHub/GitLab/Azure DevOps native integrations this is typically 'org/repo'. For deploy-key connections use HTTPS or SSH URL.", + "minLength": 1 }, "git_clone_strategy": { "type": ["string", "null"], - "description": "Git clone strategy. If omitted, auto-detected based on provider: 'deploy_key' (default), 'github_app' (GitHub native), 'deploy_token' (GitLab native), 'azure_active_directory_app' (Azure DevOps native)", - "enum": ["deploy_key", "github_app", "deploy_token", "azure_active_directory_app"], + "description": "Clone strategy. Auto-detected from integration IDs if omitted. Options: deploy_key, github_app, deploy_token, azure_active_directory_app", + "enum": ["deploy_key", "github_app", "deploy_token", "azure_active_directory_app", null], "default": null }, - "is_active": { - "type": ["boolean", "null"], - "description": "Whether the repository is active and available for use", - "default": true - }, "github_installation_id": { "type": ["integer", "null"], - "description": "[GitHub native integration only] GitHub App installation ID. Required when git_clone_strategy is 'github_app'. Ignored for non-GitHub repositories.", + "description": "[GitHub native] GitHub App installation ID", "minimum": 1, "default": null }, "gitlab_project_id": { "type": ["integer", "null"], - "description": "[GitLab native integration only] GitLab project ID (numeric). Required when git_clone_strategy is 'deploy_token'. Find in GitLab project settings > General. Ignored for non-GitLab repositories.", + "description": "[GitLab native] GitLab project ID (numeric, found in project Settings > General)", "minimum": 1, "default": null }, + "pull_request_url_template": { + "type": ["string", "null"], + "description": "Custom URL template for opening pull requests. Use {{source}} and {{destination}} placeholders.", + "default": null + }, "azure_active_directory_project_id": { "type": ["string", "null"], - "description": "[Azure DevOps native integration only] Azure DevOps project ID (UUID format). Required when git_clone_strategy is 'azure_active_directory_app'. Ignored for non-Azure DevOps repositories.", - "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$|^null$", + "description": "[Azure DevOps native] Azure DevOps project ID (UUID)", "default": null }, "azure_active_directory_repository_id": { "type": ["string", "null"], - "description": "[Azure DevOps native integration only] Azure DevOps repository ID (UUID format). Required when git_clone_strategy is 'azure_active_directory_app'. Ignored for non-Azure DevOps repositories.", - "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$|^null$", + "description": "[Azure DevOps native] Azure DevOps repository ID (UUID)", "default": null }, "azure_bypass_webhook_registration_failure": { "type": ["boolean", "null"], - "description": "[Azure DevOps native integration only] If false (default), connection fails if service user can't set webhooks. If true, connection succeeds but auto-triggering won't work. Ignored for non-Azure DevOps repositories.", + "description": "[Azure DevOps native] If true, connection succeeds even if webhook registration fails", "default": false }, "private_link_endpoint_id": { "type": ["string", "null"], - "description": "Private Link endpoint ID for VPC access. Optional for all providers.", + "description": "Private Link endpoint ID for VPC access", "default": null }, - "pull_request_url_template": { - "type": ["string", "null"], - "description": "Custom URL template for creating pull requests. If not set, uses provider-specific default. Optional for all providers.", - "default": null + "is_active": { + "type": ["boolean", "null"], + "description": "Whether the repository connection is active", + "default": true } }, "required": ["remote_url"], "additionalProperties": false }, + + "extended_attributes": { + "type": "array", + "description": "Extended attribute sets that can be applied to environments", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for referencing in environments via extended_attributes_key", "pattern": "^[a-z0-9_]+$" }, + "content": { "type": "object", "description": "Free-form YAML object with adapter-specific overrides (e.g. databricks.http_path)", "additionalProperties": true } + }, + "required": ["name", "key", "content"], + "additionalProperties": false + } + }, + "environments": { "type": "array", - "description": "List of dbt Cloud environments", + "description": "dbt Cloud environments for this project", "minItems": 1, "items": { "type": "object", "description": "Environment configuration", "properties": { - "name": { - "type": "string", - "description": "Environment name", - "minLength": 1, - "maxLength": 128 - }, - "type": { - "type": "string", - "description": "Environment type", - "enum": ["development", "deployment"], - "default": "development" - }, - "connection_id": { - "type": "integer", - "description": "dbt Cloud connection ID (find in dbt Cloud UI: Admin > Connections)", - "minimum": 1 + "name": { "type": "string", "description": "Environment display name", "minLength": 1, "maxLength": 128 }, + "key": { "type": "string", "description": "Unique key within this project for cross-referencing", "pattern": "^[a-z0-9_]+$" }, + "type": { "type": "string", "description": "Environment type", "enum": ["development", "deployment"], "default": "development" }, + "connection_key": { "type": "string", "description": "References global_connections[].key to attach a global connection" }, + "deployment_type": { + "type": ["string", "null"], + "description": "Deployment environment classification (required for type: deployment)", + "enum": ["production", "staging", "other", null], + "default": null }, + "custom_branch": { "type": ["string", "null"], "description": "Custom git branch. Null uses the project default branch.", "default": null }, + "dbt_version": { "type": ["string", "null"], "description": "dbt version override for this environment (e.g. '1.8.0')", "default": null }, + "extended_attributes_key": { "type": ["string", "null"], "description": "References extended_attributes[].key to apply adapter overrides", "default": null }, + "protected": { "type": "boolean", "description": "Protect from accidental deletion", "default": false }, + "enable_model_query_history": { "type": ["boolean", "null"], "description": "Enable model query history", "default": null }, "credential": { "type": "object", - "description": "Credential configuration", + "description": "Warehouse credential for this environment", "properties": { - "token_name": { + "credential_type": { "type": "string", - "description": "Key in token_map variable that contains the warehouse token", - "minLength": 1 + "description": "Warehouse adapter type", + "enum": ["databricks", "snowflake", "bigquery", "redshift", "postgres"] }, - "schema": { - "type": "string", - "description": "Default warehouse schema name", - "minLength": 1 - } + "schema": { "type": "string", "description": "Default target schema", "minLength": 1 }, + "catalog": { "type": "string", "description": "[Databricks] Unity Catalog catalog name" }, + "token_name": { "type": "string", "description": "[Databricks] Key in token_map variable containing the personal access token" }, + "auth_type": { "type": "string", "description": "[Snowflake] Authentication type", "enum": ["password", "keypair"] }, + "user": { "type": "string", "description": "[Snowflake] Service user name" }, + "database": { "type": "string", "description": "[Snowflake] Default database" }, + "warehouse": { "type": "string", "description": "[Snowflake] Virtual warehouse name" }, + "role": { "type": "string", "description": "[Snowflake] Default role" }, + "num_threads": { "type": "integer", "description": "Number of parallel threads", "minimum": 1, "maximum": 64 } }, - "required": ["token_name", "schema"], + "required": ["credential_type"], "additionalProperties": false + } + }, + "required": ["name", "key", "type"], + "additionalProperties": false + } + }, + + "jobs": { + "type": "array", + "description": "Jobs for this project, referencing environments by environment_key", + "items": { + "type": "object", + "description": "Job configuration", + "properties": { + "name": { "type": "string", "description": "Job display name", "minLength": 1, "maxLength": 256 }, + "key": { "type": "string", "description": "Unique key within this project", "pattern": "^[a-z0-9_]+$" }, + "environment_key": { "type": "string", "description": "References environments[].key to place this job" }, + "execute_steps": { + "type": "array", + "description": "dbt commands to execute (e.g. 'dbt build', 'dbt test')", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } }, - "dbt_version": { - "type": ["string", "null"], - "description": "dbt version (e.g., '1.5.0', '1.6.0'). Null uses default.", - "pattern": "^(\\d+\\.\\d+\\.\\d+)?$" + "triggers": { + "type": "object", + "description": "Job trigger configuration", + "properties": { + "schedule": { "type": "boolean", "description": "Enable scheduled runs", "default": false }, + "github_webhook": { "type": "boolean", "description": "Trigger on GitHub webhook (requires GitHub App integration)", "default": false }, + "git_provider_webhook": { "type": "boolean", "description": "Trigger on git provider webhook (GitLab, Azure DevOps, etc.)", "default": false }, + "on_merge": { "type": "boolean", "description": "Trigger when a PR is merged", "default": false } + }, + "required": ["schedule", "github_webhook", "git_provider_webhook", "on_merge"], + "additionalProperties": false }, - "custom_branch": { + "description": { "type": ["string", "null"], "description": "Job description", "maxLength": 512, "default": null }, + "num_threads": { "type": ["integer", "null"], "description": "Parallel thread count", "minimum": 1, "maximum": 64, "default": null }, + "timeout_seconds": { "type": ["integer", "null"], "description": "Job timeout in seconds", "minimum": 0, "default": null }, + "target_name": { "type": ["string", "null"], "description": "dbt target name override", "default": null }, + "dbt_version": { "type": ["string", "null"], "description": "Job-level dbt version override", "default": null }, + "schedule_type": { "type": ["string", "null"], - "description": "Custom git branch for this environment", + "description": "Schedule frequency type", + "enum": ["days_of_week", "every_day", "every_hour", "custom_cron", null], "default": null }, - "enable_model_query_history": { - "type": ["boolean", "null"], - "description": "Enable query history in dbt Cloud", + "schedule_days": { + "type": ["array", "null"], + "description": "Days of week to run (0=Sunday … 6=Saturday). Used with days_of_week schedule_type.", + "items": { "type": "integer", "minimum": 0, "maximum": 6 }, "default": null }, - "jobs": { - "type": "array", - "description": "List of jobs in this environment", - "items": { - "type": "object", - "description": "Job configuration", - "properties": { - "name": { - "type": "string", - "description": "Job name (must be unique within environment)", - "minLength": 1, - "maxLength": 256 - }, - "description": { - "type": ["string", "null"], - "description": "Job description", - "maxLength": 512, - "default": null - }, - "is_active": { - "type": ["boolean", "null"], - "description": "Whether job is active", - "default": true - }, - "execute_steps": { - "type": "array", - "description": "dbt commands to execute (e.g., 'dbt run', 'dbt test')", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "triggers": { - "type": "object", - "description": "Job trigger configuration (at least one must be true)", - "properties": { - "schedule": { - "type": "boolean", - "description": "Enable scheduled runs", - "default": false - }, - "github_webhook": { - "type": "boolean", - "description": "Trigger on GitHub push (requires GitHub integration)", - "default": false - }, - "git_provider_webhook": { - "type": "boolean", - "description": "Trigger on git provider webhook", - "default": false - }, - "on_merge": { - "type": "boolean", - "description": "Trigger on merge to main branch", - "default": false - } - }, - "required": ["schedule", "github_webhook", "git_provider_webhook", "on_merge"], - "additionalProperties": false - }, - "schedule_type": { - "type": ["string", "null"], - "description": "Schedule type: 'every_day', 'every_week', 'every_month', or null for custom cron", - "enum": ["every_day", "every_week", "every_month", null], - "default": null - }, - "schedule_hours": { - "type": ["array", "null"], - "description": "Hours of day to run (0-23, UTC). Use with schedule_type", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 23 - }, - "default": null - }, - "schedule_days": { - "type": ["array", "null"], - "description": "Days of week to run (0=Sunday, 6=Saturday). Use with every_week", - "items": { - "type": "integer", - "minimum": 0, - "maximum": 6 - }, - "default": null - }, - "schedule_cron": { - "type": ["string", "null"], - "description": "Cron expression for advanced scheduling (5-field format, UTC)", - "pattern": "^(([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9]|\\*|\\*/[0-9]+|[0-9]+-[0-9]+)(,([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9]|\\*|\\*/[0-9]+|[0-9]+-[0-9]+))*\\s+){4}([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9]|\\*|\\*/[0-9]+|[0-9]+-[0-9]+)(,([0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9]|5[0-9]|\\*|\\*/[0-9]+|[0-9]+-[0-9]+))*$", - "default": null - }, - "num_threads": { - "type": ["integer", "null"], - "description": "Number of parallel threads (1-16)", - "minimum": 1, - "maximum": 16, - "default": null - }, - "timeout_seconds": { - "type": ["integer", "null"], - "description": "Job timeout in seconds (300-86400)", - "minimum": 300, - "maximum": 86400, - "default": null - }, - "target_name": { - "type": ["string", "null"], - "description": "dbt target name (defaults to environment name)", - "default": null - }, - "dbt_version": { - "type": ["string", "null"], - "description": "Job-specific dbt version (overrides environment version)", - "pattern": "^(\\d+\\.\\d+\\.\\d+)?$", - "default": null - }, - "generate_docs": { - "type": ["boolean", "null"], - "description": "Generate dbt docs after run", - "default": null - }, - "run_lint": { - "type": ["boolean", "null"], - "description": "Run linters during job execution", - "default": null - }, - "run_generate_sources": { - "type": ["boolean", "null"], - "description": "Run dbt source freshness checks", - "default": null - }, - "run_compare_changes": { - "type": ["boolean", "null"], - "description": "Compare changes across runs", - "default": null - }, - "errors_on_lint_failure": { - "type": ["boolean", "null"], - "description": "Fail job if lint errors detected", - "default": null - }, - "triggers_on_draft_pr": { - "type": ["boolean", "null"], - "description": "Allow triggers on draft pull requests", - "default": null - }, - "self_deferring": { - "type": ["boolean", "null"], - "description": "Self-defer to latest run of this job", - "default": null - }, - "deferring_job_id": { - "type": ["integer", "null"], - "description": "Job ID to defer to", - "minimum": 1, - "default": null - }, - "deferring_environment_id": { - "type": ["integer", "null"], - "description": "Environment ID to defer to", - "minimum": 1, - "default": null - } - }, - "required": ["name", "execute_steps", "triggers"], - "additionalProperties": false - } - } + "schedule_hours": { + "type": ["array", "null"], + "description": "Hours of day to run (0–23, UTC). Used with days_of_week and every_day schedule types.", + "items": { "type": "integer", "minimum": 0, "maximum": 23 }, + "default": null + }, + "schedule_cron": { + "type": ["string", "null"], + "description": "Custom cron expression (5-field, UTC). Used with custom_cron schedule_type.", + "default": null + }, + "deferring_environment_key": { "type": ["string", "null"], "description": "References environments[].key to defer to that environment's most recent successful run", "default": null }, + "deferring_job_key": { "type": ["string", "null"], "description": "References another job[].key to defer to a specific job", "default": null }, + "self_deferring": { "type": ["boolean", "null"], "description": "Defer to this job's own latest successful run", "default": null }, + "generate_docs": { "type": ["boolean", "null"], "description": "Generate dbt docs after run", "default": null }, + "run_lint": { "type": ["boolean", "null"], "description": "Run SQLFluff linting", "default": null }, + "run_generate_sources": { "type": ["boolean", "null"], "description": "Run dbt source freshness", "default": null }, + "run_compare_changes": { "type": ["boolean", "null"], "description": "Enable Advanced CI change comparison", "default": null }, + "errors_on_lint_failure": { "type": ["boolean", "null"], "description": "Fail job if linting errors are found", "default": null }, + "triggers_on_draft_pr": { "type": ["boolean", "null"], "description": "Allow webhook triggers on draft PRs", "default": null }, + "is_active": { "type": ["boolean", "null"], "description": "Whether this job is active", "default": true } }, - "required": ["name", "type", "connection_id", "credential"], + "required": ["name", "key", "environment_key", "execute_steps", "triggers"], "additionalProperties": false } }, + "environment_variables": { "type": "array", - "description": "Project-level environment variables (accessible to all jobs)", + "description": "Project-level dbt environment variables", "items": { "type": "object", - "description": "Environment variable definition", "properties": { "name": { "type": "string", - "description": "Variable name (use UPPER_SNAKE_CASE)", - "pattern": "^[A-Z_][A-Z0-9_]*$", - "minLength": 1, - "maxLength": 256 + "description": "Variable name. Must start with DBT_.", + "pattern": "^DBT_[A-Z0-9_]+$", + "minLength": 1 }, "environment_values": { - "type": "object", - "description": "Values by environment name (e.g., 'Development', 'Production')", - "minProperties": 1, - "additionalProperties": { - "type": "string", - "maxLength": 4096 + "type": "array", + "description": "Per-environment values. Use env: 'project' for the project-level default.", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "env": { "type": "string", "description": "Environment name or 'project' for the project-level default", "minLength": 1 }, + "value": { "type": "string", "description": "Variable value. Prefix with 'secret_' to reference a token_map key.", "minLength": 0 } + }, + "required": ["env", "value"], + "additionalProperties": false } } }, "required": ["name", "environment_values"], "additionalProperties": false } + }, + + "profiles": { + "type": "array", + "description": "dbt Cloud IDE connection profiles", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Profile display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key for this profile", "pattern": "^[a-z0-9_]+$" }, + "connection_key": { "type": "string", "description": "References global_connections[].key" }, + "credential_key": { "type": "string", "description": "Composite key (project_key_env_key) or environment key referencing the credential to use" }, + "extended_attributes_key": { "type": ["string", "null"], "description": "References extended_attributes[].key for adapter overrides", "default": null } + }, + "required": ["name", "key", "connection_key", "credential_key"], + "additionalProperties": false + } + }, + + "lineage_integrations": { + "type": "array", + "description": "External lineage integrations (e.g. Tableau)", + "items": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "Integration display name", "minLength": 1 }, + "key": { "type": "string", "description": "Unique key. Auth token supplied via var.lineage_tokens[project_key_integration_key].", "pattern": "^[a-z0-9_]+$" }, + "host": { "type": "string", "description": "Integration host URL (e.g. https://tableau.example.com)" }, + "site_id": { "type": ["string", "null"], "description": "[Tableau] Tableau site ID", "default": null }, + "token_name": { "type": ["string", "null"], "description": "[Tableau] Tableau personal access token name", "default": null } + }, + "required": ["name", "key", "host"], + "additionalProperties": false + } + }, + + "artefacts": { + "type": "object", + "description": "Project artefact job assignments for docs and source freshness", + "properties": { + "docs_job": { "type": "string", "description": "References jobs[].key for the job that generates dbt docs" }, + "freshness_job": { "type": "string", "description": "References jobs[].key for the job that runs source freshness" } + }, + "additionalProperties": false + }, + + "semantic_layer": { + "type": "object", + "description": "Semantic Layer configuration", + "properties": { + "environment_key": { "type": "string", "description": "References environments[].key for the Semantic Layer environment" } + }, + "required": ["environment_key"], + "additionalProperties": false } }, - "required": ["name", "repository", "environments"], + + "required": ["name", "key", "repository", "environments"], "additionalProperties": false } - }, - "required": ["project"], - "additionalProperties": false + } } diff --git a/test/fixtures/basic/dbt-config.yml b/test/fixtures/basic/dbt-config.yml deleted file mode 100644 index 0cd67d0..0000000 --- a/test/fixtures/basic/dbt-config.yml +++ /dev/null @@ -1,25 +0,0 @@ -# Minimal dbt Cloud configuration for testing -project: - name: test_project - repository: - remote_url: https://github.com/test-org/test-repo.git - - environments: - - name: Development - type: development - connection_id: 999999 - credential: - token_name: dev_token - schema: dev - dbt_version: "1.5.0" - jobs: - - name: test_run - description: "Test job" - is_active: true - execute_steps: - - "dbt run" - triggers: - schedule: false - github_webhook: false - git_provider_webhook: false - on_merge: false diff --git a/test/fixtures/basic/main.tf b/test/fixtures/basic/main.tf deleted file mode 100644 index feb01c0..0000000 --- a/test/fixtures/basic/main.tf +++ /dev/null @@ -1,45 +0,0 @@ -terraform { - required_providers { - dbtcloud = { - source = "dbt-labs/dbtcloud" - version = "~> 1.3" - } - } -} - -provider "dbtcloud" { - account_id = var.dbt_account_id - token = var.dbt_token - host_url = var.dbt_host_url -} - -module "dbt_cloud" { - source = "../../" - - dbt_account_id = var.dbt_account_id - dbt_token = var.dbt_token - dbt_host_url = var.dbt_host_url - yaml_file = "${path.module}/dbt-config.yml" - target_name = var.target_name - token_map = var.token_map -} - -output "project_id" { - value = try(module.dbt_cloud.project_id, null) -} - -output "repository_id" { - value = try(module.dbt_cloud.repository_id, null) -} - -output "environment_ids" { - value = try(module.dbt_cloud.environment_ids, {}) -} - -output "credential_ids" { - value = try(module.dbt_cloud.credential_ids, {}) -} - -output "job_ids" { - value = try(module.dbt_cloud.job_ids, {}) -} diff --git a/test/fixtures/basic/variables.tf b/test/fixtures/basic/variables.tf deleted file mode 100644 index 38fec35..0000000 --- a/test/fixtures/basic/variables.tf +++ /dev/null @@ -1,33 +0,0 @@ -variable "dbt_account_id" { - type = number - description = "dbt Cloud Account ID" -} - -variable "dbt_token" { - type = string - description = "dbt Cloud API Token" - default = "test-token-not-real" - sensitive = true -} - -variable "dbt_host_url" { - type = string - description = "dbt Cloud URL" - default = "https://cloud.getdbt.com" -} - -variable "target_name" { - type = string - description = "dbt target name" - default = "dev" -} - -variable "token_map" { - type = map(string) - description = "Map of token names to warehouse tokens" - default = { - "dev_token" = "test-token-dev" - "prod_token" = "test-token-prod" - } - sensitive = true -} diff --git a/test/fixtures/complete/dbt-config.yml b/test/fixtures/complete/dbt-config.yml deleted file mode 100644 index 3162f5c..0000000 --- a/test/fixtures/complete/dbt-config.yml +++ /dev/null @@ -1,115 +0,0 @@ -# Comprehensive dbt Cloud configuration with advanced features -project: - name: advanced_project - repository: - remote_url: https://github.com/test-org/test-repo.git - - environments: - - name: Development - type: development - connection_id: 999999 - credential: - token_name: dev_token - schema: dev_schema - dbt_version: "1.5.0" - custom_branch: develop - enable_model_query_history: true - jobs: - - name: dev_run - description: "Development dbt run with tests" - is_active: true - execute_steps: - - "dbt run" - - "dbt test" - - "dbt docs generate" - triggers: - schedule: false - github_webhook: true - git_provider_webhook: false - on_merge: false - generate_docs: true - run_lint: true - num_threads: 4 - - - name: Staging - type: deployment - connection_id: 999998 - credential: - token_name: staging_token - schema: staging_schema - dbt_version: "1.5.0" - jobs: - - name: staging_build - description: "Staging daily build" - is_active: true - execute_steps: - - "dbt run" - - "dbt test" - triggers: - schedule: true - github_webhook: false - git_provider_webhook: false - on_merge: false - schedule_type: "every_day" - schedule_hours: [2, 14] - num_threads: 8 - timeout_seconds: 3600 - generate_docs: true - run_generate_sources: true - - - name: Production - type: deployment - connection_id: 999997 - credential: - token_name: prod_token - schema: prod_schema - dbt_version: "1.5.0" - jobs: - - name: prod_daily - description: "Daily production run" - is_active: true - execute_steps: - - "dbt run" - - "dbt test" - - "dbt docs generate" - triggers: - schedule: true - github_webhook: false - git_provider_webhook: false - on_merge: false - schedule_type: "every_day" - schedule_hours: [6] - num_threads: 16 - timeout_seconds: 7200 - generate_docs: true - run_compare_changes: true - errors_on_lint_failure: true - - - name: prod_weekly - description: "Weekly comprehensive analysis" - is_active: true - execute_steps: - - "dbt run --select model_tag:weekly" - - "dbt test" - triggers: - schedule: true - github_webhook: false - git_provider_webhook: false - on_merge: false - schedule_type: "every_week" - schedule_days: [0] # Sunday - schedule_hours: [1] - num_threads: 12 - - environment_variables: - - name: DBT_ENV - environment_values: - Development: dev - Staging: staging - Production: prod - - - name: LOG_LEVEL - environment_values: - Development: debug - Staging: info - Production: warn diff --git a/test/fixtures/complete/main.tf b/test/fixtures/complete/main.tf deleted file mode 100644 index 827cda5..0000000 --- a/test/fixtures/complete/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -terraform { - required_providers { - dbtcloud = { - source = "dbt-labs/dbtcloud" - version = "~> 1.3" - } - } -} - -provider "dbtcloud" { - account_id = var.dbt_account_id - token = var.dbt_token - host_url = var.dbt_host_url -} - -module "dbt_cloud" { - source = "../../" - - dbt_account_id = var.dbt_account_id - dbt_token = var.dbt_token - dbt_host_url = var.dbt_host_url - yaml_file = "${path.module}/dbt-config.yml" - target_name = var.target_name - token_map = var.token_map -} diff --git a/test/fixtures/complete/variables.tf b/test/fixtures/complete/variables.tf deleted file mode 100644 index 83e6a6b..0000000 --- a/test/fixtures/complete/variables.tf +++ /dev/null @@ -1,34 +0,0 @@ -variable "dbt_account_id" { - type = number - description = "dbt Cloud Account ID" -} - -variable "dbt_token" { - type = string - description = "dbt Cloud API Token" - default = "test-token-not-real" - sensitive = true -} - -variable "dbt_host_url" { - type = string - description = "dbt Cloud URL" - default = "https://cloud.getdbt.com" -} - -variable "target_name" { - type = string - description = "dbt target name" - default = "dev" -} - -variable "token_map" { - type = map(string) - description = "Map of token names to warehouse tokens" - default = { - "dev_token" = "test-token-dev" - "staging_token" = "test-token-staging" - "prod_token" = "test-token-prod" - } - sensitive = true -} diff --git a/test/go.mod b/test/go.mod index aa532e1..fdb97b6 100644 --- a/test/go.mod +++ b/test/go.mod @@ -1,6 +1,8 @@ -module github.com/trouze/dbt-terraform-modules-yaml/test +module github.com/trouze/terraform-dbtcloud-yaml/test -go 1.20 +go 1.21 + +toolchain go1.24.2 require ( github.com/gruntwork-io/terratest v0.46.11 @@ -8,17 +10,52 @@ require ( ) require ( - github.com/aws/aws-sdk-go v1.49.11 // indirect - github.com/gruntwork-io/go-commons v0.15.0 // indirect - github.com/gruntwork-io/go-shell v0.3.2 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v0.13.0 // indirect + cloud.google.com/go/storage v1.28.1 // indirect + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/aws/aws-sdk-go v1.44.122 // indirect + github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-getter v1.7.1 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/hcl/v2 v2.9.1 // indirect + github.com/hashicorp/terraform-json v0.13.0 // indirect + github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jinzhu/copier v1.6.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/urfave/cli v1.22.15 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.19.0 // indirect + github.com/klauspost/compress v1.15.11 // indirect + github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/tmccombs/hcl2json v0.3.3 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect + github.com/zclconf/go-cty v1.9.1 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/api v0.114.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/grpc v1.56.3 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/test/go.sum b/test/go.sum new file mode 100644 index 0000000..fca5035 --- /dev/null +++ b/test/go.sum @@ -0,0 +1,973 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= +github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= +github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= +github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/gruntwork-io/terratest v0.46.11 h1:1Z9G18I2FNuH87Ro0YtjW4NH9ky4GDpfzE7+ivkPeB8= +github.com/gruntwork-io/terratest v0.46.11/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= +github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= +github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl/v2 v2.9.1 h1:eOy4gREY0/ZQHNItlfuEZqtcQbXIxzojlP301hDpnac= +github.com/hashicorp/hcl/v2 v2.9.1/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= +github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniyEi5CM+FWGY= +github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 h1:ofNAzWCcyTALn2Zv40+8XitdzCgXY6e9qvXwN9W0YXg= +github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= +github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.8.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc= +github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= +google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/test/integration_test.go b/test/integration_test.go new file mode 100644 index 0000000..28878b6 --- /dev/null +++ b/test/integration_test.go @@ -0,0 +1,150 @@ +package test + +import ( + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// skipIfNoIntegration skips the test unless RUN_INTEGRATION_TESTS=1 is set. +func skipIfNoIntegration(t *testing.T) { + t.Helper() + if os.Getenv("RUN_INTEGRATION_TESTS") == "" { + t.Skip("skipping integration test; set RUN_INTEGRATION_TESTS=1 to run") + } +} + +// repoRoot returns the absolute path to the repository root (one level up from test/). +func repoRoot(t *testing.T) string { + t.Helper() + abs, err := filepath.Abs("../") + require.NoError(t, err) + return abs +} + +// integrationVars builds the terraform variable map from environment variables. +// Required: DBT_CLOUD_ACCOUNT_ID, DBT_CLOUD_TOKEN +// Optional: DBT_CLOUD_HOST_URL (defaults to https://cloud.getdbt.com when unset) +func integrationVars(t *testing.T, yamlFixture string) map[string]interface{} { + t.Helper() + + accountIDStr := os.Getenv("DBT_CLOUD_ACCOUNT_ID") + require.NotEmpty(t, accountIDStr, "DBT_CLOUD_ACCOUNT_ID must be set") + accountID, err := strconv.Atoi(accountIDStr) + require.NoError(t, err, "DBT_CLOUD_ACCOUNT_ID must be a valid integer") + + token := os.Getenv("DBT_CLOUD_TOKEN") + require.NotEmpty(t, token, "DBT_CLOUD_TOKEN must be set") + + vars := map[string]interface{}{ + "dbt_account_id": accountID, + "dbt_token": token, + "yaml_file": yamlFixture, + } + + if hostURL := os.Getenv("DBT_CLOUD_HOST_URL"); hostURL != "" { + vars["dbt_host_url"] = hostURL + } + + return vars +} + +// assertPositiveID checks that a key exists in the output map and its value is a positive number. +// Terraform serializes large integers as floats (e.g. "1.025871e+06"), so we parse via ParseFloat. +func assertPositiveID(t *testing.T, outputMap map[string]string, key string) { + t.Helper() + assert.Contains(t, outputMap, key, "expected key %q in output", key) + val := outputMap[key] + f, err := strconv.ParseFloat(val, 64) + assert.NoError(t, err, "output %q value %q cannot be parsed as a number", key, val) + assert.Greater(t, int(f), 0, "output %q should be a positive ID", key) +} + +// TestIntegrationCoreWorkflow applies a single project with a development environment +// and a scheduled job, then verifies IDs are real and a second plan shows no changes. +func TestIntegrationCoreWorkflow(t *testing.T) { + skipIfNoIntegration(t) + + root := repoRoot(t) + opts := &terraform.Options{ + TerraformDir: root, + Vars: integrationVars(t, "tests/fixtures/integration-core.yml"), + NoColor: true, + } + + defer terraform.Destroy(t, opts) + terraform.InitAndApply(t, opts) + + projectIDs := terraform.OutputMap(t, opts, "project_ids") + assertPositiveID(t, projectIDs, "tf_int_core") + + envIDs := terraform.OutputMap(t, opts, "environment_ids") + assertPositiveID(t, envIDs, "tf_int_core_dev") + + jobIDs := terraform.OutputMap(t, opts, "job_ids") + assertPositiveID(t, jobIDs, "tf_int_core_daily_run") + + // Idempotency: second plan must produce no changes (exit code 0). + exitCode := terraform.PlanExitCode(t, opts) + assert.Equal(t, 0, exitCode, "expected no changes after apply (exit code 0), got %d", exitCode) +} + +// TestIntegrationMultiProject applies two projects with development environments, +// verifying that composite keys are correctly populated in real state. +func TestIntegrationMultiProject(t *testing.T) { + skipIfNoIntegration(t) + + root := repoRoot(t) + opts := &terraform.Options{ + TerraformDir: root, + Vars: integrationVars(t, "tests/fixtures/integration-multi-project.yml"), + NoColor: true, + } + + defer terraform.Destroy(t, opts) + terraform.InitAndApply(t, opts) + + projectIDs := terraform.OutputMap(t, opts, "project_ids") + assert.Len(t, projectIDs, 2, "expected exactly 2 projects") + assertPositiveID(t, projectIDs, "tf_int_alpha") + assertPositiveID(t, projectIDs, "tf_int_beta") + + envIDs := terraform.OutputMap(t, opts, "environment_ids") + assertPositiveID(t, envIDs, "tf_int_alpha_dev") + assertPositiveID(t, envIDs, "tf_int_beta_dev") + + jobIDs := terraform.OutputMap(t, opts, "job_ids") + assertPositiveID(t, jobIDs, "tf_int_alpha_nightly") + + exitCode := terraform.PlanExitCode(t, opts) + assert.Equal(t, 0, exitCode, "expected no changes after apply (exit code 0), got %d", exitCode) +} + +// TestIntegrationServiceToken applies a project and a service token, verifying +// that account-level resources are created and their IDs are populated. +func TestIntegrationServiceToken(t *testing.T) { + skipIfNoIntegration(t) + + root := repoRoot(t) + opts := &terraform.Options{ + TerraformDir: root, + Vars: integrationVars(t, "tests/fixtures/integration-service-tokens.yml"), + NoColor: true, + } + + defer terraform.Destroy(t, opts) + terraform.InitAndApply(t, opts) + + projectIDs := terraform.OutputMap(t, opts, "project_ids") + assertPositiveID(t, projectIDs, "tf_int_svc") + + // service_token_ids is sensitive — read via OutputJson. + serviceTokenJSON := terraform.OutputJson(t, opts, "service_token_ids") + assert.NotEmpty(t, serviceTokenJSON, "service_token_ids output should not be empty") + assert.Contains(t, serviceTokenJSON, "tf_int_svc_token", "expected key 'tf_int_svc_token' in service_token_ids") +} diff --git a/test/terraform_test.go b/test/terraform_test.go deleted file mode 100644 index 61739a9..0000000 --- a/test/terraform_test.go +++ /dev/null @@ -1,323 +0,0 @@ -package test - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestBasicConfiguration validates the root module works with minimal YAML -func TestBasicConfiguration(t *testing.T) { - t.Parallel() - - // Setup test fixtures - tmpDir := filepath.Join(os.TempDir(), "dbt-terraform-test-basic") - defer os.RemoveAll(tmpDir) - - // Copy test configuration - err := copyDir("fixtures/basic", tmpDir) - require.NoError(t, err, "Failed to copy fixture directory") - - // Configure Terraform - terraformOptions := &terraform.Options{ - TerraformDir: tmpDir, - VarFiles: []string{"terraform.tfvars"}, - Lock: true, - NoColor: true, - Logger: t, - // Set to -json for structured output - Vars: map[string]interface{}{ - "dbt_account_id": "999999", - "dbt_host_url": "https://cloud.getdbt.com", - }, - EnvVars: map[string]string{ - "TF_INPUT": "false", - }, - } - - // Initialize and plan (not apply - we don't have real creds) - defer terraform.Destroy(t, terraformOptions) - - // Init should succeed - terraform.Init(t, terraformOptions) - - // Plan should validate syntax without errors - planOutput := terraform.Plan(t, terraformOptions) - - // Verify plan output contains expected resources - assert.Contains(t, planOutput, "module.dbt_cloud") - assert.NotContains(t, planOutput, "Error") -} - -// TestCompleteConfiguration validates advanced YAML features -func TestCompleteConfiguration(t *testing.T) { - t.Parallel() - - tmpDir := filepath.Join(os.TempDir(), "dbt-terraform-test-complete") - defer os.RemoveAll(tmpDir) - - // Copy advanced test configuration - err := copyDir("fixtures/complete", tmpDir) - require.NoError(t, err, "Failed to copy fixture directory") - - terraformOptions := &terraform.Options{ - TerraformDir: tmpDir, - VarFiles: []string{"terraform.tfvars"}, - Lock: true, - NoColor: true, - Logger: t, - Vars: map[string]interface{}{ - "dbt_account_id": "999999", - "dbt_host_url": "https://cloud.getdbt.com", - }, - EnvVars: map[string]string{ - "TF_INPUT": "false", - }, - } - - defer terraform.Destroy(t, terraformOptions) - - terraform.Init(t, terraformOptions) - planOutput := terraform.Plan(t, terraformOptions) - - // Verify complex features are handled - assert.Contains(t, planOutput, "module.dbt_cloud") - assert.NotContains(t, planOutput, "Error") -} - -// TestYAMLParsing validates YAML file is correctly parsed into Terraform locals -func TestYAMLParsing(t *testing.T) { - t.Parallel() - - tmpDir := filepath.Join(os.TempDir(), "dbt-terraform-test-yaml") - defer os.RemoveAll(tmpDir) - - err := copyDir("fixtures/basic", tmpDir) - require.NoError(t, err) - - // Read the YAML file to verify it's valid - yamlPath := filepath.Join(tmpDir, "dbt-config.yml") - yamlContent, err := os.ReadFile(yamlPath) - require.NoError(t, err, "Failed to read YAML file") - - // Verify YAML contains expected structure - yamlStr := string(yamlContent) - assert.Contains(t, yamlStr, "project:") - assert.Contains(t, yamlStr, "name:") - assert.Contains(t, yamlStr, "environments:") -} - -// TestVariableValidation validates that input variables have proper validation -func TestVariableValidation(t *testing.T) { - t.Parallel() - - tmpDir := filepath.Join(os.TempDir(), "dbt-terraform-test-vars") - defer os.RemoveAll(tmpDir) - - err := copyDir("fixtures/basic", tmpDir) - require.NoError(t, err) - - // Test invalid account ID (non-numeric) - terraformOptions := &terraform.Options{ - TerraformDir: tmpDir, - Lock: true, - NoColor: true, - Logger: t, - Vars: map[string]interface{}{ - "dbt_account_id": "invalid", - "dbt_host_url": "https://cloud.getdbt.com", - }, - EnvVars: map[string]string{ - "TF_INPUT": "false", - }, - } - - terraform.Init(t, terraformOptions) - - // Plan should fail with validation error - err = terraform.PlanE(t, terraformOptions) - assert.Error(t, err, "Expected validation error for non-numeric account_id") -} - -// TestOutputs validates that module exports expected outputs -func TestOutputs(t *testing.T) { - t.Parallel() - - tmpDir := filepath.Join(os.TempDir(), "dbt-terraform-test-outputs") - defer os.RemoveAll(tmpDir) - - err := copyDir("fixtures/basic", tmpDir) - require.NoError(t, err) - - terraformOptions := &terraform.Options{ - TerraformDir: tmpDir, - VarFiles: []string{"terraform.tfvars"}, - Lock: true, - NoColor: true, - Logger: t, - Vars: map[string]interface{}{ - "dbt_account_id": "999999", - "dbt_host_url": "https://cloud.getdbt.com", - }, - EnvVars: map[string]string{ - "TF_INPUT": "false", - }, - } - - defer terraform.Destroy(t, terraformOptions) - - terraform.Init(t, terraformOptions) - - // Check that expected outputs are defined in module - outputs := terraform.OutputAll(t, terraformOptions) - expectedOutputs := []string{"project_id", "repository_id", "environment_ids", "credential_ids", "job_ids"} - - for _, output := range expectedOutputs { - assert.Contains(t, outputs, output, fmt.Sprintf("Expected output '%s' not found", output)) - } -} - -// TestPathModule validates that path.module is used for module sources -func TestPathModule(t *testing.T) { - mainTfPath := "main.tf" - content, err := os.ReadFile(mainTfPath) - require.NoError(t, err, "Failed to read main.tf") - - mainTfStr := string(content) - - // Verify all module sources use path.module instead of relative paths - assert.NotContains(t, mainTfStr, `source = "./modules/`, "Found relative module paths instead of path.module") - assert.Contains(t, mainTfStr, `source = "${path.module}/modules/`, "Module sources should use path.module") - - // Count occurrences - should have 8 module sources using path.module - count := 0 - startIdx := 0 - searchStr := `source = "${path.module}/modules/` - for { - idx := findString(mainTfStr, searchStr, startIdx) - if idx == -1 { - break - } - count++ - startIdx = idx + 1 - } - - assert.GreaterOrEqual(t, count, 7, "Expected at least 7 module sources using path.module") -} - -// TestModuleStructure validates that all required module files exist -func TestModuleStructure(t *testing.T) { - requiredModules := []string{ - "modules/project", - "modules/repository", - "modules/project_repository", - "modules/credentials", - "modules/environments", - "modules/jobs", - "modules/environment_variables", - "modules/environment_variable_job_overrides", - } - - for _, module := range requiredModules { - // Check main.tf exists - mainTfPath := filepath.Join(module, "main.tf") - _, err := os.Stat(mainTfPath) - assert.NoError(t, err, fmt.Sprintf("Module %s missing main.tf", module)) - - // Check variables.tf exists - varsTfPath := filepath.Join(module, "variables.tf") - _, err = os.Stat(varsTfPath) - assert.NoError(t, err, fmt.Sprintf("Module %s missing variables.tf", module)) - - // Check outputs.tf exists - outputsTfPath := filepath.Join(module, "outputs.tf") - _, err = os.Stat(outputsTfPath) - assert.NoError(t, err, fmt.Sprintf("Module %s missing outputs.tf", module)) - } -} - -// TestDocumentation validates that documentation files exist and contain key sections -func TestDocumentation(t *testing.T) { - docFiles := map[string][]string{ - "README.md": {"Quick Start", "Configuration", "YAML", "Troubleshooting"}, - "QUICKSTART.md": {"Prerequisites", "Step", "module"}, - "CONTRIBUTING.md": {"Contributing", "Development"}, - "CHANGELOG.md": {"Changelog", "1.0.0"}, - } - - for file, requiredSections := range docFiles { - content, err := os.ReadFile(file) - require.NoError(t, err, fmt.Sprintf("Failed to read %s", file)) - - fileStr := string(content) - for _, section := range requiredSections { - assert.Contains(t, fileStr, section, fmt.Sprintf("%s missing section: %s", file, section)) - } - } -} - -// Helper functions - -// copyDir recursively copies a directory -func copyDir(src, dst string) error { - entries, err := os.ReadDir(src) - if err != nil { - return err - } - - if err := os.MkdirAll(dst, 0755); err != nil { - return err - } - - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - if err := copyDir(srcPath, dstPath); err != nil { - return err - } - } else { - data, err := os.ReadFile(srcPath) - if err != nil { - return err - } - - if err := os.WriteFile(dstPath, data, 0644); err != nil { - return err - } - } - } - - return nil -} - -// findString finds a substring in a string, starting from startIdx -func findString(s, substr string, startIdx int) int { - if startIdx >= len(s) { - return -1 - } - idx := -1 - for i := startIdx; i < len(s); i++ { - if i+len(substr) <= len(s) && s[i:i+len(substr)] == substr { - idx = i - break - } - } - return idx -} - -// ParseOutputJSON parses terraform output JSON -func ParseOutputJSON(output string) (map[string]interface{}, error) { - var result map[string]interface{} - if err := json.Unmarshal([]byte(output), &result); err != nil { - return nil, err - } - return result, nil -} diff --git a/tests/fixtures/basic.yml b/tests/fixtures/basic.yml new file mode 100644 index 0000000..3d35519 --- /dev/null +++ b/tests/fixtures/basic.yml @@ -0,0 +1,23 @@ +project: + name: My Test Project + key: my_project + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + protected: true + jobs: + - name: Daily Run + key: daily_run + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_type: days_of_week + schedule_days: [0, 1, 2, 3, 4] + schedule_hours: [6] diff --git a/tests/fixtures/complete.yml b/tests/fixtures/complete.yml new file mode 100644 index 0000000..2720482 --- /dev/null +++ b/tests/fixtures/complete.yml @@ -0,0 +1,44 @@ +projects: + - name: Analytics + key: analytics + environments: + - name: Development + key: dev + type: development + - name: Production + key: prod + type: deployment + deployment_type: production + protected: true + jobs: + - name: CI Check + key: ci_check + environment_key: prod + execute_steps: + - dbt build --select state:modified+ + triggers: + schedule: false + github_webhook: true + git_provider_webhook: false + on_merge: false + - name: Daily Run + key: daily_run + environment_key: prod + execute_steps: + - dbt build + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_cron: "0 6 * * 1-5" + + - name: Finance + key: finance + protected: true + environments: + - name: Production + key: prod + type: deployment + deployment_type: production + protected: true diff --git a/tests/fixtures/integration-core.yml b/tests/fixtures/integration-core.yml new file mode 100644 index 0000000..3a5c5c1 --- /dev/null +++ b/tests/fixtures/integration-core.yml @@ -0,0 +1,21 @@ +projects: + - name: TF Integration Core + key: tf_int_core + environments: + - name: Dev + key: dev + type: development + jobs: + - name: Daily Run + key: daily_run + environment_key: dev + execute_steps: + - dbt compile + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] + schedule_hours: [6] diff --git a/tests/fixtures/integration-multi-project.yml b/tests/fixtures/integration-multi-project.yml new file mode 100644 index 0000000..927751c --- /dev/null +++ b/tests/fixtures/integration-multi-project.yml @@ -0,0 +1,28 @@ +projects: + - name: TF Integration Alpha + key: tf_int_alpha + environments: + - name: Dev + key: dev + type: development + jobs: + - name: Nightly + key: nightly + environment_key: dev + execute_steps: + - dbt compile + triggers: + schedule: true + github_webhook: false + git_provider_webhook: false + on_merge: false + schedule_type: days_of_week + schedule_days: [1, 2, 3, 4, 5] + schedule_hours: [2] + + - name: TF Integration Beta + key: tf_int_beta + environments: + - name: Dev + key: dev + type: development diff --git a/tests/fixtures/integration-service-tokens.yml b/tests/fixtures/integration-service-tokens.yml new file mode 100644 index 0000000..f60de4d --- /dev/null +++ b/tests/fixtures/integration-service-tokens.yml @@ -0,0 +1,10 @@ +projects: + - name: TF Integration Svc + key: tf_int_svc + +service_tokens: + - name: TF Integration Token + key: tf_int_svc_token + permissions: + - permission_set: readonly + all_projects: true diff --git a/tests/root.tftest.hcl b/tests/root.tftest.hcl new file mode 100644 index 0000000..c395a34 --- /dev/null +++ b/tests/root.tftest.hcl @@ -0,0 +1,130 @@ +# Root module unit tests — validate YAML parsing and module orchestration. +# Uses mock providers so no real dbt Cloud credentials are required. +# Run from the repo root: terraform test -filter=tests/root.tftest.hcl + +mock_provider "dbtcloud" {} + +mock_provider "dbtcloud" { + alias = "pat_provider" +} + +# ── Shared defaults (overridden per run where needed) ───────────────────────── + +variables { + dbt_account_id = 12345 + dbt_token = "fake-token-for-testing" + yaml_file = "tests/fixtures/basic.yml" +} + +# ── Single-project YAML (project: key) ─────────────────────────────────────── + +run "single_project_yaml_produces_one_project" { + command = plan + + assert { + condition = contains(keys(output.project_ids), "my_project") + error_message = "Expected project key 'my_project' in project_ids output" + } + + assert { + condition = length(output.project_ids) == 1 + error_message = "Expected exactly one project from basic.yml" + } +} + +run "single_project_environment_ids_populated" { + command = plan + + assert { + condition = contains(keys(output.environment_ids), "my_project_prod") + error_message = "Expected environment key 'my_project_prod' in environment_ids output" + } +} + +run "single_project_job_ids_populated" { + command = plan + + assert { + condition = contains(keys(output.job_ids), "my_project_daily_run") + error_message = "Expected job key 'my_project_daily_run' in job_ids output" + } +} + +run "target_name_prefix_does_not_change_project_ids_key" { + command = plan + + variables { + target_name = "dev-" + } + + assert { + condition = contains(keys(output.project_ids), "my_project") + error_message = "project_ids key should use the YAML key, not the prefixed display name" + } +} + +# ── Multi-project YAML (projects: list) ────────────────────────────────────── + +run "multi_project_yaml_produces_two_projects" { + command = plan + + variables { + yaml_file = "tests/fixtures/complete.yml" + } + + assert { + condition = length(output.project_ids) == 2 + error_message = "Expected two projects from complete.yml" + } + + assert { + condition = contains(keys(output.project_ids), "analytics") + error_message = "Expected 'analytics' project key in output" + } + + assert { + condition = contains(keys(output.project_ids), "finance") + error_message = "Expected 'finance' project key in output" + } +} + +run "multi_project_environments_keyed_correctly" { + command = plan + + variables { + yaml_file = "tests/fixtures/complete.yml" + } + + assert { + condition = contains(keys(output.environment_ids), "analytics_dev") + error_message = "Expected composite key 'analytics_dev' in environment_ids" + } + + assert { + condition = contains(keys(output.environment_ids), "analytics_prod") + error_message = "Expected composite key 'analytics_prod' in environment_ids" + } + + assert { + condition = contains(keys(output.environment_ids), "finance_prod") + error_message = "Expected composite key 'finance_prod' in environment_ids" + } +} + +run "multi_project_jobs_keyed_correctly" { + command = plan + + variables { + yaml_file = "tests/fixtures/complete.yml" + } + + assert { + condition = contains(keys(output.job_ids), "analytics_ci_check") + error_message = "Expected composite key 'analytics_ci_check' in job_ids" + } + + assert { + condition = contains(keys(output.job_ids), "analytics_daily_run") + error_message = "Expected composite key 'analytics_daily_run' in job_ids" + } +} diff --git a/validation.tf b/validation.tf new file mode 100644 index 0000000..955937a --- /dev/null +++ b/validation.tf @@ -0,0 +1,305 @@ +############################################# +# YAML Configuration Validation +# +# Collects all configuration errors and reports +# them together at plan time via terraform_data.validate_yaml_config +# in main.tf. Add new checks here as _errors_* locals. +############################################# + +locals { + # ── Index locals: pre-computed key sets for cross-reference lookups ──────── + + _valid_global_connection_keys = toset([ + for c in try(local.yaml_content.global_connections, []) : try(c.key, c.name) + ]) + + _valid_project_keys = toset([ + for p in local.projects : try(p.key, p.name) + ]) + + _valid_group_keys = toset([ + for g in try(local.yaml_content.groups, []) : try(g.key, g.name) + ]) + + # Environment keys per project: { project_key => set(env_key) } + _env_keys_by_project = { + for p in local.projects : + try(p.key, p.name) => toset([ + for e in try(p.environments, []) : try(e.key, e.name) + ]) + } + + # Job keys per project: { project_key => set(job_key) } + _job_keys_by_project = { + for p in local.projects : + try(p.key, p.name) => toset([ + for j in try(p.jobs, []) : try(j.key, j.name) + ]) + } + + # Extended attribute keys per project: { project_key => set(ea_key) } + _ea_keys_by_project = { + for p in local.projects : + try(p.key, p.name) => toset([ + for ea in try(p.extended_attributes, []) : try(ea.key, ea.name) + ]) + } + + _valid_credential_types = toset([ + "databricks", "snowflake", "bigquery", "redshift", "postgres", + "athena", "fabric", "synapse", "starburst", "trino", + "spark", "apache_spark", "teradata", + ]) + + _valid_connection_types = toset([ + "databricks", "snowflake", "bigquery", "redshift", "postgres", + "spark", "starburst_trino", "apache_spark", "athena", "fabric", "synapse", + ]) + + # ── V-01: connection_key in environments → global_connections[].key ──────── + + _errors_connection_key = flatten([ + for p in local.projects : [ + for env in try(p.environments, []) : + try( + env.connection_key != null && !contains(local._valid_global_connection_keys, env.connection_key) + ? ["Environment '${try(env.key, env.name)}' in project '${try(p.key, p.name)}' references connection_key '${env.connection_key}' but no global_connection with that key exists. Available keys: [${join(", ", tolist(local._valid_global_connection_keys))}]"] + : [], + [] + ) + ] + ]) + + # ── V-02: environment_key in jobs → environments[].key (same project) ────── + + _errors_job_env_key = flatten([ + for p in local.projects : [ + for job in try(p.jobs, []) : ( + try(job.environment_key, null) != null && + !contains( + try(local._env_keys_by_project[try(p.key, p.name)], toset([])), + job.environment_key + ) + ) ? [ + "Job '${try(job.key, job.name)}' in project '${try(p.key, p.name)}' references environment_key '${job.environment_key}' which is not defined in this project's environments. Available keys: [${join(", ", tolist(try(local._env_keys_by_project[try(p.key, p.name)], toset([]))))}]" + ] : [] + ] + ]) + + # ── V-03: extended_attributes_key in environments → extended_attributes[].key + + _errors_ea_key = flatten([ + for p in local.projects : [ + for env in try(p.environments, []) : + try( + env.extended_attributes_key != null && !contains( + try(local._ea_keys_by_project[try(p.key, p.name)], toset([])), + env.extended_attributes_key + ) + ? ["Environment '${try(env.key, env.name)}' in project '${try(p.key, p.name)}' references extended_attributes_key '${env.extended_attributes_key}' but no extended_attribute with that key exists. Available keys: [${join(", ", tolist(try(local._ea_keys_by_project[try(p.key, p.name)], toset([]))))}]"] + : [], + [] + ) + ] + ]) + + # ── V-04: deferring_environment_key in jobs → environments[].key ─────────── + + _errors_deferring_env_key = flatten([ + for p in local.projects : [ + for job in try(p.jobs, []) : + try( + job.deferring_environment_key != null && !contains( + try(local._env_keys_by_project[try(p.key, p.name)], toset([])), + job.deferring_environment_key + ) + ? ["Job '${try(job.key, job.name)}' in project '${try(p.key, p.name)}' has deferring_environment_key '${job.deferring_environment_key}' but that environment does not exist. Available keys: [${join(", ", tolist(try(local._env_keys_by_project[try(p.key, p.name)], toset([]))))}]"] + : [], + [] + ) + ] + ]) + + # ── V-05: artefacts.docs_job / freshness_job → jobs[].key ───────────────── + + _errors_artefact_job_keys = flatten([ + for p in local.projects : try(p.artefacts, null) != null ? concat( + try(p.artefacts.docs_job, null) != null && !contains( + try(local._job_keys_by_project[try(p.key, p.name)], toset([])), + p.artefacts.docs_job + ) ? [ + "Project '${try(p.key, p.name)}' artefacts.docs_job references job key '${p.artefacts.docs_job}' which does not exist. Available job keys: [${join(", ", tolist(try(local._job_keys_by_project[try(p.key, p.name)], toset([]))))}]" + ] : [], + try(p.artefacts.freshness_job, null) != null && !contains( + try(local._job_keys_by_project[try(p.key, p.name)], toset([])), + p.artefacts.freshness_job + ) ? [ + "Project '${try(p.key, p.name)}' artefacts.freshness_job references job key '${p.artefacts.freshness_job}' which does not exist. Available job keys: [${join(", ", tolist(try(local._job_keys_by_project[try(p.key, p.name)], toset([]))))}]" + ] : [], + ) : [] + ]) + + # ── V-06: Every project must have a name ────────────────────────────────── + + _errors_project_name = [ + for p in local.projects : + "A project entry is missing the required 'name' field." + if try(p.name, null) == null + ] + + # ── V-07: type=deployment environments must have deployment_type ─────────── + + _errors_deployment_type = flatten([ + for p in local.projects : [ + for env in try(p.environments, []) : + "Environment '${try(env.key, env.name)}' in project '${try(p.key, p.name)}' has type 'deployment' but is missing deployment_type. Set deployment_type to one of: [production, staging, other]" + if try(env.type, "development") == "deployment" && try(env.deployment_type, null) == null + ] + ]) + + # ── V-08: execute_steps must be non-empty ───────────────────────────────── + + _errors_execute_steps = flatten([ + for p in local.projects : [ + for job in try(p.jobs, []) : + "Job '${try(job.key, job.name)}' in project '${try(p.key, p.name)}' has an empty execute_steps list. At least one dbt command is required (e.g. 'dbt build')." + if length(try(job.execute_steps, [])) == 0 + ] + ]) + + # ── V-09: inline credential.credential_type must be a valid warehouse type ─ + + _errors_credential_type = flatten([ + for p in local.projects : [ + for env in try(p.environments, []) : + try( + env.credential.credential_type != null && !contains(local._valid_credential_types, env.credential.credential_type) + ? ["Environment '${try(env.key, env.name)}' in project '${try(p.key, p.name)}' has credential_type '${env.credential.credential_type}' which is not a recognized warehouse type. Valid types: [${join(", ", tolist(local._valid_credential_types))}]"] + : [], + [] + ) + ] + ]) + + # ── V-10: global_connections[].type must be a valid warehouse type ───────── + + _errors_connection_type = [ + for conn in try(local.yaml_content.global_connections, []) : + "Global connection '${try(conn.key, conn.name)}' has type '${try(conn.type, "")}' which is not a recognized warehouse type. Valid types: [${join(", ", tolist(local._valid_connection_types))}]" + if !contains(local._valid_connection_types, try(conn.type, "")) + ] + + # ── V-11: schedule coherence — schedule:true requires schedule_type or cron ─ + + _errors_schedule_config = flatten([ + for p in local.projects : [ + for job in try(p.jobs, []) : + "Job '${try(job.key, job.name)}' in project '${try(p.key, p.name)}' has triggers.schedule = true but no schedule_type or schedule_cron is set. Add schedule_type (e.g. 'days_of_week') and matching schedule_hours, or use schedule_cron with a cron expression." + if( + try(job.triggers.schedule, false) == true && + try(job.schedule_type, null) == null && + try(job.schedule_cron, null) == null + ) + ] + ]) + + # ── V-12: user_groups[].group_keys → groups[].key ───────────────────────── + + _errors_user_group_keys = flatten([ + for ug in try(local.yaml_content.user_groups, []) : [ + for gk in try(ug.group_keys, []) : + "user_groups entry for user_id ${try(ug.user_id, "unknown")} references group_key '${gk}' but no group with that key is defined. Available group keys: [${join(", ", tolist(local._valid_group_keys))}]" + if !contains(local._valid_group_keys, gk) + ] + ]) + + # ── V-13: service_token permission project_key → projects[].key ──────────── + + _errors_service_token_project_keys = flatten([ + for st in try(local.yaml_content.service_tokens, []) : [ + for perm in try(st.permissions, []) : + "Service token '${try(st.key, st.name)}' has a permission referencing project_key '${perm.project_key}' which is not a defined project. Available project keys: [${join(", ", tolist(local._valid_project_keys))}]" + if( + try(perm.project_key, null) != null && + !contains(local._valid_project_keys, perm.project_key) + ) + ] + ]) + + # ── Aggregated error list ────────────────────────────────────────────────── + + _all_validation_errors = compact(concat( + local._errors_connection_key, + local._errors_job_env_key, + local._errors_ea_key, + local._errors_deferring_env_key, + local._errors_artefact_job_keys, + local._errors_project_name, + local._errors_deployment_type, + local._errors_execute_steps, + local._errors_credential_type, + local._errors_connection_type, + local._errors_schedule_config, + local._errors_user_group_keys, + local._errors_service_token_project_keys, + )) +} + +############################################# +# Best-Practice Warnings (check blocks) +# +# These produce warnings at plan/apply time but do NOT block apply. +# They catch common configuration patterns that work but may be unintentional. +############################################# + +# ── C-01: Production environments without protected: true ───────────────────── +# A 'terraform destroy' on an unprotected production environment removes it +# along with all its history and job links. + +check "production_environments_protected" { + assert { + condition = length(flatten([ + for p in local.projects : [ + for env in try(p.environments, []) : + "${try(p.key, p.name)}/${try(env.key, env.name)}" + if try(env.deployment_type, null) == "production" && !try(env.protected, false) + ] + ])) == 0 + error_message = "Best practice: the following production environments have protected: false and could be accidentally deleted by 'terraform destroy'. Add protected: true to prevent this. Environments: ${join(", ", flatten([for p in local.projects : [for env in try(p.environments, []) : "${try(p.key, p.name)}/${try(env.key, env.name)}" if try(env.deployment_type, null) == "production" && !try(env.protected, false)]]))}" + } +} + +# ── C-02: Schedule config set but triggers.schedule is false ────────────────── +# schedule_type, schedule_cron, schedule_hours, and schedule_days are silently +# ignored when triggers.schedule is false — the job simply won't run on a schedule. + +check "schedule_config_without_trigger" { + assert { + condition = length(flatten([ + for p in local.projects : [ + for job in try(p.jobs, []) : + "'${try(job.key, job.name)}' in project '${try(p.key, p.name)}'" + if !try(job.triggers.schedule, false) && ( + try(job.schedule_type, null) != null || + try(job.schedule_cron, null) != null + ) + ] + ])) == 0 + error_message = "Best practice: the following jobs have schedule configuration (schedule_type or schedule_cron) but triggers.schedule is false, so the schedule will be ignored. Set triggers.schedule: true or remove the schedule config. Jobs: ${join(", ", flatten([for p in local.projects : [for job in try(p.jobs, []) : "'${try(job.key, job.name)}' in project '${try(p.key, p.name)}'" if !try(job.triggers.schedule, false) && (try(job.schedule_type, null) != null || try(job.schedule_cron, null) != null)]]))}" + } +} + +# ── C-03: Global connections without protected: true ────────────────────────── +# Deleting a global connection detaches every environment that references it, +# which can silently break jobs. Protecting connections prevents accidental removal. + +check "global_connections_protected" { + assert { + condition = length([ + for c in try(local.yaml_content.global_connections, []) : + try(c.key, c.name) + if !try(c.protected, false) + ]) == 0 + error_message = "Best practice: the following global connections have protected: false. Deleting a connection detaches all environments that reference it. Add protected: true to prevent accidental removal. Connections: ${join(", ", [for c in try(local.yaml_content.global_connections, []) : try(c.key, c.name) if !try(c.protected, false)])}" + } +} diff --git a/variables.tf b/variables.tf index 9b88024..522397b 100644 --- a/variables.tf +++ b/variables.tf @@ -5,9 +5,10 @@ variable "dbt_account_id" { description = "dbt Cloud account ID" type = number + default = null validation { - condition = var.dbt_account_id > 0 + condition = var.dbt_account_id == null || var.dbt_account_id > 0 error_message = "dbt_account_id must be a positive integer" } } @@ -16,25 +17,28 @@ variable "dbt_token" { description = "dbt Cloud API token for authentication" type = string sensitive = true + default = null validation { - condition = length(var.dbt_token) > 0 + condition = var.dbt_token == null || length(var.dbt_token) > 0 error_message = "dbt_token cannot be empty" } } variable "dbt_pat" { - type = string - sensitive = true - default = "" + description = "dbt Cloud personal access token for GitHub App integration discovery (service tokens cannot access the integrations API)" + type = string + sensitive = true + default = null } variable "dbt_host_url" { description = "dbt Cloud host URL (e.g., https://cloud.getdbt.com or custom domain)" type = string + default = null validation { - condition = can(regex("^https://", var.dbt_host_url)) + condition = var.dbt_host_url == null || can(regex("^https://", var.dbt_host_url)) error_message = "dbt_host_url must start with https://" } } @@ -49,7 +53,12 @@ variable "yaml_file" { validation { condition = can(file(var.yaml_file)) - error_message = "yaml_file must point to a valid, readable file" + error_message = "yaml_file '${var.yaml_file}' does not exist or cannot be read. Check the path relative to where you run terraform." + } + + validation { + condition = can(yamldecode(file(var.yaml_file))) + error_message = "yaml_file '${var.yaml_file}' exists but cannot be parsed as YAML. Check for indentation errors or invalid syntax." } } @@ -64,16 +73,62 @@ variable "target_name" { ############################################# variable "token_map" { - description = "Map of credential token names to their actual values (e.g., Databricks tokens). Token names should correspond to credential.token_name in YAML." + description = "Map of credential token names to their actual values (e.g., Databricks tokens). Token names correspond to credential.token_name in YAML." type = map(string) default = {} sensitive = true } +variable "connection_credentials" { + description = "Map of global connection keys to their OAuth/auth credential objects (client_id, client_secret, private_key, etc.)" + type = map(any) + default = {} + sensitive = true +} + +variable "environment_credentials" { + description = "Map of environment credential keys to their warehouse credential objects. Key format: project_key_env_key. Supports 14 warehouse types via credential_type field." + type = map(any) + default = {} + sensitive = true +} + +variable "oauth_client_secrets" { + description = "Map of OAuth configuration keys to their client secrets" + type = map(string) + default = {} + sensitive = true +} + +variable "lineage_tokens" { + description = "Map of lineage integration keys to their authentication tokens (Tableau, Looker, etc.)" + type = map(string) + default = {} + sensitive = true +} + +############################################# +# Repository Options +############################################# + +variable "enable_gitlab_deploy_token" { + description = "Preserve native GitLab deploy_token strategy. Defaults to false due to a known API limitation (GitlabGetError on some accounts). Set to true only when GitLab OAuth access is confirmed." + type = bool + default = false +} + ############################################# # Locals ############################################# locals { - project_config = yamldecode(file(var.yaml_file)) + yaml_content = yamldecode(file(var.yaml_file)) + + # Support both single project (project:) and multi-project (projects:) YAML shapes. + # Single-project users keep their existing YAML unchanged — the project: key is + # automatically wrapped into a one-element list. + projects = try( + local.yaml_content.projects, + [local.yaml_content.project] + ) }