Merge pull request #195 from dbt-labs/release-0.2.8

Release 0.2.8

Author: b-per

.github/workflows/unit.yml

@@ -37,3 +37,4 @@ jobs:
           DBT_CLOUD_ACCOUNT_ID: ${{ secrets.TEST_DBT_CLOUD_ACCOUNT_ID }}
           DBT_CLOUD_TOKEN: ${{ secrets.TEST_DBT_CLOUD_TOKEN }}
           DBT_CLOUD_HOST_URL: ${{ secrets.TEST_DBT_CLOUD_HOST_URL }}
+          DBT_LEGACY_JOB_DEFERRAL: 1

CHANGELOG.md

@@ -2,7 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.7...HEAD)
+## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.8...HEAD)
+
+## [0.2.8](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.7...v0.2.8)
+
+## Fix
+
+- [#190](https://github.com/dbt-labs/terraform-provider-dbtcloud/issues/190) - Allow setting deferral for jobs at the environment level rather than at the job level. This is due to changes in CI in dbt Cloud. Add docs about those changes on the dbtcloud_job resource page
 
 ## [0.2.7](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.6...v0.2.7)
 

docs/data-sources/job.md

@@ -22,6 +22,7 @@ description: |-
 
 ### Read-Only
 
+- `deferring_environment_id` (Number) ID of the environment this job defers to
 - `deferring_job_id` (Number) ID of the job this job defers to
 - `environment_id` (Number) ID of the environment the job is in
 - `id` (String) The ID of this resource.

docs/resources/job.md

@@ -7,8 +7,11 @@ description: |-
 
 # dbtcloud_job (Resource)
 
-
-
+~> As of September 2023, some CI improvements are being rolled out to dbt Cloud with minor impacts to some jobs:  [more info](https://docs.getdbt.com/docs/dbt-versions/release-notes/june-2023/ci-updates-phase1-rn). 
+<br/>
+<br/>
+Those improvements include modifications to deferral which was historically set at the job level and will now be set at the environment level. 
+Deferral can still be set to "self" by setting `self_deferring` to `true` but with the new approach, deferral to other runs need to be done with `deferring_environment_id` instead of `deferring_job_id`.
 
 ## Example Usage
 
@@ -54,7 +57,8 @@ resource "dbtcloud_job" "test" {
 ### Optional
 
 - `dbt_version` (String) Version number of dbt to use in this job, usually in the format 1.2.0-latest rather than core versions
-- `deferring_job_id` (Number) Job identifier that this job defers to
+- `deferring_environment_id` (Number) Environment identifier that this job defers to (new deferring approach)
+- `deferring_job_id` (Number) Job identifier that this job defers to (legacy deferring approach)
 - `generate_docs` (Boolean) Flag for whether the job should generate documentation
 - `is_active` (Boolean) Flag for whether the job is marked active or deleted
 - `num_threads` (Number) Number of threads to use in the job
@@ -80,4 +84,4 @@ Import is supported using the following syntax:
 # Import using a job ID found in the URL or via the API.
 terraform import dbtcloud_job.test_job "job_id"
 terraform import dbtcloud_job.test_job 12345
-```
+```

docs/resources/repository.md

@@ -37,6 +37,7 @@ resource "dbtcloud_repository" "github_repo" {
 
 # repo cloned via the GitHub integration, with auto-retrieval of the `github_installation_id`
 # here, we assume that `token` and `host_url` are respectively accessible via `var.dbt_token` and `var.dbt_host_url`
+# NOTE: the following requires connecting via a user token and can't be retrieved with a service token
 data "http" "github_installations_response" {
   url = format("%s/v2/integrations/github/installations/", var.dbt_host_url)
   request_headers = {
@@ -57,6 +58,7 @@ resource "dbtcloud_repository" "github_repo_other" {
 
 
 # repo cloned via the GitLab integration
+# as of 15 Sept 2023 this resource requires using a user token and can't be set with a service token
 resource "dbtcloud_repository" "gitlab_repo" {
   project_id        = dbtcloud_project.my_project_2.id
   remote_url        = "<gitlab-group>/<gitlab-project>"

examples/resources/dbtcloud_repository/resource.tf

@@ -11,6 +11,7 @@ resource "dbtcloud_repository" "github_repo" {
 
 # repo cloned via the GitHub integration, with auto-retrieval of the `github_installation_id`
 # here, we assume that `token` and `host_url` are respectively accessible via `var.dbt_token` and `var.dbt_host_url`
+# NOTE: the following requires connecting via a user token and can't be retrieved with a service token
 data "http" "github_installations_response" {
   url = format("%s/v2/integrations/github/installations/", var.dbt_host_url)
   request_headers = {
@@ -31,6 +32,7 @@ resource "dbtcloud_repository" "github_repo_other" {
 
 
 # repo cloned via the GitLab integration
+# as of 15 Sept 2023 this resource requires using a user token and can't be set with a service token
 resource "dbtcloud_repository" "gitlab_repo" {
   project_id        = dbtcloud_project.my_project_2.id
   remote_url        = "<gitlab-group>/<gitlab-project>"

pkg/data_sources/job.go

@@ -36,6 +36,11 @@ var jobSchema = map[string]*schema.Schema{
 		Computed:    true,
 		Description: "ID of the job this job defers to",
 	},
+	"deferring_environment_id": &schema.Schema{
+		Type:        schema.TypeInt,
+		Computed:    true,
+		Description: "ID of the environment this job defers to",
+	},
 	"self_deferring": &schema.Schema{
 		Type:        schema.TypeBool,
 		Computed:    true,
@@ -95,6 +100,9 @@ func datasourceJobRead(ctx context.Context, d *schema.ResourceData, m interface{
 			return diag.FromErr(err)
 		}
 	}
+	if err := d.Set("deferring_environment_id", job.DeferringEnvironmentId); err != nil {
+		return diag.FromErr(err)
+	}
 	if err := d.Set("self_deferring", selfDeferring); err != nil {
 		return diag.FromErr(err)
 	}

pkg/dbt_cloud/job.go

@@ -48,21 +48,22 @@ type JobExecution struct {
 }
 
 type Job struct {
-	ID                   *int         `json:"id"`
-	Account_Id           int          `json:"account_id"`
-	Project_Id           int          `json:"project_id"`
-	Environment_Id       int          `json:"environment_id"`
-	Name                 string       `json:"name"`
-	Execute_Steps        []string     `json:"execute_steps"`
-	Dbt_Version          *string      `json:"dbt_version"`
-	Triggers             JobTrigger   `json:"triggers"`
-	Settings             JobSettings  `json:"settings"`
-	State                int          `json:"state"`
-	Generate_Docs        bool         `json:"generate_docs"`
-	Schedule             JobSchedule  `json:"schedule"`
-	Run_Generate_Sources bool         `json:"run_generate_sources"`
-	Deferring_Job_Id     *int         `json:"deferring_job_definition_id"`
-	Execution            JobExecution `json:"execution"`
+	ID                     *int         `json:"id"`
+	Account_Id             int          `json:"account_id"`
+	Project_Id             int          `json:"project_id"`
+	Environment_Id         int          `json:"environment_id"`
+	Name                   string       `json:"name"`
+	Execute_Steps          []string     `json:"execute_steps"`
+	Dbt_Version            *string      `json:"dbt_version"`
+	Triggers               JobTrigger   `json:"triggers"`
+	Settings               JobSettings  `json:"settings"`
+	State                  int          `json:"state"`
+	Generate_Docs          bool         `json:"generate_docs"`
+	Schedule               JobSchedule  `json:"schedule"`
+	Run_Generate_Sources   bool         `json:"run_generate_sources"`
+	Deferring_Job_Id       *int         `json:"deferring_job_definition_id"`
+	DeferringEnvironmentId *int         `json:"deferring_environment_id"`
+	Execution              JobExecution `json:"execution"`
 }
 
 func (c *Client) GetJob(jobID string) (*Job, error) {
@@ -85,7 +86,7 @@ func (c *Client) GetJob(jobID string) (*Job, error) {
 	return &jobResponse.Data, nil
 }
 
-func (c *Client) CreateJob(projectId int, environmentId int, name string, executeSteps []string, dbtVersion string, isActive bool, triggers map[string]interface{}, numThreads int, targetName string, generateDocs bool, runGenerateSources bool, scheduleType string, scheduleInterval int, scheduleHours []int, scheduleDays []int, scheduleCron string, deferringJobId int, selfDeferring bool, timeoutSeconds int) (*Job, error) {
+func (c *Client) CreateJob(projectId int, environmentId int, name string, executeSteps []string, dbtVersion string, isActive bool, triggers map[string]interface{}, numThreads int, targetName string, generateDocs bool, runGenerateSources bool, scheduleType string, scheduleInterval int, scheduleHours []int, scheduleDays []int, scheduleCron string, deferringJobId int, deferringEnvironmentID int, selfDeferring bool, timeoutSeconds int) (*Job, error) {
 	state := STATE_ACTIVE
 	if !isActive {
 		state = STATE_DELETED
@@ -165,6 +166,13 @@ func (c *Client) CreateJob(projectId int, environmentId int, name string, execut
 	}
 	if deferringJobId != 0 {
 		newJob.Deferring_Job_Id = &deferringJobId
+	} else {
+		newJob.Deferring_Job_Id = nil
+	}
+	if deferringEnvironmentID != 0 {
+		newJob.DeferringEnvironmentId = &deferringEnvironmentID
+	} else {
+		newJob.DeferringEnvironmentId = nil
 	}
 	newJobData, err := json.Marshal(newJob)
 	if err != nil {

pkg/resources/job.go

@@ -132,8 +132,14 @@ var jobSchema = map[string]*schema.Schema{
 	"deferring_job_id": &schema.Schema{
 		Type:          schema.TypeInt,
 		Optional:      true,
-		Description:   "Job identifier that this job defers to",
-		ConflictsWith: []string{"self_deferring"},
+		Description:   "Job identifier that this job defers to (legacy deferring approach)",
+		ConflictsWith: []string{"self_deferring", "deferring_environment_id"},
+	},
+	"deferring_environment_id": &schema.Schema{
+		Type:          schema.TypeInt,
+		Optional:      true,
+		Description:   "Environment identifier that this job defers to (new deferring approach)",
+		ConflictsWith: []string{"self_deferring", "deferring_job_id"},
 	},
 	"self_deferring": &schema.Schema{
 		Type:          schema.TypeBool,
@@ -237,6 +243,9 @@ func resourceJobRead(ctx context.Context, d *schema.ResourceData, m interface{})
 			return diag.FromErr(err)
 		}
 	}
+	if err := d.Set("deferring_environment_id", job.DeferringEnvironmentId); err != nil {
+		return diag.FromErr(err)
+	}
 	if err := d.Set("self_deferring", selfDeferring); err != nil {
 		return diag.FromErr(err)
 	}
@@ -278,6 +287,7 @@ func resourceJobCreate(ctx context.Context, d *schema.ResourceData, m interface{
 	scheduleDays := d.Get("schedule_days").([]interface{})
 	scheduleCron := d.Get("schedule_cron").(string)
 	deferringJobId := d.Get("deferring_job_id").(int)
+	deferringEnvironmentID := d.Get("deferring_environment_id").(int)
 	selfDeferring := d.Get("self_deferring").(bool)
 	timeoutSeconds := d.Get("timeout_seconds").(int)
 
@@ -294,7 +304,7 @@ func resourceJobCreate(ctx context.Context, d *schema.ResourceData, m interface{
 		days = append(days, day.(int))
 	}
 
-	j, err := c.CreateJob(projectId, environmentId, name, steps, dbtVersion, isActive, triggers, numThreads, targetName, generateDocs, runGenerateSources, scheduleType, scheduleInterval, hours, days, scheduleCron, deferringJobId, selfDeferring, timeoutSeconds)
+	j, err := c.CreateJob(projectId, environmentId, name, steps, dbtVersion, isActive, triggers, numThreads, targetName, generateDocs, runGenerateSources, scheduleType, scheduleInterval, hours, days, scheduleCron, deferringJobId, deferringEnvironmentID, selfDeferring, timeoutSeconds)
 	if err != nil {
 		return diag.FromErr(err)
 	}
@@ -326,6 +336,7 @@ func resourceJobUpdate(ctx context.Context, d *schema.ResourceData, m interface{
 		d.HasChange("schedule_days") ||
 		d.HasChange("schedule_cron") ||
 		d.HasChange("deferring_job_id") ||
+		d.HasChange("deferring_environment_id") ||
 		d.HasChange("self_deferring") ||
 		d.HasChange("timeout_seconds") {
 		job, err := c.GetJob(jobId)
@@ -417,7 +428,19 @@ func resourceJobUpdate(ctx context.Context, d *schema.ResourceData, m interface{
 		}
 		if d.HasChange("deferring_job_id") {
 			deferringJobId := d.Get("deferring_job_id").(int)
-			job.Deferring_Job_Id = &deferringJobId
+			if deferringJobId != 0 {
+				job.Deferring_Job_Id = &deferringJobId
+			} else {
+				job.Deferring_Job_Id = nil
+			}
+		}
+		if d.HasChange("deferring_environment_id") {
+			deferringEnvironmentId := d.Get("deferring_environment_id").(int)
+			if deferringEnvironmentId != 0 {
+				job.DeferringEnvironmentId = &deferringEnvironmentId
+			} else {
+				job.DeferringEnvironmentId = nil
+			}
 		}
 		// If self_deferring has been toggled to true, set deferring_job_id as own ID
 		// Otherwise, set it back to what deferring_job_id specifies it to be
@@ -427,7 +450,11 @@ func resourceJobUpdate(ctx context.Context, d *schema.ResourceData, m interface{
 				job.Deferring_Job_Id = &deferringJobID
 			} else {
 				deferringJobId := d.Get("deferring_job_id").(int)
-				job.Deferring_Job_Id = &deferringJobId
+				if deferringJobId != 0 {
+					job.Deferring_Job_Id = &deferringJobId
+				} else {
+					job.Deferring_Job_Id = nil
+				}
 			}
 		}
 		if d.HasChange("timeout_seconds") {

pkg/resources/job_acceptance_test.go

@@ -2,6 +2,7 @@ package resources_test
 
 import (
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
 	"testing"
@@ -20,6 +21,29 @@ func TestAccDbtCloudJobResource(t *testing.T) {
 	projectName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha))
 	environmentName := strings.ToUpper(acctest.RandStringFromCharSet(10, acctest.CharSetAlpha))
 
+	var configDeferral string
+	var checkDeferral resource.TestCheckFunc
+
+	// this logic is used as not all accounts are on the new CI approach of deferring at the env level
+	if os.Getenv("DBT_LEGACY_JOB_DEFERRAL") != "" {
+		configDeferral = testAccDbtCloudJobResourceDeferringConfig(jobName, jobName2, jobName3, projectName, environmentName, "self")
+		checkDeferral = resource.ComposeTestCheckFunc(
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job"),
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_2"),
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_3"),
+			resource.TestCheckResourceAttrSet("dbtcloud_job.test_job_2", "deferring_job_id"),
+			resource.TestCheckResourceAttrSet("dbtcloud_job.test_job_3", "self_deferring"),
+		)
+	} else {
+		configDeferral = testAccDbtCloudJobResourceDeferringConfig(jobName, jobName2, jobName3, projectName, environmentName, "env")
+		checkDeferral = resource.ComposeTestCheckFunc(
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job"),
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_2"),
+			testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_3"),
+			resource.TestCheckResourceAttrSet("dbtcloud_job.test_job_2", "deferring_environment_id"),
+		)
+	}
+
 	resource.Test(t, resource.TestCase{
 		PreCheck:     func() { testAccPreCheck(t) },
 		Providers:    testAccProviders,
@@ -57,15 +81,26 @@ func TestAccDbtCloudJobResource(t *testing.T) {
 					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "generate_docs"),
 				),
 			},
-			// DEFERRING JOBS
+			// DEFERRING JOBS (depends on whether DBT_LEGACY_JOB_DEFERRAL is set, e.g. whether the new CI is set)
+			{
+				Config: configDeferral,
+				Check:  checkDeferral,
+			},
+			// REMOVE DEFERRAL
 			{
-				Config: testAccDbtCloudJobResourceDeferringJobConfig(jobName, jobName2, jobName3, projectName, environmentName, false),
+				Config: testAccDbtCloudJobResourceFullConfig(jobName2, projectName, environmentName),
 				Check: resource.ComposeTestCheckFunc(
 					testAccCheckDbtCloudJobExists("dbtcloud_job.test_job"),
-					testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_2"),
-					testAccCheckDbtCloudJobExists("dbtcloud_job.test_job_3"),
-					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job_2", "deferring_job_id"),
-					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job_3", "self_deferring"),
+					resource.TestCheckResourceAttr("dbtcloud_job.test_job", "name", jobName2),
+					resource.TestCheckResourceAttr("dbtcloud_job.test_job", "dbt_version", DBT_CLOUD_VERSION),
+					resource.TestCheckResourceAttr("dbtcloud_job.test_job", "target_name", "test"),
+					resource.TestCheckResourceAttr("dbtcloud_job.test_job", "timeout_seconds", "180"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "project_id"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "environment_id"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "is_active"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "num_threads"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "run_generate_sources"),
+					resource.TestCheckResourceAttrSet("dbtcloud_job.test_job", "generate_docs"),
 				),
 			},
 			// IMPORT
@@ -155,10 +190,12 @@ resource "dbtcloud_job" "test_job" {
 `, projectName, environmentName, DBT_CLOUD_VERSION, environmentName, DBT_CLOUD_VERSION, jobName, DBT_CLOUD_VERSION)
 }
 
-func testAccDbtCloudJobResourceDeferringJobConfig(jobName, jobName2, jobName3, projectName, environmentName string, selfDeferring bool) string {
-	deferParam := "deferring_job_id = dbtcloud_job.test_job.id"
-	if selfDeferring {
+func testAccDbtCloudJobResourceDeferringConfig(jobName, jobName2, jobName3, projectName, environmentName string, deferring string) string {
+	deferParam := ""
+	if deferring == "self" {
 		deferParam = "self_deferring = true"
+	} else if deferring == "env" {
+		deferParam = "deferring_environment_id = dbtcloud_environment.test_job_environment_new.environment_id"
 	}
 	return fmt.Sprintf(`
 resource "dbtcloud_project" "test_job_project" {
@@ -224,9 +261,9 @@ resource "dbtcloud_job" "test_job_3" {
 	  "schedule": false,
 	  "custom_branch_only": false,
 	}
-	self_deferring = true
+	%s
   }
-`, projectName, environmentName, DBT_CLOUD_VERSION, jobName, DBT_CLOUD_VERSION, jobName2, deferParam, jobName3)
+`, projectName, environmentName, DBT_CLOUD_VERSION, jobName, DBT_CLOUD_VERSION, jobName2, deferParam, jobName3, deferParam)
 }
 
 func testAccCheckDbtCloudJobExists(resource string) resource.TestCheckFunc {

pkg/resources/license_map_acceptance_test.go

@@ -38,8 +38,8 @@ func TestAccDbtCloudLicenseMapResource(t *testing.T) {
 					testAccCheckDbtCloudLicenseMapExists("dbtcloud_license_map.test_license_map"),
 					resource.TestCheckResourceAttr("dbtcloud_license_map.test_license_map", "license_type", "developer"),
 					resource.TestCheckResourceAttr("dbtcloud_license_map.test_license_map", "sso_license_mapping_groups.#", "2"),
-					resource.TestCheckResourceAttr("dbtcloud_license_map.test_license_map", "sso_license_mapping_groups.0", groupName),
-					resource.TestCheckResourceAttr("dbtcloud_license_map.test_license_map", "sso_license_mapping_groups.1", groupName2),
+					resource.TestCheckTypeSetElemAttr("dbtcloud_license_map.test_license_map", "sso_license_mapping_groups.*", groupName),
+					resource.TestCheckTypeSetElemAttr("dbtcloud_license_map.test_license_map", "sso_license_mapping_groups.*", groupName2),
 				),
 			},
 			// IMPORT

templates/resources/job.md.tmpl

@@ -0,0 +1,26 @@
+---
+page_title: "{{.Name}} {{.Type}} - {{.RenderedProviderName}}"
+subcategory: ""
+description: |-
+{{ .Description | plainmarkdown | trimspace | prefixlines "  " }}
+---
+
+# {{.Name}} ({{.Type}})
+
+~> As of September 2023, some CI improvements are being rolled out to dbt Cloud with minor impacts to some jobs:  [more info](https://docs.getdbt.com/docs/dbt-versions/release-notes/june-2023/ci-updates-phase1-rn). 
+<br/>
+<br/>
+Those improvements include modifications to deferral which was historically set at the job level and will now be set at the environment level. 
+Deferral can still be set to "self" by setting `self_deferring` to `true` but with the new approach, deferral to other runs need to be done with `deferring_environment_id` instead of `deferring_job_id`.
+
+## Example Usage
+
+{{ tffile (printf "%s%s%s" "examples/resources/" .Name "/resource.tf") }}
+
+{{ .SchemaMarkdown | trimspace }}
+
+## Import
+
+Import is supported using the following syntax:
+
+{{ codefile "shell" (printf "%s%s%s" "examples/resources/" .Name "/import.sh") }}