feat: add cost_optimization_features to dbtcloud_job, fix CI/Merge SAO and deferral bugs (closes #664)

- Adds cost_optimization_features Set attribute as the preferred way to enable
  State-Aware Orchestration (SAO), replacing deprecated force_node_selection
- Fixes CI/Merge jobs erroring with SAO validation (405 error) by skipping
  SAO validation for CI and Merge job types
- Fixes CI/Merge jobs losing deferring_environment_id by preserving the value
  from the API response during Read
- Adds 5 new acceptance tests

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>

Author: trouze

pkg/dbt_cloud/job.go

@@ -58,31 +58,32 @@ type JobCompletionTriggerCondition struct {
 }
 
 type Job struct {
-	ID                     *int                  `json:"id"`
-	AccountId              int64                 `json:"account_id"`
-	ProjectId              int                   `json:"project_id"`
-	EnvironmentId          int                   `json:"environment_id"`
-	Name                   string                `json:"name"`
-	CompareChangesFlags    string                `json:"compare_changes_flags"`
-	DbtVersion             *string               `json:"dbt_version"`
-	DeferringEnvironmentId *int                  `json:"deferring_environment_id"`
-	DeferringJobId         *int                  `json:"deferring_job_definition_id"`
-	Description            string                `json:"description"`
-	ErrorsOnLintFailure    bool                  `json:"errors_on_lint_failure"`
-	ExecuteSteps           []string              `json:"execute_steps"`
-	Execution              JobExecution          `json:"execution"`
-	ForceNodeSelection     *bool                 `json:"force_node_selection,omitempty"`
-	GenerateDocs           bool                  `json:"generate_docs"`
-	JobCompletionTrigger   *JobCompletionTrigger `json:"job_completion_trigger_condition"`
-	JobType                string                `json:"job_type,omitempty"`
-	RunCompareChanges      bool                  `json:"run_compare_changes"`
-	RunGenerateSources     bool                  `json:"run_generate_sources"`
-	RunLint                bool                  `json:"run_lint"`
-	Schedule               JobSchedule           `json:"schedule"`
-	Settings               JobSettings           `json:"settings"`
-	State                  int                   `json:"state"`
-	TriggersOnDraftPR      bool                  `json:"triggers_on_draft_pr"`
-	Triggers               JobTrigger            `json:"triggers"`
+	ID                       *int                  `json:"id"`
+	AccountId                int64                 `json:"account_id"`
+	ProjectId                int                   `json:"project_id"`
+	EnvironmentId            int                   `json:"environment_id"`
+	Name                     string                `json:"name"`
+	CompareChangesFlags      string                `json:"compare_changes_flags"`
+	CostOptimizationFeatures []string              `json:"cost_optimization_features,omitempty"`
+	DbtVersion               *string               `json:"dbt_version"`
+	DeferringEnvironmentId   *int                  `json:"deferring_environment_id"`
+	DeferringJobId           *int                  `json:"deferring_job_definition_id"`
+	Description              string                `json:"description"`
+	ErrorsOnLintFailure      bool                  `json:"errors_on_lint_failure"`
+	ExecuteSteps             []string              `json:"execute_steps"`
+	Execution                JobExecution          `json:"execution"`
+	ForceNodeSelection       *bool                 `json:"force_node_selection,omitempty"`
+	GenerateDocs             bool                  `json:"generate_docs"`
+	JobCompletionTrigger     *JobCompletionTrigger `json:"job_completion_trigger_condition"`
+	JobType                  string                `json:"job_type,omitempty"`
+	RunCompareChanges        bool                  `json:"run_compare_changes"`
+	RunGenerateSources       bool                  `json:"run_generate_sources"`
+	RunLint                  bool                  `json:"run_lint"`
+	Schedule                 JobSchedule           `json:"schedule"`
+	Settings                 JobSettings           `json:"settings"`
+	State                    int                   `json:"state"`
+	TriggersOnDraftPR        bool                  `json:"triggers_on_draft_pr"`
+	Triggers                 JobTrigger            `json:"triggers"`
 }
 
 type JobWithEnvironment struct {
@@ -144,6 +145,7 @@ func (c *Client) CreateJob(
 	jobType string,
 	compareChangesFlags string,
 	forceNodeSelection *bool,
+	costOptimizationFeatures []string,
 ) (*Job, error) {
 	state := STATE_ACTIVE
 	if !isActive {
@@ -240,27 +242,28 @@ func (c *Client) CreateJob(
 	}
 
 	newJob := Job{
-		AccountId:            c.AccountID,
-		ProjectId:            projectId,
-		EnvironmentId:        environmentId,
-		Name:                 name,
-		Description:          description,
-		ExecuteSteps:         executeSteps,
-		State:                state,
-		Triggers:             jobTriggers,
-		Settings:             jobSettings,
-		Schedule:             jobSchedule,
-		GenerateDocs:         generateDocs,
-		RunGenerateSources:   runGenerateSources,
-		Execution:            jobExecution,
-		ForceNodeSelection:   forceNodeSelection,
-		TriggersOnDraftPR:    triggersOnDraftPR,
-		JobCompletionTrigger: jobCompletionTrigger,
-		JobType:              finalJobType,
-		RunCompareChanges:    runCompareChanges,
-		RunLint:              runLint,
-		ErrorsOnLintFailure:  errorsOnLintFailure,
-		CompareChangesFlags:  compareChangesFlags,
+		AccountId:                c.AccountID,
+		ProjectId:                projectId,
+		EnvironmentId:            environmentId,
+		Name:                     name,
+		Description:              description,
+		ExecuteSteps:             executeSteps,
+		State:                    state,
+		Triggers:                 jobTriggers,
+		Settings:                 jobSettings,
+		Schedule:                 jobSchedule,
+		GenerateDocs:             generateDocs,
+		RunGenerateSources:       runGenerateSources,
+		Execution:                jobExecution,
+		ForceNodeSelection:       forceNodeSelection,
+		CostOptimizationFeatures: costOptimizationFeatures,
+		TriggersOnDraftPR:        triggersOnDraftPR,
+		JobCompletionTrigger:     jobCompletionTrigger,
+		JobType:                  finalJobType,
+		RunCompareChanges:        runCompareChanges,
+		RunLint:                  runLint,
+		ErrorsOnLintFailure:      errorsOnLintFailure,
+		CompareChangesFlags:      compareChangesFlags,
 	}
 	if dbtVersion != "" {
 		newJob.DbtVersion = &dbtVersion

pkg/framework/objects/job/model.go

@@ -115,7 +115,8 @@ type JobResourceModel struct {
 	ExecuteSteps           []types.String `tfsdk:"execute_steps"`            // exists
 	ValidateExecuteSteps   types.Bool     `tfsdk:"validate_execute_steps"`   // opt-in validation
 	DeferringEnvironmentID types.Int64    `tfsdk:"deferring_environment_id"` // exists
-	ForceNodeSelection     types.Bool     `tfsdk:"force_node_selection"`     // exists
+	ForceNodeSelection           types.Bool `tfsdk:"force_node_selection"`            // exists
+	CostOptimizationFeatures     types.Set  `tfsdk:"cost_optimization_features"`      // replaces force_node_selection
 	Triggers               *JobTriggers   `tfsdk:"triggers"`                 // exists
 	// Settings                      *JobSettings          `tfsdk:"settings"`                 // has no of threads and target name
 	// Schedule                      *JobSchedule          `tfsdk:"schedule"`                 // has cron expression

pkg/framework/objects/job/resource.go

@@ -11,6 +11,7 @@ import (
 	"github.com/dbt-labs/terraform-provider-dbtcloud/pkg/dbt_cloud"
 	"github.com/dbt-labs/terraform-provider-dbtcloud/pkg/helper"
 	"github.com/dbt-labs/terraform-provider-dbtcloud/pkg/utils"
+	"github.com/hashicorp/terraform-plugin-framework/attr"
 	"github.com/hashicorp/terraform-plugin-framework/path"
 	"github.com/hashicorp/terraform-plugin-framework/resource"
 	"github.com/hashicorp/terraform-plugin-framework/types"
@@ -277,6 +278,11 @@ func (j *jobResource) Create(ctx context.Context, req resource.CreateRequest, re
 		forceNodeSelection = &fns
 	}
 
+	var costOptimizationFeatures []string
+	if !plan.CostOptimizationFeatures.IsNull() && !plan.CostOptimizationFeatures.IsUnknown() {
+		costOptimizationFeatures = helper.StringSetToStringSlice(plan.CostOptimizationFeatures)
+	}
+
 	var jobCompletionTriggerCondition map[string]any
 
 	if len(plan.JobCompletionTriggerCondition) != 0 {
@@ -334,6 +340,7 @@ func (j *jobResource) Create(ctx context.Context, req resource.CreateRequest, re
 		jobType,
 		compareChangesFlags,
 		forceNodeSelection,
+		costOptimizationFeatures,
 	)
 	if err != nil {
 		resp.Diagnostics.AddError(
@@ -391,6 +398,9 @@ func (j *jobResource) Create(ctx context.Context, req resource.CreateRequest, re
 		}
 	}
 
+	// Populate cost_optimization_features from API response
+	plan.CostOptimizationFeatures = sliceStringToTypesSet(createdJob.CostOptimizationFeatures)
+
 	jobIDStr := strconv.FormatInt(int64(*createdJob.ID), 10)
 
 	// Check if DeferringJobId is set and matches this job's ID for self-deferring
@@ -610,6 +620,9 @@ func (j *jobResource) Read(ctx context.Context, req resource.ReadRequest, resp *
 		state.ForceNodeSelection = types.BoolNull()
 	}
 
+	// Populate cost_optimization_features from API response
+	state.CostOptimizationFeatures = sliceStringToTypesSet(retrievedJob.CostOptimizationFeatures)
+
 	if retrievedJob.JobType != "" {
 		state.JobType = types.StringValue(retrievedJob.JobType)
 	} else {
@@ -732,7 +745,11 @@ func (j *jobResource) Update(ctx context.Context, req resource.UpdateRequest, re
 	}
 
 	if plan.DeferringEnvironmentID.IsNull() || plan.DeferringEnvironmentID.ValueInt64() == 0 {
-		job.DeferringEnvironmentId = nil
+		// For CI and Merge jobs, preserve the API's deferring_environment_id when the plan doesn't set it.
+		// Other job types clear it when not specified.
+		if job.JobType != JobTypeCI && job.JobType != JobTypeMerge {
+			job.DeferringEnvironmentId = nil
+		}
 	} else {
 		deferringEnvId := int(plan.DeferringEnvironmentID.ValueInt64())
 		job.DeferringEnvironmentId = &deferringEnvId
@@ -784,11 +801,25 @@ func (j *jobResource) Update(ctx context.Context, req resource.UpdateRequest, re
 	job.ErrorsOnLintFailure = plan.ErrorsOnLintFailure.ValueBool()
 	job.CompareChangesFlags = plan.CompareChangesFlags.ValueString()
 
-	if plan.ForceNodeSelection.IsNull() {
+	// CI and Merge jobs do not support SAO features (force_node_selection / cost_optimization_features).
+	// Sending these fields for those job types results in a 405 from the API, so we skip them.
+	isCIOrMerge := job.JobType == JobTypeCI || job.JobType == JobTypeMerge
+	if isCIOrMerge {
 		job.ForceNodeSelection = nil
+		job.CostOptimizationFeatures = nil
 	} else {
-		fns := plan.ForceNodeSelection.ValueBool()
-		job.ForceNodeSelection = &fns
+		if plan.ForceNodeSelection.IsNull() {
+			job.ForceNodeSelection = nil
+		} else {
+			fns := plan.ForceNodeSelection.ValueBool()
+			job.ForceNodeSelection = &fns
+		}
+
+		if !plan.CostOptimizationFeatures.IsNull() && !plan.CostOptimizationFeatures.IsUnknown() {
+			job.CostOptimizationFeatures = helper.StringSetToStringSlice(plan.CostOptimizationFeatures)
+		} else {
+			job.CostOptimizationFeatures = nil
+		}
 	}
 
 	// Handle job_type updates with validation
@@ -873,6 +904,17 @@ func (j *jobResource) Update(ctx context.Context, req resource.UpdateRequest, re
 		plan.ForceNodeSelection = types.BoolNull()
 	}
 
+	// Populate cost_optimization_features from API response
+	plan.CostOptimizationFeatures = sliceStringToTypesSet(updatedJob.CostOptimizationFeatures)
+
+	// For CI/Merge jobs, preserve deferring_environment_id from the API response
+	// so it is not lost in state when the plan did not explicitly configure it.
+	if updatedJob.DeferringEnvironmentId != nil {
+		plan.DeferringEnvironmentID = types.Int64Value(int64(*updatedJob.DeferringEnvironmentId))
+	} else if plan.DeferringEnvironmentID.IsNull() || plan.DeferringEnvironmentID.ValueInt64() == 0 {
+		plan.DeferringEnvironmentID = types.Int64Null()
+	}
+
 	updatedJobIDStr := strconv.FormatInt(jobID, 10)
 	updatedSelfDeferring := updatedJob.DeferringJobId != nil && strconv.Itoa(*updatedJob.DeferringJobId) == updatedJobIDStr
 	plan.SelfDeferring = types.BoolValue(updatedSelfDeferring)
@@ -884,6 +926,18 @@ func (j *jobResource) Update(ctx context.Context, req resource.UpdateRequest, re
 	}
 }
 
+// sliceStringToTypesSet returns a types.Set of strings; an empty set for nil/empty input.
+func sliceStringToTypesSet(values []string) types.Set {
+	if len(values) == 0 {
+		return types.SetValueMust(types.StringType, []attr.Value{})
+	}
+	elems := make([]attr.Value, len(values))
+	for i, v := range values {
+		elems[i] = types.StringValue(v)
+	}
+	return types.SetValueMust(types.StringType, elems)
+}
+
 func (j *jobResource) validateExecuteSteps(executeSteps []string) error {
 	dbt_flags := []string{
 		"--warn-error",