fix: only require a managed plan file for managed apply workflows

runatlantis.io/docs/custom-workflows.md

@@ -647,6 +647,11 @@ Full example, filtering output and masking matching text (`mySecret: "foo"` -> `
   * `PLANFILE` - Absolute path to the location where Atlantis expects the plan to
       either be generated (by plan) or already exist (if running apply). Can be used to
       override the built-in `plan`/`apply` commands, ex. `run: terraform plan -out $PLANFILE`.
+      A workflow whose `plan` and `apply` are both made up entirely of custom `run` steps
+      may write its plan to a path of its own choosing instead of `$PLANFILE`. Atlantis
+      does not require, hash, or delete a plan artifact for such a workflow; it still
+      validates the project's recorded plan state before running `apply`. As soon as a
+      workflow uses the built-in `plan` or `apply` step, the plan must be at `$PLANFILE`.
   * `SHOWFILE` - Absolute path to the location where Atlantis expects the plan in json format to
       either be generated (by show) or already exist (if running policy checks). Can be used to
       override the built-in `plan`/`apply` commands, ex. `run: terraform show -json $PLANFILE > $SHOWFILE`.

server/events/apply_plan_validator.go

@@ -21,6 +21,10 @@ import (
 
 type ApplyPlanValidator interface {
 	ValidateProjectPlan(ctx command.ProjectContext, absPath string) error
+	// ValidateProjectPlanStatus validates only durable plan state. It is used
+	// for workflows that do not consume the Atlantis convention plan file, so
+	// no plan artifact may be inspected or removed.
+	ValidateProjectPlanStatus(ctx command.ProjectContext) error
 }
 
 type ApplyCommandStartValidator interface {
@@ -38,6 +42,20 @@ type DefaultApplyPlanValidator struct {
 
 var errStaleCommandHead = errors.New("stale command head")
 
+// planRejectionError marks a durable-status failure that must also remove the
+// convention-managed plan artifact when one is being validated. Failures that
+// are not wrapped in it (for example a stale command head) leave the artifact
+// in place because another replica may have replaced it.
+type planRejectionError struct{ err error }
+
+func (e planRejectionError) Error() string { return e.err.Error() }
+
+func (e planRejectionError) Unwrap() error { return e.err }
+
+func rejectionErrorf(format string, args ...any) error {
+	return planRejectionError{err: fmt.Errorf(format, args...)}
+}
+
 func (v *DefaultApplyPlanValidator) ValidateCommandStartHead(ctx command.ProjectContext) error {
 	if v == nil || v.LivePullHeadFetcher == nil {
 		return nil
@@ -49,21 +67,26 @@ func (v *DefaultApplyPlanValidator) ValidateCommandStartHead(ctx command.Project
 	return validateCommandStartIdentity(ctx, livePull)
 }
 
-func (v *DefaultApplyPlanValidator) ValidateProjectPlan(ctx command.ProjectContext, absPath string) error {
+// ValidateProjectPlanStatus validates durable plan state without inspecting or
+// removing any plan artifact. Workflows with a custom apply step manage their
+// own plan file, which Atlantis neither creates nor can locate.
+func (v *DefaultApplyPlanValidator) ValidateProjectPlanStatus(ctx command.ProjectContext) error {
 	if v == nil || v.PullStatusFetcher == nil {
 		return nil
 	}
-	planPath, err := safePlanFilePath(ctx, absPath)
-	if err != nil {
-		return err
-	}
+	return v.validateProjectPlanStatus(ctx)
+}
 
+// validateProjectPlanStatus only validates durable state. Failures that should
+// also discard a convention-managed plan artifact are wrapped in
+// planRejectionError; the caller decides whether an artifact exists to remove.
+func (v *DefaultApplyPlanValidator) validateProjectPlanStatus(ctx command.ProjectContext) error {
 	pullStatus, err := v.pullStatusForApply(ctx)
 	if err != nil {
 		return fmt.Errorf("fetching current plan status: %w", err)
 	}
 	if pullStatus == nil {
-		return rejectProjectPlan(planPath, "no current plan status found; run `atlantis plan` before apply")
+		return rejectionErrorf("no current plan status found; run `atlantis plan` before apply")
 	}
 
 	livePull, err := v.getLivePullIdentity(ctx)
@@ -83,28 +106,47 @@ func (v *DefaultApplyPlanValidator) ValidateProjectPlan(ctx command.ProjectConte
 			return err
 		}
 	} else if err := pullStatusApplyEligibilityError(ctx.Pull, pullStatus.Pull, "recorded plan status"); err != nil {
-		return rejectProjectPlan(planPath, "%s", err)
+		return rejectionErrorf("%s", err)
 	}
 
 	proj := findProjectInPullStatus(pullStatus, ctx.Workspace, ctx.RepoRelDir, ctx.ProjectName)
 	if proj == nil {
-		return rejectProjectPlan(planPath,
+		return rejectionErrorf(
 			"no matching plan status exists for dir %q workspace %q project %q; run `atlantis plan`",
 			ctx.RepoRelDir, ctx.Workspace, ctx.ProjectName,
 		)
 	}
 	if !statusAllowedForApplyExecution(proj.Status) {
 		if proj.Status == models.ErroredPolicyCheckStatus {
-			return rejectProjectPlan(planPath,
+			return rejectionErrorf(
 				"policy checks have errored for dir %q workspace %q project %q and cannot be applied; run `atlantis plan`",
 				ctx.RepoRelDir, ctx.Workspace, ctx.ProjectName,
 			)
 		}
-		return rejectProjectPlan(planPath,
+		return rejectionErrorf(
 			"plan for dir %q workspace %q project %q has status %q and cannot be applied; run `atlantis plan`",
 			ctx.RepoRelDir, ctx.Workspace, ctx.ProjectName, proj.Status.String(),
 		)
 	}
+	return nil
+}
+
+func (v *DefaultApplyPlanValidator) ValidateProjectPlan(ctx command.ProjectContext, absPath string) error {
+	if v == nil || v.PullStatusFetcher == nil {
+		return nil
+	}
+	planPath, err := safePlanFilePath(ctx, absPath)
+	if err != nil {
+		return err
+	}
+
+	if err := v.validateProjectPlanStatus(ctx); err != nil {
+		var rejection planRejectionError
+		if errors.As(err, &rejection) {
+			return rejectProjectPlan(planPath, "%s", rejection.err)
+		}
+		return err
+	}
 
 	if _, err := os.Stat(planPath); err != nil {
 		if os.IsNotExist(err) {

server/events/command/project_context.go

@@ -81,6 +81,11 @@ type ProjectContext struct {
 	// ExpectedPlanHash is the SHA-256 hash of the plan file selected when the
 	// apply command was built.
 	ExpectedPlanHash string
+	// RequiresAtlantisManagedPlanFile is true when this project's workflow uses
+	// the built-in plan or apply step, meaning Atlantis owns the convention plan
+	// artifact (<workspace>.tfplan). Workflows built only from custom run steps
+	// manage their own plan file, so Atlantis must not require or inspect one.
+	RequiresAtlantisManagedPlanFile bool
 	//PullStatus is the status of the current pull request prior to this command.
 	PullStatus *models.PullStatus
 	// ProjectPolicyStatus is the status of policy sets of the current project prior to this command.

server/events/project_command_builder_internal_test.go

@@ -6,6 +6,7 @@ package events
 import (
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	version "github.com/hashicorp/go-version"
@@ -729,6 +730,10 @@ projects:
 					c.expCtx.CommandName = cmd
 					// Init fields we couldn't in our cases map.
 					c.expCtx.Steps = expSteps
+					// Atlantis owns the convention plan artifact only when the
+					// workflow uses the built-in plan or apply step.
+					c.expCtx.RequiresAtlantisManagedPlanFile = slices.Contains(c.expPlanSteps, "plan") ||
+						slices.Contains(c.expApplySteps, "apply")
 					ctx.PolicySets = emptyPolicySets
 
 					// Job ID cannot be compared since its generated at random
@@ -950,6 +955,10 @@ projects:
 					c.expCtx.CommandName = cmd
 					// Init fields we couldn't in our cases map.
 					c.expCtx.Steps = expSteps
+					// Atlantis owns the convention plan artifact only when the
+					// workflow uses the built-in plan or apply step.
+					c.expCtx.RequiresAtlantisManagedPlanFile = slices.Contains(c.expPlanSteps, "plan") ||
+						slices.Contains(c.expApplySteps, "apply")
 					ctx.PolicySets = emptyPolicySets
 
 					// Job ID cannot be compared since its generated at random
@@ -1195,6 +1204,10 @@ workflows:
 				c.expCtx.CommandName = cmd
 				// Init fields we couldn't in our cases map.
 				c.expCtx.Steps = expSteps
+				// These cases only override policy_check, so plan and apply
+				// fall back to the built-in default steps and Atlantis owns
+				// the convention plan artifact.
+				c.expCtx.RequiresAtlantisManagedPlanFile = true
 				ctx.PolicySets = emptyPolicySets
 
 				// Job ID cannot be compared since its generated at random

server/events/project_command_context_builder.go

@@ -283,58 +283,59 @@ func newProjectCommandContext(ctx *command.Context,
 	}
 
 	return command.ProjectContext{
-		CommandName:                cmd,
-		SubCommand:                 subCommand,
-		ApplyCmd:                   applyCmd,
-		ApprovePoliciesCmd:         approvePoliciesCmd,
-		BaseRepo:                   ctx.Pull.BaseRepo,
-		EscapedCommentArgs:         escapedCommentArgs,
-		AutomergeEnabled:           automergeEnabled,
-		DeleteSourceBranchOnMerge:  projCfg.DeleteSourceBranchOnMerge,
-		RepoLocksMode:              projCfg.RepoLocks.Mode,
-		CustomPolicyCheck:          projCfg.CustomPolicyCheck,
-		ParallelApplyEnabled:       parallelApplyEnabled,
-		ParallelPlanEnabled:        parallelPlanEnabled,
-		ParallelPolicyCheckEnabled: parallelPlanEnabled,
-		DependsOn:                  projCfg.DependsOn,
-		AutoplanEnabled:            projCfg.AutoplanEnabled,
-		AutoplanWhenModified:       projCfg.AutoplanWhenModified,
-		Steps:                      steps,
-		HeadRepo:                   ctx.HeadRepo,
-		Log:                        ctx.Log,
-		Scope:                      scope,
-		ProjectPlanStatus:          projectPlanStatus,
-		ProjectPolicyStatus:        projectPolicyStatus,
-		Pull:                       ctx.Pull,
-		ProjectName:                projCfg.Name,
-		PlanRequirements:           projCfg.PlanRequirements,
-		ApplyRequirements:          projCfg.ApplyRequirements,
-		ImportRequirements:         projCfg.ImportRequirements,
-		RePlanCmd:                  planCmd,
-		RepoRelDir:                 projCfg.RepoRelDir,
-		RepoConfigVersion:          projCfg.RepoCfgVersion,
-		TerraformDistribution:      projCfg.TerraformDistribution,
-		TerraformVersion:           projCfg.TerraformVersion,
-		User:                       ctx.User,
-		Verbose:                    verbose,
-		Workspace:                  projCfg.Workspace,
-		PolicySets:                 policySets,
-		PolicySetTarget:            ctx.PolicySet,
-		ClearPolicyApproval:        ctx.ClearPolicyApproval,
-		PullReqStatus:              pullReqStatus,
-		PullStatus:                 pullStatus,
-		JobID:                      uuid.New().String(),
-		ExecutionOrderGroup:        projCfg.ExecutionOrderGroup,
-		AbortOnExecutionOrderFail:  abortOnExecutionOrderFail,
-		SilencePRComments:          projCfg.SilencePRComments,
-		TeamAllowlistChecker:       teamAllowlistChecker,
-		API:                        ctx.API,
-		SkipPRRequirements:         ctx.SkipPRRequirements,
-		RunPolicyChecks:            ctx.RunPolicyChecks,
-		SuppressVCSStatus:          ctx.SuppressVCSStatus,
-		SuppressJobOutput:          ctx.SuppressJobOutput,
-		SuppressApplyWebhooks:      ctx.SuppressApplyWebhooks,
-		FailOnMissingDependencies:  ctx.FailOnMissingDependencies,
+		CommandName:                     cmd,
+		SubCommand:                      subCommand,
+		ApplyCmd:                        applyCmd,
+		ApprovePoliciesCmd:              approvePoliciesCmd,
+		BaseRepo:                        ctx.Pull.BaseRepo,
+		EscapedCommentArgs:              escapedCommentArgs,
+		AutomergeEnabled:                automergeEnabled,
+		DeleteSourceBranchOnMerge:       projCfg.DeleteSourceBranchOnMerge,
+		RepoLocksMode:                   projCfg.RepoLocks.Mode,
+		CustomPolicyCheck:               projCfg.CustomPolicyCheck,
+		ParallelApplyEnabled:            parallelApplyEnabled,
+		ParallelPlanEnabled:             parallelPlanEnabled,
+		ParallelPolicyCheckEnabled:      parallelPlanEnabled,
+		DependsOn:                       projCfg.DependsOn,
+		AutoplanEnabled:                 projCfg.AutoplanEnabled,
+		AutoplanWhenModified:            projCfg.AutoplanWhenModified,
+		Steps:                           steps,
+		RequiresAtlantisManagedPlanFile: requiresAtlantisManagedPlanFile(projCfg.Workflow),
+		HeadRepo:                        ctx.HeadRepo,
+		Log:                             ctx.Log,
+		Scope:                           scope,
+		ProjectPlanStatus:               projectPlanStatus,
+		ProjectPolicyStatus:             projectPolicyStatus,
+		Pull:                            ctx.Pull,
+		ProjectName:                     projCfg.Name,
+		PlanRequirements:                projCfg.PlanRequirements,
+		ApplyRequirements:               projCfg.ApplyRequirements,
+		ImportRequirements:              projCfg.ImportRequirements,
+		RePlanCmd:                       planCmd,
+		RepoRelDir:                      projCfg.RepoRelDir,
+		RepoConfigVersion:               projCfg.RepoCfgVersion,
+		TerraformDistribution:           projCfg.TerraformDistribution,
+		TerraformVersion:                projCfg.TerraformVersion,
+		User:                            ctx.User,
+		Verbose:                         verbose,
+		Workspace:                       projCfg.Workspace,
+		PolicySets:                      policySets,
+		PolicySetTarget:                 ctx.PolicySet,
+		ClearPolicyApproval:             ctx.ClearPolicyApproval,
+		PullReqStatus:                   pullReqStatus,
+		PullStatus:                      pullStatus,
+		JobID:                           uuid.New().String(),
+		ExecutionOrderGroup:             projCfg.ExecutionOrderGroup,
+		AbortOnExecutionOrderFail:       abortOnExecutionOrderFail,
+		SilencePRComments:               projCfg.SilencePRComments,
+		TeamAllowlistChecker:            teamAllowlistChecker,
+		API:                             ctx.API,
+		SkipPRRequirements:              ctx.SkipPRRequirements,
+		RunPolicyChecks:                 ctx.RunPolicyChecks,
+		SuppressVCSStatus:               ctx.SuppressVCSStatus,
+		SuppressJobOutput:               ctx.SuppressJobOutput,
+		SuppressApplyWebhooks:           ctx.SuppressApplyWebhooks,
+		FailOnMissingDependencies:       ctx.FailOnMissingDependencies,
 	}
 }
 
@@ -349,3 +350,30 @@ func escapeArgs(args []string) []string {
 	}
 	return escaped
 }
+
+// requiresAtlantisManagedPlanFile reports whether Atlantis owns the convention
+// plan artifact (<workspace>.tfplan) for this workflow. That is true when the
+// workflow uses the built-in plan step (Atlantis writes the file) or the
+// built-in apply step (Atlantis reads it). A workflow built only from custom
+// run steps writes its plan wherever the user's commands choose, so Atlantis
+// must not require, hash, or delete a convention plan file for it.
+func requiresAtlantisManagedPlanFile(workflow valid.Workflow) bool {
+	return hasAtlantisManagedPlanStep(workflow.Plan.Steps) || hasAtlantisManagedApplyStep(workflow.Apply.Steps)
+}
+
+func hasAtlantisManagedPlanStep(steps []valid.Step) bool {
+	return hasStepNamed(steps, "plan")
+}
+
+func hasAtlantisManagedApplyStep(steps []valid.Step) bool {
+	return hasStepNamed(steps, "apply")
+}
+
+func hasStepNamed(steps []valid.Step, name string) bool {
+	for _, step := range steps {
+		if step.StepName == name {
+			return true
+		}
+	}
+	return false
+}

server/events/project_command_runner.go

@@ -935,13 +935,21 @@ func (p *DefaultProjectCommandRunner) doApply(ctx command.ProjectContext) (apply
 		return "", "", "", err
 	}
 
+	// Workflows assembled only from custom run steps manage their own plan
+	// artifact, so Atlantis cannot require or hash a convention plan file for
+	// them. Their durable plan state is still validated.
+	managedPlanFile := requiresManagedPlanFileForApply(ctx)
 	if p.ApplyPlanValidator != nil {
-		if err := p.ApplyPlanValidator.ValidateProjectPlan(ctx, absPath); err != nil {
+		if managedPlanFile {
+			if err := p.ApplyPlanValidator.ValidateProjectPlan(ctx, absPath); err != nil {
+				return "", "", "", err
+			}
+		} else if err := p.ApplyPlanValidator.ValidateProjectPlanStatus(ctx); err != nil {
 			return "", "", "", err
 		}
 	}
 	_, usingDefaultApplyPlanValidator := p.ApplyPlanValidator.(*DefaultApplyPlanValidator)
-	if ctx.CommandName == command.Apply && ctx.ExpectedPlanHash == "" && usingDefaultApplyPlanValidator {
+	if ctx.CommandName == command.Apply && managedPlanFile && ctx.ExpectedPlanHash == "" && usingDefaultApplyPlanValidator {
 		planPath, err := safePlanFilePath(ctx, absPath)
 		if err != nil {
 			return "", "", "", err
@@ -1201,3 +1209,12 @@ func getMissingPolicySetNames(policySets []valid.PolicySet, receivedCount int) [
 	}
 	return missing
 }
+
+// requiresManagedPlanFileForApply reports whether this apply must consume the
+// Atlantis convention plan artifact. It fails closed: the steps being executed
+// are authoritative, so a context that never had
+// RequiresAtlantisManagedPlanFile populated still validates the plan file when a
+// built-in apply step will read it.
+func requiresManagedPlanFileForApply(ctx command.ProjectContext) bool {
+	return ctx.RequiresAtlantisManagedPlanFile || hasAtlantisManagedApplyStep(ctx.Steps)
+}