Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ RUN set -ex; \
curl -1sSLf "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- /usr/local/bin; \
# Install toolchain used with Atmos \
apt-get -y install --no-install-recommends terraform kubectl helmfile helm; \
# Install the helm-diff plugin required by Helmfile
helm plugin install https://github.com/databus23/helm-diff; \
# Install the helm-diff plugin required by Helmfile.
# Helm 4 requires --verify=false because helm-diff does not ship .prov signature files.
helm plugin install --verify=false https://github.com/databus23/helm-diff; \
# Clean up the package lists to keep the image clean
rm -rf /var/lib/apt/lists/*

Expand Down
3 changes: 2 additions & 1 deletion internal/exec/describe_affected_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ func setupDescribeAffectedTest(t *testing.T) (atmosConfig schema.AtmosConfigurat
PreserveTimes: false,
PreserveOwner: false,
Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) {
if strings.Contains(src, "node_modules") {
if strings.Contains(src, "node_modules") ||
strings.Contains(src, ".terraform") {
return true, nil
}
isSocket, err := u.IsSocket(src)
Expand Down
171 changes: 147 additions & 24 deletions internal/exec/stack_processor_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/go-viper/mapstructure/v2"
"github.com/pkg/errors"
"github.com/santhosh-tekuri/jsonschema/v5"
"gopkg.in/yaml.v3"

errUtils "github.com/cloudposse/atmos/errors"
cfg "github.com/cloudposse/atmos/pkg/config"
Expand Down Expand Up @@ -122,6 +123,48 @@ func buildLocalsResult(rawConfig map[string]any, localsCtx *LocalsContext) *extr
return result
}

// processTemplatesInSection processes Go templates in a section (settings, vars, or env) using the provided context.
// This allows sections to reference resolved locals and other processed sections.
// Returns the processed section or an error if template processing fails.
// IMPORTANT: Uses ignoreMissingTemplateValues=false so that templates referencing values not in the
// current context (like {{ .atmos_component }}) will fail, and the caller can fall back to raw values.
// This preserves those templates for later processing when the full context is available.
func processTemplatesInSection(atmosConfig *schema.AtmosConfiguration, section map[string]any, context map[string]any, filePath string) (map[string]any, error) {
defer perf.Track(atmosConfig, "exec.processTemplatesInSection")()

if len(section) == 0 {
return section, nil
}

// Convert section to YAML for template processing.
yamlStr, err := u.ConvertToYAML(section)
if err != nil {
return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to convert section to YAML: %w", err))
}

// Quick check: if no template markers, return as-is.
if !strings.Contains(yamlStr, "{{") {
return section, nil
}

// Process templates in the YAML string.
// Use ignoreMissingTemplateValues=false so templates with missing values fail
// (e.g., {{ .atmos_component }} when component context isn't available yet).
// The caller will fall back to raw values, preserving templates for later processing.
processed, err := ProcessTmpl(atmosConfig, filePath, yamlStr, context, false)
if err != nil {
return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to process templates in section: %w", err))
}

// Parse the processed YAML back to a map.
var result map[string]any
if err := yaml.Unmarshal([]byte(processed), &result); err != nil {
return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to parse processed section YAML: %w", err))
}

return result, nil
}

// extractAndAddLocalsToContext extracts locals from YAML and adds them to the template context.
// Returns the updated context and any error encountered during locals extraction.
// Note: The "locals" key in context is reserved for file-scoped locals and will override
Expand All @@ -143,7 +186,7 @@ func extractAndAddLocalsToContext(
// Locals are file-scoped and should NOT inherit across file boundaries.
// This ensures that each file only has access to its own locals.
if context != nil {
delete(context, "locals")
delete(context, cfg.LocalsSectionName)
}

extractResult, localsErr := extractLocalsFromRawYAML(atmosConfig, yamlContent, filePath)
Expand Down Expand Up @@ -184,27 +227,70 @@ func extractAndAddLocalsToContext(
context = make(map[string]any)
}

// Add settings, vars, env from the file to the context.
// These allow templates in the file (including locals) to reference these sections.
// This enables patterns like:
// settings:
// substage: dev
// Add resolved locals to the template context first.
// This allows settings/vars/env templates to reference locals.
context[cfg.LocalsSectionName] = extractResult.locals
log.Trace("Extracted and resolved locals", "file", relativeFilePath, "count", len(extractResult.locals))

// Process templates in settings, vars, env sections using the resolved locals.
// This enables bidirectional references between locals and settings:
// locals:
// domain: '{{ .settings.substage }}.example.com'
// stage: dev
// settings:
// context:
// stage_from_local: '{{ .locals.stage }}' # Now resolves to "dev"
// vars:
// setting_value: '{{ .settings.context.stage_from_local }}' # Now resolves to "dev"
//
// Note: We need to process these sections AFTER locals are resolved so they can reference .locals,
// but BEFORE adding them to the context so vars can reference the resolved settings values.
localsOnlyContext := map[string]any{cfg.LocalsSectionName: extractResult.locals}

// Process templates in settings if it contains template expressions.
if extractResult.settings != nil {
context[cfg.SettingsSectionName] = extractResult.settings
processedSettings, err := processTemplatesInSection(atmosConfig, extractResult.settings, localsOnlyContext, relativeFilePath)
if err != nil {
log.Debug("Failed to process templates in settings section", "file", relativeFilePath, "error", err)
// Fall back to raw settings on error.
context[cfg.SettingsSectionName] = extractResult.settings
} else {
context[cfg.SettingsSectionName] = processedSettings
}
}
if extractResult.vars != nil {
context[cfg.VarsSectionName] = extractResult.vars
// For vars, we need both locals and processed settings available.
varsContext := map[string]any{cfg.LocalsSectionName: extractResult.locals}
if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok {
varsContext[cfg.SettingsSectionName] = processedSettings
}
processedVars, err := processTemplatesInSection(atmosConfig, extractResult.vars, varsContext, relativeFilePath)
if err != nil {
log.Debug("Failed to process templates in vars section", "file", relativeFilePath, "error", err)
// Fall back to raw vars on error.
context[cfg.VarsSectionName] = extractResult.vars
} else {
context[cfg.VarsSectionName] = processedVars
}
}
if extractResult.env != nil {
context[cfg.EnvSectionName] = extractResult.env
// For env, we need locals, processed settings, and processed vars available.
envContext := map[string]any{cfg.LocalsSectionName: extractResult.locals}
if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok {
envContext[cfg.SettingsSectionName] = processedSettings
}
if processedVars, ok := context[cfg.VarsSectionName].(map[string]any); ok {
envContext[cfg.VarsSectionName] = processedVars
}
processedEnv, err := processTemplatesInSection(atmosConfig, extractResult.env, envContext, relativeFilePath)
if err != nil {
log.Debug("Failed to process templates in env section", "file", relativeFilePath, "error", err)
// Fall back to raw env on error.
context[cfg.EnvSectionName] = extractResult.env
} else {
context[cfg.EnvSectionName] = processedEnv
}
}

// Add resolved locals to the template context.
context["locals"] = extractResult.locals
log.Trace("Extracted and resolved locals", "file", relativeFilePath, "count", len(extractResult.locals))

return context, nil
}

Expand Down Expand Up @@ -599,6 +685,14 @@ func processYAMLConfigFileWithContextInternal(
return map[string]any{}, map[string]map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, nil, nil
}

// Track whether context was originally provided from outside (e.g., via import context).
// This is important because we should only process templates during import when:
// 1. The file has a .tmpl extension, OR
// 2. Context was explicitly passed from outside (not just extracted from the file itself).
// Without this check, files with locals/settings/vars/env sections would have their templates
// processed prematurely, before component-specific context (like atmos_component) is available.
originalContextProvided := len(context) > 0

// Extract and resolve file-scoped locals before template processing.
// Locals can reference other locals using {{ .locals.X }} syntax.
// The resolved locals are added to the template context so they're available during template processing.
Expand All @@ -625,22 +719,33 @@ func processYAMLConfigFileWithContextInternal(
stackManifestTemplatesProcessed := stackYamlConfig
stackManifestTemplatesErrorMessage := ""

// Process `Go` templates in the imported stack manifest if it has a template extension
// Files with .yaml.tmpl or .yml.tmpl extensions are always processed as templates
// Other .tmpl files are processed only when context is provided (backward compatibility)
// Process `Go` templates in the imported stack manifest if it has a template extension.
// Files with .yaml.tmpl or .yml.tmpl extensions are always processed as templates.
// Other .tmpl files are processed only when context is provided (backward compatibility).
// https://atmos.tools/core-concepts/stacks/imports#go-templates-in-imports
if !skipTemplatesProcessingInImports && (u.IsTemplateFile(filePath) || len(context) > 0) { //nolint:nestif // Template processing error handling requires conditional formatting based on context
var tmplErr error
stackManifestTemplatesProcessed, tmplErr = ProcessTmpl(atmosConfig, relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues)
if tmplErr != nil {
if atmosConfig.Logs.Level == u.LogLevelTrace || atmosConfig.Logs.Level == u.LogLevelDebug {
stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig)
}
wrappedErr := fmt.Errorf("%w: %w", errUtils.ErrInvalidStackManifest, tmplErr)
if mergeContext != nil {
return nil, nil, nil, nil, nil, nil, nil, nil, mergeContext.FormatError(wrappedErr, fmt.Sprintf("stack manifest '%s'%s", relativeFilePath, stackManifestTemplatesErrorMessage))
// If template processing failed and the only context is from file extraction
// (locals/settings/vars/env, not from an explicit import context), this is likely
// due to templates referencing component context (like {{ .atmos_component }}) that
// isn't available during import. Fall back to the raw content — these templates will
// be processed later in ProcessStacks when the full component context is available.
if !originalContextProvided {
log.Debug("Template processing deferred for file with file-extracted context only",
"file", relativeFilePath, "error", tmplErr)
stackManifestTemplatesProcessed = stackYamlConfig
} else {
if atmosConfig.Logs.Level == u.LogLevelTrace || atmosConfig.Logs.Level == u.LogLevelDebug {
stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig)
}
wrappedErr := fmt.Errorf("%w: %w", errUtils.ErrInvalidStackManifest, tmplErr)
if mergeContext != nil {
return nil, nil, nil, nil, nil, nil, nil, nil, mergeContext.FormatError(wrappedErr, fmt.Sprintf("stack manifest '%s'%s", relativeFilePath, stackManifestTemplatesErrorMessage))
}
return nil, nil, nil, nil, nil, nil, nil, nil, fmt.Errorf("%w: stack manifest '%s'\n%w%s", errUtils.ErrInvalidStackManifest, relativeFilePath, tmplErr, stackManifestTemplatesErrorMessage)
}
return nil, nil, nil, nil, nil, nil, nil, nil, fmt.Errorf("%w: stack manifest '%s'\n%w%s", errUtils.ErrInvalidStackManifest, relativeFilePath, tmplErr, stackManifestTemplatesErrorMessage)
}
}

Expand All @@ -662,6 +767,24 @@ func processYAMLConfigFileWithContextInternal(
}
}

// Store resolved file-level sections in stackConfigMap so they're available during describe_stacks.
// The context contains resolved values from extractAndAddLocalsToContext, but the YAML content
// (and thus stackConfigMap) may still have unresolved template expressions.
// By updating stackConfigMap with resolved values, we ensure templates like {{ .locals.X }}
// and {{ .vars.X }} can be resolved correctly.
if resolvedLocals, ok := context[cfg.LocalsSectionName].(map[string]any); ok && len(resolvedLocals) > 0 {
stackConfigMap[cfg.LocalsSectionName] = resolvedLocals
}
if resolvedVars, ok := context[cfg.VarsSectionName].(map[string]any); ok && len(resolvedVars) > 0 {
stackConfigMap[cfg.VarsSectionName] = resolvedVars
}
if resolvedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok && len(resolvedSettings) > 0 {
stackConfigMap[cfg.SettingsSectionName] = resolvedSettings
}
if resolvedEnv, ok := context[cfg.EnvSectionName].(map[string]any); ok && len(resolvedEnv) > 0 {
stackConfigMap[cfg.EnvSectionName] = resolvedEnv
}

// Enable provenance tracking in merge context if tracking is enabled
if atmosConfig.TrackProvenance && mergeContext != nil && len(positions) > 0 {
mergeContext.EnableProvenance()
Expand Down
88 changes: 88 additions & 0 deletions internal/exec/stack_processor_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1971,3 +1971,91 @@ func TestBuildLocalsResult_NilLocalsWithHasLocals(t *testing.T) {
// locals should be initialized to empty map, not nil.
assert.NotNil(t, result.locals, "locals should be initialized to empty map when hasLocals is true")
}

// TestAtmosProTemplateRegression tests that {{ .atmos_component }} templates in non-.tmpl files
// with settings sections don't fail during import processing.
// This is a regression test for issue where 1.205 inadvertently triggers template processing
// for imports when the file has settings/vars/env sections (due to locals feature changes).
func TestAtmosProTemplateRegression(t *testing.T) {
stacksBasePath := filepath.Join("..", "..", "tests", "fixtures", "scenarios", "atmos-pro-template-regression", "stacks")
filePath := filepath.Join(stacksBasePath, "deploy", "test.yaml")

atmosConfig := schema.AtmosConfiguration{
Logs: schema.Logs{
Level: "Info",
},
Templates: schema.Templates{
Settings: schema.TemplatesSettings{
Enabled: true,
},
},
}

// Process the stack manifest that imports the atmos-pro mixin.
// The mixin has a settings section and uses {{ .atmos_component }} templates.
// In 1.204, this worked because templates weren't processed during import for non-.tmpl files.
// In 1.205, the locals feature inadvertently triggers template processing because it adds
// settings/vars/env to the context, making len(context) > 0.
deepMergedConfig, importsConfig, stackConfigMap, tfInline, tfImports, hfInline, hfImports, err := ProcessYAMLConfigFile(
&atmosConfig,
stacksBasePath,
filePath,
map[string]map[string]any{},
nil, // No external context - this is key to the test.
false, // ignoreMissingFiles.
false, // skipTemplatesProcessingInImports.
false, // ignoreMissingTemplateValues - set to false to catch the error.
false, // skipIfMissing.
nil,
nil,
nil,
nil,
"",
)

// The test should pass - templates like {{ .atmos_component }} should NOT be processed
// during import when no external context is provided.
require.NoError(t, err, "Processing should not fail - templates should be deferred until component processing")
require.NotNil(t, deepMergedConfig)

// Suppress unused variable warnings - these are returned by ProcessYAMLConfigFile but not needed for this test.
_ = importsConfig
_ = stackConfigMap
_ = tfInline
_ = tfImports
_ = hfInline
_ = hfImports

// Verify the settings.pro section exists and contains unprocessed template strings.
settings, ok := deepMergedConfig["settings"].(map[string]any)
require.True(t, ok, "settings section should exist")

pro, ok := settings["pro"].(map[string]any)
require.True(t, ok, "settings.pro section should exist")

assert.Equal(t, true, pro["enabled"], "pro.enabled should be true")

// The template strings should be preserved (not processed) at this stage.
// They will be processed later in describe_stacks when component context is available.
pr, ok := pro["pull_request"].(map[string]any)
require.True(t, ok, "settings.pro.pull_request should exist")

opened, ok := pr["opened"].(map[string]any)
require.True(t, ok, "settings.pro.pull_request.opened should exist")

workflows, ok := opened["workflows"].(map[string]any)
require.True(t, ok, "settings.pro.pull_request.opened.workflows should exist")

planWorkflow, ok := workflows["atmos-terraform-plan.yaml"].(map[string]any)
require.True(t, ok, "atmos-terraform-plan.yaml workflow should exist")

inputs, ok := planWorkflow["inputs"].(map[string]any)
require.True(t, ok, "workflow inputs should exist")

// The component input should still contain the template string {{ .atmos_component }}
// because templates should NOT be processed during import for non-.tmpl files without explicit context.
componentInput, ok := inputs["component"].(string)
require.True(t, ok, "component input should be a string")
assert.Contains(t, componentInput, "atmos_component",
"Template {{ .atmos_component }} should be preserved during import, not processed")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
base_path: ""

stacks:
base_path: stacks
included_paths:
- "deploy/**/*"
excluded_paths:
- "**/_defaults.yaml"
name_template: "{{ .vars.stage }}-{{ .vars.environment }}"

components:
terraform:
base_path: components/terraform

templates:
settings:
enabled: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
variable "cidr" {
type = string
}

output "cidr" {
value = var.cidr
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Stack file that imports the atmos-pro mixin
import:
- mixins/atmos-pro

vars:
stage: test
environment: dev

components:
terraform:
vpc:
vars:
cidr: "10.0.0.0/16"
Loading