From cf4f31d04d290b2cd6571b2f5f8bb062d12b38e8 Mon Sep 17 00:00:00 2001 From: aknysh Date: Wed, 4 Feb 2026 13:55:36 -0500 Subject: [PATCH 1/2] fix: resolve Atmos Pro template regression with {{ .atmos_component }} in non-.tmpl files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting in Atmos 1.205, stack manifests using {{ .atmos_component }} or {{ .atmos_stack }} in non-template files fail during import with: "map has no entry for key atmos_component" The locals feature inadvertently triggered template processing for imported files by populating the template context with settings/vars/env, making len(context) > 0. Templates like {{ .atmos_component }} then failed because component context isn't available at import time. Fix: Track whether context was originally provided externally vs extracted from the file itself. When template processing fails and only file-extracted context is available, gracefully fall back to raw content — preserving templates for later resolution in ProcessStacks when full component context is available. Also adds processTemplatesInSection() to resolve {{ .locals.X }} references in individual sections (settings, vars, env) without processing the whole file, and persists resolved sections back into stackConfigMap. Additional fixes included: - Skip .terraform dirs in describe-affected test copies (dangling symlinks) - Add --verify=false to helm plugin install in Dockerfile (Helm 4 compat) Co-Authored-By: Claude Opus 4.5 --- Dockerfile | 5 +- internal/exec/describe_affected_test.go | 3 +- internal/exec/stack_processor_utils.go | 171 +++++++++++++++--- internal/exec/stack_processor_utils_test.go | 88 +++++++++ .../atmos-pro-template-regression/atmos.yaml | 17 ++ .../components/terraform/vpc/main.tf | 7 + .../stacks/deploy/test.yaml | 13 ++ .../stacks/mixins/atmos-pro.yaml | 38 ++++ 8 files changed, 315 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/scenarios/atmos-pro-template-regression/atmos.yaml create mode 100644 tests/fixtures/scenarios/atmos-pro-template-regression/components/terraform/vpc/main.tf create mode 100644 tests/fixtures/scenarios/atmos-pro-template-regression/stacks/deploy/test.yaml create mode 100644 tests/fixtures/scenarios/atmos-pro-template-regression/stacks/mixins/atmos-pro.yaml diff --git a/Dockerfile b/Dockerfile index 33bd18a5fb..18d621cf6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,8 +25,9 @@ RUN set -ex; \ curl -1sSLf "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- /usr/local/bin; \ # Install toolchain used with Atmos \ apt-get -y install --no-install-recommends terraform kubectl helmfile helm; \ - # Install the helm-diff plugin required by Helmfile - helm plugin install https://github.com/databus23/helm-diff; \ + # Install the helm-diff plugin required by Helmfile. + # Helm 4 requires --verify=false because helm-diff does not ship .prov signature files. + helm plugin install --verify=false https://github.com/databus23/helm-diff; \ # Clean up the package lists to keep the image clean rm -rf /var/lib/apt/lists/* diff --git a/internal/exec/describe_affected_test.go b/internal/exec/describe_affected_test.go index a900d46f18..8e912d99d5 100644 --- a/internal/exec/describe_affected_test.go +++ b/internal/exec/describe_affected_test.go @@ -262,7 +262,8 @@ func setupDescribeAffectedTest(t *testing.T) (atmosConfig schema.AtmosConfigurat PreserveTimes: false, PreserveOwner: false, Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { - if strings.Contains(src, "node_modules") { + if strings.Contains(src, "node_modules") || + strings.Contains(src, ".terraform") { return true, nil } isSocket, err := u.IsSocket(src) diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index ce0519f2ae..73b8a5ecc1 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -14,6 +14,7 @@ import ( "github.com/go-viper/mapstructure/v2" "github.com/pkg/errors" "github.com/santhosh-tekuri/jsonschema/v5" + "gopkg.in/yaml.v3" errUtils "github.com/cloudposse/atmos/errors" cfg "github.com/cloudposse/atmos/pkg/config" @@ -122,6 +123,48 @@ func buildLocalsResult(rawConfig map[string]any, localsCtx *LocalsContext) *extr return result } +// processTemplatesInSection processes Go templates in a section (settings, vars, or env) using the provided context. +// This allows sections to reference resolved locals and other processed sections. +// Returns the processed section or an error if template processing fails. +// IMPORTANT: Uses ignoreMissingTemplateValues=false so that templates referencing values not in the +// current context (like {{ .atmos_component }}) will fail, and the caller can fall back to raw values. +// This preserves those templates for later processing when the full context is available. +func processTemplatesInSection(atmosConfig *schema.AtmosConfiguration, section map[string]any, context map[string]any, filePath string) (map[string]any, error) { + defer perf.Track(atmosConfig, "exec.processTemplatesInSection")() + + if len(section) == 0 { + return section, nil + } + + // Convert section to YAML for template processing. + yamlStr, err := u.ConvertToYAML(section) + if err != nil { + return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to convert section to YAML: %w", err)) + } + + // Quick check: if no template markers, return as-is. + if !strings.Contains(yamlStr, "{{") { + return section, nil + } + + // Process templates in the YAML string. + // Use ignoreMissingTemplateValues=false so templates with missing values fail + // (e.g., {{ .atmos_component }} when component context isn't available yet). + // The caller will fall back to raw values, preserving templates for later processing. + processed, err := ProcessTmpl(atmosConfig, filePath, yamlStr, context, false) + if err != nil { + return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to process templates in section: %w", err)) + } + + // Parse the processed YAML back to a map. + var result map[string]any + if err := yaml.Unmarshal([]byte(processed), &result); err != nil { + return nil, stderrors.Join(errUtils.ErrInvalidStackManifest, fmt.Errorf("failed to parse processed section YAML: %w", err)) + } + + return result, nil +} + // extractAndAddLocalsToContext extracts locals from YAML and adds them to the template context. // Returns the updated context and any error encountered during locals extraction. // Note: The "locals" key in context is reserved for file-scoped locals and will override @@ -143,7 +186,7 @@ func extractAndAddLocalsToContext( // Locals are file-scoped and should NOT inherit across file boundaries. // This ensures that each file only has access to its own locals. if context != nil { - delete(context, "locals") + delete(context, cfg.LocalsSectionName) } extractResult, localsErr := extractLocalsFromRawYAML(atmosConfig, yamlContent, filePath) @@ -184,27 +227,70 @@ func extractAndAddLocalsToContext( context = make(map[string]any) } - // Add settings, vars, env from the file to the context. - // These allow templates in the file (including locals) to reference these sections. - // This enables patterns like: - // settings: - // substage: dev + // Add resolved locals to the template context first. + // This allows settings/vars/env templates to reference locals. + context[cfg.LocalsSectionName] = extractResult.locals + log.Trace("Extracted and resolved locals", "file", relativeFilePath, "count", len(extractResult.locals)) + + // Process templates in settings, vars, env sections using the resolved locals. + // This enables bidirectional references between locals and settings: // locals: - // domain: '{{ .settings.substage }}.example.com' + // stage: dev + // settings: + // context: + // stage_from_local: '{{ .locals.stage }}' # Now resolves to "dev" + // vars: + // setting_value: '{{ .settings.context.stage_from_local }}' # Now resolves to "dev" + // + // Note: We need to process these sections AFTER locals are resolved so they can reference .locals, + // but BEFORE adding them to the context so vars can reference the resolved settings values. + localsOnlyContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + + // Process templates in settings if it contains template expressions. if extractResult.settings != nil { - context[cfg.SettingsSectionName] = extractResult.settings + processedSettings, err := processTemplatesInSection(atmosConfig, extractResult.settings, localsOnlyContext, relativeFilePath) + if err != nil { + log.Debug("Failed to process templates in settings section", "file", relativeFilePath, "error", err) + // Fall back to raw settings on error. + context[cfg.SettingsSectionName] = extractResult.settings + } else { + context[cfg.SettingsSectionName] = processedSettings + } } if extractResult.vars != nil { - context[cfg.VarsSectionName] = extractResult.vars + // For vars, we need both locals and processed settings available. + varsContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok { + varsContext[cfg.SettingsSectionName] = processedSettings + } + processedVars, err := processTemplatesInSection(atmosConfig, extractResult.vars, varsContext, relativeFilePath) + if err != nil { + log.Debug("Failed to process templates in vars section", "file", relativeFilePath, "error", err) + // Fall back to raw vars on error. + context[cfg.VarsSectionName] = extractResult.vars + } else { + context[cfg.VarsSectionName] = processedVars + } } if extractResult.env != nil { - context[cfg.EnvSectionName] = extractResult.env + // For env, we need locals, processed settings, and processed vars available. + envContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok { + envContext[cfg.SettingsSectionName] = processedSettings + } + if processedVars, ok := context[cfg.VarsSectionName].(map[string]any); ok { + envContext[cfg.VarsSectionName] = processedVars + } + processedEnv, err := processTemplatesInSection(atmosConfig, extractResult.env, envContext, relativeFilePath) + if err != nil { + log.Debug("Failed to process templates in env section", "file", relativeFilePath, "error", err) + // Fall back to raw env on error. + context[cfg.EnvSectionName] = extractResult.env + } else { + context[cfg.EnvSectionName] = processedEnv + } } - // Add resolved locals to the template context. - context["locals"] = extractResult.locals - log.Trace("Extracted and resolved locals", "file", relativeFilePath, "count", len(extractResult.locals)) - return context, nil } @@ -599,6 +685,14 @@ func processYAMLConfigFileWithContextInternal( return map[string]any{}, map[string]map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, map[string]any{}, nil, nil } + // Track whether context was originally provided from outside (e.g., via import context). + // This is important because we should only process templates during import when: + // 1. The file has a .tmpl extension, OR + // 2. Context was explicitly passed from outside (not just extracted from the file itself). + // Without this check, files with locals/settings/vars/env sections would have their templates + // processed prematurely, before component-specific context (like atmos_component) is available. + originalContextProvided := len(context) > 0 + // Extract and resolve file-scoped locals before template processing. // Locals can reference other locals using {{ .locals.X }} syntax. // The resolved locals are added to the template context so they're available during template processing. @@ -625,22 +719,33 @@ func processYAMLConfigFileWithContextInternal( stackManifestTemplatesProcessed := stackYamlConfig stackManifestTemplatesErrorMessage := "" - // Process `Go` templates in the imported stack manifest if it has a template extension - // Files with .yaml.tmpl or .yml.tmpl extensions are always processed as templates - // Other .tmpl files are processed only when context is provided (backward compatibility) + // Process `Go` templates in the imported stack manifest if it has a template extension. + // Files with .yaml.tmpl or .yml.tmpl extensions are always processed as templates. + // Other .tmpl files are processed only when context is provided (backward compatibility). // https://atmos.tools/core-concepts/stacks/imports#go-templates-in-imports if !skipTemplatesProcessingInImports && (u.IsTemplateFile(filePath) || len(context) > 0) { //nolint:nestif // Template processing error handling requires conditional formatting based on context var tmplErr error stackManifestTemplatesProcessed, tmplErr = ProcessTmpl(atmosConfig, relativeFilePath, stackYamlConfig, context, ignoreMissingTemplateValues) if tmplErr != nil { - if atmosConfig.Logs.Level == u.LogLevelTrace || atmosConfig.Logs.Level == u.LogLevelDebug { - stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) - } - wrappedErr := fmt.Errorf("%w: %w", errUtils.ErrInvalidStackManifest, tmplErr) - if mergeContext != nil { - return nil, nil, nil, nil, nil, nil, nil, nil, mergeContext.FormatError(wrappedErr, fmt.Sprintf("stack manifest '%s'%s", relativeFilePath, stackManifestTemplatesErrorMessage)) + // If template processing failed and the only context is from file extraction + // (locals/settings/vars/env, not from an explicit import context), this is likely + // due to templates referencing component context (like {{ .atmos_component }}) that + // isn't available during import. Fall back to the raw content — these templates will + // be processed later in ProcessStacks when the full component context is available. + if !originalContextProvided { + log.Debug("Template processing deferred for file with file-extracted context only", + "file", relativeFilePath, "error", tmplErr) + stackManifestTemplatesProcessed = stackYamlConfig + } else { + if atmosConfig.Logs.Level == u.LogLevelTrace || atmosConfig.Logs.Level == u.LogLevelDebug { + stackManifestTemplatesErrorMessage = fmt.Sprintf("\n\n%s", stackYamlConfig) + } + wrappedErr := fmt.Errorf("%w: %w", errUtils.ErrInvalidStackManifest, tmplErr) + if mergeContext != nil { + return nil, nil, nil, nil, nil, nil, nil, nil, mergeContext.FormatError(wrappedErr, fmt.Sprintf("stack manifest '%s'%s", relativeFilePath, stackManifestTemplatesErrorMessage)) + } + return nil, nil, nil, nil, nil, nil, nil, nil, fmt.Errorf("%w: stack manifest '%s'\n%w%s", errUtils.ErrInvalidStackManifest, relativeFilePath, tmplErr, stackManifestTemplatesErrorMessage) } - return nil, nil, nil, nil, nil, nil, nil, nil, fmt.Errorf("%w: stack manifest '%s'\n%w%s", errUtils.ErrInvalidStackManifest, relativeFilePath, tmplErr, stackManifestTemplatesErrorMessage) } } @@ -662,6 +767,24 @@ func processYAMLConfigFileWithContextInternal( } } + // Store resolved file-level sections in stackConfigMap so they're available during describe_stacks. + // The context contains resolved values from extractAndAddLocalsToContext, but the YAML content + // (and thus stackConfigMap) may still have unresolved template expressions. + // By updating stackConfigMap with resolved values, we ensure templates like {{ .locals.X }} + // and {{ .vars.X }} can be resolved correctly. + if resolvedLocals, ok := context[cfg.LocalsSectionName].(map[string]any); ok && len(resolvedLocals) > 0 { + stackConfigMap[cfg.LocalsSectionName] = resolvedLocals + } + if resolvedVars, ok := context[cfg.VarsSectionName].(map[string]any); ok && len(resolvedVars) > 0 { + stackConfigMap[cfg.VarsSectionName] = resolvedVars + } + if resolvedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok && len(resolvedSettings) > 0 { + stackConfigMap[cfg.SettingsSectionName] = resolvedSettings + } + if resolvedEnv, ok := context[cfg.EnvSectionName].(map[string]any); ok && len(resolvedEnv) > 0 { + stackConfigMap[cfg.EnvSectionName] = resolvedEnv + } + // Enable provenance tracking in merge context if tracking is enabled if atmosConfig.TrackProvenance && mergeContext != nil && len(positions) > 0 { mergeContext.EnableProvenance() diff --git a/internal/exec/stack_processor_utils_test.go b/internal/exec/stack_processor_utils_test.go index de73f55481..01dff90238 100644 --- a/internal/exec/stack_processor_utils_test.go +++ b/internal/exec/stack_processor_utils_test.go @@ -1971,3 +1971,91 @@ func TestBuildLocalsResult_NilLocalsWithHasLocals(t *testing.T) { // locals should be initialized to empty map, not nil. assert.NotNil(t, result.locals, "locals should be initialized to empty map when hasLocals is true") } + +// TestAtmosProTemplateRegression tests that {{ .atmos_component }} templates in non-.tmpl files +// with settings sections don't fail during import processing. +// This is a regression test for issue where 1.205 inadvertently triggers template processing +// for imports when the file has settings/vars/env sections (due to locals feature changes). +func TestAtmosProTemplateRegression(t *testing.T) { + stacksBasePath := filepath.Join("..", "..", "tests", "fixtures", "scenarios", "atmos-pro-template-regression", "stacks") + filePath := filepath.Join(stacksBasePath, "deploy", "test.yaml") + + atmosConfig := schema.AtmosConfiguration{ + Logs: schema.Logs{ + Level: "Info", + }, + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + // Process the stack manifest that imports the atmos-pro mixin. + // The mixin has a settings section and uses {{ .atmos_component }} templates. + // In 1.204, this worked because templates weren't processed during import for non-.tmpl files. + // In 1.205, the locals feature inadvertently triggers template processing because it adds + // settings/vars/env to the context, making len(context) > 0. + deepMergedConfig, importsConfig, stackConfigMap, tfInline, tfImports, hfInline, hfImports, err := ProcessYAMLConfigFile( + &atmosConfig, + stacksBasePath, + filePath, + map[string]map[string]any{}, + nil, // No external context - this is key to the test. + false, // ignoreMissingFiles. + false, // skipTemplatesProcessingInImports. + false, // ignoreMissingTemplateValues - set to false to catch the error. + false, // skipIfMissing. + nil, + nil, + nil, + nil, + "", + ) + + // The test should pass - templates like {{ .atmos_component }} should NOT be processed + // during import when no external context is provided. + require.NoError(t, err, "Processing should not fail - templates should be deferred until component processing") + require.NotNil(t, deepMergedConfig) + + // Suppress unused variable warnings - these are returned by ProcessYAMLConfigFile but not needed for this test. + _ = importsConfig + _ = stackConfigMap + _ = tfInline + _ = tfImports + _ = hfInline + _ = hfImports + + // Verify the settings.pro section exists and contains unprocessed template strings. + settings, ok := deepMergedConfig["settings"].(map[string]any) + require.True(t, ok, "settings section should exist") + + pro, ok := settings["pro"].(map[string]any) + require.True(t, ok, "settings.pro section should exist") + + assert.Equal(t, true, pro["enabled"], "pro.enabled should be true") + + // The template strings should be preserved (not processed) at this stage. + // They will be processed later in describe_stacks when component context is available. + pr, ok := pro["pull_request"].(map[string]any) + require.True(t, ok, "settings.pro.pull_request should exist") + + opened, ok := pr["opened"].(map[string]any) + require.True(t, ok, "settings.pro.pull_request.opened should exist") + + workflows, ok := opened["workflows"].(map[string]any) + require.True(t, ok, "settings.pro.pull_request.opened.workflows should exist") + + planWorkflow, ok := workflows["atmos-terraform-plan.yaml"].(map[string]any) + require.True(t, ok, "atmos-terraform-plan.yaml workflow should exist") + + inputs, ok := planWorkflow["inputs"].(map[string]any) + require.True(t, ok, "workflow inputs should exist") + + // The component input should still contain the template string {{ .atmos_component }} + // because templates should NOT be processed during import for non-.tmpl files without explicit context. + componentInput, ok := inputs["component"].(string) + require.True(t, ok, "component input should be a string") + assert.Contains(t, componentInput, "atmos_component", + "Template {{ .atmos_component }} should be preserved during import, not processed") +} diff --git a/tests/fixtures/scenarios/atmos-pro-template-regression/atmos.yaml b/tests/fixtures/scenarios/atmos-pro-template-regression/atmos.yaml new file mode 100644 index 0000000000..fc4cfd871c --- /dev/null +++ b/tests/fixtures/scenarios/atmos-pro-template-regression/atmos.yaml @@ -0,0 +1,17 @@ +base_path: "" + +stacks: + base_path: stacks + included_paths: + - "deploy/**/*" + excluded_paths: + - "**/_defaults.yaml" + name_template: "{{ .vars.stage }}-{{ .vars.environment }}" + +components: + terraform: + base_path: components/terraform + +templates: + settings: + enabled: true diff --git a/tests/fixtures/scenarios/atmos-pro-template-regression/components/terraform/vpc/main.tf b/tests/fixtures/scenarios/atmos-pro-template-regression/components/terraform/vpc/main.tf new file mode 100644 index 0000000000..b5730ac8c1 --- /dev/null +++ b/tests/fixtures/scenarios/atmos-pro-template-regression/components/terraform/vpc/main.tf @@ -0,0 +1,7 @@ +variable "cidr" { + type = string +} + +output "cidr" { + value = var.cidr +} diff --git a/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/deploy/test.yaml b/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/deploy/test.yaml new file mode 100644 index 0000000000..f0242fc961 --- /dev/null +++ b/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/deploy/test.yaml @@ -0,0 +1,13 @@ +# Stack file that imports the atmos-pro mixin +import: + - mixins/atmos-pro + +vars: + stage: test + environment: dev + +components: + terraform: + vpc: + vars: + cidr: "10.0.0.0/16" diff --git a/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/mixins/atmos-pro.yaml b/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/mixins/atmos-pro.yaml new file mode 100644 index 0000000000..b4f197f2ea --- /dev/null +++ b/tests/fixtures/scenarios/atmos-pro-template-regression/stacks/mixins/atmos-pro.yaml @@ -0,0 +1,38 @@ +# This mixin file tests the regression where {{ .atmos_component }} fails +# when the file has a locals section (which populates context in 1.205). +# In 1.204, templates were not processed during import for non-.tmpl files without context. +# In 1.205, the locals feature inadvertently triggers template processing by adding +# settings/vars/env to the context, causing this error: +# "map has no entry for key atmos_component" + +# The presence of this locals section triggers the bug in 1.205 +# because extractAndAddLocalsToContext adds settings/vars/env to context +# which makes len(context) > 0, triggering template processing. +locals: + pro_enabled: true + +plan-wf-config: &plan-wf-config + atmos-terraform-plan.yaml: + inputs: + component: "{{ .atmos_component }}" + stack: "{{ .atmos_stack }}" + +apply-wf-config: &apply-wf-config + atmos-terraform-apply.yaml: + inputs: + component: "{{ .atmos_component }}" + stack: "{{ .atmos_stack }}" + github_environment: "{{ .atmos_stack }}" + +settings: + pro: + enabled: true + pull_request: + opened: + workflows: *plan-wf-config + synchronize: + workflows: *plan-wf-config + reopened: + workflows: *plan-wf-config + merged: + workflows: *apply-wf-config From 4162ad50c5c85f7bf58b0f37fd9928f1aa927fe5 Mon Sep 17 00:00:00 2001 From: aknysh Date: Wed, 4 Feb 2026 15:09:10 -0500 Subject: [PATCH 2/2] fix: include external import context in section template processing and improve test precision - Seed section contexts (settings/vars/env) with external import context so templates referencing import-provided values resolve during section processing, not just during full-file processing - Change assert.Contains to assert.Equal for template preservation assertions to verify exact template strings are preserved verbatim - Add TestExtractAndAddLocalsToContext_ExternalContext with 3 subtests covering external context resolution, cross-section propagation, and graceful fallback when context is insufficient Co-Authored-By: Claude Opus 4.5 --- internal/exec/stack_processor_utils.go | 23 +- internal/exec/stack_processor_utils_test.go | 545 +++++++++++++++++++- 2 files changed, 562 insertions(+), 6 deletions(-) diff --git a/internal/exec/stack_processor_utils.go b/internal/exec/stack_processor_utils.go index 73b8a5ecc1..ed46a57e5e 100644 --- a/internal/exec/stack_processor_utils.go +++ b/internal/exec/stack_processor_utils.go @@ -244,7 +244,14 @@ func extractAndAddLocalsToContext( // // Note: We need to process these sections AFTER locals are resolved so they can reference .locals, // but BEFORE adding them to the context so vars can reference the resolved settings values. - localsOnlyContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + // Seed section context with any external import context so that settings/vars/env + // templates referencing import-provided values can resolve during section processing. + // Locals are always overridden to ensure file-scoped locality. + localsOnlyContext := map[string]any{} + for k, v := range context { + localsOnlyContext[k] = v + } + localsOnlyContext[cfg.LocalsSectionName] = extractResult.locals // Process templates in settings if it contains template expressions. if extractResult.settings != nil { @@ -258,8 +265,11 @@ func extractAndAddLocalsToContext( } } if extractResult.vars != nil { - // For vars, we need both locals and processed settings available. - varsContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + // For vars, we need locals, external context, and processed settings available. + varsContext := map[string]any{} + for k, v := range localsOnlyContext { + varsContext[k] = v + } if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok { varsContext[cfg.SettingsSectionName] = processedSettings } @@ -273,8 +283,11 @@ func extractAndAddLocalsToContext( } } if extractResult.env != nil { - // For env, we need locals, processed settings, and processed vars available. - envContext := map[string]any{cfg.LocalsSectionName: extractResult.locals} + // For env, we need locals, external context, processed settings, and processed vars. + envContext := map[string]any{} + for k, v := range localsOnlyContext { + envContext[k] = v + } if processedSettings, ok := context[cfg.SettingsSectionName].(map[string]any); ok { envContext[cfg.SettingsSectionName] = processedSettings } diff --git a/internal/exec/stack_processor_utils_test.go b/internal/exec/stack_processor_utils_test.go index 01dff90238..bdc0c01f83 100644 --- a/internal/exec/stack_processor_utils_test.go +++ b/internal/exec/stack_processor_utils_test.go @@ -2,6 +2,7 @@ package exec import ( "errors" + "os" "path/filepath" "sort" "testing" @@ -1972,6 +1973,548 @@ func TestBuildLocalsResult_NilLocalsWithHasLocals(t *testing.T) { assert.NotNil(t, result.locals, "locals should be initialized to empty map when hasLocals is true") } +// TestProcessTemplatesInSection tests the processTemplatesInSection helper function. +func TestProcessTemplatesInSection(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{} + + t.Run("empty section returns as-is", func(t *testing.T) { + result, err := processTemplatesInSection(atmosConfig, map[string]any{}, map[string]any{}, "test.yaml") + require.NoError(t, err) + assert.Empty(t, result) + }) + + t.Run("nil section returns as-is", func(t *testing.T) { + var section map[string]any + result, err := processTemplatesInSection(atmosConfig, section, map[string]any{}, "test.yaml") + require.NoError(t, err) + assert.Nil(t, result) + }) + + t.Run("section without templates returns as-is", func(t *testing.T) { + section := map[string]any{ + "key": "plain-value", + "num": 42, + } + result, err := processTemplatesInSection(atmosConfig, section, map[string]any{}, "test.yaml") + require.NoError(t, err) + assert.Equal(t, "plain-value", result["key"]) + assert.Equal(t, 42, result["num"]) + }) + + t.Run("resolves templates with locals context", func(t *testing.T) { + section := map[string]any{ + "stage_label": "{{ .locals.stage }}-app", + } + context := map[string]any{ + "locals": map[string]any{"stage": "dev"}, + } + result, err := processTemplatesInSection(atmosConfig, section, context, "test.yaml") + require.NoError(t, err) + assert.Equal(t, "dev-app", result["stage_label"]) + }) + + t.Run("returns error on missing template values", func(t *testing.T) { + section := map[string]any{ + "value": "{{ .missing_key }}", + } + context := map[string]any{ + "locals": map[string]any{"stage": "dev"}, + } + _, err := processTemplatesInSection(atmosConfig, section, context, "test.yaml") + require.Error(t, err, "Should fail when template references missing context value") + assert.True(t, errors.Is(err, errUtils.ErrInvalidStackManifest)) + }) + + t.Run("resolves nested template values", func(t *testing.T) { + section := map[string]any{ + "nested": map[string]any{ + "name": "{{ .locals.namespace }}-{{ .locals.env }}", + }, + } + context := map[string]any{ + "locals": map[string]any{"namespace": "acme", "env": "prod"}, + } + result, err := processTemplatesInSection(atmosConfig, section, context, "test.yaml") + require.NoError(t, err) + nested, ok := result["nested"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "acme-prod", nested["name"]) + }) +} + +// TestExtractAndAddLocalsToContext_SectionProcessing tests the template processing pipeline +// in extractAndAddLocalsToContext for settings, vars, and env sections. +func TestExtractAndAddLocalsToContext_SectionProcessing(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + t.Run("settings templates resolved with locals", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +settings: + label: '{{ .locals.stage }}-config' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok, "settings should exist in context") + assert.Equal(t, "dev-config", settings["label"]) + }) + + t.Run("vars templates resolved with locals and settings", func(t *testing.T) { + yamlContent := ` +locals: + env: prod +settings: + region: us-east-1 +vars: + deploy_env: '{{ .locals.env }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + vars, ok := ctx["vars"].(map[string]any) + require.True(t, ok, "vars should exist in context") + assert.Equal(t, "prod", vars["deploy_env"]) + }) + + t.Run("env templates resolved with locals settings and vars", func(t *testing.T) { + yamlContent := ` +locals: + app: myapp +env: + APP_NAME: '{{ .locals.app }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + env, ok := ctx["env"].(map[string]any) + require.True(t, ok, "env should exist in context") + assert.Equal(t, "myapp", env["APP_NAME"]) + }) + + t.Run("settings fallback to raw on template error", func(t *testing.T) { + // Settings has a template referencing a value not in locals context. + // processTemplatesInSection should fail and fall back to raw settings. + yamlContent := ` +locals: + stage: dev +settings: + component: '{{ .atmos_component }}' + static: plain-value +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok, "settings should exist in context (raw fallback)") + // Raw fallback means the template string is preserved. + assert.Equal(t, "{{ .atmos_component }}", settings["component"].(string)) + assert.Equal(t, "plain-value", settings["static"]) + }) + + t.Run("vars fallback to raw on template error", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +vars: + stack: '{{ .atmos_stack }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + vars, ok := ctx["vars"].(map[string]any) + require.True(t, ok, "vars should exist in context (raw fallback)") + assert.Equal(t, "{{ .atmos_stack }}", vars["stack"].(string)) + }) + + t.Run("env fallback to raw on template error", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +env: + COMPONENT: '{{ .atmos_component }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + env, ok := ctx["env"].(map[string]any) + require.True(t, ok, "env should exist in context (raw fallback)") + assert.Equal(t, "{{ .atmos_component }}", env["COMPONENT"].(string)) + }) + + t.Run("clears inherited locals from parent context", func(t *testing.T) { + yamlContent := ` +locals: + own_local: mine +` + parentContext := map[string]any{ + "locals": map[string]any{"inherited": "should-be-cleared"}, + } + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", parentContext) + require.NoError(t, err) + locals, ok := ctx["locals"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "mine", locals["own_local"]) + _, hasInherited := locals["inherited"] + assert.False(t, hasInherited, "inherited locals should be cleared") + }) + + t.Run("full pipeline settings vars env with locals", func(t *testing.T) { + yamlContent := ` +locals: + ns: acme + stage: dev +settings: + context: + namespace: '{{ .locals.ns }}' +vars: + env_name: '{{ .locals.stage }}' +env: + NAMESPACE: '{{ .locals.ns }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok) + settingsCtx, ok := settings["context"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "acme", settingsCtx["namespace"]) + + vars, ok := ctx["vars"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dev", vars["env_name"]) + + env, ok := ctx["env"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "acme", env["NAMESPACE"]) + }) +} + +// TestExtractAndAddLocalsToContext_ExternalContext verifies that external import context +// is included during section template processing, enabling settings/vars/env to reference +// import-provided values alongside locals. +func TestExtractAndAddLocalsToContext_ExternalContext(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + t.Run("settings resolves with external context", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +settings: + label: '{{ .locals.stage }}-{{ .tenant }}' +` + externalCtx := map[string]any{"tenant": "acme"} + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", externalCtx) + require.NoError(t, err) + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok, "settings should exist in context") + assert.Equal(t, "dev-acme", settings["label"], + "settings should resolve templates using both locals and external context") + }) + + t.Run("vars resolves with external context and processed settings", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +settings: + env_label: '{{ .locals.stage }}-{{ .region }}' +vars: + full_label: '{{ .settings.env_label }}' +` + externalCtx := map[string]any{"region": "us-east-1"} + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", externalCtx) + require.NoError(t, err) + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dev-us-east-1", settings["env_label"]) + + vars, ok := ctx["vars"].(map[string]any) + require.True(t, ok, "vars should exist in context") + assert.Equal(t, "dev-us-east-1", vars["full_label"], + "vars should resolve settings that were resolved with external context") + }) + + t.Run("settings falls back when external context is insufficient", func(t *testing.T) { + yamlContent := ` +locals: + stage: dev +settings: + label: '{{ .locals.stage }}-{{ .missing_var }}' +` + externalCtx := map[string]any{"tenant": "acme"} + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", externalCtx) + require.NoError(t, err) + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok, "settings should exist (raw fallback)") + assert.Equal(t, "{{ .locals.stage }}-{{ .missing_var }}", settings["label"], + "settings should fall back to raw when template references unavailable values") + }) +} + +// TestProcessYAMLConfigFile_OriginalContextFallback tests the graceful fallback +// when template processing fails and only file-extracted context is available. +func TestProcessYAMLConfigFile_OriginalContextFallback(t *testing.T) { + // Create a temporary YAML file with locals and {{ .atmos_component }} templates. + // When a section mixes resolvable ({{ .locals.X }}) and unresolvable ({{ .atmos_component }}) + // templates, the section-level processing fails and falls back to raw values. + // The whole-file template processing also fails and falls back to raw content. + tmpDir := t.TempDir() + yamlContent := ` +locals: + stage: dev +settings: + component: "{{ .atmos_component }}" + stage_label: "{{ .locals.stage }}-label" +vars: + stack: "{{ .atmos_stack }}" + env_name: "{{ .locals.stage }}" +` + filePath := filepath.Join(tmpDir, "test.yaml") + err := os.WriteFile(filePath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + atmosConfig := schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + // Process with no external context (nil) — file-extracted context only. + // Template processing will fail on {{ .atmos_component }} because it's not in context. + // The fallback should return raw content preserving ALL templates for later processing. + deepMergedConfig, importsConfig, stackConfigMap, tfInline, tfImports, _, _, err := ProcessYAMLConfigFile( + &atmosConfig, + tmpDir, + filePath, + map[string]map[string]any{}, + nil, // No external context. + false, // ignoreMissingFiles. + false, // skipTemplatesProcessingInImports. + false, // ignoreMissingTemplateValues. + false, // skipIfMissing. + nil, + nil, + nil, + nil, + "", + ) + + require.NoError(t, err, "Should not fail — fallback to raw content when only file-extracted context") + require.NotNil(t, deepMergedConfig) + _, _, _ = importsConfig, stackConfigMap, tfInline // Unused return values. + _ = tfImports + + // Verify resolved locals are persisted in the config. + locals, ok := deepMergedConfig["locals"].(map[string]any) + require.True(t, ok, "resolved locals should be persisted into stackConfigMap") + assert.Equal(t, "dev", locals["stage"]) + + // Settings section has both {{ .locals.stage }} and {{ .atmos_component }}. + // processTemplatesInSection fails on {{ .atmos_component }} so the entire section + // falls back to raw values — both templates are preserved for later resolution. + settings, ok := deepMergedConfig["settings"].(map[string]any) + require.True(t, ok, "settings should exist") + assert.Equal(t, "{{ .atmos_component }}", settings["component"].(string), + "{{ .atmos_component }} should be preserved in settings fallback") + assert.Equal(t, "{{ .locals.stage }}-label", settings["stage_label"].(string), + "{{ .locals.stage }}-label is preserved when section has unresolvable templates") + + // Vars section similarly falls back because it contains {{ .atmos_stack }}. + vars, ok := deepMergedConfig["vars"].(map[string]any) + require.True(t, ok, "vars should exist") + assert.Equal(t, "{{ .atmos_stack }}", vars["stack"].(string), + "{{ .atmos_stack }} should be preserved in vars fallback") +} + +// TestProcessYAMLConfigFile_ExternalContextError tests that template errors +// are returned as errors when external context is provided (not just file-extracted). +func TestProcessYAMLConfigFile_ExternalContextError(t *testing.T) { + tmpDir := t.TempDir() + yamlContent := ` +settings: + value: "{{ .missing_key }}" +` + filePath := filepath.Join(tmpDir, "test.yaml") + err := os.WriteFile(filePath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + atmosConfig := schema.AtmosConfiguration{} + + // Process WITH external context — template errors should be returned. + externalContext := map[string]any{ + "some_key": "some_value", + } + result, importsConfig, stackCfg, tfInline, tfImports, _, _, err := ProcessYAMLConfigFile( + &atmosConfig, + tmpDir, + filePath, + map[string]map[string]any{}, + externalContext, // External context provided. + false, // ignoreMissingFiles. + false, // skipTemplatesProcessingInImports. + false, // ignoreMissingTemplateValues. + false, // skipIfMissing. + nil, + nil, + nil, + nil, + "", + ) + _ = result + _ = importsConfig + _ = stackCfg + _ = tfInline + _ = tfImports + + require.Error(t, err, "Should return error when external context is provided and template fails") + assert.True(t, errors.Is(err, errUtils.ErrInvalidStackManifest)) +} + +// TestProcessYAMLConfigFile_ResolvedSectionsPersisted tests that resolved +// sections (locals, vars, settings, env) are persisted into stackConfigMap. +func TestProcessYAMLConfigFile_ResolvedSectionsPersisted(t *testing.T) { + tmpDir := t.TempDir() + yamlContent := ` +locals: + ns: acme + env: dev +settings: + namespace: '{{ .locals.ns }}' +vars: + environment: '{{ .locals.env }}' +env: + NAMESPACE: '{{ .locals.ns }}' +` + filePath := filepath.Join(tmpDir, "test.yaml") + err := os.WriteFile(filePath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + atmosConfig := schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + deepMergedConfig, importsConfig, stackConfigMap, tfInline, tfImports, _, _, err := ProcessYAMLConfigFile( + &atmosConfig, + tmpDir, + filePath, + map[string]map[string]any{}, + nil, // No external context. + false, // ignoreMissingFiles. + false, // skipTemplatesProcessingInImports. + false, // ignoreMissingTemplateValues. + false, // skipIfMissing. + nil, + nil, + nil, + nil, + "", + ) + _, _, _ = importsConfig, stackConfigMap, tfInline // Unused return values. + _ = tfImports + + require.NoError(t, err) + + // Resolved locals should be persisted. + locals, ok := deepMergedConfig["locals"].(map[string]any) + require.True(t, ok, "locals should be persisted") + assert.Equal(t, "acme", locals["ns"]) + assert.Equal(t, "dev", locals["env"]) + + // Resolved settings should be persisted ({{ .locals.ns }} → "acme"). + settings, ok := deepMergedConfig["settings"].(map[string]any) + require.True(t, ok, "settings should be persisted") + assert.Equal(t, "acme", settings["namespace"]) + + // Resolved vars should be persisted ({{ .locals.env }} → "dev"). + vars, ok := deepMergedConfig["vars"].(map[string]any) + require.True(t, ok, "vars should be persisted") + assert.Equal(t, "dev", vars["environment"]) + + // Resolved env should be persisted ({{ .locals.ns }} → "acme"). + env, ok := deepMergedConfig["env"].(map[string]any) + require.True(t, ok, "env should be persisted") + assert.Equal(t, "acme", env["NAMESPACE"]) +} + +// TestExtractAndAddLocalsToContext_VarsWithProcessedSettings tests that vars +// can reference processed settings values through the template pipeline. +func TestExtractAndAddLocalsToContext_VarsWithProcessedSettings(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + yamlContent := ` +locals: + stage: dev +settings: + resolved_stage: '{{ .locals.stage }}' +vars: + from_settings: '{{ .settings.resolved_stage }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + + // Settings should have resolved {{ .locals.stage }} → "dev". + settings, ok := ctx["settings"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dev", settings["resolved_stage"]) + + // Vars should have resolved {{ .settings.resolved_stage }} → "dev". + vars, ok := ctx["vars"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dev", vars["from_settings"]) +} + +// TestExtractAndAddLocalsToContext_EnvWithProcessedSettingsAndVars tests that env +// can reference both processed settings and vars through the template pipeline. +func TestExtractAndAddLocalsToContext_EnvWithProcessedSettingsAndVars(t *testing.T) { + atmosConfig := &schema.AtmosConfiguration{ + Templates: schema.Templates{ + Settings: schema.TemplatesSettings{ + Enabled: true, + }, + }, + } + + yamlContent := ` +locals: + ns: acme +settings: + region: us-east-1 +vars: + app_name: '{{ .locals.ns }}-app' +env: + APP: '{{ .locals.ns }}' + REGION: '{{ .settings.region }}' +` + ctx, err := extractAndAddLocalsToContext(atmosConfig, yamlContent, "test.yaml", "test.yaml", nil) + require.NoError(t, err) + + env, ok := ctx["env"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "acme", env["APP"]) + assert.Equal(t, "us-east-1", env["REGION"]) +} + // TestAtmosProTemplateRegression tests that {{ .atmos_component }} templates in non-.tmpl files // with settings sections don't fail during import processing. // This is a regression test for issue where 1.205 inadvertently triggers template processing @@ -2056,6 +2599,6 @@ func TestAtmosProTemplateRegression(t *testing.T) { // because templates should NOT be processed during import for non-.tmpl files without explicit context. componentInput, ok := inputs["component"].(string) require.True(t, ok, "component input should be a string") - assert.Contains(t, componentInput, "atmos_component", + assert.Equal(t, "{{ .atmos_component }}", componentInput, "Template {{ .atmos_component }} should be preserved during import, not processed") }