Fix bug where -backend-config could not override attributes that weren't at the top level in the backend schema

.changes/v1.13/BUG FIXES-20250423-164150.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: 'backends: Nested attrbiutes can now be overridden during `init` using the `-backend-config` flag'
+time: 2025-04-23T16:41:50.904809+01:00
+custom:
+    Issue: "36911"

internal/backend/remote-state/inmem/backend.go

@@ -6,6 +6,8 @@ package inmem
 import (
 	"errors"
 	"fmt"
+	"maps"
+	"os"
 	"sort"
 	"sync"
 	"time"
@@ -47,20 +49,67 @@ func Reset() {
 }
 
 // New creates a new backend for Inmem remote state.
+//
+// Currently the inmem backend is available for end users to use if they know it exists (it is not in any docs), but it was intended as a test resource.
+// Making the inmem backend unavailable to end users and only available during tests is a breaking change.
+// As a compromise for now, the inmem backend includes test-specific code that is enabled by setting the TF_INMEM_TEST environment variable.
+// Test-specific behaviors include:
+// * A more complex schema for testing code related to backend configurations
+//
+// Note: The alternative choice would be to add a duplicate of inmem in the codebase that's user-inaccessible, and this would be harder to maintain.
 func New() backend.Backend {
+	if os.Getenv("TF_INMEM_TEST") != "" {
+		// We use a different schema for testing. This isn't user facing unless they
+		// dig into the code.
+		fmt.Println("TF_INMEM_TEST is set: Using test schema for the inmem backend")
+
+		return &Backend{
+			Base: backendbase.Base{
+				Schema: testSchema(),
+			},
+		}
+	}
+
+	// Default schema that's user-facing
 	return &Backend{
 		Base: backendbase.Base{
 			Schema: &configschema.Block{
-				Attributes: map[string]*configschema.Attribute{
-					"lock_id": {
-						Type:        cty.String,
-						Optional:    true,
-						Description: "initializes the state in a locked configuration",
-					},
+				Attributes: defaultSchemaAttrs,
+			},
+		},
+	}
+}
+
+var defaultSchemaAttrs = map[string]*configschema.Attribute{
+	"lock_id": {
+		Type:        cty.String,
+		Optional:    true,
+		Description: "initializes the state in a locked configuration",
+	},
+}
+
+func testSchema() *configschema.Block {
+	var newSchemaAttrs = make(map[string]*configschema.Attribute)
+	maps.Copy(newSchemaAttrs, defaultSchemaAttrs)
+
+	// Append test-specific attributes to the default attributes
+	newSchemaAttrs["test_nested_attr_single"] = &configschema.Attribute{
+		Description: "An attribute that contains nested attributes, where nesting mode is NestingSingle",
+		NestedType: &configschema.Object{
+			Nesting: configschema.NestingSingle,
+			Attributes: map[string]*configschema.Attribute{
+				"child": {
+					Type:        cty.String,
+					Optional:    true,
+					Description: "A nested attribute inside the parent attribute `test_nested_attr_single`",
 				},
 			},
 		},
 	}
+
+	return &configschema.Block{
+		Attributes: newSchemaAttrs,
+	}
 }
 
 type Backend struct {

internal/command/init.go

@@ -8,11 +8,13 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"maps"
 	"reflect"
 	"sort"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
 	svchost "github.com/hashicorp/terraform-svchost"
 	"github.com/posener/complete"
 	"github.com/zclconf/go-cty/cty"
@@ -1031,23 +1033,70 @@ func (c *InitCommand) backendConfigOverrideBody(flags arguments.FlagNameValueSli
 			flushVals() // deal with any accumulated individual values first
 			mergeBody(newBody)
 		} else {
+			// The flag value is setting a single attribute's value
 			name := item.Value[:eq]
 			rawValue := item.Value[eq+1:]
-			attrS := schema.Attributes[name]
-			if attrS == nil {
+
+			// Check the value looks like a valid attribute identifier
+			splitName := strings.Split(name, ".")
+			for _, part := range splitName {
+				if !hclsyntax.ValidIdentifier(part) {
+					diags = diags.Append(tfdiags.Sourceless(
+						tfdiags.Error,
+						"Invalid backend configuration argument",
+						fmt.Sprintf("The backend configuration argument %q given on the command line contains an invalid identifier that cannot be used for partial configuration: %q.", name, part),
+					))
+					continue
+				}
+			}
+			// Check the attribute exists in the backend's schema and is a 'leaf' attribute
+			path := cty.Path{}
+			for _, name := range splitName {
+				path = path.GetAttr(name)
+			}
+			targetAttr := schema.AttributeByPath(path)
+			if targetAttr == nil {
 				diags = diags.Append(tfdiags.Sourceless(
 					tfdiags.Error,
 					"Invalid backend configuration argument",
 					fmt.Sprintf("The backend configuration argument %q given on the command line is not expected for the selected backend type.", name),
 				))
 				continue
 			}
-			value, valueDiags := configValueFromCLI(item.String(), rawValue, attrS.Type)
-			diags = diags.Append(valueDiags)
-			if valueDiags.HasErrors() {
+			if targetAttr.NestedType != nil {
+				diags = diags.Append(tfdiags.Sourceless(
+					tfdiags.Error,
+					"Invalid backend configuration argument",
+					fmt.Sprintf("The backend configuration argument %q given on the command line specifies an attribute that contains nested attributes. Instead, use separate flags for each nested attribute inside.", name),
+				))
 				continue
 			}
-			synthVals[name] = value
+
+			if len(splitName) > 1 {
+				// The flag item is overriding a nested attribute (name contains multiple identifiers)
+				// e.g. assume_role.role_arn in the s3 backend
+				value, valueDiags := configValueFromCLI(item.String(), rawValue, targetAttr.Type)
+				diags = diags.Append(valueDiags)
+				if valueDiags.HasErrors() {
+					continue
+				}
+
+				// Synthetic values are collected as we parse each flag item
+				// Nested values need to be added in a way that doesn't affect pre-existing values
+				synthCopy := map[string]cty.Value{}
+				maps.Copy(synthCopy, synthVals)
+				synthVals = addNestedAttrsToCtyValueMap(synthCopy, splitName, value)
+			} else {
+				// The flag item is overriding a non-nested, top-level attribute
+				value, valueDiags := configValueFromCLI(item.String(), rawValue, targetAttr.Type)
+				diags = diags.Append(valueDiags)
+				if valueDiags.HasErrors() {
+					continue
+				}
+
+				// Synthetic values are collected as we parse each flag item
+				synthVals[name] = value
+			}
 		}
 	}
 
@@ -1056,6 +1105,36 @@ func (c *InitCommand) backendConfigOverrideBody(flags arguments.FlagNameValueSli
 	return ret, diags
 }
 
+// addNestedAttrsToCtyValueMap is used to assemble a map of cty Values that is used to create a cty.Object.
+// This function should be used to set nested values in the map[string]cty.Value, and cannot be used to set
+// top-level attributes (this will result in other top-level attributes being lost).
+func addNestedAttrsToCtyValueMap(accumulator map[string]cty.Value, names []string, attrValue cty.Value) map[string]cty.Value {
+	if len(names) == 1 {
+		// Base case - we are setting the attribute with the provided value
+		return map[string]cty.Value{
+			names[0]: attrValue,
+		}
+	}
+
+	// Below we are navigating the path from the final, nested attribute we want to set a value for.
+	// During this process we need to ensure that any pre-existing values in the map are not
+	// accidentally removed
+	val, ok := accumulator[names[0]]
+	if !ok {
+		accumulator[names[0]] = cty.ObjectVal(addNestedAttrsToCtyValueMap(accumulator, names[1:], attrValue))
+	} else {
+		values := val.AsValueMap()
+		newValues := addNestedAttrsToCtyValueMap(accumulator, names[1:], attrValue)
+
+		// We copy new values into the map of pre-existing values
+		maps.Copy(values, newValues)
+
+		accumulator[names[0]] = cty.ObjectVal(values)
+	}
+
+	return accumulator
+}
+
 func (c *InitCommand) AutocompleteArgs() complete.Predictor {
 	return complete.PredictDirs("")
 }