Retry config sync on concurrent changes

Snapshot the current config generation before building the merged sync view, normalize the merged role/workflow/provider definition maps off-lock, and only commit them if the generation is unchanged.

Keep the retry logic scoped to MergeConfiguration, compare and commit definitions only, and detach the snapshot through JSON so stale retries do not alias nested state. Reloaded definitions now bump the generation counter, while broader nested-mutation cleanup remains tracked in #306.

Author: michaelw

docs/development/index.md

@@ -9,3 +9,12 @@ description: Developer documentation for Thand Agent
 # Development
 
 Documentation for developers contributing to or extending the Thand Agent.
+
+## Config Mutation Invariant
+
+Configuration definition maps should be treated as immutable snapshots.
+When config changes, prefer replacing whole entries or whole definition maps
+instead of mutating nested state in place. Some older code paths still perform
+mutation-prone updates; keep new code aligned with the invariant and track
+cleanup of legacy exceptions in follow-up issues rather than extending them.
+Current cleanup work is tracked in [#306](https://github.com/thand-io/agent/issues/306).

internal/config/config.go

@@ -360,6 +360,7 @@ func (c *Config) ReloadConfig() error {
 			logrus.Infoln("Loaded workflows from external source:", len(workflows))
 			c.mu.Lock()
 			c.Workflows.Definitions = workflows
+			c.configGeneration++
 			c.mu.Unlock()
 		} else {
 			logrus.Warningln("No workflows loaded from external source")
@@ -378,6 +379,7 @@ func (c *Config) ReloadConfig() error {
 			logrus.Infoln("Loaded providers from external source:", len(providers))
 			c.mu.Lock()
 			c.Providers.Definitions = providers
+			c.configGeneration++
 			c.mu.Unlock()
 		} else {
 			logrus.Warningln("No providers loaded from external source")
@@ -394,6 +396,7 @@ func (c *Config) ReloadConfig() error {
 			logrus.Infoln("Loaded roles from external source:", len(roles))
 			c.mu.Lock()
 			c.Roles.Definitions = roles
+			c.configGeneration++
 			c.mu.Unlock()
 		} else {
 			logrus.Warningln("No roles loaded from external source")

internal/config/model.go

@@ -64,6 +64,12 @@ type Config struct {
 	logger thandLogger
 	mu     sync.RWMutex
 
+	// Incremented whenever synced config definitions actually change.
+	// Definition maps should be treated as immutable snapshots: callers should
+	// replace whole entries or whole maps rather than mutating nested state
+	// in place. Legacy mutation-prone paths are being tracked in issue #306.
+	configGeneration uint64
+
 	// Cached services client
 	initializeServiceClientOnce sync.Once
 	servicesClient              models.ServicesClientImpl