diff --git a/packages/engine/internal/cel/cel.go b/packages/engine/internal/cel/cel.go index 826b88f..b261fac 100644 --- a/packages/engine/internal/cel/cel.go +++ b/packages/engine/internal/cel/cel.go @@ -2,6 +2,7 @@ package cel import ( "fmt" + "log/slog" "os" "reflect" "strings" @@ -27,7 +28,8 @@ const maxProgramCacheSize = 10000 // Evaluator evaluates CEL expressions against a runtime context. type Evaluator struct { env *cel.Env - envCache map[string]string // cached, filtered environment variables + configEnv map[string]string // config-sourced env values from mantle.yaml + envCache map[string]string // cached, filtered environment variables programMu sync.Mutex programCache map[string]cel.Program // expression string -> compiled cel.Program } @@ -49,7 +51,7 @@ func NewEvaluator() (*Evaluator, error) { if err != nil { return nil, fmt.Errorf("creating CEL environment: %w", err) } - return &Evaluator{env: env, envCache: envVars(), programCache: make(map[string]cel.Program)}, nil + return &Evaluator{env: env, envCache: mergeEnvVars(nil), programCache: make(map[string]cel.Program)}, nil } // Eval evaluates a CEL expression and returns the result as a Go value. @@ -222,17 +224,37 @@ func (e *Evaluator) resolveValue(v any, ctx *Context) (any, error) { } } -// envVars returns only environment variables with the MANTLE_ENV_ prefix, -// stripping the prefix for cleaner access (e.g., MANTLE_ENV_FOO -> env.FOO). +// SetConfigEnv sets the config-sourced env values and rebuilds the env cache. +// This merges config values with MANTLE_ENV_* environment variables, where +// OS env vars take precedence over config values. +func (e *Evaluator) SetConfigEnv(configEnv map[string]string) { + e.configEnv = configEnv + e.envCache = mergeEnvVars(configEnv) +} + +// mergeEnvVars builds the env map available to CEL expressions. +// It starts with config-sourced values (from the env: section in mantle.yaml), +// then overlays MANTLE_ENV_* environment variables (stripping the prefix). +// When a key exists in both, the OS env var wins and an info log is emitted. // This prevents CEL expressions from reading sensitive variables like // MANTLE_ENCRYPTION_KEY, MANTLE_DATABASE_URL, or AWS_SECRET_ACCESS_KEY. -func envVars() map[string]string { +func mergeEnvVars(configEnv map[string]string) map[string]string { env := make(map[string]string) + + // Start with config values. + for k, v := range configEnv { + env[k] = v + } + + // Overlay MANTLE_ENV_* from OS environment (env var wins). const prefix = "MANTLE_ENV_" for _, e := range os.Environ() { parts := strings.SplitN(e, "=", 2) if len(parts) == 2 && strings.HasPrefix(parts[0], prefix) { key := strings.TrimPrefix(parts[0], prefix) + if _, exists := env[key]; exists { + slog.Info("env variable overrides config", "key", prefix+key, "config_key", "env."+key) + } env[key] = parts[1] } } diff --git a/packages/engine/internal/cel/cel_test.go b/packages/engine/internal/cel/cel_test.go index 99c1948..f93fa4f 100644 --- a/packages/engine/internal/cel/cel_test.go +++ b/packages/engine/internal/cel/cel_test.go @@ -296,25 +296,115 @@ func TestEval_EnvVarBlocksSensitive(t *testing.T) { } func TestEnvVars_PrefixStripping(t *testing.T) { - // Directly test the envVars function. + // Directly test the mergeEnvVars function. os.Setenv("MANTLE_ENV_TEST_KEY", "test-value") defer os.Unsetenv("MANTLE_ENV_TEST_KEY") os.Setenv("MANTLE_DATABASE_URL", "should-not-appear") defer os.Unsetenv("MANTLE_DATABASE_URL") - result := envVars() + result := mergeEnvVars(nil) if v, ok := result["TEST_KEY"]; !ok || v != "test-value" { - t.Errorf("envVars()[TEST_KEY] = %q, %v; want %q, true", v, ok, "test-value") + t.Errorf("mergeEnvVars()[TEST_KEY] = %q, %v; want %q, true", v, ok, "test-value") } if _, ok := result["MANTLE_DATABASE_URL"]; ok { - t.Error("envVars() should not contain MANTLE_DATABASE_URL") + t.Error("mergeEnvVars() should not contain MANTLE_DATABASE_URL") } if _, ok := result["DATABASE_URL"]; ok { - t.Error("envVars() should not contain DATABASE_URL (MANTLE_ prefix without ENV_)") + t.Error("mergeEnvVars() should not contain DATABASE_URL (MANTLE_ prefix without ENV_)") + } +} + +func TestEnvVars_ConfigEnvMerge(t *testing.T) { + configEnv := map[string]string{ + "APP_NAME": "my-workflow-app", + "REGION": "us-east-1", + } + + eval, err := NewEvaluator() + if err != nil { + t.Fatalf("NewEvaluator() error: %v", err) + } + eval.SetConfigEnv(configEnv) + + ctx := newTestContext() + + result, err := eval.Eval(`env.APP_NAME`, ctx) + if err != nil { + t.Fatalf("Eval(env.APP_NAME) error: %v", err) + } + if result != "my-workflow-app" { + t.Errorf("env.APP_NAME = %v, want %q", result, "my-workflow-app") + } + + result, err = eval.Eval(`env.REGION`, ctx) + if err != nil { + t.Fatalf("Eval(env.REGION) error: %v", err) + } + if result != "us-east-1" { + t.Errorf("env.REGION = %v, want %q", result, "us-east-1") + } +} + +func TestEnvVars_OsEnvOverridesConfig(t *testing.T) { + t.Setenv("MANTLE_ENV_REGION", "eu-west-1") + + configEnv := map[string]string{ + "REGION": "us-east-1", + "APP_NAME": "my-app", + } + + eval, err := NewEvaluator() + if err != nil { + t.Fatalf("NewEvaluator() error: %v", err) + } + eval.SetConfigEnv(configEnv) + + ctx := newTestContext() + + // MANTLE_ENV_REGION should override the config value. + result, err := eval.Eval(`env.REGION`, ctx) + if err != nil { + t.Fatalf("Eval(env.REGION) error: %v", err) + } + if result != "eu-west-1" { + t.Errorf("env.REGION = %v, want %q (OS env override)", result, "eu-west-1") + } + + // APP_NAME should still come from config (no OS env override). + result, err = eval.Eval(`env.APP_NAME`, ctx) + if err != nil { + t.Fatalf("Eval(env.APP_NAME) error: %v", err) + } + if result != "my-app" { + t.Errorf("env.APP_NAME = %v, want %q", result, "my-app") + } +} + +func TestEnvVars_NilConfigEnv(t *testing.T) { + eval, err := NewEvaluator() + if err != nil { + t.Fatalf("NewEvaluator() error: %v", err) + } + + // SetConfigEnv(nil) should not panic. + eval.SetConfigEnv(nil) + + ctx := newTestContext() + + // env map should still work (just no config values). + t.Setenv("MANTLE_ENV_SAFE_KEY", "safe-value") + eval.SetConfigEnv(nil) + + result, err := eval.Eval(`env.SAFE_KEY`, ctx) + if err != nil { + t.Fatalf("Eval(env.SAFE_KEY) error: %v", err) + } + if result != "safe-value" { + t.Errorf("env.SAFE_KEY = %v, want %q", result, "safe-value") } } diff --git a/packages/engine/internal/cli/run.go b/packages/engine/internal/cli/run.go index 42be4df..6e3f32e 100644 --- a/packages/engine/internal/cli/run.go +++ b/packages/engine/internal/cli/run.go @@ -65,6 +65,7 @@ func newRunCommand() *cobra.Command { return fmt.Errorf("creating engine: %w", err) } eng.MaxConcurrentExecutionsPerTeam = cfg.Engine.MaxConcurrentExecutionsPerTeam + eng.CEL.SetConfigEnv(cfg.Env) // Configure credential resolver with Postgres-backed store when encryption key is set. if cfg.Encryption.Key != "" { diff --git a/packages/engine/internal/cli/serve.go b/packages/engine/internal/cli/serve.go index 7b6abc6..8face8e 100644 --- a/packages/engine/internal/cli/serve.go +++ b/packages/engine/internal/cli/serve.go @@ -49,6 +49,7 @@ func newServeCommand() *cobra.Command { if err != nil { return fmt.Errorf("creating engine: %w", err) } + eng.CEL.SetConfigEnv(cfg.Env) eng.MaxToolRoundsLimit = cfg.Engine.MaxToolRoundsLimit eng.MaxConcurrentExecutionsPerTeam = cfg.Engine.MaxConcurrentExecutionsPerTeam diff --git a/packages/engine/internal/config/config.go b/packages/engine/internal/config/config.go index 8ad2857..3b03a57 100644 --- a/packages/engine/internal/config/config.go +++ b/packages/engine/internal/config/config.go @@ -9,6 +9,7 @@ import ( "log/slog" "net/url" "os" + "strings" "time" "github.com/dvflw/mantle/internal/budget" @@ -32,6 +33,7 @@ type Config struct { GCP GCPConfig `mapstructure:"gcp"` Azure AzureConfig `mapstructure:"azure"` Storage StorageConfig `mapstructure:"storage"` + Env map[string]string `mapstructure:"env"` } // RetentionConfig holds data retention settings. @@ -279,8 +281,6 @@ func Load(cmd *cobra.Command) (*Config, error) { } // Validate config version. - // If the user did not set "version" at all, default to 1. - // If they explicitly set it (including to 0), it must be 1. if !v.IsSet("version") { cfg.Version = 1 } @@ -289,7 +289,6 @@ func Load(cmd *cobra.Command) (*Config, error) { } // Deprecated: fall back from "tmp" section to "storage" for backward compatibility. - // Field-by-field merge so that env vars / flags on "storage.*" are not clobbered. if v.IsSet("tmp") { if sub := v.Sub("tmp"); sub != nil { var legacy StorageConfig @@ -314,6 +313,16 @@ func Load(cmd *cobra.Command) (*Config, error) { } } + // Viper lowercases all map keys. Normalize env map keys to uppercase + // so they match MANTLE_ENV_* convention and CEL expressions like env.APP_NAME. + if len(cfg.Env) > 0 { + normalized := make(map[string]string, len(cfg.Env)) + for k, v := range cfg.Env { + normalized[strings.ToUpper(k)] = v + } + cfg.Env = normalized + } + // Validate budget reset_day range. if cfg.Engine.Budget.ResetDay < 1 || cfg.Engine.Budget.ResetDay > 28 { if cfg.Engine.Budget.ResetMode == budget.ResetModeRolling { diff --git a/packages/engine/internal/config/config_test.go b/packages/engine/internal/config/config_test.go index e78e213..a780344 100644 --- a/packages/engine/internal/config/config_test.go +++ b/packages/engine/internal/config/config_test.go @@ -427,3 +427,43 @@ tmp: assert.Equal(t, "s3", cfg.Storage.Type) assert.Equal(t, "new-bucket", cfg.Storage.Bucket) } + +func TestLoad_EnvMap(t *testing.T) { + dir := t.TempDir() + configFile := filepath.Join(dir, "mantle.yaml") + err := os.WriteFile(configFile, []byte(` +env: + APP_NAME: "my-app" + REGION: "us-east-1" + DEBUG: "true" +`), 0644) + require.NoError(t, err) + + cmd := newTestCommand() + _ = cmd.Flags().Set("config", configFile) + + cfg, err := Load(cmd) + require.NoError(t, err) + + assert.Equal(t, "my-app", cfg.Env["APP_NAME"]) + assert.Equal(t, "us-east-1", cfg.Env["REGION"]) + assert.Equal(t, "true", cfg.Env["DEBUG"]) +} + +func TestLoad_EnvMapEmpty(t *testing.T) { + dir := t.TempDir() + configFile := filepath.Join(dir, "mantle.yaml") + err := os.WriteFile(configFile, []byte(` +log: + level: "debug" +`), 0644) + require.NoError(t, err) + + cmd := newTestCommand() + _ = cmd.Flags().Set("config", configFile) + + cfg, err := Load(cmd) + require.NoError(t, err) + + assert.Empty(t, cfg.Env) +} diff --git a/packages/site/src/content/docs/configuration.md b/packages/site/src/content/docs/configuration.md index ba06979..31a6719 100644 --- a/packages/site/src/content/docs/configuration.md +++ b/packages/site/src/content/docs/configuration.md @@ -48,12 +48,16 @@ engine: step_output_max_bytes: 1048576 default_max_tool_rounds: 10 default_max_tool_calls_per_round: 10 - max_concurrent_executions_per_team: 10 + max_concurrent_executions_per_team: 0 # 0 = unlimited storage: type: filesystem path: /var/lib/mantle/artifacts retention: "24h" + +env: + APP_NAME: "my-app" + API_BASE_URL: "https://api.example.com" ``` ### All Config File Fields @@ -79,12 +83,13 @@ storage: | `engine.step_output_max_bytes` | integer | `1048576` | Maximum size in bytes for a single step's output. Outputs exceeding this limit are truncated. Default is 1 MB. | | `engine.default_max_tool_rounds` | integer | `10` | Default maximum number of LLM-tool interaction rounds for AI steps with tools. Can be overridden per step with `max_tool_rounds`. | | `engine.default_max_tool_calls_per_round` | integer | `10` | Default maximum number of tool calls the LLM can make per round. Can be overridden per step with `max_tool_calls_per_round`. | -| `engine.max_concurrent_executions_per_team` | integer | `10` | Maximum number of concurrent workflow executions allowed per team. When the limit is reached, new executions are queued. Set to `0` for unlimited. | +| `engine.max_concurrent_executions_per_team` | integer | `0` | Maximum number of concurrent workflow executions allowed per team. When the limit is reached, new executions are queued. Set to `0` for unlimited. | | `storage.type` | string | -- | Artifact storage backend. One of: `s3`, `filesystem`. Required if any workflow declares artifacts. | | `storage.bucket` | string | -- | S3 bucket name. Required when `storage.type` is `s3`. | | `storage.prefix` | string | -- | S3 key prefix for artifact storage. Optional. | | `storage.path` | string | -- | Local directory path. Required when `storage.type` is `filesystem`. | | `storage.retention` | duration | -- | How long to keep artifacts after workflow completion. Uses Go duration format (e.g., `24h`). Empty means no auto-cleanup. | +| `env.*` | map[string]string | -- | Key-value pairs available to CEL workflow expressions via `env.`. See [Workflow Expression Variables](#workflow-expression-variables). | :::caution[Deprecation: `tmp` renamed to `storage`] The `tmp` configuration section has been renamed to `storage` in v0.4.0. The old `tmp` key still works but is deprecated and will be removed in a future release. Update your `mantle.yaml` to use `storage` instead. @@ -96,6 +101,67 @@ When you do not pass `--config`, Mantle searches for `mantle.yaml` in the curren When you pass `--config path/to/config.yaml` explicitly, Mantle requires that file to exist and be valid YAML. A missing or unparseable explicit config file is a hard error. +## Workflow Expression Variables + +The `env:` section in `mantle.yaml` defines key-value pairs that are available to CEL expressions in workflows via the `env` namespace. This is useful for environment-specific configuration that workflows need at runtime (API base URLs, feature flags, region names, etc.) without hardcoding values in workflow definitions. + +### Syntax + +```yaml +# mantle.yaml +env: + APP_NAME: "my-app" + API_BASE_URL: "https://api.example.com" + REGION: "us-east-1" + DEBUG: "true" +``` + +### Usage in Workflows + +Values defined in the `env:` section are accessible in any CEL expression via `env.`: + +```yaml +# workflow.yaml +name: deploy +steps: + - name: notify + connector: http/request + params: + url: "{{ env.API_BASE_URL }}/deployments" + body: '{"app": "{{ env.APP_NAME }}", "region": "{{ env.REGION }}"}' +``` + +### Precedence + +The `env:` config values merge with `MANTLE_ENV_*` OS environment variables. When the same key exists in both sources, the OS environment variable wins: + +1. **`MANTLE_ENV_*` environment variables** (highest priority) -- stripped of the `MANTLE_ENV_` prefix +2. **`env:` section in `mantle.yaml`** (lower priority) + +When an override occurs, Mantle logs an info message: + +```text +INFO env variable overrides config key=MANTLE_ENV_REGION config_key=env.REGION +``` + +This allows operators to override config-file defaults without editing YAML, which is useful in CI pipelines and container deployments. + +**Case sensitivity:** All `env:` keys from `mantle.yaml` are normalized to uppercase internally (e.g., `region` becomes `REGION`). The `MANTLE_ENV_*` prefix is stripped and the remaining key must be uppercase to match (e.g., `MANTLE_ENV_REGION` overrides `env.REGION`). Always use uppercase keys in both `mantle.yaml` and environment variables to avoid mismatches. + +**Example override:** + +```yaml +# mantle.yaml +env: + REGION: "us-east-1" +``` + +```bash +# Override REGION for this deployment +export MANTLE_ENV_REGION="eu-west-1" +mantle run deploy # env.REGION evaluates to "eu-west-1" +``` + ## Environment Variables All environment variables use the `MANTLE_` prefix with underscores replacing dots and hyphens. @@ -120,7 +186,7 @@ All environment variables use the `MANTLE_` prefix with underscores replacing do | `MANTLE_ENGINE_STEP_OUTPUT_MAX_BYTES` | `engine.step_output_max_bytes` | `1048576` | | `MANTLE_ENGINE_DEFAULT_MAX_TOOL_ROUNDS` | `engine.default_max_tool_rounds` | `10` | | `MANTLE_ENGINE_DEFAULT_MAX_TOOL_CALLS_PER_ROUND` | `engine.default_max_tool_calls_per_round` | `10` | -| `MANTLE_ENGINE_MAX_CONCURRENT_EXECUTIONS_PER_TEAM` | `engine.max_concurrent_executions_per_team` | `10` | +| `MANTLE_ENGINE_MAX_CONCURRENT_EXECUTIONS_PER_TEAM` | `engine.max_concurrent_executions_per_team` | `0` | | `MANTLE_STORAGE_TYPE` | `storage.type` | -- | | `MANTLE_STORAGE_BUCKET` | `storage.bucket` | -- | | `MANTLE_STORAGE_PREFIX` | `storage.prefix` | -- |