Merge pull request #723 from obeli-sk/exec-stdin-contract-change

exec: Move secrets to "secrets", allow passing params to stdin

Author: tomasol

CHANGELOG.md

@@ -12,9 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - *(deployment)* Verify a user-supplied `content_digest` at submit time, in addition to runtime.
 - *(deployment)* Reject at submit time deployments where two distinct deployment-owned sources (inline/owned scripts or backtrace sources) resolve to the same `file_name`, since `deployment get` could not recreate both on disk.
 - *(cli)* `deployment active` prints the ID of the currently active deployment.
+- *(deployment)* `activity_exec` components gained `params_via_stdin` (default `false`). When enabled,
+  parameters are passed to the program via the stdin JSON `params` array instead of argv, allowing
+  payloads larger than the `execve` argument-size limit.
 
 ### Changed
 
+- *(deployment)* [**breaking**] **`activity_exec` secrets stdin format changed.** Secrets are now nested under a
+  `secrets` key (`{"secrets":{"KEY":"value",...}}`) instead of being the top-level object, so they can
+  coexist with the new `params` key. Scripts that parsed `.KEY` from stdin must now read `.secrets.KEY`.
+
 - *(cli)* `deployment show <ID>` now prints the reconstructed `deployment.toml` (with local file
   references, the same TOML `deployment get` writes) instead of the raw canonical config. Pass a
   FILE argument to print a single deployment-owned source file as `deployment get` would serialize

assets/schemas/deployment-canonical.json

@@ -772,6 +772,10 @@
               "type": "null"
             }
           ]
+        },
+        "params_via_stdin": {
+          "type": "boolean",
+          "default": false
         }
       },
       "additionalProperties": false,

assets/schemas/oci-metadata-annotation.json

@@ -194,6 +194,10 @@
           ],
           "default": null
         },
+        "params_via_stdin": {
+          "type": "boolean",
+          "default": false
+        },
         "component_type": {
           "type": "string",
           "const": "activity_exec"

assets/schemas/toml/deployment.json

@@ -838,6 +838,11 @@
             }
           ],
           "default": null
+        },
+        "params_via_stdin": {
+          "description": "Pass parameters to the program via the stdin JSON `parameters` array instead\nof argv. Use this for large payloads that would exceed the `execve` argument-size\nlimit. Defaults to `false` (parameters passed as command-line arguments).",
+          "type": "boolean",
+          "default": false
         }
       },
       "additionalProperties": false,

crates/deployment-config/src/config.rs

@@ -570,6 +570,8 @@ pub struct ActivityExecComponentConfigResolved {
     pub env_vars: Vec<EnvVarConfig>,
     pub max_output_bytes: u64,
     pub secrets: Option<ExecSecretsToml>,
+    #[serde(default)]
+    pub params_via_stdin: bool,
 }
 
 #[derive(Debug, Deserialize, Serialize, Clone, Copy, JsonSchema, PartialEq)]

crates/wasm-workers/src/activity/activity_exec_worker.rs

@@ -17,6 +17,7 @@ use concepts::{
 use executor::worker::{
     FatalError, RunFinished, Worker, WorkerContext, WorkerError, WorkerResult, WorkerResultOk,
 };
+use indexmap::IndexMap;
 use secrecy::{ExposeSecret, SecretString};
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -31,7 +32,7 @@ pub enum ExecProgram {
     /// Inline script content. Written to a temp file at each execution.
     Inline(String),
     /// Path to an immutable cached script file (from OCI). Executed directly.
-    CachedFile(PathBuf),
+    CachedFile(PathBuf), // TODO: Use for CAS as well
 }
 
 /// Compiled exec activity. No WASM engine needed.
@@ -44,8 +45,12 @@ pub struct ActivityExecWorkerCompiled {
     max_output_bytes: u64,
     forward_stdout: Option<StdOutputConfig>,
     forward_stderr: Option<StdOutputConfig>,
-    /// Pre-computed stdin content from resolved secrets. Written to the child's stdin pipe.
-    stdin_content: Option<SecretString>,
+    /// Resolved secrets, nested under the `secrets` key of the stdin JSON.
+    /// `None` when no secrets are configured.
+    secrets: Option<IndexMap<String, SecretString>>,
+    /// When `true`, parameters are passed via the stdin JSON `params` array
+    /// instead of argv, sidestepping the `execve` argument-size limit.
+    params_via_stdin: bool,
     user_wasm_component: WasmComponent,
 }
 
@@ -60,7 +65,8 @@ impl ActivityExecWorkerCompiled {
         max_output_bytes: u64,
         forward_stdout: Option<StdOutputConfig>,
         forward_stderr: Option<StdOutputConfig>,
-        stdin_content: Option<SecretString>,
+        secrets: Option<IndexMap<String, SecretString>>,
+        params_via_stdin: bool,
     ) -> Result<Self, utils::wasm_tools::DecodeError> {
         let user_wasm_component = WasmComponent::new_from_fn_signature(
             &user_ffqn,
@@ -78,7 +84,8 @@ impl ActivityExecWorkerCompiled {
             max_output_bytes,
             forward_stdout,
             forward_stderr,
-            stdin_content,
+            secrets,
+            params_via_stdin,
             user_wasm_component,
         })
     }
@@ -124,7 +131,8 @@ impl ActivityExecWorkerCompiled {
             max_output_bytes: self.max_output_bytes,
             forward_stdout: stdout_config,
             forward_stderr: stderr_config,
-            stdin_content: self.stdin_content,
+            secrets: self.secrets,
+            params_via_stdin: self.params_via_stdin,
             cancel_registry,
             user_exports_noext: self.user_wasm_component.exported_functions(false).to_vec(),
         }
@@ -141,7 +149,8 @@ pub struct ActivityExecWorker {
     max_output_bytes: u64,
     forward_stdout: Option<StdOutputConfigWithSender>,
     forward_stderr: Option<StdOutputConfigWithSender>,
-    stdin_content: Option<SecretString>,
+    secrets: Option<IndexMap<String, SecretString>>,
+    params_via_stdin: bool,
     cancel_registry: CancelRegistry,
     user_exports_noext: Vec<FunctionMetadata>,
 }
@@ -241,17 +250,53 @@ impl Worker for ActivityExecWorker {
             }
         };
 
+        let json_params = ctx
+            .params
+            .as_json_values()
+            .expect("params come from database, not wasmtime");
+        assert_eq!(
+            self.user_params.len(),
+            json_params.len(),
+            "type checked in Params::from_json_values"
+        );
+
+        // Assemble the stdin JSON document `{ "secrets": {...}, "params": [...] }`.
+        // The `secrets` key is included when secrets are configured; the `params`
+        // key is included when `params_via_stdin` is set (otherwise params go to argv).
+        let stdin_content: Option<SecretString> = if self.params_via_stdin || self.secrets.is_some()
         {
+            let mut obj = serde_json::Map::new();
+            if let Some(secrets) = &self.secrets {
+                let secrets_obj = secrets
+                    .iter()
+                    .map(|(name, value)| {
+                        (
+                            name.clone(),
+                            serde_json::Value::String(value.expose_secret().to_string()),
+                        )
+                    })
+                    .collect();
+                obj.insert(
+                    "secrets".to_string(),
+                    serde_json::Value::Object(secrets_obj),
+                );
+            }
+            if self.params_via_stdin {
+                obj.insert(
+                    "params".to_string(),
+                    serde_json::Value::Array(json_params.to_vec()),
+                );
+            }
+            Some(SecretString::from(
+                serde_json::to_string(&obj).expect("JSON map serialization cannot fail"),
+            ))
+        } else {
+            None
+        };
+
+        // When params are passed via stdin, argv carries no parameters.
+        if !self.params_via_stdin {
             // Serialize each user parameter as a JSON string for command-line args.
-            let json_params = ctx
-                .params
-                .as_json_values()
-                .expect("params come from database, not wasmtime");
-            assert_eq!(
-                self.user_params.len(),
-                json_params.len(),
-                "type checked in Params::from_json_values"
-            );
             param_args.extend(json_params.iter().map(|v| {
                 serde_json::to_string(v).expect("serde_json::Value must be serializable")
             }));
@@ -272,7 +317,7 @@ impl Worker for ActivityExecWorker {
         // Capture stdout/stderr, optionally pipe stdin.
         cmd.stdout(std::process::Stdio::piped());
         cmd.stderr(std::process::Stdio::piped());
-        if self.stdin_content.is_some() {
+        if stdin_content.is_some() {
             cmd.stdin(std::process::Stdio::piped());
         }
 
@@ -288,8 +333,8 @@ impl Worker for ActivityExecWorker {
             )
         })?;
 
-        // Write stdin content if configured (e.g. resolved secrets).
-        if let Some(ref stdin_content) = self.stdin_content {
+        // Write stdin content if configured (resolved secrets and/or parameters).
+        if let Some(ref stdin_content) = stdin_content {
             use tokio::io::AsyncWriteExt;
             let mut child_stdin = child.stdin.take().expect("stdin was piped");
             child_stdin

obelisk-help-deployment.toml

@@ -159,8 +159,13 @@
 ## Not used when return_type is result (default), since the response carries no data.
 ## Default: 4096.
 # max_output_bytes = 4096
+## Pass parameters via the stdin JSON `params` array instead of argv.
+## Use for large payloads that would exceed the `execve` argument-size limit.
+## The child receives `{"params":[...]}` on stdin (alongside `secrets`, if configured).
+## Default: false.
+# params_via_stdin = false
 ## Secrets: resolved from host environment variables at startup and piped to the child's stdin as JSON.
-## The child receives `{"KEY":"value",...}` on stdin. Use `jq .KEY /dev/stdin` or equivalent to parse.
+## The child receives `{"secrets":{"KEY":"value",...}}` on stdin. Use `jq .secrets.KEY /dev/stdin` or equivalent to parse.
 # [activity_exec.secrets]
 # env_vars = [{name = "MY_SECRET", value = "${HOST_SECRET_VAR}"}] # Values support interpolation.
 

src/command/component.rs

@@ -256,6 +256,7 @@ fn find_component_for_push(
                     return_type: cfg.return_type.clone(),
                     max_output_bytes: cfg.max_output_bytes,
                     secrets: cfg.secrets.clone(),
+                    params_via_stdin: cfg.params_via_stdin,
                 },
             })
         }
@@ -486,6 +487,7 @@ fn build_component_table(
         return_type,
         max_output_bytes,
         secrets,
+        params_via_stdin,
     } = metadata
     {
         t["location"] = value(location_raw);
@@ -505,6 +507,9 @@ fn build_component_table(
             t["env_vars"] = Item::Value(toml_edit::Value::Array(arr));
         }
         t["max_output_bytes"] = value(i64::try_from(*max_output_bytes).unwrap_or(i64::MAX));
+        if *params_via_stdin {
+            t["params_via_stdin"] = value(true);
+        }
         if let Some(secrets) = secrets
             && !secrets.env_vars.is_empty()
         {
@@ -948,6 +953,7 @@ mod tests {
             return_type: Some("result<string>".to_string()),
             max_output_bytes: 1024,
             secrets: None,
+            params_via_stdin: false,
         };
         let content_digest: ContentDigest =
             "sha256:1111111111111111111111111111111111111111111111111111111111111111"

src/command/integration_tests.rs

@@ -490,6 +490,21 @@ params = [
 ]
 return_type = "result<record {{ a: u32, b: u32 }}, string>"
 
+[[activity_exec]]
+ffqn = "testing:integration/exec-stdin-args.echo-args"
+content = '''#!/usr/bin/env bash
+set -euo pipefail
+# Receives params via the stdin JSON `params` array instead of argv.
+jq -c '{{a: .params[0], b: .params[1]}}' /dev/stdin
+'''
+params = [
+  {{ name = "a", type = "u32" }},
+  {{ name = "b", type = "u32" }},
+]
+return_type = "result<record {{ a: u32, b: u32 }}, string>"
+env_vars = ["PATH"] # for jq
+params_via_stdin = true
+
 [[activity_exec]]
 ffqn = "testing:integration/exec-stream.stream-test"
 content = '''#!/usr/bin/env bash
@@ -3927,10 +3942,13 @@ async fn activity_exec_stdin_secrets() {
         .await;
     assert_eq!(resp.status().as_u16(), 201);
     let body: Value = resp.json().await.unwrap();
-    // Secrets are serialized as a JSON object to stdin; the script wraps it as a JSON string.
+    // Secrets are serialized as a JSON object under the `secrets` key to stdin; the script wraps it as a JSON string.
     let ok_val = body["ok"].as_str().expect("expected ok string");
     let parsed: Value = serde_json::from_str(ok_val).expect("inner value must be valid JSON");
-    assert_eq!(parsed, json!({ "MY_SECRET": "s3cret_value" }));
+    assert_eq!(
+        parsed,
+        json!({ "secrets": { "MY_SECRET": "s3cret_value" } })
+    );
     server.shutdown().await;
 }
 
@@ -3975,6 +3993,23 @@ async fn activity_exec_args_passthrough() {
     server.shutdown().await;
 }
 
+#[tokio::test]
+async fn activity_exec_args_via_stdin() {
+    let server = TestServer::start(test_addr!(82)).await;
+    let resp = server
+        .submit_follow(
+            "testing:integration/exec-stdin-args.echo-args",
+            vec![json!(10), json!(20)],
+        )
+        .await;
+    assert_eq!(resp.status().as_u16(), 201);
+    let body: Value = resp.json().await.unwrap();
+    // With params_via_stdin, params arrive in the stdin JSON `params` array
+    // (argv carries none); the script echoes them back as {"a": 10, "b": 20}.
+    assert_eq!(body, json!({ "ok": { "a": 10, "b": 20 } }));
+    server.shutdown().await;
+}
+
 #[tokio::test]
 async fn activity_exec_env_vars() {
     let server = TestServer::start(test_addr!(59)).await;

src/command/server.rs

@@ -103,7 +103,6 @@ use grpc::extractor::accept_trace;
 use grpc::grpc_gen;
 use hashbrown::HashMap;
 use indexmap::IndexMap;
-use secrecy::SecretString;
 use serde_json::json;
 use sha2::{Digest as _, Sha256};
 use std::fmt::Debug;
@@ -3823,19 +3822,9 @@ fn prespawn_activity_exec(
 
     let program = activity_exec.program;
 
-    // Compute stdin_content from resolved secrets.
-    // Secrets are serialized as a JSON object and piped to the child's stdin.
-    let stdin_content = activity_exec.secrets.map(|secrets| {
-        use secrecy::ExposeSecret;
-        let mut obj = serde_json::Map::new();
-        for (name, secret_val) in &secrets.env_vars {
-            obj.insert(
-                name.clone(),
-                serde_json::Value::String(secret_val.expose_secret().to_string()),
-            );
-        }
-        SecretString::from(serde_json::to_string(&obj).expect("JSON map serialization cannot fail"))
-    });
+    // The worker nests these secrets under the `secrets` key of the stdin JSON document
+    // at execution time.
+    let secrets = activity_exec.secrets.map(|secrets| secrets.env_vars);
 
     let worker = ActivityExecWorkerCompiled::new(
         program,
@@ -3846,7 +3835,8 @@ fn prespawn_activity_exec(
         activity_exec.max_output_bytes,
         activity_exec.forward_stdout,
         activity_exec.forward_stderr,
-        stdin_content,
+        secrets,
+        activity_exec.params_via_stdin,
     )
     .with_context(|| format!("cannot create exec activity worker for {component_id}"))?;
     let wit = worker.wit();