Skip to content

Commit 4b74986

Browse files
committed
feat(http): add HTTP and SSE streams extension with delivery semantics
1 parent 441546f commit 4b74986

35 files changed

Lines changed: 9978 additions & 76 deletions

Cargo.lock

Lines changed: 522 additions & 10 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,17 @@ name = "vm"
2929
default = ["runtime", "cli", "cranelift-jit"]
3030
runtime = []
3131
async = ["runtime", "dep:tokio"]
32+
http-client = [
33+
"async",
34+
"dep:futures-util",
35+
"dep:http-body-util",
36+
"dep:hyper",
37+
"dep:hyper-util",
38+
"dep:rustls",
39+
"dep:tokio-rustls",
40+
"dep:url",
41+
"dep:webpki-roots",
42+
]
3243
sqlite = ["runtime", "dep:rusqlite"]
3344
edge-abi = [
3445
"dep:edge_abi",
@@ -79,7 +90,6 @@ cranelift-module = { version = "0.129.1", optional = true }
7990
cranelift-native = { version = "0.129.1", optional = true }
8091
pd-host-function = { path = "./pd-host-function", version = "0.1.0" }
8192
rusqlite = { version = "0.32", default-features = false, features = ["bundled", "hooks", "limits"], optional = true }
82-
tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process"], optional = true }
8393
edge_abi = { package = "pd-edge-abi", version = "0.1.1", default-features = false, optional = true }
8494
futures-channel = "0.3"
8595
paste = "1"
@@ -90,6 +100,17 @@ rt-format = "0.3.1"
90100
self_cell = "1"
91101
rustyline = { version = "14", optional = true }
92102

103+
[target.'cfg(not(target_family = "wasm"))'.dependencies]
104+
http-body-util = { version = "0.1", optional = true }
105+
hyper = { version = "1", default-features = false, features = ["client", "http1"], optional = true }
106+
hyper-util = { version = "0.1", default-features = false, features = ["tokio"], optional = true }
107+
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"], optional = true }
108+
tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"], optional = true }
109+
url = { version = "2", optional = true }
110+
futures-util = { version = "0.3", optional = true }
111+
tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync", "fs", "io-util", "process", "macros"], optional = true }
112+
webpki-roots = { version = "1", optional = true }
113+
93114
[target.'cfg(windows)'.dependencies]
94115
windows-sys = { version = "0.59", features = ["Win32_System_Diagnostics_Debug", "Win32_System_Memory", "Win32_System_ProcessStatus", "Win32_System_Threading"] }
95116

@@ -99,6 +120,11 @@ libc = "0.2"
99120
[dev-dependencies]
100121
pd-host-schema = { path = "./crates/pd-host-schema", version = "0.1.0" }
101122
syn = { version = "2", features = ["full"] }
123+
124+
[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
125+
tokio = { version = "1", features = ["macros", "rt-multi-thread", "time", "sync"] }
126+
127+
[target.'cfg(target_family = "wasm")'.dev-dependencies]
102128
tokio = { version = "1", features = ["macros", "rt", "time", "sync"] }
103129

104130
[[test]]
@@ -126,6 +152,26 @@ name = "host_context_arch_tests"
126152
path = "tests/host_context_arch_tests.rs"
127153
required-features = ["runtime"]
128154

155+
[[test]]
156+
name = "http_host_tests"
157+
path = "tests/vm/http_host_tests.rs"
158+
required-features = ["runtime", "http-client"]
159+
160+
[[test]]
161+
name = "http_sse_tests"
162+
path = "tests/vm/http_sse_tests.rs"
163+
required-features = ["runtime", "http-client"]
164+
165+
[[test]]
166+
name = "io_http_coexistence_tests"
167+
path = "tests/vm/io_http_coexistence_tests.rs"
168+
required-features = ["runtime", "http-client"]
169+
170+
[[test]]
171+
name = "http_feature_gating_tests"
172+
path = "tests/http_feature_gating_tests.rs"
173+
required-features = ["runtime"]
174+
129175
[build-dependencies]
130176
pd-host-schema = { path = "./crates/pd-host-schema", version = "0.1.0" }
131177
syn = { version = "2", features = ["full"] }

build.rs

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,17 @@ struct NamespaceDecl {
129129
runtime_supported_on_wasm: bool,
130130
}
131131

132+
/// HTTP/SSE is a native transport extension. Keep this predicate identical to
133+
/// the `cfg` boundary used by the runtime and public exports: the Cargo
134+
/// feature remains selectable on wasm, but it must not publish transport
135+
/// sources or generated host/catalog entries there.
136+
pub(crate) fn http_transport_enabled(http_client_feature: bool, target_family: &str) -> bool {
137+
http_client_feature
138+
&& !target_family
139+
.split(',')
140+
.any(|family| family.trim() == "wasm")
141+
}
142+
132143
#[derive(Clone, Debug)]
133144
struct Group<'a> {
134145
key: String,
@@ -166,7 +177,8 @@ fn main() {
166177
catalog.retain(|entry| !entry.source_name.starts_with("sqlite::"));
167178
}
168179

169-
let host_sources = vec![
180+
let target_family = env::var("CARGO_CFG_TARGET_FAMILY").expect("missing target family");
181+
let mut host_sources = vec![
170182
SourceSpec {
171183
path: "src/builtins/runtime/host.rs".to_string(),
172184
module: "host".to_string(),
@@ -178,6 +190,21 @@ fn main() {
178190
category: SourceCategory::DefaultHost,
179191
},
180192
];
193+
if http_transport_enabled(
194+
env::var_os("CARGO_FEATURE_HTTP_CLIENT").is_some(),
195+
&target_family,
196+
) {
197+
host_sources.push(SourceSpec {
198+
path: "src/builtins/runtime/http/mod.rs".to_string(),
199+
module: "http".to_string(),
200+
category: SourceCategory::DefaultHost,
201+
});
202+
host_sources.push(SourceSpec {
203+
path: "src/builtins/runtime/http/sse.rs".to_string(),
204+
module: "http::sse".to_string(),
205+
category: SourceCategory::DefaultHost,
206+
});
207+
}
181208
let async_enabled = env::var_os("CARGO_FEATURE_ASYNC").is_some();
182209
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("missing target architecture");
183210
let builtin_sources = builtin_source_specs(&namespaces, async_enabled, &target_arch);
@@ -2216,11 +2243,21 @@ fn find_matching_paren(source: &str) -> usize {
22162243
#[cfg(test)]
22172244
mod tests {
22182245
use super::{
2219-
HostExecutionKind, NamespaceDecl, SourceCategory, builtin_source_specs, parse_source_file,
2220-
select_io_source_path,
2246+
HostExecutionKind, NamespaceDecl, SourceCategory, builtin_source_specs,
2247+
http_transport_enabled, parse_source_file, select_io_source_path,
22212248
};
22222249
use std::path::Path;
22232250

2251+
#[test]
2252+
fn http_transport_predicate_matches_source_and_catalog_boundary() {
2253+
assert!(http_transport_enabled(true, "unix"));
2254+
assert!(http_transport_enabled(true, "windows"));
2255+
assert!(!http_transport_enabled(true, "wasm"));
2256+
assert!(!http_transport_enabled(true, "wasm,unix"));
2257+
assert!(!http_transport_enabled(false, "unix"));
2258+
assert!(!http_transport_enabled(false, "wasm"));
2259+
}
2260+
22242261
fn io_namespace() -> NamespaceDecl {
22252262
NamespaceDecl {
22262263
namespace: "io".to_string(),

docs/http-client.md

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Native HTTP client and SSE feature
2+
3+
The `http-client` Cargo feature enables the buffered HTTP request and callable
4+
SSE host builtins on supported native targets. The feature name remains valid
5+
on every target so workspace feature selection stays uniform, but the native
6+
transport implementation is target-gated.
7+
8+
## Target boundary
9+
10+
The transport is compiled when both conditions hold:
11+
12+
- the `http-client` feature is enabled; and
13+
- the target is not in Rust's `wasm` target family (`not(target_family = "wasm")`).
14+
15+
On `wasm32-unknown-unknown` and other wasm-family targets, enabling
16+
`http-client` is intentionally a no-op for transport publication. Cargo does
17+
not build the native Tokio networking, Hyper, Rustls, URL, or HTTP-body
18+
transport dependencies. The HTTP module, `HttpConfig`/`HttpExtension`/
19+
`HttpHostExt` exports, HTTP builtins and callables, HTTP catalog functions, and
20+
LSP standard-catalog entries are absent. Browser or other wasm networking must
21+
be supplied by the embedding host instead.
22+
23+
The build script uses the same target-family boundary when it selects host
24+
source files for generated dispatch and catalog metadata. This keeps the
25+
compiled runtime surface and generated metadata synchronized.
26+
27+
## Native API
28+
29+
On a supported native target, enabling `http-client` preserves the public API:
30+
31+
- `HttpConfig` controls request and stream limits, redirects, timeouts, and
32+
capability policy;
33+
- `HttpExtension` and `HttpHostExt` install the native HTTP host integration;
34+
- `register_http_builtin_module` and `http_host_catalog` expose the native
35+
resource schema and callable metadata;
36+
- `http::client::request` returns a bounded response map; and
37+
- `http::client::sse` drives a bounded SSE stream through a script callback.
38+
39+
The HTTP and SSE behavior, resource lifecycle, cancellation, and native async
40+
bridge contracts are unchanged by the wasm boundary. See
41+
[`callable-runtime.md`](callable-runtime.md) for the general callable and
42+
host-runtime contract.
43+
44+
## Feature checks
45+
46+
For a native HTTP build:
47+
48+
```bash
49+
cargo test -p pd-vm --no-default-features --features runtime,http-client --test http_feature_gating_tests
50+
```
51+
52+
For the wasm gating check, keep the feature enabled while selecting a wasm
53+
package or target:
54+
55+
```bash
56+
cargo check -p pd-vm --no-default-features --features runtime,http-client \
57+
--target wasm32-unknown-unknown
58+
```
59+
60+
This verifies that feature selection is accepted without publishing the native
61+
transport surface.

src/builtins/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ mod metadata;
55
#[cfg(feature = "runtime")]
66
pub(crate) mod runtime;
77

8-
#[cfg(test)]
8+
#[allow(unused_imports)]
99
pub use self::metadata::CallableType;
1010
pub use self::metadata::{
1111
CallableDef, CallableParam, CallableParamType, CallableSignature, HostExecution,
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
use std::time::Duration;
2+
3+
use crate::vm::{VmError, VmResult};
4+
5+
/// Bounded network policy for the built-in HTTP client and future streaming adapters.
6+
#[derive(Clone, Debug, PartialEq, Eq)]
7+
pub struct HttpConfig {
8+
pub allowed_schemes: Vec<String>,
9+
pub allowed_hosts: Vec<String>,
10+
pub allowed_ports: Vec<u16>,
11+
pub max_redirects: usize,
12+
pub max_request_body_bytes: usize,
13+
/// Maximum number of caller-supplied request header fields. Client-managed
14+
/// fields such as `Host` are outside this extension surface.
15+
pub max_request_header_count: usize,
16+
/// Maximum serialized size of the caller-supplied request header block.
17+
/// Every field contributes `name + ": " + value + "\\r\\n"`; the final
18+
/// terminating `"\\r\\n"` is included as well.
19+
pub max_request_header_bytes: usize,
20+
pub max_response_body_bytes: usize,
21+
pub connect_timeout: Duration,
22+
pub request_timeout: Duration,
23+
pub allow_private_ips: bool,
24+
pub max_stream_item_bytes: usize,
25+
pub max_stream_total_bytes: usize,
26+
pub max_sse_line_bytes: usize,
27+
pub max_stream_duration: Duration,
28+
pub stream_idle_timeout: Duration,
29+
}
30+
31+
impl HttpConfig {
32+
/// Validates limits that must remain positive for every streaming adapter.
33+
pub fn validate(&self) -> VmResult<()> {
34+
let positive_limits = [
35+
("max_stream_item_bytes", self.max_stream_item_bytes),
36+
("max_stream_total_bytes", self.max_stream_total_bytes),
37+
("max_sse_line_bytes", self.max_sse_line_bytes),
38+
];
39+
if let Some((name, _)) = positive_limits.iter().find(|(_, value)| *value == 0) {
40+
return Err(VmError::HostError(format!(
41+
"HTTP configuration field '{name}' must be positive"
42+
)));
43+
}
44+
let positive_header_limits = [
45+
("max_request_header_count", self.max_request_header_count),
46+
("max_request_header_bytes", self.max_request_header_bytes),
47+
];
48+
if let Some((name, _)) = positive_header_limits.iter().find(|(_, value)| *value == 0) {
49+
return Err(VmError::HostError(format!(
50+
"HTTP configuration field '{name}' must be positive"
51+
)));
52+
}
53+
let positive_timeouts = [
54+
("connect_timeout", self.connect_timeout),
55+
("request_timeout", self.request_timeout),
56+
("max_stream_duration", self.max_stream_duration),
57+
("stream_idle_timeout", self.stream_idle_timeout),
58+
];
59+
if let Some((name, _)) = positive_timeouts
60+
.iter()
61+
.find(|(_, timeout)| timeout.is_zero())
62+
{
63+
return Err(VmError::HostError(format!(
64+
"HTTP configuration field '{name}' must be positive"
65+
)));
66+
}
67+
if let Some((name, _)) = positive_timeouts
68+
.iter()
69+
.find(|(_, timeout)| std::time::Instant::now().checked_add(*timeout).is_none())
70+
{
71+
return Err(VmError::HostError(format!(
72+
"HTTP configuration field '{name}' is too large"
73+
)));
74+
}
75+
Ok(())
76+
}
77+
}
78+
79+
impl Default for HttpConfig {
80+
fn default() -> Self {
81+
Self {
82+
allowed_schemes: vec!["https".to_string()],
83+
allowed_hosts: Vec::new(),
84+
allowed_ports: Vec::new(),
85+
max_redirects: 5,
86+
max_request_body_bytes: 1024 * 1024,
87+
max_request_header_count: 100,
88+
max_request_header_bytes: 64 * 1024,
89+
max_response_body_bytes: 8 * 1024 * 1024,
90+
connect_timeout: Duration::from_secs(10),
91+
request_timeout: Duration::from_secs(30),
92+
allow_private_ips: false,
93+
max_stream_item_bytes: 1024 * 1024,
94+
max_stream_total_bytes: 64 * 1024 * 1024,
95+
max_sse_line_bytes: 64 * 1024,
96+
max_stream_duration: Duration::from_secs(5 * 60),
97+
stream_idle_timeout: Duration::from_secs(30),
98+
}
99+
}
100+
}

0 commit comments

Comments
 (0)