Skip to content

multi-process support for dynamic loading #911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
qianxichen233 wants to merge 4 commits into
base: dylink_impl_integration
Choose a base branch
from
Open

multi-process support for dynamic loading #911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b4d7fb9
draft multi-process support for dynamic loading
qianxichen233 Mar 9, 2026
2b09aa4
execve support with dynamic loading
qianxichen233 Mar 9, 2026
712e529
signal support for dynamic loading
qianxichen233 Mar 9, 2026
d145779
threading support for dynamic loading
qianxichen233 Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion scripts/append_tls_relocs_export.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@ if ! command -v wasm2wat >/dev/null 2>&1 || ! command -v wat2wasm >/dev/null 2>&
exit 1
fi

INSERT_LINE=" (export \"__wasm_apply_tls_relocs\" (func \$__wasm_apply_tls_relocs))"
# INSERT_LINE=" (export \"__wasm_apply_tls_relocs\" (func \$__wasm_apply_tls_relocs))"
INSERT_LINE=" (export \"__wasm_apply_tls_relocs\" (func \$__wasm_apply_tls_relocs))\n (export \"__wasm_apply_global_relocs\" (func \$__wasm_apply_global_relocs))"
ANCHOR_LINE=" (export \"__wasm_apply_data_relocs\" (func \$__wasm_apply_data_relocs))"

tmpdir="$(mktemp -d)"
Expand Down
2 changes: 1 addition & 1 deletion scripts/make_glibc_and_sysroot.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ SYSROOT_ARCHIVE="$SYSROOT/lib/wasm32-wasi/libc.a"
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

# Define common flags
CFLAGS="--target=wasm32-unknown-wasi -v -Wno-int-conversion -DNO_HIDDEN -DNO_ASYNCIFY -std=gnu11 -fgnu89-inline -matomics -mbulk-memory -O2 -g -fPIC"
CFLAGS="--target=wasm32-unknown-wasi -v -Wno-int-conversion -DNO_HIDDEN -std=gnu11 -fgnu89-inline -matomics -mbulk-memory -O2 -g -fPIC"
WARNINGS="-Wall -Wwrite-strings -Wundef -Wstrict-prototypes -Wold-style-definition"
EXTRA_FLAGS="-fmerge-all-constants -ftrapping-math -fno-stack-protector -fno-common"
EXTRA_FLAGS+=" -Wp,-U_FORTIFY_SOURCE -fmath-errno -fPIE -ftls-model=local-exec"
Expand Down
4 changes: 3 additions & 1 deletion scripts/make_shared_glibc.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3120,4 +3120,6 @@ wasm-ld \

mkdir -p /home/lind/lind-wasm/lindfs/lib

sudo ./scripts/append_tls_relocs_export.sh "$SYSROOT/lib/wasm32-wasi/libc.so" /home/lind/lind-wasm/lindfs/lib/libc.so
sudo ./scripts/append_tls_relocs_export.sh "$SYSROOT/lib/wasm32-wasi/libc.so" /home/lind/lind-wasm/lindfs/lib/libc.wasm
sudo /home/lind/binaryen-epoch-injection/bin/wasm-opt --enable-bulk-memory --enable-threads --epoch-injection --pass-arg=epoch-import --asyncify --pass-arg=asyncify-import-globals -O2 --debuginfo /home/lind/lind-wasm/lindfs/lib/libc.wasm -o /home/lind/lind-wasm/lindfs/lib/libc.wasm
sudo /home/lind/lind-wasm/scripts/lind_compile --precompile-only /home/lind/lind-wasm/lindfs/lib/libc.wasm
200 changes: 103 additions & 97 deletions src/lind-boot/src/lind_wasmtime/execute.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,6 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
// new cage is created
lind_manager.increment();

// create Global Offset Table for dynamic loading
// #[cfg(feature = "dylink-support")]
let mut lind_got = Arc::new(Mutex::new(LindGOT::new()));

// Initialize vmctx pool
init_vmctx_pool();
// Initialize trampoline entry function pointer for wasmtime runtime.
Expand All @@ -82,6 +78,55 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
panic!("[lind-boot] egister syscall handlers (clone/exec/exit) with 3i failed");
}

// -- Run the first module in the first cage --
let result = execute_with_lind(lindboot_cli, lind_manager.clone(), CAGE_START_ID as u64);

match result {
Ok(ref _res) => {
// we wait until all other cage exits
lind_manager.wait();
}
Err(e) => {
// Exit the process if Wasmtime understands the error;
// otherwise, fall back on Rust's default error printing/return
// code.
return Err(wasi_common::maybe_exit_on_error(e));
}
}

result
}

/// Executes a Wasm program *within an existing Lind runtime* as part of an `exec()` path.
///
/// This function is not used for the initial launch. Instead, it is invoked only when
/// the guest issues `exec`, at which point the runtime must load a new Wasm module
/// (or the same module with different args/env) into a target cage while keeping the
/// global Lind/RawPOSIX runtime alive.
///
/// Unlike `execute()`, this function does not call `rawposix_start`, does not create
/// the initial cage manager, and does not register 3i handlers. The handler table is
/// already present: forked processes inherit it via RawPOSIX table cloning, and exec
/// does not require mutating it. The goal here is to perform the minimal work needed
/// to re-create a Wasmtime engine/store, attach host APIs, instantiate the module
/// inside the provided `cageid`, and transfer control to the new guest entrypoint.
pub fn execute_with_lind(
lind_boot: CliOptions,
lind_manager: Arc<LindCageManager>,
cageid: u64,
) -> Result<Vec<Val>> {
// -- Initialize the Wasmtime execution environment --
let wasm_file_path = Path::new(lind_boot.wasm_file());
let args = lind_boot.args.clone();
let wt_config = make_wasmtime_config(lind_boot.wasmtime_backtrace);
let engine = Engine::new(&wt_config).context("failed to create execution engine")?;
let host = HostCtx::default();
let mut wstore = Store::new(&engine, host);

// create Global Offset Table for dynamic loading
// #[cfg(feature = "dylink-support")]
let mut lind_got = Arc::new(Mutex::new(LindGOT::new()));

// -- Load module and Attach host APIs --
// Set up the WASI. In lind-wasm, we predefine all the features we need are `thread` and `wasipreview1`
// so we manually add them to the linker without checking the input
Expand Down Expand Up @@ -173,7 +218,7 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
let table_base = wasmtime::Global::new(
&mut wstore,
wasmtime::GlobalType::new(ValType::I32, wasmtime::Mutability::Const),
Val::I32(0),
Val::I32(1),
)
.unwrap();
linker.define(&mut wstore, "env", "__memory_base", memory_base);
Expand All @@ -186,20 +231,47 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
drop(got_guard);
}

let epoch_handler = {
let mut linker = linker.lock().unwrap();
let __asyncify_state = wasmtime::Global::new(
&mut wstore,
wasmtime::GlobalType::new(ValType::I32, wasmtime::Mutability::Var),
Val::I32(0),
)
.unwrap();
let __asyncify_data = wasmtime::Global::new(
&mut wstore,
wasmtime::GlobalType::new(ValType::I32, wasmtime::Mutability::Var),
Val::I32(0),
)
.unwrap();
let lind_epoch = wasmtime::Global::new(
&mut wstore,
wasmtime::GlobalType::new(ValType::I64, wasmtime::Mutability::Var),
Val::I64(0),
)
.unwrap();
linker.define(&mut wstore, "env", "__asyncify_state", __asyncify_state);
linker.define(&mut wstore, "env", "__asyncify_data", __asyncify_data);
linker.define(&mut wstore, "lind", "epoch", lind_epoch);

lind_epoch.get_handler_as_u64(&mut wstore) as u64
};

attach_api(
&mut wstore,
&mut linker,
&module,
lind_manager.clone(),
lindboot_cli.clone(),
None,
lind_got.clone()
lind_boot.clone(),
cageid as i32,
lind_got.clone(),
)?;

// Load the preload wasm modules.
let mut modules = Vec::new();
modules.push((String::new(), module.clone()));
for (name, path) in lindboot_cli.preloads.iter() {
for (name, path) in lind_boot.preloads.iter() {
// Read the wasm module binary either as `*.wat` or a raw binary
let module = read_wasm_or_cwasm(&engine, path)?;
modules.push((name.clone(), module.clone()));
Expand All @@ -223,6 +295,7 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {

// Add the module's functions to the linker.
for (name, module) in modules.iter().skip(1) {
println!("[debug] link module {}", name);
let mut lib_linker = linker.lock().unwrap();

// Read dylink metadata for this preloaded (library) module.
Expand Down Expand Up @@ -262,7 +335,7 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
&module,
&mut table,
table_start,
&*guard,
Some(&*guard),
)
.context(format!("failed to process preload `{}`", name,))?;
}
Expand All @@ -282,83 +355,13 @@ pub fn execute_wasmtime(lindboot_cli: CliOptions) -> anyhow::Result<Vec<Val>> {
guard.warning_undefined();
}

// -- Run the first module in the first cage --
let result = wasmtime_wasi::runtime::with_ambient_tokio_runtime(|| {
load_main_module(
&mut wstore,
&mut linker,
&module,
CAGE_START_ID as u64,
&args,
lind_got.clone(),
&mut table,
)
.with_context(|| format!("failed to run main module"))
});

match result {
Ok(ref _res) => {
// we wait until all other cage exits
lind_manager.wait();
}
Err(e) => {
// Exit the process if Wasmtime understands the error;
// otherwise, fall back on Rust's default error printing/return
// code.
return Err(wasi_common::maybe_exit_on_error(e));
}
{
let mut ctx = wstore.data_mut().lind_fork_ctx.as_mut().unwrap();
let mut linker = linker.lock().unwrap();
ctx.update_linker(linker.clone());
ctx.update_modules(modules.clone());
}

result
}

/// Executes a Wasm program *within an existing Lind runtime* as part of an `exec()` path.
///
/// This function is not used for the initial launch. Instead, it is invoked only when
/// the guest issues `exec`, at which point the runtime must load a new Wasm module
/// (or the same module with different args/env) into a target cage while keeping the
/// global Lind/RawPOSIX runtime alive.
///
/// Unlike `execute()`, this function does not call `rawposix_start`, does not create
/// the initial cage manager, and does not register 3i handlers. The handler table is
/// already present: forked processes inherit it via RawPOSIX table cloning, and exec
/// does not require mutating it. The goal here is to perform the minimal work needed
/// to re-create a Wasmtime engine/store, attach host APIs, instantiate the module
/// inside the provided `cageid`, and transfer control to the new guest entrypoint.
pub fn execute_with_lind(
lind_boot: CliOptions,
lind_manager: Arc<LindCageManager>,
cageid: u64,
) -> Result<Vec<Val>> {
// -- Initialize the Wasmtime execution environment --
let wasm_file_path = Path::new(lind_boot.wasm_file());
let args = lind_boot.args.clone();
let wt_config = make_wasmtime_config(lind_boot.wasmtime_backtrace);
let engine = Engine::new(&wt_config).context("failed to create execution engine")?;
let host = HostCtx::default();
let mut wstore = Store::new(&engine, host);

// -- Load module and Attach host APIs --
// Set up the WASI. In lind-wasm, we predefine all the features we need are `thread` and `wasipreview1`
// so we manually add them to the linker without checking the input
let module = read_wasm_or_cwasm(&engine, wasm_file_path)?;
let mut linker = Arc::new(Mutex::new(Linker::new(&engine)));
// create Global Offset Table for dynamic loading
// #[cfg(feature = "dylink-support")]
let mut lind_got = Arc::new(Mutex::new(LindGOT::new()));
let ty = wasmtime::TableType::new(wasmtime::RefType::FUNCREF, 0, None);
let mut table = wasmtime::Table::new(&mut wstore, ty, wasmtime::Ref::Func(None)).unwrap();

attach_api(
&mut wstore,
&mut linker,
&module,
lind_manager.clone(),
lind_boot.clone(),
Some(cageid as i32),
lind_got.clone(),
)?;

// -- Run the module in the cage --
let result = wasmtime_wasi::runtime::with_ambient_tokio_runtime(|| {
load_main_module(
Expand All @@ -369,6 +372,7 @@ pub fn execute_with_lind(
&args,
lind_got,
&mut table,
epoch_handler,
)
.with_context(|| format!("failed to run main module"))
});
Expand Down Expand Up @@ -488,7 +492,7 @@ fn attach_api(
module: &Module,
lind_manager: Arc<LindCageManager>,
lindboot_cli: CliOptions,
cageid: Option<i32>,
cageid: i32,
got: Arc<Mutex<LindGOT>>,
) -> Result<()> {
// Setup WASI-p1
Expand Down Expand Up @@ -565,7 +569,7 @@ fn attach_api(

// attach Lind-Multi-Process-Context to the host
let _ = wstore.data_mut().lind_fork_ctx = Some(LindCtx::new(
module.clone(),
Vec::<(String, Module)>::new(),
linker_guard.clone(),
lind_manager.clone(),
lindboot_cli.clone(),
Expand Down Expand Up @@ -603,6 +607,7 @@ fn load_main_module(
args: &[String],
got: Arc<Mutex<LindGOT>>,
table: &mut wasmtime::Table,
epoch_handler: u64,
) -> Result<Vec<Val>> {
let mut linker_guard = linker.lock().unwrap();

Expand Down Expand Up @@ -633,8 +638,8 @@ fn load_main_module(

// let stack_low = instance.get_stack_low(store.as_context_mut()).unwrap();
// let stack_pointer = instance.get_stack_pointer(store.as_context_mut()).unwrap();
let stack_low = 0;
let stack_pointer = 0;
let stack_low = 1024;
Copy link
Contributor

@rennergade rennergade Mar 9, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lets define constants and not use magic numbers

let stack_pointer = 1024 + 8388608;
store.as_context_mut().set_stack_base(stack_pointer as u64);
store.as_context_mut().set_stack_top(stack_low as u64);

Expand All @@ -645,14 +650,15 @@ fn load_main_module(
if #[cfg(feature = "disable_signals")] {
let pointer = 0;
} else {
// retrieve the epoch global
let lind_epoch = instance
.get_export(&mut *store, "epoch")
.and_then(|export| export.into_global())
.expect("Failed to find epoch global export!");

// retrieve the handler (underlying pointer) for the epoch global
let pointer = lind_epoch.get_handler_as_u64(&mut *store);
// // retrieve the epoch global
// let lind_epoch = instance
// .get_export(&mut *store, "epoch")
// .and_then(|export| export.into_global())
// .expect("Failed to find epoch global export!");

// // retrieve the handler (underlying pointer) for the epoch global
// let pointer = lind_epoch.get_handler_as_u64(&mut *store);
let pointer = epoch_handler;
}
}

Expand Down
7 changes: 4 additions & 3 deletions src/wasmtime/crates/lind-common/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@ pub fn add_to_linker<
"lind",
"lind-get-cage-id",
move |mut caller: Caller<'_, T>| -> u64 {
wasmtime_lind_multi_process::current_cageid(&mut caller) as u64
let cageid = wasmtime_lind_multi_process::current_cageid(&mut caller) as u64;
cageid
},
)?;

Expand Down Expand Up @@ -223,8 +224,8 @@ pub fn add_to_linker<
linker.func_wrap(
"debug",
"lind_debug_str",
move |caller: Caller<'_, T>, ptr: i32| -> i32 {
let mem_base = get_memory_base(&caller);
move |mut caller: Caller<'_, T>, ptr: i32| -> i32 {
let mem_base = get_memory_base(&mut caller);
if let Ok(msg) = get_cstr(mem_base + (ptr as u32) as u64) {
eprintln!("[LIND DEBUG STR]: {}", msg);
}
Expand Down
Loading
Loading