diff --git a/crates/wasmi/src/engine/executor/handler/args.rs b/crates/wasmi/src/engine/executor/handler/args.rs index baa4464e9c..c584eb4a6e 100644 --- a/crates/wasmi/src/engine/executor/handler/args.rs +++ b/crates/wasmi/src/engine/executor/handler/args.rs @@ -152,7 +152,7 @@ impl Args { /// Returns the bytes of the default memory at index 0. #[inline] - pub fn fetch_default_memory_bytes<'a>(&mut self) -> &'a mut [u8] { + pub fn fetch_default_memory_bytes<'a>(&mut self, _state: &'a mut VmState) -> &'a mut [u8] { state::mem0_bytes::<'a>(self.mem0_ptr, self.mem0_len) } @@ -166,7 +166,10 @@ impl Args { where Inst: LoadEntity, { - self.instance.load(state.store.inner_mut(), addr) + // SAFETY: `addr` stems from a Wasmi IR operator and thus addresses a memory entry of + // `self.instance` whose cache was warmed at instantiation. The `state` borrow + // scopes the returned reference. + unsafe { self.instance.load_entity_mut(state.store, addr) } } /// Returns an exclusive reference to the global at `index`. @@ -179,7 +182,10 @@ impl Args { where Inst: LoadEntity, { - self.instance.load(state.store.inner_mut(), addr) + // SAFETY: `addr` stems from a Wasmi IR operator and thus addresses a global entry of + // `self.instance` whose cache was warmed at instantiation. The `state` borrow + // scopes the returned reference. + unsafe { self.instance.load_entity_mut(state.store, addr) } } /// Returns an exclusive reference to the table at `index`. @@ -192,7 +198,10 @@ impl Args { where Inst: LoadEntity, { - self.instance.load(state.store.inner_mut(), addr) + // SAFETY: `addr` stems from a Wasmi IR operator and thus addresses a table entry of + // `self.instance` whose cache was warmed at instantiation. The `state` borrow + // scopes the returned reference. + unsafe { self.instance.load_entity_mut(state.store, addr) } } /// Returns an exclusive reference to the element segment at `index`. @@ -205,7 +214,10 @@ impl Args { where Inst: LoadEntity, { - self.instance.load(state.store.inner_mut(), addr) + // SAFETY: `addr` stems from a Wasmi IR operator and thus addresses an element segment + // entry of `self.instance` whose cache was warmed at instantiation. The `state` + // borrow scopes the returned reference. + unsafe { self.instance.load_entity_mut(state.store, addr) } } /// Returns an exclusive reference to the data segment at `index`. @@ -218,7 +230,10 @@ impl Args { where Inst: LoadEntity, { - self.instance.load(state.store.inner_mut(), addr) + // SAFETY: `addr` stems from a Wasmi IR operator and thus addresses a data segment entry + // of `self.instance` whose cache was warmed at instantiation. The `state` borrow + // scopes the returned reference. + unsafe { self.instance.load_entity_mut(state.store, addr) } } /// Reloads the data pointer and length of the default memory at index 0 from `state`. diff --git a/crates/wasmi/src/engine/executor/handler/exec.rs b/crates/wasmi/src/engine/executor/handler/exec.rs index d86e0e5536..b3e1f88ec8 100644 --- a/crates/wasmi/src/engine/executor/handler/exec.rs +++ b/crates/wasmi/src/engine/executor/handler/exec.rs @@ -11,7 +11,6 @@ use super::{ Args, dispatch::Done, state::{Freg32, Freg64, Inst, Ip, Ireg, Mem0Len, Mem0Ptr, Sp, VmState}, - utils::fetch_func, }; #[cfg(feature = "simd")] use crate::V128; @@ -25,7 +24,7 @@ use crate::{ Control, dispatch::Break, state::DoneReason, - utils::{self, GetValue, IntoControl as _}, + utils::{self, GetValue, IntoControl as _, LoadEntity, LoadHandle as _}, }, utils::unreachable_unchecked, }, @@ -405,7 +404,7 @@ execution_handler! { delta, } = unsafe { args.decode_op() }; let delta: u64 = args.get(delta); - let memref = utils::fetch_memory(instance, memory); + let memref = unsafe { instance.load_handle(memory) }; let return_value = match state.store.grow_memory(&memref, delta) { Ok(return_value) => { // The `memory.grow` operation might have invalidated the cached @@ -419,7 +418,8 @@ execution_handler! { Err(StoreError::External( MemoryError::OutOfBoundsGrowth | MemoryError::OutOfSystemMemory, )) => { - let memory_ty = utils::resolve_memory(state.store, &memref).ty(); + let memory = unsafe { instance.load_entity_mut(state.store, memory) }; + let memory_ty = memory.ty(); match memory_ty.is_64() { true => u64::MAX, false => u64::from(u32::MAX), @@ -480,8 +480,8 @@ execution_handler! { memory_copy_within(state, &mut args, ip, dst_memory, dst_index, src_index, len)?; dispatch!(state, args) } - let src_ptr = unsafe { utils::load_memory_ptr(instance, src_memory) }; - let mut dst_ptr = unsafe { utils::load_memory_ptr(instance, dst_memory) }; + let src_ptr = unsafe { instance.load_entity_ptr(src_memory) }; + let mut dst_ptr = unsafe { instance.load_entity_ptr(dst_memory) }; if src_ptr == dst_ptr { // Distinct memory indices can still resolve to the same store entity (e.g. the same // memory imported under two names), so branch on the resolved entity before forming @@ -520,7 +520,7 @@ fn memory_copy_within( len: usize, ) -> Control<(), Break> { // SAFETY: `args.instance` is live and warmed up; `memory` is the only entity accessed here. - let memory = unsafe { utils::load_memory_ptr(args.instance, dst_memory).as_mut() }; + let memory = unsafe { args.instance.load_entity_ptr(dst_memory).as_mut() }; let fuel = state.store.inner_mut().fuel_mut(); // These accesses just perform the bounds checks required by the Wasm spec. utils::memory_slice(memory, src_index, len).into_control()?; @@ -562,7 +562,7 @@ execution_handler! { trap!(TrapCode::MemoryOutOfBounds) }; // SAFETY: `instance` is live and warmed up; `memory` is the only entity accessed here. - let memory = unsafe { utils::load_memory_ptr(instance, memory).as_mut() }; + let memory = unsafe { instance.load_entity_ptr(memory).as_mut() }; let fuel = state.store.inner_mut().fuel_mut(); let slice = utils::memory_slice_mut(memory, dst, len).into_control()?; consume_fuel!(state, ip, args, fuel, |costs| costs.fuel_for_copying_values::(len as u64)); @@ -605,8 +605,8 @@ execution_handler! { }; // SAFETY: `instance` is live and warmed up. `memory` and `data` live in different // arenas, so their references are to distinct entities and cannot alias. - let memory = unsafe { utils::load_memory_ptr(instance, memory).as_mut() }; - let data = unsafe { utils::load_data_ptr(instance, data).as_ref() }; + let memory = unsafe { instance.load_entity_ptr(memory).as_mut() }; + let data = unsafe { instance.load_entity_ptr(data).as_ref() }; let fuel = state.store.inner_mut().fuel_mut(); let memory = utils::memory_slice_mut(memory, dst_index, len).into_control()?; let Some(data) = data @@ -680,13 +680,13 @@ execution_handler! { delta, value, } = unsafe { args.decode_op() }; - let table = utils::fetch_table(instance, table); + let table_ref = unsafe { instance.load_handle(table) }; let delta = args.get(delta); let value = args.get(value); - let return_value = match state.store.grow_table(&table, delta, value) { + let return_value = match state.store.grow_table(&table_ref, delta, value) { Ok(return_value) => return_value, Err(StoreError::External(TableError::GrowOutOfBounds | TableError::OutOfSystemMemory)) => { - let table = utils::resolve_table(state.store, &table); + let table = unsafe { instance.load_entity_mut(state.store, table) }; match table.ty().is_64() { true => u64::MAX, false => u64::from(u32::MAX), @@ -733,8 +733,8 @@ execution_handler! { let dst: u64 = args.get(dst); let src: u64 = args.get(src); let len: u64 = args.get(len); - let src_ptr = unsafe { utils::load_table_ptr(instance, src_table) }; - let mut dst_ptr = unsafe { utils::load_table_ptr(instance, dst_table) }; + let src_ptr = unsafe { instance.load_entity_ptr(src_table) }; + let mut dst_ptr = unsafe { instance.load_entity_ptr(dst_table) }; // Distinct table indices can still resolve to the same store entity (e.g. the same table // imported under two names), so branch on the resolved entity, not just the index. let result = if src_ptr == dst_ptr { @@ -791,7 +791,7 @@ execution_handler! { let len: u64 = args.get(len); let value: RawRef = args.get(value); // SAFETY: `instance` is live and warmed up; `table` is the only entity accessed here. - let table = unsafe { utils::load_table_ptr(instance, table).as_mut() }; + let table = unsafe { instance.load_entity_ptr(table).as_mut() }; let fuel = state.store.inner_mut().fuel_mut(); if let Err(error) = table.fill_raw(dst, value, len, Some(fuel)) { let trap_code = match error { @@ -834,8 +834,8 @@ execution_handler! { let len: u32 = args.get(len); // SAFETY: `args.instance` is live and warmed up. `table` and `elem` live in different // arenas, so their references are to distinct entities and cannot alias. - let table = unsafe { utils::load_table_ptr(args.instance, table).as_mut() }; - let element = unsafe { utils::load_elem_ptr(args.instance, elem).as_ref() }; + let table = unsafe { args.instance.load_entity_ptr(table).as_mut() }; + let element = unsafe { args.instance.load_entity_ptr(elem).as_ref() }; let fuel = state.store.inner_mut().fuel_mut(); if let Err(error) = table.init(element.as_ref(), dst, src, len, Some(fuel)) { let trap_code = match error { @@ -961,7 +961,7 @@ execution_handler! { ) -> Done = { let mut args = Args::from_parts(ip, sp, mem0, mem0_len, instance, ireg, freg32, freg64); let crate::ir::decode::RefFunc { func, result } = unsafe { args.decode_op() }; - let func = fetch_func(instance, func); + let func = unsafe { instance.load_handle(func) }; let Some(rawref) = func.unwrap_raw(&*state.store) else { unsafe { unreachable_unchecked!("store mismatch with: {func:?}") } }; diff --git a/crates/wasmi/src/engine/executor/handler/exec/macros.rs b/crates/wasmi/src/engine/executor/handler/exec/macros.rs index 620bf9bb72..301afcb4b2 100644 --- a/crates/wasmi/src/engine/executor/handler/exec/macros.rs +++ b/crates/wasmi/src/engine/executor/handler/exec/macros.rs @@ -200,7 +200,7 @@ macro_rules! handler_load_mem0_offset16 { } = unsafe { args.decode_op() }; let ptr = args.get(ptr); let offset = args.get(offset); - let bytes = args.fetch_default_memory_bytes(); + let bytes = args.fetch_default_memory_bytes(state); let loaded = $load(bytes, ptr, u64::from(offset)).into_control()?; args.set(result, loaded); dispatch!(state, args) @@ -268,7 +268,7 @@ macro_rules! handler_store_mem0_offset16 { let ptr = args.get(ptr); let offset = args.get(offset); let value: $hint = args.get(value); - let bytes = args.fetch_default_memory_bytes(); + let bytes = args.fetch_default_memory_bytes(state); $store(bytes, ptr, u64::from(offset), value.into()).into_control()?; dispatch!(state, args) } diff --git a/crates/wasmi/src/engine/executor/handler/exec/simd.rs b/crates/wasmi/src/engine/executor/handler/exec/simd.rs index 0e458e315f..1247c06374 100644 --- a/crates/wasmi/src/engine/executor/handler/exec/simd.rs +++ b/crates/wasmi/src/engine/executor/handler/exec/simd.rs @@ -540,7 +540,7 @@ macro_rules! handler_store_lane_mem0_offset16_ss { let ptr = args.get(ptr); let offset = args.get(offset); let value = args.get(value); - let bytes = args.fetch_default_memory_bytes(); + let bytes = args.fetch_default_memory_bytes(state); $eval(bytes, ptr, u64::from(offset), value, lane).into_control()?; dispatch!(state, args) } diff --git a/crates/wasmi/src/engine/executor/handler/utils.rs b/crates/wasmi/src/engine/executor/handler/utils.rs index a2e4c4051d..74398c7362 100644 --- a/crates/wasmi/src/engine/executor/handler/utils.rs +++ b/crates/wasmi/src/engine/executor/handler/utils.rs @@ -2,25 +2,21 @@ use super::state::{Freg32, Freg64, Inst, Ip, Ireg, Mem0Len, Mem0Ptr, Sp, VmState #[cfg(feature = "simd")] use crate::core::simd::ImmLaneIdx; #[cfg(doc)] -use crate::instance::InstanceEntity; +use crate::instance::{InstanceEntity, ThinPtr}; use crate::{ + DataSegment, + ElementSegment, Error, Func, + Global, + Handle, Memory, Nullable, RefType, Table, TrapCode, V128, - core::{ - CoreElementSegment as ElementSegmentEntity, - CoreGlobal as GlobalEntity, - CoreMemory as MemoryEntity, - CoreTable as TableEntity, - CoreTable, - RawVal, - ShiftAmount, - }, + core::{CoreMemory as MemoryEntity, CoreTable as TableEntity, RawVal, ShiftAmount}, engine::{ FuncEntry, InOutParams, @@ -32,7 +28,7 @@ use crate::{ utils::unreachable_unchecked, }, func::{FuncEntity, HostFuncEntity, Trampoline}, - instance::{DataAddr, ElemAddr, FuncAddr, GlobalAddr, MemoryAddr, TableAddr}, + instance::{DataAddr, ElemAddr, FuncAddr, GlobalAddr, HandleAndEntity, MemoryAddr, TableAddr}, ir::{ self, Address, @@ -45,8 +41,7 @@ use crate::{ SlotSpan, Table0, }, - memory::DataSegmentEntity, - store::{CallHooks, PrunedStore, StoreError, StoreInner}, + store::{CallHooks, PrunedStore, StoreError}, }; use core::{num::NonZero, ptr::NonNull}; @@ -560,8 +555,11 @@ pub fn extract_mem0(_store: &mut PrunedStore, inst: Inst) -> (Mem0Ptr, Mem0Len) let Some(mem0) = (unsafe { inst.as_ptr().get_memory(addr) }) else { unsafe { unreachable_unchecked!("missing memory at: {addr:?}") } }; - // SAFETY: warmed at instantiation; the `_store` borrow scopes exclusive memory access. + // SAFETY: the entry stems from the instance layout and is thus in bounds; the reference + // does not outlive this function. + let mem0 = unsafe { mem0.as_ref() }; let mem0 = mem0.entity(); + // SAFETY: warmed at instantiation; the `_store` borrow scopes exclusive memory access. let mem0 = unsafe { &mut *mem0.as_ptr() }.data_mut(); let mem0_ptr = mem0.as_mut_ptr(); let mem0_len = mem0.len(); @@ -588,107 +586,195 @@ pub fn memory_slice_mut( .ok_or(TrapCode::MemoryOutOfBounds) } -/// Extension trait for [`Inst`] to safely(ish) load entities from their addresses. -pub trait LoadEntity { - /// The loaded entity type. - type Entity; +/// Extension trait for [`Inst`] to load typed entries from their addresses. +trait LoadEntry { + /// The type of the loaded entry. + type Entry; - /// Loads the entity from the warmed up instance cache. + /// Returns a pointer to the entry at `addr` of the instance cache. /// - /// The `store` borrow is not read; it ties the returned reference so that it - /// cannot alias a concurrent store mutation making this a safe wrapper. - fn load(self, store: &mut StoreInner, addr: Addr) -> &mut Self::Entity; + /// # Safety + /// + /// In addition to the requirements of [`ThinPtr::as_ref`] the caller must ensure that + /// `addr` is in bounds of `self` and that the entry it addresses stores a handle of the + /// associated kind. Wasmi's translation guarantees both for every address encoded into a + /// Wasmi IR operator. + /// + /// The returned pointer is only sound to dereference while `self` refers to a live + /// [`InstanceEntity`]. + unsafe fn load_entry_ptr(self, addr: Addr) -> NonNull; } -macro_rules! impl_load_entity_for_inst { +macro_rules! impl_load_entry_for_inst { ( - $( fn $fn:ident(inst: Inst, store: &mut StoreInner, $param:ident: ir::$param_ty:ident) -> &mut $ret:ty = $entry:ident );* $(;)? + $( + ir::$addr:ident -> $handle:ident = $getter:ident + );* $(;)? ) => { $( - impl LoadEntity for Inst { - type Entity = $ret; + impl LoadEntry for Inst { + type Entry = HandleAndEntity<$handle>; #[inline] - fn load(self, _store: &mut StoreInner, $param: ir::$param_ty) -> &mut Self::Entity { - let addr = <$param_ty>::from(::core::primitive::u32::from($param)); + unsafe fn load_entry_ptr(self, addr: ir::$addr) -> NonNull { + let addr = <$addr>::from(::core::primitive::u32::from(addr)); // SAFETY: `addr` addresses an entry of this kind by translation invariant. - let Some(entry) = (unsafe { self.as_ptr().$entry(addr) }) else { + let Some(entry) = (unsafe { self.as_ptr().$getter(addr) }) else { unsafe { $crate::engine::utils::unreachable_unchecked!( - ::core::concat!("missing ", ::core::stringify!($param), " at: {:?}"), + ::core::concat!("missing ", ::core::stringify!($handle), " at: {:?}"), addr, ) } }; - // SAFETY: warmed at instantiation; the `_store` borrow scopes the reference. - unsafe { &mut *entry.entity().as_ptr() } + entry + } + } + )* + }; +} +impl_load_entry_for_inst! { + ir::GlobalAddr -> Global = get_global; + ir::MemoryAddr -> Memory = get_memory; + ir::TableAddr -> Table = get_table; + ir::FuncAddr -> Func = get_func; + ir::DataAddr -> DataSegment = get_data; + ir::ElemAddr -> ElementSegment = get_elem; +} + +/// Extension trait for [`Inst`] to load entities from their addresses. +pub trait LoadEntity { + /// The type of the loaded handle entity. + type Entity; + + /// Returns a pointer to the entity at `addr` of the warmed up instance cache. + /// + /// Unlike [`LoadEntity::load_entity_mut`] this returns a raw [`NonNull`] instead of a + /// reference tied to the store. This lets the caller hold several entity pointers (or an + /// entity pointer alongside a `&mut` borrow of `fuel`) at once and materialize a reference + /// only in the narrow scope where the entity is actually accessed. + /// + /// # Safety + /// + /// Same as for [`LoadEntry::load_entry_ptr`]. Additionally the entry's cache must have been + /// warmed up, and the returned pointer must not be turned into a reference that aliases a + /// concurrent store mutation or another live reference to the same entity. + unsafe fn load_entity_ptr(self, addr: Addr) -> NonNull; + + /// Returns an exclusive reference to the entity at `addr` of the warmed up instance cache. + /// + /// The `store` borrow is not read; it ties the returned reference so that it + /// cannot alias a concurrent store mutation. + /// + /// # Safety + /// + /// Same as for [`LoadEntity::load_entity_ptr`]. The `store` borrow rules out store mutations + /// for the lifetime of the returned reference but not a second reference to the same entity + /// loaded from `self`. + unsafe fn load_entity_mut(self, _store: &mut PrunedStore, addr: Addr) -> &mut Self::Entity; +} + +macro_rules! impl_load_entity_for_inst { + ( + $( + ir::$addr:ident -> $handle:ident + );* $(;)? + ) => { + $( + impl LoadEntity for Inst { + type Entity = <$handle as Handle>::Entity; + + #[inline] + unsafe fn load_entity_ptr(self, addr: ir::$addr) -> NonNull { + // SAFETY: guaranteed by the caller; the reference does not outlive the call + // and the returned pointer is a copy of the cached entity pointer, + // so it does not derive its provenance from that reference. + unsafe { self.load_entry_ptr(addr).as_ref() }.entity() + } + + #[inline] + unsafe fn load_entity_mut(self, _store: &mut PrunedStore, addr: ir::$addr) -> &mut Self::Entity { + // SAFETY: guaranteed by the caller; the `_store` borrow scopes the returned + // reference against a concurrent store mutation. + unsafe { self.load_entity_ptr(addr).as_mut() } } } )* }; } impl_load_entity_for_inst! { - fn load_data(inst: Inst, store: &mut StoreInner, data: ir::DataAddr) -> &mut DataSegmentEntity = get_data; - fn load_elem(inst: Inst, store: &mut StoreInner, elem: ir::ElemAddr) -> &mut ElementSegmentEntity = get_elem; - fn load_global(inst: Inst, store: &mut StoreInner, global: ir::GlobalAddr) -> &mut GlobalEntity = get_global; - fn load_memory(inst: Inst, store: &mut StoreInner, memory: ir::MemoryAddr) -> &mut MemoryEntity = get_memory; - fn load_table(inst: Inst, store: &mut StoreInner, table: ir::TableAddr) -> &mut TableEntity = get_table; + ir::GlobalAddr -> Global; + ir::MemoryAddr -> Memory; + ir::TableAddr -> Table; + ir::FuncAddr -> Func; + ir::ElemAddr -> ElementSegment; + ir::DataAddr -> DataSegment; } impl LoadEntity for Inst { type Entity = TableEntity; #[inline] - fn load(self, _store: &mut StoreInner, _addr: Table0) -> &mut Self::Entity { - // SAFETY: `addr` addresses an entry of this kind by translation invariant. + unsafe fn load_entity_ptr(self, _addr: Table0) -> NonNull { + // SAFETY: guaranteed by the caller; an instance with a `(table 0)` caches its entity + // pointer in its header at instantiation. let Some(entity) = (unsafe { self.as_ptr().get_table0() }) else { unsafe { unreachable_unchecked!("missing table entity for table 0") } }; - // SAFETY: warmed at instantiation; the `_store` borrow scopes the reference. - unsafe { &mut *entity.as_ptr() } + entity } + + #[inline] + unsafe fn load_entity_mut(self, _store: &mut PrunedStore, addr: Table0) -> &mut Self::Entity { + // SAFETY: guaranteed by the caller; the `_store` borrow scopes the returned reference + // against a concurrent store mutation. + unsafe { self.load_entity_ptr(addr).as_mut() } + } +} + +/// Extension trait for [`Inst`] to load handles from their addresses. +pub trait LoadHandle { + /// The type of the loaded handle. + type Handle; + + /// Returns the handle at `addr` of the instance. + /// + /// Unlike the [`LoadEntity`] methods this does not touch the entity cache and therefore + /// does not require it to be warmed up. + /// + /// # Safety + /// + /// Same as for [`LoadEntry::load_entry_ptr`]. + unsafe fn load_handle(self, addr: Addr) -> Self::Handle; } -macro_rules! impl_resolve_ptr_from_instance { +macro_rules! impl_load_handle_for_inst { ( - $( fn $fn:ident(inst: Inst, $param:ident: ir::$param_ty:ident) -> $ret:ty = $entry:ident );* $(;)? + $( + ir::$addr:ident -> $handle:ident + );* $(;)? ) => { $( - /// Resolves a pointer to the entity from the warmed up instance cache. - /// - /// Unlike `>::load` and friends this returns a raw [`NonNull`] - /// instead of a reference tied to the store. This lets the caller hold several entity pointers - /// (or an entity pointer alongside a `&mut` borrow of `fuel`) at once and materialize - /// a reference only in the narrow scope where the entity is actually accessed. - /// - /// # Safety - /// - /// The caller must ensure that `inst` refers to a live, warmed up [`InstanceEntity`] - /// and that `$param` is a valid address within it. The returned pointer is only sound - /// to dereference while the instance cache remains warmed up, and the caller must - /// ensure the resulting reference does not alias any other live reference. - #[inline] - pub unsafe fn $fn(inst: Inst, $param: ir::$param_ty) -> NonNull<$ret> { - let addr = <$param_ty>::from(::core::primitive::u32::from($param)); - // Safety: guaranteed by the caller. - let Some(entry) = (unsafe { inst.as_ptr().$entry(addr) }) else { - unsafe { - $crate::engine::utils::unreachable_unchecked!( - ::core::concat!("missing ", ::core::stringify!($param), " at: {:?}"), - addr, - ) - } - }; - entry.entity() + impl LoadHandle for Inst { + type Handle = $handle; + + #[inline] + unsafe fn load_handle(self, addr: ir::$addr) -> Self::Handle { + // SAFETY: guaranteed by the caller; the reference does not outlive the call. + let entry_ref = unsafe { self.load_entry_ptr(addr).as_ref() }; + entry_ref.handle() + } } )* }; } -impl_resolve_ptr_from_instance! { - fn load_memory_ptr(inst: Inst, memory: ir::MemoryAddr) -> MemoryEntity = get_memory; - fn load_table_ptr(inst: Inst, table: ir::TableAddr) -> TableEntity = get_table; - fn load_data_ptr(inst: Inst, data: ir::DataAddr) -> DataSegmentEntity = get_data; - fn load_elem_ptr(inst: Inst, elem: ir::ElemAddr) -> ElementSegmentEntity = get_elem; +impl_load_handle_for_inst! { + // ir::GlobalAddr -> Global; + ir::MemoryAddr -> Memory; + ir::TableAddr -> Table; + ir::FuncAddr -> Func; + // ir::ElemAddr -> ElementSegment; + // ir::DataAddr -> DataSegment; } /// Resolves the [`Func`] handle and its warmed up cached [`FuncEntity`] pointer from `inst`. @@ -697,68 +783,12 @@ impl_resolve_ptr_from_instance! { /// and must not be turned into a reference that aliases a concurrent store mutation. #[inline] pub fn load_func_entry(inst: Inst, func: ir::FuncAddr) -> (Func, NonNull) { - let addr = FuncAddr::from(u32::from(func)); - // SAFETY: `addr` addresses a func entry by translation invariant and its cache was - // warmed at instantiation. - let Some(entry) = (unsafe { inst.as_ptr().get_func(addr) }) else { - unsafe { unreachable_unchecked!("missing func at: {addr:?}") } - }; - (entry.handle(), entry.entity()) -} - -macro_rules! impl_fetch_from_instance { - ( - $( fn $fn:ident($param:ident: ir::$param_ty:ident) -> $ret:ty = $entry:ident );* $(;)? - ) => { - $( - pub fn $fn(instance: Inst, $param: ir::$param_ty) -> $ret { - let addr = <$param_ty>::from(::core::primitive::u32::from($param)); - // SAFETY: `addr` addresses an entry of this kind by translation invariant. - let Some(entry) = (unsafe { instance.as_ptr().$entry(addr) }) else { - unsafe { - $crate::engine::utils::unreachable_unchecked!( - ::core::concat!("missing ", ::core::stringify!($param), " at: {:?}"), - addr, - ) - } - }; - entry.handle() - } - )* - }; -} -impl_fetch_from_instance! { - fn fetch_func(func: ir::FuncAddr) -> Func = get_func; - fn fetch_memory(memory: ir::MemoryAddr) -> Memory = get_memory; - fn fetch_table(table: ir::TableAddr) -> Table = get_table; -} - -macro_rules! impl_resolve_from_store { - ( - $( fn $fn:ident($param:ident: $ty:ty) -> $ret:ty = $getter:expr );* $(;)? - ) => { - $( - pub fn $fn<'a>(store: &'a mut PrunedStore, $param: $ty) -> $ret { - match $getter(store.inner_mut(), $param) { - ::core::result::Result::Ok($param) => $param, - ::core::result::Result::Err(error) => unsafe { - $crate::engine::utils::unreachable_unchecked!( - ::core::concat!("could not resolve stored ", ::core::stringify!($param), ": {:?}"), - error, - ) - }, - } - } - )* - }; -} -impl_resolve_from_store! { - // fn resolve_elem(elem: &ElementSegment) -> &'a CoreElementSegment = StoreInner::try_resolve_element; - // fn resolve_global(global: &Global) -> &'a CoreGlobal = StoreInner::try_resolve_global; - fn resolve_memory(memory: &Memory) -> &'a MemoryEntity = StoreInner::try_resolve_memory; - fn resolve_table(table: &Table) -> &'a CoreTable = StoreInner::try_resolve_table; - // fn resolve_instance(instance: &Instance) -> &'a InstanceEntity = StoreInner::try_resolve_instance; - // fn resolve_func_type(func_type: DedupFuncType) -> DedupFuncType = StoreInner::resolve_func_type; + // SAFETY: `func` addresses a func entry by translation invariant and its cache was + // warmed at instantiation; the reference does not outlive this function. + let entry_ref = unsafe { inst.load_entry_ptr(func).as_ref() }; + let handle = entry_ref.handle(); + let entity = entry_ref.entity(); + (handle, entity) } #[inline] diff --git a/crates/wasmi/src/instance/cache.rs b/crates/wasmi/src/instance/cache.rs index 8252278970..27f710467f 100644 --- a/crates/wasmi/src/instance/cache.rs +++ b/crates/wasmi/src/instance/cache.rs @@ -93,14 +93,32 @@ impl AnyHandleAndEntity { } } + /// Returns `this` as a pointer to a [`HandleAndEntity`]. + /// + /// # Safety + /// + /// The caller must ensure that `this` points to a live entry that stores a `T` handle and + /// that is not mutably accessed for the duration of the call. + /// Debug builds validate the handle kind against the stored handle kind tag. + #[inline] + pub unsafe fn into_typed_ptr( + this: NonNull, + ) -> NonNull> { + // Safety: guaranteed by the caller. + unsafe { this.as_ref() }.handle.assert_kind::(); + // Safety: `HandleAndEntity` is a `repr(transparent)` wrapper around + // `AnyHandleAndEntity` and the caller guarantees the handle kind. + this.cast::>() + } + /// Returns `self` as a shared [`HandleAndEntity`]. /// /// # Safety /// - /// The caller must ensure that `self` stores a `T` handle. Debug builds validate this - /// against the stored handle kind tag. + /// The caller must ensure that `self` stores a `T` handle. + /// Debug builds validate this against the stored handle kind tag. #[inline] - pub unsafe fn typed_ref(&self) -> &HandleAndEntity { + pub unsafe fn as_typed_ref(&self) -> &HandleAndEntity { self.handle.assert_kind::(); // Safety: `HandleAndEntity` is a `repr(transparent)` wrapper around // `AnyHandleAndEntity` and the caller guarantees the handle kind. @@ -111,9 +129,10 @@ impl AnyHandleAndEntity { /// /// # Safety /// - /// Same as for [`AnyHandleAndEntity::typed_ref`]. + /// The caller must ensure that `self` stores a `T` handle. + /// Debug builds validate this against the stored handle kind tag. #[inline] - pub unsafe fn typed_mut(&mut self) -> &mut HandleAndEntity { + pub unsafe fn as_typed_mut(&mut self) -> &mut HandleAndEntity { self.handle.assert_kind::(); // Safety: `HandleAndEntity` is a `repr(transparent)` wrapper around // `AnyHandleAndEntity` and the caller guarantees the handle kind. diff --git a/crates/wasmi/src/instance/entity.rs b/crates/wasmi/src/instance/entity.rs index 823df26cc9..1c1a3d6764 100644 --- a/crates/wasmi/src/instance/entity.rs +++ b/crates/wasmi/src/instance/entity.rs @@ -292,7 +292,7 @@ impl InstanceEntity { while let Some(addr) = layout.$addr(index) { let entry = &mut self.handles[u32::from(addr) as usize]; // Safety: the `InstanceLayout` only yields addresses of its own group. - let entry = unsafe { entry.typed_mut::<$handle>() }; + let entry = unsafe { entry.as_typed_mut::<$handle>() }; entry.warmup(store); index += 1; } @@ -308,7 +308,7 @@ impl InstanceEntity { if let Some(addr) = self.header.layout.table_addr(0) { let entry = &self.handles[u32::from(addr) as usize]; // Safety: the `InstanceLayout` only yields addresses of its own group. - let entry = unsafe { entry.typed_ref::() }; + let entry = unsafe { entry.as_typed_ref::
() }; self.header.table0 = Some(Table0Ptr::new(entry.entity())); } } @@ -318,7 +318,7 @@ impl InstanceEntity { pub fn get_memory(&self, addr: MemoryAddr) -> Option { let entry = self.entry(addr)?; // Safety: `addr` is a `MemoryAddr` and thus addresses a [`Memory`] entry. - Some(unsafe { entry.typed_ref::() }.handle()) + Some(unsafe { entry.as_typed_ref::() }.handle()) } /// Returns the [`Table`] at the `addr` if any. @@ -326,7 +326,7 @@ impl InstanceEntity { pub fn get_table(&self, addr: TableAddr) -> Option
{ let entry = self.entry(addr)?; // Safety: `addr` is a `TableAddr` and thus addresses a [`Table`] entry. - Some(unsafe { entry.typed_ref::
() }.handle()) + Some(unsafe { entry.as_typed_ref::
() }.handle()) } /// Returns the [`Global`] at the `addr` if any. @@ -334,7 +334,7 @@ impl InstanceEntity { pub fn get_global(&self, addr: GlobalAddr) -> Option { let entry = self.entry(addr)?; // Safety: `addr` is a `GlobalAddr` and thus addresses a [`Global`] entry. - Some(unsafe { entry.typed_ref::() }.handle()) + Some(unsafe { entry.as_typed_ref::() }.handle()) } /// Returns the [`Func`] at the `addr` if any. @@ -342,7 +342,7 @@ impl InstanceEntity { pub fn get_func(&self, addr: FuncAddr) -> Option { let entry = self.entry(addr)?; // Safety: `addr` is a `FuncAddr` and thus addresses a [`Func`] entry. - Some(unsafe { entry.typed_ref::() }.handle()) + Some(unsafe { entry.as_typed_ref::() }.handle()) } /// Returns the [`DataSegment`] at the `addr` if any. @@ -350,7 +350,7 @@ impl InstanceEntity { pub fn get_data(&self, addr: DataAddr) -> Option { let entry = self.entry(addr)?; // Safety: `addr` is a `DataAddr` and thus addresses a [`DataSegment`] entry. - Some(unsafe { entry.typed_ref::() }.handle()) + Some(unsafe { entry.as_typed_ref::() }.handle()) } /// Returns the [`ElementSegment`] at the `addr` if any. @@ -358,7 +358,7 @@ impl InstanceEntity { pub fn get_elem(&self, addr: ElemAddr) -> Option { let entry = self.entry(addr)?; // Safety: `addr` is a `ElemAddr` and thus addresses a [`ElementSegment`] entry. - Some(unsafe { entry.typed_ref::() }.handle()) + Some(unsafe { entry.as_typed_ref::() }.handle()) } /// Returns the value exported to the given `name` if any. @@ -394,33 +394,6 @@ impl InstanceEntity { } } -macro_rules! impl_get_entry { - ( - $( - $(#[$attr:meta])* - pub unsafe fn $get:ident(self, addr: $addr_ty:ty) -> &HandleAndEntity<$handle:ty>; - )* - ) => { - $( - $(#[$attr])* - /// - /// # Safety - /// - /// In addition to the requirements of [`ThinPtr::as_ref`] the caller must ensure - #[doc = concat!("that the entry at `addr` stores a [`", stringify!($handle), "`] handle.")] - /// Wasmi's translation guarantees this for every address encoded into a Wasmi IR - /// operator. - #[inline] - pub unsafe fn $get<'a>(self, addr: $addr_ty) -> Option<&'a HandleAndEntity<$handle>> { - // Safety: guaranteed by the caller. - let entry = unsafe { self.entry(u32::from(addr)) }?; - // Safety: guaranteed by the caller. - Some(unsafe { entry.typed_ref::<$handle>() }) - } - )* - }; -} - /// The thin-pointer API of [`InstanceEntity`]. /// /// # Note @@ -469,13 +442,13 @@ impl ThinPtr { /// /// Same as for [`ThinPtr::as_ref`]. #[inline] - unsafe fn entry<'a>(self, addr: u32) -> Option<&'a AnyHandleAndEntity> { + unsafe fn entry(self, addr: u32) -> Option> { // Safety: guaranteed by the caller. if addr >= unsafe { self.header() }.len_handles() { return None; } // Safety: guaranteed by the caller and the bounds check above. - Some(unsafe { self.handles().add(addr as usize).as_ref() }) + Some(unsafe { self.handles().add(addr as usize) }) } /// Returns a shared reference to the [`InstanceLayout`] of the pointee. @@ -494,21 +467,6 @@ impl ThinPtr { unsafe { self.header() }.table0.map(Table0Ptr::get) } - impl_get_entry! { - /// Returns the [`HandleAndEntity`] of the [`Memory`] at `addr`. - pub unsafe fn get_memory(self, addr: MemoryAddr) -> &HandleAndEntity; - /// Returns the [`HandleAndEntity`] of the [`Global`] at `addr`. - pub unsafe fn get_global(self, addr: GlobalAddr) -> &HandleAndEntity; - /// Returns the [`HandleAndEntity`] of the [`Table`] at `addr`. - pub unsafe fn get_table(self, addr: TableAddr) -> &HandleAndEntity
; - /// Returns the [`HandleAndEntity`] of the [`Func`] at `addr`. - pub unsafe fn get_func(self, addr: FuncAddr) -> &HandleAndEntity; - /// Returns the [`HandleAndEntity`] of the [`ElementSegment`] at `addr`. - pub unsafe fn get_elem(self, addr: ElemAddr) -> &HandleAndEntity; - /// Returns the [`HandleAndEntity`] of the [`DataSegment`] at `addr`. - pub unsafe fn get_data(self, addr: DataAddr) -> &HandleAndEntity; - } - /// Returns a shared reference to the [`InstanceEntity`] pointee. /// /// # Safety @@ -536,3 +494,48 @@ impl ThinPtr { unsafe { &*ptr } } } + +macro_rules! impl_get_entry { + ( + $( + $(#[$attr:meta])* + pub unsafe fn $get:ident(self, addr: $addr_ty:ty) -> Option>>; + )* + ) => { + $( + $(#[$attr])* + /// + /// # Safety + /// + /// In addition to the requirements of [`ThinPtr::as_ref`] the caller must ensure + #[doc = concat!("that the entry at `addr` stores a [`", stringify!($handle), "`] handle.")] + /// Wasmi's translation guarantees this for every address encoded into a Wasmi IR + /// operator. + #[inline] + pub unsafe fn $get(self, addr: $addr_ty) -> Option>> { + // Safety: guaranteed by the caller. + let entry = unsafe { self.entry(u32::from(addr)) }?; + // Safety: guaranteed by the caller. + let typed = unsafe { AnyHandleAndEntity::into_typed_ptr::<$handle>(entry) }; + Some(typed) + } + )* + }; +} + +impl ThinPtr { + impl_get_entry! { + /// Returns the [`HandleAndEntity`] of the [`Memory`] at `addr`. + pub unsafe fn get_memory(self, addr: MemoryAddr) -> Option>>; + /// Returns the [`HandleAndEntity`] of the [`Global`] at `addr`. + pub unsafe fn get_global(self, addr: GlobalAddr) -> Option>>; + /// Returns the [`HandleAndEntity`] of the [`Table`] at `addr`. + pub unsafe fn get_table(self, addr: TableAddr) -> Option>>; + /// Returns the [`HandleAndEntity`] of the [`Func`] at `addr`. + pub unsafe fn get_func(self, addr: FuncAddr) -> Option>>; + /// Returns the [`HandleAndEntity`] of the [`ElementSegment`] at `addr`. + pub unsafe fn get_elem(self, addr: ElemAddr) -> Option>>; + /// Returns the [`HandleAndEntity`] of the [`DataSegment`] at `addr`. + pub unsafe fn get_data(self, addr: DataAddr) -> Option>>; + } +} diff --git a/crates/wasmi/src/instance/mod.rs b/crates/wasmi/src/instance/mod.rs index 8d4684fe34..daf2d2824a 100644 --- a/crates/wasmi/src/instance/mod.rs +++ b/crates/wasmi/src/instance/mod.rs @@ -1,5 +1,5 @@ -pub(crate) use self::builder::InstanceEntityBuilder; -use self::cache::{AnyHandleAndEntity, HandleAndEntity, Table0Ptr}; +use self::cache::{AnyHandleAndEntity, Table0Ptr}; +pub(crate) use self::{builder::InstanceEntityBuilder, cache::HandleAndEntity}; pub use self::{ entity::InstanceEntity, exports::{Export, ExportsIter, Extern, ExternType}, diff --git a/crates/wasmi/src/lib.rs b/crates/wasmi/src/lib.rs index 364cf7164f..98f9b760f9 100644 --- a/crates/wasmi/src/lib.rs +++ b/crates/wasmi/src/lib.rs @@ -231,7 +231,7 @@ use self::{ func::FuncEntity, handle::{Handle, RawHandle}, instance::{InstanceEntity, InstanceEntityBuilder}, - memory::DataSegmentEntity, + memory::{DataSegment, DataSegmentEntity}, table::ElementSegment, }; pub use wasmi_core::{