@@ -7486,6 +7486,144 @@ impl BexVm {
74867486 })
74877487 }
74887488
7489+ /// Enter a bytecode function or closure called through `CallIndirect` the
7490+ /// way `CallExactArgs` enters its callee, when the site pushed exactly the
7491+ /// callee's slots. Returns the entered function; the caller then switches
7492+ /// dispatch to it. `None` leaves everything untouched for the general
7493+ /// path, which handles compiled, hooked and pass-through callees, a
7494+ /// pending trace or type lane, layouts that need remapping, and every
7495+ /// other callable.
7496+ ///
7497+ /// Kept out of line: inlined into `run_compact`, it made every ordinary
7498+ /// exact call about ten instructions slower.
7499+ #[inline(never)]
7500+ fn enter_indirect_bytecode_call(
7501+ &mut self,
7502+ callee_ptr: HeapPtr,
7503+ frame_idx: usize,
7504+ caller: &'static Function,
7505+ ) -> Result<Option<&'static Function>, VmError> {
7506+ if self.pending_call_trace.is_some() || !self.pending_call_type_args.is_empty() {
7507+ return Ok(None);
7508+ }
7509+ // SAFETY: the active heap permit prevents collection during
7510+ // frame/stack growth, as at `CallExactArgs`; a closure's function
7511+ // lives at least as long as the closure.
7512+ let (callee, callee_fn_ptr, captured_type_args) = match unsafe { callee_ptr.get() } {
7513+ Object::Function(callee) => (callee, callee_ptr, &[][..]),
7514+ Object::Closure(closure) => match unsafe { closure.function.get() } {
7515+ Object::Function(callee) => {
7516+ (callee, closure.function, &closure.captured_type_args[..])
7517+ }
7518+ _ => return Ok(None),
7519+ },
7520+ _ => return Ok(None),
7521+ };
7522+ if !matches!(callee.kind, FunctionKind::Bytecode)
7523+ || callee.compiled_body.is_some()
7524+ || callee.trace_passthrough
7525+ || callee
7526+ .bytecode
7527+ .compact
7528+ .as_ref()
7529+ .is_none_or(|code| code.trace_hook_finish_pc.is_some())
7530+ || Self::recorded_call_layout(caller, self.cur_pc)
7531+ .is_some_and(|layout| !callee.argument_layout_is(layout))
7532+ {
7533+ return Ok(None);
7534+ }
7535+
7536+ let _popped_callee = self.stack.ensure_pop();
7537+ // Eager ok_or constructs/drops VmInternalError even on success.
7538+ #[expect(clippy::unnecessary_lazy_evaluations)]
7539+ let locals_offset = StackIndex::from_raw(
7540+ self.stack
7541+ .len()
7542+ .checked_sub(callee.arity)
7543+ .ok_or_else(|| VmInternalError::NotEnoughItemsOnStack(callee.arity))?,
7544+ );
7545+ let Frame::Bytecode(caller_frame) = &mut self.frames[frame_idx] else {
7546+ verifier_unreachable!()
7547+ };
7548+ caller_frame.faulting_pc = self.cur_pc;
7549+ self.reserve_call_frame()?;
7550+ self.reserve_stack(callee.real_local_count)?;
7551+ let context = self.current_context().clone();
7552+ if let Some(telemetry) = &mut self.telemetry {
7553+ telemetry.set_context(context.clone());
7554+ }
7555+ let frame_telemetry = if self.telemetry.is_some() {
7556+ self.enter_indirect_bytecode_telemetry(
7557+ frame_idx,
7558+ callee,
7559+ callee_fn_ptr,
7560+ locals_offset,
7561+ captured_type_args,
7562+ )
7563+ } else {
7564+ None
7565+ };
7566+ // Written in place, as in the general path.
7567+ self.frames.push_reserved_with(|| {
7568+ Frame::Bytecode(BytecodeFrame {
7569+ function: callee_ptr,
7570+ instruction_ptr: 0,
7571+ locals_offset,
7572+ // A closure made in a generic context reads the type
7573+ // arguments it captured.
7574+ type_args: captured_type_args.to_vec(),
7575+ faulting_pc: 0,
7576+ telemetry: frame_telemetry,
7577+ context,
7578+ })
7579+ });
7580+ if callee.real_local_count != 0 {
7581+ let new_len = self.stack.len() + callee.real_local_count;
7582+ self.resize_stack(new_len, Value::NULL)?;
7583+ }
7584+ Ok(Some(callee))
7585+ }
7586+
7587+ /// Telemetry entry for a bytecode callee `CallIndirect` enters directly:
7588+ /// what `execute_call_from_locals_offset` records for an untraced call
7589+ /// that passes no type arguments.
7590+ #[inline(never)]
7591+ fn enter_indirect_bytecode_telemetry(
7592+ &mut self,
7593+ frame_idx: usize,
7594+ callee: &Function,
7595+ callee_fn_ptr: HeapPtr,
7596+ locals_offset: StackIndex,
7597+ carried: &[bex_vm_types::RealizedTy],
7598+ ) -> Option<FrameTelemetry> {
7599+ // As the general path records it, with this instruction as the call site.
7600+ let caller = self.bytecode_caller(frame_idx);
7601+ let actual_caller = self
7602+ .visible_bytecode_caller(frame_idx)
7603+ .and_then(|at| self.frame_identity(at));
7604+ let caller_is_observed = self.frame_receipt(caller).is_some();
7605+ let caller_pc = self.caller_pc(caller, frame_idx, self.cur_pc);
7606+ let args = &self.stack.as_slice()
7607+ [locals_offset.raw()..locals_offset.raw().saturating_add(callee.arity)];
7608+ // SAFETY: arguments and callable remain live under the heap permit.
7609+ self.telemetry.as_mut().and_then(|telemetry| unsafe {
7610+ telemetry.enter_bytecode_with_trace(
7611+ callee,
7612+ callee_fn_ptr,
7613+ actual_caller,
7614+ caller_pc,
7615+ caller_is_observed,
7616+ args,
7617+ None,
7618+ CallTypeArgs {
7619+ carried,
7620+ passed: &[],
7621+ },
7622+ |caller, callee| Self::register_call_path_functions(&self.heap, caller, callee),
7623+ )
7624+ })
7625+ }
7626+
74897627 fn execute_call_from_locals_offset(
74907628 &mut self,
74917629 callee_ptr: HeapPtr,
@@ -10575,6 +10713,22 @@ impl BexVm {
1057510713 let callee_value = self.stack[callee_slot];
1057610714 let callee_ptr =
1057710715 self.as_object_ptr(callee_value, FunctionType::Callable.into())?;
10716+ // Bytecode functions and closures stay in compact
10717+ // dispatch, as at `CallExactArgs`.
10718+ if let Some(callee) =
10719+ self.enter_indirect_bytecode_call(callee_ptr, *frame_idx, function)?
10720+ {
10721+ *frame_idx = self.frames.len() - 1;
10722+ *function = callee;
10723+ runtime_package = self.runtime_package_of(function);
10724+ if self.should_early_yield() {
10725+ return Ok(Some(VmExecState::EarlyYield));
10726+ }
10727+ *pc = 0;
10728+ *code = &callee.bytecode.compact.as_ref().unwrap().code;
10729+ *dispatch_frame_idx = *frame_idx;
10730+ continue;
10731+ }
1057810732 let obj = self.get_object(callee_ptr);
1057910733
1058010734 if let Object::HostClosure(host_closure) = obj {
0 commit comments