wasm/execution/interpreter_loop/
mod.rs

1//! This module solely contains the actual interpretation loop that matches instructions, interpreting the WASM bytecode
2//!
3//!
4//! # Note to Developer:
5//!
6//! 1. There must be only imports and one `impl` with one function (`run`) in it.
7//! 2. This module must only use [`RuntimeError`] and never [`Error`](crate::core::error::ValidationError).
8
9use alloc::vec::Vec;
10use core::{array, num::NonZeroU64, ops::ControlFlow};
11
12use crate::{
13    addrs::{AddrVec, DataAddr, ElemAddr, FuncAddr, MemAddr, ModuleAddr, TableAddr},
14    assert_validated::UnwrapValidatedExt,
15    core::{
16        indices::{DataIdx, ElemIdx, MemIdx, TableIdx},
17        reader::{types::memarg::MemArg, WasmReader},
18        sidetable::Sidetable,
19        utils::ToUsizeExt,
20    },
21    execution::{
22        config::Config,
23        interpreter_loop::dispatch_tables::{
24            HasBaseDispatchTable, HasFcDispatchTable, HasFdDispatchTable,
25        },
26        store::Hostcode,
27    },
28    instances::{DataInst, ElemInst, FuncInst, MemInst, ModuleInst, TableInst},
29    resumable::WasmResumable,
30    unreachable_validated,
31    value_stack::Stack,
32    RuntimeError, TrapError, Value,
33};
34
35use super::{little_endian::LittleEndianBytes, store::Store, store::StoreInner};
36
37mod control;
38mod memory;
39mod numeric;
40mod parametric;
41mod reference;
42mod table;
43mod variable;
44mod vector;
45
46mod dispatch_tables;
47
48/// A non-error outcome of execution of the interpreter loop
49pub enum InterpreterLoopOutcome {
50    /// Execution has returned normally, i.e. the end of the bottom-most
51    /// function on the stack was reached.
52    ExecutionReturned,
53    /// Execution was preempted because there was not enough fuel in the
54    /// [`WasmResumable`] object.
55    ///
56    OutOfFuel {
57        /// The amount of fuel required to continue execution at least the next
58        /// instruction.
59        required_fuel: NonZeroU64,
60    },
61    HostCalled {
62        func_addr: FuncAddr,
63        // TODO this allocation might be preventable. mutably borrow the stack
64        // instead
65        params: Vec<Value>,
66        hostcode: Hostcode,
67    },
68}
69
70type InstructionHandlerFn =
71    for<'wasm, 'modules> unsafe fn(
72        wasm: &mut WasmReader<'wasm>,
73        resumable: &mut WasmResumable,
74        current_sidetable: &mut &'modules Sidetable,
75        store_inner: &mut StoreInner,
76        modules: &'modules AddrVec<ModuleAddr, ModuleInst<'wasm>>,
77        current_module: &mut ModuleAddr,
78        current_function_end_marker: &mut usize,
79    )
80        -> Result<ControlFlow<InterpreterLoopOutcome>, RuntimeError>;
81
82// A placeholder instruction for unassigned instruction bytes. This function is by definition dead
83// code!
84define_instruction_fn! {unset, fuel_check = omit, |Args { .. }| {
85    // Access T to circumvent warning that it is unused by this function. #[allow] does not work for
86    // macros.
87    let _ = T::DISPATCH_TABLE;
88    unreachable_validated!();
89}}
90
91/// Interprets wasm native functions. Wasm parameters and Wasm return values are passed on the stack.
92/// Returns `Ok(ControlFlow::Continue(()))` in case execution successfully terminates, `Ok(Some(required_fuel))` if execution
93/// terminates due to insufficient fuel, indicating how much fuel is required to resume with `required_fuel`,
94/// and `[Error::RuntimeError]` otherwise.
95///
96/// # Safety
97///
98/// The given resumable must be valid in the given [`Store`].
99#[inline(never)]
100pub(super) unsafe fn run<T: Config>(
101    resumable: &mut WasmResumable,
102    store: &mut Store<T>,
103) -> Result<InterpreterLoopOutcome, RuntimeError> {
104    let current_func_addr = resumable.current_func_addr;
105    let pc = resumable.pc;
106    // SAFETY: The caller ensures that the resumable and thus also its function
107    // address is valid in the current store.
108    let func_inst = unsafe { store.inner.functions.get(current_func_addr) };
109    let FuncInst::WasmFunc(wasm_func_inst) = &func_inst else {
110        unreachable!(
111            "the interpreter loop shall only be executed with native wasm functions as root call"
112        );
113    };
114    let mut current_module = wasm_func_inst.module_addr;
115
116    // Start reading the function's instructions
117    // SAFETY: This module address was just read from the current store. Every
118    // store guarantees all addresses contained in it to be valid within itself.
119    let module = unsafe { store.modules.get(current_module) };
120    let wasm_bytecode = module.wasm_bytecode;
121    let wasm = &mut WasmReader::new(wasm_bytecode);
122
123    let mut current_sidetable: &Sidetable = &module.sidetable;
124
125    let mut current_function_end_marker =
126        wasm_func_inst.code_expr.from() + wasm_func_inst.code_expr.len();
127
128    let store_inner = &mut store.inner;
129
130    // local variable for holding where the function code ends (last END instr address + 1) to avoid lookup at every END instr
131
132    wasm.pc = pc;
133
134    loop {
135        // call the instruction hook
136        store.user_data.instruction_hook(wasm_bytecode, wasm.pc);
137
138        let prev_pc = wasm.pc;
139
140        let first_instr_byte = wasm.read_u8().unwrap_validated();
141
142        #[cfg(debug_assertions)]
143        trace!(
144            "Executing instruction {}",
145            crate::opcodes::opcode_byte_to_str(first_instr_byte)
146        );
147
148        let instruction_fn = T::DISPATCH_TABLE
149            .get(usize::from(first_instr_byte))
150            .expect("the instruction to be valid because the code is validated");
151
152        // SAFETY: All possible instruction handler functions use the same safety requirements, as
153        // they are defined through the same macro: The caller ensures that the resumable is valid
154        // in the current store. Also all other address types passed via the `Args` must come from
155        // the current store itself. Therefore, they are automatically valid in this store.
156        let instruction_result = unsafe {
157            instruction_fn(
158                wasm,
159                resumable,
160                &mut current_sidetable,
161                store_inner,
162                &store.modules,
163                &mut current_module,
164                &mut current_function_end_marker,
165            )
166        };
167
168        if let ControlFlow::Break(interpreter_loop_outcome) = instruction_result? {
169            if let InterpreterLoopOutcome::OutOfFuel { .. } = interpreter_loop_outcome {
170                wasm.pc = prev_pc;
171            }
172
173            resumable.pc = wasm.pc;
174            return Ok(interpreter_loop_outcome);
175        }
176    }
177}
178
179//helper function for avoiding code duplication at intraprocedural jumps
180fn do_sidetable_control_transfer(
181    wasm: &mut WasmReader,
182    stack: &mut Stack,
183    current_stp: &mut usize,
184    current_sidetable: &Sidetable,
185) -> Result<(), RuntimeError> {
186    let sidetable_entry = &current_sidetable[*current_stp];
187
188    stack.remove_in_between(sidetable_entry.popcnt, sidetable_entry.valcnt);
189
190    *current_stp = sidetable_entry.stp;
191    wasm.pc = sidetable_entry.pc;
192
193    Ok(())
194}
195
196#[inline(always)]
197fn calculate_mem_address(memarg: &MemArg, relative_address: u32) -> Result<usize, RuntimeError> {
198    memarg
199        .offset
200        // The spec states that this should be a 33 bit integer, e.g. it is not legal to wrap if the
201        // sum of offset and relative_address exceeds u32::MAX. To emulate this behavior, we use a
202        // checked addition.
203        // See: https://webassembly.github.io/spec/core/syntax/instructions.html#memory-instructions
204        .checked_add(relative_address)
205        .ok_or(TrapError::MemoryOrDataAccessOutOfBounds)?
206        .try_into()
207        .map_err(|_| TrapError::MemoryOrDataAccessOutOfBounds.into())
208}
209
210//helpers for avoiding code duplication during module instantiation
211/// # Safety
212///
213/// 1. The module address `current_module` must be valid in `store_modules` for a module instance `module_inst`.
214/// 2. The table index `table_idx` must be valid in `module_inst` for a table address `table_addr`.
215/// 3. `table_addr` must be valid in `store_tables`.
216/// 4. The element index `elem_idx` must be valid in `module_inst` for an element address `elem_addr`.
217/// 5. `elem_addr` must be valid in `store_elements`.
218// TODO instead of passing all module instances and the current module addr
219// separately, directly pass a `&ModuleInst`.
220#[inline(always)]
221#[allow(clippy::too_many_arguments)]
222pub(super) unsafe fn table_init(
223    store_modules: &AddrVec<ModuleAddr, ModuleInst>,
224    store_tables: &mut AddrVec<TableAddr, TableInst>,
225    store_elements: &AddrVec<ElemAddr, ElemInst>,
226    current_module: ModuleAddr,
227    elem_idx: ElemIdx,
228    table_idx: TableIdx,
229    n: u32,
230    s: i32,
231    d: i32,
232) -> Result<(), RuntimeError> {
233    let n = n.into_usize();
234    let s = s.cast_unsigned().into_usize();
235    let d = d.cast_unsigned().into_usize();
236
237    // SAFETY: The caller ensures that this module address is valid in this
238    // address vector (1).
239    let module_inst = unsafe { store_modules.get(current_module) };
240    // SAFETY: The caller ensures that `table_idx` is valid for this specific
241    // `IdxVec` (2).
242    let table_addr = *unsafe { module_inst.table_addrs.get(table_idx) };
243    // SAFETY: The caller ensures that `elem_idx` is valid for this specific
244    // `IdxVec` (4).
245    let elem_addr = *unsafe { module_inst.elem_addrs.get(elem_idx) };
246    // SAFETY: The caller ensures that this table address is valid in this
247    // address vector (3).
248    let tab = unsafe { store_tables.get_mut(table_addr) };
249    // SAFETY: The caller ensures that this element address is valid in this
250    // address vector (5).
251    let elem = unsafe { store_elements.get(elem_addr) };
252
253    trace!(
254        "Instruction: table.init '{}' '{}' [{} {} {}] -> []",
255        elem_idx,
256        table_idx,
257        d,
258        s,
259        n
260    );
261
262    let final_src_offset = s
263        .checked_add(n)
264        .filter(|&res| res <= elem.len())
265        .ok_or(TrapError::TableOrElementAccessOutOfBounds)?;
266
267    if d.checked_add(n).filter(|&res| res <= tab.len()).is_none() {
268        return Err(TrapError::TableOrElementAccessOutOfBounds.into());
269    }
270
271    let dest = &mut tab.elem[d..];
272    let src = &elem.references[s..final_src_offset];
273    dest[..src.len()].copy_from_slice(src);
274    Ok(())
275}
276
277/// # Safety
278///
279/// 1. The module address `current_module` must be valid in `store_modules` for some module instance `module_inst`.
280/// 2. The element index `elem_idx` must be valid in `module_inst` for some element address `elem_addr`.
281/// 3. `elem_addr` must be valid in `store_elements`.
282#[inline(always)]
283pub(super) unsafe fn elem_drop(
284    store_modules: &AddrVec<ModuleAddr, ModuleInst>,
285    store_elements: &mut AddrVec<ElemAddr, ElemInst>,
286    current_module: ModuleAddr,
287    elem_idx: ElemIdx,
288) {
289    // WARN: i'm not sure if this is okay or not
290
291    // SAFETY: The caller ensures that this module address is valid in this
292    // address vector (1).
293    let module_inst = unsafe { store_modules.get(current_module) };
294    // SAFETY: The caller ensures that `elem_idx` is valid for this specific
295    // `IdxVec` (2).
296    let elem_addr = *unsafe { module_inst.elem_addrs.get(elem_idx) };
297
298    // SAFETY: The caller ensures that this element address is valid in this
299    // address vector (3).
300    let elem = unsafe { store_elements.get_mut(elem_addr) };
301
302    elem.references.clear();
303}
304
305/// # Safety
306///
307/// 1. The module address `current_module` must be valid in `store_modules` for some module instance `module_inst`.
308/// 2. The memory index `mem_idx` must be valid in `module_inst` for some memory address `mem_addr`.
309/// 3. `mem_addr` must be valid in `store_memories` for some memory instance `mem.
310/// 4. The data index `data_idx` must be valid in `module_inst` for some data address `data_addr`.
311/// 5. `data_addr` must be valid in `store_data`.
312#[inline(always)]
313#[allow(clippy::too_many_arguments)]
314pub(super) unsafe fn memory_init(
315    store_modules: &AddrVec<ModuleAddr, ModuleInst>,
316    store_memories: &mut AddrVec<MemAddr, MemInst>,
317    store_data: &AddrVec<DataAddr, DataInst>,
318    current_module: ModuleAddr,
319    data_idx: DataIdx,
320    mem_idx: MemIdx,
321    n: u32,
322    s: u32,
323    d: u32,
324) -> Result<(), RuntimeError> {
325    let n = n.into_usize();
326    let s = s.into_usize();
327    let d = d.into_usize();
328
329    // SAFETY: The caller ensures that this is module address is valid in this
330    // address vector (1).
331    let module_inst = unsafe { store_modules.get(current_module) };
332    // SAFETY: The caller ensures that `mem_idx` is valid for this specific
333    // `IdxVec` (2).
334    let mem_addr = *unsafe { module_inst.mem_addrs.get(mem_idx) };
335    // SAFETY: The caller ensures that this memory address is valid in this
336    // address vector (3).
337    let mem = unsafe { store_memories.get(mem_addr) };
338    // SAFETY: The caller ensures that `data_idx` is valid for this specific
339    // `IdxVec` (4).
340    let data_addr = *unsafe { module_inst.data_addrs.get(data_idx) };
341    // SAFETY: The caller ensures that this data address is valid in this
342    // address vector (5).
343    let data = unsafe { store_data.get(data_addr) };
344
345    mem.mem.init(d, &data.data, s, n)?;
346
347    trace!("Instruction: memory.init");
348    Ok(())
349}
350
351/// # Safety
352///
353/// 1. The module address `current_module` must be valid in `store_modules` for some module instance `module_inst`.
354/// 2. The data index `data_idx` must be valid in `module_inst` for some data address `data_addr`.
355/// 3. `data_addr` must be valid in `store_data`.
356#[inline(always)]
357pub(super) unsafe fn data_drop(
358    store_modules: &AddrVec<ModuleAddr, ModuleInst>,
359    store_data: &mut AddrVec<DataAddr, DataInst>,
360    current_module: ModuleAddr,
361    data_idx: DataIdx,
362) {
363    // Here is debatable
364    // If we were to be on par with the spec we'd have to use a DataInst struct
365    // But since memory.init is specifically made for Passive data segments
366    // I thought that using DataMode would be better because we can see if the
367    // data segment is passive or active
368
369    // Also, we should set data to null here (empty), which we do by clearing it
370    // SAFETY: The caller guarantees this module to be valid in this address
371    // vector (1).
372    let module_inst = unsafe { store_modules.get(current_module) };
373    // SAFETY: The caller ensures that `data_idx` is valid for this specific
374    // `IdxVec` (2).
375    let data_addr = *unsafe { module_inst.data_addrs.get(data_idx) };
376    // SAFETY: The caller ensures that this data address is valid in this
377    // address vector (3).
378    let data = unsafe { store_data.get_mut(data_addr) };
379
380    data.data.clear();
381}
382
383#[inline(always)]
384pub(crate) fn to_lanes<const M: usize, const N: usize, T: LittleEndianBytes<M>>(
385    data: [u8; 16],
386) -> [T; N] {
387    assert_eq!(M * N, 16);
388
389    let mut lanes = data
390        .chunks(M)
391        .map(|chunk| T::from_le_bytes(chunk.try_into().unwrap()));
392    array::from_fn(|_| lanes.next().unwrap())
393}
394
395#[inline(always)]
396pub(crate) fn from_lanes<const M: usize, const N: usize, T: LittleEndianBytes<M>>(
397    lanes: [T; N],
398) -> [u8; 16] {
399    assert_eq!(M * N, 16);
400
401    let mut bytes = lanes.into_iter().flat_map(T::to_le_bytes);
402    array::from_fn(|_| bytes.next().unwrap())
403}
404
405pub(crate) struct Args<'a, 'sidetable, 'wasm, 'other, 'resumable> {
406    wasm: &'a mut WasmReader<'wasm>,
407    resumable: &'resumable mut WasmResumable,
408    current_sidetable: &'a mut &'sidetable Sidetable,
409    store_inner: &'other mut StoreInner,
410    modules: &'sidetable AddrVec<ModuleAddr, ModuleInst<'wasm>>,
411    current_module: &'a mut ModuleAddr,
412    current_function_end_marker: &'a mut usize,
413}
414
415macro_rules! define_instruction_fn {
416    ($name:ident, fuel_check = omit, $contents:expr) => {
417        /// # Safety
418        ///
419        /// The given [`WasmResumable`](crate::execution::resumable::WasmResumable) and all address
420        /// types contained in the [`Args`](crate::execution::interpreter_loop::Args) must be valid
421        /// in the [`StoreInner`](crate::execution::store::StoreInner) that is also contained in the
422        /// [`Args`](crate::execution::interpreter_loop::Args).
423        // Disable inlining to inspect the emitted code of individual instruction handlers:
424        // #[inline(never)]
425        pub(crate) unsafe fn $name<'wasm, 'modules, T: $crate::config::Config>(
426            wasm: &mut $crate::core::reader::WasmReader<'wasm>,
427            resumable: &mut $crate::execution::resumable::WasmResumable,
428            current_sidetable: &mut &'modules $crate::core::sidetable::Sidetable,
429            store_inner: &mut $crate::execution::store::StoreInner,
430            modules: &'modules $crate::execution::store::addrs::AddrVec<
431                $crate::execution::store::addrs::ModuleAddr,
432                $crate::execution::store::instances::ModuleInst<'wasm>,
433            >,
434            current_module: &mut $crate::execution::store::addrs::ModuleAddr,
435            current_function_end_marker: &mut usize,
436        ) -> Result<
437            core::ops::ControlFlow<$crate::execution::interpreter_loop::InterpreterLoopOutcome>,
438            $crate::RuntimeError,
439        > {
440            let args = $crate::execution::interpreter_loop::Args {
441                store_inner,
442                modules,
443                wasm,
444                current_module,
445                current_function_end_marker,
446                current_sidetable,
447                resumable,
448            };
449
450            $contents(args)
451        }
452    };
453
454    ($name:ident, fuel_check = flat($opcode:expr), $contents:expr) => {
455        define_instruction_fn! {
456            $name,
457            fuel_check = omit,
458            |args: $crate::execution::interpreter_loop::Args| {
459                if let core::ops::ControlFlow::Break(outcome) =
460                    $crate::execution::interpreter_loop::decrement_fuel(
461                        T::get_flat_cost($opcode),
462                        &mut args.resumable.maybe_fuel,
463                    )
464                {
465                    return Ok(core::ops::ControlFlow::Break(outcome));
466                }
467
468                $contents(args)
469            }
470        }
471    };
472
473    ($name: ident, fuel_check = flat_fc($opcode: expr), $contents:expr) => {
474        define_instruction_fn! {
475            $name,
476            fuel_check = omit,
477            |args: $crate::execution::interpreter_loop::Args| {
478                if let core::ops::ControlFlow::Break(outcome) =
479                    $crate::execution::interpreter_loop::decrement_fuel(
480                        T::get_fc_extension_flat_cost($opcode),
481                        &mut args.resumable.maybe_fuel,
482                    )
483                {
484                    return Ok(core::ops::ControlFlow::Break(outcome));
485                }
486
487                $contents(args)
488            }
489        }
490    };
491
492    ($name: ident, fuel_check = flat_fd($opcode: expr), $contents:expr) => {
493        define_instruction_fn! {
494            $name,
495            fuel_check = omit,
496            |args: $crate::execution::interpreter_loop::Args| {
497                if let core::ops::ControlFlow::Break(outcome) =
498                    $crate::execution::interpreter_loop::decrement_fuel(
499                        T::get_fd_extension_flat_cost($opcode),
500                        &mut args.resumable.maybe_fuel,
501                    )
502                {
503                    return Ok(core::ops::ControlFlow::Break(outcome));
504                }
505
506                $contents(args)
507            }
508        }
509    };
510}
511
512pub(crate) use define_instruction_fn;
513
514#[inline(always)]
515fn decrement_fuel(cost: u64, maybe_fuel: &mut Option<u64>) -> ControlFlow<InterpreterLoopOutcome> {
516    if let Some(fuel) = maybe_fuel {
517        if *fuel >= cost {
518            *fuel -= cost;
519        } else {
520            return ControlFlow::Break(InterpreterLoopOutcome::OutOfFuel {
521                required_fuel: NonZeroU64::new(cost - *fuel)
522                    .expect("the last check guarantees that the current fuel is smaller than cost"),
523            });
524        }
525    }
526
527    ControlFlow::Continue(())
528}
529
530define_instruction_fn! {fc_extensions, fuel_check = omit, |args: Args| {
531    // should we call instruction hook here as well? multibyte instruction
532    let second_instr = args.wasm.read_var_u32().unwrap_validated();
533
534    let instruction_fn = T::FC_DISPATCH_TABLE
535        .get(second_instr.into_usize())
536        .expect("the instruction to be valid because the code is validated");
537
538    // SAFETY: All possible instruction handler functions use the same safety requirements, as
539    // they are defined through the same macro: The caller ensures that the resumable is valid
540    // in the current store. Also all other address types passed via the `Args` must come from
541    // the current store itself. Therefore, they are automatically valid in this store.
542    unsafe {
543        instruction_fn(
544            args.wasm,
545            args.resumable,
546            args.current_sidetable,
547            args.store_inner,
548            args.modules,
549            args.current_module,
550            args.current_function_end_marker,
551        )
552    }
553}}
554
555define_instruction_fn! {fd_extensions, fuel_check = omit, |args: Args| {
556    // Should we call instruction hook here as well? Multibyte instruction
557    let second_instr = args.wasm.read_var_u32().unwrap_validated();
558
559    let instruction_fn = T::FD_DISPATCH_TABLE
560        .get(second_instr.into_usize())
561        .expect("the instruction to be valid because the code is validated");
562
563    // SAFETY: All possible instruction handler functions use the same safety requirements, as
564    // they are defined through the same macro: The caller ensures that the resumable is valid
565    // in the current store. Also all other address types passed via the `Args` must come from
566    // the current store itself. Therefore, they are automatically valid in this store.
567    unsafe {
568        instruction_fn(
569            args.wasm,
570            args.resumable,
571            args.current_sidetable,
572            args.store_inner,
573            args.modules,
574            args.current_module,
575            args.current_function_end_marker,
576        )
577    }
578}}
wasm/execution/interpreter_loop/mod.rs

wasm/execution/interpreter_loop/
mod.rs
