wasmtime: Implement fast Wasm stack walking

Cargo.toml

@@ -126,9 +126,6 @@ harness = false
 name = "tokio"
 required-features = ["wasmtime-wasi/tokio"]
 
-[profile.dev.package.backtrace]
-debug = false # FIXME(#1813)
-
 [[bench]]
 name = "instantiation"
 harness = false

benches/call.rs

@@ -368,7 +368,7 @@ fn wasm_to_host(c: &mut Criterion) {
         desc: &str,
         is_async: IsAsync,
     ) {
-        group.bench_function(&format!("wasm-to-host - nop - {}", desc), |b| {
+        group.bench_function(&format!("wasm-to-host - {} - nop", desc), |b| {
             let run = instance
                 .get_typed_func::<u64, (), _>(&mut *store, "run-nop")
                 .unwrap();
@@ -383,7 +383,7 @@ fn wasm_to_host(c: &mut Criterion) {
             })
         });
         group.bench_function(
-            &format!("wasm-to-host - nop-params-and-results - {}", desc),
+            &format!("wasm-to-host - {} - nop-params-and-results", desc),
             |b| {
                 let run = instance
                     .get_typed_func::<u64, (), _>(&mut *store, "run-nop-params-and-results")

crates/asm-macros/src/lib.rs

@@ -5,12 +5,16 @@
 //! attributes correct (e.g. ELF symbols get a size and are flagged as a
 //! function) and additionally handles visibility across platforms. All symbols
 //! should be visible to Rust but not visible externally outside of a `*.so`.
+//!
+//! It also exports a an `asm_sym!` macro which can be used to reference symbols
+//! from within `global_asm!`-defined functions, and handles adding the leading
+//! underscore that macOS prepends to symbols for you.
 
 cfg_if::cfg_if! {
     if #[cfg(target_os = "macos")] {
         #[macro_export]
         macro_rules! asm_func {
-            ($name:tt, $($body:tt)*) => {
+            ($name:expr, $($body:tt)*) => {
                 std::arch::global_asm!(concat!(
                     ".p2align 4\n",
                     ".private_extern _", $name, "\n",
@@ -23,7 +27,28 @@ cfg_if::cfg_if! {
 
         #[macro_export]
         macro_rules! asm_sym {
-            ($name:tt) => (concat!("_", $name))
+            ( $( $name:tt )* ) => ( concat!("_", $( $name )* ) )
+        }
+    } else if #[cfg(target_os = "windows")] {
+        #[macro_export]
+        macro_rules! asm_func {
+            ($name:expr, $($body:tt)*) => {
+                std::arch::global_asm!(concat!(
+                    ".def ", $name, "\n",
+                    ".scl 2\n",
+                    ".type 32\n",
+                    ".endef\n",
+                    ".global ", $name, "\n",
+                    ".p2align 4\n",
+                    $name, ":\n",
+                    $($body)*
+                ));
+            };
+        }
+
+        #[macro_export]
+        macro_rules! asm_sym {
+            ( $( $name:tt )* ) => ( $( $name )* )
         }
     } else {
         // Note that for now this "else" clause just assumes that everything
@@ -45,22 +70,22 @@ cfg_if::cfg_if! {
 
         #[macro_export]
         macro_rules! asm_func {
-            ($name:tt, $($body:tt)*) => {
+            ($name:expr, $($body:tt)*) => {
                 std::arch::global_asm!(concat!(
                     ".p2align 4\n",
                     ".hidden ", $name, "\n",
                     ".global ", $name, "\n",
                     $crate::elf_func_type_header!($name),
                     $name, ":\n",
-                    $($body)*
+                    concat!($($body)*),
                     ".size ", $name, ",.-", $name,
                 ));
             };
         }
 
         #[macro_export]
         macro_rules! asm_sym {
-            ($name:tt) => ($name)
+            ( $( $name:tt )* ) => ( $( $name )* )
         }
     }
 }

crates/cranelift/src/func_environ.rs

@@ -30,7 +30,7 @@ macro_rules! declare_function_signatures {
     (
         $(
             $( #[$attr:meta] )*
-            $name:ident( $( $param:ident ),* ) -> ( $( $result:ident ),* );
+            $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
         )*
     ) => {
         /// A struct with an `Option<ir::SigRef>` member for every builtin
@@ -94,7 +94,7 @@ macro_rules! declare_function_signatures {
                     let sig = self.$name.unwrap_or_else(|| {
                         func.import_signature(Signature {
                             params: vec![ $( self.$param() ),* ],
-                            returns: vec![ $( self.$result() ),* ],
+                            returns: vec![ $( self.$result() )? ],
                             call_conv: self.call_conv,
                         })
                     });
@@ -1197,11 +1197,11 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m
                 let builtin_sig = self
                     .builtin_function_signatures
                     .drop_externref(builder.func);
-                let (_vmctx, builtin_addr) = self
+                let (vmctx, builtin_addr) = self
                     .translate_load_builtin_function_address(&mut builder.cursor(), builtin_idx);
                 builder
                     .ins()
-                    .call_indirect(builtin_sig, builtin_addr, &[current_elem]);
+                    .call_indirect(builtin_sig, builtin_addr, &[vmctx, current_elem]);
                 builder.ins().jump(continue_block, &[]);
 
                 builder.switch_to_block(continue_block);

crates/environ/src/builtin.rs

@@ -4,53 +4,53 @@ macro_rules! foreach_builtin_function {
     ($mac:ident) => {
         $mac! {
             /// Returns an index for wasm's `memory.grow` builtin function.
-            memory32_grow(vmctx, i64, i32) -> (pointer);
+            memory32_grow(vmctx: vmctx, delta: i64, index: i32) -> pointer;
             /// Returns an index for wasm's `table.copy` when both tables are locally
             /// defined.
-            table_copy(vmctx, i32, i32, i32, i32, i32) -> ();
+            table_copy(vmctx: vmctx, dst_index: i32, src_index: i32, dst: i32, src: i32, len: i32);
             /// Returns an index for wasm's `table.init`.
-            table_init(vmctx, i32, i32, i32, i32, i32) -> ();
+            table_init(vmctx: vmctx, table: i32, elem: i32, dst: i32, src: i32, len: i32);
             /// Returns an index for wasm's `elem.drop`.
-            elem_drop(vmctx, i32) -> ();
+            elem_drop(vmctx: vmctx, elem: i32);
             /// Returns an index for wasm's `memory.copy`
-            memory_copy(vmctx, i32, i64, i32, i64, i64) -> ();
+            memory_copy(vmctx: vmctx, dst_index: i32, dst: i64, src_index: i32, src: i64, len: i64);
             /// Returns an index for wasm's `memory.fill` instruction.
-            memory_fill(vmctx, i32, i64, i32, i64) -> ();
+            memory_fill(vmctx: vmctx, memory: i32, dst: i64, val: i32, len: i64);
             /// Returns an index for wasm's `memory.init` instruction.
-            memory_init(vmctx, i32, i32, i64, i32, i32) -> ();
+            memory_init(vmctx: vmctx, memory: i32, data: i32, dst: i64, src: i32, len: i32);
             /// Returns a value for wasm's `ref.func` instruction.
-            ref_func(vmctx, i32) -> (pointer);
+            ref_func(vmctx: vmctx, func: i32) -> pointer;
             /// Returns an index for wasm's `data.drop` instruction.
-            data_drop(vmctx, i32) -> ();
+            data_drop(vmctx: vmctx, data: i32);
             /// Returns a table entry after lazily initializing it.
-            table_get_lazy_init_funcref(vmctx, i32, i32) -> (pointer);
+            table_get_lazy_init_funcref(vmctx: vmctx, table: i32, index: i32) -> pointer;
             /// Returns an index for Wasm's `table.grow` instruction for `funcref`s.
-            table_grow_funcref(vmctx, i32, i32, pointer) -> (i32);
+            table_grow_funcref(vmctx: vmctx, table: i32, delta: i32, init: pointer) -> i32;
             /// Returns an index for Wasm's `table.grow` instruction for `externref`s.
-            table_grow_externref(vmctx, i32, i32, reference) -> (i32);
+            table_grow_externref(vmctx: vmctx, table: i32, delta: i32, init: reference) -> i32;
             /// Returns an index for Wasm's `table.fill` instruction for `externref`s.
-            table_fill_externref(vmctx, i32, i32, reference, i32) -> ();
+            table_fill_externref(vmctx: vmctx, table: i32, dst: i32, val: reference, len: i32);
             /// Returns an index for Wasm's `table.fill` instruction for `funcref`s.
-            table_fill_funcref(vmctx, i32, i32, pointer, i32) -> ();
+            table_fill_funcref(vmctx: vmctx, table: i32, dst: i32, val: pointer, len: i32);
             /// Returns an index to drop a `VMExternRef`.
-            drop_externref(pointer) -> ();
+            drop_externref(vmctx: vmctx, val: pointer);
             /// Returns an index to do a GC and then insert a `VMExternRef` into the
             /// `VMExternRefActivationsTable`.
-            activations_table_insert_with_gc(vmctx, reference) -> ();
+            activations_table_insert_with_gc(vmctx: vmctx, val: reference);
             /// Returns an index for Wasm's `global.get` instruction for `externref`s.
-            externref_global_get(vmctx, i32) -> (reference);
+            externref_global_get(vmctx: vmctx, global: i32) -> reference;
             /// Returns an index for Wasm's `global.get` instruction for `externref`s.
-            externref_global_set(vmctx, i32, reference) -> ();
+            externref_global_set(vmctx: vmctx, global: i32, val: reference);
             /// Returns an index for wasm's `memory.atomic.notify` instruction.
-            memory_atomic_notify(vmctx, i32, pointer, i32) -> (i32);
+            memory_atomic_notify(vmctx: vmctx, memory: i32, addr: pointer, count: i32) -> i32;
             /// Returns an index for wasm's `memory.atomic.wait32` instruction.
-            memory_atomic_wait32(vmctx, i32, pointer, i32, i64) -> (i32);
+            memory_atomic_wait32(vmctx: vmctx, memory: i32, addr: pointer, expected: i32, timeout: i64) -> i32;
             /// Returns an index for wasm's `memory.atomic.wait64` instruction.
-            memory_atomic_wait64(vmctx, i32, pointer, i64, i64) -> (i32);
+            memory_atomic_wait64(vmctx: vmctx, memory: i32, addr: pointer, expected: i64, timeout: i64) -> i32;
             /// Invoked when fuel has run out while executing a function.
-            out_of_gas(vmctx) -> ();
+            out_of_gas(vmctx: vmctx);
             /// Invoked when we reach a new epoch.
-            new_epoch(vmctx) -> (i64);
+            new_epoch(vmctx: vmctx) -> i64;
         }
     };
 }
@@ -75,7 +75,7 @@ macro_rules! declare_indexes {
     (
         $(
             $( #[$attr:meta] )*
-            $name:ident( $( $param:ident ),* ) -> ( $( $result:ident ),* );
+            $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
         )*
     ) => {
         impl BuiltinFunctionIndex {

crates/environ/src/vmoffsets.rs

@@ -5,7 +5,9 @@
 //
 // struct VMContext {
 //      magic: u32,
+//      _padding: u32, // (On 64-bit systems)
 //      runtime_limits: *const VMRuntimeLimits,
+//      callee: *mut VMFunctionBody,
 //      externref_activations_table: *mut VMExternRefActivationsTable,
 //      store: *mut dyn Store,
 //      builtins: *mut VMBuiltinFunctionsArray,
@@ -78,6 +80,7 @@ pub struct VMOffsets<P> {
     // precalculated offsets of various member fields
     magic: u32,
     runtime_limits: u32,
+    callee: u32,
     epoch_ptr: u32,
     externref_activations_table: u32,
     store: u32,
@@ -269,6 +272,7 @@ impl<P: PtrSize> VMOffsets<P> {
             store: "jit store state",
             externref_activations_table: "jit host externref state",
             epoch_ptr: "jit current epoch state",
+            callee: "callee function pointer",
             runtime_limits: "jit runtime limits state",
             magic: "magic value",
         }
@@ -290,6 +294,7 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             num_escaped_funcs: fields.num_escaped_funcs,
             magic: 0,
             runtime_limits: 0,
+            callee: 0,
             epoch_ptr: 0,
             externref_activations_table: 0,
             store: 0,
@@ -340,6 +345,7 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
             size(magic) = 4u32,
             align(u32::from(ret.ptr.size())),
             size(runtime_limits) = ret.ptr.size(),
+            size(callee) = ret.ptr.size(),
             size(epoch_ptr) = ret.ptr.size(),
             size(externref_activations_table) = ret.ptr.size(),
             size(store) = ret.ptr.size() * 2,
@@ -556,7 +562,22 @@ impl<P: PtrSize> VMOffsets<P> {
     /// Return the offset of the `epoch_deadline` field of `VMRuntimeLimits`
     #[inline]
     pub fn vmruntime_limits_epoch_deadline(&self) -> u8 {
-        self.pointer_size() + 8 // `stack_limit` is a pointer; `fuel_consumed` is an `i64`
+        self.vmruntime_limits_fuel_consumed() + 8 // `stack_limit` is a pointer; `fuel_consumed` is an `i64`
+    }
+
+    /// Return the offset of the `last_wasm_exit_fp` field of `VMRuntimeLimits`.
+    pub fn vmruntime_limits_last_wasm_exit_fp(&self) -> u8 {
+        self.vmruntime_limits_epoch_deadline() + 8
+    }
+
+    /// Return the offset of the `last_wasm_exit_pc` field of `VMRuntimeLimits`.
+    pub fn vmruntime_limits_last_wasm_exit_pc(&self) -> u8 {
+        self.vmruntime_limits_last_wasm_exit_fp() + self.pointer_size()
+    }
+
+    /// Return the offset of the `last_enty_sp` field of `VMRuntimeLimits`.
+    pub fn vmruntime_limits_last_wasm_entry_sp(&self) -> u8 {
+        self.vmruntime_limits_last_wasm_exit_pc() + self.pointer_size()
     }
 }
 
@@ -574,6 +595,11 @@ impl<P: PtrSize> VMOffsets<P> {
         self.runtime_limits
     }
 
+    /// Return the offset to the `callee` member in this `VMContext`.
+    pub fn vmctx_callee(&self) -> u32 {
+        self.callee
+    }
+
     /// Return the offset to the `*const AtomicU64` epoch-counter
     /// pointer.
     #[inline]
@@ -824,6 +850,12 @@ impl<P: PtrSize> VMOffsets<P> {
     }
 }
 
+/// Equivalent of `VMCONTEXT_MAGIC` except for host functions.
+///
+/// This is stored at the start of all `VMHostFuncContext` structures and
+/// double-checked on `VMHostFuncContext::from_opaque`.
+pub const VM_HOST_FUNC_MAGIC: u32 = u32::from_le_bytes(*b"host");
+
 #[cfg(test)]
 mod tests {
     use crate::vmoffsets::align;

crates/fuzzing/src/generators.rs

@@ -17,6 +17,7 @@ mod memory;
 mod module_config;
 mod single_inst_module;
 mod spec_test;
+mod stacks;
 pub mod table_ops;
 
 pub use codegen_settings::CodegenSettings;
@@ -27,3 +28,4 @@ pub use memory::{MemoryConfig, NormalMemoryConfig, UnalignedMemory, UnalignedMem
 pub use module_config::ModuleConfig;
 pub use single_inst_module::SingleInstModule;
 pub use spec_test::SpecTest;
+pub use stacks::Stacks;