perf: pick the tail-call handlers' calling convention per target

The Unbudgeted handlers take five integer arguments. The Rust ABI passes
six in registers on x86-64 System V but four on Windows, which puts the
Instruction on the stack on every dispatch there. `rust-preserve-none`
(nightly, like `become`) passes twelve in registers on every OS and keeps
only RSP and RBP callee-saved, so a handler saves at most RBP.

On arm64_32 (watchOS) the Rust ABI passes the 8-byte Instruction by
reference, so every dispatch stores it and the next handler loads it back.
The C convention passes it in a register, and `C-unwind` still lets a
host function's panic unwind.

Other targets keep the Rust ABI.

Author: matthargett

crates/tinywasm/src/interpreter/executor/dispatch_become.rs

@@ -3,8 +3,45 @@ use super::*;
 struct Unbudgeted;
 struct Bounded;
 
-type UnbudgetedHandler = for<'store> fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>;
-type BoundedHandler = for<'store> fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>;
+// Between two handlers only what the calling convention passes in registers stays out of memory,
+// so the handlers' convention depends on the target:
+// - x86-64: the Rust ABI passes six integer arguments in registers on System V but four on
+//   Windows, which would put the Instruction on the stack. `rust-preserve-none`
+//   passes twelve on every OS and keeps only RSP and RBP callee-saved, so a handler saves at most
+//   RBP, and only when it uses it.
+// - arm64_32 (watchOS): the Rust ABI passes an aggregate larger than a pointer, the 8-byte
+//   `Instruction`, by reference, so every dispatch would store it and the next handler load it
+//   back. The C convention passes it in a register; `C-unwind` still lets a host function's
+//   panic unwind.
+// - elsewhere, the Rust ABI (arm64 passes 8 integer arguments in registers).
+macro_rules! handler_fn {
+    ($(#[$meta:meta])* fn $($rest:tt)*) => {
+        #[cfg(target_arch = "x86_64")]
+        $(#[$meta])* extern "rust-preserve-none" fn $($rest)*
+        #[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))]
+        #[allow(improper_ctypes_definitions)]
+        $(#[$meta])* extern "C-unwind" fn $($rest)*
+        #[cfg(not(any(target_arch = "x86_64", all(target_arch = "aarch64", target_pointer_width = "32"))))]
+        $(#[$meta])* fn $($rest)*
+    };
+}
+
+macro_rules! handler_types {
+    ($($abi:literal)?) => {
+        // Both sides are Rust, so the C convention's view of these types need not be FFI-safe.
+        #[allow(improper_ctypes_definitions)]
+        type UnbudgetedHandler =
+            for<'store> $(extern $abi)? fn(&mut Executor<'store>, &[Instruction], usize, Instruction) -> ExecResult<()>;
+        #[allow(improper_ctypes_definitions)]
+        type BoundedHandler = for<'store> $(extern $abi)? fn(&mut Executor<'store>, usize, Instruction, u32) -> ExecResult<()>;
+    };
+}
+#[cfg(target_arch = "x86_64")]
+handler_types!("rust-preserve-none");
+#[cfg(all(target_arch = "aarch64", target_pointer_width = "32"))]
+handler_types!("C-unwind");
+#[cfg(not(any(target_arch = "x86_64", all(target_arch = "aarch64", target_pointer_width = "32"))))]
+handler_types!();
 
 macro_rules! define_unbudgeted_tail_dispatch {
     ($executor:ident, $instr_ptr:ident, $dispatch_next:ident, $dispatch_flow:ident;
@@ -19,7 +56,7 @@ macro_rules! define_unbudgeted_tail_dispatch {
             HANDLERS[opcode as usize]
         }
 
-        $(
+        $(handler_fn! {
             #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)]
             fn $variant(
                 $executor: &mut Executor<'_>,
@@ -59,7 +96,7 @@ macro_rules! define_unbudgeted_tail_dispatch {
                 $body;
                 $dispatch_next!($instr_ptr + 1)
             }
-        )*
+        })*
     };
 }
 
@@ -76,7 +113,7 @@ macro_rules! define_bounded_tail_dispatch {
             HANDLERS[opcode as usize]
         }
 
-        $(
+        $(handler_fn! {
             #[allow(non_snake_case, unreachable_code, unused_imports, unused_macros, unused_variables)]
             fn $variant(
                 $executor: &mut Executor<'_>,
@@ -122,7 +159,7 @@ macro_rules! define_bounded_tail_dispatch {
                 $body;
                 $dispatch_next!($instr_ptr + 1)
             }
-        )*
+        })*
     };
 }
 
@@ -132,16 +169,20 @@ impl Unbudgeted {
     // The handlers tail-call these cold paths instead of calling them: a call would make every
     // handler save a stack frame.
 
-    #[cold]
-    #[inline(never)]
-    fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> {
-        unreachable!("instruction handler mismatch")
+    handler_fn! {
+        #[cold]
+        #[inline(never)]
+        fn handler_mismatch(_: &mut Executor<'_>, _: &[Instruction], _: usize, _: Instruction) -> ExecResult<()> {
+            unreachable!("instruction handler mismatch")
+        }
     }
 
-    #[cold]
-    #[inline(never)]
-    fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> {
-        unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
+    handler_fn! {
+        #[cold]
+        #[inline(never)]
+        fn invalid_instr_ptr(_: &mut Executor<'_>, _: &[Instruction], instr_ptr: usize, _: Instruction) -> ExecResult<()> {
+            unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
+        }
     }
 }
 
@@ -150,16 +191,20 @@ impl Bounded {
 
     // Tail-called like `Unbudgeted`'s.
 
-    #[cold]
-    #[inline(never)]
-    fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> {
-        unreachable!("instruction handler mismatch")
+    handler_fn! {
+        #[cold]
+        #[inline(never)]
+        fn handler_mismatch(_: &mut Executor<'_>, _: usize, _: Instruction, _: u32) -> ExecResult<()> {
+            unreachable!("instruction handler mismatch")
+        }
     }
 
-    #[cold]
-    #[inline(never)]
-    fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> {
-        unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
+    handler_fn! {
+        #[cold]
+        #[inline(never)]
+        fn invalid_instr_ptr(_: &mut Executor<'_>, instr_ptr: usize, _: Instruction, _: u32) -> ExecResult<()> {
+            unreachable!("instruction pointer {instr_ptr} out of range, this is a bug")
+        }
     }
 
     #[inline(always)]

crates/tinywasm/src/lib.rs

@@ -2,6 +2,7 @@
 #![cfg_attr(feature = "nightly-tail-calls", allow(incomplete_features))]
 #![cfg_attr(feature = "nightly-tail-calls", feature(explicit_tail_calls))]
 #![cfg_attr(all(feature = "nightly-tail-calls", not(debug_assertions)), feature(abort_immediate))]
+#![cfg_attr(all(feature = "nightly-tail-calls", target_arch = "x86_64"), feature(rust_preserve_none_cc))]
 #![doc(test(
     no_crate_inject,
     attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_assignments, unused_variables))