Skip to content

Commit 44f7ec2

Browse files
committed
perf: keep each value stack's height outside its Vec
The height of a value-stack lane is its own field, over a Vec that holds every slot the lane has reached. Slots above the height hold stale values that are written before they are read again. The tail-call handlers can then pass the height between them in a register and write it back without unsafe code. Entering a function writes its locals, and its operand-stack reservation if that is at most 64 slots, so later entries at that height compare once. A larger reservation stays capacity, and a push writes each slot the first time it reaches it, so a function whose deep branch rarely runs keeps no pages for it. With 64 stores of a module whose untaken branch declares a 30000-deep i64 stack, each store keeps next's footprint (35 KB on macOS, 16-19 KB on glibc) instead of 273 KB and 250 KB. v128.any_true and i8x16.all_true test the vector with integer arithmetic: LLVM scalarized their iterator forms once pushes gained that path. Only builds with debug assertions check indices against the height. CI also runs the tests optimized with them, keeping the release profile's wrapping arithmetic. Entering a function zeroes one or two locals with direct stores, and only the ones in between with memset.
1 parent bff2507 commit 44f7ec2

3 files changed

Lines changed: 195 additions & 47 deletions

File tree

‎.github/workflows/test.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ jobs:
4646
name: "Linux x86 (nightly tail calls)"
4747
target: x86_64-unknown-linux-gnu
4848
args: "--features tinywasm/nightly-tail-calls"
49+
- os: ubuntu-26.04
50+
rust: nightly
51+
name: "Linux x86 (nightly tail calls, release with debug assertions)"
52+
target: x86_64-unknown-linux-gnu
53+
args: "--release --config profile.release.debug-assertions=true --features tinywasm/nightly-tail-calls"
4954
- os: ubuntu-26.04
5055
rust: stable
5156
name: "Linux x86 (stable, no default features)"

‎crates/tinywasm/src/interpreter/simd/instructions.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ impl Value128 {
3030
pub(crate) fn v128_any_true(self) -> bool {
3131
simd_impl! {
3232
wasm => { wasm::v128_any_true(self.to_wasm_v128()) }
33-
generic => { self.0.iter().any(|&b| b != 0) }
33+
generic => { u128::from_ne_bytes(self.0) != 0 }
3434
}
3535
}
3636

@@ -498,7 +498,11 @@ impl Value128 {
498498
"f64x2.extract_lane" => extract_lane_f64(self, lane: u8) -> f64 => f64::from_bits(self.extract_lane_i64(lane) as u64);
499499

500500
// Truth checks, popcount, and averaging
501-
"i8x16.all_true" => i8x16_all_true(self) -> bool => self.0.iter().all(|&b| b != 0);
501+
"i8x16.all_true" => i8x16_all_true(self) -> bool => {
502+
// No byte is zero: the classic zero-byte test on the whole vector.
503+
let v = u128::from_ne_bytes(self.0);
504+
v.wrapping_sub(u128::from_ne_bytes([1; 16])) & !v & u128::from_ne_bytes([0x80; 16]) == 0
505+
};
502506
"i16x8.all_true" => i16x8_all_true(self) -> bool => self.as_i16x8().iter().all(|&x| x != 0);
503507
"i32x4.all_true" => i32x4_all_true(self) -> bool => self.as_i32x4().iter().all(|&x| x != 0);
504508
"i64x2.all_true" => i64x2_all_true(self) -> bool => self.as_i64x2().iter().all(|&x| x != 0);

‎crates/tinywasm/src/interpreter/stack/value_stack.rs‎

Lines changed: 184 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -18,31 +18,59 @@ pub(crate) struct ValueStack {
1818
}
1919

2020
#[cfg_attr(feature = "debug", derive(Debug))]
21+
/// One value lane: a stack of `len` values over `data`.
22+
///
23+
/// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale
24+
/// values that are written before they are read again. A large operand-stack reservation stays
25+
/// capacity, so its slots are written, and their pages touched, only when a push first reaches them.
26+
/// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and
27+
/// write it back without `unsafe`.
2128
pub(crate) struct Stack<T: Copy + Default> {
2229
data: Vec<T>,
30+
len: usize,
2331
max_size: usize,
2432
dynamic: bool,
2533
}
2634

35+
/// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them.
36+
const WRITTEN_RESERVATION: usize = 64;
37+
2738
impl<T: Copy + Default> Stack<T> {
2839
pub(crate) fn new(config: StackConfig) -> Self {
29-
Self { data: Vec::with_capacity(config.initial_size), max_size: config.max_size, dynamic: config.dynamic }
40+
Self {
41+
data: Vec::with_capacity(config.initial_size),
42+
len: 0,
43+
max_size: config.max_size,
44+
dynamic: config.dynamic,
45+
}
3046
}
3147

3248
pub(crate) fn clear(&mut self) {
33-
self.data.clear();
49+
self.len = 0;
3450
}
3551

3652
#[inline(always)]
3753
pub(crate) fn len(&self) -> usize {
38-
self.data.len()
54+
self.len
3955
}
4056

4157
/// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand
42-
/// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own
43-
/// growth path, so the instruction handlers make no calls.
58+
/// stack, so the stack is never full here, and the handlers make no calls.
4459
#[inline(always)]
4560
pub(crate) fn push(&mut self, value: T) {
61+
let len = self.len;
62+
match self.data.get_mut(len) {
63+
Some(slot) => *slot = value,
64+
None => self.push_first(value),
65+
}
66+
self.len = len + 1;
67+
}
68+
69+
/// Adds the first slot at a height the stack has not reached before, within the reserved
70+
/// capacity: it never reallocates. On the tail-call build this makes no call.
71+
#[inline(always)]
72+
fn push_first(&mut self, value: T) {
73+
core::hint::cold_path();
4674
if self.data.len() == self.data.capacity() {
4775
crate::invariant_violated("value stack push beyond the function's reservation");
4876
}
@@ -53,10 +81,12 @@ impl<T: Copy + Default> Stack<T> {
5381
/// covers, so a dynamic stack grows here if needed.
5482
#[inline(always)]
5583
pub(crate) fn push_or_grow(&mut self, value: T) -> Result<(), Trap> {
56-
if self.data.len() == self.data.capacity() {
57-
return self.push_grow(value);
84+
let len = self.len;
85+
match self.data.get_mut(len) {
86+
Some(slot) => *slot = value,
87+
None => return self.push_grow(value),
5888
}
59-
self.data.push(value);
89+
self.len = len + 1;
6090
Ok(())
6191
}
6292

@@ -69,33 +99,38 @@ impl<T: Copy + Default> Stack<T> {
6999
#[cold]
70100
#[inline(never)]
71101
fn push_grow(&mut self, value: T) -> Result<(), Trap> {
72-
// Check the limit only at capacity to avoid an extra hot-path check. Vec growth may
73-
// intentionally overshoot max_size.
74-
if !self.dynamic || self.data.len() >= self.max_size {
102+
// Only the reserved capacity is free to use. Past it, check the limit, which Vec growth
103+
// may intentionally overshoot.
104+
if self.data.len() == self.data.capacity() && (!self.dynamic || self.data.len() >= self.max_size) {
75105
return Err(Trap::ValueStackOverflow);
76106
}
77107
self.data.push(value);
108+
self.len = self.data.len();
78109
Ok(())
79110
}
80111

112+
/// Pops the top value. On an empty stack the index wraps around, so the bounds check in
113+
/// [`Self::get`] also catches an underflow.
81114
#[inline(always)]
82115
pub(crate) fn pop(&mut self) -> T {
83-
match self.data.pop() {
84-
Some(value) => value,
85-
None => crate::invariant_violated("value stack underflow"),
86-
}
116+
let index = self.len.wrapping_sub(1);
117+
let value = *self.get(index);
118+
self.len = index;
119+
value
87120
}
88121

89122
#[inline(always)]
90123
pub(crate) fn last(&self) -> &T {
91-
match self.data.last() {
92-
Some(value) => value,
93-
None => crate::invariant_violated("value stack underflow"),
94-
}
124+
self.get(self.len.wrapping_sub(1))
95125
}
96126

127+
/// The slot at `index`, which validation keeps below the height. The check is against the
128+
/// slots in use at any height, which is what memory safety needs. Builds with debug assertions
129+
/// also check the height, and CI runs the tests optimized with them, with the release profile's
130+
/// wrapping arithmetic.
97131
#[inline(always)]
98132
pub(crate) fn get(&self, index: usize) -> &T {
133+
debug_assert!(index < self.len);
99134
match self.data.get(index) {
100135
Some(value) => value,
101136
None => crate::invariant_violated("value stack index out of range"),
@@ -104,6 +139,7 @@ impl<T: Copy + Default> Stack<T> {
104139

105140
#[inline(always)]
106141
pub(crate) fn set(&mut self, index: usize, value: T) {
142+
debug_assert!(index < self.len);
107143
match self.data.get_mut(index) {
108144
Some(slot) => *slot = value,
109145
None => crate::invariant_violated("value stack index out of range"),
@@ -118,32 +154,36 @@ impl<T: Copy + Default> Stack<T> {
118154

119155
#[inline(always)]
120156
pub(crate) fn truncate_keep(&mut self, n: usize, end_keep: usize) {
121-
let len = self.data.len();
157+
let len = self.len;
122158
debug_assert!(n <= len);
123159
if n >= len {
124160
return;
125161
}
126162

127163
let keep = len.wrapping_sub(n).min(end_keep);
128-
self.data.copy_within(len.wrapping_sub(keep)..len, n);
129-
self.data.truncate(n.wrapping_add(keep));
164+
if keep != 0 {
165+
// Copying to the start of the values above `n` leaves copy_within no check to fail.
166+
match self.data.get_mut(n..len) {
167+
Some(above) => above.copy_within(above.len().wrapping_sub(keep).., 0),
168+
None => crate::invariant_violated("value stack index out of range"),
169+
}
170+
}
171+
self.len = n.wrapping_add(keep);
130172
}
131173

132174
#[inline(always)]
133175
pub(crate) fn truncate_to(&mut self, n: usize) {
134-
debug_assert!(n <= self.data.len());
135-
self.data.truncate(n);
176+
debug_assert!(n <= self.len);
177+
self.len = n;
136178
}
137179

138180
/// Moves the top value down to index `n` and drops everything above it (a function's result
139181
/// replacing its frame).
140182
#[inline(always)]
141183
pub(crate) fn truncate_to_one_tail(&mut self, n: usize) {
142-
let len = self.data.len();
143-
debug_assert!(n < len);
144-
let last = self.data[len.wrapping_sub(1)];
145-
self.data[n] = last;
146-
self.data.truncate(n + 1);
184+
debug_assert!(n < self.len);
185+
self.copy(self.len.wrapping_sub(1), n);
186+
self.len = n + 1;
147187
}
148188

149189
/// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves
@@ -157,36 +197,75 @@ impl<T: Copy + Default> Stack<T> {
157197
max_stack: usize,
158198
) -> Result<u32, Trap> {
159199
debug_assert!(param_count <= local_count);
160-
debug_assert!(param_count <= self.data.len());
200+
debug_assert!(param_count <= self.len);
161201

162-
let len = self.data.len();
202+
let len = self.len;
163203
let start = len - param_count;
164204
let end = start + local_count;
165205
let reserve = end + max_stack;
166206

167-
if reserve > self.data.capacity() {
207+
if reserve > self.data.len() {
168208
core::hint::cold_path();
169-
if reserve > self.max_size || !self.dynamic {
170-
return Err(Trap::ValueStackOverflow);
171-
}
172-
let cap = self.data.capacity();
173-
let target = reserve.max(cap.max(1).saturating_mul(2)).min(self.max_size);
174-
if self.data.try_reserve(target - len).is_err() {
175-
return Err(Trap::ValueStackOverflow);
176-
}
209+
self.reserve_slots(end, reserve)?;
177210
}
178211

179-
self.data.resize(end, T::default());
212+
// Most functions have no or few locals in a lane. Store the first and last directly and
213+
// fill only what lies between: `fill` becomes a memset call even for one value.
214+
if end > len {
215+
match self.data.get_mut(len..end) {
216+
Some([a]) => *a = T::default(),
217+
Some([a, middle @ .., b]) => {
218+
if !middle.is_empty() {
219+
middle.fill(T::default());
220+
}
221+
(*a, *b) = (T::default(), T::default());
222+
}
223+
_ => crate::invariant_violated("value stack index out of range"),
224+
}
225+
}
226+
self.len = end;
180227
Ok(start as u32)
181228
}
182229

230+
/// Makes slots for a function's locals and reserves its operand stack when they reach past the
231+
/// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out
232+
/// with the locals, so later entries at this height skip this. A larger one stays capacity and
233+
/// its slots are written as they are pushed, so a function whose deep branch rarely runs does
234+
/// not keep pages for it; its entries come back here.
235+
#[cold]
236+
#[inline(never)]
237+
fn reserve_slots(&mut self, end: usize, reserve: usize) -> Result<(), Trap> {
238+
if reserve > self.data.capacity() {
239+
self.grow_to(reserve)?;
240+
}
241+
let slots = if reserve - self.data.len() <= WRITTEN_RESERVATION { reserve } else { end };
242+
if slots > self.data.len() {
243+
self.data.resize(slots, T::default());
244+
}
245+
Ok(())
246+
}
247+
248+
/// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it.
249+
#[cold]
250+
#[inline(never)]
251+
fn grow_to(&mut self, reserve: usize) -> Result<(), Trap> {
252+
if reserve > self.max_size || !self.dynamic {
253+
return Err(Trap::ValueStackOverflow);
254+
}
255+
let target = reserve.max(self.data.capacity().max(1).saturating_mul(2)).min(self.max_size);
256+
if self.data.try_reserve(target - self.data.len()).is_err() {
257+
return Err(Trap::ValueStackOverflow);
258+
}
259+
Ok(())
260+
}
261+
183262
#[inline(always)]
184263
pub(crate) fn select_many(&mut self, count: usize, condition: bool) {
185264
if count == 0 {
186265
return;
187266
}
188267

189-
let len = self.data.len();
268+
let len = self.len;
190269
let needed = count.wrapping_mul(2);
191270

192271
if len < needed {
@@ -196,10 +275,13 @@ impl<T: Copy + Default> Stack<T> {
196275
if !condition {
197276
let dst = len.wrapping_sub(needed);
198277
let src = len.wrapping_sub(count);
199-
self.data.copy_within(src..len, dst);
278+
match self.data.get_mut(..len) {
279+
Some(live) => live.copy_within(src..len, dst),
280+
None => crate::invariant_violated("value stack index out of range"),
281+
}
200282
}
201283

202-
self.data.truncate(len.wrapping_sub(count));
284+
self.len = len.wrapping_sub(count);
203285
}
204286
}
205287

@@ -208,7 +290,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack<T> {
208290
type IntoIter = core::slice::Iter<'a, T>;
209291

210292
fn into_iter(self) -> Self::IntoIter {
211-
self.data.iter()
293+
self.data[..self.len].iter()
212294
}
213295
}
214296
impl ValueStack {
@@ -327,3 +409,60 @@ impl ValueStack {
327409
}
328410
}
329411
}
412+
413+
#[cfg(test)]
414+
mod tests {
415+
use super::*;
416+
417+
/// A stack of height 2 whose third slot still holds a popped value.
418+
#[cfg(debug_assertions)]
419+
fn popped() -> Stack<Value32> {
420+
let mut stack = Stack::new(StackConfig::fixed(8));
421+
for value in [1, 2, 3] {
422+
stack.push_or_grow(value).unwrap();
423+
}
424+
assert_eq!(stack.pop(), 3);
425+
stack
426+
}
427+
428+
#[test]
429+
#[cfg(debug_assertions)]
430+
#[should_panic]
431+
fn get_above_the_height() {
432+
popped().get(2);
433+
}
434+
435+
#[test]
436+
#[cfg(debug_assertions)]
437+
#[should_panic]
438+
fn set_above_the_height() {
439+
popped().set(2, 0);
440+
}
441+
442+
/// A large operand-stack reservation is not written: only the locals and the values pushed
443+
/// become slots. A small one is written with the locals.
444+
#[test]
445+
fn large_reservations_are_written_as_they_are_pushed() {
446+
let mut stack = Stack::<Value64>::new(StackConfig::fixed(1024));
447+
assert!(matches!(stack.enter_locals(0, 2, 500), Ok(0)));
448+
assert_eq!(stack.data.len(), 2);
449+
for value in 0..500 {
450+
stack.push(value);
451+
}
452+
assert_eq!((stack.len(), stack.data.len()), (502, 502));
453+
454+
// Below the slots the stack already has, entering a function writes only its locals.
455+
stack.truncate_to(2);
456+
stack.push(7);
457+
assert!(matches!(stack.enter_locals(0, 2, 10), Ok(3)));
458+
assert_eq!((stack.len(), stack.data.len()), (5, 502));
459+
460+
stack.truncate_to(0);
461+
for value in 0..500 {
462+
stack.push_or_grow(value).unwrap();
463+
}
464+
assert!(matches!(stack.enter_locals(0, 1, 8), Ok(500)));
465+
assert_eq!(stack.data.len(), 509);
466+
assert!(matches!(stack.enter_locals(0, 2, 1000), Err(Trap::ValueStackOverflow)));
467+
}
468+
}

0 commit comments

Comments
 (0)