@@ -18,31 +18,59 @@ pub(crate) struct ValueStack {
1818}
1919
2020#[ cfg_attr( feature = "debug" , derive( Debug ) ) ]
21+ /// One value lane: a stack of `len` values over `data`.
22+ ///
23+ /// `data` holds every slot the stack has reached and only grows. The slots above `len` hold stale
24+ /// values that are written before they are read again. A large operand-stack reservation stays
25+ /// capacity, so its slots are written, and their pages touched, only when a push first reaches them.
26+ /// Keeping the height outside the `Vec` lets the tail-call handlers carry it in a register and
27+ /// write it back without `unsafe`.
2128pub ( crate ) struct Stack < T : Copy + Default > {
2229 data : Vec < T > ,
30+ len : usize ,
2331 max_size : usize ,
2432 dynamic : bool ,
2533}
2634
35+ /// The most slots [`Stack::enter_locals`] writes ahead of the pushes that reach them.
36+ const WRITTEN_RESERVATION : usize = 64 ;
37+
2738impl < T : Copy + Default > Stack < T > {
2839 pub ( crate ) fn new ( config : StackConfig ) -> Self {
29- Self { data : Vec :: with_capacity ( config. initial_size ) , max_size : config. max_size , dynamic : config. dynamic }
40+ Self {
41+ data : Vec :: with_capacity ( config. initial_size ) ,
42+ len : 0 ,
43+ max_size : config. max_size ,
44+ dynamic : config. dynamic ,
45+ }
3046 }
3147
3248 pub ( crate ) fn clear ( & mut self ) {
33- self . data . clear ( ) ;
49+ self . len = 0 ;
3450 }
3551
3652 #[ inline( always) ]
3753 pub ( crate ) fn len ( & self ) -> usize {
38- self . data . len ( )
54+ self . len
3955 }
4056
4157 /// Pushes a value inside a function body. `enter_locals` reserved the function's whole operand
42- /// stack, so the stack is never full here. After this check `Vec::push` cannot reach its own
43- /// growth path, so the instruction handlers make no calls.
58+ /// stack, so the stack is never full here, and the handlers make no calls.
4459 #[ inline( always) ]
4560 pub ( crate ) fn push ( & mut self , value : T ) {
61+ let len = self . len ;
62+ match self . data . get_mut ( len) {
63+ Some ( slot) => * slot = value,
64+ None => self . push_first ( value) ,
65+ }
66+ self . len = len + 1 ;
67+ }
68+
69+ /// Adds the first slot at a height the stack has not reached before, within the reserved
70+ /// capacity: it never reallocates. On the tail-call build this makes no call.
71+ #[ inline( always) ]
72+ fn push_first ( & mut self , value : T ) {
73+ core:: hint:: cold_path ( ) ;
4674 if self . data . len ( ) == self . data . capacity ( ) {
4775 crate :: invariant_violated ( "value stack push beyond the function's reservation" ) ;
4876 }
@@ -53,10 +81,12 @@ impl<T: Copy + Default> Stack<T> {
5381 /// covers, so a dynamic stack grows here if needed.
5482 #[ inline( always) ]
5583 pub ( crate ) fn push_or_grow ( & mut self , value : T ) -> Result < ( ) , Trap > {
56- if self . data . len ( ) == self . data . capacity ( ) {
57- return self . push_grow ( value) ;
84+ let len = self . len ;
85+ match self . data . get_mut ( len) {
86+ Some ( slot) => * slot = value,
87+ None => return self . push_grow ( value) ,
5888 }
59- self . data . push ( value ) ;
89+ self . len = len + 1 ;
6090 Ok ( ( ) )
6191 }
6292
@@ -69,33 +99,38 @@ impl<T: Copy + Default> Stack<T> {
6999 #[ cold]
70100 #[ inline( never) ]
71101 fn push_grow ( & mut self , value : T ) -> Result < ( ) , Trap > {
72- // Check the limit only at capacity to avoid an extra hot-path check. Vec growth may
73- // intentionally overshoot max_size .
74- if !self . dynamic || self . data . len ( ) >= self . max_size {
102+ // Only the reserved capacity is free to use. Past it, check the limit, which Vec growth
103+ // may intentionally overshoot.
104+ if self . data . len ( ) == self . data . capacity ( ) && ( !self . dynamic || self . data . len ( ) >= self . max_size ) {
75105 return Err ( Trap :: ValueStackOverflow ) ;
76106 }
77107 self . data . push ( value) ;
108+ self . len = self . data . len ( ) ;
78109 Ok ( ( ) )
79110 }
80111
112+ /// Pops the top value. On an empty stack the index wraps around, so the bounds check in
113+ /// [`Self::get`] also catches an underflow.
81114 #[ inline( always) ]
82115 pub ( crate ) fn pop ( & mut self ) -> T {
83- match self . data . pop ( ) {
84- Some ( value) => value ,
85- None => crate :: invariant_violated ( "value stack underflow" ) ,
86- }
116+ let index = self . len . wrapping_sub ( 1 ) ;
117+ let value = * self . get ( index ) ;
118+ self . len = index ;
119+ value
87120 }
88121
89122 #[ inline( always) ]
90123 pub ( crate ) fn last ( & self ) -> & T {
91- match self . data . last ( ) {
92- Some ( value) => value,
93- None => crate :: invariant_violated ( "value stack underflow" ) ,
94- }
124+ self . get ( self . len . wrapping_sub ( 1 ) )
95125 }
96126
127+ /// The slot at `index`, which validation keeps below the height. The check is against the
128+ /// slots in use at any height, which is what memory safety needs. Builds with debug assertions
129+ /// also check the height, and CI runs the tests optimized with them, with the release profile's
130+ /// wrapping arithmetic.
97131 #[ inline( always) ]
98132 pub ( crate ) fn get ( & self , index : usize ) -> & T {
133+ debug_assert ! ( index < self . len) ;
99134 match self . data . get ( index) {
100135 Some ( value) => value,
101136 None => crate :: invariant_violated ( "value stack index out of range" ) ,
@@ -104,6 +139,7 @@ impl<T: Copy + Default> Stack<T> {
104139
105140 #[ inline( always) ]
106141 pub ( crate ) fn set ( & mut self , index : usize , value : T ) {
142+ debug_assert ! ( index < self . len) ;
107143 match self . data . get_mut ( index) {
108144 Some ( slot) => * slot = value,
109145 None => crate :: invariant_violated ( "value stack index out of range" ) ,
@@ -118,32 +154,36 @@ impl<T: Copy + Default> Stack<T> {
118154
119155 #[ inline( always) ]
120156 pub ( crate ) fn truncate_keep ( & mut self , n : usize , end_keep : usize ) {
121- let len = self . data . len ( ) ;
157+ let len = self . len ;
122158 debug_assert ! ( n <= len) ;
123159 if n >= len {
124160 return ;
125161 }
126162
127163 let keep = len. wrapping_sub ( n) . min ( end_keep) ;
128- self . data . copy_within ( len. wrapping_sub ( keep) ..len, n) ;
129- self . data . truncate ( n. wrapping_add ( keep) ) ;
164+ if keep != 0 {
165+ // Copying to the start of the values above `n` leaves copy_within no check to fail.
166+ match self . data . get_mut ( n..len) {
167+ Some ( above) => above. copy_within ( above. len ( ) . wrapping_sub ( keep) .., 0 ) ,
168+ None => crate :: invariant_violated ( "value stack index out of range" ) ,
169+ }
170+ }
171+ self . len = n. wrapping_add ( keep) ;
130172 }
131173
132174 #[ inline( always) ]
133175 pub ( crate ) fn truncate_to ( & mut self , n : usize ) {
134- debug_assert ! ( n <= self . data . len( ) ) ;
135- self . data . truncate ( n ) ;
176+ debug_assert ! ( n <= self . len) ;
177+ self . len = n ;
136178 }
137179
138180 /// Moves the top value down to index `n` and drops everything above it (a function's result
139181 /// replacing its frame).
140182 #[ inline( always) ]
141183 pub ( crate ) fn truncate_to_one_tail ( & mut self , n : usize ) {
142- let len = self . data . len ( ) ;
143- debug_assert ! ( n < len) ;
144- let last = self . data [ len. wrapping_sub ( 1 ) ] ;
145- self . data [ n] = last;
146- self . data . truncate ( n + 1 ) ;
184+ debug_assert ! ( n < self . len) ;
185+ self . copy ( self . len . wrapping_sub ( 1 ) , n) ;
186+ self . len = n + 1 ;
147187 }
148188
149189 /// Enters a function: turns its parameters into the first locals, zeroes the rest, and reserves
@@ -157,36 +197,75 @@ impl<T: Copy + Default> Stack<T> {
157197 max_stack : usize ,
158198 ) -> Result < u32 , Trap > {
159199 debug_assert ! ( param_count <= local_count) ;
160- debug_assert ! ( param_count <= self . data . len( ) ) ;
200+ debug_assert ! ( param_count <= self . len) ;
161201
162- let len = self . data . len ( ) ;
202+ let len = self . len ;
163203 let start = len - param_count;
164204 let end = start + local_count;
165205 let reserve = end + max_stack;
166206
167- if reserve > self . data . capacity ( ) {
207+ if reserve > self . data . len ( ) {
168208 core:: hint:: cold_path ( ) ;
169- if reserve > self . max_size || !self . dynamic {
170- return Err ( Trap :: ValueStackOverflow ) ;
171- }
172- let cap = self . data . capacity ( ) ;
173- let target = reserve. max ( cap. max ( 1 ) . saturating_mul ( 2 ) ) . min ( self . max_size ) ;
174- if self . data . try_reserve ( target - len) . is_err ( ) {
175- return Err ( Trap :: ValueStackOverflow ) ;
176- }
209+ self . reserve_slots ( end, reserve) ?;
177210 }
178211
179- self . data . resize ( end, T :: default ( ) ) ;
212+ // Most functions have no or few locals in a lane. Store the first and last directly and
213+ // fill only what lies between: `fill` becomes a memset call even for one value.
214+ if end > len {
215+ match self . data . get_mut ( len..end) {
216+ Some ( [ a] ) => * a = T :: default ( ) ,
217+ Some ( [ a, middle @ .., b] ) => {
218+ if !middle. is_empty ( ) {
219+ middle. fill ( T :: default ( ) ) ;
220+ }
221+ ( * a, * b) = ( T :: default ( ) , T :: default ( ) ) ;
222+ }
223+ _ => crate :: invariant_violated ( "value stack index out of range" ) ,
224+ }
225+ }
226+ self . len = end;
180227 Ok ( start as u32 )
181228 }
182229
230+ /// Makes slots for a function's locals and reserves its operand stack when they reach past the
231+ /// slots the stack has had. A reservation of up to [`WRITTEN_RESERVATION`] slots is written out
232+ /// with the locals, so later entries at this height skip this. A larger one stays capacity and
233+ /// its slots are written as they are pushed, so a function whose deep branch rarely runs does
234+ /// not keep pages for it; its entries come back here.
235+ #[ cold]
236+ #[ inline( never) ]
237+ fn reserve_slots ( & mut self , end : usize , reserve : usize ) -> Result < ( ) , Trap > {
238+ if reserve > self . data . capacity ( ) {
239+ self . grow_to ( reserve) ?;
240+ }
241+ let slots = if reserve - self . data . len ( ) <= WRITTEN_RESERVATION { reserve } else { end } ;
242+ if slots > self . data . len ( ) {
243+ self . data . resize ( slots, T :: default ( ) ) ;
244+ }
245+ Ok ( ( ) )
246+ }
247+
248+ /// Makes room for `reserve` slots, growing the allocation if a dynamic stack allows it.
249+ #[ cold]
250+ #[ inline( never) ]
251+ fn grow_to ( & mut self , reserve : usize ) -> Result < ( ) , Trap > {
252+ if reserve > self . max_size || !self . dynamic {
253+ return Err ( Trap :: ValueStackOverflow ) ;
254+ }
255+ let target = reserve. max ( self . data . capacity ( ) . max ( 1 ) . saturating_mul ( 2 ) ) . min ( self . max_size ) ;
256+ if self . data . try_reserve ( target - self . data . len ( ) ) . is_err ( ) {
257+ return Err ( Trap :: ValueStackOverflow ) ;
258+ }
259+ Ok ( ( ) )
260+ }
261+
183262 #[ inline( always) ]
184263 pub ( crate ) fn select_many ( & mut self , count : usize , condition : bool ) {
185264 if count == 0 {
186265 return ;
187266 }
188267
189- let len = self . data . len ( ) ;
268+ let len = self . len ;
190269 let needed = count. wrapping_mul ( 2 ) ;
191270
192271 if len < needed {
@@ -196,10 +275,13 @@ impl<T: Copy + Default> Stack<T> {
196275 if !condition {
197276 let dst = len. wrapping_sub ( needed) ;
198277 let src = len. wrapping_sub ( count) ;
199- self . data . copy_within ( src..len, dst) ;
278+ match self . data . get_mut ( ..len) {
279+ Some ( live) => live. copy_within ( src..len, dst) ,
280+ None => crate :: invariant_violated ( "value stack index out of range" ) ,
281+ }
200282 }
201283
202- self . data . truncate ( len. wrapping_sub ( count) ) ;
284+ self . len = len . wrapping_sub ( count) ;
203285 }
204286}
205287
@@ -208,7 +290,7 @@ impl<'a, T: Copy + Default> IntoIterator for &'a Stack<T> {
208290 type IntoIter = core:: slice:: Iter < ' a , T > ;
209291
210292 fn into_iter ( self ) -> Self :: IntoIter {
211- self . data . iter ( )
293+ self . data [ .. self . len ] . iter ( )
212294 }
213295}
214296impl ValueStack {
@@ -327,3 +409,60 @@ impl ValueStack {
327409 }
328410 }
329411}
412+
413+ #[ cfg( test) ]
414+ mod tests {
415+ use super :: * ;
416+
417+ /// A stack of height 2 whose third slot still holds a popped value.
418+ #[ cfg( debug_assertions) ]
419+ fn popped ( ) -> Stack < Value32 > {
420+ let mut stack = Stack :: new ( StackConfig :: fixed ( 8 ) ) ;
421+ for value in [ 1 , 2 , 3 ] {
422+ stack. push_or_grow ( value) . unwrap ( ) ;
423+ }
424+ assert_eq ! ( stack. pop( ) , 3 ) ;
425+ stack
426+ }
427+
428+ #[ test]
429+ #[ cfg( debug_assertions) ]
430+ #[ should_panic]
431+ fn get_above_the_height ( ) {
432+ popped ( ) . get ( 2 ) ;
433+ }
434+
435+ #[ test]
436+ #[ cfg( debug_assertions) ]
437+ #[ should_panic]
438+ fn set_above_the_height ( ) {
439+ popped ( ) . set ( 2 , 0 ) ;
440+ }
441+
442+ /// A large operand-stack reservation is not written: only the locals and the values pushed
443+ /// become slots. A small one is written with the locals.
444+ #[ test]
445+ fn large_reservations_are_written_as_they_are_pushed ( ) {
446+ let mut stack = Stack :: < Value64 > :: new ( StackConfig :: fixed ( 1024 ) ) ;
447+ assert ! ( matches!( stack. enter_locals( 0 , 2 , 500 ) , Ok ( 0 ) ) ) ;
448+ assert_eq ! ( stack. data. len( ) , 2 ) ;
449+ for value in 0 ..500 {
450+ stack. push ( value) ;
451+ }
452+ assert_eq ! ( ( stack. len( ) , stack. data. len( ) ) , ( 502 , 502 ) ) ;
453+
454+ // Below the slots the stack already has, entering a function writes only its locals.
455+ stack. truncate_to ( 2 ) ;
456+ stack. push ( 7 ) ;
457+ assert ! ( matches!( stack. enter_locals( 0 , 2 , 10 ) , Ok ( 3 ) ) ) ;
458+ assert_eq ! ( ( stack. len( ) , stack. data. len( ) ) , ( 5 , 502 ) ) ;
459+
460+ stack. truncate_to ( 0 ) ;
461+ for value in 0 ..500 {
462+ stack. push_or_grow ( value) . unwrap ( ) ;
463+ }
464+ assert ! ( matches!( stack. enter_locals( 0 , 1 , 8 ) , Ok ( 500 ) ) ) ;
465+ assert_eq ! ( stack. data. len( ) , 509 ) ;
466+ assert ! ( matches!( stack. enter_locals( 0 , 2 , 1000 ) , Err ( Trap :: ValueStackOverflow ) ) ) ;
467+ }
468+ }
0 commit comments