11use std:: cmp:: { Eq , PartialEq } ;
2+ use std:: convert:: TryFrom ;
23use std:: fmt:: { self , Display , Formatter } ;
34use std:: vec:: IntoIter ;
45
5- use itertools:: Itertools ;
66use proc_macro:: { Delimiter , Span , TokenStream , TokenTree } ;
7- use proc_macro2:: Span as Span2 ;
7+ use proc_macro2:: { Span as Span2 , TokenStream as TokenStream2 } ;
8+ use syn;
89
910#[ derive( Clone , Copy , Debug ) ]
1011pub ( crate ) struct Pos {
@@ -32,19 +33,21 @@ impl Pos {
3233 }
3334}
3435
35- fn span_pos ( span : & Span ) -> ( Pos , Pos ) {
36+ fn span_pos ( span : & Span ) -> Result < ( Pos , Pos ) , TokenStream2 > {
3637 let span2: Span2 = ( * span) . into ( ) ;
3738 let start = span2. start ( ) ;
3839 let end = span2. end ( ) ;
3940
4041 // Rust 1.88 stabilized Span APIs, so this branch must be unreachable
4142 if start. line == 0 || end. line == 0 {
42- proc_macro_error2:: abort_call_site!(
43- "cannot retrieve span location information; mlua requires nightly Rust or stable >= 1.88"
44- ) ;
43+ return Err ( syn:: Error :: new (
44+ Span2 :: call_site ( ) ,
45+ "cannot retrieve span location information; mlua requires nightly Rust or stable >= 1.88" ,
46+ )
47+ . to_compile_error ( ) ) ;
4548 }
4649
47- ( Pos :: new ( start. line , start. column ) , Pos :: new ( end. line , end. column ) )
50+ Ok ( ( Pos :: new ( start. line , start. column ) , Pos :: new ( end. line , end. column ) ) )
4851}
4952
5053/// Attribute of token.
@@ -74,33 +77,32 @@ impl PartialEq for Token {
7477impl Eq for Token { }
7578
7679impl Token {
77- fn new ( tree : TokenTree ) -> Self {
78- let ( start, end) = span_pos ( & tree. span ( ) ) ;
80+ fn new ( tree : TokenTree ) -> Result < Self , TokenStream2 > {
81+ let ( start, end) = span_pos ( & tree. span ( ) ) ? ;
7982 let source = tree. span ( ) . source_text ( ) . unwrap_or_else ( || tree. to_string ( ) ) ;
80- Self {
83+ Ok ( Self {
8184 source,
8285 start,
8386 end,
8487 tree,
8588 attr : TokenAttr :: None ,
86- }
89+ } )
8790 }
8891
89- fn new_delim ( source : String , tree : TokenTree , open : bool ) -> Self {
90- let ( start, end) = span_pos ( & tree. span ( ) ) ;
92+ fn new_delim ( source : String , tree : TokenTree , open : bool ) -> Result < Self , TokenStream2 > {
93+ let ( start, end) = span_pos ( & tree. span ( ) ) ? ;
9194 let ( start, end) = if open {
9295 ( start, start. right ( ) )
9396 } else {
9497 ( end. left ( ) , end)
9598 } ;
96-
97- Self {
99+ Ok ( Self {
98100 source,
99101 tree,
100102 start,
101103 end,
102104 attr : TokenAttr :: None ,
103- }
105+ } )
104106 }
105107
106108 pub ( crate ) fn tree ( & self ) -> & TokenTree {
@@ -133,27 +135,33 @@ impl Token {
133135pub ( crate ) struct Tokens ( pub ( crate ) Vec < Token > ) ;
134136
135137impl Tokens {
136- pub ( crate ) fn retokenize ( tt : TokenStream ) -> Tokens {
137- Tokens (
138- tt. into_iter ( )
139- . flat_map ( Tokens :: from)
140- . batching ( |iter| {
141- // Find variable tokens: `$` + `ident` => `$ident`
142- let t = iter. next ( ) ?;
143- if t. is ( "$" ) {
144- if let Some ( next) = iter. next ( )
145- && matches ! ( next. tree, TokenTree :: Ident ( _) )
146- {
147- Some ( next. attr ( TokenAttr :: Cap ) )
148- } else {
149- proc_macro_error2:: abort!( t. tree. span( ) , "`$` must be followed by an identifier" ) ;
150- }
151- } else {
152- Some ( t)
153- }
154- } )
155- . collect ( ) ,
156- )
138+ pub ( crate ) fn retokenize ( tt : TokenStream ) -> Result < Tokens , TokenStream2 > {
139+ let mut flat = Vec :: new ( ) ;
140+ for tree in tt. into_iter ( ) {
141+ flat. extend ( Tokens :: try_from ( tree) ?. 0 ) ;
142+ }
143+
144+ let mut tokens = Vec :: new ( ) ;
145+ let mut iter = flat. into_iter ( ) ;
146+ while let Some ( t) = iter. next ( ) {
147+ // Find variable tokens: `$` + `ident` => `$ident`
148+ if t. is ( "$" ) {
149+ if let Some ( next) = iter. next ( )
150+ && matches ! ( next. tree, TokenTree :: Ident ( _) )
151+ {
152+ tokens. push ( next. attr ( TokenAttr :: Cap ) ) ;
153+ } else {
154+ return Err ( syn:: Error :: new (
155+ t. tree . span ( ) . into ( ) ,
156+ "`$` must be followed by an identifier" ,
157+ )
158+ . to_compile_error ( ) ) ;
159+ }
160+ } else {
161+ tokens. push ( t) ;
162+ }
163+ }
164+ Ok ( Tokens ( tokens) )
157165 }
158166}
159167
@@ -166,8 +174,10 @@ impl IntoIterator for Tokens {
166174 }
167175}
168176
169- impl From < TokenTree > for Tokens {
170- fn from ( tt : TokenTree ) -> Self {
177+ impl TryFrom < TokenTree > for Tokens {
178+ type Error = TokenStream2 ;
179+
180+ fn try_from ( tt : TokenTree ) -> Result < Self , TokenStream2 > {
171181 let tts = match tt. clone ( ) {
172182 TokenTree :: Group ( g) => {
173183 let ( b, e) = match g. delimiter ( ) {
@@ -178,15 +188,16 @@ impl From<TokenTree> for Tokens {
178188 } ;
179189 let ( b, e) = ( b. into ( ) , e. into ( ) ) ;
180190
181- vec ! [ Token :: new_delim( b, tt. clone( ) , true ) ]
182- . into_iter ( )
183- . chain ( g. stream ( ) . into_iter ( ) . flat_map ( Tokens :: from) )
184- . chain ( vec ! [ Token :: new_delim( e, tt, false ) ] )
185- . collect ( )
191+ let mut result = vec ! [ Token :: new_delim( b, tt. clone( ) , true ) ?] ;
192+ for inner in g. stream ( ) . into_iter ( ) {
193+ result. extend ( Tokens :: try_from ( inner) ?. 0 ) ;
194+ }
195+ result. push ( Token :: new_delim ( e, tt, false ) ?) ;
196+ result
186197 }
187- _ => vec ! [ Token :: new( tt) ] ,
198+ _ => vec ! [ Token :: new( tt) ? ] ,
188199 } ;
189- Tokens ( tts)
200+ Ok ( Tokens ( tts) )
190201 }
191202}
192203
0 commit comments