From a51c69e2d5711e071af48307d432ba2d97e3dedf Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Sun, 5 Nov 2017 18:25:45 +0200 Subject: [PATCH 1/2] proc_macro: process proc_macro tokens instead of libsyntax ones in the quasi-quoter. --- src/libproc_macro/lib.rs | 2 +- src/libproc_macro/quote.rs | 318 +++++++++++++++++++------------------ 2 files changed, 167 insertions(+), 153 deletions(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index cf30966fa8971..33dd6c9a94170 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -191,7 +191,7 @@ impl Default for Span { /// This is needed to implement a custom quoter. #[unstable(feature = "proc_macro", issue = "38356")] pub fn quote_span(span: Span) -> TokenStream { - TokenStream(quote::Quote::quote(&span.0)) + quote::Quote::quote(span) } macro_rules! diagnostic_method { diff --git a/src/libproc_macro/quote.rs b/src/libproc_macro/quote.rs index 8c1f6bfc11a5f..49a9d71e83c02 100644 --- a/src/libproc_macro/quote.rs +++ b/src/libproc_macro/quote.rs @@ -15,13 +15,12 @@ //! including re-exported API `libsyntax`, to build a `syntax::tokenstream::TokenStream` //! and wrap it into a `proc_macro::TokenStream`. -use syntax::ast::Ident; +use {Delimiter, Literal, Spacing, Span, Term, TokenNode, TokenStream, TokenTree}; + +use std::iter; use syntax::ext::base::{ExtCtxt, ProcMacro}; -use syntax::parse::token::{self, Token, Lit}; -use syntax::symbol::Symbol; -use syntax::tokenstream::{Delimited, TokenTree, TokenStream, TokenStreamBuilder}; -use syntax_pos::{DUMMY_SP, Span}; -use syntax_pos::hygiene::SyntaxContext; +use syntax::parse::token; +use syntax::tokenstream; pub struct Quoter; @@ -30,216 +29,231 @@ pub mod __rt { pub use syntax::parse::token; pub use syntax::symbol::Symbol; pub use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree, Delimited}; - pub use super::{ctxt, span}; + + use syntax_pos::Span; + use syntax_pos::hygiene::SyntaxContext; pub fn unquote + Clone>(tokens: &T) -> TokenStream { T::into(tokens.clone()).0 } -} -pub fn ctxt() -> SyntaxContext { - ::__internal::with_sess(|(_, mark)| SyntaxContext::empty().apply_mark(mark)) -} + pub fn ctxt() -> SyntaxContext { + ::__internal::with_sess(|(_, mark)| SyntaxContext::empty().apply_mark(mark)) + } -pub fn span() -> Span { - ::Span::default().0 + pub fn span() -> Span { + ::Span::default().0 + } } pub trait Quote { - fn quote(&self) -> TokenStream; + fn quote(self) -> TokenStream; } macro_rules! quote_tok { - (,) => { Token::Comma }; - (.) => { Token::Dot }; - (:) => { Token::Colon }; - (::) => { Token::ModSep }; - (!) => { Token::Not }; - (<) => { Token::Lt }; - (>) => { Token::Gt }; - (_) => { Token::Underscore }; - (0) => { Token::Literal(token::Lit::Integer(Symbol::intern("0")), None) }; - (&) => { Token::BinOp(token::And) }; - ($i:ident) => { Token::Ident(Ident { name: Symbol::intern(stringify!($i)), ctxt: ctxt() }) }; + (,) => { TokenNode::Op(',', Spacing::Alone) }; + (.) => { TokenNode::Op('.', Spacing::Alone) }; + (:) => { TokenNode::Op(':', Spacing::Alone) }; + (::) => { + [ + TokenNode::Op(':', Spacing::Joint), + TokenNode::Op(':', Spacing::Alone) + ].iter().cloned().collect::() + }; + (!) => { TokenNode::Op('!', Spacing::Alone) }; + (<) => { TokenNode::Op('<', Spacing::Alone) }; + (>) => { TokenNode::Op('>', Spacing::Alone) }; + (_) => { TokenNode::Op('_', Spacing::Alone) }; + (0) => { TokenNode::Literal(::Literal::integer(0)) }; + (&) => { TokenNode::Op('&', Spacing::Alone) }; + ($i:ident) => { TokenNode::Term(Term::intern(stringify!($i))) }; } macro_rules! quote_tree { - ((unquote $($t:tt)*)) => { TokenStream::from($($t)*) }; + ((unquote $($t:tt)*)) => { $($t)* }; ((quote $($t:tt)*)) => { ($($t)*).quote() }; - (($($t:tt)*)) => { delimit(token::Paren, quote!($($t)*)) }; - ([$($t:tt)*]) => { delimit(token::Bracket, quote!($($t)*)) }; - ({$($t:tt)*}) => { delimit(token::Brace, quote!($($t)*)) }; + (($($t:tt)*)) => { TokenNode::Group(Delimiter::Parenthesis, quote!($($t)*)) }; + ([$($t:tt)*]) => { TokenNode::Group(Delimiter::Bracket, quote!($($t)*)) }; + ({$($t:tt)*}) => { TokenNode::Group(Delimiter::Brace, quote!($($t)*)) }; (rt) => { quote!(::__internal::__rt) }; - ($t:tt) => { TokenStream::from(TokenTree::Token(span(), quote_tok!($t))) }; -} - -fn delimit(delim: token::DelimToken, stream: TokenStream) -> TokenStream { - TokenTree::Delimited(span(), Delimited { delim: delim, tts: stream.into() }).into() + ($t:tt) => { quote_tok!($t) }; } macro_rules! quote { () => { TokenStream::empty() }; - ($($t:tt)*) => { [ $( quote_tree!($t), )* ].iter().cloned().collect::() }; + ($($t:tt)*) => { + [ + $(TokenStream::from(quote_tree!($t)),)* + ].iter().cloned().collect::() + }; } impl ProcMacro for Quoter { - fn expand<'cx>(&self, cx: &'cx mut ExtCtxt, _: Span, stream: TokenStream) -> TokenStream { + fn expand<'cx>(&self, cx: &'cx mut ExtCtxt, + _: ::syntax_pos::Span, + stream: tokenstream::TokenStream) + -> tokenstream::TokenStream { let mut info = cx.current_expansion.mark.expn_info().unwrap(); info.callee.allow_internal_unstable = true; cx.current_expansion.mark.set_expn_info(info); - ::__internal::set_sess(cx, || quote!(::TokenStream { 0: (quote stream) })) + ::__internal::set_sess(cx, || quote!(::TokenStream { + 0: (quote TokenStream(stream)) + }).0) } } impl Quote for Option { - fn quote(&self) -> TokenStream { - match *self { - Some(ref t) => quote!(Some((quote t))), + fn quote(self) -> TokenStream { + match self { + Some(t) => quote!(Some((quote t))), None => quote!(None), } } } impl Quote for TokenStream { - fn quote(&self) -> TokenStream { - let mut builder = TokenStreamBuilder::new(); - builder.push(quote!(rt::TokenStreamBuilder::new())); - - let mut trees = self.trees(); - loop { - let (mut tree, mut is_joint) = match trees.next_as_stream() { - Some(next) => next.as_tree(), - None => return builder.add(quote!(.build())).build(), - }; - if let TokenTree::Token(_, Token::Dollar) = tree { - let (next_tree, next_is_joint) = match trees.next_as_stream() { - Some(next) => next.as_tree(), - None => panic!("unexpected trailing `$` in `quote!`"), - }; - match next_tree { - TokenTree::Token(_, Token::Ident(..)) => { - builder.push(quote!(.add(rt::unquote(&(unquote next_tree))))); - continue + fn quote(self) -> TokenStream { + let mut after_dollar = false; + let stream = iter::once(quote!(rt::TokenStreamBuilder::new())) + .chain(self.into_iter().filter_map(|tree| { + if after_dollar { + after_dollar = false; + match tree.kind { + TokenNode::Term(_) => { + return Some(quote!(.add(rt::unquote(&(unquote tree))))); + } + TokenNode::Op('$', _) => {} + _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), } - TokenTree::Token(_, Token::Dollar) => { - tree = next_tree; - is_joint = next_is_joint; - } - _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), + } else if let TokenNode::Op('$', _) = tree.kind { + after_dollar = true; + return None; } - } - builder.push(match is_joint { - true => quote!(.add((quote tree).joint())), - false => quote!(.add(rt::TokenStream::from((quote tree)))), - }); + Some(quote!(.add(rt::TokenStream::from((quote tree))))) + })) + .chain(iter::once(quote!(.build()))).collect(); + + if after_dollar { + panic!("unexpected trailing `$` in `quote!`"); } + + stream } } impl Quote for TokenTree { - fn quote(&self) -> TokenStream { - match *self { - TokenTree::Token(span, ref token) => quote! { - rt::TokenTree::Token((quote span), (quote token)) + fn quote(self) -> TokenStream { + let (op, kind) = match self.kind { + TokenNode::Op(op, kind) => (op, kind), + TokenNode::Group(delimiter, tokens) => { + return quote! { + rt::TokenTree::Delimited((quote self.span), rt::Delimited { + delim: (quote delimiter), + tts: (quote tokens).into() + }) + }; + }, + TokenNode::Term(term) => { + let variant = if term.as_str().starts_with("'") { + quote!(Lifetime) + } else { + quote!(Ident) + }; + return quote! { + rt::TokenTree::Token((quote self.span), + rt::token::(unquote variant)(rt::Ident { + name: (quote term), + ctxt: rt::ctxt() + })) + }; + } + TokenNode::Literal(lit) => { + return quote! { + rt::TokenTree::Token((quote self.span), (quote lit)) + }; + } + }; + + let token = match op { + '=' => quote!(Eq), + '<' => quote!(Lt), + '>' => quote!(Gt), + '!' => quote!(Not), + '~' => quote!(Tilde), + '+' => quote!(BinOp(rt::token::BinOpToken::Plus)), + '-' => quote!(BinOp(rt::token::BinOpToken::Minus)), + '*' => quote!(BinOp(rt::token::BinOpToken::Star)), + '/' => quote!(BinOp(rt::token::BinOpToken::Slash)), + '%' => quote!(BinOp(rt::token::BinOpToken::Percent)), + '^' => quote!(BinOp(rt::token::BinOpToken::Caret)), + '&' => quote!(BinOp(rt::token::BinOpToken::And)), + '|' => quote!(BinOp(rt::token::BinOpToken::Or)), + '@' => quote!(At), + '.' => quote!(Dot), + ',' => quote!(Comma), + ';' => quote!(Semi), + ':' => quote!(Colon), + '#' => quote!(Pound), + '$' => quote!(Dollar), + '?' => quote!(Question), + '_' => quote!(Underscore), + _ => panic!("unsupported character {}", op), + }; + + match kind { + Spacing::Alone => quote! { + rt::TokenTree::Token((quote self.span), rt::token::(unquote token)) }, - TokenTree::Delimited(span, ref delimited) => quote! { - rt::TokenTree::Delimited((quote span), (quote delimited)) + Spacing::Joint => quote! { + rt::TokenTree::Token((quote self.span), rt::token::(unquote token)).joint() }, } } } -impl Quote for Delimited { - fn quote(&self) -> TokenStream { - quote!(rt::Delimited { delim: (quote self.delim), tts: (quote self.stream()).into() }) - } -} - impl<'a> Quote for &'a str { - fn quote(&self) -> TokenStream { - TokenTree::Token(span(), Token::Literal(token::Lit::Str_(Symbol::intern(self)), None)) - .into() + fn quote(self) -> TokenStream { + TokenNode::Literal(Literal::string(self)).into() } } impl Quote for usize { - fn quote(&self) -> TokenStream { - let integer_symbol = Symbol::intern(&self.to_string()); - TokenTree::Token(DUMMY_SP, Token::Literal(token::Lit::Integer(integer_symbol), None)) - .into() - } -} - -impl Quote for Ident { - fn quote(&self) -> TokenStream { - quote!(rt::Ident { name: (quote self.name), ctxt: rt::ctxt() }) + fn quote(self) -> TokenStream { + TokenNode::Literal(Literal::integer(self as i128)).into() } } -impl Quote for Symbol { - fn quote(&self) -> TokenStream { - quote!(rt::Symbol::intern((quote &*self.as_str()))) +impl Quote for Term { + fn quote(self) -> TokenStream { + quote!(rt::Symbol::intern((quote self.as_str()))) } } impl Quote for Span { - fn quote(&self) -> TokenStream { + fn quote(self) -> TokenStream { quote!(rt::span()) } } -impl Quote for Token { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*; $($t:tt)*) => { - match *self { - $( Token::$i => quote!(rt::token::$i), )* - $( $t )* - } - } - } - - gen_match! { - Eq, Lt, Le, EqEq, Ne, Ge, Gt, AndAnd, OrOr, Not, Tilde, At, Dot, DotDot, DotDotDot, - DotDotEq, Comma, Semi, Colon, ModSep, RArrow, LArrow, FatArrow, Pound, Dollar, - Question, Underscore; - - Token::OpenDelim(delim) => quote!(rt::token::OpenDelim((quote delim))), - Token::CloseDelim(delim) => quote!(rt::token::CloseDelim((quote delim))), - Token::BinOp(tok) => quote!(rt::token::BinOp((quote tok))), - Token::BinOpEq(tok) => quote!(rt::token::BinOpEq((quote tok))), - Token::Ident(ident) => quote!(rt::token::Ident((quote ident))), - Token::Lifetime(ident) => quote!(rt::token::Lifetime((quote ident))), - Token::Literal(lit, sfx) => quote!(rt::token::Literal((quote lit), (quote sfx))), - _ => panic!("Unhandled case!"), - } - } -} - -impl Quote for token::BinOpToken { - fn quote(&self) -> TokenStream { - macro_rules! gen_match { - ($($i:ident),*) => { - match *self { - $( token::BinOpToken::$i => quote!(rt::token::BinOpToken::$i), )* - } - } - } - - gen_match!(Plus, Minus, Star, Slash, Percent, Caret, And, Or, Shl, Shr) - } -} +impl Quote for Literal { + fn quote(self) -> TokenStream { + let (lit, sfx) = match self.0 { + token::Literal(lit, sfx) => (lit, sfx.map(Term)), + _ => panic!("unsupported literal {:?}", self.0), + }; -impl Quote for Lit { - fn quote(&self) -> TokenStream { macro_rules! gen_match { ($($i:ident),*; $($raw:ident),*) => { - match *self { - $( Lit::$i(lit) => quote!(rt::token::Lit::$i((quote lit))), )* - $( Lit::$raw(lit, n) => { - quote!(::syntax::parse::token::Lit::$raw((quote lit), (quote n))) - })* + match lit { + $(token::Lit::$i(lit) => quote! { + rt::token::Literal(rt::token::Lit::$i((quote Term(lit))), + (quote sfx)) + },)* + $(token::Lit::$raw(lit, n) => quote! { + rt::token::Literal(rt::token::Lit::$raw((quote Term(lit)), (quote n)), + (quote sfx)) + },)* } } } @@ -248,16 +262,16 @@ impl Quote for Lit { } } -impl Quote for token::DelimToken { - fn quote(&self) -> TokenStream { +impl Quote for Delimiter { + fn quote(self) -> TokenStream { macro_rules! gen_match { - ($($i:ident),*) => { - match *self { - $(token::DelimToken::$i => { quote!(rt::token::DelimToken::$i) })* + ($($i:ident => $j:ident),*) => { + match self { + $(Delimiter::$i => { quote!(rt::token::DelimToken::$j) })* } } } - gen_match!(Paren, Bracket, Brace, NoDelim) + gen_match!(Parenthesis => Paren, Brace => Brace, Bracket => Bracket, None => NoDelim) } } From fbcc6733d46c1582c1a417b5909a2ff0a587f0ac Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Sun, 5 Nov 2017 20:21:05 +0200 Subject: [PATCH 2/2] proc_macro: use the proc_macro API at runtime to construct quasi-quoted TokenStream's. --- src/libproc_macro/lib.rs | 2 +- src/libproc_macro/quote.rs | 246 ++++++++++++++++++------------------- 2 files changed, 118 insertions(+), 130 deletions(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 33dd6c9a94170..8a400f3e6360e 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -728,7 +728,7 @@ impl TokenTree { #[unstable(feature = "proc_macro_internals", issue = "27812")] #[doc(hidden)] pub mod __internal { - pub use quote::{Quoter, __rt}; + pub use quote::{LiteralKind, Quoter, unquote}; use std::cell::Cell; diff --git a/src/libproc_macro/quote.rs b/src/libproc_macro/quote.rs index 49a9d71e83c02..26f88ad6bf649 100644 --- a/src/libproc_macro/quote.rs +++ b/src/libproc_macro/quote.rs @@ -11,39 +11,19 @@ //! # Quasiquoter //! This file contains the implementation internals of the quasiquoter provided by `quote!`. -//! This quasiquoter uses macros 2.0 hygiene to reliably use items from `__rt`, -//! including re-exported API `libsyntax`, to build a `syntax::tokenstream::TokenStream` -//! and wrap it into a `proc_macro::TokenStream`. +//! This quasiquoter uses macros 2.0 hygiene to reliably access +//! items from `proc_macro`, to build a `proc_macro::TokenStream`. use {Delimiter, Literal, Spacing, Span, Term, TokenNode, TokenStream, TokenTree}; -use std::iter; use syntax::ext::base::{ExtCtxt, ProcMacro}; use syntax::parse::token; use syntax::tokenstream; pub struct Quoter; -pub mod __rt { - pub use syntax::ast::Ident; - pub use syntax::parse::token; - pub use syntax::symbol::Symbol; - pub use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree, Delimited}; - - use syntax_pos::Span; - use syntax_pos::hygiene::SyntaxContext; - - pub fn unquote + Clone>(tokens: &T) -> TokenStream { - T::into(tokens.clone()).0 - } - - pub fn ctxt() -> SyntaxContext { - ::__internal::with_sess(|(_, mark)| SyntaxContext::empty().apply_mark(mark)) - } - - pub fn span() -> Span { - ::Span::default().0 - } +pub fn unquote + Clone>(tokens: &T) -> TokenStream { + T::into(tokens.clone()) } pub trait Quote { @@ -75,7 +55,6 @@ macro_rules! quote_tree { (($($t:tt)*)) => { TokenNode::Group(Delimiter::Parenthesis, quote!($($t)*)) }; ([$($t:tt)*]) => { TokenNode::Group(Delimiter::Bracket, quote!($($t)*)) }; ({$($t:tt)*}) => { TokenNode::Group(Delimiter::Brace, quote!($($t)*)) }; - (rt) => { quote!(::__internal::__rt) }; ($t:tt) => { quote_tok!($t) }; } @@ -96,9 +75,7 @@ impl ProcMacro for Quoter { let mut info = cx.current_expansion.mark.expn_info().unwrap(); info.callee.allow_internal_unstable = true; cx.current_expansion.mark.set_expn_info(info); - ::__internal::set_sess(cx, || quote!(::TokenStream { - 0: (quote TokenStream(stream)) - }).0) + ::__internal::set_sess(cx, || TokenStream(stream).quote().0) } } @@ -113,102 +90,61 @@ impl Quote for Option { impl Quote for TokenStream { fn quote(self) -> TokenStream { + if self.is_empty() { + return quote!(::TokenStream::empty()); + } let mut after_dollar = false; - let stream = iter::once(quote!(rt::TokenStreamBuilder::new())) - .chain(self.into_iter().filter_map(|tree| { - if after_dollar { - after_dollar = false; - match tree.kind { - TokenNode::Term(_) => { - return Some(quote!(.add(rt::unquote(&(unquote tree))))); - } - TokenNode::Op('$', _) => {} - _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), + let tokens = self.into_iter().filter_map(|tree| { + if after_dollar { + after_dollar = false; + match tree.kind { + TokenNode::Term(_) => { + return Some(quote!(::__internal::unquote(&(unquote tree)),)); } - } else if let TokenNode::Op('$', _) = tree.kind { - after_dollar = true; - return None; + TokenNode::Op('$', _) => {} + _ => panic!("`$` must be followed by an ident or `$` in `quote!`"), } + } else if let TokenNode::Op('$', _) = tree.kind { + after_dollar = true; + return None; + } - Some(quote!(.add(rt::TokenStream::from((quote tree))))) - })) - .chain(iter::once(quote!(.build()))).collect(); + Some(quote!(::TokenStream::from((quote tree)),)) + }).collect::(); if after_dollar { panic!("unexpected trailing `$` in `quote!`"); } - stream + quote!([(unquote tokens)].iter().cloned().collect::<::TokenStream>()) } } impl Quote for TokenTree { fn quote(self) -> TokenStream { - let (op, kind) = match self.kind { - TokenNode::Op(op, kind) => (op, kind), - TokenNode::Group(delimiter, tokens) => { - return quote! { - rt::TokenTree::Delimited((quote self.span), rt::Delimited { - delim: (quote delimiter), - tts: (quote tokens).into() - }) - }; - }, - TokenNode::Term(term) => { - let variant = if term.as_str().starts_with("'") { - quote!(Lifetime) - } else { - quote!(Ident) - }; - return quote! { - rt::TokenTree::Token((quote self.span), - rt::token::(unquote variant)(rt::Ident { - name: (quote term), - ctxt: rt::ctxt() - })) - }; - } - TokenNode::Literal(lit) => { - return quote! { - rt::TokenTree::Token((quote self.span), (quote lit)) - }; + quote!(::TokenTree { span: (quote self.span), kind: (quote self.kind) }) + } +} + +impl Quote for TokenNode { + fn quote(self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident($($arg:ident),+)),*) => { + match self { + $(TokenNode::$i($($arg),+) => quote! { + ::TokenNode::$i($((quote $arg)),+) + },)* + } } - }; - - let token = match op { - '=' => quote!(Eq), - '<' => quote!(Lt), - '>' => quote!(Gt), - '!' => quote!(Not), - '~' => quote!(Tilde), - '+' => quote!(BinOp(rt::token::BinOpToken::Plus)), - '-' => quote!(BinOp(rt::token::BinOpToken::Minus)), - '*' => quote!(BinOp(rt::token::BinOpToken::Star)), - '/' => quote!(BinOp(rt::token::BinOpToken::Slash)), - '%' => quote!(BinOp(rt::token::BinOpToken::Percent)), - '^' => quote!(BinOp(rt::token::BinOpToken::Caret)), - '&' => quote!(BinOp(rt::token::BinOpToken::And)), - '|' => quote!(BinOp(rt::token::BinOpToken::Or)), - '@' => quote!(At), - '.' => quote!(Dot), - ',' => quote!(Comma), - ';' => quote!(Semi), - ':' => quote!(Colon), - '#' => quote!(Pound), - '$' => quote!(Dollar), - '?' => quote!(Question), - '_' => quote!(Underscore), - _ => panic!("unsupported character {}", op), - }; - - match kind { - Spacing::Alone => quote! { - rt::TokenTree::Token((quote self.span), rt::token::(unquote token)) - }, - Spacing::Joint => quote! { - rt::TokenTree::Token((quote self.span), rt::token::(unquote token)).joint() - }, } + + gen_match! { Op(op, kind), Group(delim, tokens), Term(term), Literal(lit) } + } +} + +impl Quote for char { + fn quote(self) -> TokenStream { + TokenNode::Literal(Literal::character(self)).into() } } @@ -226,52 +162,104 @@ impl Quote for usize { impl Quote for Term { fn quote(self) -> TokenStream { - quote!(rt::Symbol::intern((quote self.as_str()))) + quote!(::Term::intern((quote self.as_str()))) } } impl Quote for Span { fn quote(self) -> TokenStream { - quote!(rt::span()) + quote!(::Span::default()) } } -impl Quote for Literal { - fn quote(self) -> TokenStream { - let (lit, sfx) = match self.0 { - token::Literal(lit, sfx) => (lit, sfx.map(Term)), - _ => panic!("unsupported literal {:?}", self.0), - }; +macro_rules! literals { + ($($i:ident),*; $($raw:ident),*) => { + pub enum LiteralKind { + $($i,)* + $($raw(usize),)* + } - macro_rules! gen_match { - ($($i:ident),*; $($raw:ident),*) => { - match lit { - $(token::Lit::$i(lit) => quote! { - rt::token::Literal(rt::token::Lit::$i((quote Term(lit))), - (quote sfx)) + impl LiteralKind { + pub fn with_contents_and_suffix(self, contents: Term, suffix: Option) + -> Literal { + let contents = contents.0; + let suffix = suffix.map(|t| t.0); + match self { + $(LiteralKind::$i => { + Literal(token::Literal(token::Lit::$i(contents), suffix)) + })* + $(LiteralKind::$raw(n) => { + Literal(token::Literal(token::Lit::$raw(contents, n), suffix)) + })* + } + } + } + + impl Literal { + fn kind_contents_and_suffix(self) -> (LiteralKind, Term, Option) { + let (lit, suffix) = match self.0 { + token::Literal(lit, suffix) => (lit, suffix), + _ => panic!("unsupported literal {:?}", self.0), + }; + + let (kind, contents) = match lit { + $(token::Lit::$i(contents) => (LiteralKind::$i, contents),)* + $(token::Lit::$raw(contents, n) => (LiteralKind::$raw(n), contents),)* + }; + (kind, Term(contents), suffix.map(Term)) + } + } + + impl Quote for LiteralKind { + fn quote(self) -> TokenStream { + match self { + $(LiteralKind::$i => quote! { + ::__internal::LiteralKind::$i },)* - $(token::Lit::$raw(lit, n) => quote! { - rt::token::Literal(rt::token::Lit::$raw((quote Term(lit)), (quote n)), - (quote sfx)) + $(LiteralKind::$raw(n) => quote! { + ::__internal::LiteralKind::$raw((quote n)) },)* } } } - gen_match!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw) + impl Quote for Literal { + fn quote(self) -> TokenStream { + let (kind, contents, suffix) = self.kind_contents_and_suffix(); + quote! { + (quote kind).with_contents_and_suffix((quote contents), (quote suffix)) + } + } + } } } +literals!(Byte, Char, Float, Str_, Integer, ByteStr; StrRaw, ByteStrRaw); + impl Quote for Delimiter { fn quote(self) -> TokenStream { macro_rules! gen_match { - ($($i:ident => $j:ident),*) => { + ($($i:ident),*) => { + match self { + $(Delimiter::$i => { quote!(::Delimiter::$i) })* + } + } + } + + gen_match!(Parenthesis, Brace, Bracket, None) + } +} + +impl Quote for Spacing { + fn quote(self) -> TokenStream { + macro_rules! gen_match { + ($($i:ident),*) => { match self { - $(Delimiter::$i => { quote!(rt::token::DelimToken::$j) })* + $(Spacing::$i => { quote!(::Spacing::$i) })* } } } - gen_match!(Parenthesis => Paren, Brace => Brace, Bracket => Bracket, None => NoDelim) + gen_match!(Alone, Joint) } }