Skip to content

Commit 6e849e8

Browse files
nnethercotennethercote
committed
Improve token stream pretty-printing.
As well as nicer output, this fixes several FIXMEs in `tests/ui/macros/stringify.rs`, where the current output is sub-optimal.
1 parent de286f4 commit 6e849e8

File tree

61 files changed

+432
-339
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+432
-339
lines changed

‎compiler/rustc_ast_pretty/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
#![deny(rustc::diagnostic_outside_of_impl)]
66
#![feature(associated_type_bounds)]
77
#![feature(box_patterns)]
8+
#![feature(if_let_guard)]
9+
#![feature(let_chains)]
810
#![feature(with_negative_coherence)]
911
#![recursion_limit = "256"]
1012

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 129 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -146,47 +146,146 @@ pub fn print_crate<'a>(
146146
s.s.eof()
147147
}
148148

149-
/// Should two consecutive tokens be printed with a space between them?
149+
fn is_punct(tt: &TokenTree) -> bool {
150+
matches!(tt, TokenTree::Token(tok, _) if tok.is_punct())
151+
}
152+
153+
/// Should two consecutive token trees be printed with a space between them?
154+
///
155+
/// NOTE: should always be false if both token trees are punctuation, so that
156+
/// any old proc macro that parses pretty-printed code won't glue together
157+
/// tokens that shouldn't be glued.
150158
///
151159
/// Note: some old proc macros parse pretty-printed output, so changes here can
152160
/// break old code. For example:
153161
/// - #63896: `#[allow(unused,` must be printed rather than `#[allow(unused ,`
154162
/// - #73345: `#[allow(unused)] must be printed rather than `# [allow(unused)]
155163
///
156-
fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
164+
fn space_between(prev: Option<&TokenTree>, tt1: &TokenTree, tt2: &TokenTree) -> bool {
157165
use token::*;
158166
use Delimiter::*;
159167
use TokenTree::Delimited as Del;
160168
use TokenTree::Token as Tok;
161169

162-
// Each match arm has one or more examples in comments. The default is to
163-
// insert space between adjacent tokens, except for the cases listed in
164-
// this match.
170+
// Each match arm has one or more examples in comments.
165171
match (tt1, tt2) {
166172
// No space after line doc comments.
167173
(Tok(Token { kind: DocComment(CommentKind::Line, ..), .. }, _), _) => false,
168174

169-
// `.` + ANYTHING: `x.y`, `tup.0`
170-
// `$` + ANYTHING: `$e`
171-
(Tok(Token { kind: Dot | Dollar, .. }, _), _) => false,
172-
173-
// ANYTHING + `,`: `foo,`
174-
// ANYTHING + `.`: `x.y`, `tup.0`
175-
// ANYTHING + `!`: `foo! { ... }`
176-
//
177-
// FIXME: Incorrect cases:
178-
// - Logical not: `x =! y`, `if! x { f(); }`
179-
// - Never type: `Fn() ->!`
180-
(_, Tok(Token { kind: Comma | Dot | Not, .. }, _)) => false,
175+
// `.` + NON-PUNCT: `x.y`, `tup.0`
176+
// `$` + NON-PUNCT: `$e`
177+
(Tok(Token { kind: Dot | Dollar, .. }, _), tt2) if !is_punct(tt2) => false,
178+
179+
// NON-PUNCT + `,`: `foo,`
180+
// NON-PUNCT + `;`: `x = 3;`, `[T; 3]`
181+
// NON-PUNCT + `.`: `x.y`, `tup.0`
182+
// NON-PUNCT + `:`: `'a: loop { ... }`, `x: u8`, `where T: U`,
183+
// `<Self as T>::x`, `Trait<'a>: Sized`, `X<Y<Z>>: Send`,
184+
// `let (a, b): (u32, u32);`
185+
(tt1, Tok(Token { kind: Comma | Semi | Dot | Colon, .. }, _)) if !is_punct(tt1) => false,
186+
187+
// ANYTHING-BUT-`,`|`:`|`mut`|`<` + `[`: `<expr>[1]`, `vec![]`, `#[attr]`,
188+
// `#![attr]`, but not `data: [T; 0]`, `f(a, [])`, `&mut [T]`,
189+
// `NonNull< [T] >`
190+
(Tok(Token { kind: Comma | Colon | Lt, .. }, _), Del(_, Bracket, _)) => true,
191+
(Tok(Token { kind: Ident(sym, is_raw), .. }, _), Del(_, Bracket, _))
192+
if *sym == kw::Mut && !is_raw =>
193+
{
194+
true
195+
}
196+
(Tok(_, _), Del(_, Bracket, _)) => false,
197+
198+
// IDENT|`fn`|`Self`|`pub` + `(`: `f(3)`, `fn(x: u8)`, `Self()`, `pub(crate)`,
199+
// but `let (a, b) = (1, 2)` needs a space after the `let`
200+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Del(_, Parenthesis, _))
201+
if !Ident::new(*sym, *span).is_reserved()
202+
|| *sym == kw::Fn
203+
|| *sym == kw::SelfUpper
204+
|| *sym == kw::Pub
205+
|| *is_raw =>
206+
{
207+
false
208+
}
209+
210+
// IDENT|`self`|`Self`|`$crate`|`crate`|`super` + `::`: `x::y`,
211+
// `Self::a`, `$crate::x`, `crate::x`, `super::x`, but
212+
// `if ::a::b() { ... }` needs a space after the `if`.
213+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: ModSep, .. }, _))
214+
if !Ident::new(*sym, *span).is_reserved()
215+
|| *sym == kw::SelfLower
216+
|| *sym == kw::SelfUpper
217+
|| *sym == kw::DollarCrate
218+
|| *sym == kw::Crate
219+
|| *sym == kw::Super
220+
|| *is_raw =>
221+
{
222+
false
223+
}
224+
225+
// `::` + IDENT: `foo::bar`
226+
// `::` + `{`: `use a::{b, c}`
227+
(
228+
Tok(Token { kind: ModSep, .. }, _),
229+
Tok(Token { kind: Ident(..), .. }, _) | Del(_, Brace, _),
230+
) => false,
231+
232+
// `impl` + `<`: `impl<T> Foo<T> { ... }`
233+
// `for` + `<`: `for<'a> fn()`
234+
(Tok(Token { kind: Ident(sym, is_raw), .. }, _), Tok(Token { kind: Lt, .. }, _))
235+
if (*sym == kw::Impl || *sym == kw::For) && !is_raw =>
236+
{
237+
false
238+
}
239+
240+
// `fn` + IDENT + `<`: `fn f<T>(t: T) { ... }`
241+
(Tok(Token { kind: Ident(..), .. }, _), Tok(Token { kind: Lt, .. }, _))
242+
if let Some(prev) = prev
243+
&& let Tok(Token { kind: Ident(sym, is_raw), .. }, _) = prev
244+
&& *sym == kw::Fn
245+
&& !is_raw =>
246+
{
247+
false
248+
}
249+
250+
// `>` + `(`: `f::<u8>()`
251+
// `>>` + `(`: `collect::<Vec<_>>()`
252+
(Tok(Token { kind: Gt | BinOp(Shr), .. }, _), Del(_, Parenthesis, _)) => false,
253+
254+
// IDENT + `!`: `println!()`, but `if !x { ... }` needs a space after the `if`
255+
(Tok(Token { kind: Ident(sym, is_raw), span }, _), Tok(Token { kind: Not, .. }, _))
256+
if !Ident::new(*sym, *span).is_reserved() || *is_raw =>
257+
{
258+
false
259+
}
260+
261+
// ANYTHING-BUT-`macro_rules` + `!` + NON-PUNCT-OR-BRACE: `foo!()`, `vec![]`,
262+
// `if !cond { ... }`, but not `macro_rules! m { ... }`
263+
(Tok(Token { kind: Not, .. }, _), tt2) if is_punct(tt2) => true,
264+
(Tok(Token { kind: Not, .. }, _), Del(_, Brace, _)) => true,
265+
(Tok(Token { kind: Not, .. }, _), _) =>
266+
if let Some(prev) = prev
267+
&& let Tok(Token { kind: Ident(sym, is_raw), .. }, _) = prev
268+
&& *sym == sym::macro_rules
269+
&& !is_raw
270+
{
271+
true
272+
} else {
273+
false
274+
}
181275

182-
// IDENT + `(`: `f(3)`
183-
//
184-
// FIXME: Incorrect cases:
185-
// - Let: `let(a, b) = (1, 2)`
186-
(Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false,
276+
// `~` + `const`: `impl ~const Clone`
277+
(Tok(Token { kind: Tilde, .. }, _), Tok(Token { kind: Ident(sym, is_raw), .. }, _))
278+
if *sym == kw::Const && !is_raw =>
279+
{
280+
false
281+
}
187282

188-
// `#` + `[`: `#[attr]`
189-
(Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false,
283+
// `?` + `Sized`: `dyn ?Sized`
284+
(Tok(Token { kind: Question, .. }, _), Tok(Token { kind: Ident(sym, is_raw), .. }, _))
285+
if *sym == sym::Sized && !is_raw =>
286+
{
287+
false
288+
}
190289

191290
_ => true,
192291
}
@@ -583,14 +682,19 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
583682
}
584683

585684
fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
685+
let mut prev = None;
586686
let mut iter = tts.trees().peekable();
587687
while let Some(tt) = iter.next() {
588688
self.print_tt(tt, convert_dollar_crate);
589689
if let Some(next) = iter.peek() {
590-
if space_between(tt, next) {
690+
if space_between(prev, tt, next) {
591691
self.space();
692+
} else {
693+
// There must be a space between two punctuation tokens.
694+
assert!(!is_punct(tt) || !is_punct(next));
592695
}
593696
}
697+
prev = Some(tt);
594698
}
595699
}
596700

‎compiler/rustc_span/src/symbol.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ symbols! {
302302
Saturating,
303303
Send,
304304
SeqCst,
305+
Sized,
305306
SliceIndex,
306307
SliceIter,
307308
Some,

‎tests/pretty/ast-stmt-expr-attr.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,11 @@ fn syntax() {
113113
let _ = #[attr] continue;
114114
let _ = #[attr] return;
115115
let _ = #[attr] foo!();
116-
let _ = #[attr] foo!(#! [attr]);
116+
let _ = #[attr] foo!(# ![attr]);
117117
let _ = #[attr] foo![];
118-
let _ = #[attr] foo![#! [attr]];
118+
let _ = #[attr] foo![# ![attr]];
119119
let _ = #[attr] foo! {};
120-
let _ = #[attr] foo! { #! [attr] };
120+
let _ = #[attr] foo! { # ![attr] };
121121
let _ = #[attr] Foo { bar: baz };
122122
let _ = #[attr] Foo { ..foo };
123123
let _ = #[attr] Foo { bar: baz, ..foo };

‎tests/pretty/cast-lt.pp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@
88
// pretty-mode:expanded
99
// pp-exact:cast-lt.pp
1010

11-
macro_rules! negative { ($e : expr) => { $e < 0 } }
11+
macro_rules! negative { ($e: expr) => { $e < 0 } }
1212

1313
fn main() { (1 as i32) < 0; }

‎tests/pretty/delimited-token-groups.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,23 @@
22

33
#![feature(rustc_attrs)]
44

5-
macro_rules! mac { ($($tt : tt) *) => () }
5+
macro_rules! mac { ($($tt: tt) *) => () }
66

77
mac! {
8-
struct S { field1 : u8, field2 : u16, } impl Clone for S
8+
struct S { field1: u8, field2: u16, } impl Clone for S
99
{
1010
fn clone() -> S
1111
{
12-
panic! () ;
12+
panic!();
1313

1414
}
1515
}
1616
}
1717

1818
mac! {
1919
a(aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
20-
aaaaaaaa aaaaaaaa) a
21-
[aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
20+
aaaaaaaa aaaaaaaa)
21+
a[aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa
2222
aaaaaaaa aaaaaaaa] a
2323
{
2424
aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa aaaaaaaa

‎tests/pretty/macro.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22

33
#![feature(decl_macro)]
44

5-
pub(crate) macro mac { ($arg : expr) => { $arg + $arg } }
5+
pub(crate) macro mac { ($arg: expr) => { $arg + $arg } }
66

77
fn main() {}

‎tests/pretty/macro_rules.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
// pp-exact
22

3-
macro_rules! brace { () => {} ; }
3+
macro_rules! brace { () => {}; }
44

5-
macro_rules! bracket[() => {} ;];
5+
macro_rules! bracket[() => {};];
66

7-
macro_rules! paren(() => {} ;);
7+
macro_rules! paren(() => {};);
88

99
macro_rules! matcher_brackets {
10-
(paren) => {} ; (bracket) => {} ; (brace) => {} ;
10+
(paren) => {}; (bracket) => {}; (brace) => {};
1111
}
1212

1313
macro_rules! all_fragments {
14-
($b : block, $e : expr, $i : ident, $it : item, $l : lifetime, $lit :
15-
literal, $m : meta, $p : pat, $pth : path, $s : stmt, $tt : tt, $ty : ty,
16-
$vis : vis) => {} ;
14+
($b: block, $e: expr, $i: ident, $it: item, $l: lifetime, $lit: literal,
15+
$m: meta, $p: pat, $pth: path, $s: stmt, $tt: tt, $ty: ty, $vis: vis) =>
16+
{};
1717
}
1818

1919
fn main() {}

‎tests/pretty/offset_of.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
// pp-exact
22
#![feature(offset_of)]
33

4-
fn main() { std::mem::offset_of!(std :: ops :: Range < usize >, end); }
4+
fn main() { std::mem::offset_of!(std::ops::Range < usize > , end); }

‎tests/pretty/stmt_expr_attributes.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ fn _8() {
113113
}
114114

115115
fn _9() {
116-
macro_rules! stmt_mac { () => { let _ = () ; } }
116+
macro_rules! stmt_mac { () => { let _ = (); } }
117117

118118
#[rustc_dummy]
119119
stmt_mac!();

0 commit comments

Comments
 (0)