rustc_ast/
token.rs

1use std::borrow::Cow;
2use std::fmt;
3
4pub use LitKind::*;
5pub use NtExprKind::*;
6pub use NtPatKind::*;
7pub use TokenKind::*;
8use rustc_macros::{Decodable, Encodable, HashStable_Generic};
9use rustc_span::edition::Edition;
10use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym};
11#[allow(clippy::useless_attribute)] // FIXME: following use of `hidden_glob_reexports` incorrectly triggers `useless_attribute` lint.
12#[allow(hidden_glob_reexports)]
13use rustc_span::{Ident, Symbol};
14
15use crate::ast;
16use crate::util::case::Case;
17
18#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
19pub enum CommentKind {
20    Line,
21    Block,
22}
23
24// This type must not implement `Hash` due to the unusual `PartialEq` impl below.
25#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)]
26pub enum InvisibleOrigin {
27    // From the expansion of a metavariable in a declarative macro.
28    MetaVar(MetaVarKind),
29
30    // Converted from `proc_macro::Delimiter` in
31    // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
32    ProcMacro,
33}
34
35impl InvisibleOrigin {
36    // Should the parser skip these invisible delimiters? Ideally this function
37    // will eventually disappear and no invisible delimiters will be skipped.
38    #[inline]
39    pub fn skip(&self) -> bool {
40        match self {
41            InvisibleOrigin::MetaVar(_) => false,
42            InvisibleOrigin::ProcMacro => true,
43        }
44    }
45}
46
47impl PartialEq for InvisibleOrigin {
48    #[inline]
49    fn eq(&self, _other: &InvisibleOrigin) -> bool {
50        // When we had AST-based nonterminals we couldn't compare them, and the
51        // old `Nonterminal` type had an `eq` that always returned false,
52        // resulting in this restriction:
53        // https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment
54        // This `eq` emulates that behaviour. We could consider lifting this
55        // restriction now but there are still cases involving invisible
56        // delimiters that make it harder than it first appears.
57        false
58    }
59}
60
61/// Annoyingly similar to `NonterminalKind`, but the slight differences are important.
62#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
63pub enum MetaVarKind {
64    Item,
65    Block,
66    Stmt,
67    Pat(NtPatKind),
68    Expr {
69        kind: NtExprKind,
70        // This field is needed for `Token::can_begin_literal_maybe_minus`.
71        can_begin_literal_maybe_minus: bool,
72        // This field is needed for `Token::can_begin_string_literal`.
73        can_begin_string_literal: bool,
74    },
75    Ty {
76        is_path: bool,
77    },
78    Ident,
79    Lifetime,
80    Literal,
81    Meta {
82        /// Will `AttrItem::meta` succeed on this, if reparsed?
83        has_meta_form: bool,
84    },
85    Path,
86    Vis,
87    TT,
88}
89
90impl fmt::Display for MetaVarKind {
91    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
92        let sym = match self {
93            MetaVarKind::Item => sym::item,
94            MetaVarKind::Block => sym::block,
95            MetaVarKind::Stmt => sym::stmt,
96            MetaVarKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
97            MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
98            MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
99            MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
100            MetaVarKind::Ty { .. } => sym::ty,
101            MetaVarKind::Ident => sym::ident,
102            MetaVarKind::Lifetime => sym::lifetime,
103            MetaVarKind::Literal => sym::literal,
104            MetaVarKind::Meta { .. } => sym::meta,
105            MetaVarKind::Path => sym::path,
106            MetaVarKind::Vis => sym::vis,
107            MetaVarKind::TT => sym::tt,
108        };
109        write!(f, "{sym}")
110    }
111}
112
113/// Describes how a sequence of token trees is delimited.
114/// Cannot use `proc_macro::Delimiter` directly because this
115/// structure should implement some additional traits.
116#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
117pub enum Delimiter {
118    /// `( ... )`
119    Parenthesis,
120    /// `{ ... }`
121    Brace,
122    /// `[ ... ]`
123    Bracket,
124    /// `∅ ... ∅`
125    /// An invisible delimiter, that may, for example, appear around tokens coming from a
126    /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
127    /// `$var * 3` where `$var` is `1 + 2`.
128    /// Invisible delimiters might not survive roundtrip of a token stream through a string.
129    Invisible(InvisibleOrigin),
130}
131
132impl Delimiter {
133    // Should the parser skip these delimiters? Only happens for certain kinds
134    // of invisible delimiters. Ideally this function will eventually disappear
135    // and no invisible delimiters will be skipped.
136    #[inline]
137    pub fn skip(&self) -> bool {
138        match self {
139            Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
140            Delimiter::Invisible(origin) => origin.skip(),
141        }
142    }
143
144    // This exists because `InvisibleOrigin`s should be compared. It is only used for assertions.
145    pub fn eq_ignoring_invisible_origin(&self, other: &Delimiter) -> bool {
146        match (self, other) {
147            (Delimiter::Parenthesis, Delimiter::Parenthesis) => true,
148            (Delimiter::Brace, Delimiter::Brace) => true,
149            (Delimiter::Bracket, Delimiter::Bracket) => true,
150            (Delimiter::Invisible(_), Delimiter::Invisible(_)) => true,
151            _ => false,
152        }
153    }
154
155    pub fn as_open_token_kind(&self) -> TokenKind {
156        match *self {
157            Delimiter::Parenthesis => OpenParen,
158            Delimiter::Brace => OpenBrace,
159            Delimiter::Bracket => OpenBracket,
160            Delimiter::Invisible(origin) => OpenInvisible(origin),
161        }
162    }
163
164    pub fn as_close_token_kind(&self) -> TokenKind {
165        match *self {
166            Delimiter::Parenthesis => CloseParen,
167            Delimiter::Brace => CloseBrace,
168            Delimiter::Bracket => CloseBracket,
169            Delimiter::Invisible(origin) => CloseInvisible(origin),
170        }
171    }
172}
173
174// Note that the suffix is *not* considered when deciding the `LitKind` in this
175// type. This means that float literals like `1f32` are classified by this type
176// as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
177// given the `Float` kind.
178#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
179pub enum LitKind {
180    Bool, // AST only, must never appear in a `Token`
181    Byte,
182    Char,
183    Integer, // e.g. `1`, `1u8`, `1f32`
184    Float,   // e.g. `1.`, `1.0`, `1e3f32`
185    Str,
186    StrRaw(u8), // raw string delimited by `n` hash symbols
187    ByteStr,
188    ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols
189    CStr,
190    CStrRaw(u8),
191    Err(ErrorGuaranteed),
192}
193
194/// A literal token.
195#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
196pub struct Lit {
197    pub kind: LitKind,
198    pub symbol: Symbol,
199    pub suffix: Option<Symbol>,
200}
201
202impl Lit {
203    pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
204        Lit { kind, symbol, suffix }
205    }
206
207    /// Returns `true` if this is semantically a float literal. This includes
208    /// ones like `1f32` that have an `Integer` kind but a float suffix.
209    pub fn is_semantic_float(&self) -> bool {
210        match self.kind {
211            LitKind::Float => true,
212            LitKind::Integer => match self.suffix {
213                Some(sym) => sym == sym::f32 || sym == sym::f64,
214                None => false,
215            },
216            _ => false,
217        }
218    }
219
220    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
221    /// `Parser::eat_token_lit` (excluding unary negation).
222    pub fn from_token(token: &Token) -> Option<Lit> {
223        match token.uninterpolate().kind {
224            Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
225            Literal(token_lit) => Some(token_lit),
226            OpenInvisible(InvisibleOrigin::MetaVar(
227                MetaVarKind::Literal | MetaVarKind::Expr { .. },
228            )) => {
229                // Unreachable with the current test suite.
230                panic!("from_token metavar");
231            }
232            _ => None,
233        }
234    }
235}
236
237impl fmt::Display for Lit {
238    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
239        let Lit { kind, symbol, suffix } = *self;
240        match kind {
241            Byte => write!(f, "b'{symbol}'")?,
242            Char => write!(f, "'{symbol}'")?,
243            Str => write!(f, "\"{symbol}\"")?,
244            StrRaw(n) => write!(
245                f,
246                "r{delim}\"{string}\"{delim}",
247                delim = "#".repeat(n as usize),
248                string = symbol
249            )?,
250            ByteStr => write!(f, "b\"{symbol}\"")?,
251            ByteStrRaw(n) => write!(
252                f,
253                "br{delim}\"{string}\"{delim}",
254                delim = "#".repeat(n as usize),
255                string = symbol
256            )?,
257            CStr => write!(f, "c\"{symbol}\"")?,
258            CStrRaw(n) => {
259                write!(f, "cr{delim}\"{symbol}\"{delim}", delim = "#".repeat(n as usize))?
260            }
261            Integer | Float | Bool | Err(_) => write!(f, "{symbol}")?,
262        }
263
264        if let Some(suffix) = suffix {
265            write!(f, "{suffix}")?;
266        }
267
268        Ok(())
269    }
270}
271
272impl LitKind {
273    /// An English article for the literal token kind.
274    pub fn article(self) -> &'static str {
275        match self {
276            Integer | Err(_) => "an",
277            _ => "a",
278        }
279    }
280
281    pub fn descr(self) -> &'static str {
282        match self {
283            Bool => "boolean",
284            Byte => "byte",
285            Char => "char",
286            Integer => "integer",
287            Float => "float",
288            Str | StrRaw(..) => "string",
289            ByteStr | ByteStrRaw(..) => "byte string",
290            CStr | CStrRaw(..) => "C string",
291            Err(_) => "error",
292        }
293    }
294
295    pub(crate) fn may_have_suffix(self) -> bool {
296        matches!(self, Integer | Float | Err(_))
297    }
298}
299
300pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool {
301    let ident_token = Token::new(Ident(name, is_raw), span);
302
303    !ident_token.is_reserved_ident()
304        || ident_token.is_path_segment_keyword()
305        || [
306            kw::Async,
307            kw::Do,
308            kw::Box,
309            kw::Break,
310            kw::Const,
311            kw::Continue,
312            kw::False,
313            kw::For,
314            kw::Gen,
315            kw::If,
316            kw::Let,
317            kw::Loop,
318            kw::Match,
319            kw::Move,
320            kw::Return,
321            kw::True,
322            kw::Try,
323            kw::Unsafe,
324            kw::While,
325            kw::Yield,
326            kw::Safe,
327            kw::Static,
328        ]
329        .contains(&name)
330}
331
332fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool {
333    let ident_token = Token::new(Ident(name, is_raw), span);
334
335    !ident_token.is_reserved_ident()
336        || ident_token.is_path_segment_keyword()
337        || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
338            .contains(&name)
339}
340
341#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic)]
342pub enum IdentIsRaw {
343    No,
344    Yes,
345}
346
347impl From<bool> for IdentIsRaw {
348    fn from(b: bool) -> Self {
349        if b { Self::Yes } else { Self::No }
350    }
351}
352
353impl From<IdentIsRaw> for bool {
354    fn from(is_raw: IdentIsRaw) -> bool {
355        matches!(is_raw, IdentIsRaw::Yes)
356    }
357}
358
359#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
360pub enum TokenKind {
361    /* Expression-operator symbols. */
362    /// `=`
363    Eq,
364    /// `<`
365    Lt,
366    /// `<=`
367    Le,
368    /// `==`
369    EqEq,
370    /// `!=`
371    Ne,
372    /// `>=`
373    Ge,
374    /// `>`
375    Gt,
376    /// `&&`
377    AndAnd,
378    /// `||`
379    OrOr,
380    /// `!`
381    Bang,
382    /// `~`
383    Tilde,
384    // `+`
385    Plus,
386    // `-`
387    Minus,
388    // `*`
389    Star,
390    // `/`
391    Slash,
392    // `%`
393    Percent,
394    // `^`
395    Caret,
396    // `&`
397    And,
398    // `|`
399    Or,
400    // `<<`
401    Shl,
402    // `>>`
403    Shr,
404    // `+=`
405    PlusEq,
406    // `-=`
407    MinusEq,
408    // `*=`
409    StarEq,
410    // `/=`
411    SlashEq,
412    // `%=`
413    PercentEq,
414    // `^=`
415    CaretEq,
416    // `&=`
417    AndEq,
418    // `|=`
419    OrEq,
420    // `<<=`
421    ShlEq,
422    // `>>=`
423    ShrEq,
424
425    /* Structural symbols */
426    /// `@`
427    At,
428    /// `.`
429    Dot,
430    /// `..`
431    DotDot,
432    /// `...`
433    DotDotDot,
434    /// `..=`
435    DotDotEq,
436    /// `,`
437    Comma,
438    /// `;`
439    Semi,
440    /// `:`
441    Colon,
442    /// `::`
443    PathSep,
444    /// `->`
445    RArrow,
446    /// `<-`
447    LArrow,
448    /// `=>`
449    FatArrow,
450    /// `#`
451    Pound,
452    /// `$`
453    Dollar,
454    /// `?`
455    Question,
456    /// Used by proc macros for representing lifetimes, not generated by lexer right now.
457    SingleQuote,
458    /// `(`
459    OpenParen,
460    /// `)`
461    CloseParen,
462    /// `{`
463    OpenBrace,
464    /// `}`
465    CloseBrace,
466    /// `[`
467    OpenBracket,
468    /// `]`
469    CloseBracket,
470    /// Invisible opening delimiter, produced by a macro.
471    OpenInvisible(InvisibleOrigin),
472    /// Invisible closing delimiter, produced by a macro.
473    CloseInvisible(InvisibleOrigin),
474
475    /* Literals */
476    Literal(Lit),
477
478    /// Identifier token.
479    /// Do not forget about `NtIdent` when you want to match on identifiers.
480    /// It's recommended to use `Token::{ident,uninterpolate}` and
481    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
482    /// identifiers in the same way.
483    Ident(Symbol, IdentIsRaw),
484    /// This identifier (and its span) is the identifier passed to the
485    /// declarative macro. The span in the surrounding `Token` is the span of
486    /// the `ident` metavariable in the macro's RHS.
487    NtIdent(Ident, IdentIsRaw),
488
489    /// Lifetime identifier token.
490    /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
491    /// It's recommended to use `Token::{ident,uninterpolate}` and
492    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
493    /// identifiers in the same way.
494    Lifetime(Symbol, IdentIsRaw),
495    /// This identifier (and its span) is the lifetime passed to the
496    /// declarative macro. The span in the surrounding `Token` is the span of
497    /// the `lifetime` metavariable in the macro's RHS.
498    NtLifetime(Ident, IdentIsRaw),
499
500    /// A doc comment token.
501    /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
502    /// similarly to symbols in string literal tokens.
503    DocComment(CommentKind, ast::AttrStyle, Symbol),
504
505    /// End Of File
506    Eof,
507}
508
509#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
510pub struct Token {
511    pub kind: TokenKind,
512    pub span: Span,
513}
514
515impl TokenKind {
516    pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
517        Literal(Lit::new(kind, symbol, suffix))
518    }
519
520    /// An approximation to proc-macro-style single-character operators used by
521    /// rustc parser. If the operator token can be broken into two tokens, the
522    /// first of which has `n` (1 or 2) chars, then this function performs that
523    /// operation, otherwise it returns `None`.
524    pub fn break_two_token_op(&self, n: u32) -> Option<(TokenKind, TokenKind)> {
525        assert!(n == 1 || n == 2);
526        Some(match (self, n) {
527            (Le, 1) => (Lt, Eq),
528            (EqEq, 1) => (Eq, Eq),
529            (Ne, 1) => (Bang, Eq),
530            (Ge, 1) => (Gt, Eq),
531            (AndAnd, 1) => (And, And),
532            (OrOr, 1) => (Or, Or),
533            (Shl, 1) => (Lt, Lt),
534            (Shr, 1) => (Gt, Gt),
535            (PlusEq, 1) => (Plus, Eq),
536            (MinusEq, 1) => (Minus, Eq),
537            (StarEq, 1) => (Star, Eq),
538            (SlashEq, 1) => (Slash, Eq),
539            (PercentEq, 1) => (Percent, Eq),
540            (CaretEq, 1) => (Caret, Eq),
541            (AndEq, 1) => (And, Eq),
542            (OrEq, 1) => (Or, Eq),
543            (ShlEq, 1) => (Lt, Le),  // `<` + `<=`
544            (ShlEq, 2) => (Shl, Eq), // `<<` + `=`
545            (ShrEq, 1) => (Gt, Ge),  // `>` + `>=`
546            (ShrEq, 2) => (Shr, Eq), // `>>` + `=`
547            (DotDot, 1) => (Dot, Dot),
548            (DotDotDot, 1) => (Dot, DotDot), // `.` + `..`
549            (DotDotDot, 2) => (DotDot, Dot), // `..` + `.`
550            (DotDotEq, 2) => (DotDot, Eq),
551            (PathSep, 1) => (Colon, Colon),
552            (RArrow, 1) => (Minus, Gt),
553            (LArrow, 1) => (Lt, Minus),
554            (FatArrow, 1) => (Eq, Gt),
555            _ => return None,
556        })
557    }
558
559    /// Returns tokens that are likely to be typed accidentally instead of the current token.
560    /// Enables better error recovery when the wrong token is found.
561    pub fn similar_tokens(&self) -> &[TokenKind] {
562        match self {
563            Comma => &[Dot, Lt, Semi],
564            Semi => &[Colon, Comma],
565            Colon => &[Semi],
566            FatArrow => &[Eq, RArrow, Ge, Gt],
567            _ => &[],
568        }
569    }
570
571    pub fn should_end_const_arg(&self) -> bool {
572        matches!(self, Gt | Ge | Shr | ShrEq)
573    }
574
575    pub fn is_delim(&self) -> bool {
576        self.open_delim().is_some() || self.close_delim().is_some()
577    }
578
579    pub fn open_delim(&self) -> Option<Delimiter> {
580        match *self {
581            OpenParen => Some(Delimiter::Parenthesis),
582            OpenBrace => Some(Delimiter::Brace),
583            OpenBracket => Some(Delimiter::Bracket),
584            OpenInvisible(origin) => Some(Delimiter::Invisible(origin)),
585            _ => None,
586        }
587    }
588
589    pub fn close_delim(&self) -> Option<Delimiter> {
590        match *self {
591            CloseParen => Some(Delimiter::Parenthesis),
592            CloseBrace => Some(Delimiter::Brace),
593            CloseBracket => Some(Delimiter::Bracket),
594            CloseInvisible(origin) => Some(Delimiter::Invisible(origin)),
595            _ => None,
596        }
597    }
598
599    pub fn is_close_delim_or_eof(&self) -> bool {
600        match self {
601            CloseParen | CloseBrace | CloseBracket | CloseInvisible(_) | Eof => true,
602            _ => false,
603        }
604    }
605}
606
607impl Token {
608    pub fn new(kind: TokenKind, span: Span) -> Self {
609        Token { kind, span }
610    }
611
612    /// Some token that will be thrown away later.
613    pub fn dummy() -> Self {
614        Token::new(TokenKind::Question, DUMMY_SP)
615    }
616
617    /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
618    pub fn from_ast_ident(ident: Ident) -> Self {
619        Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
620    }
621
622    pub fn is_range_separator(&self) -> bool {
623        [DotDot, DotDotDot, DotDotEq].contains(&self.kind)
624    }
625
626    pub fn is_punct(&self) -> bool {
627        match self.kind {
628            Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Bang | Tilde | Plus | Minus
629            | Star | Slash | Percent | Caret | And | Or | Shl | Shr | PlusEq | MinusEq | StarEq
630            | SlashEq | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | Dot | DotDot
631            | DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow
632            | FatArrow | Pound | Dollar | Question | SingleQuote => true,
633
634            OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
635            | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | DocComment(..) | Ident(..)
636            | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
637        }
638    }
639
640    pub fn is_like_plus(&self) -> bool {
641        matches!(self.kind, Plus | PlusEq)
642    }
643
644    /// Returns `true` if the token can appear at the start of an expression.
645    ///
646    /// **NB**: Take care when modifying this function, since it will change
647    /// the stable set of tokens that are allowed to match an expr nonterminal.
648    pub fn can_begin_expr(&self) -> bool {
649        match self.uninterpolate().kind {
650            Ident(name, is_raw)              =>
651                ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
652            OpenParen                         | // tuple
653            OpenBrace                         | // block
654            OpenBracket                       | // array
655            Literal(..)                       | // literal
656            Bang                              | // operator not
657            Minus                             | // unary minus
658            Star                              | // dereference
659            Or | OrOr                         | // closure
660            And                               | // reference
661            AndAnd                            | // double reference
662            // DotDotDot is no longer supported, but we need some way to display the error
663            DotDot | DotDotDot | DotDotEq     | // range notation
664            Lt | Shl                          | // associated path
665            PathSep                           | // global path
666            Lifetime(..)                      | // labeled loop
667            Pound                             => true, // expression attributes
668            OpenInvisible(InvisibleOrigin::MetaVar(
669                MetaVarKind::Block |
670                MetaVarKind::Expr { .. } |
671                MetaVarKind::Literal |
672                MetaVarKind::Path
673            )) => true,
674            _ => false,
675        }
676    }
677
678    /// Returns `true` if the token can appear at the start of a pattern.
679    ///
680    /// Shamelessly borrowed from `can_begin_expr`, only used for diagnostics right now.
681    pub fn can_begin_pattern(&self, pat_kind: NtPatKind) -> bool {
682        match &self.uninterpolate().kind {
683            // box, ref, mut, and other identifiers (can stricten)
684            Ident(..) | NtIdent(..) |
685            OpenParen |                          // tuple pattern
686            OpenBracket |                        // slice pattern
687            And |                                // reference
688            Minus |                              // negative literal
689            AndAnd |                             // double reference
690            Literal(_) |                         // literal
691            DotDot |                             // range pattern (future compat)
692            DotDotDot |                          // range pattern (future compat)
693            PathSep |                            // path
694            Lt |                                 // path (UFCS constant)
695            Shl => true,                         // path (double UFCS)
696            Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
697            OpenInvisible(InvisibleOrigin::MetaVar(
698                MetaVarKind::Expr { .. } |
699                MetaVarKind::Literal |
700                MetaVarKind::Meta { .. } |
701                MetaVarKind::Pat(_) |
702                MetaVarKind::Path |
703                MetaVarKind::Ty { .. }
704            )) => true,
705            _ => false,
706        }
707    }
708
709    /// Returns `true` if the token can appear at the start of a type.
710    pub fn can_begin_type(&self) -> bool {
711        match self.uninterpolate().kind {
712            Ident(name, is_raw) =>
713                ident_can_begin_type(name, self.span, is_raw), // type name or keyword
714            OpenParen                         | // tuple
715            OpenBracket                       | // array
716            Bang                              | // never
717            Star                              | // raw pointer
718            And                               | // reference
719            AndAnd                            | // double reference
720            Question                          | // maybe bound in trait object
721            Lifetime(..)                      | // lifetime bound in trait object
722            Lt | Shl                          | // associated path
723            PathSep => true,                    // global path
724            OpenInvisible(InvisibleOrigin::MetaVar(
725                MetaVarKind::Ty { .. } |
726                MetaVarKind::Path
727            )) => true,
728            // For anonymous structs or unions, which only appear in specific positions
729            // (type of struct fields or union fields), we don't consider them as regular types
730            _ => false,
731        }
732    }
733
734    /// Returns `true` if the token can appear at the start of a const param.
735    pub fn can_begin_const_arg(&self) -> bool {
736        match self.kind {
737            OpenBrace | Literal(..) | Minus => true,
738            Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
739            OpenInvisible(InvisibleOrigin::MetaVar(
740                MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
741            )) => true,
742            _ => false,
743        }
744    }
745
746    /// Returns `true` if the token can appear at the start of an item.
747    pub fn can_begin_item(&self) -> bool {
748        match self.kind {
749            Ident(name, _) => [
750                kw::Fn,
751                kw::Use,
752                kw::Struct,
753                kw::Enum,
754                kw::Pub,
755                kw::Trait,
756                kw::Extern,
757                kw::Impl,
758                kw::Unsafe,
759                kw::Const,
760                kw::Safe,
761                kw::Static,
762                kw::Union,
763                kw::Macro,
764                kw::Mod,
765                kw::Type,
766            ]
767            .contains(&name),
768            _ => false,
769        }
770    }
771
772    /// Returns `true` if the token is any literal.
773    pub fn is_lit(&self) -> bool {
774        matches!(self.kind, Literal(..))
775    }
776
777    /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
778    /// for example a '-42', or one of the boolean idents).
779    ///
780    /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
781    ///
782    /// Keep this in sync with `Lit::from_token` and `Parser::eat_token_lit`
783    /// (excluding unary negation).
784    pub fn can_begin_literal_maybe_minus(&self) -> bool {
785        match self.uninterpolate().kind {
786            Literal(..) | Minus => true,
787            Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
788            OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
789                MetaVarKind::Literal => true,
790                MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
791                    can_begin_literal_maybe_minus
792                }
793                _ => false,
794            },
795            _ => false,
796        }
797    }
798
799    pub fn can_begin_string_literal(&self) -> bool {
800        match self.uninterpolate().kind {
801            Literal(..) => true,
802            OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
803                MetaVarKind::Literal => true,
804                MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
805                _ => false,
806            },
807            _ => false,
808        }
809    }
810
811    /// A convenience function for matching on identifiers during parsing.
812    /// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
813    /// into the regular identifier or lifetime token it refers to,
814    /// otherwise returns the original token.
815    pub fn uninterpolate(&self) -> Cow<'_, Token> {
816        match self.kind {
817            NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
818            NtLifetime(ident, is_raw) => {
819                Cow::Owned(Token::new(Lifetime(ident.name, is_raw), ident.span))
820            }
821            _ => Cow::Borrowed(self),
822        }
823    }
824
825    /// Returns an identifier if this token is an identifier.
826    #[inline]
827    pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> {
828        // We avoid using `Token::uninterpolate` here because it's slow.
829        match self.kind {
830            Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
831            NtIdent(ident, is_raw) => Some((ident, is_raw)),
832            _ => None,
833        }
834    }
835
836    /// Returns a lifetime identifier if this token is a lifetime.
837    #[inline]
838    pub fn lifetime(&self) -> Option<(Ident, IdentIsRaw)> {
839        // We avoid using `Token::uninterpolate` here because it's slow.
840        match self.kind {
841            Lifetime(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
842            NtLifetime(ident, is_raw) => Some((ident, is_raw)),
843            _ => None,
844        }
845    }
846
847    /// Returns `true` if the token is an identifier.
848    pub fn is_ident(&self) -> bool {
849        self.ident().is_some()
850    }
851
852    /// Returns `true` if the token is a lifetime.
853    pub fn is_lifetime(&self) -> bool {
854        self.lifetime().is_some()
855    }
856
857    /// Returns `true` if the token is an identifier whose name is the given
858    /// string slice.
859    pub fn is_ident_named(&self, name: Symbol) -> bool {
860        self.ident().is_some_and(|(ident, _)| ident.name == name)
861    }
862
863    /// Is this a pre-parsed expression dropped into the token stream
864    /// (which happens while parsing the result of macro expansion)?
865    pub fn is_metavar_expr(&self) -> bool {
866        matches!(
867            self.is_metavar_seq(),
868            Some(
869                MetaVarKind::Expr { .. }
870                    | MetaVarKind::Literal
871                    | MetaVarKind::Path
872                    | MetaVarKind::Block
873            )
874        )
875    }
876
877    /// Are we at a block from a metavar (`$b:block`)?
878    pub fn is_metavar_block(&self) -> bool {
879        matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
880    }
881
882    /// Returns `true` if the token is either the `mut` or `const` keyword.
883    pub fn is_mutability(&self) -> bool {
884        self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
885    }
886
887    pub fn is_qpath_start(&self) -> bool {
888        self == &Lt || self == &Shl
889    }
890
891    pub fn is_path_start(&self) -> bool {
892        self == &PathSep
893            || self.is_qpath_start()
894            || matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
895            || self.is_path_segment_keyword()
896            || self.is_ident() && !self.is_reserved_ident()
897    }
898
899    /// Returns `true` if the token is a given keyword, `kw`.
900    pub fn is_keyword(&self, kw: Symbol) -> bool {
901        self.is_non_raw_ident_where(|id| id.name == kw)
902    }
903
904    /// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this
905    /// token is an identifier equal to `kw` ignoring the case.
906    pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
907        self.is_keyword(kw)
908            || (case == Case::Insensitive
909                && self.is_non_raw_ident_where(|id| {
910                    // Do an ASCII case-insensitive match, because all keywords are ASCII.
911                    id.name.as_str().eq_ignore_ascii_case(kw.as_str())
912                }))
913    }
914
915    pub fn is_path_segment_keyword(&self) -> bool {
916        self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
917    }
918
919    /// Returns true for reserved identifiers used internally for elided lifetimes,
920    /// unnamed method parameters, crate root module, error recovery etc.
921    pub fn is_special_ident(&self) -> bool {
922        self.is_non_raw_ident_where(Ident::is_special)
923    }
924
925    /// Returns `true` if the token is a keyword used in the language.
926    pub fn is_used_keyword(&self) -> bool {
927        self.is_non_raw_ident_where(Ident::is_used_keyword)
928    }
929
930    /// Returns `true` if the token is a keyword reserved for possible future use.
931    pub fn is_unused_keyword(&self) -> bool {
932        self.is_non_raw_ident_where(Ident::is_unused_keyword)
933    }
934
935    /// Returns `true` if the token is either a special identifier or a keyword.
936    pub fn is_reserved_ident(&self) -> bool {
937        self.is_non_raw_ident_where(Ident::is_reserved)
938    }
939
940    /// Returns `true` if the token is the identifier `true` or `false`.
941    pub fn is_bool_lit(&self) -> bool {
942        self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
943    }
944
945    pub fn is_numeric_lit(&self) -> bool {
946        matches!(
947            self.kind,
948            Literal(Lit { kind: LitKind::Integer, .. }) | Literal(Lit { kind: LitKind::Float, .. })
949        )
950    }
951
952    /// Returns `true` if the token is the integer literal.
953    pub fn is_integer_lit(&self) -> bool {
954        matches!(self.kind, Literal(Lit { kind: LitKind::Integer, .. }))
955    }
956
957    /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
958    pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
959        match self.ident() {
960            Some((id, IdentIsRaw::No)) => pred(id),
961            _ => false,
962        }
963    }
964
965    /// Is this an invisible open delimiter at the start of a token sequence
966    /// from an expanded metavar?
967    pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
968        match self.kind {
969            OpenInvisible(InvisibleOrigin::MetaVar(kind)) => Some(kind),
970            _ => None,
971        }
972    }
973
974    pub fn glue(&self, joint: &Token) -> Option<Token> {
975        let kind = match (&self.kind, &joint.kind) {
976            (Eq, Eq) => EqEq,
977            (Eq, Gt) => FatArrow,
978            (Eq, _) => return None,
979
980            (Lt, Eq) => Le,
981            (Lt, Lt) => Shl,
982            (Lt, Le) => ShlEq,
983            (Lt, Minus) => LArrow,
984            (Lt, _) => return None,
985
986            (Gt, Eq) => Ge,
987            (Gt, Gt) => Shr,
988            (Gt, Ge) => ShrEq,
989            (Gt, _) => return None,
990
991            (Bang, Eq) => Ne,
992            (Bang, _) => return None,
993
994            (Plus, Eq) => PlusEq,
995            (Plus, _) => return None,
996
997            (Minus, Eq) => MinusEq,
998            (Minus, Gt) => RArrow,
999            (Minus, _) => return None,
1000
1001            (Star, Eq) => StarEq,
1002            (Star, _) => return None,
1003
1004            (Slash, Eq) => SlashEq,
1005            (Slash, _) => return None,
1006
1007            (Percent, Eq) => PercentEq,
1008            (Percent, _) => return None,
1009
1010            (Caret, Eq) => CaretEq,
1011            (Caret, _) => return None,
1012
1013            (And, Eq) => AndEq,
1014            (And, And) => AndAnd,
1015            (And, _) => return None,
1016
1017            (Or, Eq) => OrEq,
1018            (Or, Or) => OrOr,
1019            (Or, _) => return None,
1020
1021            (Shl, Eq) => ShlEq,
1022            (Shl, _) => return None,
1023
1024            (Shr, Eq) => ShrEq,
1025            (Shr, _) => return None,
1026
1027            (Dot, Dot) => DotDot,
1028            (Dot, DotDot) => DotDotDot,
1029            (Dot, _) => return None,
1030
1031            (DotDot, Dot) => DotDotDot,
1032            (DotDot, Eq) => DotDotEq,
1033            (DotDot, _) => return None,
1034
1035            (Colon, Colon) => PathSep,
1036            (Colon, _) => return None,
1037
1038            (SingleQuote, Ident(name, is_raw)) => {
1039                Lifetime(Symbol::intern(&format!("'{name}")), *is_raw)
1040            }
1041            (SingleQuote, _) => return None,
1042
1043            (
1044                Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq
1045                | PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
1046                | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
1047                | OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
1048                | OpenInvisible(_) | CloseInvisible(_) | Literal(..) | Ident(..) | NtIdent(..)
1049                | Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof,
1050                _,
1051            ) => {
1052                return None;
1053            }
1054        };
1055
1056        Some(Token::new(kind, self.span.to(joint.span)))
1057    }
1058}
1059
1060impl PartialEq<TokenKind> for Token {
1061    #[inline]
1062    fn eq(&self, rhs: &TokenKind) -> bool {
1063        self.kind == *rhs
1064    }
1065}
1066
1067#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
1068pub enum NtPatKind {
1069    // Matches or-patterns. Was written using `pat` in edition 2021 or later.
1070    PatWithOr,
1071    // Doesn't match or-patterns.
1072    // - `inferred`: was written using `pat` in edition 2015 or 2018.
1073    // - `!inferred`: was written using `pat_param`.
1074    PatParam { inferred: bool },
1075}
1076
1077#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
1078pub enum NtExprKind {
1079    // Matches expressions using the post-edition 2024. Was written using
1080    // `expr` in edition 2024 or later.
1081    Expr,
1082    // Matches expressions using the pre-edition 2024 rules.
1083    // - `inferred`: was written using `expr` in edition 2021 or earlier.
1084    // - `!inferred`: was written using `expr_2021`.
1085    Expr2021 { inferred: bool },
1086}
1087
1088#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
1089pub enum NonterminalKind {
1090    Item,
1091    Block,
1092    Stmt,
1093    Pat(NtPatKind),
1094    Expr(NtExprKind),
1095    Ty,
1096    Ident,
1097    Lifetime,
1098    Literal,
1099    Meta,
1100    Path,
1101    Vis,
1102    TT,
1103}
1104
1105impl NonterminalKind {
1106    /// The `edition` closure is used to get the edition for the given symbol. Doing
1107    /// `span.edition()` is expensive, so we do it lazily.
1108    pub fn from_symbol(
1109        symbol: Symbol,
1110        edition: impl FnOnce() -> Edition,
1111    ) -> Option<NonterminalKind> {
1112        Some(match symbol {
1113            sym::item => NonterminalKind::Item,
1114            sym::block => NonterminalKind::Block,
1115            sym::stmt => NonterminalKind::Stmt,
1116            sym::pat => {
1117                if edition().at_least_rust_2021() {
1118                    NonterminalKind::Pat(PatWithOr)
1119                } else {
1120                    NonterminalKind::Pat(PatParam { inferred: true })
1121                }
1122            }
1123            sym::pat_param => NonterminalKind::Pat(PatParam { inferred: false }),
1124            sym::expr => {
1125                if edition().at_least_rust_2024() {
1126                    NonterminalKind::Expr(Expr)
1127                } else {
1128                    NonterminalKind::Expr(Expr2021 { inferred: true })
1129                }
1130            }
1131            sym::expr_2021 => NonterminalKind::Expr(Expr2021 { inferred: false }),
1132            sym::ty => NonterminalKind::Ty,
1133            sym::ident => NonterminalKind::Ident,
1134            sym::lifetime => NonterminalKind::Lifetime,
1135            sym::literal => NonterminalKind::Literal,
1136            sym::meta => NonterminalKind::Meta,
1137            sym::path => NonterminalKind::Path,
1138            sym::vis => NonterminalKind::Vis,
1139            sym::tt => NonterminalKind::TT,
1140            _ => return None,
1141        })
1142    }
1143
1144    fn symbol(self) -> Symbol {
1145        match self {
1146            NonterminalKind::Item => sym::item,
1147            NonterminalKind::Block => sym::block,
1148            NonterminalKind::Stmt => sym::stmt,
1149            NonterminalKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat,
1150            NonterminalKind::Pat(PatParam { inferred: false }) => sym::pat_param,
1151            NonterminalKind::Expr(Expr2021 { inferred: true } | Expr) => sym::expr,
1152            NonterminalKind::Expr(Expr2021 { inferred: false }) => sym::expr_2021,
1153            NonterminalKind::Ty => sym::ty,
1154            NonterminalKind::Ident => sym::ident,
1155            NonterminalKind::Lifetime => sym::lifetime,
1156            NonterminalKind::Literal => sym::literal,
1157            NonterminalKind::Meta => sym::meta,
1158            NonterminalKind::Path => sym::path,
1159            NonterminalKind::Vis => sym::vis,
1160            NonterminalKind::TT => sym::tt,
1161        }
1162    }
1163}
1164
1165impl fmt::Display for NonterminalKind {
1166    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1167        write!(f, "{}", self.symbol())
1168    }
1169}
1170
1171// Some types are used a lot. Make sure they don't unintentionally get bigger.
1172#[cfg(target_pointer_width = "64")]
1173mod size_asserts {
1174    use rustc_data_structures::static_assert_size;
1175
1176    use super::*;
1177    // tidy-alphabetical-start
1178    static_assert_size!(Lit, 12);
1179    static_assert_size!(LitKind, 2);
1180    static_assert_size!(Token, 24);
1181    static_assert_size!(TokenKind, 16);
1182    // tidy-alphabetical-end
1183}