rustc_lexer/lib.rs
1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use LiteralKind::*;
34use TokenKind::*;
35use cursor::EOF_CHAR;
36pub use cursor::{Cursor, FrontmatterAllowed};
37use unicode_properties::UnicodeEmoji;
38pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
39
40/// Parsed token.
41/// It doesn't contain information about data that has been parsed,
42/// only the type of the token and its size.
43#[derive(Debug)]
44pub struct Token {
45 pub kind: TokenKind,
46 pub len: u32,
47}
48
49impl Token {
50 fn new(kind: TokenKind, len: u32) -> Token {
51 Token { kind, len }
52 }
53}
54
55/// Enum representing common lexeme types.
56#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57pub enum TokenKind {
58 /// A line comment, e.g. `// comment`.
59 LineComment {
60 doc_style: Option<DocStyle>,
61 },
62
63 /// A block comment, e.g. `/* block comment */`.
64 ///
65 /// Block comments can be recursive, so a sequence like `/* /* */`
66 /// will not be considered terminated and will result in a parsing error.
67 BlockComment {
68 doc_style: Option<DocStyle>,
69 terminated: bool,
70 },
71
72 /// Any whitespace character sequence.
73 Whitespace,
74
75 Frontmatter {
76 has_invalid_preceding_whitespace: bool,
77 invalid_infostring: bool,
78 },
79
80 /// An identifier or keyword, e.g. `ident` or `continue`.
81 Ident,
82
83 /// An identifier that is invalid because it contains emoji.
84 InvalidIdent,
85
86 /// A raw identifier, e.g. "r#ident".
87 RawIdent,
88
89 /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
90 /// literal prefixes that contain emoji, which are considered "invalid".
91 ///
92 /// Note that only the
93 /// prefix (`foo`) is included in the token, not the separator (which is
94 /// lexed as its own distinct token). In Rust 2021 and later, reserved
95 /// prefixes are reported as errors; in earlier editions, they result in a
96 /// (allowed by default) lint, and are treated as regular identifier
97 /// tokens.
98 UnknownPrefix,
99
100 /// An unknown prefix in a lifetime, like `'foo#`.
101 ///
102 /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
103 /// and not the separator.
104 UnknownPrefixLifetime,
105
106 /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
107 /// several tokens: `'r` and `#` and `foo`.
108 RawLifetime,
109
110 /// Guarded string literal prefix: `#"` or `##`.
111 ///
112 /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
113 /// Split into the component tokens on older editions.
114 GuardedStrPrefix,
115
116 /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
117 /// suffix, but may be present here on string and float literals. Users of
118 /// this type will need to check for and reject that case.
119 ///
120 /// See [LiteralKind] for more details.
121 Literal {
122 kind: LiteralKind,
123 suffix_start: u32,
124 },
125
126 /// A lifetime, e.g. `'a`.
127 Lifetime {
128 starts_with_number: bool,
129 },
130
131 /// `;`
132 Semi,
133 /// `,`
134 Comma,
135 /// `.`
136 Dot,
137 /// `(`
138 OpenParen,
139 /// `)`
140 CloseParen,
141 /// `{`
142 OpenBrace,
143 /// `}`
144 CloseBrace,
145 /// `[`
146 OpenBracket,
147 /// `]`
148 CloseBracket,
149 /// `@`
150 At,
151 /// `#`
152 Pound,
153 /// `~`
154 Tilde,
155 /// `?`
156 Question,
157 /// `:`
158 Colon,
159 /// `$`
160 Dollar,
161 /// `=`
162 Eq,
163 /// `!`
164 Bang,
165 /// `<`
166 Lt,
167 /// `>`
168 Gt,
169 /// `-`
170 Minus,
171 /// `&`
172 And,
173 /// `|`
174 Or,
175 /// `+`
176 Plus,
177 /// `*`
178 Star,
179 /// `/`
180 Slash,
181 /// `^`
182 Caret,
183 /// `%`
184 Percent,
185
186 /// Unknown token, not expected by the lexer, e.g. "№"
187 Unknown,
188
189 /// End of input.
190 Eof,
191}
192
193#[derive(Clone, Copy, Debug, PartialEq, Eq)]
194pub enum DocStyle {
195 Outer,
196 Inner,
197}
198
199/// Enum representing the literal types supported by the lexer.
200///
201/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
202/// this type. This means that float literals like `1f32` are classified by this
203/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
204/// `rustc_ast::ast::LitKind`).
205#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
206pub enum LiteralKind {
207 /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
208 Int { base: Base, empty_int: bool },
209 /// `12.34f32`, `1e3`, but not `1f32`.
210 Float { base: Base, empty_exponent: bool },
211 /// `'a'`, `'\\'`, `'''`, `';`
212 Char { terminated: bool },
213 /// `b'a'`, `b'\\'`, `b'''`, `b';`
214 Byte { terminated: bool },
215 /// `"abc"`, `"abc`
216 Str { terminated: bool },
217 /// `b"abc"`, `b"abc`
218 ByteStr { terminated: bool },
219 /// `c"abc"`, `c"abc`
220 CStr { terminated: bool },
221 /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
222 /// an invalid literal.
223 RawStr { n_hashes: Option<u8> },
224 /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
225 /// indicates an invalid literal.
226 RawByteStr { n_hashes: Option<u8> },
227 /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
228 RawCStr { n_hashes: Option<u8> },
229}
230
231/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
232///
233/// Can capture fewer closing hashes than starting hashes,
234/// for more efficient lexing and better backwards diagnostics.
235#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
236pub struct GuardedStr {
237 pub n_hashes: u32,
238 pub terminated: bool,
239 pub token_len: u32,
240}
241
242#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
243pub enum RawStrError {
244 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
245 InvalidStarter { bad_char: char },
246 /// The string was not terminated, e.g. `r###"abcde"##`.
247 /// `possible_terminator_offset` is the number of characters after `r` or
248 /// `br` where they may have intended to terminate it.
249 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
250 /// More than 255 `#`s exist.
251 TooManyDelimiters { found: u32 },
252}
253
254/// Base of numeric literal encoding according to its prefix.
255#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
256pub enum Base {
257 /// Literal starts with "0b".
258 Binary = 2,
259 /// Literal starts with "0o".
260 Octal = 8,
261 /// Literal doesn't contain a prefix.
262 Decimal = 10,
263 /// Literal starts with "0x".
264 Hexadecimal = 16,
265}
266
267/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
268/// but shebang isn't a part of rust syntax.
269pub fn strip_shebang(input: &str) -> Option<usize> {
270 // Shebang must start with `#!` literally, without any preceding whitespace.
271 // For simplicity we consider any line starting with `#!` a shebang,
272 // regardless of restrictions put on shebangs by specific platforms.
273 if let Some(input_tail) = input.strip_prefix("#!") {
274 // Ok, this is a shebang but if the next non-whitespace token is `[`,
275 // then it may be valid Rust code, so consider it Rust code.
276 let next_non_whitespace_token =
277 tokenize(input_tail, FrontmatterAllowed::No).map(|tok| tok.kind).find(|tok| {
278 !matches!(
279 tok,
280 TokenKind::Whitespace
281 | TokenKind::LineComment { doc_style: None }
282 | TokenKind::BlockComment { doc_style: None, .. }
283 )
284 });
285 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
286 // No other choice than to consider this a shebang.
287 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
288 }
289 }
290 None
291}
292
293/// Validates a raw string literal. Used for getting more information about a
294/// problem with a `RawStr`/`RawByteStr` with a `None` field.
295#[inline]
296pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
297 debug_assert!(!input.is_empty());
298 let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
299 // Move past the leading `r` or `br`.
300 for _ in 0..prefix_len {
301 cursor.bump().unwrap();
302 }
303 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
304}
305
306/// Creates an iterator that produces tokens from the input string.
307///
308/// When parsing a full Rust document,
309/// first [`strip_shebang`] and then allow frontmatters with [`FrontmatterAllowed::Yes`].
310///
311/// When tokenizing a slice of a document, be sure to disallow frontmatters with [`FrontmatterAllowed::No`]
312pub fn tokenize(
313 input: &str,
314 frontmatter_allowed: FrontmatterAllowed,
315) -> impl Iterator<Item = Token> {
316 let mut cursor = Cursor::new(input, frontmatter_allowed);
317 std::iter::from_fn(move || {
318 let token = cursor.advance_token();
319 if token.kind != TokenKind::Eof { Some(token) } else { None }
320 })
321}
322
323/// True if `c` is considered a whitespace according to Rust language definition.
324/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
325/// for definitions of these classes.
326pub fn is_whitespace(c: char) -> bool {
327 // This is Pattern_White_Space.
328 //
329 // Note that this set is stable (ie, it doesn't change with different
330 // Unicode versions), so it's ok to just hard-code the values.
331
332 matches!(
333 c,
334 // End-of-line characters
335 | '\u{000A}' // line feed (\n)
336 | '\u{000B}' // vertical tab
337 | '\u{000C}' // form feed
338 | '\u{000D}' // carriage return (\r)
339 | '\u{0085}' // next line (from latin1)
340 | '\u{2028}' // LINE SEPARATOR
341 | '\u{2029}' // PARAGRAPH SEPARATOR
342
343 // `Default_Ignorable_Code_Point` characters
344 | '\u{200E}' // LEFT-TO-RIGHT MARK
345 | '\u{200F}' // RIGHT-TO-LEFT MARK
346
347 // Horizontal space characters
348 | '\u{0009}' // tab (\t)
349 | '\u{0020}' // space
350 )
351}
352
353/// True if `c` is considered horizontal whitespace according to Rust language definition.
354pub fn is_horizontal_whitespace(c: char) -> bool {
355 // This is Pattern_White_Space.
356 //
357 // Note that this set is stable (ie, it doesn't change with different
358 // Unicode versions), so it's ok to just hard-code the values.
359
360 matches!(
361 c,
362 // Horizontal space characters
363 '\u{0009}' // tab (\t)
364 | '\u{0020}' // space
365 )
366}
367
368/// True if `c` is valid as a first character of an identifier.
369/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
370/// a formal definition of valid identifier name.
371pub fn is_id_start(c: char) -> bool {
372 // This is XID_Start OR '_' (which formally is not a XID_Start).
373 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
374}
375
376/// True if `c` is valid as a non-first character of an identifier.
377/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
378/// a formal definition of valid identifier name.
379pub fn is_id_continue(c: char) -> bool {
380 unicode_xid::UnicodeXID::is_xid_continue(c)
381}
382
383/// The passed string is lexically an identifier.
384pub fn is_ident(string: &str) -> bool {
385 let mut chars = string.chars();
386 if let Some(start) = chars.next() {
387 is_id_start(start) && chars.all(is_id_continue)
388 } else {
389 false
390 }
391}
392
393impl Cursor<'_> {
394 /// Parses a token from the input string.
395 pub fn advance_token(&mut self) -> Token {
396 let Some(first_char) = self.bump() else {
397 return Token::new(TokenKind::Eof, 0);
398 };
399
400 let token_kind = match first_char {
401 c if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
402 && is_whitespace(c) =>
403 {
404 let mut last = first_char;
405 while is_whitespace(self.first()) {
406 let Some(c) = self.bump() else {
407 break;
408 };
409 last = c;
410 }
411 // invalid frontmatter opening as whitespace preceding it isn't newline.
412 // combine the whitespace and the frontmatter to a single token as we shall
413 // error later.
414 if last != '\n' && self.as_str().starts_with("---") {
415 self.bump();
416 self.frontmatter(true)
417 } else {
418 Whitespace
419 }
420 }
421 '-' if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
422 && self.as_str().starts_with("--") =>
423 {
424 // happy path
425 self.frontmatter(false)
426 }
427 // Slash, comment or block comment.
428 '/' => match self.first() {
429 '/' => self.line_comment(),
430 '*' => self.block_comment(),
431 _ => Slash,
432 },
433
434 // Whitespace sequence.
435 c if is_whitespace(c) => self.whitespace(),
436
437 // Raw identifier, raw string literal or identifier.
438 'r' => match (self.first(), self.second()) {
439 ('#', c1) if is_id_start(c1) => self.raw_ident(),
440 ('#', _) | ('"', _) => {
441 let res = self.raw_double_quoted_string(1);
442 let suffix_start = self.pos_within_token();
443 if res.is_ok() {
444 self.eat_literal_suffix();
445 }
446 let kind = RawStr { n_hashes: res.ok() };
447 Literal { kind, suffix_start }
448 }
449 _ => self.ident_or_unknown_prefix(),
450 },
451
452 // Byte literal, byte string literal, raw byte string literal or identifier.
453 'b' => self.c_or_byte_string(
454 |terminated| ByteStr { terminated },
455 |n_hashes| RawByteStr { n_hashes },
456 Some(|terminated| Byte { terminated }),
457 ),
458
459 // c-string literal, raw c-string literal or identifier.
460 'c' => self.c_or_byte_string(
461 |terminated| CStr { terminated },
462 |n_hashes| RawCStr { n_hashes },
463 None,
464 ),
465
466 // Identifier (this should be checked after other variant that can
467 // start as identifier).
468 c if is_id_start(c) => self.ident_or_unknown_prefix(),
469
470 // Numeric literal.
471 c @ '0'..='9' => {
472 let literal_kind = self.number(c);
473 let suffix_start = self.pos_within_token();
474 self.eat_literal_suffix();
475 TokenKind::Literal { kind: literal_kind, suffix_start }
476 }
477
478 // Guarded string literal prefix: `#"` or `##`
479 '#' if matches!(self.first(), '"' | '#') => {
480 self.bump();
481 TokenKind::GuardedStrPrefix
482 }
483
484 // One-symbol tokens.
485 ';' => Semi,
486 ',' => Comma,
487 '.' => Dot,
488 '(' => OpenParen,
489 ')' => CloseParen,
490 '{' => OpenBrace,
491 '}' => CloseBrace,
492 '[' => OpenBracket,
493 ']' => CloseBracket,
494 '@' => At,
495 '#' => Pound,
496 '~' => Tilde,
497 '?' => Question,
498 ':' => Colon,
499 '$' => Dollar,
500 '=' => Eq,
501 '!' => Bang,
502 '<' => Lt,
503 '>' => Gt,
504 '-' => Minus,
505 '&' => And,
506 '|' => Or,
507 '+' => Plus,
508 '*' => Star,
509 '^' => Caret,
510 '%' => Percent,
511
512 // Lifetime or character literal.
513 '\'' => self.lifetime_or_char(),
514
515 // String literal.
516 '"' => {
517 let terminated = self.double_quoted_string();
518 let suffix_start = self.pos_within_token();
519 if terminated {
520 self.eat_literal_suffix();
521 }
522 let kind = Str { terminated };
523 Literal { kind, suffix_start }
524 }
525 // Identifier starting with an emoji. Only lexed for graceful error recovery.
526 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
527 _ => Unknown,
528 };
529 if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
530 && !matches!(token_kind, Whitespace)
531 {
532 // stop allowing frontmatters after first non-whitespace token
533 self.frontmatter_allowed = FrontmatterAllowed::No;
534 }
535 let res = Token::new(token_kind, self.pos_within_token());
536 self.reset_pos_within_token();
537 res
538 }
539
540 /// Given that one `-` was eaten, eat the rest of the frontmatter.
541 fn frontmatter(&mut self, has_invalid_preceding_whitespace: bool) -> TokenKind {
542 debug_assert_eq!('-', self.prev());
543
544 let pos = self.pos_within_token();
545 self.eat_while(|c| c == '-');
546
547 // one `-` is eaten by the caller.
548 let length_opening = self.pos_within_token() - pos + 1;
549
550 // must be ensured by the caller
551 debug_assert!(length_opening >= 3);
552
553 // whitespace between the opening and the infostring.
554 self.eat_while(|ch| ch != '\n' && is_horizontal_whitespace(ch));
555
556 // copied from `eat_identifier`, but allows `-` and `.` in infostring to allow something like
557 // `---Cargo.toml` as a valid opener
558 if is_id_start(self.first()) {
559 self.bump();
560 self.eat_while(|c| is_id_continue(c) || c == '-' || c == '.');
561 }
562
563 self.eat_while(|ch| ch != '\n' && is_horizontal_whitespace(ch));
564 let invalid_infostring = self.first() != '\n';
565
566 let mut found = false;
567 let nl_fence_pattern = format!("\n{:-<1$}", "", length_opening as usize);
568 if let Some(closing) = self.as_str().find(&nl_fence_pattern) {
569 // candidate found
570 self.bump_bytes(closing + nl_fence_pattern.len());
571 // in case like
572 // ---cargo
573 // --- blahblah
574 // or
575 // ---cargo
576 // ----
577 // combine those stuff into this frontmatter token such that it gets detected later.
578 self.eat_until(b'\n');
579 found = true;
580 }
581
582 if !found {
583 // recovery strategy: a closing statement might have preceding whitespace/newline
584 // but not have enough dashes to properly close. In this case, we eat until there,
585 // and report a mismatch in the parser.
586 let mut rest = self.as_str();
587 // We can look for a shorter closing (starting with four dashes but closing with three)
588 // and other indications that Rust has started and the infostring has ended.
589 let mut potential_closing = rest
590 .find("\n---")
591 // n.b. only in the case where there are dashes, we move the index to the line where
592 // the dashes start as we eat to include that line. For other cases those are Rust code
593 // and not included in the frontmatter.
594 .map(|x| x + 1)
595 .or_else(|| rest.find("\nuse "))
596 .or_else(|| rest.find("\n//!"))
597 .or_else(|| rest.find("\n#!["));
598
599 if potential_closing.is_none() {
600 // a less fortunate recovery if all else fails which finds any dashes preceded by whitespace
601 // on a standalone line. Might be wrong.
602 while let Some(closing) = rest.find("---") {
603 let preceding_chars_start = rest[..closing].rfind("\n").map_or(0, |i| i + 1);
604 if rest[preceding_chars_start..closing].chars().all(is_horizontal_whitespace) {
605 // candidate found
606 potential_closing = Some(closing);
607 break;
608 } else {
609 rest = &rest[closing + 3..];
610 }
611 }
612 }
613
614 if let Some(potential_closing) = potential_closing {
615 // bump to the potential closing, and eat everything on that line.
616 self.bump_bytes(potential_closing);
617 self.eat_until(b'\n');
618 } else {
619 // eat everything. this will get reported as an unclosed frontmatter.
620 self.eat_while(|_| true);
621 }
622 }
623
624 Frontmatter { has_invalid_preceding_whitespace, invalid_infostring }
625 }
626
627 fn line_comment(&mut self) -> TokenKind {
628 debug_assert!(self.prev() == '/' && self.first() == '/');
629 self.bump();
630
631 let doc_style = match self.first() {
632 // `//!` is an inner line doc comment.
633 '!' => Some(DocStyle::Inner),
634 // `////` (more than 3 slashes) is not considered a doc comment.
635 '/' if self.second() != '/' => Some(DocStyle::Outer),
636 _ => None,
637 };
638
639 self.eat_until(b'\n');
640 LineComment { doc_style }
641 }
642
643 fn block_comment(&mut self) -> TokenKind {
644 debug_assert!(self.prev() == '/' && self.first() == '*');
645 self.bump();
646
647 let doc_style = match self.first() {
648 // `/*!` is an inner block doc comment.
649 '!' => Some(DocStyle::Inner),
650 // `/***` (more than 2 stars) is not considered a doc comment.
651 // `/**/` is not considered a doc comment.
652 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
653 _ => None,
654 };
655
656 let mut depth = 1usize;
657 while let Some(c) = self.bump() {
658 match c {
659 '/' if self.first() == '*' => {
660 self.bump();
661 depth += 1;
662 }
663 '*' if self.first() == '/' => {
664 self.bump();
665 depth -= 1;
666 if depth == 0 {
667 // This block comment is closed, so for a construction like "/* */ */"
668 // there will be a successfully parsed block comment "/* */"
669 // and " */" will be processed separately.
670 break;
671 }
672 }
673 _ => (),
674 }
675 }
676
677 BlockComment { doc_style, terminated: depth == 0 }
678 }
679
680 fn whitespace(&mut self) -> TokenKind {
681 debug_assert!(is_whitespace(self.prev()));
682 self.eat_while(is_whitespace);
683 Whitespace
684 }
685
686 fn raw_ident(&mut self) -> TokenKind {
687 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
688 // Eat "#" symbol.
689 self.bump();
690 // Eat the identifier part of RawIdent.
691 self.eat_identifier();
692 RawIdent
693 }
694
695 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
696 debug_assert!(is_id_start(self.prev()));
697 // Start is already eaten, eat the rest of identifier.
698 self.eat_while(is_id_continue);
699 // Known prefixes must have been handled earlier. So if
700 // we see a prefix here, it is definitely an unknown prefix.
701 match self.first() {
702 '#' | '"' | '\'' => UnknownPrefix,
703 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
704 _ => Ident,
705 }
706 }
707
708 fn invalid_ident(&mut self) -> TokenKind {
709 // Start is already eaten, eat the rest of identifier.
710 self.eat_while(|c| {
711 const ZERO_WIDTH_JOINER: char = '\u{200d}';
712 is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
713 });
714 // An invalid identifier followed by '#' or '"' or '\'' could be
715 // interpreted as an invalid literal prefix. We don't bother doing that
716 // because the treatment of invalid identifiers and invalid prefixes
717 // would be the same.
718 InvalidIdent
719 }
720
721 fn c_or_byte_string(
722 &mut self,
723 mk_kind: fn(bool) -> LiteralKind,
724 mk_kind_raw: fn(Option<u8>) -> LiteralKind,
725 single_quoted: Option<fn(bool) -> LiteralKind>,
726 ) -> TokenKind {
727 match (self.first(), self.second(), single_quoted) {
728 ('\'', _, Some(single_quoted)) => {
729 self.bump();
730 let terminated = self.single_quoted_string();
731 let suffix_start = self.pos_within_token();
732 if terminated {
733 self.eat_literal_suffix();
734 }
735 let kind = single_quoted(terminated);
736 Literal { kind, suffix_start }
737 }
738 ('"', _, _) => {
739 self.bump();
740 let terminated = self.double_quoted_string();
741 let suffix_start = self.pos_within_token();
742 if terminated {
743 self.eat_literal_suffix();
744 }
745 let kind = mk_kind(terminated);
746 Literal { kind, suffix_start }
747 }
748 ('r', '"', _) | ('r', '#', _) => {
749 self.bump();
750 let res = self.raw_double_quoted_string(2);
751 let suffix_start = self.pos_within_token();
752 if res.is_ok() {
753 self.eat_literal_suffix();
754 }
755 let kind = mk_kind_raw(res.ok());
756 Literal { kind, suffix_start }
757 }
758 _ => self.ident_or_unknown_prefix(),
759 }
760 }
761
762 fn number(&mut self, first_digit: char) -> LiteralKind {
763 debug_assert!('0' <= self.prev() && self.prev() <= '9');
764 let mut base = Base::Decimal;
765 if first_digit == '0' {
766 // Attempt to parse encoding base.
767 match self.first() {
768 'b' => {
769 base = Base::Binary;
770 self.bump();
771 if !self.eat_decimal_digits() {
772 return Int { base, empty_int: true };
773 }
774 }
775 'o' => {
776 base = Base::Octal;
777 self.bump();
778 if !self.eat_decimal_digits() {
779 return Int { base, empty_int: true };
780 }
781 }
782 'x' => {
783 base = Base::Hexadecimal;
784 self.bump();
785 if !self.eat_hexadecimal_digits() {
786 return Int { base, empty_int: true };
787 }
788 }
789 // Not a base prefix; consume additional digits.
790 '0'..='9' | '_' => {
791 self.eat_decimal_digits();
792 }
793
794 // Also not a base prefix; nothing more to do here.
795 '.' | 'e' | 'E' => {}
796
797 // Just a 0.
798 _ => return Int { base, empty_int: false },
799 }
800 } else {
801 // No base prefix, parse number in the usual way.
802 self.eat_decimal_digits();
803 }
804
805 match self.first() {
806 // Don't be greedy if this is actually an
807 // integer literal followed by field/method access or a range pattern
808 // (`0..2` and `12.foo()`)
809 '.' if self.second() != '.' && !is_id_start(self.second()) => {
810 // might have stuff after the ., and if it does, it needs to start
811 // with a number
812 self.bump();
813 let mut empty_exponent = false;
814 if self.first().is_ascii_digit() {
815 self.eat_decimal_digits();
816 match self.first() {
817 'e' | 'E' => {
818 self.bump();
819 empty_exponent = !self.eat_float_exponent();
820 }
821 _ => (),
822 }
823 }
824 Float { base, empty_exponent }
825 }
826 'e' | 'E' => {
827 self.bump();
828 let empty_exponent = !self.eat_float_exponent();
829 Float { base, empty_exponent }
830 }
831 _ => Int { base, empty_int: false },
832 }
833 }
834
835 fn lifetime_or_char(&mut self) -> TokenKind {
836 debug_assert!(self.prev() == '\'');
837
838 let can_be_a_lifetime = if self.second() == '\'' {
839 // It's surely not a lifetime.
840 false
841 } else {
842 // If the first symbol is valid for identifier, it can be a lifetime.
843 // Also check if it's a number for a better error reporting (so '0 will
844 // be reported as invalid lifetime and not as unterminated char literal).
845 is_id_start(self.first()) || self.first().is_ascii_digit()
846 };
847
848 if !can_be_a_lifetime {
849 let terminated = self.single_quoted_string();
850 let suffix_start = self.pos_within_token();
851 if terminated {
852 self.eat_literal_suffix();
853 }
854 let kind = Char { terminated };
855 return Literal { kind, suffix_start };
856 }
857
858 if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
859 // Eat "r" and `#`, and identifier start characters.
860 self.bump();
861 self.bump();
862 self.bump();
863 self.eat_while(is_id_continue);
864 return RawLifetime;
865 }
866
867 // Either a lifetime or a character literal with
868 // length greater than 1.
869 let starts_with_number = self.first().is_ascii_digit();
870
871 // Skip the literal contents.
872 // First symbol can be a number (which isn't a valid identifier start),
873 // so skip it without any checks.
874 self.bump();
875 self.eat_while(is_id_continue);
876
877 match self.first() {
878 // Check if after skipping literal contents we've met a closing
879 // single quote (which means that user attempted to create a
880 // string with single quotes).
881 '\'' => {
882 self.bump();
883 let kind = Char { terminated: true };
884 Literal { kind, suffix_start: self.pos_within_token() }
885 }
886 '#' if !starts_with_number => UnknownPrefixLifetime,
887 _ => Lifetime { starts_with_number },
888 }
889 }
890
891 fn single_quoted_string(&mut self) -> bool {
892 debug_assert!(self.prev() == '\'');
893 // Check if it's a one-symbol literal.
894 if self.second() == '\'' && self.first() != '\\' {
895 self.bump();
896 self.bump();
897 return true;
898 }
899
900 // Literal has more than one symbol.
901
902 // Parse until either quotes are terminated or error is detected.
903 loop {
904 match self.first() {
905 // Quotes are terminated, finish parsing.
906 '\'' => {
907 self.bump();
908 return true;
909 }
910 // Probably beginning of the comment, which we don't want to include
911 // to the error report.
912 '/' => break,
913 // Newline without following '\'' means unclosed quote, stop parsing.
914 '\n' if self.second() != '\'' => break,
915 // End of file, stop parsing.
916 EOF_CHAR if self.is_eof() => break,
917 // Escaped slash is considered one character, so bump twice.
918 '\\' => {
919 self.bump();
920 self.bump();
921 }
922 // Skip the character.
923 _ => {
924 self.bump();
925 }
926 }
927 }
928 // String was not terminated.
929 false
930 }
931
932 /// Eats double-quoted string and returns true
933 /// if string is terminated.
934 fn double_quoted_string(&mut self) -> bool {
935 debug_assert!(self.prev() == '"');
936 while let Some(c) = self.bump() {
937 match c {
938 '"' => {
939 return true;
940 }
941 '\\' if self.first() == '\\' || self.first() == '"' => {
942 // Bump again to skip escaped character.
943 self.bump();
944 }
945 _ => (),
946 }
947 }
948 // End of file reached.
949 false
950 }
951
952 /// Attempt to lex for a guarded string literal.
953 ///
954 /// Used by `rustc_parse::lexer` to lex for guarded strings
955 /// conditionally based on edition.
956 ///
957 /// Note: this will not reset the `Cursor` when a
958 /// guarded string is not found. It is the caller's
959 /// responsibility to do so.
960 pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
961 debug_assert!(self.prev() != '#');
962
963 let mut n_start_hashes: u32 = 0;
964 while self.first() == '#' {
965 n_start_hashes += 1;
966 self.bump();
967 }
968
969 if self.first() != '"' {
970 return None;
971 }
972 self.bump();
973 debug_assert!(self.prev() == '"');
974
975 // Lex the string itself as a normal string literal
976 // so we can recover that for older editions later.
977 let terminated = self.double_quoted_string();
978 if !terminated {
979 let token_len = self.pos_within_token();
980 self.reset_pos_within_token();
981
982 return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
983 }
984
985 // Consume closing '#' symbols.
986 // Note that this will not consume extra trailing `#` characters:
987 // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
988 // followed by a `#` token.
989 let mut n_end_hashes = 0;
990 while self.first() == '#' && n_end_hashes < n_start_hashes {
991 n_end_hashes += 1;
992 self.bump();
993 }
994
995 // Reserved syntax, always an error, so it doesn't matter if
996 // `n_start_hashes != n_end_hashes`.
997
998 self.eat_literal_suffix();
999
1000 let token_len = self.pos_within_token();
1001 self.reset_pos_within_token();
1002
1003 Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
1004 }
1005
1006 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
1007 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
1008 // Wrap the actual function to handle the error with too many hashes.
1009 // This way, it eats the whole raw string.
1010 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
1011 // Only up to 255 `#`s are allowed in raw strings
1012 match u8::try_from(n_hashes) {
1013 Ok(num) => Ok(num),
1014 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
1015 }
1016 }
1017
1018 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
1019 debug_assert!(self.prev() == 'r');
1020 let start_pos = self.pos_within_token();
1021 let mut possible_terminator_offset = None;
1022 let mut max_hashes = 0;
1023
1024 // Count opening '#' symbols.
1025 let mut eaten = 0;
1026 while self.first() == '#' {
1027 eaten += 1;
1028 self.bump();
1029 }
1030 let n_start_hashes = eaten;
1031
1032 // Check that string is started.
1033 match self.bump() {
1034 Some('"') => (),
1035 c => {
1036 let c = c.unwrap_or(EOF_CHAR);
1037 return Err(RawStrError::InvalidStarter { bad_char: c });
1038 }
1039 }
1040
1041 // Skip the string contents and on each '#' character met, check if this is
1042 // a raw string termination.
1043 loop {
1044 self.eat_until(b'"');
1045
1046 if self.is_eof() {
1047 return Err(RawStrError::NoTerminator {
1048 expected: n_start_hashes,
1049 found: max_hashes,
1050 possible_terminator_offset,
1051 });
1052 }
1053
1054 // Eat closing double quote.
1055 self.bump();
1056
1057 // Check that amount of closing '#' symbols
1058 // is equal to the amount of opening ones.
1059 // Note that this will not consume extra trailing `#` characters:
1060 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
1061 // followed by a `#` token.
1062 let mut n_end_hashes = 0;
1063 while self.first() == '#' && n_end_hashes < n_start_hashes {
1064 n_end_hashes += 1;
1065 self.bump();
1066 }
1067
1068 if n_end_hashes == n_start_hashes {
1069 return Ok(n_start_hashes);
1070 } else if n_end_hashes > max_hashes {
1071 // Keep track of possible terminators to give a hint about
1072 // where there might be a missing terminator
1073 possible_terminator_offset =
1074 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
1075 max_hashes = n_end_hashes;
1076 }
1077 }
1078 }
1079
1080 fn eat_decimal_digits(&mut self) -> bool {
1081 let mut has_digits = false;
1082 loop {
1083 match self.first() {
1084 '_' => {
1085 self.bump();
1086 }
1087 '0'..='9' => {
1088 has_digits = true;
1089 self.bump();
1090 }
1091 _ => break,
1092 }
1093 }
1094 has_digits
1095 }
1096
1097 fn eat_hexadecimal_digits(&mut self) -> bool {
1098 let mut has_digits = false;
1099 loop {
1100 match self.first() {
1101 '_' => {
1102 self.bump();
1103 }
1104 '0'..='9' | 'a'..='f' | 'A'..='F' => {
1105 has_digits = true;
1106 self.bump();
1107 }
1108 _ => break,
1109 }
1110 }
1111 has_digits
1112 }
1113
1114 /// Eats the float exponent. Returns true if at least one digit was met,
1115 /// and returns false otherwise.
1116 fn eat_float_exponent(&mut self) -> bool {
1117 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
1118 if self.first() == '-' || self.first() == '+' {
1119 self.bump();
1120 }
1121 self.eat_decimal_digits()
1122 }
1123
1124 // Eats the suffix of the literal, e.g. "u8".
1125 fn eat_literal_suffix(&mut self) {
1126 self.eat_identifier();
1127 }
1128
1129 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
1130 // identifier.
1131 fn eat_identifier(&mut self) {
1132 if !is_id_start(self.first()) {
1133 return;
1134 }
1135 self.bump();
1136
1137 self.eat_while(is_id_continue);
1138 }
1139}