rustc_parse/
lib.rs

1//! The main parser interface.
2
3// tidy-alphabetical-start
4#![allow(rustc::diagnostic_outside_of_impl)]
5#![allow(rustc::untranslatable_diagnostic)]
6#![feature(assert_matches)]
7#![feature(box_patterns)]
8#![feature(debug_closure_helpers)]
9#![feature(if_let_guard)]
10#![feature(iter_intersperse)]
11#![recursion_limit = "256"]
12// tidy-alphabetical-end
13
14use std::path::{Path, PathBuf};
15use std::str::Utf8Error;
16use std::sync::Arc;
17
18use rustc_ast as ast;
19use rustc_ast::tokenstream::{DelimSpan, TokenStream};
20use rustc_ast::{AttrItem, Attribute, MetaItemInner, token};
21use rustc_ast_pretty::pprust;
22use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize};
23use rustc_lexer::FrontmatterAllowed;
24use rustc_session::parse::ParseSess;
25use rustc_span::source_map::SourceMap;
26use rustc_span::{FileName, SourceFile, Span};
27pub use unicode_normalization::UNICODE_VERSION as UNICODE_NORMALIZATION_VERSION;
28
29pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");
30
31#[macro_use]
32pub mod parser;
33use parser::Parser;
34use rustc_ast::token::Delimiter;
35
36pub mod lexer;
37
38mod errors;
39
40rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
41
42// Unwrap the result if `Ok`, otherwise emit the diagnostics and abort.
43pub fn unwrap_or_emit_fatal<T>(expr: Result<T, Vec<Diag<'_>>>) -> T {
44    match expr {
45        Ok(expr) => expr,
46        Err(errs) => {
47            for err in errs {
48                err.emit();
49            }
50            FatalError.raise()
51        }
52    }
53}
54
55/// Creates a new parser from a source string. On failure, the errors must be consumed via
56/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are
57/// dropped.
58pub fn new_parser_from_source_str(
59    psess: &ParseSess,
60    name: FileName,
61    source: String,
62) -> Result<Parser<'_>, Vec<Diag<'_>>> {
63    let source_file = psess.source_map().new_source_file(name, source);
64    new_parser_from_source_file(psess, source_file)
65}
66
67/// Creates a new parser from a filename. On failure, the errors must be consumed via
68/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are
69/// dropped.
70///
71/// If a span is given, that is used on an error as the source of the problem.
72pub fn new_parser_from_file<'a>(
73    psess: &'a ParseSess,
74    path: &Path,
75    sp: Option<Span>,
76) -> Result<Parser<'a>, Vec<Diag<'a>>> {
77    let sm = psess.source_map();
78    let source_file = sm.load_file(path).unwrap_or_else(|e| {
79        let msg = format!("couldn't read `{}`: {}", path.display(), e);
80        let mut err = psess.dcx().struct_fatal(msg);
81        if let Ok(contents) = std::fs::read(path)
82            && let Err(utf8err) = String::from_utf8(contents.clone())
83        {
84            utf8_error(
85                sm,
86                &path.display().to_string(),
87                sp,
88                &mut err,
89                utf8err.utf8_error(),
90                &contents,
91            );
92        }
93        if let Some(sp) = sp {
94            err.span(sp);
95        }
96        err.emit();
97    });
98    new_parser_from_source_file(psess, source_file)
99}
100
101pub fn utf8_error<E: EmissionGuarantee>(
102    sm: &SourceMap,
103    path: &str,
104    sp: Option<Span>,
105    err: &mut Diag<'_, E>,
106    utf8err: Utf8Error,
107    contents: &[u8],
108) {
109    // The file exists, but it wasn't valid UTF-8.
110    let start = utf8err.valid_up_to();
111    let note = format!("invalid utf-8 at byte `{start}`");
112    let msg = if let Some(len) = utf8err.error_len() {
113        format!(
114            "byte{s} `{bytes}` {are} not valid utf-8",
115            bytes = if len == 1 {
116                format!("{:?}", contents[start])
117            } else {
118                format!("{:?}", &contents[start..start + len])
119            },
120            s = pluralize!(len),
121            are = if len == 1 { "is" } else { "are" },
122        )
123    } else {
124        note.clone()
125    };
126    let contents = String::from_utf8_lossy(contents).to_string();
127    let source = sm.new_source_file(PathBuf::from(path).into(), contents);
128    let span = Span::with_root_ctxt(
129        source.normalized_byte_pos(start as u32),
130        source.normalized_byte_pos(start as u32),
131    );
132    if span.is_dummy() {
133        err.note(note);
134    } else {
135        if sp.is_some() {
136            err.span_note(span, msg);
137        } else {
138            err.span(span);
139            err.span_label(span, msg);
140        }
141    }
142}
143
144/// Given a session and a `source_file`, return a parser. Returns any buffered errors from lexing
145/// the initial token stream.
146fn new_parser_from_source_file(
147    psess: &ParseSess,
148    source_file: Arc<SourceFile>,
149) -> Result<Parser<'_>, Vec<Diag<'_>>> {
150    let end_pos = source_file.end_position();
151    let stream = source_file_to_stream(psess, source_file, None, FrontmatterAllowed::Yes)?;
152    let mut parser = Parser::new(psess, stream, None);
153    if parser.token == token::Eof {
154        parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None);
155    }
156    Ok(parser)
157}
158
159pub fn source_str_to_stream(
160    psess: &ParseSess,
161    name: FileName,
162    source: String,
163    override_span: Option<Span>,
164) -> Result<TokenStream, Vec<Diag<'_>>> {
165    let source_file = psess.source_map().new_source_file(name, source);
166    // used mainly for `proc_macro` and the likes, not for our parsing purposes, so don't parse
167    // frontmatters as frontmatters.
168    source_file_to_stream(psess, source_file, override_span, FrontmatterAllowed::No)
169}
170
171/// Given a source file, produces a sequence of token trees. Returns any buffered errors from
172/// parsing the token stream.
173fn source_file_to_stream<'psess>(
174    psess: &'psess ParseSess,
175    source_file: Arc<SourceFile>,
176    override_span: Option<Span>,
177    frontmatter_allowed: FrontmatterAllowed,
178) -> Result<TokenStream, Vec<Diag<'psess>>> {
179    let src = source_file.src.as_ref().unwrap_or_else(|| {
180        psess.dcx().bug(format!(
181            "cannot lex `source_file` without source: {}",
182            psess.source_map().filename_for_diagnostics(&source_file.name)
183        ));
184    });
185
186    lexer::lex_token_trees(
187        psess,
188        src.as_str(),
189        source_file.start_pos,
190        override_span,
191        frontmatter_allowed,
192    )
193}
194
195/// Runs the given subparser `f` on the tokens of the given `attr`'s item.
196pub fn parse_in<'a, T>(
197    psess: &'a ParseSess,
198    tts: TokenStream,
199    name: &'static str,
200    mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
201) -> PResult<'a, T> {
202    let mut parser = Parser::new(psess, tts, Some(name));
203    let result = f(&mut parser)?;
204    if parser.token != token::Eof {
205        parser.unexpected()?;
206    }
207    Ok(result)
208}
209
210pub fn fake_token_stream_for_item(psess: &ParseSess, item: &ast::Item) -> TokenStream {
211    let source = pprust::item_to_string(item);
212    let filename = FileName::macro_expansion_source_code(&source);
213    unwrap_or_emit_fatal(source_str_to_stream(psess, filename, source, Some(item.span)))
214}
215
216pub fn fake_token_stream_for_crate(psess: &ParseSess, krate: &ast::Crate) -> TokenStream {
217    let source = pprust::crate_to_string_for_macros(krate);
218    let filename = FileName::macro_expansion_source_code(&source);
219    unwrap_or_emit_fatal(source_str_to_stream(
220        psess,
221        filename,
222        source,
223        Some(krate.spans.inner_span),
224    ))
225}
226
227pub fn parse_cfg_attr(
228    cfg_attr: &Attribute,
229    psess: &ParseSess,
230) -> Option<(MetaItemInner, Vec<(AttrItem, Span)>)> {
231    const CFG_ATTR_GRAMMAR_HELP: &str = "#[cfg_attr(condition, attribute, other_attribute, ...)]";
232    const CFG_ATTR_NOTE_REF: &str = "for more information, visit \
233        <https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute>";
234
235    match cfg_attr.get_normal_item().args {
236        ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens })
237            if !tokens.is_empty() =>
238        {
239            check_cfg_attr_bad_delim(psess, dspan, delim);
240            match parse_in(psess, tokens.clone(), "`cfg_attr` input", |p| p.parse_cfg_attr()) {
241                Ok(r) => return Some(r),
242                Err(e) => {
243                    e.with_help(format!("the valid syntax is `{CFG_ATTR_GRAMMAR_HELP}`"))
244                        .with_note(CFG_ATTR_NOTE_REF)
245                        .emit();
246                }
247            }
248        }
249        _ => {
250            psess.dcx().emit_err(errors::MalformedCfgAttr {
251                span: cfg_attr.span,
252                sugg: CFG_ATTR_GRAMMAR_HELP,
253            });
254        }
255    }
256    None
257}
258
259fn check_cfg_attr_bad_delim(psess: &ParseSess, span: DelimSpan, delim: Delimiter) {
260    if let Delimiter::Parenthesis = delim {
261        return;
262    }
263    psess.dcx().emit_err(errors::CfgAttrBadDelim {
264        span: span.entire(),
265        sugg: errors::MetaBadDelimSugg { open: span.open, close: span.close },
266    });
267}
rustc_parse/lib.rs

rustc_parse/
lib.rs
