bootstrap/core/build_steps/
format.rs

1//! Runs rustfmt on the repository.
2
3use std::collections::VecDeque;
4use std::path::{Path, PathBuf};
5use std::process::Command;
6use std::sync::Mutex;
7use std::sync::mpsc::SyncSender;
8
9use build_helper::git::get_git_modified_files;
10use ignore::WalkBuilder;
11
12use crate::core::builder::{Builder, Kind};
13use crate::utils::build_stamp::BuildStamp;
14use crate::utils::exec::command;
15use crate::utils::helpers::{self, t};
16
17#[must_use]
18enum RustfmtStatus {
19    InProgress,
20    Ok,
21    Failed,
22}
23
24fn rustfmt(
25    src: &Path,
26    rustfmt: &Path,
27    paths: &[PathBuf],
28    check: bool,
29) -> impl FnMut(bool) -> RustfmtStatus + use<> {
30    let mut cmd = Command::new(rustfmt);
31    // Avoid the submodule config paths from coming into play. We only allow a single global config
32    // for the workspace for now.
33    cmd.arg("--config-path").arg(src.canonicalize().unwrap());
34    cmd.arg("--edition").arg("2024");
35    cmd.arg("--unstable-features");
36    cmd.arg("--skip-children");
37    if check {
38        cmd.arg("--check");
39    }
40    cmd.args(paths);
41    let mut cmd = cmd.spawn().expect("running rustfmt");
42    // Poor man's async: return a closure that might wait for rustfmt's completion (depending on
43    // the value of the `block` argument).
44    move |block: bool| -> RustfmtStatus {
45        let status = if !block {
46            match cmd.try_wait() {
47                Ok(Some(status)) => Ok(status),
48                Ok(None) => return RustfmtStatus::InProgress,
49                Err(err) => Err(err),
50            }
51        } else {
52            cmd.wait()
53        };
54        if status.unwrap().success() { RustfmtStatus::Ok } else { RustfmtStatus::Failed }
55    }
56}
57
58fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, BuildStamp)> {
59    let stamp_file = BuildStamp::new(&build.out).with_prefix("rustfmt");
60
61    let mut cmd = command(build.config.initial_rustfmt.as_ref()?);
62    cmd.arg("--version");
63
64    let output = cmd.allow_failure().run_capture(build);
65    if output.is_failure() {
66        return None;
67    }
68    Some((output.stdout(), stamp_file))
69}
70
71/// Return whether the format cache can be reused.
72fn verify_rustfmt_version(build: &Builder<'_>) -> bool {
73    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
74        return false;
75    };
76    stamp_file.add_stamp(version).is_up_to_date()
77}
78
79/// Updates the last rustfmt version used.
80fn update_rustfmt_version(build: &Builder<'_>) {
81    let Some((version, stamp_file)) = get_rustfmt_version(build) else {
82        return;
83    };
84
85    t!(stamp_file.add_stamp(version).write());
86}
87
88/// Returns the Rust files modified between the last merge commit and what is now on the disk.
89/// Does not include removed files.
90///
91/// Returns `None` if all files should be formatted.
92fn get_modified_rs_files(build: &Builder<'_>) -> Result<Option<Vec<String>>, String> {
93    // In CI `get_git_modified_files` returns something different to normal environment.
94    // This shouldn't be called in CI anyway.
95    assert!(!build.config.is_running_on_ci);
96
97    if !verify_rustfmt_version(build) {
98        return Ok(None);
99    }
100
101    get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &["rs"]).map(Some)
102}
103
104#[derive(serde_derive::Deserialize)]
105struct RustfmtConfig {
106    ignore: Vec<String>,
107}
108
109// Prints output describing a collection of paths, with lines such as "formatted modified file
110// foo/bar/baz" or "skipped 20 untracked files".
111fn print_paths(verb: &str, adjective: Option<&str>, paths: &[String]) {
112    let len = paths.len();
113    let adjective =
114        if let Some(adjective) = adjective { format!("{adjective} ") } else { String::new() };
115    if len <= 10 {
116        for path in paths {
117            println!("fmt: {verb} {adjective}file {path}");
118        }
119    } else {
120        println!("fmt: {verb} {len} {adjective}files");
121    }
122}
123
124pub fn format(build: &Builder<'_>, check: bool, all: bool, paths: &[PathBuf]) {
125    if build.kind == Kind::Format && build.top_stage != 0 {
126        eprintln!("ERROR: `x fmt` only supports stage 0.");
127        eprintln!("HELP: Use `x run rustfmt` to run in-tree rustfmt.");
128        crate::exit!(1);
129    }
130
131    if !paths.is_empty() {
132        eprintln!(
133            "fmt error: path arguments are no longer accepted; use `--all` to format everything"
134        );
135        crate::exit!(1);
136    };
137    if build.config.dry_run() {
138        return;
139    }
140
141    // By default, we only check modified files locally to speed up runtime. Exceptions are if
142    // `--all` is specified or we are in CI. We check all files in CI to avoid bugs in
143    // `get_modified_rs_files` letting regressions slip through; we also care about CI time less
144    // since this is still very fast compared to building the compiler.
145    let all = all || build.config.is_running_on_ci;
146
147    let mut builder = ignore::types::TypesBuilder::new();
148    builder.add_defaults();
149    builder.select("rust");
150    let matcher = builder.build().unwrap();
151    let rustfmt_config = build.src.join("rustfmt.toml");
152    if !rustfmt_config.exists() {
153        eprintln!("fmt error: Not running formatting checks; rustfmt.toml does not exist.");
154        eprintln!("fmt error: This may happen in distributed tarballs.");
155        return;
156    }
157    let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config));
158    let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config));
159    let mut override_builder = ignore::overrides::OverrideBuilder::new(&build.src);
160    for ignore in rustfmt_config.ignore {
161        if ignore.starts_with('!') {
162            // A `!`-prefixed entry could be added as a whitelisted entry in `override_builder`,
163            // i.e. strip the `!` prefix. But as soon as whitelisted entries are added, an
164            // `OverrideBuilder` will only traverse those whitelisted entries, and won't traverse
165            // any files that aren't explicitly mentioned. No bueno! Maybe there's a way to combine
166            // explicit whitelisted entries and traversal of unmentioned files, but for now just
167            // forbid such entries.
168            eprintln!("fmt error: `!`-prefixed entries are not supported in rustfmt.toml, sorry");
169            crate::exit!(1);
170        } else {
171            override_builder.add(&format!("!{ignore}")).expect(&ignore);
172        }
173    }
174    let git_available =
175        helpers::git(None).allow_failure().arg("--version").run_capture(build).is_success();
176
177    let mut adjective = None;
178    if git_available {
179        let in_working_tree = helpers::git(Some(&build.src))
180            .allow_failure()
181            .arg("rev-parse")
182            .arg("--is-inside-work-tree")
183            .run_capture(build)
184            .is_success();
185        if in_working_tree {
186            let untracked_paths_output = helpers::git(Some(&build.src))
187                .arg("status")
188                .arg("--porcelain")
189                .arg("-z")
190                .arg("--untracked-files=normal")
191                .run_capture_stdout(build)
192                .stdout();
193            let untracked_paths: Vec<_> = untracked_paths_output
194                .split_terminator('\0')
195                .filter_map(
196                    |entry| entry.strip_prefix("?? "), // returns None if the prefix doesn't match
197                )
198                .map(|x| x.to_string())
199                .collect();
200            print_paths("skipped", Some("untracked"), &untracked_paths);
201
202            for untracked_path in untracked_paths {
203                // The leading `/` makes it an exact match against the
204                // repository root, rather than a glob. Without that, if you
205                // have `foo.rs` in the repository root it will also match
206                // against anything like `compiler/rustc_foo/src/foo.rs`,
207                // preventing the latter from being formatted.
208                override_builder.add(&format!("!/{untracked_path}")).expect(&untracked_path);
209            }
210            if !all {
211                adjective = Some("modified");
212                match get_modified_rs_files(build) {
213                    Ok(Some(files)) => {
214                        if files.is_empty() {
215                            println!("fmt info: No modified files detected for formatting.");
216                            return;
217                        }
218
219                        for file in files {
220                            override_builder.add(&format!("/{file}")).expect(&file);
221                        }
222                    }
223                    Ok(None) => {
224                        // NOTE: `Ok(None)` signifies that we need to format all files.
225                        // The tricky part here is that if `override_builder` isn't given any white
226                        // list files (i.e. files to be formatted, added without leading `!`), it
227                        // will instead look for *all* files. So, by doing nothing here, we are
228                        // actually making it so we format all files.
229                    }
230                    Err(err) => {
231                        eprintln!("fmt warning: Something went wrong running git commands:");
232                        eprintln!("fmt warning: {err}");
233                        eprintln!("fmt warning: Falling back to formatting all files.");
234                    }
235                }
236            }
237        } else {
238            eprintln!("fmt: warning: Not in git tree. Skipping git-aware format checks");
239        }
240    } else {
241        eprintln!("fmt: warning: Could not find usable git. Skipping git-aware format checks");
242    }
243
244    let override_ = override_builder.build().unwrap(); // `override` is a reserved keyword
245
246    let rustfmt_path = build.config.initial_rustfmt.clone().unwrap_or_else(|| {
247        eprintln!("fmt error: `x fmt` is not supported on this channel");
248        crate::exit!(1);
249    });
250    assert!(rustfmt_path.exists(), "{}", rustfmt_path.display());
251    let src = build.src.clone();
252    let (tx, rx): (SyncSender<PathBuf>, _) = std::sync::mpsc::sync_channel(128);
253    let walker = WalkBuilder::new(src.clone()).types(matcher).overrides(override_).build_parallel();
254
255    // There is a lot of blocking involved in spawning a child process and reading files to format.
256    // Spawn more processes than available concurrency to keep the CPU busy.
257    let max_processes = build.jobs() as usize * 2;
258
259    // Spawn child processes on a separate thread so we can batch entries we have received from
260    // ignore.
261    let thread = std::thread::spawn(move || {
262        let mut result = Ok(());
263
264        let mut children = VecDeque::new();
265        while let Ok(path) = rx.recv() {
266            // Try getting more paths from the channel to amortize the overhead of spawning
267            // processes.
268            let paths: Vec<_> = rx.try_iter().take(63).chain(std::iter::once(path)).collect();
269
270            let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check);
271            children.push_back(child);
272
273            // Poll completion before waiting.
274            for i in (0..children.len()).rev() {
275                match children[i](false) {
276                    RustfmtStatus::InProgress => {}
277                    RustfmtStatus::Failed => {
278                        result = Err(());
279                        children.swap_remove_back(i);
280                        break;
281                    }
282                    RustfmtStatus::Ok => {
283                        children.swap_remove_back(i);
284                        break;
285                    }
286                }
287            }
288
289            if children.len() >= max_processes {
290                // Await oldest child.
291                match children.pop_front().unwrap()(true) {
292                    RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
293                    RustfmtStatus::Failed => result = Err(()),
294                }
295            }
296        }
297
298        // Await remaining children.
299        for mut child in children {
300            match child(true) {
301                RustfmtStatus::InProgress | RustfmtStatus::Ok => {}
302                RustfmtStatus::Failed => result = Err(()),
303            }
304        }
305
306        result
307    });
308
309    let formatted_paths = Mutex::new(Vec::new());
310    let formatted_paths_ref = &formatted_paths;
311    walker.run(|| {
312        let tx = tx.clone();
313        Box::new(move |entry| {
314            let cwd = std::env::current_dir();
315            let entry = t!(entry);
316            if entry.file_type().is_some_and(|t| t.is_file()) {
317                formatted_paths_ref.lock().unwrap().push({
318                    // `into_path` produces an absolute path. Try to strip `cwd` to get a shorter
319                    // relative path.
320                    let mut path = entry.clone().into_path();
321                    if let Ok(cwd) = cwd {
322                        if let Ok(path2) = path.strip_prefix(cwd) {
323                            path = path2.to_path_buf();
324                        }
325                    }
326                    path.display().to_string()
327                });
328                t!(tx.send(entry.into_path()));
329            }
330            ignore::WalkState::Continue
331        })
332    });
333    let mut paths = formatted_paths.into_inner().unwrap();
334    paths.sort();
335    print_paths(if check { "checked" } else { "formatted" }, adjective, &paths);
336
337    drop(tx);
338
339    let result = thread.join().unwrap();
340
341    if result.is_err() {
342        crate::exit!(1);
343    }
344
345    // Update `build/.rustfmt-stamp`, allowing this code to ignore files which have not been changed
346    // since last merge.
347    //
348    // NOTE: Because of the exit above, this is only reachable if formatting / format checking
349    // succeeded. So we are not commiting the version if formatting was not good.
350    update_rustfmt_version(build);
351}
bootstrap/core/build_steps/format.rs

bootstrap/core/build_steps/
format.rs
