build_helper/git.rs
1use std::path::Path;
2use std::process::{Command, Stdio};
3
4use crate::ci::CiEnv;
5
6#[derive(Debug)]
7pub struct GitConfig<'a> {
8 pub nightly_branch: &'a str,
9 pub git_merge_commit_email: &'a str,
10}
11
12/// Runs a command and returns the output
13pub fn output_result(cmd: &mut Command) -> Result<String, String> {
14 let output = match cmd.stderr(Stdio::inherit()).output() {
15 Ok(status) => status,
16 Err(e) => return Err(format!("failed to run command: {:?}: {}", cmd, e)),
17 };
18 if !output.status.success() {
19 return Err(format!(
20 "command did not execute successfully: {:?}\n\
21 expected success, got: {}\n{}",
22 cmd,
23 output.status,
24 String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))?
25 ));
26 }
27 String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))
28}
29
30/// Represents the result of checking whether a set of paths
31/// have been modified locally or not.
32#[derive(PartialEq, Debug, Clone)]
33pub enum PathFreshness {
34 /// Artifacts should be downloaded from this upstream commit,
35 /// there are no local modifications.
36 LastModifiedUpstream { upstream: String },
37 /// There are local modifications to a certain set of paths.
38 /// "Local" essentially means "not-upstream" here.
39 /// `upstream` is the latest upstream merge commit that made modifications to the
40 /// set of paths.
41 HasLocalModifications { upstream: String },
42 /// No upstream commit was found.
43 /// This should not happen in most reasonable circumstances, but one never knows.
44 MissingUpstream,
45}
46
47/// This function figures out if a set of paths was last modified upstream or
48/// if there are some local modifications made to them.
49/// It can be used to figure out if we should download artifacts from CI or rather
50/// build them locally.
51///
52/// The function assumes that at least a single upstream bors merge commit is in the
53/// local git history.
54///
55/// `target_paths` should be a non-empty slice of paths (git `pathspec`s) relative to `git_dir`
56/// whose modifications would invalidate the artifact.
57/// Each pathspec can also be a negative match, i.e. `:!foo`. This matches changes outside
58/// the `foo` directory.
59/// See <https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-aiddefpathspecapathspec>
60/// for how git `pathspec` works.
61///
62/// The function behaves differently in CI and outside CI.
63///
64/// - Outside CI, we want to find out if `target_paths` were modified in some local commit on
65/// top of the latest upstream commit that is available in local git history.
66/// If not, we try to find the most recent upstream commit (which we assume are commits
67/// made by bors) that modified `target_paths`.
68/// We don't want to simply take the latest master commit to avoid changing the output of
69/// this function frequently after rebasing on the latest master branch even if `target_paths`
70/// were not modified upstream in the meantime. In that case we would be redownloading CI
71/// artifacts unnecessarily.
72///
73/// - In CI, we use a shallow clone of depth 2, i.e., we fetch only a single parent commit
74/// (which will be the most recent bors merge commit) and do not have access
75/// to the full git history. Luckily, we only need to distinguish between two situations:
76/// 1) The current PR made modifications to `target_paths`.
77/// In that case, a build is typically necessary.
78/// 2) The current PR did not make modifications to `target_paths`.
79/// In that case we simply take the latest upstream commit, because on CI there is no need to avoid
80/// redownloading.
81pub fn check_path_modifications(
82 git_dir: &Path,
83 config: &GitConfig<'_>,
84 target_paths: &[&str],
85 ci_env: CiEnv,
86) -> Result<PathFreshness, String> {
87 assert!(!target_paths.is_empty());
88 for path in target_paths {
89 assert!(Path::new(path.trim_start_matches(":!")).is_relative());
90 }
91
92 let upstream_sha = if matches!(ci_env, CiEnv::GitHubActions) {
93 // Here the situation is different for PR CI and try/auto CI.
94 // For PR CI, we have the following history:
95 // <merge commit made by GitHub>
96 // 1-N PR commits
97 // upstream merge commit made by bors
98 //
99 // For try/auto CI, we have the following history:
100 // <**non-upstream** merge commit made by bors>
101 // 1-N PR commits
102 // upstream merge commit made by bors
103 //
104 // But on both cases, HEAD should be a merge commit.
105 // So if HEAD contains modifications of `target_paths`, our PR has modified
106 // them. If not, we can use the only available upstream commit for downloading
107 // artifacts.
108
109 // Do not include HEAD, as it is never an upstream commit
110 // If we do not find an upstream commit in CI, something is seriously wrong.
111 Some(
112 get_closest_upstream_commit(Some(git_dir), config, ci_env)?
113 .expect("No upstream commit was found on CI"),
114 )
115 } else {
116 // Outside CI, we want to find the most recent upstream commit that
117 // modified the set of paths, to have an upstream reference that does not change
118 // unnecessarily often.
119 // However, if such commit is not found, we can fall back to the latest upstream commit
120 let upstream_with_modifications =
121 get_latest_upstream_commit_that_modified_files(git_dir, config, target_paths)?;
122 match upstream_with_modifications {
123 Some(sha) => Some(sha),
124 None => get_closest_upstream_commit(Some(git_dir), config, ci_env)?,
125 }
126 };
127
128 let Some(upstream_sha) = upstream_sha else {
129 return Ok(PathFreshness::MissingUpstream);
130 };
131
132 // For local environments, we want to find out if something has changed
133 // from the latest upstream commit.
134 // However, that should be equivalent to checking if something has changed
135 // from the latest upstream commit *that modified `target_paths`*, and
136 // with this approach we do not need to invoke git an additional time.
137 if has_changed_since(git_dir, &upstream_sha, target_paths) {
138 Ok(PathFreshness::HasLocalModifications { upstream: upstream_sha })
139 } else {
140 Ok(PathFreshness::LastModifiedUpstream { upstream: upstream_sha })
141 }
142}
143
144/// Returns true if any of the passed `paths` have changed since the `base` commit.
145pub fn has_changed_since(git_dir: &Path, base: &str, paths: &[&str]) -> bool {
146 let mut git = Command::new("git");
147 git.current_dir(git_dir);
148
149 git.args(["diff-index", "--quiet", base, "--"]).args(paths);
150
151 // Exit code 0 => no changes
152 // Exit code 1 => some changes were detected
153 !git.status().expect("cannot run git diff-index").success()
154}
155
156/// Returns the latest upstream commit that modified `target_paths`, or `None` if no such commit
157/// was found.
158fn get_latest_upstream_commit_that_modified_files(
159 git_dir: &Path,
160 git_config: &GitConfig<'_>,
161 target_paths: &[&str],
162) -> Result<Option<String>, String> {
163 let mut git = Command::new("git");
164 git.current_dir(git_dir);
165
166 // In theory, we could just use
167 // `git rev-list --first-parent HEAD --author=<merge-bot> -- <paths>`
168 // to find the latest upstream commit that modified `<paths>`.
169 // However, this does not work if you are in a subtree sync branch that contains merge commits
170 // which have the subtree history as their first parent, and the rustc history as second parent:
171 // `--first-parent` will just walk up the subtree history and never see a single rustc commit.
172 // We thus have to take a two-pronged approach. First lookup the most recent upstream commit
173 // by *date* (this should work even in a subtree sync branch), and then start the lookup for
174 // modified paths starting from that commit.
175 //
176 // See https://github.com/rust-lang/rust/pull/138591#discussion_r2037081858 for more details.
177 let upstream = get_closest_upstream_commit(Some(git_dir), git_config, CiEnv::None)?
178 .unwrap_or_else(|| "HEAD".to_string());
179
180 git.args([
181 "rev-list",
182 "--first-parent",
183 "-n1",
184 &upstream,
185 "--author",
186 git_config.git_merge_commit_email,
187 ]);
188
189 if !target_paths.is_empty() {
190 git.arg("--").args(target_paths);
191 }
192 let output = output_result(&mut git)?.trim().to_owned();
193 if output.is_empty() { Ok(None) } else { Ok(Some(output)) }
194}
195
196/// Returns the most recent (ordered chronologically) commit found in the local history that
197/// should exist upstream. We identify upstream commits by the e-mail of the commit
198/// author.
199///
200/// If we are in CI, we simply return our first parent.
201fn get_closest_upstream_commit(
202 git_dir: Option<&Path>,
203 config: &GitConfig<'_>,
204 env: CiEnv,
205) -> Result<Option<String>, String> {
206 let base = match env {
207 CiEnv::None => "HEAD",
208 CiEnv::GitHubActions => {
209 // On CI, we should always have a non-upstream merge commit at the tip,
210 // and our first parent should be the most recently merged upstream commit.
211 // We thus simply return our first parent.
212 return resolve_commit_sha(git_dir, "HEAD^1").map(Some);
213 }
214 };
215
216 let mut git = Command::new("git");
217
218 if let Some(git_dir) = git_dir {
219 git.current_dir(git_dir);
220 }
221
222 // We do not use `--first-parent`, because we can be in a situation (outside CI) where we have
223 // a subtree merge that actually has the main rustc history as its second parent.
224 // Using `--first-parent` would recurse into the history of the subtree, which could have some
225 // old bors commits that are not relevant to us.
226 // With `--author-date-order`, git recurses into all parent subtrees, and returns the most
227 // chronologically recent bors commit.
228 // Here we assume that none of our subtrees use bors anymore, and that all their old bors
229 // commits are way older than recent rustc bors commits!
230 git.args([
231 "rev-list",
232 "--author-date-order",
233 &format!("--author={}", config.git_merge_commit_email),
234 "-n1",
235 &base,
236 ]);
237
238 let output = output_result(&mut git)?.trim().to_owned();
239 if output.is_empty() { Ok(None) } else { Ok(Some(output)) }
240}
241
242/// Resolve the commit SHA of `commit_ref`.
243fn resolve_commit_sha(git_dir: Option<&Path>, commit_ref: &str) -> Result<String, String> {
244 let mut git = Command::new("git");
245
246 if let Some(git_dir) = git_dir {
247 git.current_dir(git_dir);
248 }
249
250 git.args(["rev-parse", commit_ref]);
251
252 Ok(output_result(&mut git)?.trim().to_owned())
253}
254
255/// Returns the files that have been modified in the current branch compared to the master branch.
256/// This includes committed changes, uncommitted changes, and changes that are not even staged.
257///
258/// The `extensions` parameter can be used to filter the files by their extension.
259/// Does not include removed files.
260/// If `extensions` is empty, all files will be returned.
261pub fn get_git_modified_files(
262 config: &GitConfig<'_>,
263 git_dir: Option<&Path>,
264 extensions: &[&str],
265) -> Result<Vec<String>, String> {
266 let Some(merge_base) = get_closest_upstream_commit(git_dir, config, CiEnv::None)? else {
267 return Err("No upstream commit was found".to_string());
268 };
269
270 let mut git = Command::new("git");
271 if let Some(git_dir) = git_dir {
272 git.current_dir(git_dir);
273 }
274 let files = output_result(git.args(["diff-index", "--name-status", merge_base.trim()]))?
275 .lines()
276 .filter_map(|f| {
277 let (status, name) = f.trim().split_once(char::is_whitespace).unwrap();
278 if status == "D" {
279 None
280 } else if Path::new(name).extension().map_or(extensions.is_empty(), |ext| {
281 // If there is no extension, we allow the path if `extensions` is empty
282 // If there is an extension, we allow it if `extension` is empty or it contains the
283 // extension.
284 extensions.is_empty() || extensions.contains(&ext.to_str().unwrap())
285 }) {
286 Some(name.to_owned())
287 } else {
288 None
289 }
290 })
291 .collect();
292 Ok(files)
293}
294
295/// Returns the files that haven't been added to git yet.
296pub fn get_git_untracked_files(git_dir: Option<&Path>) -> Result<Option<Vec<String>>, String> {
297 let mut git = Command::new("git");
298 if let Some(git_dir) = git_dir {
299 git.current_dir(git_dir);
300 }
301
302 let files = output_result(git.arg("ls-files").arg("--others").arg("--exclude-standard"))?
303 .lines()
304 .map(|s| s.trim().to_owned())
305 .collect();
306 Ok(Some(files))
307}