compiletest/
read2.rs

1// FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs`
2// Consider unify the read2() in libstd, cargo and this to prevent further code duplication.
3
4#[cfg(test)]
5mod tests;
6
7use std::io::{self, Write};
8use std::process::{Child, Output};
9
10pub use self::imp::read2;
11
12#[derive(Copy, Clone, Debug)]
13pub enum Truncated {
14    Yes,
15    No,
16}
17
18pub fn read2_abbreviated(
19    mut child: Child,
20    filter_paths_from_len: &[String],
21) -> io::Result<(Output, Truncated)> {
22    let mut stdout = ProcOutput::new();
23    let mut stderr = ProcOutput::new();
24
25    drop(child.stdin.take());
26    read2(
27        child.stdout.take().unwrap(),
28        child.stderr.take().unwrap(),
29        &mut |is_stdout, data, _| {
30            if is_stdout { &mut stdout } else { &mut stderr }.extend(data, filter_paths_from_len);
31            data.clear();
32        },
33    )?;
34    let status = child.wait()?;
35
36    let truncated =
37        if stdout.truncated() || stderr.truncated() { Truncated::Yes } else { Truncated::No };
38    Ok((Output { status, stdout: stdout.into_bytes(), stderr: stderr.into_bytes() }, truncated))
39}
40
41const MAX_OUT_LEN: usize = 512 * 1024;
42
43// Whenever a path is filtered when counting the length of the output, we need to add some
44// placeholder length to ensure a compiler emitting only filtered paths doesn't cause a OOM.
45//
46// 32 was chosen semi-arbitrarily: it was the highest power of two that still allowed the test
47// suite to pass at the moment of implementing path filtering.
48const FILTERED_PATHS_PLACEHOLDER_LEN: usize = 32;
49
50enum ProcOutput {
51    Full { bytes: Vec<u8>, filtered_len: usize },
52    Abbreviated { head: Vec<u8>, skipped: usize },
53}
54
55impl ProcOutput {
56    fn new() -> Self {
57        ProcOutput::Full { bytes: Vec::new(), filtered_len: 0 }
58    }
59
60    fn truncated(&self) -> bool {
61        matches!(self, Self::Abbreviated { .. })
62    }
63
64    fn extend(&mut self, data: &[u8], filter_paths_from_len: &[String]) {
65        let new_self = match *self {
66            ProcOutput::Full { ref mut bytes, ref mut filtered_len } => {
67                let old_len = bytes.len();
68                bytes.extend_from_slice(data);
69                *filtered_len += data.len();
70
71                // We had problems in the past with tests failing only in some environments,
72                // due to the length of the base path pushing the output size over the limit.
73                //
74                // To make those failures deterministic across all environments we ignore known
75                // paths when calculating the string length, while still including the full
76                // path in the output. This could result in some output being larger than the
77                // threshold, but it's better than having nondeterministic failures.
78                //
79                // The compiler emitting only excluded strings is addressed by adding a
80                // placeholder size for each excluded segment, which will eventually reach
81                // the configured threshold.
82                for path in filter_paths_from_len {
83                    let path_bytes = path.as_bytes();
84                    // We start matching `path_bytes - 1` into the previously loaded data,
85                    // to account for the fact a path_bytes might be included across multiple
86                    // `extend` calls. Starting from `- 1` avoids double-counting paths.
87                    let matches = (&bytes[(old_len.saturating_sub(path_bytes.len() - 1))..])
88                        .windows(path_bytes.len())
89                        .filter(|window| window == &path_bytes)
90                        .count();
91                    *filtered_len -= matches * path_bytes.len();
92
93                    // We can't just remove the length of the filtered path from the output length,
94                    // otherwise a compiler emitting only filtered paths would OOM compiletest. Add
95                    // a fixed placeholder length for each path to prevent that.
96                    *filtered_len += matches * FILTERED_PATHS_PLACEHOLDER_LEN;
97                }
98
99                let new_len = bytes.len();
100                if (*filtered_len).min(new_len) <= MAX_OUT_LEN {
101                    return;
102                }
103
104                let mut head = std::mem::take(bytes);
105                // Don't truncate if this as a whole line.
106                // That should make it less likely that we cut a JSON line in half.
107                if head.last() != Some(&b'\n') {
108                    head.truncate(MAX_OUT_LEN);
109                }
110                let skipped = new_len - head.len();
111                ProcOutput::Abbreviated { head, skipped }
112            }
113            ProcOutput::Abbreviated { ref mut skipped, .. } => {
114                *skipped += data.len();
115                return;
116            }
117        };
118        *self = new_self;
119    }
120
121    fn into_bytes(self) -> Vec<u8> {
122        match self {
123            ProcOutput::Full { bytes, .. } => bytes,
124            ProcOutput::Abbreviated { mut head, skipped } => {
125                let head_note =
126                    format!("<<<<<< TRUNCATED, SHOWING THE FIRST {} BYTES >>>>>>\n\n", head.len());
127                head.splice(0..0, head_note.into_bytes());
128                write!(&mut head, "\n\n<<<<<< TRUNCATED, DROPPED {} BYTES >>>>>>", skipped)
129                    .unwrap();
130                head
131            }
132        }
133    }
134}
135
136#[cfg(not(any(unix, windows)))]
137mod imp {
138    use std::io::{self, Read};
139    use std::process::{ChildStderr, ChildStdout};
140
141    pub fn read2(
142        out_pipe: ChildStdout,
143        err_pipe: ChildStderr,
144        data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
145    ) -> io::Result<()> {
146        let mut buffer = Vec::new();
147        out_pipe.read_to_end(&mut buffer)?;
148        data(true, &mut buffer, true);
149        buffer.clear();
150        err_pipe.read_to_end(&mut buffer)?;
151        data(false, &mut buffer, true);
152        Ok(())
153    }
154}
155
156#[cfg(unix)]
157mod imp {
158    use std::io::prelude::*;
159    use std::os::unix::prelude::*;
160    use std::process::{ChildStderr, ChildStdout};
161    use std::{io, mem};
162
163    pub fn read2(
164        mut out_pipe: ChildStdout,
165        mut err_pipe: ChildStderr,
166        data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
167    ) -> io::Result<()> {
168        // FIXME(#139616): justify why this is sound.
169        unsafe {
170            libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
171            libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
172        }
173
174        let mut out_done = false;
175        let mut err_done = false;
176        let mut out = Vec::new();
177        let mut err = Vec::new();
178
179        // FIXME(#139616): justify why this is sound.
180        let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() };
181        fds[0].fd = out_pipe.as_raw_fd();
182        fds[0].events = libc::POLLIN;
183        fds[1].fd = err_pipe.as_raw_fd();
184        fds[1].events = libc::POLLIN;
185        let mut nfds = 2;
186        let mut errfd = 1;
187
188        while nfds > 0 {
189            // wait for either pipe to become readable using `select`
190            // FIXME(#139616): justify why this is sound.
191            let r = unsafe { libc::poll(fds.as_mut_ptr(), nfds, -1) };
192            if r == -1 {
193                let err = io::Error::last_os_error();
194                if err.kind() == io::ErrorKind::Interrupted {
195                    continue;
196                }
197                return Err(err);
198            }
199
200            // Read as much as we can from each pipe, ignoring EWOULDBLOCK or
201            // EAGAIN. If we hit EOF, then this will happen because the underlying
202            // reader will return Ok(0), in which case we'll see `Ok` ourselves. In
203            // this case we flip the other fd back into blocking mode and read
204            // whatever's leftover on that file descriptor.
205            let handle = |res: io::Result<_>| match res {
206                Ok(_) => Ok(true),
207                Err(e) => {
208                    if e.kind() == io::ErrorKind::WouldBlock {
209                        Ok(false)
210                    } else {
211                        Err(e)
212                    }
213                }
214            };
215            if !err_done && fds[errfd].revents != 0 && handle(err_pipe.read_to_end(&mut err))? {
216                err_done = true;
217                nfds -= 1;
218            }
219            data(false, &mut err, err_done);
220            if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? {
221                out_done = true;
222                fds[0].fd = err_pipe.as_raw_fd();
223                errfd = 0;
224                nfds -= 1;
225            }
226            data(true, &mut out, out_done);
227        }
228        Ok(())
229    }
230}
231
232#[cfg(windows)]
233mod imp {
234    use std::os::windows::prelude::*;
235    use std::process::{ChildStderr, ChildStdout};
236    use std::{io, slice};
237
238    use miow::Overlapped;
239    use miow::iocp::{CompletionPort, CompletionStatus};
240    use miow::pipe::NamedPipe;
241    use windows::Win32::Foundation::ERROR_BROKEN_PIPE;
242
243    struct Pipe<'a> {
244        dst: &'a mut Vec<u8>,
245        overlapped: Overlapped,
246        pipe: NamedPipe,
247        done: bool,
248    }
249
250    pub fn read2(
251        out_pipe: ChildStdout,
252        err_pipe: ChildStderr,
253        data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
254    ) -> io::Result<()> {
255        let mut out = Vec::new();
256        let mut err = Vec::new();
257
258        let port = CompletionPort::new(1)?;
259        port.add_handle(0, &out_pipe)?;
260        port.add_handle(1, &err_pipe)?;
261
262        // FIXME(#139616): justify why this is sound.
263        unsafe {
264            let mut out_pipe = Pipe::new(out_pipe, &mut out);
265            let mut err_pipe = Pipe::new(err_pipe, &mut err);
266
267            out_pipe.read()?;
268            err_pipe.read()?;
269
270            let mut status = [CompletionStatus::zero(), CompletionStatus::zero()];
271
272            while !out_pipe.done || !err_pipe.done {
273                for status in port.get_many(&mut status, None)? {
274                    if status.token() == 0 {
275                        out_pipe.complete(status);
276                        data(true, out_pipe.dst, out_pipe.done);
277                        out_pipe.read()?;
278                    } else {
279                        err_pipe.complete(status);
280                        data(false, err_pipe.dst, err_pipe.done);
281                        err_pipe.read()?;
282                    }
283                }
284            }
285
286            Ok(())
287        }
288    }
289
290    impl<'a> Pipe<'a> {
291        // FIXME(#139616): document caller contract.
292        unsafe fn new<P: IntoRawHandle>(p: P, dst: &'a mut Vec<u8>) -> Pipe<'a> {
293            Pipe {
294                dst,
295                // FIXME(#139616): justify why this is sound.
296                pipe: unsafe { NamedPipe::from_raw_handle(p.into_raw_handle()) },
297                overlapped: Overlapped::zero(),
298                done: false,
299            }
300        }
301
302        // FIXME(#139616): document caller contract.
303        unsafe fn read(&mut self) -> io::Result<()> {
304            // FIXME(#139616): justify why this is sound.
305            let dst = unsafe { slice_to_end(self.dst) };
306            // FIXME(#139616): justify why this is sound.
307            match unsafe { self.pipe.read_overlapped(dst, self.overlapped.raw()) } {
308                Ok(_) => Ok(()),
309                Err(e) => {
310                    if e.raw_os_error() == Some(ERROR_BROKEN_PIPE.0 as i32) {
311                        self.done = true;
312                        Ok(())
313                    } else {
314                        Err(e)
315                    }
316                }
317            }
318        }
319
320        // FIXME(#139616): document caller contract.
321        unsafe fn complete(&mut self, status: &CompletionStatus) {
322            let prev = self.dst.len();
323            // FIXME(#139616): justify why this is sound.
324            unsafe { self.dst.set_len(prev + status.bytes_transferred() as usize) };
325            if status.bytes_transferred() == 0 {
326                self.done = true;
327            }
328        }
329    }
330
331    // FIXME(#139616): document caller contract.
332    unsafe fn slice_to_end(v: &mut Vec<u8>) -> &mut [u8] {
333        if v.capacity() == 0 {
334            v.reserve(16);
335        }
336        if v.capacity() == v.len() {
337            v.reserve(1);
338        }
339        // FIXME(#139616): justify why this is sound.
340        unsafe {
341            slice::from_raw_parts_mut(
342                v.as_mut_ptr().offset(v.len() as isize),
343                v.capacity() - v.len(),
344            )
345        }
346    }
347}
compiletest/read2.rs

compiletest/
read2.rs
