Rollup merge of rust-lang#63175 - jsgf:argsfile, r=alexcrichton

rustc: implement argsfiles for command line

Many tools, such as gcc and gnu-ld, support "args files" - that is, being able to specify @file on the command line.  This causes `file` to be opened and parsed for command line options. They're separated with whitespace; whitespace can be quoted with double or single quotes, and everything can be \\-escaped. Args files may recursively include other args files via `@file2`.

See https://sourceware.org/binutils/docs/ld/Options.html#Options for the documentation of gnu-ld's @file parameters.

This is useful for very large command lines, or when command lines are being generated into files by other tooling.

Author: Centril

src/doc/rustc/src/command-line-arguments.md

@@ -304,3 +304,10 @@ to customize the output:
 
 Note that it is invalid to combine the `--json` argument with the `--color`
 argument, and it is required to combine `--json` with `--error-format=json`.
+
+## `@path`: load command-line flags from a path
+
+If you specify `@path` on the command-line, then it will open `path` and read
+command line options from it. These options are one per line; a blank line indicates
+an empty option. The file can use Unix or Windows style line endings, and must be
+encoded as UTF-8.

src/librustc_driver/args/mod.rs

@@ -0,0 +1,84 @@
+use std::env;
+use std::error;
+use std::fmt;
+use std::fs;
+use std::io;
+use std::str;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+#[cfg(test)]
+mod tests;
+
+static USED_ARGSFILE_FEATURE: AtomicBool = AtomicBool::new(false);
+
+pub fn used_unstable_argsfile() -> bool {
+    USED_ARGSFILE_FEATURE.load(Ordering::Relaxed)
+}
+
+pub struct ArgsIter {
+    base: env::ArgsOs,
+    file: std::vec::IntoIter<String>,
+}
+
+impl ArgsIter {
+    pub fn new() -> Self {
+        ArgsIter { base: env::args_os(), file: vec![].into_iter() }
+    }
+}
+
+impl Iterator for ArgsIter {
+    type Item = Result<String, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(line) = self.file.next() {
+                return Some(Ok(line));
+            }
+
+            let arg =
+                self.base.next().map(|arg| arg.into_string().map_err(|_| Error::Utf8Error(None)));
+            match arg {
+                Some(Err(err)) => return Some(Err(err)),
+                Some(Ok(ref arg)) if arg.starts_with("@") => {
+                    let path = &arg[1..];
+                    let file = match fs::read_to_string(path) {
+                        Ok(file) => {
+                            USED_ARGSFILE_FEATURE.store(true, Ordering::Relaxed);
+                            file
+                        }
+                        Err(ref err) if err.kind() == io::ErrorKind::InvalidData => {
+                            return Some(Err(Error::Utf8Error(Some(path.to_string()))));
+                        }
+                        Err(err) => return Some(Err(Error::IOError(path.to_string(), err))),
+                    };
+                    self.file =
+                        file.lines().map(ToString::to_string).collect::<Vec<_>>().into_iter();
+                }
+                Some(Ok(arg)) => return Some(Ok(arg)),
+                None => return None,
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Utf8Error(Option<String>),
+    IOError(String, io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::Utf8Error(None) => write!(fmt, "Utf8 error"),
+            Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {}", path),
+            Error::IOError(path, err) => write!(fmt, "IO Error: {}: {}", path, err),
+        }
+    }
+}
+
+impl error::Error for Error {
+    fn description(&self) -> &'static str {
+        "argument error"
+    }
+}

src/librustc_driver/args/tests.rs

@@ -0,0 +1,145 @@
+use super::*;
+
+use std::str;
+
+fn want_args(v: impl IntoIterator<Item = &'static str>) -> Vec<String> {
+    v.into_iter().map(String::from).collect()
+}
+
+fn got_args(file: &[u8]) -> Result<Vec<String>, Error> {
+    let ret = str::from_utf8(file)
+        .map_err(|_| Error::Utf8Error(None))?
+        .lines()
+        .map(ToString::to_string)
+        .collect::<Vec<_>>();
+    Ok(ret)
+}
+
+#[test]
+fn nothing() {
+    let file = b"";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec![]));
+}
+
+#[test]
+fn empty() {
+    let file = b"\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec![""]));
+}
+
+#[test]
+fn simple() {
+    let file = b"foo";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn simple_eol() {
+    let file = b"foo\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn multi() {
+    let file = b"foo\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_eol() {
+    let file = b"foo\nbar\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_empty() {
+    let file = b"foo\n\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_eol() {
+    let file = b"foo\n\nbar\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_start() {
+    let file = b"\nfoo\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["", "foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_end() {
+    let file = b"foo\nbar\n\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar", ""]));
+}
+
+#[test]
+fn simple_eol_crlf() {
+    let file = b"foo\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn multi_crlf() {
+    let file = b"foo\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_eol_crlf() {
+    let file = b"foo\r\nbar\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_crlf() {
+    let file = b"foo\r\n\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_eol_crlf() {
+    let file = b"foo\r\n\r\nbar\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_start_crlf() {
+    let file = b"\r\nfoo\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["", "foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_end_crlf() {
+    let file = b"foo\r\nbar\r\n\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar", ""]));
+}
+
+#[test]
+fn bad_utf8() {
+    let file = b"foo\x80foo";
+
+    match got_args(file).unwrap_err() {
+        Error::Utf8Error(_) => (),
+        bad => panic!("bad err: {:?}", bad),
+    }
+}

src/librustc_driver/lib.rs

@@ -66,6 +66,7 @@ use syntax::symbol::sym;
 use syntax_pos::{DUMMY_SP, MultiSpan, FileName};
 
 pub mod pretty;
+mod args;
 
 /// Exit status code used for successful compilation and help output.
 pub const EXIT_SUCCESS: i32 = 0;
@@ -777,13 +778,19 @@ fn usage(verbose: bool, include_unstable_options: bool) {
     } else {
         "\n    --help -v           Print the full set of options rustc accepts"
     };
-    println!("{}\nAdditional help:
+    let at_path = if verbose && nightly_options::is_nightly_build() {
+        "    @path               Read newline separated options from `path`\n"
+    } else {
+        ""
+    };
+    println!("{options}{at_path}\nAdditional help:
     -C help             Print codegen options
     -W help             \
-              Print 'lint' options and default settings{}{}\n",
-             options.usage(message),
-             nightly_help,
-             verbose_help);
+              Print 'lint' options and default settings{nightly}{verbose}\n",
+             options = options.usage(message),
+             at_path = at_path,
+             nightly = nightly_help,
+             verbose = verbose_help);
 }
 
 fn print_wall_help() {
@@ -1008,6 +1015,12 @@ pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
     //   (unstable option being used on stable)
     nightly_options::check_nightly_options(&matches, &config::rustc_optgroups());
 
+    // Late check to see if @file was used without unstable options enabled
+    if crate::args::used_unstable_argsfile() && !nightly_options::is_unstable_enabled(&matches) {
+        early_error(ErrorOutputType::default(),
+            "@path is unstable - use -Z unstable-options to enable its use");
+    }
+
     if matches.opt_present("h") || matches.opt_present("help") {
         // Only show unstable options in --help if we accept unstable options.
         usage(matches.opt_present("verbose"), nightly_options::is_unstable_enabled(&matches));
@@ -1186,10 +1199,10 @@ pub fn main() {
     init_rustc_env_logger();
     let mut callbacks = TimePassesCallbacks::default();
     let result = report_ices_to_stderr_if_any(|| {
-        let args = env::args_os().enumerate()
-            .map(|(i, arg)| arg.into_string().unwrap_or_else(|arg| {
+        let args = args::ArgsIter::new().enumerate()
+            .map(|(i, arg)| arg.unwrap_or_else(|err| {
                 early_error(ErrorOutputType::default(),
-                            &format!("Argument {} is not valid Unicode: {:?}", i, arg))
+                            &format!("Argument {} is not valid: {}", i, err))
             }))
             .collect::<Vec<_>>();
         run_compiler(&args, &mut callbacks, None, None)

src/test/ui/commandline-argfile-badutf8.args

@@ -0,0 +1,2 @@
+--cfg
+unbroken�

src/test/ui/commandline-argfile-badutf8.rs

@@ -0,0 +1,14 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-fail
+// normalize-stderr-test: "Argument \d+" -> "Argument $$N"
+// compile-flags: --cfg cmdline_set @{{src-base}}/commandline-argfile-badutf8.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unbroken))]
+compile_error!("unbroken not set");
+
+fn main() {
+}

src/test/ui/commandline-argfile-badutf8.stderr

@@ -0,0 +1,2 @@
+error: Argument $N is not valid: Utf8 error in $DIR/commandline-argfile-badutf8.args
+

src/test/ui/commandline-argfile-missing.rs

@@ -0,0 +1,15 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-fail
+// normalize-stderr-test: "Argument \d+" -> "Argument $$N"
+// normalize-stderr-test: "os error \d+" -> "os error $$ERR"
+// compile-flags: --cfg cmdline_set @{{src-base}}/commandline-argfile-missing.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unbroken))]
+compile_error!("unbroken not set");
+
+fn main() {
+}

src/test/ui/commandline-argfile-missing.stderr

@@ -0,0 +1,2 @@
+error: Argument $N is not valid: IO Error: $DIR/commandline-argfile-missing.args: No such file or directory (os error $ERR)
+

src/test/ui/commandline-argfile.args

@@ -0,0 +1,2 @@
+--cfg
+unbroken

src/test/ui/commandline-argfile.rs

@@ -0,0 +1,13 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-pass
+// compile-flags: --cfg cmdline_set @{{src-base}}/commandline-argfile.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unbroken))]
+compile_error!("unbroken not set");
+
+fn main() {
+}