Auto merge of #40236 - petrochenkov:btweak, r=alexcrichton

rustbuild: A few tweaks

Fixes #40016
Fixes #39507

r? @alexcrichton

Author: bors

Diff for: src/Cargo.lock

@@ -8,6 +8,7 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
+from __future__ import print_function
 import argparse
 import contextlib
 import datetime
@@ -501,7 +502,7 @@ def build_triple(self):
 
         return "{}-{}".format(cputype, ostype)
 
-def main():
+def bootstrap():
     parser = argparse.ArgumentParser(description='Build rust')
     parser.add_argument('--config')
     parser.add_argument('--clean', action='store_true')
@@ -564,8 +565,6 @@ def main():
     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
     rb._cargo_rev = data['cargo']
 
-    start_time = time()
-
     # Fetch/build the bootstrap
     rb.build = rb.build_triple()
     rb.download_stage0()
@@ -582,9 +581,19 @@ def main():
     env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
     rb.run(args, env)
 
-    end_time = time()
-
-    print("Build completed in %s" % format_build_time(end_time - start_time))
+def main():
+    start_time = time()
+    try:
+        bootstrap()
+        print("Build completed successfully in %s" % format_build_time(time() - start_time))
+    except (SystemExit, KeyboardInterrupt) as e:
+        if hasattr(e, 'code') and isinstance(e.code, int):
+            exit_code = e.code
+        else:
+            exit_code = 1
+            print(e)
+        print("Build completed unsuccessfully in %s" % format_build_time(time() - start_time))
+        sys.exit(exit_code)
 
 if __name__ == '__main__':
     main()

Diff for: src/bootstrap/bootstrap.py

@@ -740,7 +740,7 @@ impl Build {
         } else {
             let base = self.llvm_out(&self.config.build).join("build");
             let exe = exe("FileCheck", target);
-            if self.config.build.contains("msvc") {
+            if !self.config.ninja && self.config.build.contains("msvc") {
                 base.join("Release/bin").join(exe)
             } else {
                 base.join("bin").join(exe)

Diff for: src/bootstrap/lib.rs

@@ -41,28 +41,32 @@ pub fn llvm(build: &Build, target: &str) {
         }
     }
 
-    // If the cleaning trigger is newer than our built artifacts (or if the
-    // artifacts are missing) then we keep going, otherwise we bail out.
-    let dst = build.llvm_out(target);
-    let stamp = build.src.join("src/rustllvm/llvm-auto-clean-trigger");
-    let mut stamp_contents = String::new();
-    t!(t!(File::open(&stamp)).read_to_string(&mut stamp_contents));
-    let done_stamp = dst.join("llvm-finished-building");
+    let clean_trigger = build.src.join("src/rustllvm/llvm-auto-clean-trigger");
+    let mut clean_trigger_contents = String::new();
+    t!(t!(File::open(&clean_trigger)).read_to_string(&mut clean_trigger_contents));
+
+    let out_dir = build.llvm_out(target);
+    let done_stamp = out_dir.join("llvm-finished-building");
     if done_stamp.exists() {
         let mut done_contents = String::new();
         t!(t!(File::open(&done_stamp)).read_to_string(&mut done_contents));
-        if done_contents == stamp_contents {
+
+        // LLVM was already built previously.
+        // We don't track changes in LLVM sources, so we need to choose between reusing
+        // what was built previously, or cleaning the directory and doing a fresh build.
+        // The choice depends on contents of the clean-trigger file.
+        // If the contents are the same as during the previous build, then no action is required.
+        // If the contents differ from the previous build, then cleaning is triggered.
+        if done_contents == clean_trigger_contents {
             return
+        } else {
+            t!(fs::remove_dir_all(&out_dir));
         }
     }
-    drop(fs::remove_dir_all(&dst));
 
     println!("Building LLVM for {}", target);
-
     let _time = util::timeit();
-    let _ = fs::remove_dir_all(&dst.join("build"));
-    t!(fs::create_dir_all(&dst.join("build")));
-    let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"};
+    t!(fs::create_dir_all(&out_dir));
 
     // http://llvm.org/docs/CMake.html
     let mut cfg = cmake::Config::new(build.src.join("src/llvm"));
@@ -82,9 +86,11 @@ pub fn llvm(build: &Build, target: &str) {
         None => "X86;ARM;AArch64;Mips;PowerPC;SystemZ;JSBackend;MSP430;Sparc;NVPTX",
     };
 
+    let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"};
+
     cfg.target(target)
        .host(&build.config.build)
-       .out_dir(&dst)
+       .out_dir(&out_dir)
        .profile(profile)
        .define("LLVM_ENABLE_ASSERTIONS", assertions)
        .define("LLVM_TARGETS_TO_BUILD", llvm_targets)
@@ -142,7 +148,7 @@ pub fn llvm(build: &Build, target: &str) {
     //        tools and libs on all platforms.
     cfg.build();
 
-    t!(t!(File::create(&done_stamp)).write_all(stamp_contents.as_bytes()));
+    t!(t!(File::create(&done_stamp)).write_all(clean_trigger_contents.as_bytes()));
 }
 
 fn check_llvm_version(build: &Build, llvm_config: &Path) {

Diff for: src/bootstrap/native.rs

@@ -312,6 +312,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  });
         };
 
+        suite("check-ui", "src/test/ui", "ui", "ui");
         suite("check-rpass", "src/test/run-pass", "run-pass", "run-pass");
         suite("check-cfail", "src/test/compile-fail", "compile-fail", "compile-fail");
         suite("check-pfail", "src/test/parse-fail", "parse-fail", "parse-fail");
@@ -372,7 +373,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
                  });
         };
 
-        suite("check-ui", "src/test/ui", "ui", "ui");
+        suite("check-ui-full", "src/test/ui-fulldeps", "ui", "ui-fulldeps");
         suite("check-rpass-full", "src/test/run-pass-fulldeps",
               "run-pass", "run-pass-fulldeps");
         suite("check-rfail-full", "src/test/run-fail-fulldeps",
@@ -1530,7 +1531,8 @@ mod tests {
         assert!(plan.iter().all(|s| s.host == "A"));
         assert!(plan.iter().all(|s| s.target == "C"));
 
-        assert!(!plan.iter().any(|s| s.name.contains("-ui")));
+        assert!(plan.iter().any(|s| s.name.contains("-ui")));
+        assert!(!plan.iter().any(|s| s.name.contains("ui-full")));
         assert!(plan.iter().any(|s| s.name.contains("cfail")));
         assert!(!plan.iter().any(|s| s.name.contains("cfail-full")));
         assert!(plan.iter().any(|s| s.name.contains("codegen-units")));

Diff for: src/bootstrap/step.rs

@@ -12,7 +12,8 @@
 
 extern crate filetime;
 
-use std::fs;
+use std::{fs, env};
+use std::fs::File;
 use std::process::{Command, Stdio};
 use std::path::{Path, PathBuf};
 
@@ -166,6 +167,47 @@ pub fn up_to_date(src: &Path, dst: &Path) -> bool {
     }
 }
 
+#[must_use]
+pub struct NativeLibBoilerplate {
+    pub src_dir: PathBuf,
+    pub out_dir: PathBuf,
+}
+
+impl Drop for NativeLibBoilerplate {
+    fn drop(&mut self) {
+        t!(File::create(self.out_dir.join("rustbuild.timestamp")));
+    }
+}
+
+// Perform standard preparations for native libraries that are build only once for all stages.
+// Emit rerun-if-changed and linking attributes for Cargo, check if any source files are
+// updated, calculate paths used later in actual build with CMake/make or C/C++ compiler.
+// If Err is returned, then everything is up-to-date and further build actions can be skipped.
+// Timestamps are created automatically when the result of `native_lib_boilerplate` goes out
+// of scope, so all the build actions should be completed until then.
+pub fn native_lib_boilerplate(src_name: &str,
+                              out_name: &str,
+                              link_name: &str,
+                              search_subdir: &str)
+                              -> Result<NativeLibBoilerplate, ()> {
+    let current_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());
+    let src_dir = current_dir.join("..").join(src_name);
+    rerun_if_changed_anything_in_dir(&src_dir);
+
+    let out_dir = env::var_os("RUSTBUILD_NATIVE_DIR").unwrap_or(env::var_os("OUT_DIR").unwrap());
+    let out_dir = PathBuf::from(out_dir).join(out_name);
+    let _ = fs::create_dir_all(&out_dir);
+    println!("cargo:rustc-link-lib=static={}", link_name);
+    println!("cargo:rustc-link-search=native={}", out_dir.join(search_subdir).display());
+
+    let timestamp = out_dir.join("rustbuild.timestamp");
+    if !up_to_date(Path::new("build.rs"), &timestamp) || !up_to_date(&src_dir, &timestamp) {
+        Ok(NativeLibBoilerplate { src_dir: src_dir, out_dir: out_dir })
+    } else {
+        Err(())
+    }
+}
+
 fn dir_up_to_date(src: &Path, threshold: &FileTime) -> bool {
     t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| {
         let meta = t!(e.metadata());

Diff for: src/build_helper/lib.rs

@@ -10,19 +10,15 @@
 
 #![deny(warnings)]
 
-#[macro_use]
 extern crate build_helper;
 extern crate gcc;
 
 use std::env;
-use std::fs::{self, File};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::Command;
-use build_helper::{run, rerun_if_changed_anything_in_dir, up_to_date};
+use build_helper::{run, native_lib_boilerplate};
 
 fn main() {
-    println!("cargo:rerun-if-changed=build.rs");
-
     // FIXME: This is a hack to support building targets that don't
     // support jemalloc alongside hosts that do. The jemalloc build is
     // controlled by a feature of the std crate, and if that feature
@@ -61,22 +57,11 @@ fn main() {
         return;
     }
 
-    let build_dir = env::var_os("RUSTBUILD_NATIVE_DIR").unwrap_or(env::var_os("OUT_DIR").unwrap());
-    let build_dir = PathBuf::from(build_dir).join("jemalloc");
-    let _ = fs::create_dir_all(&build_dir);
-
-    if target.contains("windows") {
-        println!("cargo:rustc-link-lib=static=jemalloc");
-    } else {
-        println!("cargo:rustc-link-lib=static=jemalloc_pic");
-    }
-    println!("cargo:rustc-link-search=native={}/lib", build_dir.display());
-    let src_dir = env::current_dir().unwrap().join("../jemalloc");
-    rerun_if_changed_anything_in_dir(&src_dir);
-    let timestamp = build_dir.join("rustbuild.timestamp");
-    if up_to_date(&Path::new("build.rs"), &timestamp) && up_to_date(&src_dir, &timestamp) {
-        return
-    }
+    let link_name = if target.contains("windows") { "jemalloc" } else { "jemalloc_pic" };
+    let native = match native_lib_boilerplate("jemalloc", "jemalloc", link_name, "lib") {
+        Ok(native) => native,
+        _ => return,
+    };
 
     let compiler = gcc::Config::new().get_compiler();
     // only msvc returns None for ar so unwrap is okay
@@ -88,12 +73,12 @@ fn main() {
         .join(" ");
 
     let mut cmd = Command::new("sh");
-    cmd.arg(src_dir.join("configure")
-                   .to_str()
-                   .unwrap()
-                   .replace("C:\\", "/c/")
-                   .replace("\\", "/"))
-       .current_dir(&build_dir)
+    cmd.arg(native.src_dir.join("configure")
+                          .to_str()
+                          .unwrap()
+                          .replace("C:\\", "/c/")
+                          .replace("\\", "/"))
+       .current_dir(&native.out_dir)
        .env("CC", compiler.path())
        .env("EXTRA_CFLAGS", cflags.clone())
        // jemalloc generates Makefile deps using GCC's "-MM" flag. This means
@@ -166,7 +151,7 @@ fn main() {
     run(&mut cmd);
 
     let mut make = Command::new(build_helper::make(&host));
-    make.current_dir(&build_dir)
+    make.current_dir(&native.out_dir)
         .arg("build_lib_static");
 
     // mingw make seems... buggy? unclear...
@@ -187,6 +172,4 @@ fn main() {
             .file("pthread_atfork_dummy.c")
             .compile("libpthread_atfork_dummy.a");
     }
-
-    t!(File::create(&timestamp));
 }

Diff for: src/liballoc_jemalloc/build.rs

@@ -15,4 +15,5 @@ doc = false
 core = { path = "../libcore" }
 
 [build-dependencies]
+build_helper = { path = "../build_helper" }
 gcc = "0.3.27"