Unrolled build for rust-lang#137373

Rollup merge of rust-lang#137373 - Kobzol:tool-stage0-improve, r=jieyouxu

Compile run-make-support and run-make tests with the bootstrap compiler

It does not seem necessary to have to recompile run-make-support on changes to the local compiler/stdlib. This PR simplifies the implementation of a few tools, then switches rms to stage0 and also makes the handling of environment variables in run-make tests simpler.

Best reviewed commit-by-commit. I can split it into multiple PRs if you want.

Also tested that `COMPILETEST_FORCE_STAGE0=1 ./x test tests/run-make --stage 0` still works. Incredibly, it looks like it even passes more tests than on `master` 😆

r? ``@jieyouxu``

Author: rust-timer

Diff for: src/bootstrap/src/core/build_steps/test.rs

@@ -1242,59 +1242,6 @@ macro_rules! test {
     };
 }
 
-#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
-pub struct RunMakeSupport {
-    pub compiler: Compiler,
-    pub target: TargetSelection,
-}
-
-impl Step for RunMakeSupport {
-    type Output = PathBuf;
-    const DEFAULT: bool = true;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.never()
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
-        run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() });
-    }
-
-    /// Builds run-make-support and returns the path to the resulting rlib.
-    fn run(self, builder: &Builder<'_>) -> PathBuf {
-        builder.ensure(compile::Std::new(self.compiler, self.target));
-
-        let cargo = tool::prepare_tool_cargo(
-            builder,
-            self.compiler,
-            Mode::ToolStd,
-            self.target,
-            Kind::Build,
-            "src/tools/run-make-support",
-            SourceType::InTree,
-            &[],
-        );
-
-        let _guard = builder.msg_tool(
-            Kind::Build,
-            Mode::ToolStd,
-            "run-make-support",
-            self.compiler.stage,
-            &self.compiler.host,
-            &self.target,
-        );
-        cargo.into_cmd().run(builder);
-
-        let lib_name = "librun_make_support.rlib";
-        let lib = builder.tools_dir(self.compiler).join(lib_name);
-
-        let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(lib_name);
-        builder.copy_link(&cargo_out, &lib);
-        lib
-    }
-}
-
 /// Runs `cargo test` on the `src/tools/run-make-support` crate.
 /// That crate is used by run-make tests.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -1446,40 +1393,7 @@ test!(Pretty {
     only_hosts: true,
 });
 
-/// Special-handling is needed for `run-make`, so don't use `test!` for defining `RunMake`
-/// tests.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
-pub struct RunMake {
-    pub compiler: Compiler,
-    pub target: TargetSelection,
-}
-
-impl Step for RunMake {
-    type Output = ();
-    const DEFAULT: bool = true;
-    const ONLY_HOSTS: bool = false;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.suite_path("tests/run-make")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple());
-        run.builder.ensure(RunMakeSupport { compiler, target: run.build_triple() });
-        run.builder.ensure(RunMake { compiler, target: run.target });
-    }
-
-    fn run(self, builder: &Builder<'_>) {
-        builder.ensure(Compiletest {
-            compiler: self.compiler,
-            target: self.target,
-            mode: "run-make",
-            suite: "run-make",
-            path: "tests/run-make",
-            compare_mode: None,
-        });
-    }
-}
+test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make", default: true });
 
 test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly", default: true });
 
@@ -1722,6 +1636,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
                 host: target,
             });
         }
+        if suite == "run-make" {
+            builder.tool_exe(Tool::RunMakeSupport);
+        }
 
         // Also provide `rust_test_helpers` for the host.
         builder.ensure(TestHelpers { target: compiler.host });
@@ -1774,6 +1691,11 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
             };
 
             cmd.arg("--cargo-path").arg(cargo_path);
+
+            // We need to pass the compiler that was used to compile run-make-support,
+            // because we have to use the same compiler to compile rmake.rs recipes.
+            let stage0_rustc_path = builder.compiler(0, compiler.host);
+            cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path));
         }
 
         // Avoid depending on rustdoc when we don't need it.

Diff for: src/bootstrap/src/core/build_steps/tool.rs

@@ -34,6 +34,12 @@ pub enum SourceType {
     Submodule,
 }
 
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub enum ToolArtifactKind {
+    Binary,
+    Library,
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 struct ToolBuild {
     compiler: Compiler,
@@ -47,6 +53,8 @@ struct ToolBuild {
     allow_features: &'static str,
     /// Additional arguments to pass to the `cargo` invocation.
     cargo_args: Vec<String>,
+    /// Whether the tool builds a binary or a library.
+    artifact_kind: ToolArtifactKind,
 }
 
 impl Builder<'_> {
@@ -79,7 +87,7 @@ impl Builder<'_> {
 /// for using this type as `type Output = ToolBuildResult;`
 #[derive(Clone)]
 pub struct ToolBuildResult {
-    /// Executable path of the corresponding tool that was built.
+    /// Artifact path of the corresponding tool that was built.
     pub tool_path: PathBuf,
     /// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`.
     /// For `ToolRustc` this is one stage before of the `target_compiler`.
@@ -179,8 +187,14 @@ impl Step for ToolBuild {
             if tool == "tidy" {
                 tool = "rust-tidy";
             }
-            let tool_path =
-                copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool);
+            let tool_path = match self.artifact_kind {
+                ToolArtifactKind::Binary => {
+                    copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool)
+                }
+                ToolArtifactKind::Library => builder
+                    .cargo_out(self.compiler, self.mode, self.target)
+                    .join(format!("lib{tool}.rlib")),
+            };
 
             ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler }
         }
@@ -330,6 +344,7 @@ macro_rules! bootstrap_tool {
         $(,is_unstable_tool = $unstable:expr)*
         $(,allow_features = $allow_features:expr)?
         $(,submodules = $submodules:expr)?
+        $(,artifact_kind = $artifact_kind:expr)?
         ;
     )+) => {
         #[derive(PartialEq, Eq, Clone)]
@@ -389,6 +404,7 @@ macro_rules! bootstrap_tool {
                         builder.require_submodule(submodule, None);
                     }
                 )*
+
                 builder.ensure(ToolBuild {
                     compiler: self.compiler,
                     target: self.target,
@@ -407,7 +423,12 @@ macro_rules! bootstrap_tool {
                     },
                     extra_features: vec![],
                     allow_features: concat!($($allow_features)*),
-                    cargo_args: vec![]
+                    cargo_args: vec![],
+                    artifact_kind: if false $(|| $artifact_kind == ToolArtifactKind::Library)* {
+                        ToolArtifactKind::Library
+                    } else {
+                        ToolArtifactKind::Binary
+                    }
                 })
             }
         }
@@ -445,51 +466,14 @@ bootstrap_tool!(
     WasmComponentLd, "src/tools/wasm-component-ld", "wasm-component-ld", is_unstable_tool = true, allow_features = "min_specialization";
     UnicodeTableGenerator, "src/tools/unicode-table-generator", "unicode-table-generator";
     FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump";
+    OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"];
+    RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library;
 );
 
 /// These are the submodules that are required for rustbook to work due to
 /// depending on mdbook plugins.
 pub static SUBMODULES_FOR_RUSTBOOK: &[&str] = &["src/doc/book", "src/doc/reference"];
 
-#[derive(Debug, Clone, Hash, PartialEq, Eq)]
-pub struct OptimizedDist {
-    pub compiler: Compiler,
-    pub target: TargetSelection,
-}
-
-impl Step for OptimizedDist {
-    type Output = ToolBuildResult;
-
-    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/opt-dist")
-    }
-
-    fn make_run(run: RunConfig<'_>) {
-        run.builder.ensure(OptimizedDist {
-            compiler: run.builder.compiler(0, run.builder.config.build),
-            target: run.target,
-        });
-    }
-
-    fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
-        // We need to ensure the rustc-perf submodule is initialized when building opt-dist since
-        // the tool requires it to be in place to run.
-        builder.require_submodule("src/tools/rustc-perf", None);
-
-        builder.ensure(ToolBuild {
-            compiler: self.compiler,
-            target: self.target,
-            tool: "opt-dist",
-            mode: Mode::ToolBootstrap,
-            path: "src/tools/opt-dist",
-            source_type: SourceType::InTree,
-            extra_features: Vec::new(),
-            allow_features: "",
-            cargo_args: Vec::new(),
-        })
-    }
-}
-
 /// The [rustc-perf](https://github.com/rust-lang/rustc-perf) benchmark suite, which is added
 /// as a submodule at `src/tools/rustc-perf`.
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
@@ -529,6 +513,7 @@ impl Step for RustcPerf {
             // Only build the collector package, which is used for benchmarking through
             // a CLI.
             cargo_args: vec!["-p".to_string(), "collector".to_string()],
+            artifact_kind: ToolArtifactKind::Binary,
         };
         let res = builder.ensure(tool.clone());
         // We also need to symlink the `rustc-fake` binary to the corresponding directory,
@@ -586,6 +571,7 @@ impl Step for ErrorIndex {
             extra_features: Vec::new(),
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         })
     }
 }
@@ -621,6 +607,7 @@ impl Step for RemoteTestServer {
             extra_features: Vec::new(),
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         })
     }
 }
@@ -725,6 +712,7 @@ impl Step for Rustdoc {
                 extra_features,
                 allow_features: "",
                 cargo_args: Vec::new(),
+                artifact_kind: ToolArtifactKind::Binary,
             });
 
         // don't create a stage0-sysroot/bin directory.
@@ -779,6 +767,7 @@ impl Step for Cargo {
             extra_features: Vec::new(),
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         })
     }
 }
@@ -827,6 +816,7 @@ impl Step for LldWrapper {
             extra_features: Vec::new(),
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         });
 
         let libdir_bin = builder.sysroot_target_bindir(self.target_compiler, target);
@@ -887,6 +877,7 @@ impl Step for RustAnalyzer {
             source_type: SourceType::InTree,
             allow_features: RustAnalyzer::ALLOW_FEATURES,
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         })
     }
 }
@@ -931,6 +922,7 @@ impl Step for RustAnalyzerProcMacroSrv {
             source_type: SourceType::InTree,
             allow_features: RustAnalyzer::ALLOW_FEATURES,
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         });
 
         // Copy `rust-analyzer-proc-macro-srv` to `<sysroot>/libexec/`
@@ -985,6 +977,7 @@ impl Step for LlvmBitcodeLinker {
             extra_features: self.extra_features,
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         });
 
         if tool_result.target_compiler.stage > 0 {
@@ -1164,6 +1157,7 @@ fn run_tool_build_step(
             source_type: SourceType::InTree,
             allow_features: "",
             cargo_args: vec![],
+            artifact_kind: ToolArtifactKind::Binary,
         });
 
     // FIXME: This should just be an if-let-chain, but those are unstable.
@@ -1242,6 +1236,7 @@ impl Step for TestFloatParse {
             extra_features: Vec::new(),
             allow_features: "",
             cargo_args: Vec::new(),
+            artifact_kind: ToolArtifactKind::Binary,
         })
     }
 }

Diff for: src/tools/compiletest/src/common.rs

@@ -190,6 +190,9 @@ pub struct Config {
     /// The cargo executable.
     pub cargo_path: Option<PathBuf>,
 
+    /// Rustc executable used to compile run-make recipes.
+    pub stage0_rustc_path: Option<PathBuf>,
+
     /// The rustdoc executable.
     pub rustdoc_path: Option<PathBuf>,
 

Diff for: src/tools/compiletest/src/lib.rs

@@ -54,6 +54,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
         .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
         .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH")
+        .optopt(
+            "",
+            "stage0-rustc-path",
+            "path to rustc to use for compiling run-make recipes",
+            "PATH",
+        )
         .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
         .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
         .reqopt("", "python", "path to python to use for doc tests", "PATH")
@@ -320,6 +326,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
         rustc_path: opt_path(matches, "rustc-path"),
         cargo_path: matches.opt_str("cargo-path").map(PathBuf::from),
+        stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(PathBuf::from),
         rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
         coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
         python: matches.opt_str("python").unwrap(),