Use download-rustc = "if-unchanged" in x86_64-gnu

.github/workflows/ci.yml

@@ -37,6 +37,7 @@ jobs:
   pr:
     name: "PR - ${{ matrix.name }}"
     env:
+      DOWNLOAD_RUSTC: 1
       PR_CI_JOB: 1
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
@@ -55,7 +56,7 @@ jobs:
           - name: mingw-check-tidy
             os: ubuntu-20.04-16core-64gb
             env: {}
-          - name: x86_64-gnu-llvm-14
+          - name: x86_64-gnu
             os: ubuntu-20.04-16core-64gb
             env: {}
           - name: x86_64-gnu-tools
@@ -268,8 +269,9 @@ jobs:
             os: ubuntu-20.04-8core-32gb
             env: {}
           - name: x86_64-gnu
+            env:
+              DOWNLOAD_RUSTC: 1
             os: ubuntu-20.04-4core-16gb
-            env: {}
           - name: x86_64-gnu-stable
             env:
               IMAGE: x86_64-gnu

src/bootstrap/compile.rs

@@ -645,8 +645,8 @@ fn cp_rustc_component_to_ci_sysroot(
     contents: Vec<String>,
 ) {
     let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false });
+    let ci_rustc_dir = builder.config.ci_rustc_dir();
 
-    let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc");
     for file in contents {
         let src = ci_rustc_dir.join(&file);
         let dst = sysroot.join(file);
@@ -1381,7 +1381,7 @@ impl Step for Sysroot {
                 // FIXME: this is wrong when compiler.host != build, but we don't support that today
                 OsStr::new(std::env::consts::DLL_EXTENSION),
             ];
-            let ci_rustc_dir = builder.ci_rustc_dir(builder.config.build);
+            let ci_rustc_dir = builder.config.ci_rustc_dir();
             builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| {
                 if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) {
                     return true;

src/bootstrap/config.rs

@@ -23,6 +23,7 @@ use crate::channel::{self, GitInfo};
 pub use crate::flags::Subcommand;
 use crate::flags::{Color, Flags, Warnings};
 use crate::util::{exe, output, t};
+use build_helper::ci::CiEnv;
 use build_helper::detail_exit_macro;
 use once_cell::sync::OnceCell;
 use semver::Version;
@@ -1328,6 +1329,25 @@ impl Config {
         let mut omit_git_hash = None;
 
         if let Some(rust) = toml.rust {
+            set(&mut config.channel, rust.channel);
+
+            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
+            // This list is incomplete, please help by expanding it!
+            if config.download_rustc_commit.is_some() {
+                // We need the channel used by the downloaded compiler to match the one we set for rustdoc;
+                // otherwise rustdoc-ui tests break.
+                let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel")));
+                let ci_channel = ci_channel.trim_end();
+                if config.channel != ci_channel
+                    && !(config.channel == "dev" && ci_channel == "nightly")
+                {
+                    panic!(
+                        "setting rust.channel={} is incompatible with download-rustc",
+                        config.channel
+                    );
+                }
+            }
+
             debug = rust.debug;
             debug_assertions = rust.debug_assertions;
             debug_assertions_std = rust.debug_assertions_std;
@@ -1339,6 +1359,7 @@ impl Config {
             debuginfo_level_std = rust.debuginfo_level_std;
             debuginfo_level_tools = rust.debuginfo_level_tools;
             debuginfo_level_tests = rust.debuginfo_level_tests;
+
             config.rust_split_debuginfo = rust
                 .split_debuginfo
                 .as_deref()
@@ -1354,7 +1375,6 @@ impl Config {
             set(&mut config.jemalloc, rust.jemalloc);
             set(&mut config.test_compare_mode, rust.test_compare_mode);
             set(&mut config.backtrace, rust.backtrace);
-            set(&mut config.channel, rust.channel);
             config.description = rust.description;
             set(&mut config.rust_dist_src, rust.dist_src);
             set(&mut config.verbose_tests, rust.verbose_tests);
@@ -1395,8 +1415,6 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
-
             config.rust_lto = rust
                 .lto
                 .as_deref()
@@ -1508,6 +1526,11 @@ impl Config {
                 let mut target = Target::from_triple(&triple);
 
                 if let Some(ref s) = cfg.llvm_config {
+                    if config.download_rustc_commit.is_some() && triple == &*config.build.triple {
+                        panic!(
+                            "setting llvm_config for the host is incompatible with download-rustc"
+                        );
+                    }
                     target.llvm_config = Some(config.src.join(s));
                 }
                 target.llvm_has_rust_patches = cfg.llvm_has_rust_patches;
@@ -1778,6 +1801,13 @@ impl Config {
         self.out.join(&*self.build.triple).join("ci-llvm")
     }
 
+    /// Directory where the extracted `rustc-dev` component is stored.
+    pub(crate) fn ci_rustc_dir(&self) -> PathBuf {
+        // assert!(self.download_rustc_commit.is_some());
+        assert!(self.download_rustc());
+        self.out.join(self.build.triple).join("ci-rustc")
+    }
+
     /// Determine whether llvm should be linked dynamically.
     ///
     /// If `false`, llvm should be linked statically.
@@ -1813,11 +1843,11 @@ impl Config {
         self.download_rustc_commit().is_some()
     }
 
-    pub(crate) fn download_rustc_commit(&self) -> Option<&'static str> {
+    pub(crate) fn download_rustc_commit(&self) -> Option<&str> {
         static DOWNLOAD_RUSTC: OnceCell<Option<String>> = OnceCell::new();
         if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() {
             // avoid trying to actually download the commit
-            return None;
+            return self.download_rustc_commit.as_deref();
         }
 
         DOWNLOAD_RUSTC
@@ -1956,11 +1986,15 @@ impl Config {
 
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
+
+        // If we are running in CI, the current commit will never have artifacts already built.
+        let tip_commit = if CiEnv::is_ci() { "HEAD^" } else { "HEAD" };
+        // FIXME(#113250): This may not be the best way to find the right commit; it certainly doesn't match what we do for LLVM.
         let merge_base = output(
             self.git()
                 .arg("rev-list")
                 .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email))
-                .args(&["-n1", "--first-parent", "HEAD"]),
+                .args(&["-n1", "--first-parent", tip_commit]),
         );
         let commit = merge_base.trim_end();
         if commit.is_empty() {

src/bootstrap/download.rs

@@ -402,7 +402,11 @@ impl Config {
 
     fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
         assert!(self.download_rustc());
-        let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc");
+        if self.dry_run() {
+            return vec![];
+        }
+
+        let ci_rustc_dir = self.ci_rustc_dir();
         let stamp_file = ci_rustc_dir.join(stamp_file);
         let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
         t!(BufReader::new(contents_file).lines().collect())
@@ -419,7 +423,7 @@ impl Config {
         self.download_toolchain(
             &version,
             "ci-rustc",
-            commit,
+            &format!("{commit}-{}", self.llvm_assertions),
             &extra_components,
             Self::download_ci_component,
         );
@@ -495,8 +499,15 @@ impl Config {
 
     /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
     // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
-    fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) {
-        Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
+    fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) {
+        Self::download_component(
+            self,
+            DownloadSource::CI,
+            filename,
+            prefix,
+            commit_with_assertions,
+            "ci-rustc",
+        )
     }
 
     fn download_component(
@@ -516,11 +527,18 @@ impl Config {
         let bin_root = self.out.join(self.build.triple).join(destination);
         let tarball = cache_dir.join(&filename);
         let (base_url, url, should_verify) = match mode {
-            DownloadSource::CI => (
-                self.stage0_metadata.config.artifacts_server.clone(),
-                format!("{key}/{filename}"),
-                false,
-            ),
+            DownloadSource::CI => {
+                let dist_server = if self.llvm_assertions {
+                    self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone()
+                } else {
+                    self.stage0_metadata.config.artifacts_server.clone()
+                };
+                let url = format!(
+                    "{}/{filename}",
+                    key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap()
+                );
+                (dist_server, url, false)
+            }
             DownloadSource::Dist => {
                 let dist_server = env::var("RUSTUP_DIST_SERVER")
                     .unwrap_or(self.stage0_metadata.config.dist_server.to_string());

src/bootstrap/lib.rs

@@ -822,11 +822,6 @@ impl Build {
         self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir())
     }
 
-    /// Directory where the extracted `rustc-dev` component is stored.
-    fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf {
-        self.out.join(&*target.triple).join("ci-rustc")
-    }
-
     /// Root output directory for LLVM compiled for `target`
     ///
     /// Note that if LLVM is configured externally then the directory returned

src/bootstrap/test.rs

@@ -2348,7 +2348,7 @@ impl Step for Crate {
                 // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`,
                 // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`.
                 // Override it.
-                if builder.download_rustc() {
+                if builder.download_rustc() && compiler.stage > 0 {
                     let sysroot = builder
                         .out
                         .join(compiler.host.triple)

src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile

@@ -4,7 +4,6 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
-  gcc-multilib \
   make \
   ninja-build \
   file \
@@ -26,11 +25,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   mingw-w64 \
   && rm -rf /var/lib/apt/lists/*
 
-# Install powershell (universal package) so we can test x.ps1 on Linux
-RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \
-    dpkg -i powershell.deb && \
-    rm -f powershell.deb
-
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
@@ -49,6 +43,4 @@ ENV RUST_CONFIGURE_ARGS \
       --enable-llvm-link-shared \
       --set rust.thin-lto-import-instr-limit=10
 
-COPY host-x86_64/x86_64-gnu-llvm-14/script.sh /tmp/
-
-ENV SCRIPT /tmp/script.sh
+ENV SCRIPT ../x.py --stage 2 test

src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:20.04
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
   g++ \
+  gcc-multilib \
   make \
   ninja-build \
   file \
@@ -19,6 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   mingw-w64 \
   && rm -rf /var/lib/apt/lists/*
 
+# Install powershell (universal package) so we can test x.ps1 on Linux
+RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \
+    dpkg -i powershell.deb && \
+    rm -f powershell.deb
+
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
@@ -27,4 +33,7 @@ ENV RUST_CONFIGURE_ARGS \
  --enable-sanitizers \
  --enable-profiler \
  --enable-compiler-docs
-ENV SCRIPT python3 ../x.py --stage 2 test
+
+COPY host-x86_64/x86_64-gnu/script.sh /tmp/
+
+ENV SCRIPT /tmp/script.sh

src/ci/docker/run.sh

@@ -254,6 +254,7 @@ docker \
   --env DEPLOY \
   --env DEPLOY_ALT \
   --env CI \
+  --env DOWNLOAD_RUSTC \
   --env GITHUB_ACTIONS \
   --env GITHUB_REF \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \

src/ci/github-actions/ci.yml

@@ -310,6 +310,7 @@ jobs:
     <<: *base-ci-job
     name: PR - ${{ matrix.name }}
     env:
+      DOWNLOAD_RUSTC: 1
       <<: [*shared-ci-variables, *public-variables]
       PR_CI_JOB: 1
     if: github.event_name == 'pull_request'
@@ -323,7 +324,7 @@ jobs:
           - name: mingw-check-tidy
             <<: *job-linux-16c
 
-          - name: x86_64-gnu-llvm-14
+          - name: x86_64-gnu
             <<: *job-linux-16c
 
           - name: x86_64-gnu-tools
@@ -433,6 +434,8 @@ jobs:
             <<: *job-linux-8c
 
           - name: x86_64-gnu
+            env:
+              DOWNLOAD_RUSTC: 1
             <<: *job-linux-4c
 
           # This job ensures commits landing on nightly still pass the full

src/ci/run.sh

@@ -103,6 +103,11 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi
+
+  if [ "$DOWNLOAD_RUSTC" = 1 ]; then
+    echo "error: DOWNLOAD_RUSTC should not be set in dist builders!" >&2
+    exit 1
+  fi
 else
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
@@ -121,6 +126,10 @@ else
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
   fi
 
+  if [ "$DOWNLOAD_RUSTC" = 1 ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
+  fi
+
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
   # We enable this for non-dist builders, since those aren't trying to produce

tests/ui-fulldeps/missing-rustc-driver-error.rs

@@ -3,6 +3,8 @@
 // compile-flags: --emit link
 // The exactly list of required crates depends on the target. as such only test Unix targets.
 // only-unix
+// normalize-stderr-test ".*crate .* required.*\n\n" -> ""
+// normalize-stderr-test: "aborting due to [0-9]+" -> "aborting due to NUMBER"
 
 #![feature(rustc_private)]
 

tests/ui-fulldeps/missing-rustc-driver-error.stderr

@@ -2,15 +2,5 @@ error: crate `rustc_serialize` required to be available in rlib format, but was
    |
    = help: try adding `extern crate rustc_driver;` at the top level of this crate
 
-error: crate `smallvec` required to be available in rlib format, but was not found in this form
-
-error: crate `thin_vec` required to be available in rlib format, but was not found in this form
-
-error: crate `indexmap` required to be available in rlib format, but was not found in this form
-
-error: crate `hashbrown` required to be available in rlib format, but was not found in this form
-
-error: crate `equivalent` required to be available in rlib format, but was not found in this form
-
-error: aborting due to 6 previous errors
+error: aborting due to NUMBER previous errors