Rustbuild: enable -Zsplit-metadata for stage != 0

Author: bjorn3

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -33,7 +33,7 @@ use rustc_span::{
 };
 use tracing::{debug, instrument, trace};
 
-use crate::errors::{FailCreateFileEncoder, FailWriteFile};
+use crate::errors::{FailCreateFileEncoder, FailWriteFile, FailedCreateFile};
 use crate::rmeta::*;
 
 pub(super) struct EncodeContext<'a, 'tcx> {
@@ -2340,6 +2340,10 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: &Path) {
             });
             header.position.get()
         });
+    } else {
+        std::fs::File::create(&ref_path).unwrap_or_else(|err| {
+            tcx.dcx().emit_fatal(FailedCreateFile { filename: &ref_path, err });
+        });
     }
 }
 

src/bootstrap/src/bin/rustc.rs

@@ -130,6 +130,12 @@ fn main() {
         }
     }
 
+    if orig_args.iter().any(|arg| arg == "-Zsplit-metadata")
+        && orig_args.windows(2).any(|args| args[0] == "--crate-type" && args[1] == "dylib")
+    {
+        cmd.arg("--emit").arg("metadata");
+    }
+
     // Print backtrace in case of ICE
     if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() {
         cmd.env("RUST_BACKTRACE", "1");

src/bootstrap/src/core/build_steps/compile.rs

@@ -2050,14 +2050,12 @@ pub fn run_cargo(
                 || filename.ends_with(".a")
                 || is_debug_info(&filename)
                 || is_dylib(Path::new(&*filename))
+                || filename.ends_with(".rmeta")
             {
-                // Always keep native libraries, rust dylibs and debuginfo
+                // Always keep native libraries, rust dylibs, debuginfo and crate metadata
                 keep = true;
             }
-            if is_check && filename.ends_with(".rmeta") {
-                // During check builds we need to keep crate metadata
-                keep = true;
-            } else if rlib_only_metadata {
+            if !is_check && rlib_only_metadata {
                 if filename.contains("jemalloc_sys")
                     || filename.contains("rustc_smir")
                     || filename.contains("stable_mir")
@@ -2069,7 +2067,6 @@ pub fn run_cargo(
                     // Distribute the rest of the rustc crates as rmeta files only to reduce
                     // the tarball sizes by about 50%. The object files are linked into
                     // librustc_driver.so, so it is still possible to link against them.
-                    keep |= filename.ends_with(".rmeta");
                 }
             } else {
                 // In all other cases keep all rlibs
@@ -2115,7 +2112,12 @@ pub fn run_cargo(
             let file_stem = parts.next().unwrap().to_owned();
             let extension = parts.next().unwrap().to_owned();
 
-            toplevel.push((file_stem, extension, expected_len));
+            if extension == "so" || extension == "dylib" {
+                // FIXME workaround for the fact that cargo doesn't understand `-Zsplit-metadata`
+                toplevel.push((file_stem.clone(), "rmeta".to_owned(), None));
+            }
+
+            toplevel.push((file_stem, extension, Some(expected_len)));
         }
     });
 
@@ -2136,7 +2138,7 @@ pub fn run_cargo(
         .collect::<Vec<_>>();
     for (prefix, extension, expected_len) in toplevel {
         let candidates = contents.iter().filter(|&(_, filename, meta)| {
-            meta.len() == expected_len
+            expected_len.is_none_or(|expected_len| meta.len() == expected_len)
                 && filename
                     .strip_prefix(&prefix[..])
                     .map(|s| s.starts_with('-') && s.ends_with(&extension[..]))
@@ -2147,6 +2149,7 @@ pub fn run_cargo(
         });
         let path_to_add = match max {
             Some(triple) => triple.0.to_str().unwrap(),
+            None if extension == "rmeta" => continue, // cfg(not(bootstrap)) remove this once -Zsplit-metadata is passed for all stages
             None => panic!("no output generated for {prefix:?} {extension:?}"),
         };
         if is_dylib(Path::new(path_to_add)) {

src/bootstrap/src/core/builder/cargo.rs

@@ -626,6 +626,18 @@ impl Builder<'_> {
         hostflags.arg("-Zunstable-options");
         hostflags.arg("--check-cfg=cfg(bootstrap)");
 
+        match mode {
+            Mode::Std | Mode::Rustc | Mode::Codegen => {
+                // cfg(bootstrap) unconditionally pass this once the bootstrap compiler understands it
+                if stage != 0 {
+                    // FIXME remove once cargo enables this by default
+                    rustflags.arg("-Zsplit-metadata");
+                }
+            }
+            // Tools may not expect to be compiled with -Zsplit-metadata
+            Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => {}
+        }
+
         // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`,
         // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See
         // #71458.

tests/ui/duplicate_entry_error.rs

@@ -1,4 +1,4 @@
-//@ normalize-stderr: "loaded from .*libstd-.*.rlib" -> "loaded from SYSROOT/libstd-*.rlib"
+//@ normalize-stderr: "loaded from .*libstd-.*.rmeta" -> "loaded from SYSROOT/libstd-*.rmeta"
 // note-pattern: first defined in crate `std`.
 
 // Test for issue #31788 and E0152
@@ -11,7 +11,7 @@ use core::panic::PanicInfo;
 
 #[lang = "panic_impl"]
 fn panic_impl(info: &PanicInfo) -> ! {
-//~^ ERROR: found duplicate lang item `panic_impl`
+    //~^ ERROR: found duplicate lang item `panic_impl`
     loop {}
 }
 

tests/ui/duplicate_entry_error.stderr

@@ -8,7 +8,7 @@ LL | | }
    | |_^
    |
    = note: the lang item is first defined in crate `std` (which `duplicate_entry_error` depends on)
-   = note: first definition in `std` loaded from SYSROOT/libstd-*.rlib
+   = note: first definition in `std` loaded from SYSROOT/libstd-*.rmeta
    = note: second definition in the local crate (`duplicate_entry_error`)
 
 error: aborting due to 1 previous error

tests/ui/error-codes/E0152.rs

@@ -1,8 +1,7 @@
-//@ normalize-stderr: "loaded from .*liballoc-.*.rlib" -> "loaded from SYSROOT/liballoc-*.rlib"
+//@ normalize-stderr: "loaded from .*liballoc-.*.rmeta" -> "loaded from SYSROOT/liballoc-*.rmeta"
 #![feature(lang_items)]
 
 #[lang = "owned_box"]
 struct Foo<T>(T); //~ ERROR E0152
 
-fn main() {
-}
+fn main() {}

tests/ui/error-codes/E0152.stderr

@@ -5,7 +5,7 @@ LL | struct Foo<T>(T);
    | ^^^^^^^^^^^^^^^^^
    |
    = note: the lang item is first defined in crate `alloc` (which `std` depends on)
-   = note: first definition in `alloc` loaded from SYSROOT/liballoc-*.rlib
+   = note: first definition in `alloc` loaded from SYSROOT/liballoc-*.rmeta
    = note: second definition in the local crate (`E0152`)
 
 error: aborting due to 1 previous error

tests/ui/lang-items/duplicate.rs

@@ -1,4 +1,4 @@
-//@ normalize-stderr: "loaded from .*libcore-.*.rlib" -> "loaded from SYSROOT/libcore-*.rlib"
+//@ normalize-stderr: "loaded from .*libcore-.*.rmeta" -> "loaded from SYSROOT/libcore-*.rmeta"
 #![feature(lang_items)]
 
 #[lang = "sized"]

tests/ui/lang-items/duplicate.stderr

@@ -5,7 +5,7 @@ LL | trait Sized {}
    | ^^^^^^^^^^^^^^
    |
    = note: the lang item is first defined in crate `core` (which `std` depends on)
-   = note: first definition in `core` loaded from SYSROOT/libcore-*.rlib
+   = note: first definition in `core` loaded from SYSROOT/libcore-*.rmeta
    = note: second definition in the local crate (`duplicate`)
 
 error: aborting due to 1 previous error

tests/ui/panic-handler/panic-handler-std.rs

@@ -1,4 +1,4 @@
-//@ normalize-stderr: "loaded from .*libstd-.*.rlib" -> "loaded from SYSROOT/libstd-*.rlib"
+//@ normalize-stderr: "loaded from .*libstd-.*.rmeta" -> "loaded from SYSROOT/libstd-*.rmeta"
 //@ error-pattern: found duplicate lang item `panic_impl`
 
 extern crate core;

tests/ui/panic-handler/panic-handler-std.stderr

@@ -7,7 +7,7 @@ LL | | }
    | |_^
    |
    = note: the lang item is first defined in crate `std` (which `panic_handler_std` depends on)
-   = note: first definition in `std` loaded from SYSROOT/libstd-*.rlib
+   = note: first definition in `std` loaded from SYSROOT/libstd-*.rmeta
    = note: second definition in the local crate (`panic_handler_std`)
 
 error: aborting due to 1 previous error