[TEST] Test out Fjall memory allocator

Cargo.lock

@@ -1319,6 +1319,17 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "fjall"
+version = "0.1.0"
+source = "git+https://github.com/Zoxc/fjall.git#3b34a9ada76834ff9c2b5b4622f40709910ae54b"
+dependencies = [
+ "bitflags 2.5.0",
+ "libc",
+ "sptr",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.28"
@@ -2205,7 +2216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.4",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -3387,7 +3398,7 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 name = "rustc-main"
 version = "0.0.0"
 dependencies = [
- "jemalloc-sys",
+ "fjall",
  "rustc_codegen_ssa",
  "rustc_driver",
  "rustc_driver_impl",
@@ -3777,6 +3788,7 @@ dependencies = [
 name = "rustc_driver"
 version = "0.0.0"
 dependencies = [
+ "fjall",
  "rustc_driver_impl",
 ]
 
@@ -5178,6 +5190,12 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "sptr"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a"
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.0"

compiler/rustc/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 
+fjall = { git = "https://github.com/Zoxc/fjall.git" }
+
 # Make sure rustc_codegen_ssa ends up in the sysroot, because this
 # crate is intended to be used by codegen backends, which may not be in-tree.
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
@@ -20,14 +22,9 @@ rustc_smir = { path = "../rustc_smir" }
 stable_mir = { path = "../stable_mir" }
 # tidy-alphabetical-end
 
-[dependencies.jemalloc-sys]
-version = "0.5.0"
-optional = true
-features = ['unprefixed_malloc_on_supported_platforms']
-
 [features]
 # tidy-alphabetical-start
-jemalloc = ['jemalloc-sys']
+jemalloc = []
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']

compiler/rustc/src/main.rs

@@ -1,4 +1,6 @@
 #![feature(unix_sigpipe)]
+// We need this feature as it changes `dylib` linking behavior and allows us to link to `rustc_driver`.
+#![feature(rustc_private)]
 
 // A note about jemalloc: rustc uses jemalloc when built for CI and
 // distribution. The obvious way to do this is with the `#[global_allocator]`
@@ -34,27 +36,62 @@
 // https://github.com/rust-lang/rust/commit/b90cfc887c31c3e7a9e6d462e2464db1fe506175#diff-43914724af6e464c1da2171e4a9b6c7e607d5bc1203fa95c0ab85be4122605ef
 // for an example of how to do so.
 
+use std::os::raw::{c_char, c_int, c_void};
+
+#[no_mangle]
+unsafe extern "C" fn calloc(items: usize, size: usize) -> *mut c_void {
+    fjall::c::calloc(items, size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn posix_memalign(ptr: *mut *mut c_void, size: usize, align: usize) -> c_int {
+    fjall::c::posix_memalign(ptr, size, align)
+}
+
+#[no_mangle]
+unsafe extern "C" fn aligned_alloc(size: usize, align: usize) -> *mut c_void {
+    fjall::c::aligned_alloc(size, align)
+}
+
+#[no_mangle]
+unsafe extern "C" fn malloc(size: usize) -> *mut c_void {
+    fjall::c::malloc(size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn realloc(ptr: *mut c_void, size: usize) -> *mut c_void {
+    fjall::c::realloc(ptr, size)
+}
+
+#[no_mangle]
+unsafe extern "C" fn free(ptr: *mut c_void) {
+    fjall::c::free(ptr);
+}
+
+#[no_mangle]
+unsafe extern "C" fn strdup(ptr: *const c_char) -> *mut c_char {
+    fjall::c::strdup(ptr)
+}
+
 #[unix_sigpipe = "sig_dfl"]
 fn main() {
     // See the comment at the top of this file for an explanation of this.
-    #[cfg(feature = "jemalloc-sys")]
     {
-        use std::os::raw::{c_int, c_void};
-
         #[used]
-        static _F1: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::calloc;
+        static _F1: unsafe extern "C" fn(usize, usize) -> *mut c_void = calloc;
         #[used]
-        static _F2: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int =
-            jemalloc_sys::posix_memalign;
+        static _F2: unsafe extern "C" fn(*mut *mut c_void, usize, usize) -> c_int = posix_memalign;
         #[used]
-        static _F3: unsafe extern "C" fn(usize, usize) -> *mut c_void = jemalloc_sys::aligned_alloc;
+        static _F3: unsafe extern "C" fn(usize, usize) -> *mut c_void = aligned_alloc;
         #[used]
-        static _F4: unsafe extern "C" fn(usize) -> *mut c_void = jemalloc_sys::malloc;
+        static _F4: unsafe extern "C" fn(usize) -> *mut c_void = malloc;
         #[used]
-        static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = jemalloc_sys::realloc;
+        static _F5: unsafe extern "C" fn(*mut c_void, usize) -> *mut c_void = realloc;
         #[used]
-        static _F6: unsafe extern "C" fn(*mut c_void) = jemalloc_sys::free;
-
+        static _F6: unsafe extern "C" fn(*mut c_void) = free;
+        #[used]
+        static _F7: unsafe extern "C" fn(*const c_char) -> *mut c_char = strdup;
+        /*
         // On OSX, jemalloc doesn't directly override malloc/free, but instead
         // registers itself with the allocator's zone APIs in a ctor. However,
         // the linker doesn't seem to consider ctors as "used" when statically
@@ -67,7 +104,7 @@ fn main() {
 
             #[used]
             static _F7: unsafe extern "C" fn() = _rjem_je_zone_register;
-        }
+        }*/
     }
 
     rustc_driver::main()

compiler/rustc_driver/Cargo.toml

@@ -8,5 +8,6 @@ crate-type = ["dylib"]
 
 [dependencies]
 # tidy-alphabetical-start
+fjall = { git = "https://github.com/Zoxc/fjall.git" }
 rustc_driver_impl = { path = "../rustc_driver_impl" }
 # tidy-alphabetical-end

compiler/rustc_driver/src/lib.rs

@@ -5,4 +5,8 @@
 #![feature(rustdoc_internals)]
 #![doc(rust_logo)]
 
+#[cfg(not(bootstrap))]
+#[global_allocator]
+static GLOBAL: fjall::Alloc = fjall::Alloc;
+
 pub use rustc_driver_impl::*;

compiler/rustc_metadata/messages.ftl

@@ -41,6 +41,9 @@ metadata_crate_dep_multiple =
 metadata_crate_dep_not_static =
     `{$crate_name}` was unavailable as a static crate, preventing fully static linking
 
+metadata_crate_dep_rustc_driver =
+    `feature(rustc_private)` is needed to link to the compiler's `rustc_driver` library
+
 metadata_crate_location_unknown_type =
     extern location for {$crate_name} is of an unknown type: {$path}
 

compiler/rustc_metadata/src/dependency_format.rs

@@ -54,16 +54,18 @@
 use crate::creader::CStore;
 use crate::errors::{
     BadPanicStrategy, CrateDepMultiple, IncompatiblePanicInDropStrategy, LibRequired,
-    NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcLibRequired, TwoPanicRuntimes,
+    NonStaticCrateDep, RequiredPanicStrategy, RlibRequired, RustcDriverHelp, RustcLibRequired,
+    TwoPanicRuntimes,
 };
 
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::CrateNum;
 use rustc_middle::middle::dependency_format::{Dependencies, DependencyList, Linkage};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::CrateType;
 use rustc_session::cstore::CrateDepKind;
 use rustc_session::cstore::LinkagePreference::{self, RequireDynamic, RequireStatic};
+use rustc_span::sym;
 
 pub(crate) fn calculate(tcx: TyCtxt<'_>) -> Dependencies {
     tcx.crate_types()
@@ -158,25 +160,49 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         Linkage::Dynamic | Linkage::IncludedFromDylib => {}
     }
 
+    let all_dylibs = || {
+        tcx.crates(()).iter().filter(|&&cnum| {
+            !tcx.dep_kind(cnum).macros_only() && tcx.used_crate_source(cnum).dylib.is_some()
+        })
+    };
+
+    let mut upstream_in_dylibs = FxHashSet::default();
+
+    if tcx.features().rustc_private {
+        // We need this to prevent users of `rustc_driver` from linking to dynamically to `std`
+        // which does not work as `std` is also statically linked into `rustc_driver`.
+
+        // Find all libraries statically linked to upstream dylibs.
+        for &cnum in all_dylibs() {
+            let deps = tcx.dylib_dependency_formats(cnum);
+            for &(depnum, style) in deps.iter() {
+                if let RequireStatic = style {
+                    upstream_in_dylibs.insert(depnum);
+                }
+            }
+        }
+    }
+
     let mut formats = FxHashMap::default();
 
     // Sweep all crates for found dylibs. Add all dylibs, as well as their
     // dependencies, ensuring there are no conflicts. The only valid case for a
     // dependency to be relied upon twice is for both cases to rely on a dylib.
-    for &cnum in tcx.crates(()).iter() {
-        if tcx.dep_kind(cnum).macros_only() {
+    for &cnum in all_dylibs() {
+        if upstream_in_dylibs.contains(&cnum) {
+            info!("skipping dylib: {}", tcx.crate_name(cnum));
+            // If this dylib is also available statically linked to another dylib
+            // we try to use that instead.
             continue;
         }
+
         let name = tcx.crate_name(cnum);
-        let src = tcx.used_crate_source(cnum);
-        if src.dylib.is_some() {
-            info!("adding dylib: {}", name);
-            add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static);
-            let deps = tcx.dylib_dependency_formats(cnum);
-            for &(depnum, style) in deps.iter() {
-                info!("adding {:?}: {}", style, tcx.crate_name(depnum));
-                add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static);
-            }
+        info!("adding dylib: {}", name);
+        add_library(tcx, cnum, RequireDynamic, &mut formats, &mut unavailable_as_static);
+        let deps = tcx.dylib_dependency_formats(cnum);
+        for &(depnum, style) in deps.iter() {
+            info!("adding {:?}: {}", style, tcx.crate_name(depnum));
+            add_library(tcx, depnum, style, &mut formats, &mut unavailable_as_static);
         }
     }
 
@@ -266,12 +292,15 @@ fn add_library(
             // This error is probably a little obscure, but I imagine that it
             // can be refined over time.
             if link2 != link || link == RequireStatic {
+                let linking_to_rustc_driver = tcx.sess.psess.unstable_features.is_nightly_build()
+                    && tcx.crates(()).iter().any(|&cnum| tcx.crate_name(cnum) == sym::rustc_driver);
                 tcx.dcx().emit_err(CrateDepMultiple {
                     crate_name: tcx.crate_name(cnum),
                     non_static_deps: unavailable_as_static
                         .drain(..)
                         .map(|cnum| NonStaticCrateDep { crate_name: tcx.crate_name(cnum) })
                         .collect(),
+                    rustc_driver_help: linking_to_rustc_driver.then_some(RustcDriverHelp),
                 });
             }
         }

compiler/rustc_metadata/src/errors.rs

@@ -40,6 +40,8 @@ pub struct CrateDepMultiple {
     pub crate_name: Symbol,
     #[subdiagnostic]
     pub non_static_deps: Vec<NonStaticCrateDep>,
+    #[subdiagnostic]
+    pub rustc_driver_help: Option<RustcDriverHelp>,
 }
 
 #[derive(Subdiagnostic)]
@@ -48,6 +50,10 @@ pub struct NonStaticCrateDep {
     pub crate_name: Symbol,
 }
 
+#[derive(Subdiagnostic)]
+#[help(metadata_crate_dep_rustc_driver)]
+pub struct RustcDriverHelp;
+
 #[derive(Diagnostic)]
 #[diag(metadata_two_panic_runtimes)]
 pub struct TwoPanicRuntimes {

compiler/rustc_span/src/symbol.rs

@@ -1526,6 +1526,7 @@ symbols! {
         rustc_dirty,
         rustc_do_not_const_check,
         rustc_doc_primitive,
+        rustc_driver,
         rustc_dummy,
         rustc_dump_env_program_clauses,
         rustc_dump_program_clauses,

src/bootstrap/src/bin/rustc.rs

@@ -91,6 +91,23 @@ fn main() {
         rustc_real
     };
 
+    // Get the name of the crate we're compiling, if any.
+    let crate_name = arg("--crate-name");
+
+    // We want everything statically linked into `rustc_driver`, so remove `-C prefer-dynamic`
+    if crate_name == Some("rustc_driver") && stage != "0" {
+        // Remove `-C prefer-dynamic` to link `std` statically into `rustc_driver`
+        if let Some(pos) = args.iter().enumerate().position(|(i, a)| {
+            a == "-C" && args.get(i + 1).map(|a| a == "prefer-dynamic").unwrap_or(false)
+        }) {
+            args.remove(pos);
+            args.remove(pos);
+        }
+        if let Some(pos) = args.iter().position(|a| a == "-Cprefer-dynamic") {
+            args.remove(pos);
+        }
+    }
+
     let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER_REAL") {
         let mut cmd = Command::new(wrapper);
         cmd.arg(rustc_driver);
@@ -100,9 +117,6 @@ fn main() {
     };
     cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
 
-    // Get the name of the crate we're compiling, if any.
-    let crate_name = arg("--crate-name");
-
     if let Some(crate_name) = crate_name {
         if let Some(target) = env::var_os("RUSTC_TIME") {
             if target == "all"

src/bootstrap/src/core/build_steps/compile.rs

@@ -1794,7 +1794,12 @@ impl Step for Assemble {
         let src_libdir = builder.sysroot_libdir(build_compiler, host);
         for f in builder.read_dir(&src_libdir) {
             let filename = f.file_name().into_string().unwrap();
-            if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename)
+            let can_be_rustc_dep = filename.starts_with("rustc_driver-")
+                || filename.starts_with("librustc_driver-")
+                || build_compiler.stage == 0;
+            if can_be_rustc_dep
+                && (is_dylib(&filename) || is_debug_info(&filename))
+                && !proc_macros.contains(&filename)
             {
                 builder.copy_link(&f.path(), &rustc_libdir.join(&filename));
             }

src/bootstrap/src/core/builder.rs

@@ -2031,7 +2031,7 @@ impl<'a> Builder<'a> {
         // When we build Rust dylibs they're all intended for intermediate
         // usage, so make sure we pass the -Cprefer-dynamic flag instead of
         // linking all deps statically into the dylib.
-        if matches!(mode, Mode::Std | Mode::Rustc) {
+        if matches!(mode, Mode::Std) {
             rustflags.arg("-Cprefer-dynamic");
         }
 

src/ci/scripts/run-build-from-ci.sh

@@ -7,6 +7,8 @@ IFS=$'\n\t'
 
 source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
 
+export RUSTC_BACKTRACE_ON_ICE=1
+export RUST_BACKTRACE=1
 export CI="true"
 export SRC=.
 

src/tools/clippy/src/main.rs

@@ -1,3 +1,6 @@
+// We need this feature as it changes `dylib` linking behavior and allows us to link to
+// `rustc_driver`.
+#![feature(rustc_private)]
 #![cfg_attr(feature = "deny-warnings", deny(warnings))]
 // warn on lints, that are included in `rust-lang/rust`s bootstrap
 #![warn(rust_2018_idioms, unused_lifetimes)]