Auto merge of #148671 - ZuseZ4:offload-bootstrap, r=<try>

Offload bootstrap

Author: rust-bors[bot]

bootstrap.example.toml

@@ -106,6 +106,9 @@
 # Whether to build LLVM with support for it's gpu offload runtime.
 #llvm.offload = false
 
+# absolute path to ClangConfig.cmake
+#llvm.offload-clang-dir = ""
+
 # When true, link libstdc++ statically into the rustc_llvm.
 # This is useful if you don't want to use the dynamic version of that
 # library provided by LLVM.

src/bootstrap/configure.py

@@ -120,6 +120,7 @@ def v(*args):
 o("llvm-assertions", "llvm.assertions", "build LLVM with assertions")
 o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme")
 o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support")
+o("llvm-offload-clang-dir", "llvm.offload-clang-dir", "pass the absolute directory of ClangConfig.cmake")
 o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface")
 o("debug-assertions", "rust.debug-assertions", "build with debugging assertions")
 o(

src/bootstrap/src/core/build_steps/compile.rs

@@ -1440,10 +1440,12 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect
     if builder.config.llvm_enzyme {
         cargo.env("LLVM_ENZYME", "1");
     }
+    let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target });
     if builder.config.llvm_offload {
+        builder.ensure(llvm::OmpOffload { target });
         cargo.env("LLVM_OFFLOAD", "1");
     }
-    let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target });
+
     cargo.env("LLVM_CONFIG", &host_llvm_config);
 
     // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script

src/bootstrap/src/core/build_steps/llvm.rs

@@ -454,16 +454,6 @@ impl Step for Llvm {
             enabled_llvm_runtimes.push("compiler-rt");
         }
 
-        // This is an experimental flag, which likely builds more than necessary.
-        // We will optimize it when we get closer to releasing it on nightly.
-        if builder.config.llvm_offload {
-            enabled_llvm_runtimes.push("offload");
-            //FIXME(ZuseZ4): LLVM intends to drop the offload dependency on openmp.
-            //Remove this line once they achieved it.
-            enabled_llvm_runtimes.push("openmp");
-            enabled_llvm_projects.push("compiler-rt");
-        }
-
         if !enabled_llvm_projects.is_empty() {
             enabled_llvm_projects.sort();
             enabled_llvm_projects.dedup();
@@ -896,6 +886,128 @@ fn get_var(var_base: &str, host: &str, target: &str) -> Option<OsString> {
         .or_else(|| env::var_os(var_base))
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct OmpOffload {
+    pub target: TargetSelection,
+}
+
+impl Step for OmpOffload {
+    type Output = PathBuf;
+    const IS_HOST: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/llvm-project/offload")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(OmpOffload { target: run.target });
+    }
+
+    /// Compile OpenMP offload runtimes for `target`.
+    #[allow(unused)]
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        if builder.config.dry_run() {
+            return PathBuf::from("/");
+        }
+        let target = self.target;
+
+        let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target });
+
+        let out_dir = builder.llvm_out(target);
+        static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
+        let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
+            generate_smart_stamp_hash(
+                builder,
+                &builder.config.src.join("src/llvm-project/offload"),
+                builder.offload_info.sha().unwrap_or_default(),
+            )
+        });
+        let stamp = BuildStamp::new(&out_dir).with_prefix("offload").add_stamp(smart_stamp_hash);
+
+        trace!("checking build stamp to see if we need to rebuild offload/openmp artifacts");
+        if stamp.is_up_to_date() {
+            trace!(?out_dir, "offload/openmp build artifacts are up to date");
+            if stamp.stamp().is_empty() {
+                builder.info(
+                    "Could not determine the Offload submodule commit hash. \
+                     Assuming that an Offload rebuild is not necessary.",
+                );
+                builder.info(&format!(
+                    "To force Offload/OpenMP to rebuild, remove the file `{}`",
+                    stamp.path().display()
+                ));
+            }
+            return out_dir;
+        }
+
+        trace!(?target, "(re)building offload/openmp artifacts");
+        builder.info(&format!("Building OpenMP/Offload for {target}"));
+        t!(stamp.remove());
+        let _time = helpers::timeit(builder);
+        t!(fs::create_dir_all(&out_dir));
+
+        builder.config.update_submodule(Path::new("src").join("llvm-project").to_str().unwrap());
+        let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/runtimes/"));
+        configure_cmake(builder, target, &mut cfg, true, LdFlags::default(), &[]);
+
+        // Re-use the same flags as llvm to control the level of debug information
+        // generated by Enzyme.
+        // FIXME(ZuseZ4): Find a nicer way to use Enzyme Debug builds.
+        let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) {
+            (false, _) => "Debug",
+            (true, false) => "Release",
+            (true, true) => "RelWithDebInfo",
+        };
+        trace!(?profile);
+
+        //let cc = if let Some(p) = &builder.build.config.llvm_offload_cc {
+        //    //p.clone()
+        //    builder.cc(target)
+        //} else {
+        //    builder.cc(target)
+        //};
+        //let cxx = if let Some(p) = &builder.build.config.llvm_offload_cxx {
+        //    //p.clone()
+        //    builder.cxx(target).unwrap()
+        //} else {
+        //    builder.cxx(target).unwrap()
+        //};
+        let root = if let Some(p) = &builder.build.config.llvm_root_offload {
+            p.clone()
+        } else {
+            builder.llvm_out(target).join("build")
+        };
+        let clang_dir = if !builder.config.llvm_clang {
+            // We must have an external clang to use.
+            assert!(&builder.build.config.llvm_clang_dir.is_some());
+            builder.build.config.llvm_clang_dir.clone()
+        } else {
+            // No need to specify it, since we use the in-tree clang
+            None
+        };
+
+        cfg.out_dir(&out_dir)
+            .profile(profile)
+            .env("LLVM_CONFIG_REAL", &host_llvm_config)
+            .define("LLVM_ENABLE_ASSERTIONS", "ON")
+            .define("LLVM_ENABLE_RUNTIMES", "openmp;offload")
+            .define("LLVM_INCLUDE_TESTS", "OFF")
+            .define("OFFLOAD_INCLUDE_TESTS", "OFF")
+            //.define("CMAKE_C_COMPILER", cc)
+            //.define("CMAKE_CXX_COMPILER", cxx)
+            .define("OPENMP_STANDALONE_BUILD", "ON")
+            .define("LLVM_ROOT", root)
+            .define("LLVM_DIR", builder.llvm_out(target).join("lib").join("cmake").join("llvm"));
+        if let Some(p) = clang_dir {
+            cfg.define("Clang_DIR", p);
+        }
+        cfg.build();
+
+        t!(stamp.write());
+        out_dir
+    }
+}
+
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Enzyme {
     pub target: TargetSelection,

src/bootstrap/src/core/config/config.rs

@@ -169,6 +169,10 @@ pub struct Config {
     pub llvm_link_jobs: Option<u32>,
     pub llvm_version_suffix: Option<String>,
     pub llvm_use_linker: Option<String>,
+    pub llvm_offload_cc: Option<PathBuf>,
+    pub llvm_offload_cxx: Option<PathBuf>,
+    pub llvm_root_offload: Option<PathBuf>,
+    pub llvm_clang_dir: Option<PathBuf>,
     pub llvm_allow_old_toolchain: bool,
     pub llvm_polly: bool,
     pub llvm_clang: bool,
@@ -290,6 +294,7 @@ pub struct Config {
     pub miri_info: channel::GitInfo,
     pub rustfmt_info: channel::GitInfo,
     pub enzyme_info: channel::GitInfo,
+    pub offload_info: channel::GitInfo,
     pub in_tree_llvm_info: channel::GitInfo,
     pub in_tree_gcc_info: channel::GitInfo,
 
@@ -601,6 +606,10 @@ impl Config {
             ldflags: llvm_ldflags,
             use_libcxx: llvm_use_libcxx,
             use_linker: llvm_use_linker,
+            offload_cc: llvm_offload_cc,
+            offload_cxx: llvm_offload_cxx,
+            root_offload: llvm_root_offload,
+            clang_dir: llvm_clang_dir,
             allow_old_toolchain: llvm_allow_old_toolchain,
             offload: llvm_offload,
             polly: llvm_polly,
@@ -1257,6 +1266,8 @@ impl Config {
         let in_tree_gcc_info = git_info(&exec_ctx, false, &src.join("src/gcc"));
         let in_tree_llvm_info = git_info(&exec_ctx, false, &src.join("src/llvm-project"));
         let enzyme_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/enzyme"));
+        let offload_info =
+            git_info(&exec_ctx, omit_git_hash, &src.join("src/llvm-project/offload"));
         let miri_info = git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/miri"));
         let rust_analyzer_info =
             git_info(&exec_ctx, omit_git_hash, &src.join("src/tools/rust-analyzer"));
@@ -1356,10 +1367,12 @@ impl Config {
             llvm_cflags,
             llvm_clang: llvm_clang.unwrap_or(false),
             llvm_clang_cl,
+            llvm_clang_dir: llvm_clang_dir.map(PathBuf::from),
             llvm_cxxflags,
             llvm_enable_warnings: llvm_enable_warnings.unwrap_or(false),
             llvm_enzyme: llvm_enzyme.unwrap_or(false),
             llvm_experimental_targets,
+            llvm_root_offload: llvm_root_offload.map(PathBuf::from),
             llvm_from_ci,
             llvm_ldflags,
             llvm_libunwind_default: rust_llvm_libunwind
@@ -1374,6 +1387,8 @@ impl Config {
                     .or((!llvm_from_ci && llvm_thin_lto.unwrap_or(false)).then_some(true)),
             ),
             llvm_offload: llvm_offload.unwrap_or(false),
+            llvm_offload_cc: llvm_offload_cc.map(PathBuf::from),
+            llvm_offload_cxx: llvm_offload_cxx.map(PathBuf::from),
             llvm_optimize: llvm_optimize.unwrap_or(true),
             llvm_plugins: llvm_plugin.unwrap_or(false),
             llvm_polly: llvm_polly.unwrap_or(false),
@@ -1396,6 +1411,7 @@ impl Config {
             musl_root: rust_musl_root.map(PathBuf::from),
             ninja_in_file: llvm_ninja.unwrap_or(true),
             nodejs: build_nodejs.map(PathBuf::from),
+            offload_info,
             omit_git_hash,
             on_fail: flags_on_fail,
             optimized_compiler_builtins,

src/bootstrap/src/core/config/toml/llvm.rs

@@ -32,6 +32,10 @@ define_config! {
         ldflags: Option<String> = "ldflags",
         use_libcxx: Option<bool> = "use-libcxx",
         use_linker: Option<String> = "use-linker",
+        offload_cc: Option<String> = "offload-cc",
+        offload_cxx: Option<String> = "offload-cxx",
+        root_offload: Option<String> = "root-offload",
+        clang_dir: Option<String> = "offload-clang-dir",
         allow_old_toolchain: Option<bool> = "allow-old-toolchain",
         offload: Option<bool> = "offload",
         polly: Option<bool> = "polly",
@@ -110,6 +114,10 @@ pub fn check_incompatible_options_for_ci_llvm(
         ldflags,
         use_libcxx,
         use_linker,
+        offload_cc: _,
+        offload_cxx: _,
+        root_offload: _,
+        clang_dir: _,
         allow_old_toolchain,
         offload,
         polly,

src/bootstrap/src/lib.rs

@@ -212,12 +212,15 @@ pub struct Build {
     src: PathBuf,
     out: PathBuf,
     bootstrap_out: PathBuf,
+    //offload_cc: PathBuf,
+    //offload_cxx: PathBuf,
     cargo_info: GitInfo,
     rust_analyzer_info: GitInfo,
     clippy_info: GitInfo,
     miri_info: GitInfo,
     rustfmt_info: GitInfo,
     enzyme_info: GitInfo,
+    offload_info: GitInfo,
     in_tree_llvm_info: GitInfo,
     in_tree_gcc_info: GitInfo,
     local_rebuild: bool,
@@ -476,6 +479,7 @@ impl Build {
         let miri_info = config.miri_info.clone();
         let rustfmt_info = config.rustfmt_info.clone();
         let enzyme_info = config.enzyme_info.clone();
+        let offload_info = config.offload_info.clone();
         let in_tree_llvm_info = config.in_tree_llvm_info.clone();
         let in_tree_gcc_info = config.in_tree_gcc_info.clone();
 
@@ -568,6 +572,7 @@ impl Build {
             miri_info,
             rustfmt_info,
             enzyme_info,
+            offload_info,
             in_tree_llvm_info,
             in_tree_gcc_info,
             cc: HashMap::new(),

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -90,6 +90,8 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
+      --set llvm.offload=true \
+      --set llvm.offload-clang-dir="/rustroot/lib/cmake/clang" \
       --set llvm.libzstd=true \
       --set rust.jemalloc \
       --set rust.bootstrap-override-lld=true \