diff --git a/lib/MPIPreferences/src/parse_cray_cc.jl b/lib/MPIPreferences/src/parse_cray_cc.jl index d1eae42bc..8922d294d 100644 --- a/lib/MPIPreferences/src/parse_cray_cc.jl +++ b/lib/MPIPreferences/src/parse_cray_cc.jl @@ -64,7 +64,14 @@ function other_libs(libs) end function analyze_cray_cc() - cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"])) + opts_file = get(ENV, "JULIA_MPI_CC_OPTS_FILE", "") + cray_opts = "" + if ! isempty(opts_file) + @warn "Loading Cray parameters from file at: JULIA_MPI_CC_OPTS_FILE=$(opts_file)" + cray_opts = readchomp(opts_file) + else + cray_opts = readchomp(Cmd(["cc", "--cray-print-opts=all"])) + end ld_paths = SubString{String}[] libs = SubString{String}[] diff --git a/lib/MPIPreferences/test/Project.toml b/lib/MPIPreferences/test/Project.toml new file mode 100644 index 000000000..cfd31377a --- /dev/null +++ b/lib/MPIPreferences/test/Project.toml @@ -0,0 +1,5 @@ +[deps] +MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/lib/MPIPreferences/test/runtests.jl b/lib/MPIPreferences/test/runtests.jl new file mode 100644 index 000000000..72c7e895b --- /dev/null +++ b/lib/MPIPreferences/test/runtests.jl @@ -0,0 +1,56 @@ +using Pkg, Test, MPIPreferences + +project_path = dirname(Pkg.project().path) +test_dir = @__DIR__ +test_data = joinpath(@__DIR__, "test_data") + +cmd(f; a="") = `$(Base.julia_cmd()) --startup-file=no --project=$(project_path) $(joinpath(test_dir, f)) $(a)` + +function test_generated_prefs(suffix) + MPIPreferences.use_system_binary(vendor="cray"; mpiexec="srun") + + local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml") + local_pref_2 = joinpath(project_path, "LocalPreferences.toml") + pref_file = isfile(local_pref_1) ? local_pref_1 : local_pref_2 + + genereated_settings = readchomp(pref_file) + expected_settings = readchomp( + joinpath(test_data, "LocalPreferences_$(suffix).toml") + ) + + @test genereated_settings == expected_settings + r = run(cmd("test_libmpi.jl")) + @test success(r) +end + +function test_existing_prefs(suffix) + local_pref_1 = joinpath(project_path, "JuliaLocalPreferences.toml") + local_pref_2 = joinpath(project_path, "LocalPreferences.toml") + + # ensure any previous tests have been cleaned up + if isfile(local_pref_1) rm(local_pref_1) end + if isfile(local_pref_2) rm(local_pref_2) end + + cp( + joinpath(test_data, "LocalPreferences_$(suffix).toml"), + joinpath(project_path, "LocalPreferences.toml") + ) + + r = run(cmd("test_libmpi.jl")) + @test success(r) +end + +withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cuda.txt")) do + test_generated_prefs("cuda") +end + +withenv("JULIA_MPI_CC_OPTS_FILE" => joinpath(test_data, "cc_opts_cpu.txt")) do + test_generated_prefs("cpu") +end + +test_existing_prefs("cuda") +r = run(cmd("test_gtl_preload.jl"; a="cuda")) +@test success(r) +test_existing_prefs("cpu") +r = run(cmd("test_gtl_preload.jl"; a="cpu")) +@test success(r) \ No newline at end of file diff --git a/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml b/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml new file mode 100644 index 000000000..a6949d610 --- /dev/null +++ b/lib/MPIPreferences/test/test_data/LocalPreferences_cpu.toml @@ -0,0 +1,9 @@ +[MPIPreferences] +__clear__ = ["preloads"] +_format = "1.1" +abi = "MPICH" +binary = "system" +cclibs = ["sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"] +libmpi = "libmpi_gnu_91.so" +mpiexec = "srun" +preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED" diff --git a/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml b/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml new file mode 100644 index 000000000..bf1b9844d --- /dev/null +++ b/lib/MPIPreferences/test/test_data/LocalPreferences_cuda.toml @@ -0,0 +1,9 @@ +[MPIPreferences] +_format = "1.1" +abi = "MPICH" +binary = "system" +cclibs = ["cupti", "cudart", "cuda", "sci_gnu_82_mpi", "sci_gnu_82", "dl", "dsmml", "xpmem"] +libmpi = "libmpi_gnu_91.so" +mpiexec = "srun" +preloads = ["libmpi_gtl_cuda.so"] +preloads_env_switch = "MPICH_GPU_SUPPORT_ENABLED" diff --git a/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt b/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt new file mode 100644 index 000000000..0452a61be --- /dev/null +++ b/lib/MPIPreferences/test/test_data/cc_opts_cpu.txt @@ -0,0 +1 @@ +-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem diff --git a/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt b/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt new file mode 100644 index 000000000..aa6a30d87 --- /dev/null +++ b/lib/MPIPreferences/test/test_data/cc_opts_cuda.txt @@ -0,0 +1 @@ +-I/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/include -I/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/include -I/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/include -I/opt/cray/pe/dsmml/0.2.2/dsmml//include -I/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/include -L/opt/cray/pe/mpich/8.1.25/ofi/gnu/9.1/lib -L/opt/cray/pe/mpich/8.1.25/gtl/lib -L/opt/cray/pe/libsci/23.02.1.1/GNU/9.1/x86_64/lib -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64/stubs -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/nvvm/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/CUPTI/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/cuda/11.7/extras/Debugger/lib64 -L/opt/nvidia/hpc_sdk/Linux_x86_64/22.7/math_libs/11.7/lib64 -L/opt/cray/pe/dsmml/0.2.2/dsmml//lib -L/opt/cray/xpmem/2.6.2-2.5_2.27__gd067c3f.shasta/lib64 -Wl,--as-needed,-lcupti,-lcudart,--no-as-needed -lcuda -Wl,--as-needed,-lmpi_gnu_91,--no-as-needed -lmpi_gtl_cuda -Wl,--as-needed,-lsci_gnu_82_mpi,--no-as-needed -Wl,--as-needed,-lsci_gnu_82,--no-as-needed -ldl -Wl,--as-needed,-ldsmml,--no-as-needed -lxpmem diff --git a/lib/MPIPreferences/test/test_gtl_preload.jl b/lib/MPIPreferences/test/test_gtl_preload.jl new file mode 100644 index 000000000..938fe3862 --- /dev/null +++ b/lib/MPIPreferences/test/test_gtl_preload.jl @@ -0,0 +1,7 @@ +using Test, MPIPreferences + +if ARGS[1] == "cuda" + @test only(MPIPreferences.System.preloads) == "libmpi_gtl_cuda.so" +elseif ARGS[1] == "cpu" + @test isnothing(MPIPreferences.System.preloads) +end \ No newline at end of file diff --git a/lib/MPIPreferences/test/test_libmpi.jl b/lib/MPIPreferences/test/test_libmpi.jl new file mode 100644 index 000000000..250a3cdfe --- /dev/null +++ b/lib/MPIPreferences/test/test_libmpi.jl @@ -0,0 +1,3 @@ +using Test, MPI + +@test MPI.libmpi == "libmpi_gnu_91.so" \ No newline at end of file