diff --git a/CMakeLists.txt b/CMakeLists.txt index 9388c96a6..335e9e182 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,9 +70,9 @@ endif("${isSystemDir}" STREQUAL "-1") ## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them). # set runpath, too -# if(NOT APPLE) -# set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") -# endif() +if(NOT APPLE) + set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags") +endif() set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${THIRD_PARTY_ROOT}/lib") @@ -203,6 +203,11 @@ endif() # MPI (mostly for booting, but we also use MPI communication in some places now) find_package(MPI REQUIRED) include_directories(${MPI_INCLUDE_PATH}) +# Add MPI libs dirs to RPATH +foreach(lib ${MPI_CXX_LIBRARIES}) + get_filename_component(libdir "${lib}" PATH) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${libdir}") +endforeach() macro(add_grappa_exe target exe ) add_executable(${target} EXCLUDE_FROM_ALL ${ARGN}) diff --git a/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb b/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb index 63f69065d..d78f7a85b 100755 --- a/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb +++ b/applications/NPB/GRAPPA/IS/igor_grappa_intsort.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../../util/igor_common.rb' +require_relative_to_symlink '../../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/NPB/MPI/igor_mpi_intsort.rb b/applications/NPB/MPI/igor_mpi_intsort.rb index f1bb30e36..04aad8764 100755 --- a/applications/NPB/MPI/igor_mpi_intsort.rb +++ b/applications/NPB/MPI/igor_mpi_intsort.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/grappa/igor_ckpt_gen.rb b/applications/graph500/grappa/igor_ckpt_gen.rb index ea37ff51a..872d06a80 100755 --- a/applications/graph500/grappa/igor_ckpt_gen.rb +++ b/applications/graph500/grappa/igor_ckpt_gen.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/applications/graph500/grappa/igor_grappa_bfs.rb b/applications/graph500/grappa/igor_grappa_bfs.rb index f4caa2e42..a44605bb2 100755 --- a/applications/graph500/grappa/igor_grappa_bfs.rb +++ b/applications/graph500/grappa/igor_grappa_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/grappa/igor_new_bfs.rb b/applications/graph500/grappa/igor_new_bfs.rb index e09b091cc..f747fbd1c 100755 --- a/applications/graph500/grappa/igor_new_bfs.rb +++ b/applications/graph500/grappa/igor_new_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/grappa/igor_new_cc.rb b/applications/graph500/grappa/igor_new_cc.rb index 6edc8079e..55d540e09 100755 --- a/applications/graph500/grappa/igor_new_cc.rb +++ b/applications/graph500/grappa/igor_new_cc.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/graph500/mpi/igor_mpi_bfs.rb b/applications/graph500/mpi/igor_mpi_bfs.rb index 1a842c724..eaf89787a 100755 --- a/applications/graph500/mpi/igor_mpi_bfs.rb +++ b/applications/graph500/mpi/igor_mpi_bfs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../../util/igor_common.rb' +require_relative_to_symlink '../../../util/igor_common.rb' Igor do include Isolatable diff --git a/applications/pagerank/igor_grappa_pagerank.rb b/applications/pagerank/igor_grappa_pagerank.rb index 6e09f97d6..8e479efb8 100755 --- a/applications/pagerank/igor_grappa_pagerank.rb +++ b/applications/pagerank/igor_grappa_pagerank.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../util/igor_common.rb' +require_relative_to_symlink '../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/applications/uts/igor_grappa_uts.rb b/applications/uts/igor_grappa_uts.rb index eb600da65..92c3fb270 100755 --- a/applications/uts/igor_grappa_uts.rb +++ b/applications/uts/igor_grappa_uts.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../../util/igor_common.rb' +require_relative_to_symlink '../../util/igor_common.rb' def expand_flags(*names) names.map{|n| "--#{n}=%{#{n}}"}.join(' ') diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index 7bc795332..e469340e7 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -1,9 +1,33 @@ -# file(COPY . DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file(GLOB scripts "*") -foreach(file ${scripts}) + +set(symlinked_scripts + distcc_make + distcc_ninja + grappa_srun + launch_distcc.sh + srun_epilog.sh +) + +set(configured_scripts + srun_prolog.rb +) + +foreach(file ${symlinked_scripts}) get_filename_component(name ${file} NAME) - file(RELATIVE_PATH relative_file ${CMAKE_CURRENT_BINARY_DIR} ${file}) execute_process(COMMAND - ln -sf ${relative_file} ${CMAKE_CURRENT_BINARY_DIR}/${name} + ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/${file} ${CMAKE_CURRENT_BINARY_DIR}/${name} ) endforeach() + +set(GRAPPA_LD_DIRS "") +if(VAMPIR_ROOT) + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${VAMPIR_ROOT}/lib") +endif() +if(THIRD_PARTY_ROOT) + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${THIRD_PARTY_ROOT}/lib") +else() + set(GRAPPA_LD_DIRS "${GRAPPA_LD_DIRS}:${EXTERN_BUILD_DIR}/lib") +endif() + +foreach(f ${configured_scripts}) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${f}.in ${CMAKE_CURRENT_BINARY_DIR}/${f} @ONLY) +endforeach() diff --git a/bin/grappa_srun b/bin/grappa_srun index 39850d056..9402dafa8 100755 --- a/bin/grappa_srun +++ b/bin/grappa_srun @@ -39,7 +39,7 @@ srun_flags = %w[ --cpu_bind=rank --label --kill-on-bad-exit ] \ << "--task-epilog=#{DIR}/srun_epilog.sh" # "source" prolog -require_relative "srun_prolog.rb" +require "#{DIR}/srun_prolog.rb" setarch = "" diff --git a/bin/srun_prolog.rb b/bin/srun_prolog.rb.in similarity index 90% rename from bin/srun_prolog.rb rename to bin/srun_prolog.rb.in index 9c52ee721..23ffb4282 100755 --- a/bin/srun_prolog.rb +++ b/bin/srun_prolog.rb.in @@ -4,6 +4,11 @@ interactive = true end +unless ENV["IGOR_OVERRIDE_LIBRARY_PATH"] + # prepend VampirTrace & third-party dirs to LD_LIBRARY_PATH + ENV["LD_LIBRARY_PATH"] = "@GRAPPA_LD_DIRS@:#{ENV["LD_LIBRARY_PATH"]}" +end + ## set up Google logging defaults ENV["GLOG_logtostderr"] = "1" ENV["GLOG_v"] = "0" diff --git a/system/Grappa.cpp b/system/Grappa.cpp index 88915f56f..a35a616a5 100644 --- a/system/Grappa.cpp +++ b/system/Grappa.cpp @@ -185,13 +185,17 @@ static void stats_dump_sighandler( int signum ) { // function to call when google logging library detect a failure namespace Grappa { namespace impl { + + bool freeze_on_error = false; + /// called on failures to backtrace and pause for debugger - void failure_function() { + void failure_function(int ignore = 0) { google::FlushLogFiles(google::GLOG_INFO); - google::DumpStackTrace(); - gasnett_freezeForDebuggerErr(); + // google::DumpStackTrace(); + if (freeze_on_error) gasnett_freezeForDebuggerErr(); gasnet_exit(1); } + } } @@ -207,7 +211,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt // } // make sure gasnet is ready to backtrace - gasnett_backtrace_init( (*argv_p)[0] ); + // gasnett_backtrace_init( (*argv_p)[0] ); // help generate unique profile filename Grappa::impl::set_exe_name( (*argv_p)[0] ); @@ -217,8 +221,11 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt // activate logging google::InitGoogleLogging( *argv_p[0] ); - google::InstallFailureFunction( &Grappa::impl::failure_function ); - google::OverrideDefaultSignalHandler( &gasnet_pause_sighandler ); + + char * env_freeze = getenv("GASNET_FREEZE_ON_ERROR"); + if (env_freeze && strncmp(env_freeze,"1",1) == 0) freeze_on_error = true; + + google::OverrideDefaultSignalHandler( &Grappa::impl::failure_function ); DVLOG(1) << "Initializing Grappa library...."; #ifdef HEAPCHECK_ENABLE @@ -248,7 +255,7 @@ void Grappa_init( int * argc_p, char ** argv_p[], int64_t global_memory_size_byt sigemptyset( &sigabrt_sa.sa_mask ); sigabrt_sa.sa_flags = 0; sigabrt_sa.sa_handler = &gasnet_pause_sighandler; - CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed."; + // CHECK_EQ( 0, sigaction( SIGABRT, &sigabrt_sa, 0 ) ) << "SIGABRT signal handler installation failed."; // Asynchronous IO // initialize completed stack @@ -385,6 +392,8 @@ void Grappa_activate() { DVLOG(1) << "Activating Grappa library...."; global_communicator.activate(); + google::InstallFailureSignalHandler(); + locale_shared_memory.activate(); global_task_manager.activate(); Grappa::comm_barrier(); diff --git a/system/LocaleSharedMemory.cpp b/system/LocaleSharedMemory.cpp index ff0cee30c..c9c2f2a40 100644 --- a/system/LocaleSharedMemory.cpp +++ b/system/LocaleSharedMemory.cpp @@ -38,7 +38,7 @@ namespace Grappa { namespace impl { /// called on failures to backtrace and pause for debugger -extern void failure_function(); +extern void failure_function(int ignore = 0); /// how much memory do we expect to allocate? extern int64_t global_memory_size_bytes; diff --git a/system/RDMAAggregator.cpp b/system/RDMAAggregator.cpp index d27504272..24b50336c 100644 --- a/system/RDMAAggregator.cpp +++ b/system/RDMAAggregator.cpp @@ -33,7 +33,7 @@ namespace Grappa { namespace impl { // defined in Grappa.cpp - extern void failure_function(); + extern void failure_function(int ignore = 0); } } diff --git a/system/tests/igor_context_switch.rb b/system/tests/igor_context_switch.rb index 69cd44575..b907cc8b5 100755 --- a/system/tests/igor_context_switch.rb +++ b/system/tests/igor_context_switch.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do include Isolatable diff --git a/system/tests/igor_datastructs.rb b/system/tests/igor_datastructs.rb index f6d7399ee..88612ae36 100755 --- a/system/tests/igor_datastructs.rb +++ b/system/tests/igor_datastructs.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do database '~/exp/pgas.sqlite', :queue diff --git a/system/tests/igor_tests.rb b/system/tests/igor_tests.rb index b189aed15..879979586 100755 --- a/system/tests/igor_tests.rb +++ b/system/tests/igor_tests.rb @@ -2,7 +2,7 @@ require 'igor' # inherit parser, sbatch_flags -require_relative '../util/igor_common.rb' +require_relative_to_symlink '../util/igor_common.rb' Igor do database '~/exp/test.sqlite', :vector diff --git a/util/igor_common.rb b/util/igor_common.rb index a7ac9947e..8570b4758 100644 --- a/util/igor_common.rb +++ b/util/igor_common.rb @@ -69,7 +69,7 @@ def isolate(exes, shared_dir=nil) end end FileUtils.cp(exe, @ldir) - libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH} ldd #{exe}"` + libs = `bash -c "LD_LIBRARY_PATH=#{$GRAPPA_LIBPATH}:$LD_LIBRARY_PATH ldd #{exe}"` .split(/\n/) .map{|s| s[/.*> (.*) \(.*/,1] } .select{|s| s != nil and s != "" and @@ -97,15 +97,16 @@ def command(c=nil) @params[:tdir] = tdir ignore :tdir, :command c = %Q[ - if [[ ! -d "#{tdir}" ]]; then - srun mkdir -p #{tdir}; - ls #{tdir}; - echo $(hostname); - for l in $(ls #{@ldir}); do - echo $l; sbcast #{@ldir}/$l #{tdir}/${l}; - done; - fi; - #{c} + srun bash -c ' + if [[ ! -d "#{tdir}" ]]; then + mkdir -p #{tdir}; + ls #{tdir}; + echo $(hostname); + fi'; + for l in $(ls #{@ldir}); do + echo $l; sbcast -f #{@ldir}/$l #{tdir}/${l}; + done; + LD_LIBRARY_PATH=#{tdir} IGOR_OVERRIDE_LIBRARY_PATH=1 #{c} ].tr("\n "," ") end return super(c)