diff --git a/ompi/communicator/comm_cid.c b/ompi/communicator/comm_cid.c index 8546b700401..cbf9653cb9c 100644 --- a/ompi/communicator/comm_cid.c +++ b/ompi/communicator/comm_cid.c @@ -58,7 +58,7 @@ #include "pmix.h" /* for use when we don't have a PMIx that supports CID generation */ -opal_atomic_int64_t ompi_comm_next_base_cid = 1; +opal_atomic_int64_t ompi_comm_next_base_cid = OPAL_ATOMIC_VAR_INIT(1); /* A macro comparing two CIDs */ #define OMPI_COMM_CID_IS_LOWER(comm1,comm2) ( ((comm1)->c_index < (comm2)->c_index)? 1:0) diff --git a/ompi/communicator/comm_request.c b/ompi/communicator/comm_request.c index 89ff7fff7af..c81a839cfbc 100644 --- a/ompi/communicator/comm_request.c +++ b/ompi/communicator/comm_request.c @@ -109,7 +109,7 @@ int ompi_comm_request_schedule_append_w_flags(ompi_comm_request_t *request, ompi static int ompi_comm_request_progress (void) { ompi_comm_request_t *request, *next; - static opal_atomic_int32_t progressing = 0; + static opal_atomic_int32_t progressing = OPAL_ATOMIC_VAR_INIT(0); int completed = 0; /* don't allow re-entry */ @@ -175,7 +175,7 @@ static int ompi_comm_request_progress (void) } opal_mutex_unlock (&ompi_comm_request_mutex); - progressing = 0; + opal_atomic_store(&progressing, 0); return completed; } diff --git a/ompi/communicator/ft/comm_ft_detector.c b/ompi/communicator/ft/comm_ft_detector.c index 7b835e84f81..f280f35b13e 100644 --- a/ompi/communicator/ft/comm_ft_detector.c +++ b/ompi/communicator/ft/comm_ft_detector.c @@ -94,7 +94,7 @@ static opal_event_base_t* fd_event_base = NULL; static void fd_event_cb(int fd, short flags, void* pdetector); static bool comm_detector_use_thread = false; -static opal_atomic_int32_t fd_thread_active = 0; +static opal_atomic_int32_t fd_thread_active = OPAL_ATOMIC_VAR_INIT(0); static opal_thread_t fd_thread; static void* fd_progress(opal_object_t* obj); @@ -168,8 +168,8 @@ int ompi_comm_failure_detector_init(void) { fd_thread.t_arg = NULL; ret = opal_thread_start(&fd_thread); if( OPAL_SUCCESS != ret ) goto cleanup; - while( 0 == fd_thread_active ); /* wait for the fd thread initialization */ - if( 0 > fd_thread_active ) goto cleanup; + while( 0 == opal_atomic_load(&fd_thread_active) ); /* wait for the fd thread initialization */ + if( 0 > opal_atomic_load(&fd_thread_active) ) goto cleanup; } return OMPI_SUCCESS; @@ -218,18 +218,18 @@ int ompi_comm_failure_detector_finalize(void) { #endif while( observing == detector->hb_observing ) { /* If observed process changed, recheck if local*/ - if( !(0 < fd_thread_active) ) + if( !(0 < opal_atomic_load(&fd_thread_active)) ) { opal_progress(); } } } - if( 0 < fd_thread_active ) { + if( 0 < opal_atomic_load(&fd_thread_active) ) { void* tret; /* this is not a race condition. Accesses are serialized, we use the * atomic for the mfence part of it. */ - OPAL_THREAD_ADD_FETCH32(&fd_thread_active, -fd_thread_active); + OPAL_THREAD_ADD_FETCH32(&fd_thread_active, -opal_atomic_load(&fd_thread_active)); opal_event_base_loopbreak(fd_event_base); opal_thread_join(&fd_thread, &tret); } @@ -587,9 +587,9 @@ void* fd_progress(opal_object_t* obj) { return OPAL_THREAD_CANCELLED; } OPAL_THREAD_ADD_FETCH32(&fd_thread_active, 1); - while( 1 == fd_thread_active ); /* wait for init stage 2: start_detector */ + while( 1 == opal_atomic_load(&fd_thread_active) ); /* wait for init stage 2: start_detector */ ret = MCA_PML_CALL(irecv(NULL, 0, MPI_BYTE, 0, MCA_COLL_BASE_TAG_FT_END, &ompi_mpi_comm_self.comm, &req)); - while( fd_thread_active ) { + while( opal_atomic_load(&fd_thread_active) ) { opal_event_loop(fd_event_base, OPAL_EVLOOP_ONCE); #if 0 /* This test disabled because rdma emulation over TCP would not work without diff --git a/ompi/errhandler/errhandler.h b/ompi/errhandler/errhandler.h index 40909f3f140..bfda84a0713 100644 --- a/ompi/errhandler/errhandler.h +++ b/ompi/errhandler/errhandler.h @@ -213,7 +213,7 @@ extern opal_atomic_int32_t ompi_instance_count; */ #define OMPI_ERR_INIT_FINALIZE(name) \ { \ - if (OPAL_UNLIKELY(0 == ompi_instance_count)) { \ + if (OPAL_UNLIKELY(0 == opal_atomic_load(&ompi_instance_count))) { \ ompi_errhandler_invoke(NULL, NULL, -1, \ ompi_errcode_get_mpi_code(MPI_ERR_ARG), \ name); \ diff --git a/ompi/instance/instance.c b/ompi/instance/instance.c index ff140e98891..6be375fcf8a 100644 --- a/ompi/instance/instance.c +++ b/ompi/instance/instance.c @@ -90,7 +90,7 @@ enum { OMPI_INSTANCE_FINALIZING = -2, }; -opal_atomic_int32_t ompi_instance_count = 0; +opal_atomic_int32_t ompi_instance_count = OPAL_ATOMIC_VAR_INIT(0); static const char *ompi_instance_builtin_psets[] = { "mpi://WORLD", diff --git a/ompi/mca/coll/xhc/coll_xhc_intrinsic.h b/ompi/mca/coll/xhc/coll_xhc_intrinsic.h index ee61b8d7db3..03650022297 100644 --- a/ompi/mca/coll/xhc/coll_xhc_intrinsic.h +++ b/ompi/mca/coll/xhc/coll_xhc_intrinsic.h @@ -43,11 +43,13 @@ typedef size_t __attribute__((aligned(SIZEOF_SIZE_T))) xf_size_t; // https://github.com/open-mpi/ompi/issues/9722 #if OPAL_USE_GCC_BUILTIN_ATOMICS || OPAL_USE_C11_ATOMICS - #define xhc_atomic_load_int(addr) __atomic_load_n(addr, __ATOMIC_RELAXED) - #define xhc_atomic_store_int(addr, val) __atomic_store_n(addr, val, __ATOMIC_RELAXED) + #define xhc_atomic_load_int(addr) opal_atomic_load((opal_atomic_int_t *)(addr)) + #define xhc_atomic_store_int(addr, val) \ + opal_atomic_store((opal_atomic_int_t *)(addr), (val)) - #define xhc_atomic_load_size_t(addr) __atomic_load_n(addr, __ATOMIC_RELAXED) - #define xhc_atomic_store_size_t(addr, val) __atomic_store_n(addr, val, __ATOMIC_RELAXED) + #define xhc_atomic_load_size_t(addr) opal_atomic_load((opal_atomic_size_t *)(addr)) + #define xhc_atomic_store_size_t(addr, val) \ + opal_atomic_store((opal_atomic_size_t *)(addr), (val)) #else #define xhc_atomic_load_int(addr) (*(addr)) #define xhc_atomic_store_int(addr, val) (*(addr) = (val)) diff --git a/ompi/mca/common/monitoring/common_monitoring.c b/ompi/mca/common/monitoring/common_monitoring.c index 14e69fb4326..57a62b2b045 100644 --- a/ompi/mca/common/monitoring/common_monitoring.c +++ b/ompi/mca/common/monitoring/common_monitoring.c @@ -39,7 +39,7 @@ /*** Monitoring specific variables ***/ /* Keep tracks of how many components are currently using the common part */ -static opal_atomic_int32_t mca_common_monitoring_hold = 0; +static opal_atomic_int32_t mca_common_monitoring_hold = OPAL_ATOMIC_VAR_INIT(0); /* Output parameters */ int mca_common_monitoring_output_stream_id = -1; static opal_output_stream_t mca_common_monitoring_output_stream_obj = { diff --git a/ompi/mca/common/ompio/common_ompio_buffer.c b/ompi/mca/common/ompio/common_ompio_buffer.c index c8ce40c0561..ab24ee1d3ef 100644 --- a/ompi/mca/common/ompio/common_ompio_buffer.c +++ b/ompi/mca/common/ompio/common_ompio_buffer.c @@ -35,7 +35,7 @@ static opal_mutex_t mca_common_ompio_buffer_mutex; /* lock for thread s static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL; static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL; -static opal_atomic_int32_t mca_common_ompio_buffer_init = 0; +static opal_atomic_int32_t mca_common_ompio_buffer_init = OPAL_ATOMIC_VAR_INIT(0); static int32_t mca_common_ompio_pagesize=4096; static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size ); static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf ); @@ -145,7 +145,7 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize ) { char *tmp=NULL; - if ( !mca_common_ompio_buffer_init ){ + if ( !opal_atomic_load(&mca_common_ompio_buffer_init) ){ mca_common_ompio_buffer_alloc_init (); } @@ -159,7 +159,7 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize ) void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf ) { - if ( !mca_common_ompio_buffer_init ){ + if ( !opal_atomic_load(&mca_common_ompio_buffer_init) ){ /* Should not happen. You can not release a buf without ** having it allocated first. */ diff --git a/ompi/mca/osc/monitoring/osc_monitoring_module.h b/ompi/mca/osc/monitoring/osc_monitoring_module.h index d352a8a4c50..7c21c74ea80 100644 --- a/ompi/mca/osc/monitoring/osc_monitoring_module.h +++ b/ompi/mca/osc/monitoring/osc_monitoring_module.h @@ -50,7 +50,7 @@ OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \ { \ /* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \ - opal_atomic_int32_t init_done = 0; \ + opal_atomic_int32_t init_done = OPAL_ATOMIC_VAR_INIT(0); \ /* Define and set the ompi_osc_monitoring_## template \ * ##_template variable. The functions recorded here are \ * linked to the original functions of the original \ diff --git a/ompi/mca/pml/base/pml_base_bsend.c b/ompi/mca/pml/base/pml_base_bsend.c index d1c38e90209..d145d305107 100644 --- a/ompi/mca/pml/base/pml_base_bsend.c +++ b/ompi/mca/pml/base/pml_base_bsend.c @@ -54,7 +54,7 @@ static size_t mca_pml_bsend_size; /* adjusted size of user buffe static size_t mca_pml_bsend_count; /* number of outstanding requests */ static size_t mca_pml_bsend_pagesz; /* mmap page size */ static int mca_pml_bsend_pagebits; /* number of bits in pagesz */ -static opal_atomic_int32_t mca_pml_bsend_init = 0; +static opal_atomic_int32_t mca_pml_bsend_init = OPAL_ATOMIC_VAR_INIT(0); /* defined in pml_base_open.c */ extern char *ompi_pml_base_bsend_allocator_name; diff --git a/ompi/mca/pml/base/pml_base_sendreq.c b/ompi/mca/pml/base/pml_base_sendreq.c index fd25b2f4898..bd1e9bef5f9 100644 --- a/ompi/mca/pml/base/pml_base_sendreq.c +++ b/ompi/mca/pml/base/pml_base_sendreq.c @@ -54,7 +54,7 @@ static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req) #if MPI_VERSION >= 4 int mca_pml_cancel_send_callback(struct ompi_request_t *request, int flag) { - static opal_atomic_int32_t send_deprecate_count = 0; + static opal_atomic_int32_t send_deprecate_count = OPAL_ATOMIC_VAR_INIT(0); int32_t val; val = opal_atomic_add_fetch_32(&send_deprecate_count, 1); diff --git a/ompi/mca/pml/ob1/pml_ob1_progress.c b/ompi/mca/pml/ob1/pml_ob1_progress.c index 930d5b7311e..6b7d08bdbc7 100644 --- a/ompi/mca/pml/ob1/pml_ob1_progress.c +++ b/ompi/mca/pml/ob1/pml_ob1_progress.c @@ -52,7 +52,7 @@ static inline int mca_pml_ob1_process_pending_accelerator_async_copies(void) return count; } -static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0; +static opal_atomic_int32_t mca_pml_ob1_progress_needed = OPAL_ATOMIC_VAR_INIT(0); int mca_pml_ob1_enable_progress(int32_t count) { int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count); diff --git a/ompi/request/grequestx.c b/ompi/request/grequestx.c index 60347854d78..f8023409a9c 100644 --- a/ompi/request/grequestx.c +++ b/ompi/request/grequestx.c @@ -28,7 +28,7 @@ static bool requests_initialized = false; static opal_list_t requests; -static opal_atomic_int32_t active_requests = 0; +static opal_atomic_int32_t active_requests = OPAL_ATOMIC_VAR_INIT(0); static bool in_progress = false; static opal_mutex_t lock = OPAL_MUTEX_STATIC_INIT; diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 787e1e10249..0667e0877c0 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -125,7 +125,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING; * Global variables and symbols for the MPI layer */ -opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; +opal_atomic_int32_t ompi_mpi_state = OPAL_ATOMIC_VAR_INIT(OMPI_MPI_STATE_NOT_INITIALIZED); volatile bool ompi_rte_initialized = false; bool ompi_mpi_thread_multiple = false; @@ -371,7 +371,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided, // silently return successfully once the initializing // thread has completed. if (reinit_ok) { - while (ompi_mpi_state < OMPI_MPI_STATE_INIT_COMPLETED) { + while (opal_atomic_load(&ompi_mpi_state) < OMPI_MPI_STATE_INIT_COMPLETED) { usleep(1); } return MPI_SUCCESS; diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index cc047b59d3d..3931f446f29 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -54,6 +54,20 @@ #include "opal/opal_portable_platform.h" #include "opal_stdatomic.h" +#if OPAL_USE_C11_ATOMICS +# define opal_atomic_load(addr) atomic_load_explicit(&((addr)->v), memory_order_relaxed) +# define opal_atomic_store(addr, value) \ + atomic_store_explicit(&((addr)->v), (value), memory_order_relaxed) +# define opal_atomic_load_explicit(addr, order) atomic_load_explicit(&((addr)->v), (order)) +# define opal_atomic_store_explicit(addr, value, order) \ + atomic_store_explicit(&((addr)->v), (value), (order)) +#else +# define opal_atomic_load(addr) (*(addr)) +# define opal_atomic_store(addr, value) (*(addr) = (value)) +# define opal_atomic_load_explicit(addr, order) opal_atomic_load(addr) +# define opal_atomic_store_explicit(addr, value, order) opal_atomic_store(addr, value) +#endif + BEGIN_C_DECLS /********************************************************************** diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h index 5eca86c9edb..2c997a04c19 100644 --- a/opal/include/opal/sys/atomic_stdc.h +++ b/opal/include/opal/sys/atomic_stdc.h @@ -33,6 +33,8 @@ # include # include +# define opal_atomic_address(addr) (&((addr)->v)) + /********************************************************************** * * Memory Barriers @@ -72,39 +74,41 @@ static inline void opal_atomic_rmb(void) *********************************************************************/ # define opal_atomic_compare_exchange_strong_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_relaxed, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_acquire, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_release, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_relaxed, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_acquire, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_release, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_relaxed, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_relaxed, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_acquire, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_acquire, memory_order_relaxed) # define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) \ - atomic_compare_exchange_strong_explicit(addr, compare, value, memory_order_release, \ - memory_order_relaxed) + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_release, memory_order_relaxed) # if OPAL_HAVE_C11_CSWAP_INT128 /* the C11 atomic compare-exchange is lock free so use it */ -# define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong +# define opal_atomic_compare_exchange_strong_128(addr, compare, value) \ + atomic_compare_exchange_strong_explicit(opal_atomic_address(addr), compare, value, \ + memory_order_relaxed, memory_order_relaxed) # define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 @@ -116,7 +120,7 @@ __opal_attribute_always_inline__ static inline bool opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { - opal_int128_t prev = __sync_val_compare_and_swap(addr, *oldval, newval); + opal_int128_t prev = __sync_val_compare_and_swap(&((addr)->v), *oldval, newval); bool ret = prev == *oldval; *oldval = prev; return ret; @@ -138,11 +142,11 @@ opal_atomic_compare_exchange_strong_128(opal_atomic_int128_t *addr, opal_int128_ *********************************************************************/ # define opal_atomic_swap_32(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned int *) addr, value, memory_order_relaxed) + atomic_exchange_explicit((_Atomic unsigned int *) opal_atomic_address(addr), value, memory_order_relaxed) # define opal_atomic_swap_64(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed) + atomic_exchange_explicit((_Atomic unsigned long *) opal_atomic_address(addr), value, memory_order_relaxed) # define opal_atomic_swap_ptr(addr, value) \ - atomic_exchange_explicit((_Atomic unsigned long *) addr, value, memory_order_relaxed) + atomic_exchange_explicit((_Atomic unsigned long *) opal_atomic_address(addr), value, memory_order_relaxed) /********************************************************************** @@ -184,12 +188,13 @@ static inline void opal_atomic_unlock(opal_atomic_lock_t *lock) # define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ static inline type opal_atomic_fetch_##op##_##bits(opal_atomic_##type *addr, type value) \ { \ - return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed); \ + return atomic_fetch_##op##_explicit(&((addr)->v), value, memory_order_relaxed); \ } \ \ static inline type opal_atomic_##op##_fetch_##bits(opal_atomic_##type *addr, type value) \ { \ - return atomic_fetch_##op##_explicit(addr, value, memory_order_relaxed) operator value; \ + return atomic_fetch_##op##_explicit(&((addr)->v), value, memory_order_relaxed) \ + operator value; \ } OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) @@ -208,7 +213,7 @@ OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) # define opal_atomic_add(addr, value) \ - (void) atomic_fetch_add_explicit(addr, value, memory_order_relaxed) + (void) atomic_fetch_add_explicit(opal_atomic_address(addr), value, memory_order_relaxed) #include "opal/sys/atomic_impl_minmax_math.h" diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h index f7dd8353d3b..4b895df31bc 100644 --- a/opal/include/opal_stdatomic.h +++ b/opal/include/opal_stdatomic.h @@ -42,18 +42,38 @@ enum { OPAL_ATOMIC_LOCK_UNLOCKED = 0, # include -typedef atomic_int opal_atomic_int_t; -typedef atomic_long opal_atomic_long_t; - -typedef _Atomic int32_t opal_atomic_int32_t; -typedef _Atomic uint32_t opal_atomic_uint32_t; -typedef _Atomic int64_t opal_atomic_int64_t; -typedef _Atomic uint64_t opal_atomic_uint64_t; - -typedef _Atomic size_t opal_atomic_size_t; -typedef _Atomic ssize_t opal_atomic_ssize_t; -typedef _Atomic intptr_t opal_atomic_intptr_t; -typedef _Atomic uintptr_t opal_atomic_uintptr_t; +typedef struct { + _Atomic int v; +} opal_atomic_int_t; +typedef struct { + _Atomic long v; +} opal_atomic_long_t; + +typedef struct { + _Atomic int32_t v; +} opal_atomic_int32_t; +typedef struct { + _Atomic uint32_t v; +} opal_atomic_uint32_t; +typedef struct { + _Atomic int64_t v; +} opal_atomic_int64_t; +typedef struct { + _Atomic uint64_t v; +} opal_atomic_uint64_t; + +typedef struct { + _Atomic size_t v; +} opal_atomic_size_t; +typedef struct { + _Atomic ssize_t v; +} opal_atomic_ssize_t; +typedef struct { + _Atomic intptr_t v; +} opal_atomic_intptr_t; +typedef struct { + _Atomic uintptr_t v; +} opal_atomic_uintptr_t; typedef atomic_flag opal_atomic_lock_t; @@ -68,7 +88,9 @@ typedef atomic_flag opal_atomic_lock_t; # if OPAL_USE_C11_ATOMICS && OPAL_HAVE_C11_CSWAP_INT128 -typedef _Atomic opal_int128_t opal_atomic_int128_t; +typedef struct { + _Atomic opal_int128_t v; +} opal_atomic_int128_t; # else @@ -78,4 +100,10 @@ typedef volatile opal_int128_t opal_atomic_int128_t __opal_attribute_aligned__(1 # endif +# if OPAL_USE_C11_ATOMICS +# define OPAL_ATOMIC_VAR_INIT(value) { .v = (value) } +# else +# define OPAL_ATOMIC_VAR_INIT(value) (value) +# endif + #endif /* !defined(OPAL_STDATOMIC_H) */ diff --git a/opal/mca/btl/sm/btl_sm_component.c b/opal/mca/btl/sm/btl_sm_component.c index 21e1bcf2a6c..31ab62eb3bf 100644 --- a/opal/mca/btl/sm/btl_sm_component.c +++ b/opal/mca/btl/sm/btl_sm_component.c @@ -549,7 +549,7 @@ static void mca_btl_sm_progress_endpoints(void) static int mca_btl_sm_component_progress(void) { - static opal_atomic_int32_t lock = 0; + static opal_atomic_int32_t lock = OPAL_ATOMIC_VAR_INIT(0); int count = 0; if (opal_using_threads()) { @@ -566,13 +566,13 @@ static int mca_btl_sm_component_progress(void) mca_btl_sm_progress_endpoints(); if (SM_FIFO_FREE == mca_btl_sm_component.my_fifo->fifo_head) { - lock = 0; + opal_atomic_store(&lock, 0); return count; } count += mca_btl_sm_poll_fifo(); opal_atomic_mb(); - lock = 0; + opal_atomic_store(&lock, 0); return count; } diff --git a/opal/mca/btl/uct/btl_uct_device_context.h b/opal/mca/btl/uct/btl_uct_device_context.h index 7e25e0bef19..b138bacb72c 100644 --- a/opal/mca/btl/uct/btl_uct_device_context.h +++ b/opal/mca/btl/uct/btl_uct_device_context.h @@ -61,7 +61,7 @@ static inline void mca_btl_uct_context_unlock(mca_btl_uct_device_context_t *cont static inline int mca_btl_uct_get_context_index(void) { - static opal_atomic_uint32_t next_uct_index = 0; + static opal_atomic_uint32_t next_uct_index = OPAL_ATOMIC_VAR_INIT(0); int context_id; # if OPAL_C_HAVE__THREAD_LOCAL @@ -79,7 +79,9 @@ static inline int mca_btl_uct_get_context_index(void) # endif /* avoid using atomics in this. i doubt it improves performance to ensure atomicity on the * next index in this case. */ - context_id = next_uct_index++ % mca_btl_uct_component.num_contexts_per_module; + context_id = opal_atomic_load(&next_uct_index); + opal_atomic_store(&next_uct_index, context_id + 1); + context_id %= mca_btl_uct_component.num_contexts_per_module; # if OPAL_C_HAVE__THREAD_LOCAL } # endif diff --git a/opal/mca/common/ucx/common_ucx_wpool.c b/opal/mca/common/ucx/common_ucx_wpool.c index ae290201710..b3d859eaf34 100644 --- a/opal/mca/common/ucx/common_ucx_wpool.c +++ b/opal/mca/common/ucx/common_ucx_wpool.c @@ -32,8 +32,8 @@ __thread int initialized = 0; #endif bool opal_common_ucx_thread_enabled = false; -opal_atomic_int64_t opal_common_ucx_ep_counts = 0; -opal_atomic_int64_t opal_common_ucx_unpacked_rkey_counts = 0; +opal_atomic_int64_t opal_common_ucx_ep_counts = OPAL_ATOMIC_VAR_INIT(0); +opal_atomic_int64_t opal_common_ucx_unpacked_rkey_counts = OPAL_ATOMIC_VAR_INIT(0); static _ctx_record_t *_tlocal_add_ctx_rec(opal_common_ucx_ctx_t *ctx); static inline _ctx_record_t *_tlocal_get_ctx_rec(opal_tsd_tracked_key_t tls_key); diff --git a/opal/mca/threads/base/wait_sync.c b/opal/mca/threads/base/wait_sync.c index 2458a9614be..a125e2ac950 100644 --- a/opal/mca/threads/base/wait_sync.c +++ b/opal/mca/threads/base/wait_sync.c @@ -47,7 +47,7 @@ void opal_threads_base_wait_sync_global_wakeup_mt(int status) opal_mutex_unlock(&wait_sync_lock); } -static opal_atomic_int32_t num_thread_in_progress = 0; +static opal_atomic_int32_t num_thread_in_progress = OPAL_ATOMIC_VAR_INIT(0); #define WAIT_SYNC_PASS_OWNERSHIP(who) \ do { \ @@ -97,7 +97,8 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync) * - our sync has been triggered. */ check_status: - if (sync != opal_threads_base_wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) { + if (sync != opal_threads_base_wait_sync_list && + opal_atomic_load(&num_thread_in_progress) >= opal_max_thread_in_progress) { opal_thread_internal_cond_wait(&sync->condition, &sync->lock); /** diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c index 7bca660b9d6..49bcd0d9173 100644 --- a/opal/runtime/opal_progress.c +++ b/opal/runtime/opal_progress.c @@ -72,13 +72,13 @@ static opal_timer_t event_progress_last_time = 0; static opal_timer_t event_progress_delta = 0; #else /* current count down until we tick the event library */ -static opal_atomic_int32_t event_progress_counter = 0; +static opal_atomic_int32_t event_progress_counter = OPAL_ATOMIC_VAR_INIT(0); /* reset value for counter when it hits 0 */ static int32_t event_progress_delta = 0; #endif /* users of the event library from MPI cause the tick rate to be every time */ -static opal_atomic_int32_t num_event_users = 0; +static opal_atomic_int32_t num_event_users = OPAL_ATOMIC_VAR_INIT(0); #if OPAL_ENABLE_DEBUG static int debug_output = -1; @@ -159,7 +159,8 @@ int opal_progress_init(void) (debug_output, "progress: initialized event flag to: %x", opal_progress_event_flag)); OPAL_OUTPUT((debug_output, "progress: initialized yield_when_idle to: %s", opal_progress_yield_when_idle ? "true" : "false")); - OPAL_OUTPUT((debug_output, "progress: initialized num users to: %d", num_event_users)); + OPAL_OUTPUT((debug_output, "progress: initialized num users to: %d", + opal_atomic_load(&num_event_users))); OPAL_OUTPUT( (debug_output, "progress: initialized poll rate to: %ld", (long) event_progress_delta)); @@ -170,7 +171,7 @@ int opal_progress_init(void) static int opal_progress_events(void) { - static opal_atomic_int32_t lock = 0; + static opal_atomic_int32_t lock = OPAL_ATOMIC_VAR_INIT(0); int events = 0; if (opal_progress_event_flag != 0 && !OPAL_THREAD_SWAP_32(&lock, 1)) { @@ -183,7 +184,9 @@ static int opal_progress_events(void) /* trip the event library if we've reached our tick rate and we are enabled */ if (now - event_progress_last_time > event_progress_delta) { - event_progress_last_time = (num_event_users > 0) ? now - event_progress_delta : now; + event_progress_last_time = (opal_atomic_load(&num_event_users) > 0) + ? now - event_progress_delta + : now; events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag); } @@ -192,11 +195,13 @@ static int opal_progress_events(void) /* trip the event library if we've reached our tick rate and we are enabled */ if (OPAL_THREAD_ADD_FETCH32(&event_progress_counter, -1) <= 0) { - event_progress_counter = (num_event_users > 0) ? 0 : event_progress_delta; + opal_atomic_store(&event_progress_counter, + (opal_atomic_load(&num_event_users) > 0) ? 0 + : event_progress_delta); events += opal_event_loop(opal_sync_event_base, opal_progress_event_flag); } #endif /* OPAL_PROGRESS_USE_TIMERS */ - lock = 0; + opal_atomic_store(&lock, 0); } return events; @@ -236,7 +241,7 @@ int opal_progress(void) } opal_progress_events(); - } else if (num_event_users > 0) { + } else if (opal_atomic_load(&num_event_users) > 0) { opal_progress_events(); } @@ -280,7 +285,7 @@ void opal_progress_event_users_increment(void) event_progress_last_time -= event_progress_delta; #else /* always reset the tick rate - can't hurt */ - event_progress_counter = 0; + opal_atomic_store(&event_progress_counter, 0); #endif } @@ -298,7 +303,7 @@ void opal_progress_event_users_decrement(void) #if !OPAL_PROGRESS_USE_TIMERS /* start now in delaying if it's easy */ if (val >= 0) { - event_progress_counter = event_progress_delta; + opal_atomic_store(&event_progress_counter, event_progress_delta); } #endif } @@ -326,7 +331,8 @@ void opal_progress_set_event_poll_rate(int polltime) event_progress_last_time = opal_timer_base_get_cycles(); # endif #else - event_progress_counter = event_progress_delta = 0; + event_progress_delta = 0; + opal_atomic_store(&event_progress_counter, 0); #endif if (polltime == 0) {