From 4f908887f9083d5a5c69d03a6ace5437dd89632e Mon Sep 17 00:00:00 2001 From: Wayne Mitchell Date: Fri, 17 Oct 2025 23:56:48 +0000 Subject: [PATCH 1/5] Account for sycl device in umpire build --- src/utilities/memory.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utilities/memory.c b/src/utilities/memory.c index d4b8292085..ecee7a4da0 100644 --- a/src/utilities/memory.c +++ b/src/utilities/memory.c @@ -1889,11 +1889,17 @@ HYPRE_Int hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) { hypre_Handle *handle = hypre_handle(); - const hypre_int device_id = hypre_HandleDevice(handle); char resource_name[16]; const char *pool_name = hypre_HandleUmpireDevicePoolName(handle); +#if defined(HYPRE_USING_SYCL) + HYPRE_Int device_id; + hypre_GetDevice(&device_id); + hypre_sprintf(resource_name, "%s::%d", "DEVICE", device_id); +#else + const hypre_int device_id = hypre_HandleDevice(handle); hypre_sprintf(resource_name, "%s::%d", "DEVICE", device_id); +#endif umpire_resourcemanager *rm_ptr = &hypre_HandleUmpireResourceMan(handle); umpire_allocator pooled_allocator; From b4c473f9d5602f927f8771b427adc8069fd8662a Mon Sep 17 00:00:00 2001 From: Wayne Mitchell Date: Thu, 30 Oct 2025 22:59:03 +0000 Subject: [PATCH 2/5] Needed to specify umpire when setting some things for building umpire in cmake --- src/config/cmake/HYPRE_CMakeUtilities.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 21f2abd47e..579d4c0bbf 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -649,9 +649,9 @@ function(maybe_build_umpire) set(ENABLE_EXAMPLES OFF CACHE BOOL "Disable Umpire examples" FORCE) set(ENABLE_DOCS OFF CACHE BOOL "Disable Umpire docs" FORCE) set(ENABLE_TESTS OFF CACHE BOOL "Disable Umpire tests" FORCE) - set(ENABLE_CUDA ${HYPRE_ENABLE_CUDA} CACHE BOOL "Enable CUDA in Umpire" FORCE) - set(ENABLE_HIP ${HYPRE_ENABLE_HIP} CACHE BOOL "Enable HIP in Umpire" FORCE) - set(ENABLE_SYCL ${HYPRE_ENABLE_SYCL} CACHE BOOL "Enable SYCL in Umpire" FORCE) + set(UMPIRE_ENABLE_CUDA ${HYPRE_ENABLE_CUDA} CACHE BOOL "Enable CUDA in Umpire" FORCE) + set(UMPIRE_ENABLE_HIP ${HYPRE_ENABLE_HIP} CACHE BOOL "Enable HIP in Umpire" FORCE) + set(UMPIRE_ENABLE_SYCL ${HYPRE_ENABLE_SYCL} CACHE BOOL "Enable SYCL in Umpire" FORCE) # Ensure Umpire installs to the same prefix as hypre set(CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix" FORCE) From a745d08452c3497ac1851fb487dedb5a01d17406 Mon Sep 17 00:00:00 2001 From: Wayne Mitchell Date: Fri, 31 Oct 2025 15:58:07 +0000 Subject: [PATCH 3/5] Revert to just ENABLE_CUDA/HIP where appropriate --- src/config/cmake/HYPRE_CMakeUtilities.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 579d4c0bbf..8c3689be04 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -649,8 +649,8 @@ function(maybe_build_umpire) set(ENABLE_EXAMPLES OFF CACHE BOOL "Disable Umpire examples" FORCE) set(ENABLE_DOCS OFF CACHE BOOL "Disable Umpire docs" FORCE) set(ENABLE_TESTS OFF CACHE BOOL "Disable Umpire tests" FORCE) - set(UMPIRE_ENABLE_CUDA ${HYPRE_ENABLE_CUDA} CACHE BOOL "Enable CUDA in Umpire" FORCE) - set(UMPIRE_ENABLE_HIP ${HYPRE_ENABLE_HIP} CACHE BOOL "Enable HIP in Umpire" FORCE) + set(ENABLE_CUDA ${HYPRE_ENABLE_CUDA} CACHE BOOL "Enable CUDA in Umpire" FORCE) + set(ENABLE_HIP ${HYPRE_ENABLE_HIP} CACHE BOOL "Enable HIP in Umpire" FORCE) set(UMPIRE_ENABLE_SYCL ${HYPRE_ENABLE_SYCL} CACHE BOOL "Enable SYCL in Umpire" FORCE) # Ensure Umpire installs to the same prefix as hypre From 678072021616a4e1dcbdeefffe5d133e0402ae71 Mon Sep 17 00:00:00 2001 From: Wayne Mitchell Date: Thu, 13 Nov 2025 20:35:14 +0000 Subject: [PATCH 4/5] Trying to pass our sycl queue to umpire. So far, no dice... this does not seem to have the intended effect. --- src/utilities/memory.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/utilities/memory.c b/src/utilities/memory.c index 2bac44ae78..09cf575a2c 100644 --- a/src/utilities/memory.c +++ b/src/utilities/memory.c @@ -1720,6 +1720,10 @@ hypre_umpire_host_pooled_realloc(void *ptr, size_t size) /*-------------------------------------------------------------------------- * hypre_umpire_device_pooled_allocate *--------------------------------------------------------------------------*/ +#include "umpire/Umpire.hpp" +#include "umpire/strategy/NamedAllocationStrategy.hpp" +#include "umpire/util/MemoryResourceTraits.hpp" +#include "umpire/ResourceManager.hpp" HYPRE_Int hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) @@ -1730,6 +1734,7 @@ hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) #if defined(HYPRE_USING_SYCL) HYPRE_Int device_id; + /* WM: TODO - the way we count/identify devices is not the same as in umpire... is this an issue? What's the best solution? */ hypre_GetDevice(&device_id); hypre_sprintf(resource_name, "%s::%d", "DEVICE", device_id); #else @@ -1747,7 +1752,15 @@ hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) else { umpire_allocator allocator; +#if defined(HYPRE_USING_SYCL) + umpire::ResourceManager *rm = static_cast(rm_ptr->addr); + auto traits{umpire::get_default_resource_traits(resource_name)}; + traits.queue = hypre_HandleComputeStream(hypre_handle()); + auto sycl_allocator{rm->makeResource(resource_name, traits)}; + allocator.addr = (void*)&sycl_allocator; +#else umpire_resourcemanager_get_allocator_by_name(rm_ptr, resource_name, &allocator); +#endif hypre_umpire_resourcemanager_make_allocator_pool(rm_ptr, pool_name, allocator, hypre_HandleUmpireDevicePoolSize(handle), hypre_HandleUmpireBlockSize(handle), &pooled_allocator); @@ -1755,6 +1768,9 @@ hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) hypre_HandleOwnUmpireDevicePool(handle) = 1; } + umpire::Allocator *pooled_allocator_obj = static_cast(pooled_allocator.addr); + hypre_printf("WM: debug - allocator queue = %p\n", pooled_allocator_obj->getAllocationStrategy()->getTraits().queue); + hypre_printf("WM: debug - hypre queue = %p\n", hypre_HandleComputeStream(hypre_handle())); *ptr = umpire_allocator_allocate(&pooled_allocator, nbytes); return hypre_error_flag; From 6c2c373795fe1c56eda671b6d7f22a833aadfc8e Mon Sep 17 00:00:00 2001 From: Wayne Mitchell Date: Fri, 14 Nov 2025 18:37:41 +0000 Subject: [PATCH 5/5] Tried some more things for passing the sycl queue to umpire, but still no luck. Commenting this out for now. --- src/utilities/memory.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/utilities/memory.c b/src/utilities/memory.c index 09cf575a2c..55d3ec547a 100644 --- a/src/utilities/memory.c +++ b/src/utilities/memory.c @@ -1721,7 +1721,7 @@ hypre_umpire_host_pooled_realloc(void *ptr, size_t size) * hypre_umpire_device_pooled_allocate *--------------------------------------------------------------------------*/ #include "umpire/Umpire.hpp" -#include "umpire/strategy/NamedAllocationStrategy.hpp" +#include "umpire/strategy/QuickPool.hpp" #include "umpire/util/MemoryResourceTraits.hpp" #include "umpire/ResourceManager.hpp" @@ -1751,26 +1751,31 @@ hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes) } else { +/* #if defined(HYPRE_USING_SYCL) */ + /* WM: the code below doesn't work... I get: Allocator with name "DEVICE::0" is already registered */ + /* umpire::ResourceManager *rm = static_cast(rm_ptr->addr); */ + /* auto traits{umpire::get_default_resource_traits(resource_name)}; */ + /* traits.queue = hypre_HandleComputeStream(hypre_handle()); */ + /* auto resource_allocator{rm->makeResource(resource_name, traits)}; */ + /* umpire::Allocator *pooled_allocator_obj_ptr = new umpire::Allocator; */ + /* *pooled_allocator_obj_ptr = rm->makeAllocator( */ + /* resource_name, resource_allocator, hypre_HandleUmpireDevicePoolSize(handle), hypre_HandleUmpireBlockSize(handle)); */ + /* pooled_allocator.addr = (void *) pooled_allocator_obj_ptr; */ + /* pooled_allocator.idtor = 1; */ +/* #else */ umpire_allocator allocator; -#if defined(HYPRE_USING_SYCL) - umpire::ResourceManager *rm = static_cast(rm_ptr->addr); - auto traits{umpire::get_default_resource_traits(resource_name)}; - traits.queue = hypre_HandleComputeStream(hypre_handle()); - auto sycl_allocator{rm->makeResource(resource_name, traits)}; - allocator.addr = (void*)&sycl_allocator; -#else umpire_resourcemanager_get_allocator_by_name(rm_ptr, resource_name, &allocator); -#endif hypre_umpire_resourcemanager_make_allocator_pool(rm_ptr, pool_name, allocator, hypre_HandleUmpireDevicePoolSize(handle), hypre_HandleUmpireBlockSize(handle), &pooled_allocator); +/* #endif */ hypre_HandleOwnUmpireDevicePool(handle) = 1; } - umpire::Allocator *pooled_allocator_obj = static_cast(pooled_allocator.addr); - hypre_printf("WM: debug - allocator queue = %p\n", pooled_allocator_obj->getAllocationStrategy()->getTraits().queue); - hypre_printf("WM: debug - hypre queue = %p\n", hypre_HandleComputeStream(hypre_handle())); + /* umpire::Allocator *pooled_allocator_obj = static_cast(pooled_allocator.addr); */ + /* hypre_printf("WM: debug - allocator queue = %p\n", pooled_allocator_obj->getAllocationStrategy()->getTraits().queue); */ + /* hypre_printf("WM: debug - hypre queue = %p\n", hypre_HandleComputeStream(hypre_handle())); */ *ptr = umpire_allocator_allocate(&pooled_allocator, nbytes); return hypre_error_flag;