Sycl umpire

src/config/cmake/HYPRE_CMakeUtilities.cmake

@@ -651,7 +651,7 @@ function(maybe_build_umpire)
   set(ENABLE_TESTS                 OFF CACHE BOOL "Disable Umpire tests" FORCE)
   set(ENABLE_CUDA ${HYPRE_ENABLE_CUDA} CACHE BOOL "Enable CUDA in Umpire" FORCE)
   set(ENABLE_HIP  ${HYPRE_ENABLE_HIP}  CACHE BOOL "Enable HIP in Umpire" FORCE)
-  set(ENABLE_SYCL ${HYPRE_ENABLE_SYCL} CACHE BOOL "Enable SYCL in Umpire" FORCE)
+  set(UMPIRE_ENABLE_SYCL ${HYPRE_ENABLE_SYCL} CACHE BOOL "Enable SYCL in Umpire" FORCE)
 
   # Ensure Umpire installs to the same prefix as hypre
   set(CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Install prefix" FORCE)

src/utilities/memory.c

@@ -1720,16 +1720,27 @@ hypre_umpire_host_pooled_realloc(void *ptr, size_t size)
 /*--------------------------------------------------------------------------
  * hypre_umpire_device_pooled_allocate
  *--------------------------------------------------------------------------*/
+#include "umpire/Umpire.hpp"
+#include "umpire/strategy/QuickPool.hpp"
+#include "umpire/util/MemoryResourceTraits.hpp"
+#include "umpire/ResourceManager.hpp"
 
 HYPRE_Int
 hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes)
 {
    hypre_Handle *handle = hypre_handle();
-   const hypre_int device_id = hypre_HandleDevice(handle);
    char resource_name[16];
    const char *pool_name = hypre_HandleUmpireDevicePoolName(handle);
 
+#if defined(HYPRE_USING_SYCL)
+   HYPRE_Int device_id;
+   /* WM: TODO - the way we count/identify devices is not the same as in umpire... is this an issue? What's the best solution? */
+   hypre_GetDevice(&device_id);
+   hypre_sprintf(resource_name, "%s::%d", "DEVICE", device_id);
+#else
+   const hypre_int device_id = hypre_HandleDevice(handle);
    hypre_sprintf(resource_name, "%s::%d", "DEVICE", device_id);
+#endif
 
    umpire_resourcemanager *rm_ptr = &hypre_HandleUmpireResourceMan(handle);
    umpire_allocator pooled_allocator;
@@ -1740,15 +1751,31 @@ hypre_umpire_device_pooled_allocate(void **ptr, size_t nbytes)
    }
    else
    {
+/* #if defined(HYPRE_USING_SYCL) */
+      /* WM: the code below doesn't work... I get:  Allocator with name "DEVICE::0" is already registered */
+      /* umpire::ResourceManager *rm = static_cast<umpire::ResourceManager *>(rm_ptr->addr); */
+      /* auto traits{umpire::get_default_resource_traits(resource_name)}; */
+      /* traits.queue = hypre_HandleComputeStream(hypre_handle()); */
+      /* auto resource_allocator{rm->makeResource(resource_name, traits)}; */
+      /* umpire::Allocator *pooled_allocator_obj_ptr = new umpire::Allocator; */
+      /* *pooled_allocator_obj_ptr = rm->makeAllocator<umpire::strategy::QuickPool>( */
+      /*     resource_name, resource_allocator, hypre_HandleUmpireDevicePoolSize(handle), hypre_HandleUmpireBlockSize(handle)); */
+      /* pooled_allocator.addr = (void *) pooled_allocator_obj_ptr; */
+      /* pooled_allocator.idtor = 1; */
+/* #else */
       umpire_allocator allocator;
       umpire_resourcemanager_get_allocator_by_name(rm_ptr, resource_name, &allocator);
       hypre_umpire_resourcemanager_make_allocator_pool(rm_ptr, pool_name, allocator,
                                                        hypre_HandleUmpireDevicePoolSize(handle),
                                                        hypre_HandleUmpireBlockSize(handle), &pooled_allocator);
+/* #endif */
 
       hypre_HandleOwnUmpireDevicePool(handle) = 1;
    }
 
+   /* umpire::Allocator *pooled_allocator_obj = static_cast<umpire::Allocator *>(pooled_allocator.addr); */
+   /* hypre_printf("WM: debug - allocator queue = %p\n", pooled_allocator_obj->getAllocationStrategy()->getTraits().queue); */
+   /* hypre_printf("WM: debug - hypre queue = %p\n", hypre_HandleComputeStream(hypre_handle())); */
    *ptr = umpire_allocator_allocate(&pooled_allocator, nbytes);
 
    return hypre_error_flag;