From eae6350c4e95be97e25d3e879be1af91bd27b091 Mon Sep 17 00:00:00 2001 From: Willem Deconinck Date: Fri, 26 Apr 2024 15:48:29 +0000 Subject: [PATCH] Trace GPU events when ATLAS_TRACE_MEMORY=1 --- src/atlas/array/ArrayDataStore.cc | 2 +- src/atlas/array/ArrayDataStore.h | 4 +- src/atlas/array/native/NativeDataStore.h | 47 ++++++++++++++++++++---- src/atlas/field/detail/FieldImpl.cc | 9 +++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/atlas/array/ArrayDataStore.cc b/src/atlas/array/ArrayDataStore.cc index 5b4ee9e62..9d8b4803c 100644 --- a/src/atlas/array/ArrayDataStore.cc +++ b/src/atlas/array/ArrayDataStore.cc @@ -16,7 +16,7 @@ namespace atlas { namespace array { -void throw_OutOfRange(const std::string& class_name, char idx_str, int idx, int max) { +void throw_OutOfRange(std::string_view class_name, char idx_str, int idx, int max) { std::ostringstream msg; msg << class_name << " index " << idx << " out of bounds: " << idx << " >= " << max; throw_Exception(msg.str(), Here()); diff --git a/src/atlas/array/ArrayDataStore.h b/src/atlas/array/ArrayDataStore.h index 2c4ff0c2a..f5963ecf0 100644 --- a/src/atlas/array/ArrayDataStore.h +++ b/src/atlas/array/ArrayDataStore.h @@ -10,7 +10,7 @@ #pragma once -#include +#include #include "atlas/array/ArrayIdx.h" #include "atlas/array/ArrayLayout.h" @@ -81,7 +81,7 @@ static constexpr char array_dim() { return Dim == 0 ? 'i' : (Dim == 1 ? 'j' : (Dim == 2 ? 'k' : (Dim == 3 ? 'l' : (Dim == 4 ? 'm' : ('*'))))); } -void throw_OutOfRange(const std::string& class_name, char idx_str, int idx, int max); +void throw_OutOfRange(std::string_view class_name, char idx_str, int idx, int max); #endif //------------------------------------------------------------------------------------------------------ diff --git a/src/atlas/array/native/NativeDataStore.h b/src/atlas/array/native/NativeDataStore.h index 895e41291..2499219b8 100644 --- a/src/atlas/array/native/NativeDataStore.h +++ b/src/atlas/array/native/NativeDataStore.h @@ -20,12 +20,14 @@ #include #endif +#include "eckit/log/Bytes.h" + #include "atlas/array/ArrayDataStore.h" #include "atlas/library/Library.h" #include "atlas/library/config.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" -#include "eckit/log/Bytes.h" +#include "atlas/util/RegisterPointerInfo.h" #if ATLAS_HAVE_ACC #include "atlas_acc_support/atlas_acc_map_data.h" @@ -117,11 +119,18 @@ class DataStore : public ArrayDataStore { void updateDevice() const override { #if ATLAS_HAVE_CUDA if (not device_allocated_) { + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "updateDevice(" << name() << ") : device not allocated" << std::endl; + } allocateDevice(); } + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "updateDevice(" << name() << ") : cudaMemcpyHostToDevice( device_ptr:" << device_data_ << " , host_ptr:"<< host_data_ << " , " << eckit::Bytes(size_*sizeof(Value)) << " ) " << std::endl; + } + cudaError_t err = cudaMemcpy(device_data_, host_data_, size_*sizeof(Value), cudaMemcpyHostToDevice); if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateDevice: "+std::string(cudaGetErrorString(err)), Here()); + throw_AssertionFailed("Failed to updateDevice("+std::string(name())+") : "+std::string(cudaGetErrorString(err)), Here()); } device_updated_ = true; #endif @@ -130,9 +139,12 @@ class DataStore : public ArrayDataStore { void updateHost() const override { #if ATLAS_HAVE_CUDA if (device_allocated_) { + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "updateHost(" << name() << ") : cudaMemcpyDeviceToHost( host_ptr:" << host_data_ << " , device_ptr:"<< device_data_ << " , " << eckit::Bytes(size_*sizeof(Value)) << " ) " << std::endl; + } cudaError_t err = cudaMemcpy(host_data_, device_data_, size_*sizeof(Value), cudaMemcpyDeviceToHost); if (err != cudaSuccess) { - throw_AssertionFailed("Failed to updateHost: "+std::string(cudaGetErrorString(err)), Here()); + throw_AssertionFailed("Failed to updateHost("+std::string(name())+") : "+std::string(cudaGetErrorString(err)), Here()); } host_updated_ = true; } @@ -169,9 +181,13 @@ class DataStore : public ArrayDataStore { return; } if (size_) { - cudaError_t err = cudaMalloc((void**)&device_data_, sizeof(Value)*size_); + size_t bytes = sizeof(Value)*size_; + cudaError_t err = cudaMalloc((void**)&device_data_, bytes); if (err != cudaSuccess) { - throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + throw_AssertionFailed("allocateDevice("+std::string(name())+") Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + } + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "allocateDevice(" << name() << ") : cudaMalloc( device_ptr:" << device_data_ << " , " << eckit::Bytes(bytes) << " )" << std::endl; } device_allocated_ = true; accMap(); @@ -182,11 +198,15 @@ class DataStore : public ArrayDataStore { void deallocateDevice() const override { #if ATLAS_HAVE_CUDA if (device_allocated_) { - accUnmap(); + size_t bytes = sizeof(Value)*size_; + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "deallocateDevice(" << name() << ") : cudaFree( device_ptr:" << device_data_ << " , " << eckit::Bytes(bytes) << " )" << std::endl; + } cudaError_t err = cudaFree(device_data_); if (err != cudaSuccess) { - throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(cudaGetErrorString(err)), Here()); + throw_AssertionFailed("Failed to deallocateDevice("+std::string(name())+") : " + std::string(cudaGetErrorString(err)), Here()); } + accUnmap(); device_data_ = nullptr; device_allocated_ = false; } @@ -214,7 +234,10 @@ class DataStore : public ArrayDataStore { void accMap() const override { #if ATLAS_HAVE_ACC if (not acc_mapped_) { - ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated"); + ATLAS_ASSERT(deviceAllocated(),"Could not accMap("+std::string(name())+") as device data is not allocated"); + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "accMap("+std::string(name())+") : atlas_acc_map_data( host_ptr:" << host_data_ << " , device_ptr:" << device_data_ << " , " << eckit::Bytes(size_ * sizeof(Value)) << " )" << std::endl; + } atlas_acc_map_data((void*)host_data_, (void*)device_data_, size_ * sizeof(Value)); acc_mapped_ = true; } @@ -228,6 +251,9 @@ class DataStore : public ArrayDataStore { void accUnmap() const override { #if ATLAS_HAVE_ACC if (acc_mapped_) { + if (atlas::Library::instance().traceMemory()) { + Log::trace() << "accUnmap(" << name() << ") : atlas_acc_unmap_data( host_ptr:" << host_data_ << " )" << std::endl; + } atlas_acc_unmap_data(host_data_); acc_mapped_ = false; } @@ -276,6 +302,11 @@ class DataStore : public ArrayDataStore { size_t footprint() const { return sizeof(Value) * size_; } + + std::string_view name() const { + return util::registered_pointer_name(this); + } + size_t size_; Value* host_data_; mutable Value* device_data_{nullptr}; diff --git a/src/atlas/field/detail/FieldImpl.cc b/src/atlas/field/detail/FieldImpl.cc index 78d929f23..f4d5ec889 100644 --- a/src/atlas/field/detail/FieldImpl.cc +++ b/src/atlas/field/detail/FieldImpl.cc @@ -12,12 +12,14 @@ #include #include "atlas/library/config.h" +#include "atlas/library/Library.h" #include "atlas/array/MakeView.h" #include "atlas/field/FieldCreator.h" #include "atlas/field/detail/FieldImpl.h" #include "atlas/runtime/Exception.h" #include "atlas/runtime/Log.h" +#include "atlas/util/RegisterPointerInfo.h" #if ATLAS_HAVE_FUNCTIONSPACE #include "atlas/functionspace/FunctionSpace.h" @@ -99,7 +101,11 @@ FieldImpl::~FieldImpl() { for (auto& f : callback_on_destruction_) { f(); } + const void* ds = &array_->data_store(); delete array_; + if( atlas::Library::instance().traceMemory()) { + util::unregister_pointer_name(ds); + } } #if ATLAS_HAVE_FUNCTIONSPACE delete functionspace_; @@ -152,6 +158,9 @@ void FieldImpl::rename(const std::string& name) { for (FieldObserver* observer : field_observers_) { observer->onFieldRename(*this); } + if( atlas::Library::instance().traceMemory()) { + util::register_pointer_name(&array().data_store(), name); + } } const std::string& FieldImpl::name() const {