Skip to content

Commit

Permalink
Trace GPU events when ATLAS_TRACE_MEMORY=1
Browse files Browse the repository at this point in the history
  • Loading branch information
@wdeconinck
wdeconinck committed Apr 29, 2024
1 parent f36acd9 commit eae6350
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 11 deletions.
2 changes: 1 addition & 1 deletion src/atlas/array/ArrayDataStore.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
namespace atlas {
namespace array {

void throw_OutOfRange(const std::string& class_name, char idx_str, int idx, int max) {
void throw_OutOfRange(std::string_view class_name, char idx_str, int idx, int max) {
std::ostringstream msg;
msg << class_name << " index " << idx << " out of bounds: " << idx << " >= " << max;
throw_Exception(msg.str(), Here());
Expand Down
4 changes: 2 additions & 2 deletions src/atlas/array/ArrayDataStore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

#pragma once

#include <string>
#include <string_view>

#include "atlas/array/ArrayIdx.h"
#include "atlas/array/ArrayLayout.h"
Expand Down Expand Up @@ -81,7 +81,7 @@ static constexpr char array_dim() {
return Dim == 0 ? 'i' : (Dim == 1 ? 'j' : (Dim == 2 ? 'k' : (Dim == 3 ? 'l' : (Dim == 4 ? 'm' : ('*')))));
}

void throw_OutOfRange(const std::string& class_name, char idx_str, int idx, int max);
void throw_OutOfRange(std::string_view class_name, char idx_str, int idx, int max);
#endif

//------------------------------------------------------------------------------------------------------
Expand Down
47 changes: 39 additions & 8 deletions src/atlas/array/native/NativeDataStore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@
#include <cuda_runtime.h>
#endif

#include "eckit/log/Bytes.h"

#include "atlas/array/ArrayDataStore.h"
#include "atlas/library/Library.h"
#include "atlas/library/config.h"
#include "atlas/runtime/Exception.h"
#include "atlas/runtime/Log.h"
#include "eckit/log/Bytes.h"
#include "atlas/util/RegisterPointerInfo.h"

#if ATLAS_HAVE_ACC
#include "atlas_acc_support/atlas_acc_map_data.h"
Expand Down Expand Up @@ -117,11 +119,18 @@ class DataStore : public ArrayDataStore {
void updateDevice() const override {
#if ATLAS_HAVE_CUDA
if (not device_allocated_) {
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "updateDevice(" << name() << ") : device not allocated" << std::endl;
}
allocateDevice();
}
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "updateDevice(" << name() << ") : cudaMemcpyHostToDevice( device_ptr:" << device_data_ << " , host_ptr:"<< host_data_ << " , " << eckit::Bytes(size_*sizeof(Value)) << " ) " << std::endl;
}

cudaError_t err = cudaMemcpy(device_data_, host_data_, size_*sizeof(Value), cudaMemcpyHostToDevice);
if (err != cudaSuccess) {
throw_AssertionFailed("Failed to updateDevice: "+std::string(cudaGetErrorString(err)), Here());
throw_AssertionFailed("Failed to updateDevice("+std::string(name())+") : "+std::string(cudaGetErrorString(err)), Here());
}
device_updated_ = true;
#endif
Expand All @@ -130,9 +139,12 @@ class DataStore : public ArrayDataStore {
void updateHost() const override {
#if ATLAS_HAVE_CUDA
if (device_allocated_) {
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "updateHost(" << name() << ") : cudaMemcpyDeviceToHost( host_ptr:" << host_data_ << " , device_ptr:"<< device_data_ << " , " << eckit::Bytes(size_*sizeof(Value)) << " ) " << std::endl;
}
cudaError_t err = cudaMemcpy(host_data_, device_data_, size_*sizeof(Value), cudaMemcpyDeviceToHost);
if (err != cudaSuccess) {
throw_AssertionFailed("Failed to updateHost: "+std::string(cudaGetErrorString(err)), Here());
throw_AssertionFailed("Failed to updateHost("+std::string(name())+") : "+std::string(cudaGetErrorString(err)), Here());
}
host_updated_ = true;
}
Expand Down Expand Up @@ -169,9 +181,13 @@ class DataStore : public ArrayDataStore {
return;
}
if (size_) {
cudaError_t err = cudaMalloc((void**)&device_data_, sizeof(Value)*size_);
size_t bytes = sizeof(Value)*size_;
cudaError_t err = cudaMalloc((void**)&device_data_, bytes);
if (err != cudaSuccess) {
throw_AssertionFailed("Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here());
throw_AssertionFailed("allocateDevice("+std::string(name())+") Failed to allocate GPU memory: " + std::string(cudaGetErrorString(err)), Here());
}
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "allocateDevice(" << name() << ") : cudaMalloc( device_ptr:" << device_data_ << " , " << eckit::Bytes(bytes) << " )" << std::endl;
}
device_allocated_ = true;
accMap();
Expand All @@ -182,11 +198,15 @@ class DataStore : public ArrayDataStore {
void deallocateDevice() const override {
#if ATLAS_HAVE_CUDA
if (device_allocated_) {
accUnmap();
size_t bytes = sizeof(Value)*size_;
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "deallocateDevice(" << name() << ") : cudaFree( device_ptr:" << device_data_ << " , " << eckit::Bytes(bytes) << " )" << std::endl;
}
cudaError_t err = cudaFree(device_data_);
if (err != cudaSuccess) {
throw_AssertionFailed("Failed to deallocate GPU memory: " + std::string(cudaGetErrorString(err)), Here());
throw_AssertionFailed("Failed to deallocateDevice("+std::string(name())+") : " + std::string(cudaGetErrorString(err)), Here());
}
accUnmap();
device_data_ = nullptr;
device_allocated_ = false;
}
Expand Down Expand Up @@ -214,7 +234,10 @@ class DataStore : public ArrayDataStore {
void accMap() const override {
#if ATLAS_HAVE_ACC
if (not acc_mapped_) {
ATLAS_ASSERT(deviceAllocated(),"Could not accMap as device data is not allocated");
ATLAS_ASSERT(deviceAllocated(),"Could not accMap("+std::string(name())+") as device data is not allocated");
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "accMap("+std::string(name())+") : atlas_acc_map_data( host_ptr:" << host_data_ << " , device_ptr:" << device_data_ << " , " << eckit::Bytes(size_ * sizeof(Value)) << " )" << std::endl;
}
atlas_acc_map_data((void*)host_data_, (void*)device_data_, size_ * sizeof(Value));
acc_mapped_ = true;
}
Expand All @@ -228,6 +251,9 @@ class DataStore : public ArrayDataStore {
void accUnmap() const override {
#if ATLAS_HAVE_ACC
if (acc_mapped_) {
if (atlas::Library::instance().traceMemory()) {
Log::trace() << "accUnmap(" << name() << ") : atlas_acc_unmap_data( host_ptr:" << host_data_ << " )" << std::endl;
}
atlas_acc_unmap_data(host_data_);
acc_mapped_ = false;
}
Expand Down Expand Up @@ -276,6 +302,11 @@ class DataStore : public ArrayDataStore {

size_t footprint() const { return sizeof(Value) * size_; }


std::string_view name() const {
return util::registered_pointer_name(this);
}

size_t size_;
Value* host_data_;
mutable Value* device_data_{nullptr};
Expand Down
9 changes: 9 additions & 0 deletions src/atlas/field/detail/FieldImpl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
#include <sstream>

#include "atlas/library/config.h"
#include "atlas/library/Library.h"

#include "atlas/array/MakeView.h"
#include "atlas/field/FieldCreator.h"
#include "atlas/field/detail/FieldImpl.h"
#include "atlas/runtime/Exception.h"
#include "atlas/runtime/Log.h"
#include "atlas/util/RegisterPointerInfo.h"

#if ATLAS_HAVE_FUNCTIONSPACE
#include "atlas/functionspace/FunctionSpace.h"
Expand Down Expand Up @@ -99,7 +101,11 @@ FieldImpl::~FieldImpl() {
for (auto& f : callback_on_destruction_) {
f();
}
const void* ds = &array_->data_store();
delete array_;
if( atlas::Library::instance().traceMemory()) {
util::unregister_pointer_name(ds);
}
}
#if ATLAS_HAVE_FUNCTIONSPACE
delete functionspace_;
Expand Down Expand Up @@ -152,6 +158,9 @@ void FieldImpl::rename(const std::string& name) {
for (FieldObserver* observer : field_observers_) {
observer->onFieldRename(*this);
}
if( atlas::Library::instance().traceMemory()) {
util::register_pointer_name(&array().data_store(), name);
}
}

const std::string& FieldImpl::name() const {
Expand Down

0 comments on commit eae6350

Please sign in to comment.