Skip to content

Commit

Permalink
Vulkan objects cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@ksuprynowicz
ksuprynowicz committed Dec 20, 2024
1 parent 8e4ed8e commit d7f6a0b
Show file tree
Hide file tree
Showing 13 changed files with 256 additions and 164 deletions.
2 changes: 1 addition & 1 deletion libraries/display-plugins/src/display-plugins/VulkanDisplayPlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1125,7 +1125,7 @@ void VulkanDisplayPlugin::setupRenderPass() {
auto vkBackend = std::dynamic_pointer_cast<gpu::vk::VKBackend>(getBackend());
auto device = vkBackend->getContext().device->logicalDevice;
if (_renderPass) {
vkDestroyRenderPass(device, _renderPass, nullptr);
vkBackend->getContext().recycler.trashVkRenderPass(_renderPass);
}

VkAttachmentDescription attachment{};
Expand Down
108 changes: 42 additions & 66 deletions libraries/gpu-vk/src/gpu/vk/VKBackend.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,12 @@ VKBackend::VKBackend() {
}

VKBackend::~VKBackend() {
Q_ASSERT(isBackendShutdownComplete);
// FIXME queue up all the trash calls
// VKTODO: move to context
_context.destroyContext();
}

void VKBackend::shutdown() {
VK_CHECK_RESULT(vkQueueWaitIdle(_context.graphicsQueue));
VK_CHECK_RESULT(vkQueueWaitIdle(_context.transferQueue) );
VK_CHECK_RESULT(vkDeviceWaitIdle(_context.device->logicalDevice));
Expand All @@ -127,6 +131,8 @@ VKBackend::~VKBackend() {
_framesToReuse.resize(0);
_framePool.resize(0);

beforeShutdownCleanup();

{
size_t pipelineCacheDataSize;
VK_CHECK_RESULT(vkGetPipelineCacheData(_context.device->logicalDevice, _pipelineCache, &pipelineCacheDataSize, nullptr));
Expand All @@ -138,7 +144,7 @@ VKBackend::~VKBackend() {
}
}

_context.destroyContext();
isBackendShutdownComplete = true;
}

const std::string& VKBackend::getVersion() const {
Expand Down Expand Up @@ -649,6 +655,8 @@ void VKBackend::executeFrame(const FramePointer& frame) {
// Create descriptor pool
// VKTODO: delete descriptor pool after it's not needed
//_frameData._descriptorPool
// VKTODO: make sure that previous frame finished rendering
perFrameCleanup();
acquireFrameData();

int batch_count = 0;
Expand Down Expand Up @@ -818,7 +826,6 @@ void VKBackend::executeFrame(const FramePointer& frame) {
// Move pointer to current frame to property that will store it while it's being rendered and before it's recycled.
_currentlyRenderedFrame = _currentFrame;
releaseFrameData();

// loop through commands

//
Expand Down Expand Up @@ -1657,12 +1664,14 @@ void VKBackend::setupStereoSide(int side) {

vk::VKFramebuffer* VKBackend::syncGPUObject(const Framebuffer& framebuffer) {
// VKTODO
return vk::VKFramebuffer::sync(*this, framebuffer);
auto object = vk::VKFramebuffer::sync(*this, framebuffer);
return object;
}

VKBuffer* VKBackend::syncGPUObject(const Buffer& buffer) {
// VKTODO
return vk::VKBuffer::sync(*this, buffer);
auto object = vk::VKBuffer::sync(*this, buffer);
return object;
}

VKTexture* VKBackend::syncGPUObject(const Texture& texture) {
Expand Down Expand Up @@ -1756,12 +1765,14 @@ VKTexture* VKBackend::syncGPUObject(const Texture& texture) {
}
}

_textures.insert(object);
return object;
}

VKQuery* VKBackend::syncGPUObject(const Query& query) {
// VKTODO
return vk::VKQuery::sync(*this, query);
auto object = vk::VKQuery::sync(*this, query);
return object;
}

void VKBackend::blitToFramebuffer(gpu::Texture& input, gpu::Texture& output) {
Expand Down Expand Up @@ -2021,78 +2032,34 @@ void VKBackend::waitForGPU() {
VK_CHECK_RESULT(vkDeviceWaitIdle(_context.device->logicalDevice));
}

void VKBackend::Recycler::trashVkSampler(VkSampler& sampler) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkSamplers.push_back(sampler);
}

void VKBackend::Recycler::trashVkFramebuffer(VkFramebuffer& framebuffer) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkFramebuffer.push_back(framebuffer);
}

void VKBackend::Recycler::trashVkImageView(VkImageView& imageView) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkImageViews.push_back(imageView);
}

void VKBackend::Recycler::trashVkImage(VkImage& image) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkImages.push_back(image);
}

void VKBackend::Recycler::trashVkBuffer(VkBuffer& buffer) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkBuffers.push_back(buffer);
}

void VKBackend::Recycler::trashVkRenderPass(VkRenderPass& renderPass) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkRenderPasses.push_back(renderPass);
}

void VKBackend::Recycler::trashVkPipeline(VkPipeline& pipeline) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkPipelines.push_back(pipeline);
}

void VKBackend::Recycler::trashVkShaderModule(VkShaderModule& module) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
vkShaderModules.push_back(module);
}

void VKBackend::Recycler::trashVmaAllocation(VmaAllocation& allocation) {
std::lock_guard<std::recursive_mutex> lockGuard(recyclerMutex);
}

void VKBackend::perFrameCleanup() {
auto &recycler = _context.recycler;
std::lock_guard<std::recursive_mutex> lockGuard(recycler.recyclerMutex);
// Remove pointers to objects that were deleted during the frame.
for (auto framebuffer : recycler.deletedFramebuffers) {
framebuffers.erase(framebuffer);
_framebuffers.erase(framebuffer);
}

size_t capacityBeforeClear = recycler.deletedFramebuffers.capacity();
recycler.deletedFramebuffers.resize(0);
recycler.deletedFramebuffers.reserve(capacityBeforeClear);

for (auto buffer : recycler.deletedBuffers) {
buffers.erase(buffer);
_buffers.erase(buffer);
}

capacityBeforeClear = recycler.deletedBuffers.capacity();
recycler.deletedBuffers.resize(0);
recycler.deletedBuffers.reserve(capacityBeforeClear);

for (auto texture : recycler.deletedTextures) {
textures.erase(texture);
_textures.erase(texture);
}

capacityBeforeClear = recycler.deletedTextures.capacity();
recycler.deletedTextures.resize(0);
recycler.deletedTextures.reserve(capacityBeforeClear);

for (auto query : recycler.deletedQueries) {
queries.erase(query);
_queries.erase(query);
}

capacityBeforeClear = recycler.deletedQueries.capacity();
recycler.deletedQueries.resize(0);
recycler.deletedQueries.reserve(capacityBeforeClear);
Expand Down Expand Up @@ -2122,38 +2089,47 @@ void VKBackend::perFrameCleanup() {
vkDestroyPipeline(device, pipeline, nullptr);
}

for (auto swapchain : recycler.vkSwapchainsKHR) {
vkDestroySwapchainKHR(device, swapchain, nullptr);
}

for (auto surface: recycler.vkSurfacesKHR) {
vkDestroySurfaceKHR(_context.instance, surface, nullptr);
}

for (auto allocation : recycler.vmaAllocations) {
vmaFreeMemory(vks::Allocation::getAllocator(), allocation);
}
}

void VKBackend::beforeShutdownCleanup() {
auto &recycler = _context.recycler;
// Lock prevents destroying objects while hashes
std::lock_guard<std::recursive_mutex> lockGuard(recycler.recyclerMutex);
// Remove pointers to objects that were deleted during the frame.
// This prevents access-after-delete.
perFrameCleanup();

// Delete remaining backend objects.
for (auto framebuffer : framebuffers) {
for (auto framebuffer : _framebuffers) {
framebuffer->_gpuObject.gpuObject.setGPUObject(nullptr);
}
framebuffers.clear();
_framebuffers.clear();

for (auto buffer : buffers) {
for (auto buffer : _buffers) {
buffer->_gpuObject.gpuObject.setGPUObject(nullptr);
}
buffers.clear();
_buffers.clear();

for (auto texture : textures) {
for (auto texture : _textures) {
texture->_gpuObject.gpuObject.setGPUObject(nullptr);
}
textures.clear();
_textures.clear();

for (auto query : queries) {
for (auto query : _queries) {
query->_gpuObject.gpuObject.setGPUObject(nullptr);
}
queries.clear();
_queries.clear();

// Deleted objects got added to recycler, so they need to be cleaned since sets are already empty.
recycler.deletedFramebuffers.clear();
Expand Down
50 changes: 7 additions & 43 deletions libraries/gpu-vk/src/gpu/vk/VKBackend.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,45 +287,6 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
VKBackend *_backend;
};

// Contains objects that need to be deleted on Vulkan backend thread after frame is rendered.
// It's filled by destructors of objects like gpu::Texture and gpu::Buffer, since these destroy
// backend counterpart of their objects.
public:
class Recycler {
public:
// This means that for every GPU object mutex will be locked and unlocked several times.
// VKTODO: It would be good to do profiling and check if it impacts performance or not.
void trashVkSampler(VkSampler &sampler);
void trashVkFramebuffer(VkFramebuffer &framebuffer);
void trashVkImageView(VkImageView &imageView);
void trashVkImage(VkImage &image);
void trashVkBuffer(VkBuffer &buffer);
void trashVkRenderPass(VkRenderPass &renderPass);
void trashVkPipeline(VkPipeline &pipeline);
void trashVkShaderModule(VkShaderModule &module);
void trashVmaAllocation(VmaAllocation &allocation);

private:
std::recursive_mutex recyclerMutex;

std::vector<VkSampler> vkSamplers;
std::vector<VkFramebuffer> vkFramebuffer;
std::vector<VkImageView> vkImageViews;
std::vector<VkImage> vkImages;
std::vector<VkBuffer> vkBuffers;
std::vector<VkRenderPass> vkRenderPasses;
std::vector<VkPipeline> vkPipelines;
std::vector<VkShaderModule> vkShaderModules;
std::vector<VmaAllocation> vmaAllocations;

// List of pointers to objects that were deleted and need to be removed from backend object sets.
std::vector<VKFramebuffer*> deletedFramebuffers;
std::vector<VKBuffer*> deletedBuffers;
std::vector<VKTexture*> deletedTextures;
std::vector<VKQuery*> deletedQueries;
friend class VKBackend;
} recycler;

private:
void draw(VkPrimitiveTopology mode, uint32 numVertices, uint32 startVertex);
void renderPassTransfer(const Batch& batch);
Expand All @@ -345,6 +306,7 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
public:
VKBackend();
~VKBackend();
void shutdown() override;
vks::Context& getContext() { return _context; }
void syncProgram(const gpu::ShaderPointer& program) override {}
void syncCache() override {}
Expand Down Expand Up @@ -460,10 +422,10 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
// These are filled by syncGPUObject() calls, and are needed to track backend objects so that they can be destroyed before
// destroying backend.
// Access to these objects happens only from the backend thread. Destructors don't access them directly, but through a recycler.
std::unordered_set<VKFramebuffer*> framebuffers;
std::unordered_set<VKBuffer*> buffers;
std::unordered_set<VKTexture*> textures;
std::unordered_set<VKQuery*> queries;
std::unordered_set<VKFramebuffer*> _framebuffers;
std::unordered_set<VKBuffer*> _buffers;
std::unordered_set<VKTexture*> _textures;
std::unordered_set<VKQuery*> _queries;
void perFrameCleanup();
// Called by the destructor
void beforeShutdownCleanup();
Expand Down Expand Up @@ -493,6 +455,8 @@ class VKBackend : public Backend, public std::enable_shared_from_this<VKBackend>
std::shared_ptr<FrameData> _currentFrame;
// Frame for which command buffer is already generated and it's currently being rendered.
std::shared_ptr<FrameData> _currentlyRenderedFrame;
// Safety check to ensure that shutdown was completed before destruction.
std::atomic<bool> isBackendShutdownComplete{ false };

typedef void (VKBackend::*CommandCall)(const Batch&, size_t);
static std::array<VKBackend::CommandCall, Batch::NUM_COMMANDS> _commandCalls;
Expand Down
6 changes: 4 additions & 2 deletions libraries/gpu-vk/src/gpu/vk/VKBuffer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,10 @@ VKBuffer::~VKBuffer() {
Backend::bufferCount.decrement();
auto backend = _backend.lock();
if (backend) {
backend->recycler.trashVkBuffer(buffer);
backend->recycler.trashVmaAllocation(allocation);
auto &recycler = backend->getContext().recycler;
recycler.trashVkBuffer(buffer);
recycler.trashVmaAllocation(allocation);
recycler.bufferDeleted(this);
}
}

3 changes: 3 additions & 0 deletions libraries/gpu-vk/src/gpu/vk/VKFramebuffer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,9 @@ bool gpu::vk::VKFramebuffer::checkStatus(gpu::vk::VKFramebuffer::FramebufferStat
}

gpu::vk::VKFramebuffer::~VKFramebuffer() {
auto backend = _backend.lock();
auto &recycler = backend->getContext().recycler;
recycler.framebufferDeleted(this);
//VKTODO
/*if (_id) {
auto backend = _backend.lock();
Expand Down
3 changes: 3 additions & 0 deletions libraries/gpu-vk/src/gpu/vk/VKQuery.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ class VKQuery : public VKObject<Query> {
// VKTODO: We need a check on backend.lock(), or to pass backend reference instead
VKQuery(const std::weak_ptr<VKBackend>& backend, const Query& query) : Parent(*backend.lock(), query) {}
~VKQuery() {
auto backend = _backend.lock();
auto &recycler = backend->getContext().recycler;
recycler.queryDeleted(this);
// Vulkan handle is used instead
/*if (_id) {
// VKTODO
Expand Down
21 changes: 13 additions & 8 deletions libraries/gpu-vk/src/gpu/vk/VKTexture.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,9 @@ VKTexture::VKTexture(const std::weak_ptr<VKBackend>& backend, const Texture& tex
}

VKTexture::~VKTexture() {
qDebug() << "VKTexture destroyed: " << this;
auto backend = _backend.lock();
backend->getContext().recycler.textureDeleted(this);
if (backend && _vkImage == VK_NULL_HANDLE) {
// VKTODO
// backend->releaseTexture(_id, 0);
Expand Down Expand Up @@ -244,13 +246,15 @@ VKAttachmentTexture::~VKAttachmentTexture() {
// VKTODO: Redo destructors for cleanup to happen on present thread
auto backend = _backend.lock();
auto device = backend->getContext().device->logicalDevice;
auto &recycler = backend->getContext().recycler;
if (_vkImageView) {
vkDestroyImageView(device, _vkImageView, nullptr);
recycler.trashVkImageView(_vkImageView);
}
if (_vkSampler) {
vkDestroySampler(device, _vkSampler, nullptr);
recycler.trashVkSampler(_vkSampler);
}
vmaDestroyImage(vks::Allocation::getAllocator(), _vkImage, _vmaAllocation);
recycler.trashVkImage(_vkImage);
recycler.trashVmaAllocation(_vmaAllocation);
}

VkDescriptorImageInfo VKAttachmentTexture::getDescriptorImageInfo() {
Expand Down Expand Up @@ -541,12 +545,13 @@ void VKStrictResourceTexture::postTransfer(VKBackend &backend) {
VKStrictResourceTexture::~VKStrictResourceTexture() {
auto backend = _backend.lock();
auto device = backend->getContext().device->logicalDevice;
vkDestroyImageView(device, _vkImageView, nullptr);
if (_vkSampler)
{
vkDestroySampler(device, _vkSampler, nullptr);
auto &recycler = backend->getContext().recycler;
recycler.trashVkImageView(_vkImageView);
if (_vkSampler) {
recycler.trashVkSampler(_vkSampler);
}
vmaDestroyImage(vks::Allocation::getAllocator(), _vkImage, _vmaAllocation);
recycler.trashVkImage(_vkImage);
recycler.trashVmaAllocation(_vmaAllocation);
}

/*Size VKTexture::copyMipFaceFromTexture(uint16_t sourceMip, uint16_t targetMip, uint8_t face) const {
Expand Down
Loading

0 comments on commit d7f6a0b

Please sign in to comment.