From 1b962c1640c3d74b0afa19015a41ceaf6521aa49 Mon Sep 17 00:00:00 2001 From: Panagiotis Apostolou Date: Mon, 19 Jun 2023 12:37:47 +0300 Subject: [PATCH] Using userfaultfd in page guard manager --- android/layer/build.gradle | 2 +- android/tools/replay/build.gradle | 2 +- framework/util/page_guard_manager.cpp | 525 ++++++++++++++++++++++++-- framework/util/page_guard_manager.h | 50 ++- 4 files changed, 540 insertions(+), 39 deletions(-) diff --git a/android/layer/build.gradle b/android/layer/build.gradle index 1cea7b834e..d8a2074376 100644 --- a/android/layer/build.gradle +++ b/android/layer/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'com.android.library' android { compileSdkVersion 27 - ndkVersion '21.3.6528147' + ndkVersion '24.0.8215888' defaultConfig { minSdkVersion 26 targetSdkVersion 27 diff --git a/android/tools/replay/build.gradle b/android/tools/replay/build.gradle index 5bd6f58f1d..63ebcb1828 100644 --- a/android/tools/replay/build.gradle +++ b/android/tools/replay/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'com.android.application' android { compileSdkVersion 27 - ndkVersion '21.3.6528147' + ndkVersion '24.0.8215888' defaultConfig { applicationId "com.lunarg.gfxreconstruct.replay" minSdkVersion 26 diff --git a/framework/util/page_guard_manager.cpp b/framework/util/page_guard_manager.cpp index a5ab529c95..5a1c158ff6 100644 --- a/framework/util/page_guard_manager.cpp +++ b/framework/util/page_guard_manager.cpp @@ -29,6 +29,16 @@ #include #include +#include + +#if defined(USE_USERFAULTFD) +#include +#include +#include +#include +#include +#include +#endif GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(util) @@ -199,6 +209,377 @@ void PageGuardManager::InitializeSystemExceptionContext(void) #endif } +#if defined(USE_USERFAULTFD) +bool PageGuardManager::UffdInit() +{ + // open the userfault fd + uffd_fd_ = syscall(SYS_userfaultfd, UFFD_USER_MODE_ONLY | O_CLOEXEC | O_NONBLOCK); + if (uffd_fd_ == -1) + { + GFXRECON_LOG_DEBUG_ONCE("syscall/userfaultfd: %s", strerror(errno)); + return false; + } + + // enable for api version and check features + struct uffdio_api uffdio_api; + uffdio_api.api = UFFD_API; + uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP; + if (ioctl(uffd_fd_, UFFDIO_API, &uffdio_api) == -1) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_api: %s", strerror(errno)); + + uffd_fd_ = -1; + return false; + } + + if (uffdio_api.api != UFFD_API) + { + GFXRECON_LOG_ERROR("unsupported userfaultfd api"); + + uffd_fd_ = -1; + return false; + } + + const uint64_t required_features[] = { UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + for (size_t i = 0; i < sizeof(required_features) / sizeof(required_features[0]); ++i) + { + if ((uffdio_api.features & required_features[i]) != required_features[i]) + { + GFXRECON_LOG_ERROR("unsupported userfaultfd feature: 0x%" PRIx64 "\n", required_features[i]); + + uffd_fd_ = -1; + return false; + } + } + + const uint64_t requested_ioctls[] = { 1 << _UFFDIO_REGISTER }; + for (size_t i = 0; i < sizeof(requested_ioctls) / sizeof(requested_ioctls[0]); ++i) + { + if ((uffdio_api.ioctls & requested_ioctls[i]) != requested_ioctls[i]) + { + GFXRECON_LOG_ERROR("unsupported userfaultfd ioctl: 0x%" PRIx64 "\n", requested_ioctls[i]); + + uffd_fd_ = -1; + return false; + } + } + + return true; +} + +std::atomic PageGuardManager::uffd_stop_handler_thread = { false }; + +bool PageGuardManager::UffdResetEntryProtection(MemoryInfo* memory_info) +{ + UffdUnregisterMemory(memory_info->shadow_memory, memory_info->shadow_range); + + void* shadow_memory = memory_info->shadow_memory; + if (munmap(memory_info->shadow_memory, memory_info->shadow_range)) + { + GFXRECON_LOG_ERROR("munmap error: %s", strerror(errno)); + return false; + } + + memory_info->shadow_memory = mmap(memory_info->shadow_memory, + memory_info->shadow_range, + PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, + -1, + 0); + if (memory_info->shadow_memory == MAP_FAILED) + { + GFXRECON_LOG_ERROR("mmap error: %s", strerror(errno)); + memory_info->shadow_memory = nullptr; + return false; + } + else if (memory_info->shadow_memory != shadow_memory) + { + GFXRECON_LOG_ERROR("MAP_FIXED was not honored when remapping memory"); + return false; + } + + UffdRegisterMemory(memory_info->shadow_memory, memory_info->shadow_range); + + return true; +} + +bool PageGuardManager::UffdResetEntriesProtection() +{ + bool success = true; + + for (auto& entry : memory_info_) + { + auto memory_info = &entry.second; + + if (memory_info->is_modified) + { + if (!UffdResetEntryProtection(memory_info)) + { + success = false; + } + else + { + memory_info->is_modified = true; + } + } + } + + return success; +} + +void PageGuardManager::UffdHandleFault(int uffd_fd, uint64_t address, uint64_t flags) +{ + MemoryInfo* memory_info = nullptr; + + std::lock_guard lock(tracked_memory_lock_); + + bool found = FindMemory(reinterpret_cast(address), &memory_info); + if (found) + { + assert((memory_info != nullptr) && (memory_info->aligned_address != nullptr)); + assert(static_cast(address) >= reinterpret_cast(memory_info->aligned_address)); + + const bool is_write = (flags & UFFD_PAGEFAULT_FLAG_WRITE) == UFFD_PAGEFAULT_FLAG_WRITE; + + memory_info->is_modified = true; + + // Get the offset from the start of the first protected memory page to the current address. + size_t start_offset = reinterpret_cast(address) - static_cast(memory_info->aligned_address); + + size_t page_index = start_offset >> system_page_pot_shift_; + size_t page_offset = page_index << system_page_pot_shift_; + void* page_address = static_cast(memory_info->aligned_address) + page_offset; + size_t segment_size = GetMemorySegmentSize(memory_info, page_index); + + if (is_write) + { + memory_info->status_tracker.SetActiveWriteBlock(page_index, true); + } + else + { + memory_info->status_tracker.SetActiveReadBlock(page_index, true); + + if (enable_read_write_same_page_) + { + // The page guard has been removed from this page. If we expect both reads and writes to the page, + // it needs to be marked for active write. + memory_info->status_tracker.SetActiveWriteBlock(page_index, true); + } + } + + // Write protect fault + if ((flags & UFFD_PAGEFAULT_FLAG_WP) == UFFD_PAGEFAULT_FLAG_WP) + { + // Write fault + struct uffdio_writeprotect wp; + wp.range.start = address; + wp.range.len = system_page_size_; + wp.mode = 0; + + if (ioctl(uffd_fd, UFFDIO_WRITEPROTECT, &wp) == -1) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_writeprotect: %s", strerror(errno)); + } + } + else + { + // Missing page fault + + // Copy from the mapped memory to the shadow memory. + if (page_index == 0) + { + segment_size -= memory_info->aligned_offset; + } + else + { + page_offset -= memory_info->aligned_offset; + } + + assert(memory_info->shadow_memory != nullptr); + uint8_t* destination_address = static_cast(memory_info->shadow_memory) + page_offset; + uint8_t* source_address = static_cast(memory_info->mapped_memory) + page_offset; + + struct uffdio_copy copy; + copy.dst = reinterpret_cast(destination_address); + copy.src = reinterpret_cast(source_address); + copy.len = system_page_size_; + copy.mode = 0; + + assert(uffd_fd != -1); + if (ioctl(uffd_fd, UFFDIO_COPY, ©)) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_copy: %s", strerror(errno)); + } + + if (copy.copy != system_page_size_) + { + GFXRECON_LOG_ERROR("Unexpected copy.copy\n"); + } + } + } +} + +void* PageGuardManager::UffdHandlerThread(void* args) +{ + GFXRECON_UNREFERENCED_PARAMETER(args); + assert(uffd_fd_ != -1); + + for (;;) + { + struct pollfd pollfd[1]; + pollfd[0].fd = uffd_fd_; + pollfd[0].events = POLLIN | POLLERR | POLLPRI; + + // wait for a userfaultfd event to occur + int pollres = poll(pollfd, 1, 200); + + if (uffd_stop_handler_thread.load()) + { + break; + } + + switch (pollres) + { + case -1: + GFXRECON_LOG_WARNING("poll/userfaultfd: %s", strerror(errno)); + continue; + case 0: + continue; + case 1: + break; + default: + GFXRECON_LOG_WARNING("unexpected poll result: %d", pollres); + continue; + } + + if (pollfd[0].revents & POLLERR) + { + GFXRECON_LOG_ERROR("pollerr: %s", strerror(errno)); + continue; + } + + if ((pollfd[0].revents & POLLIN) != POLLIN) + { + continue; + } + + static struct uffd_msg msg[256]; + int readres = read(uffd_fd_, &msg, sizeof(msg)); + if (readres == -1) + { + if (errno == EAGAIN) + { + GFXRECON_LOG_ERROR("read/userfaultfd EAGAIN: %s", strerror(errno)); + continue; + } + + GFXRECON_LOG_ERROR("read/userfaultfd: %s", strerror(errno)); + break; + } + + if (readres % sizeof(uffd_msg)) + { + GFXRECON_LOG_ERROR("Unexpected read size\n"); + break; + } + + for (int i = 0; i < readres / sizeof(struct uffd_msg); ++i) + { + if (msg[i].event == UFFD_EVENT_PAGEFAULT) + { + UffdHandleFault(uffd_fd_, msg[i].arg.pagefault.address, msg[i].arg.pagefault.flags); + } + } + } + + uffd_stop_handler_thread = true; + + return nullptr; +} + +bool PageGuardManager::UffdStartHandlerThread() +{ + assert(uffd_fd_ != -1); + + if (pthread_create(&uffd_handler_thread_, nullptr, UffdHandlerThreadHelper, this)) + { + GFXRECON_LOG_ERROR("pthread_create: %s", strerror(errno)); + return false; + } + + return true; +} + +bool PageGuardManager::InitializeUserFaultFd() +{ + + if (!UffdInit()) + { + return false; + } + + if (!UffdStartHandlerThread()) + { + return false; + } + + return true; +} + +bool PageGuardManager::UffdRegisterMemory(const void* region, size_t region_size) +{ + assert(uffd_fd_ != -1); + + struct uffdio_register uffdio_register; + uffdio_register.range.start = (unsigned long)region; + uffdio_register.range.len = region_size; + uffdio_register.mode = UFFDIO_REGISTER_MODE_WP | UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd_fd_, UFFDIO_REGISTER, &uffdio_register) == -1) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_register: %s", strerror(errno)); + return false; + } + + const uint64_t zero_page_ioctl = (uint64_t)1 << _UFFDIO_ZEROPAGE; + const uint64_t copy_ioctl = (uint64_t)1 << _UFFDIO_COPY; + const uint64_t expected_ioctls = zero_page_ioctl | copy_ioctl; + if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls) + { + GFXRECON_LOG_ERROR("unexpected userfaultfd ioctl set (expected: 0x%llx got: 0x%llx)\n", + expected_ioctls, + uffdio_register.ioctls); + return false; + } + + struct uffdio_writeprotect prot_p; + prot_p.range.start = (unsigned long)region; + prot_p.range.len = region_size; + prot_p.mode = UFFDIO_WRITEPROTECT_MODE_WP; + if (ioctl(uffd_fd_, UFFDIO_WRITEPROTECT, &prot_p)) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_writeprotect: %s\n", strerror(errno)); + + return false; + } + + return true; +} + +void PageGuardManager::UffdUnregisterMemory(const void* address, size_t length) +{ + assert(uffd_fd_ != -1); + + struct uffdio_register uffdio_register; + uffdio_register.range.start = reinterpret_cast(address); + uffdio_register.range.len = static_cast(length); + uffdio_register.mode = 0; + if (ioctl(uffd_fd_, UFFDIO_UNREGISTER, &uffdio_register) == -1) + { + GFXRECON_LOG_ERROR("ioctl/uffdio_unregister: %s", strerror(errno)); + } +} +#endif // defined(USE_USERFAULTFD) + PageGuardManager::PageGuardManager() : exception_handler_(nullptr), exception_handler_count_(0), system_page_size_(util::platform::GetSystemPageSize()), system_page_pot_shift_(GetSystemPagePotShift()), enable_copy_on_map_(kDefaultEnableCopyOnMap), @@ -207,24 +588,67 @@ PageGuardManager::PageGuardManager() : signal_handler_watcher_max_restores_(kDefaultSignalHandlerWatcherMaxRestores), enable_read_write_same_page_(kDefaultEnableReadWriteSamePage) { - InitializeSystemExceptionContext(); +#if defined(USE_MPROTECT) + protection_mode_ = kMProtectMode; +#else + protection_mode_ = kUserFaultFdMode; + uffd_fd_ = -1; +#endif + + if (kMProtectMode == protection_mode_) + { + InitializeSystemExceptionContext(); + } + +#if defined(USE_USERFAULTFD) + else + { + if (!InitializeUserFaultFd()) + { + GFXRECON_LOG_ERROR("Failed to initialize UserFaultFD. Falling back to mprotect method."); + + protection_mode_ = kMProtectMode; + InitializeSystemExceptionContext(); + } + } +#endif } -PageGuardManager::PageGuardManager(bool enable_copy_on_map, - bool enable_separate_read, - bool expect_read_write_same_page, - bool unblock_SIGSEGV, - bool enable_signal_handler_watcher, - int signal_handler_watcher_max_restores) : +PageGuardManager::PageGuardManager(bool enable_copy_on_map, + bool enable_separate_read, + bool expect_read_write_same_page, + bool unblock_SIGSEGV, + bool enable_signal_handler_watcher, + int signal_handler_watcher_max_restores, + MemoryProtectionMode protection_mode) : exception_handler_(nullptr), exception_handler_count_(0), system_page_size_(util::platform::GetSystemPageSize()), system_page_pot_shift_(GetSystemPagePotShift()), enable_copy_on_map_(enable_copy_on_map), enable_separate_read_(enable_separate_read), unblock_sigsegv_(unblock_SIGSEGV), enable_signal_handler_watcher_(enable_signal_handler_watcher), signal_handler_watcher_max_restores_(signal_handler_watcher_max_restores), - enable_read_write_same_page_(expect_read_write_same_page) + enable_read_write_same_page_(expect_read_write_same_page), protection_mode_(protection_mode) { - InitializeSystemExceptionContext(); +#if !defined(USE_MPROTECT) + uffd_fd_ = -1; +#endif + + if (kMProtectMode == protection_mode_) + { + InitializeSystemExceptionContext(); + } +#if defined(USE_USERFAULTFD) + else + { + if (!InitializeUserFaultFd()) + { + GFXRECON_LOG_ERROR("Failed to initialize UserFaultFD. Falling back to mprotect method."); + + protection_mode_ = kMProtectMode; + InitializeSystemExceptionContext(); + } + } +#endif } PageGuardManager::~PageGuardManager() @@ -319,12 +743,19 @@ void PageGuardManager::Create(bool enable_copy_on_map, { if (instance_ == nullptr) { +#if defined(USE_MPROTECT) + MemoryProtectionMode mode = kMProtectMode; +#else + MemoryProtectionMode mode = kUserFaultFdMode; +#endif + instance_ = new PageGuardManager(enable_copy_on_map, enable_separate_read, expect_read_write_same_page, unblock_SIGSEGV, enable_signal_handler_watcher, - signal_handler_watcher_max_restores); + signal_handler_watcher_max_restores, + mode); #if !defined(WIN32) if (enable_signal_handler_watcher && @@ -748,6 +1179,12 @@ void PageGuardManager::ProcessEntry(uint64_t memory_id, { ProcessActiveRange(memory_id, memory_info, start_index, memory_info->total_pages, handle_modified); } + +#if defined(USE_USERFAULTFD) + { + UffdResetEntryProtection(memory_info); + } +#endif } void PageGuardManager::ProcessActiveRange(uint64_t memory_id, @@ -777,7 +1214,10 @@ void PageGuardManager::ProcessActiveRange(uint64_t memory_id, // Page guard was disabled when these pages were accessed. We enable it now for write, to // trap any writes made to the memory while we are performing the copy from shadow memory // to mapped memory. - SetMemoryProtection(guard_address, guard_range, kGuardReadOnlyProtect); + if (kMProtectMode == protection_mode_) + { + SetMemoryProtection(guard_address, guard_range, kGuardReadOnlyProtect); + } // Copy from shadow memory to the original mapped memory. if (start_index == 0) @@ -797,8 +1237,11 @@ void PageGuardManager::ProcessActiveRange(uint64_t memory_id, // the memory range. handle_modified(memory_id, memory_info->shadow_memory, page_offset, page_range); - // Reset page guard to detect both read and write accesses when using shadow memory. - SetMemoryProtection(guard_address, guard_range, kGuardReadWriteProtect); + if (kMProtectMode == protection_mode_) + { + // Reset page guard to detect both read and write accesses when using shadow memory. + SetMemoryProtection(guard_address, guard_range, kGuardReadWriteProtect); + } } else { @@ -806,8 +1249,11 @@ void PageGuardManager::ProcessActiveRange(uint64_t memory_id, { void* guard_address = static_cast(memory_info->aligned_address) + page_offset; - // Reset page guard to detect only write accesses when not using shadow memory. - SetMemoryProtection(guard_address, page_range, kGuardReadOnlyProtect); + if (kMProtectMode == protection_mode_) + { + // Reset page guard to detect only write accesses when not using shadow memory. + SetMemoryProtection(guard_address, page_range, kGuardReadOnlyProtect); + } } // Copy directly from the mapped memory. @@ -990,25 +1436,35 @@ void* PageGuardManager::AddTrackedMemory(uint64_t memory_id, start_address = shadow_memory; } - std::lock_guard lock(tracked_memory_lock_); - if (!use_write_watch) { - AddExceptionHandler(); - - // When using shadow memory, enable page guard for read and write operations so that shadow memory can be - // synchronized with the mapped memory on both read and write access. When not using shadow memory, only - // detect write access. - if (use_shadow_memory) +#if defined(USE_MPROTECT) { - success = SetMemoryProtection(aligned_address, guard_range, kGuardReadWriteProtect); + AddExceptionHandler(); + + // When using shadow memory, enable page guard for read and write operations so that shadow memory can + // be synchronized with the mapped memory on both read and write access. When not using shadow memory, + // only detect write access. + if (use_shadow_memory) + { + success = SetMemoryProtection(aligned_address, guard_range, kGuardReadWriteProtect); + } + else + { + success = SetMemoryProtection(aligned_address, guard_range, kGuardReadOnlyProtect); + } } - else +#else { - success = SetMemoryProtection(aligned_address, guard_range, kGuardReadOnlyProtect); + assert(kUserFaultFdMode == protection_mode_); + assert(shadow_memory == aligned_address); + success = UffdRegisterMemory(aligned_address, shadow_size); } +#endif } + std::lock_guard lock(tracked_memory_lock_); + if (success) { assert(memory_info_.find(memory_id) == memory_info_.end()); @@ -1033,8 +1489,16 @@ void* PageGuardManager::AddTrackedMemory(uint64_t memory_id, { if (!use_write_watch) { - RemoveExceptionHandler(); - SetMemoryProtection(aligned_address, guard_range, kGuardNoProtect); +#if defined(USE_MPROTECT) + { + RemoveExceptionHandler(); + SetMemoryProtection(aligned_address, guard_range, kGuardNoProtect); + } +#else + { + UffdUnregisterMemory(aligned_address, guard_range); + } +#endif } if (shadow_memory != nullptr) @@ -1067,11 +1531,14 @@ void PageGuardManager::RemoveTrackedMemory(uint64_t memory_id) if (!memory_info.use_write_watch) { +#if defined(USE_MPROTECT) RemoveExceptionHandler(); SetMemoryProtection( memory_info.aligned_address, memory_info.mapped_range + memory_info.aligned_offset, kGuardNoProtect); +#else + UffdUnregisterMemory(memory_info.shadow_memory, memory_info.shadow_range); +#endif } - if ((memory_info.shadow_memory != nullptr) && memory_info.own_shadow_memory) { FreeMemory(memory_info.shadow_memory, memory_info.shadow_range); diff --git a/framework/util/page_guard_manager.h b/framework/util/page_guard_manager.h index 467b314432..8c3cbb850d 100644 --- a/framework/util/page_guard_manager.h +++ b/framework/util/page_guard_manager.h @@ -41,6 +41,12 @@ #include #endif +#if defined(__ANDROID__) || defined(WIN32) +#define USE_MPROTECT +#else +#define USE_USERFAULTFD +#endif + GFXRECON_BEGIN_NAMESPACE(gfxrecon) GFXRECON_BEGIN_NAMESPACE(util) @@ -117,15 +123,22 @@ class PageGuardManager void FreePersistentShadowMemory(uintptr_t shadow_memory_handle); + enum MemoryProtectionMode + { + kMProtectMode, + kUserFaultFdMode + }; + protected: PageGuardManager(); - PageGuardManager(bool enable_copy_on_map, - bool enable_separate_read, - bool expect_read_write_same_page, - bool unblock_SIGSEGV, - bool enable_signal_handler_watcher, - int signal_handler_watcher_max_restores); + PageGuardManager(bool enable_copy_on_map, + bool enable_separate_read, + bool expect_read_write_same_page, + bool unblock_SIGSEGV, + bool enable_signal_handler_watcher, + int signal_handler_watcher_max_restores, + MemoryProtectionMode protection_mode); ~PageGuardManager(); @@ -245,13 +258,34 @@ class PageGuardManager const bool enable_read_write_same_page_; #if !defined(WIN32) - pthread_t signal_handler_watcher_thread_; + pthread_t signal_handler_watcher_thread_; + static uint32_t signal_handler_watcher_restores_; static void* SignalHandlerWatcher(void*); static bool CheckSignalHandler(); void MarkAllTrackedMemoryAsDirty(); +#endif - static uint32_t signal_handler_watcher_restores_; + MemoryProtectionMode protection_mode_; + +#if defined(USE_USERFAULTFD) + int uffd_fd_; + pthread_t uffd_handler_thread_; + static std::atomic uffd_stop_handler_thread; + + bool InitializeUserFaultFd(); + bool UffdInit(); + bool UffdStartHandlerThread(); + bool UffdRegisterMemory(const void* region, size_t region_size); + void UffdUnregisterMemory(const void* address, size_t length); + void UffdHandleFault(int uffd_fd, uint64_t address, uint64_t flags); + void* UffdHandlerThread(void* args); + bool UffdResetEntriesProtection(); + bool UffdResetEntryProtection(MemoryInfo* memory_entry); + static void* UffdHandlerThreadHelper(void* this_) + { + return (reinterpret_cast(this_)->UffdHandlerThread(nullptr)); + } #endif };