From 07fb06ce018fa133d7eb3f206ff7e0c829918b71 Mon Sep 17 00:00:00 2001 From: jinlong Date: Thu, 13 Aug 2026 16:19:08 +0800 Subject: [PATCH] perf(hypervisor): speed up incremental snapshots with pagemap bit 61 Cut metadata scanning from one /proc/kpageflags syscall per present page to a single sequential pagemap read. Classify CoW pages with must_save = swapped || (present && !PM_FILE). Use bit 61 on Linux 6.6.44+ / 6.11+ / 7+; keep kpageflags on older kernels. Signed-off-by: jinlong --- hypervisor/vm-migration/src/lib.rs | 3 +- hypervisor/vmm/src/kernel_release.rs | 221 +++++++++++++++ hypervisor/vmm/src/lib.rs | 1 + hypervisor/vmm/src/memory_manager.rs | 3 +- hypervisor/vmm/src/pagemap_anon.rs | 329 +++++++++++++++++------ hypervisor/vmm/src/pagemap_anon_bench.rs | 173 ++++++++++++ hypervisor/vmm/src/soft_dirty.rs | 16 +- 7 files changed, 663 insertions(+), 83 deletions(-) create mode 100644 hypervisor/vmm/src/kernel_release.rs create mode 100644 hypervisor/vmm/src/pagemap_anon_bench.rs diff --git a/hypervisor/vm-migration/src/lib.rs b/hypervisor/vm-migration/src/lib.rs index 215e5eef5..df0aa9e36 100644 --- a/hypervisor/vm-migration/src/lib.rs +++ b/hypervisor/vm-migration/src/lib.rs @@ -111,7 +111,8 @@ pub enum SnapshotType { /// Full snapshot - saves complete VM memory #[default] Full, - /// Incremental snapshot - only saves CoW anonymous pages via pagemap + kpageflags + /// Incremental snapshot - only saves CoW anonymous pages (pagemap bit 61 + /// on Linux 6.6.44+ / 6.11+ / 7+; `/proc/kpageflags` on older kernels) Incremental, /// Soft-dirty snapshot - only saves pages written since the previous /// soft-dirty snapshot (true delta), via /proc/self/clear_refs + diff --git a/hypervisor/vmm/src/kernel_release.rs b/hypervisor/vmm/src/kernel_release.rs new file mode 100644 index 000000000..51ead7a60 --- /dev/null +++ b/hypervisor/vmm/src/kernel_release.rs @@ -0,0 +1,221 @@ +// Copyright © 2026 Tencent Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +//! Host kernel release (`uname -r`) and the pagemap bit-61 version gate. +//! +//! File-PMD `PM_FILE` reporting was fixed in upstream `3f9f022` (6.6.44 on +//! the 6.6 stable line, about 6.11 on mainline). A naive +//! `(major, minor, patch) >= (6, 6, 44)` compare would treat 6.7–6.10 as +//! new enough; those trees still omit `PM_FILE` on file PMDs. + +use log::info; +use once_cell::sync::Lazy; +use std::fmt; + +/// Parsed leading `major.minor.patch` from a `uname -r` string. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct KernelRelease { + pub major: u32, + pub minor: u32, + pub patch: u32, +} + +impl KernelRelease { + /// Parse leading `major.minor[.patch]` from a `uname -r` string. + /// + /// Distro suffixes are ignored (`6.6.44-1.el9` → `6.6.44`). A missing + /// patch is `0` (`6.6` → `6.6.0`). Returns `None` when the string does + /// not start with at least `major.minor`. + pub(crate) fn parse(release: &str) -> Option { + let mut nums = [0u32; 3]; + let mut count = 0usize; + let bytes = release.as_bytes(); + let mut i = 0usize; + while count < 3 { + if i >= bytes.len() || !bytes[i].is_ascii_digit() { + break; + } + let start = i; + while i < bytes.len() && bytes[i].is_ascii_digit() { + i += 1; + } + nums[count] = release[start..i].parse().ok()?; + count += 1; + if i < bytes.len() && bytes[i] == b'.' { + i += 1; + continue; + } + break; + } + (count >= 2).then_some(Self { + major: nums[0], + minor: nums[1], + patch: nums[2], + }) + } + + /// Host kernel release string (`uname -r`). + /// + /// Prefers `/proc/sys/kernel/osrelease` so the gate does not depend on the + /// `libc` `utsname` layout. Falls back to `uname(2)`. Empty if both fail. + pub(crate) fn uname_string() -> String { + if let Ok(s) = std::fs::read_to_string("/proc/sys/kernel/osrelease") { + let s = s.trim(); + if !s.is_empty() { + return s.to_string(); + } + } + let mut uts = std::mem::MaybeUninit::::uninit(); + // SAFETY: `uname` writes a complete `utsname` on success. + if unsafe { libc::uname(uts.as_mut_ptr()) } != 0 { + return String::new(); + } + // SAFETY: `uname` succeeded, so `uts` is initialized and `release` is + // a NUL-terminated kernel string. + let uts = unsafe { uts.assume_init() }; + let release = unsafe { std::ffi::CStr::from_ptr(uts.release.as_ptr()) }; + release.to_string_lossy().into_owned() + } + + /// Whether this kernel is known to set `PM_FILE` on file PMDs. + pub(crate) fn supports_pm_file_pmd(self) -> bool { + match (self.major, self.minor, self.patch) { + (6, 6, patch) => patch >= 44, + (6, minor, _) => minor >= 11, + (major, _, _) => major >= 7, + } + } +} + +impl fmt::Display for KernelRelease { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}.{}.{}", self.major, self.minor, self.patch) + } +} + +/// Cached scan-path decision for `pagemap_anon`. +/// +/// Bit 61 (`PM_FILE`) is only used when the kernel is known to set it on +/// file PMDs. Everything else, including an unparseable release, uses +/// `kpageflags`. +pub(crate) struct PagemapScanPath { + use_bit61: bool, +} + +impl PagemapScanPath { + fn detect() -> Self { + let release = KernelRelease::uname_string(); + let use_bit61 = + KernelRelease::parse(&release).is_some_and(KernelRelease::supports_pm_file_pmd); + info!( + "pagemap_anon: kernel={} path={}", + if release.is_empty() { + "unknown" + } else { + release.as_str() + }, + if use_bit61 { "bit61" } else { "kpageflags" } + ); + Self { use_bit61 } + } + + /// Host decision, probed once from `uname`. + pub(crate) fn cached() -> &'static Self { + static PATH: Lazy = Lazy::new(PagemapScanPath::detect); + &PATH + } + + pub(crate) fn use_bit61(&self) -> bool { + self.use_bit61 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_uname_release_strings() { + assert_eq!( + KernelRelease::parse("6.6.44"), + Some(KernelRelease { + major: 6, + minor: 6, + patch: 44 + }) + ); + assert_eq!( + KernelRelease::parse("6.6.44-1.el9"), + Some(KernelRelease { + major: 6, + minor: 6, + patch: 44 + }) + ); + assert_eq!( + KernelRelease::parse("6.6"), + Some(KernelRelease { + major: 6, + minor: 6, + patch: 0 + }) + ); + assert_eq!( + KernelRelease::parse("7.0.0-28-generic"), + Some(KernelRelease { + major: 7, + minor: 0, + patch: 0 + }) + ); + assert_eq!( + KernelRelease::parse("5.15.0-91-generic"), + Some(KernelRelease { + major: 5, + minor: 15, + patch: 0 + }) + ); + assert_eq!( + KernelRelease::parse("6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2"), + Some(KernelRelease { + major: 6, + minor: 6, + patch: 69 + }) + ); + assert_eq!(KernelRelease::parse(""), None); + assert_eq!(KernelRelease::parse("abc"), None); + assert_eq!(KernelRelease::parse("6"), None); + assert_eq!(KernelRelease::parse("linux-6.6.44"), None); + } + + #[test] + fn pm_file_pmd_gate() { + let cases = [ + ("6.6.43", false), + ("6.6.44", true), + ("6.6.45", true), + ( + "6.6.69-opencloudos9.cubesandbox.pvm.host-gb85200d80fa2", + true, + ), + ("6.6.44-1.el9", true), + ("6.6", false), + ("6.7.0", false), + ("6.10.0", false), + ("6.11.0", true), + ("6.12.1", true), + ("7.0.0-28-generic", true), + ("5.15.0", false), + ("", false), + ("not-a-version", false), + ]; + for (release, expected) in cases { + let got = + KernelRelease::parse(release).is_some_and(KernelRelease::supports_pm_file_pmd); + assert_eq!(got, expected, "release={release}"); + } + } +} diff --git a/hypervisor/vmm/src/lib.rs b/hypervisor/vmm/src/lib.rs index 1a1d7bd49..c4c320a12 100644 --- a/hypervisor/vmm/src/lib.rs +++ b/hypervisor/vmm/src/lib.rs @@ -77,6 +77,7 @@ pub mod device_tree; #[cfg(feature = "guest_debug")] mod gdb; pub mod interrupt; +mod kernel_release; pub mod memory_manager; pub mod migration; pub mod pagemap_anon; diff --git a/hypervisor/vmm/src/memory_manager.rs b/hypervisor/vmm/src/memory_manager.rs index fd9616a08..93da96843 100644 --- a/hypervisor/vmm/src/memory_manager.rs +++ b/hypervisor/vmm/src/memory_manager.rs @@ -2396,7 +2396,8 @@ impl MemoryManager { let guest_memory = self.guest_memory.memory(); - // Use pagemap + kpageflags to filter memory ranges, keeping only anonymous pages (CoW) + // Classify CoW anonymous pages: pagemap bit 61 on kernels that set + // PM_FILE on file PMDs, otherwise /proc/kpageflags. let (filtered_ranges, stats) = filter_memory_ranges_by_pagemap_anon(&guest_memory, &self.snapshot_memory_ranges) .map_err(|e| { diff --git a/hypervisor/vmm/src/pagemap_anon.rs b/hypervisor/vmm/src/pagemap_anon.rs index f4e6bc436..132fe9a06 100644 --- a/hypervisor/vmm/src/pagemap_anon.rs +++ b/hypervisor/vmm/src/pagemap_anon.rs @@ -2,20 +2,16 @@ // // SPDX-License-Identifier: Apache-2.0 -//! PagemapAnon snapshot support +//! Incremental snapshots of Guest-written CoW anonymous pages. //! -//! This module provides functionality for creating pagemap-anon-based snapshots -//! that only save CoW anonymous pages (pages actually written by the Guest) -//! by inspecting `/proc/self/pagemap` and `/proc/kpageflags`. +//! A page is saved when `swapped || (present && !PM_FILE)`. +//! Linux 6.6.44+ / 6.11+ / 7+ reads that from pagemap bit 61; older kernels +//! use `/proc/kpageflags` (`KPF_ANON`, needs `CAP_SYS_ADMIN`). //! -//! In a `MAP_PRIVATE` mmap restore scenario: -//! - Pages only read by Guest remain as file-backed page cache (KPF_ANON=0) -//! - Pages written by Guest trigger CoW and become anonymous pages (KPF_ANON=1) -//! - Pages never accessed have no PTE (present=0) -//! -//! This module filters out only the anonymous pages, significantly reducing -//! snapshot size compared to mincore which also saves read-only page cache pages. +//! Under `MAP_PRIVATE` restore, unread and read-only file pages are skipped; +//! Guest writes become private anon and are saved. +use crate::kernel_release::PagemapScanPath; use log::{debug, trace}; use once_cell::sync::Lazy; use std::fs::File; @@ -27,10 +23,10 @@ use vm_migration::protocol::{MemoryRange, MemoryRangeTable}; /// Host page size in bytes, probed once from `sysconf(_SC_PAGESIZE)`. /// /// This is 4 KiB on x86_64 but 64 KiB on ARM64 hosts configured with 64 KiB -/// base pages. `/proc/self/pagemap` and `/proc/kpageflags` are indexed in -/// units of the kernel's real page size, so every page-index, seek-offset and -/// range-length computation must use this value — hardcoding 4096 would -/// mis-index the pagemap and silently corrupt snapshots on 64 KiB kernels. +/// base pages. `/proc/self/pagemap` is indexed in units of the kernel's real +/// page size, so every page-index, seek-offset and range-length computation +/// must use this value — hardcoding 4096 would mis-index the pagemap and +/// silently corrupt snapshots on 64 KiB kernels. /// /// The value is fixed for the process lifetime, so probe it once and cache it. static HOST_PAGE_SIZE: Lazy = Lazy::new(|| { @@ -98,21 +94,31 @@ pub(crate) fn coalesce_pages_to_ranges( /// Size of a pagemap entry in bytes const PAGEMAP_ENTRY_SIZE: u64 = 8; -/// Size of a kpageflags entry in bytes -const KPAGEFLAGS_ENTRY_SIZE: u64 = 8; - /// Bit 63: page is present in RAM const PAGEMAP_PRESENT_BIT: u64 = 1 << 63; /// Bit 62: page is in swap const PAGEMAP_SWAPPED_BIT: u64 = 1 << 62; -/// Mask for PFN (bits 0-54) +/// Bit 61: `PM_FILE` (file-backed or shared-anon; private CoW anon has this clear). +const PAGEMAP_FILE_BIT: u64 = 1 << 61; + +/// Present-entry PFN. Zero without `CAP_SYS_ADMIN`. const PAGEMAP_PFN_MASK: u64 = (1 << 55) - 1; -/// Bit 12 in kpageflags: KPF_ANON (anonymous page) +const KPAGEFLAGS_ENTRY_SIZE: u64 = 8; + +/// `/proc/kpageflags` bit 12. const KPF_ANON: u64 = 1 << 12; +/// `true` if this pagemap entry must be written into an incremental snapshot. +pub(crate) fn pagemap_entry_is_cow_anon(entry: u64) -> bool { + let swapped = (entry & PAGEMAP_SWAPPED_BIT) != 0; + let present = (entry & PAGEMAP_PRESENT_BIT) != 0; + let pm_file = (entry & PAGEMAP_FILE_BIT) != 0; + swapped || (present && !pm_file) +} + /// Errors related to pagemap_anon operations #[derive(Debug, Error)] pub enum PagemapAnonError { @@ -143,7 +149,7 @@ pub enum PagemapAnonError { #[error("Memory region not aligned to page boundary")] NotPageAligned, - #[error("No CAP_SYS_ADMIN permission: PFN is zero for a present page, cannot read kpageflags")] + #[error("CAP_SYS_ADMIN is required to read pagemap PFNs for /proc/kpageflags")] NoCapSysAdmin, } @@ -175,17 +181,20 @@ impl PagemapAnonStats { } } -/// Get the anonymous page bitmap for a memory region by reading -/// `/proc/self/pagemap` and `/proc/kpageflags`. -/// -/// # Arguments -/// * `host_addr` - Host virtual address of the memory region (must be page-aligned) -/// * `length` - Length of the memory region in bytes -/// -/// # Returns -/// A vector of bools where each bool indicates if the corresponding page -/// is an anonymous page (CoW written by Guest). +/// Per-page CoW-anon bitmap. Bit 61 on new enough kernels, else kpageflags. pub fn get_anon_pages(host_addr: u64, length: u64) -> Result> { + Ok(scan_anon_pages(host_addr, length)?.0) +} + +fn scan_anon_pages(host_addr: u64, length: u64) -> Result<(Vec, u64)> { + if PagemapScanPath::cached().use_bit61() { + scan_pagemap_cow_anon(host_addr, length) + } else { + scan_kpageflags_anon(host_addr, length) + } +} + +fn read_pagemap_entries(host_addr: u64, length: u64) -> Result> { let page_size = host_page_size(); if host_addr % page_size != 0 { return Err(PagemapAnonError::NotPageAligned); @@ -194,20 +203,12 @@ pub fn get_anon_pages(host_addr: u64, length: u64) -> Result> { let num_pages = length.div_ceil(page_size) as usize; let start_page = host_addr / page_size; - // Open /proc/self/pagemap and /proc/kpageflags let mut pagemap_file = File::open("/proc/self/pagemap").map_err(|e| PagemapAnonError::OpenFailed { path: "/proc/self/pagemap".to_string(), source: e, })?; - let mut kpageflags_file = - File::open("/proc/kpageflags").map_err(|e| PagemapAnonError::OpenFailed { - path: "/proc/kpageflags".to_string(), - source: e, - })?; - - // Batch read all pagemap entries for this region let pagemap_offset = start_page * PAGEMAP_ENTRY_SIZE; pagemap_file .seek(SeekFrom::Start(pagemap_offset)) @@ -225,64 +226,71 @@ pub fn get_anon_pages(host_addr: u64, length: u64) -> Result> { source: e, })?; - let mut result = vec![false; num_pages]; - let mut kpageflags_buf = [0u8; KPAGEFLAGS_ENTRY_SIZE as usize]; + Ok(pagemap_buf + .chunks_exact(PAGEMAP_ENTRY_SIZE as usize) + .map(|chunk| u64::from_ne_bytes(chunk.try_into().unwrap())) + .collect()) +} - for (i, item) in result.iter_mut().enumerate().take(num_pages) { - let entry_offset = i * PAGEMAP_ENTRY_SIZE as usize; - let entry = u64::from_ne_bytes( - pagemap_buf[entry_offset..entry_offset + PAGEMAP_ENTRY_SIZE as usize] - .try_into() - .unwrap(), - ); +/// Bit-61 scan. Returns (must-save bitmap, swapped-page count). +pub(crate) fn scan_pagemap_cow_anon(host_addr: u64, length: u64) -> Result<(Vec, u64)> { + let entries = read_pagemap_entries(host_addr, length)?; + let mut result = vec![false; entries.len()]; + let mut swapped_pages = 0u64; - let present = (entry & PAGEMAP_PRESENT_BIT) != 0; - let swapped = (entry & PAGEMAP_SWAPPED_BIT) != 0; + for (item, entry) in result.iter_mut().zip(entries.iter()) { + if (entry & PAGEMAP_SWAPPED_BIT) != 0 { + swapped_pages += 1; + } + *item = pagemap_entry_is_cow_anon(*entry); + } - // Swapped anonymous pages are also Guest-written pages that must be saved. - // When an anonymous page is swapped out, present=0 but swapped=1. - if swapped { + Ok((result, swapped_pages)) +} + +/// kpageflags scan. Needs `CAP_SYS_ADMIN`. +pub(crate) fn scan_kpageflags_anon(host_addr: u64, length: u64) -> Result<(Vec, u64)> { + let entries = read_pagemap_entries(host_addr, length)?; + let mut kpageflags = + File::open("/proc/kpageflags").map_err(|e| PagemapAnonError::OpenFailed { + path: "/proc/kpageflags".to_string(), + source: e, + })?; + + let mut result = vec![false; entries.len()]; + let mut swapped_pages = 0u64; + let mut flags_buf = [0u8; KPAGEFLAGS_ENTRY_SIZE as usize]; + + for (item, entry) in result.iter_mut().zip(entries.into_iter()) { + if (entry & PAGEMAP_SWAPPED_BIT) != 0 { + swapped_pages += 1; *item = true; continue; } - - if !present { + if (entry & PAGEMAP_PRESENT_BIT) == 0 { continue; } let pfn = entry & PAGEMAP_PFN_MASK; if pfn == 0 { - // PFN is zero for a present page — this means we don't have - // CAP_SYS_ADMIN permission to read PFN from pagemap. return Err(PagemapAnonError::NoCapSysAdmin); } - - // Read kpageflags for this PFN - let kpageflags_offset = pfn * KPAGEFLAGS_ENTRY_SIZE; - kpageflags_file - .seek(SeekFrom::Start(kpageflags_offset)) + kpageflags + .seek(SeekFrom::Start(pfn * KPAGEFLAGS_ENTRY_SIZE)) .map_err(|e| PagemapAnonError::SeekFailed { path: "/proc/kpageflags".to_string(), source: e, })?; - - kpageflags_file - .read_exact(&mut kpageflags_buf) + kpageflags + .read_exact(&mut flags_buf) .map_err(|e| PagemapAnonError::ReadFailed { path: "/proc/kpageflags".to_string(), source: e, })?; - - let flags = u64::from_ne_bytes(kpageflags_buf); - - // KPF_ANON (bit 12) indicates this is an anonymous page, - // meaning it was created by CoW when Guest wrote to it. - if (flags & KPF_ANON) != 0 { - *item = true; - } + *item = (u64::from_ne_bytes(flags_buf) & KPF_ANON) != 0; } - Ok(result) + Ok((result, swapped_pages)) } /// Filter memory ranges by pagemap_anon, returning only ranges with anonymous (CoW) pages. @@ -329,12 +337,12 @@ pub fn filter_memory_ranges_by_pagemap_anon {} + Ok((kpf, _)) => { + let under = bit61 + .iter() + .zip(&kpf) + .filter(|(want, got)| **want && !**got) + .count(); + assert_eq!(under, 0, "kpageflags under-saved bit61 CoW pages"); + fixture.assert_expected_bitmap(&kpf); + } + Err(e) => panic!("unexpected kpageflags error: {e}"), + } + } + + struct MapPrivateCowFixture { + ptr: *mut libc::c_void, + len: usize, + num_pages: usize, + } + + impl MapPrivateCowFixture { + fn new() -> Self { + use std::io::Write; + use std::os::unix::io::AsRawFd; + + let page_size = host_page_size() as usize; + const NUM_PAGES: usize = 8; + let len = page_size * NUM_PAGES; + + let path = std::env::temp_dir().join(format!( + "cube_pagemap_anon_map_private_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .expect("create temp file for MAP_PRIVATE fixture"); + let mut contents = vec![0u8; len]; + for page in 0..NUM_PAGES { + contents[page * page_size] = 0xA0 + page as u8; + } + file.write_all(&contents) + .expect("write pattern into temp file"); + let _ = std::fs::remove_file(&path); + + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE, + file.as_raw_fd(), + 0, + ) + }; + assert_ne!( + ptr, + libc::MAP_FAILED, + "mmap MAP_PRIVATE failed: {}", + std::io::Error::last_os_error() + ); + + let base = ptr as *mut u8; + unsafe { + let _ = std::ptr::read_volatile(base.add(page_size)); + std::ptr::write_volatile(base.add(2 * page_size), 0xc2); + std::ptr::write_volatile(base.add(4 * page_size), 0xc4); + } + + Self { + ptr, + len, + num_pages: NUM_PAGES, + } + } + + fn host_addr(&self) -> u64 { + self.ptr as u64 + } + + fn length(&self) -> u64 { + self.len as u64 + } + + fn assert_expected_bitmap(&self, bitmap: &[bool]) { + assert_eq!(bitmap.len(), self.num_pages); + assert!(!bitmap[0], "untouched page must not be saved"); + assert!(!bitmap[1], "read-only file page must not be saved"); + assert!(bitmap[2], "written CoW page must be saved"); + assert!(!bitmap[3], "untouched page must not be saved"); + assert!(bitmap[4], "written CoW page must be saved"); + for i in 5..self.num_pages { + assert!(!bitmap[i], "untouched page {i} must not be saved"); + } + } + } + + impl Drop for MapPrivateCowFixture { + fn drop(&mut self) { + unsafe { + libc::munmap(self.ptr, self.len); + } + } + } } diff --git a/hypervisor/vmm/src/pagemap_anon_bench.rs b/hypervisor/vmm/src/pagemap_anon_bench.rs new file mode 100644 index 000000000..de0372ad4 --- /dev/null +++ b/hypervisor/vmm/src/pagemap_anon_bench.rs @@ -0,0 +1,173 @@ +// Copyright © 2026 Tencent Corporation +// +// SPDX-License-Identifier: Apache-2.0 + +//! Privileged micro-benchmark for the pagemap bit-61 optimization. +//! +//! Kept separate from `pagemap_anon.rs` so the production implementation and +//! its ordinary unit tests are not obscured by benchmark-only scaffolding. +//! +//! Ignored by default (`CAP_SYS_ADMIN` is required to open `/proc/kpageflags`). +//! Run with `--ignored`. Optional env: `CUBE_PAGEMAP_BENCH_MIB` (comma-separated +//! sizes, default `64,256,1024`) and `CUBE_PAGEMAP_BENCH_ITERS` (default `7`). + +use super::*; +use std::hint::black_box; +use std::io::Write; +use std::os::unix::io::AsRawFd; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +struct PrivateFileMapping { + ptr: *mut libc::c_void, + length: usize, +} + +impl PrivateFileMapping { + fn new(size_mib: u64, page_size: u64) -> Self { + let length = size_mib * 1024 * 1024; + let path = std::env::temp_dir().join(format!( + "cube_pagemap_bench_{}_{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let mut file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .expect("create benchmark backing file"); + + let pattern = vec![0x5a; 1024 * 1024]; + for _ in 0..size_mib { + file.write_all(&pattern) + .expect("populate benchmark backing file"); + } + + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + length as usize, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE, + file.as_raw_fd(), + 0, + ) + }; + assert_ne!(ptr, libc::MAP_FAILED, "MAP_PRIVATE fixture failed"); + std::fs::remove_file(path).expect("unlink benchmark backing file"); + + // Restore-like workload: read every file page, then CoW-write 10%. + let base = ptr as *mut u8; + let pages = length / page_size; + for page in 0..pages { + unsafe { + black_box(std::ptr::read_volatile( + base.add((page * page_size) as usize), + )); + } + } + for page in (0..pages).step_by(10) { + unsafe { + std::ptr::write_volatile(base.add((page * page_size) as usize), 0xc2); + } + } + + Self { + ptr, + length: length as usize, + } + } +} + +impl Drop for PrivateFileMapping { + fn drop(&mut self) { + unsafe { + libc::munmap(self.ptr, self.length); + } + } +} + +fn median_ns(mut samples: Vec) -> u128 { + samples.sort_unstable(); + samples[samples.len() / 2] +} + +fn measure(iterations: usize, mut scan: impl FnMut() -> Vec) -> u128 { + black_box(scan()); // warmup + median_ns( + (0..iterations) + .map(|_| { + let start = Instant::now(); + black_box(scan()); + start.elapsed().as_nanos() + }) + .collect(), + ) +} + +fn env_values(name: &str, default: &str) -> Vec { + std::env::var(name) + .unwrap_or_else(|_| default.to_string()) + .split(',') + .map(|value| value.trim().parse().expect("invalid benchmark setting")) + .collect() +} + +/// Compare the actual old and new VMM implementations on the same mapping. +#[test] +#[ignore = "requires CAP_SYS_ADMIN; cargo test -- --ignored (CUBE_PAGEMAP_BENCH_MIB / CUBE_PAGEMAP_BENCH_ITERS)"] +fn benchmark_get_anon_pages_before_after() { + let page_size = host_page_size(); + let iterations = env_values("CUBE_PAGEMAP_BENCH_ITERS", "7")[0] as usize; + println!( + "pagemap_anon release micro-benchmark: page_size={page_size}, iterations={iterations}" + ); + println!( + "{:<8} {:>12} {:>12} {:>10} {:>12} {:>12} {:>10} {:>10}", + "MiB", "old_ms", "new_ms", "speedup", "old_pages", "new_pages", "under", "over" + ); + + for size_mib in env_values("CUBE_PAGEMAP_BENCH_MIB", "64,256,1024") { + let mapping = PrivateFileMapping::new(size_mib, page_size); + let host_addr = mapping.ptr as u64; + let length = mapping.length as u64; + + let old_ns = measure(iterations, || { + scan_kpageflags_anon(host_addr, length) + .expect("kpageflags scan failed") + .0 + }); + let new_ns = measure(iterations, || { + scan_pagemap_cow_anon(host_addr, length) + .expect("bit61 scan failed") + .0 + }); + + let old = scan_kpageflags_anon(host_addr, length) + .expect("kpageflags comparison failed") + .0; + let new = scan_pagemap_cow_anon(host_addr, length) + .expect("bit61 comparison failed") + .0; + let under = old.iter().zip(&new).filter(|(a, b)| **a && !**b).count(); + let over = old.iter().zip(&new).filter(|(a, b)| !**a && **b).count(); + let old_pages = old.iter().filter(|&&save| save).count(); + let new_pages = new.iter().filter(|&&save| save).count(); + + println!( + "{:<8} {:>12.3} {:>12.3} {:>9.1}x {:>12} {:>12} {:>10} {:>10}", + size_mib, + old_ns as f64 / 1_000_000.0, + new_ns as f64 / 1_000_000.0, + old_ns as f64 / new_ns as f64, + old_pages, + new_pages, + under, + over + ); + assert_eq!(under, 0, "bit61 path under-saved legacy KPF_ANON pages"); + } +} diff --git a/hypervisor/vmm/src/soft_dirty.rs b/hypervisor/vmm/src/soft_dirty.rs index e5e39e0c2..4bc0a8502 100644 --- a/hypervisor/vmm/src/soft_dirty.rs +++ b/hypervisor/vmm/src/soft_dirty.rs @@ -760,8 +760,8 @@ mod tests { /// `filter_memory_ranges_by_soft_dirty` does. /// /// Skipped silently when the host kernel lacks `CONFIG_MEM_SOFT_DIRTY=y`, - /// or when the test process lacks CAP_SYS_ADMIN to read PFNs from - /// /proc/self/pagemap and classify anonymous pages through /proc/kpageflags. + /// or when the kpageflags fallback needs `CAP_SYS_ADMIN` and the test + /// process does not have it. #[test] fn test_filter_memory_ranges_by_anon_and_soft_dirty_end_to_end() { let _guard = CLEAR_REFS_LOCK.lock().unwrap_or_else(|e| e.into_inner()); @@ -823,7 +823,17 @@ mod tests { Ok(result) => result, Err(SoftDirtyError::AnonProbe(pagemap_anon::PagemapAnonError::NoCapSysAdmin)) => { eprintln!( - "no CAP_SYS_ADMIN to read pagemap PFNs; skipping {}", + "kpageflags path needs CAP_SYS_ADMIN; skipping {}", + "test_filter_memory_ranges_by_anon_and_soft_dirty_end_to_end" + ); + return; + } + Err(SoftDirtyError::AnonProbe(pagemap_anon::PagemapAnonError::OpenFailed { + path, + .. + })) if path == "/proc/kpageflags" => { + eprintln!( + "cannot open /proc/kpageflags; skipping {}", "test_filter_memory_ranges_by_anon_and_soft_dirty_end_to_end" ); return;