From 83b2f68641987afdf83ad868d2e55327147487fd Mon Sep 17 00:00:00 2001 From: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> Date: Sat, 12 Dec 2020 03:14:34 +0000 Subject: [PATCH 1/3] Implement guest page size policy Signed-off-by: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> --- src/mmap.rs | 52 +++++++++++++++++++++++++++++++++++++++------ src/mmap_unix.rs | 31 ++++++++++++++++++++++++--- src/mmap_windows.rs | 12 ++++++++--- 3 files changed, 83 insertions(+), 12 deletions(-) diff --git a/src/mmap.rs b/src/mmap.rs index f950381b..e46d5b2d 100644 --- a/src/mmap.rs +++ b/src/mmap.rs @@ -70,6 +70,17 @@ pub enum Error { UnsortedMemoryRegions, } +/// Page configuration types for controlling allocation size and behavior +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum PageSizePolicy { + /// Base pages are the smallest page-size unit available on the system. + BasePages, + /// Transparent hugepages, if available, are managed by the host operating system. + TransparentHugepages, + /// Explicit hugepages swear a lot. Especially if the addresses aren't aligned. + ExplicitHugepages, +} + impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { @@ -426,16 +437,44 @@ impl GuestMemoryMmap { /// /// Valid memory regions are specified as a slice of (Address, Size) tuples sorted by Address. pub fn from_ranges(ranges: &[(GuestAddress, usize)]) -> result::Result { - Self::from_ranges_with_files(ranges.iter().map(|r| (r.0, r.1, None))) + Self::from_ranges_with_options( + ranges + .iter() + .map(|r| (r.0, r.1, PageSizePolicy::BasePages, None)), + ) + } + + /// Creates a container and allocates anonymous memory for guest memory regions. + /// + /// Valid memory regions are specified as asequence of (Address, Size, PageSizePolicy) + /// tuples sorted by Address. + pub fn from_ranges_with_policy( + ranges: &[(GuestAddress, usize, PageSizePolicy)], + ) -> result::Result { + Self::from_ranges_with_options(ranges.iter().map(|r| (r.0, r.1, r.2, None))) + } + + /// Creates a container and allocates anonymous memory for guest memory regions. + /// + /// Valid memory regions are specified as a sequence of (Address, Size, Option) + /// tuples sorted by Address. + pub fn from_ranges_with_files( + ranges: &[(GuestAddress, usize, Option)], + ) -> result::Result { + Self::from_ranges_with_options( + ranges + .iter() + .map(|r| (r.0, r.1, PageSizePolicy::BasePages, r.2.clone())), + ) } /// Creates a container and allocates anonymous memory for guest memory regions. /// /// Valid memory regions are specified as a sequence of (Address, Size, Option) /// tuples sorted by Address. - pub fn from_ranges_with_files(ranges: T) -> result::Result + pub fn from_ranges_with_options(ranges: T) -> result::Result where - A: Borrow<(GuestAddress, usize, Option)>, + A: Borrow<(GuestAddress, usize, PageSizePolicy, Option)>, T: IntoIterator, { Self::from_regions( @@ -444,11 +483,12 @@ impl GuestMemoryMmap { .map(|x| { let guest_base = x.borrow().0; let size = x.borrow().1; + let policy = x.borrow().2; - if let Some(ref f_off) = x.borrow().2 { - MmapRegion::from_file(f_off.clone(), size) + if let Some(ref f_off) = x.borrow().3 { + MmapRegion::from_file(f_off.clone(), size, policy) } else { - MmapRegion::new(size) + MmapRegion::new(size, policy) } .map_err(Error::MmapRegion) .and_then(|r| GuestRegionMmap::new(r, guest_base)) diff --git a/src/mmap_unix.rs b/src/mmap_unix.rs index 5d23de03..fb2fc23f 100644 --- a/src/mmap_unix.rs +++ b/src/mmap_unix.rs @@ -18,7 +18,7 @@ use std::ptr::null_mut; use std::result; use crate::guest_memory::FileOffset; -use crate::mmap::{check_file_offset, AsSlice}; +use crate::mmap::{check_file_offset, AsSlice, PageSizePolicy}; use crate::volatile_memory::{self, compute_offset, VolatileMemory, VolatileSlice}; /// Error conditions that may arise when creating a new `MmapRegion` object. @@ -90,6 +90,7 @@ pub struct MmapRegion { prot: i32, flags: i32, owned: bool, + policy: PageSizePolicy, } // Send and Sync aren't automatically inherited for the raw address pointer. @@ -104,12 +105,13 @@ impl MmapRegion { /// /// # Arguments /// * `size` - The size of the memory region in bytes. - pub fn new(size: usize) -> Result { + pub fn new(size: usize, policy: PageSizePolicy) -> Result { Self::build( None, size, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_ANONYMOUS | libc::MAP_NORESERVE | libc::MAP_PRIVATE, + policy, ) } @@ -119,12 +121,13 @@ impl MmapRegion { /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file /// referred to by `file_offset.file`. /// * `size` - The size of the memory region in bytes. - pub fn from_file(file_offset: FileOffset, size: usize) -> Result { + pub fn from_file(file_offset: FileOffset, size: usize, policy: PageSizePolicy) -> Result { Self::build( Some(file_offset), size, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_NORESERVE | libc::MAP_SHARED, + policy, ) } @@ -143,6 +146,7 @@ impl MmapRegion { size: usize, prot: i32, flags: i32, + policy: PageSizePolicy, ) -> Result { // Forbid MAP_FIXED, as it doesn't make sense in this context, and is pretty dangerous // in general. @@ -157,6 +161,13 @@ impl MmapRegion { (-1, 0) }; + // Support explicit (pre-reserved) hugepages if requested + let flags = if policy == PageSizePolicy::ExplicitHugepages { + flags | libc::MAP_HUGETLB + } else { + flags + }; + // This is safe because we're not allowing MAP_FIXED, and invalid parameters cannot break // Rust safety guarantees (things may change if we're mapping /dev/mem or some wacky file). let addr = unsafe { libc::mmap(null_mut(), size, prot, flags, fd, offset as libc::off_t) }; @@ -165,6 +176,13 @@ impl MmapRegion { return Err(Error::Mmap(io::Error::last_os_error())); } + // Support transparent hugepages if requested + if policy == PageSizePolicy::TransparentHugepages { + unsafe { + libc::madvise(addr, size, libc::MADV_HUGEPAGE); + }; + } + Ok(Self { addr: addr as *mut u8, size, @@ -172,6 +190,7 @@ impl MmapRegion { prot, flags, owned: true, + policy, }) } @@ -213,6 +232,7 @@ impl MmapRegion { prot, flags, owned: false, + policy: PageSizePolicy::BasePages, }) } @@ -248,6 +268,11 @@ impl MmapRegion { self.owned } + /// Returns information regarding the page size policy backing this region. + pub fn policy(&self) -> PageSizePolicy { + self.policy + } + /// Checks whether this region and `other` are backed by overlapping /// [`FileOffset`](struct.FileOffset.html) objects. /// diff --git a/src/mmap_windows.rs b/src/mmap_windows.rs index ce58999a..9a71867d 100644 --- a/src/mmap_windows.rs +++ b/src/mmap_windows.rs @@ -13,7 +13,7 @@ use libc::{c_void, size_t}; use winapi::um::errhandlingapi::GetLastError; use crate::guest_memory::FileOffset; -use crate::mmap::AsSlice; +use crate::mmap::{AsSlice, PageSizePolicy}; use crate::volatile_memory::{self, compute_offset, VolatileMemory, VolatileSlice}; #[allow(non_snake_case)] @@ -88,7 +88,8 @@ impl MmapRegion { /// /// # Arguments /// * `size` - The size of the memory region in bytes. - pub fn new(size: usize) -> io::Result { + /// * `policy` - Unimplemented on Windows platforms. + pub fn new(size: usize, _policy: PageSizePolicy) -> io::Result { if (size == 0) || (size > MM_HIGHEST_VAD_ADDRESS as usize) { return Err(io::Error::from_raw_os_error(libc::EINVAL)); } @@ -111,7 +112,12 @@ impl MmapRegion { /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file /// referred to by `file_offset.file`. /// * `size` - The size of the memory region in bytes. - pub fn from_file(file_offset: FileOffset, size: usize) -> io::Result { + /// * `policy` - Unimplemented on Windows platforms. + pub fn from_file( + file_offset: FileOffset, + size: usize, + _policy: PageSizePolicy, + ) -> io::Result { let handle = file_offset.file().as_raw_handle(); if handle == INVALID_HANDLE_VALUE { return Err(io::Error::from_raw_os_error(libc::EBADF)); From aa4fa1b3c5b5a2a375ba4ddb5b67ebcf185427e2 Mon Sep 17 00:00:00 2001 From: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> Date: Sat, 12 Dec 2020 04:11:08 +0000 Subject: [PATCH 2/3] Add page size policy documentation Signed-off-by: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> --- DESIGN.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index 1e420e8d..3fcb4902 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -122,6 +122,22 @@ let buf = &mut [0u8; 5]; let result = guest_memory_mmap.write(buf, addr); ``` +#### Page Size Policy + +For regions backed by a call to `mmap`, the user may specify the desired page size +and behavior expected by the operating system. The current options include: + +- `BasePages`: The standard page size provided by the operating system. +- `TransparentHugepages`: (Implemented only for Unix-like systems) Hints to the operating + system that base pages can be combined transparently into larger page sizes. Concretely, + mappings with this policy will invoke `madvise` with the `MADV_HUGEPAGE` flag. +- `ExplicitHugepages`: Requests that the entire mapping be explicitly mapped to a pre-reserved + pool of hugepages. Concretely, mappings with this policy will include the `MAP_HUGETLB` flag + in the call to `mmap`. **NOTE:** If the operating system has no available hugepages (e.g. on Linux, + if `cat /proc/sys/vm/nr_hugepages` reads 0), then the mapping, or attempts to dereference addresses + within the region, may fail. It is the responsibility of the VMM to ensure that hugepages are + available for use before constructing a mapping with this policy. + ### Utilities and Helpers The following utilities and helper traits/macros are imported from the From e043095923b1bee0813a77b496833eee04f6b17a Mon Sep 17 00:00:00 2001 From: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> Date: Sat, 12 Dec 2020 05:24:01 +0000 Subject: [PATCH 3/3] Preserve backward-compatibility Signed-off-by: Michael Jarrett <1077485+EmeraldShift@users.noreply.github.com> --- src/mmap.rs | 25 +++++++++++++++---------- src/mmap_unix.rs | 33 +++++++++++++++++++++++++++++++-- src/mmap_windows.rs | 22 ++++++++++++++++++++-- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/src/mmap.rs b/src/mmap.rs index e46d5b2d..18047f00 100644 --- a/src/mmap.rs +++ b/src/mmap.rs @@ -458,14 +458,19 @@ impl GuestMemoryMmap { /// /// Valid memory regions are specified as a sequence of (Address, Size, Option) /// tuples sorted by Address. - pub fn from_ranges_with_files( - ranges: &[(GuestAddress, usize, Option)], - ) -> result::Result { - Self::from_ranges_with_options( - ranges - .iter() - .map(|r| (r.0, r.1, PageSizePolicy::BasePages, r.2.clone())), - ) + pub fn from_ranges_with_files(ranges: T) -> result::Result + where + A: Borrow<(GuestAddress, usize, Option)>, + T: IntoIterator, + { + Self::from_ranges_with_options(ranges.into_iter().map(|r| { + ( + r.borrow().0, + r.borrow().1, + PageSizePolicy::BasePages, + r.borrow().2.clone(), + ) + })) } /// Creates a container and allocates anonymous memory for guest memory regions. @@ -486,9 +491,9 @@ impl GuestMemoryMmap { let policy = x.borrow().2; if let Some(ref f_off) = x.borrow().3 { - MmapRegion::from_file(f_off.clone(), size, policy) + MmapRegion::from_file_with_policy(f_off.clone(), size, policy) } else { - MmapRegion::new(size, policy) + MmapRegion::with_policy(size, policy) } .map_err(Error::MmapRegion) .and_then(|r| GuestRegionMmap::new(r, guest_base)) diff --git a/src/mmap_unix.rs b/src/mmap_unix.rs index fb2fc23f..06fb2563 100644 --- a/src/mmap_unix.rs +++ b/src/mmap_unix.rs @@ -105,7 +105,16 @@ impl MmapRegion { /// /// # Arguments /// * `size` - The size of the memory region in bytes. - pub fn new(size: usize, policy: PageSizePolicy) -> Result { + pub fn new(size: usize) -> Result { + Self::with_policy(size, PageSizePolicy::BasePages) + } + + /// Creates a shared anonymous mapping of `size` bytes. + /// + /// # Arguments + /// * `size` - The size of the memory region in bytes. + /// * `policy` - The page size policy of the memory region. + pub fn with_policy(size: usize, policy: PageSizePolicy) -> Result { Self::build( None, size, @@ -121,7 +130,22 @@ impl MmapRegion { /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file /// referred to by `file_offset.file`. /// * `size` - The size of the memory region in bytes. - pub fn from_file(file_offset: FileOffset, size: usize, policy: PageSizePolicy) -> Result { + pub fn from_file(file_offset: FileOffset, size: usize) -> Result { + Self::from_file_with_policy(file_offset, size, PageSizePolicy::BasePages) + } + + /// Creates a shared file mapping of `size` bytes. + /// + /// # Arguments + /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file + /// referred to by `file_offset.file`. + /// * `size` - The size of the memory region in bytes. + /// * `policy` - The page size policy of the memory region. + pub fn from_file_with_policy( + file_offset: FileOffset, + size: usize, + policy: PageSizePolicy, + ) -> Result { Self::build( Some(file_offset), size, @@ -412,6 +436,7 @@ mod tests { size, prot, flags, + PageSizePolicy::BasePages, ); assert_eq!(format!("{:?}", r.unwrap_err()), "InvalidOffsetLength"); @@ -421,6 +446,7 @@ mod tests { size, prot, flags, + PageSizePolicy::BasePages, ); assert_eq!(format!("{:?}", r.unwrap_err()), "MappingPastEof"); @@ -430,6 +456,7 @@ mod tests { size, prot, flags | libc::MAP_FIXED, + PageSizePolicy::BasePages, ); assert_eq!(format!("{:?}", r.unwrap_err()), "MapFixed"); @@ -442,6 +469,7 @@ mod tests { size, prot, flags, + PageSizePolicy::BasePages, ); assert_eq!(r.unwrap_err().raw_os_error(), libc::EINVAL); @@ -451,6 +479,7 @@ mod tests { size, prot, flags, + PageSizePolicy::BasePages, ) .unwrap(); diff --git a/src/mmap_windows.rs b/src/mmap_windows.rs index 9a71867d..2b62d9da 100644 --- a/src/mmap_windows.rs +++ b/src/mmap_windows.rs @@ -84,12 +84,20 @@ unsafe impl Send for MmapRegion {} unsafe impl Sync for MmapRegion {} impl MmapRegion { + /// Creates a shared anonymous mapping of `size` bytes. + /// + /// # Arguments + /// * `size` - The size of the memory region in bytes. + pub fn new(size: usize) -> io::Result { + Self::with_policy(size, PageSizePolicy::BasePages) + } + /// Creates a shared anonymous mapping of `size` bytes. /// /// # Arguments /// * `size` - The size of the memory region in bytes. /// * `policy` - Unimplemented on Windows platforms. - pub fn new(size: usize, _policy: PageSizePolicy) -> io::Result { + pub fn with_policy(size: usize, _policy: PageSizePolicy) -> io::Result { if (size == 0) || (size > MM_HIGHEST_VAD_ADDRESS as usize) { return Err(io::Error::from_raw_os_error(libc::EINVAL)); } @@ -106,6 +114,16 @@ impl MmapRegion { }) } + /// Creates a shared file mapping of `size` bytes. + /// + /// # Arguments + /// * `file_offset` - The mapping will be created at offset `file_offset.start` in the file + /// referred to by `file_offset.file`. + /// * `size` - The size of the memory region in bytes. + pub fn from_file(file_offset: FileOffset, size: usize) -> io::Result { + Self::from_file_with_policy(file_offset, size, PageSizePolicy::BasePages) + } + /// Creates a shared file mapping of `size` bytes. /// /// # Arguments @@ -113,7 +131,7 @@ impl MmapRegion { /// referred to by `file_offset.file`. /// * `size` - The size of the memory region in bytes. /// * `policy` - Unimplemented on Windows platforms. - pub fn from_file( + pub fn from_file_with_policy( file_offset: FileOffset, size: usize, _policy: PageSizePolicy,