diff --git a/Cargo.lock b/Cargo.lock index c59334ea..a1c5ad65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,9 +22,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "bare-metal" @@ -49,9 +49,9 @@ checksum = "ed8765909f9009617974ab6b7d332625b320b33c326b1e9321382ef1999b5d56" [[package]] name = "bit_field" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" [[package]] name = "bitflags" @@ -61,9 +61,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" [[package]] name = "bitmap-allocator" @@ -102,9 +102,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "cortex-a" @@ -142,6 +142,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "heapless" version = "0.8.0" @@ -157,14 +163,15 @@ name = "hvisor" version = "0.1.0" dependencies = [ "aarch64-cpu", - "bit_field 0.10.2", - "bitflags 2.9.1", + "bit_field 0.10.3", + "bitflags 2.9.4", "bitmap-allocator", "bitvec", "buddy_system_allocator", "cfg-if", "cortex-a", "heapless", + "hvisor-pt", "lazy_static", "log", "loongArch64", @@ -181,25 +188,45 @@ dependencies = [ "sbi-spec 0.0.8", "spin 0.10.0", "tock-registers", + "vstd", "x2apic", "x86", "x86_64", ] +[[package]] +name = "hvisor-pt" +version = "0.1.0" +source = "git+https://github.com/PKTH-Jx/hvisor-pt#88a1f48eba13e702a14f1c15c985060e57ac1fc2" +dependencies = [ + "bitflags 2.9.4", + "vstd", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.5.2", + "spin 0.9.8", ] [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -207,9 +234,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "loongArch64" @@ -217,15 +244,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c9f0d275c70310e2a9d2fc23250c5ac826a73fa828a5f256401f85c5c554283" dependencies = [ - "bit_field 0.10.2", - "bitflags 2.9.1", + "bit_field 0.10.3", + "bitflags 2.9.4", ] [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "numeric-enum-macro" @@ -241,9 +268,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -286,9 +313,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", @@ -298,9 +325,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" dependencies = [ "aho-corasick", "memchr", @@ -309,9 +336,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "riscv" @@ -319,7 +346,7 @@ version = "0.6.0" source = "git+https://github.com/rcore-os/riscv#11d43cf7cccb3b62a3caaf3e07a1db7449588f9a" dependencies = [ "bare-metal", - "bit_field 0.10.2", + "bit_field 0.10.3", "bitflags 1.3.2", "log", "riscv-target", @@ -340,9 +367,9 @@ dependencies = [ [[package]] name = "riscv-decode" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec7a6dc0b0bb96a4d23271864a45c0d24dcd9dde2a1b630a35f79fa29c588bf" +checksum = "68b59d645e392e041ad18f5e529ed13242d8405c66bb192f59703ea2137017d0" [[package]] name = "riscv-macros" @@ -393,9 +420,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "sbi-rt" @@ -441,15 +468,15 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "spin" -version = "0.5.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +checksum = "13287b4da9d1207a4f4929ac390916d64eacfe236a487e9a9f5b3be392be5162" [[package]] name = "spin" -version = "0.7.1" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13287b4da9d1207a4f4929ac390916d64eacfe236a487e9a9f5b3be392be5162" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "spin" @@ -468,15 +495,26 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.100" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tap" version = "1.0.1" @@ -491,9 +529,62 @@ checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" + +[[package]] +name = "verus_builtin" +version = "0.0.0-2025-08-12-1837" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d00c22a221fbf8b3398002ebdd1c89017e5381c2f05d6933a3706d87e90e14" + +[[package]] +name = "verus_builtin_macros" +version = "0.0.0-2025-08-12-1837" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4725e5f5c417d90aa8ac92b47daea50b8494874390491b9a957e2f8ed3fe05d7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", + "verus_prettyplease", + "verus_syn", +] + +[[package]] +name = "verus_prettyplease" +version = "0.0.0-2025-08-12-1837" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "1f62091543498033a2d17682164c789de934c42bcdc8ff497fb381c995f1cad5" +dependencies = [ + "proc-macro2", + "verus_syn", +] + +[[package]] +name = "verus_state_machines_macros" +version = "0.0.0-2025-08-12-1837" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "734a9a8e44ddb3cf61dd80de4aee6f3cf6f46db25ad0e0b381e11d0ee8f1e077" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "verus_syn", +] + +[[package]] +name = "verus_syn" +version = "0.0.0-2025-08-12-1837" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30b28d965c1c0259f28a4813e1c393f126940a1051870ef75d668860a2fd0bb" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] [[package]] name = "volatile" @@ -501,6 +592,17 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "442887c63f2c839b346c192d047a7c87e73d0689c9157b00b53dcc27dd5ea793" +[[package]] +name = "vstd" +version = "0.0.0-2025-08-12-1837" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b5f531340ad04fc039853c28cfe984b7baa3982aba0d37dfac01406e4d9f10f" +dependencies = [ + "verus_builtin", + "verus_builtin_macros", + "verus_state_machines_macros", +] + [[package]] name = "wyz" version = "0.5.1" @@ -529,7 +631,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2781db97787217ad2a2845c396a5efe286f87467a5810836db6d74926e94a385" dependencies = [ - "bit_field 0.10.2", + "bit_field 0.10.3", "bitflags 1.3.2", "raw-cpuid", ] @@ -540,7 +642,7 @@ version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "100555a863c0092238c2e0e814c1096c1e5cf066a309c696a87e907b5f8c5d69" dependencies = [ - "bit_field 0.10.2", + "bit_field 0.10.3", "bitflags 1.3.2", "rustversion", "volatile", diff --git a/Cargo.toml b/Cargo.toml index f61cca26..45b892c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ cortex-a = "8.1.1" cfg-if = "1.0" bitvec = { version="1.0.1", default-features = false, features = ["atomic", "alloc"] } heapless = { version = "0.8.0 "} +vstd = { version = "0.0.0-2025-08-12-1837", default-features = false } +hvisor-pt = { git = "https://github.com/PKTH-Jx/hvisor-pt", features = ["aarch64"] } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4.0" diff --git a/src/arch/aarch64/paging.rs b/src/arch/aarch64/paging.rs index 18544053..1d631fb7 100644 --- a/src/arch/aarch64/paging.rs +++ b/src/arch/aarch64/paging.rs @@ -22,6 +22,15 @@ use alloc::{sync::Arc, vec::Vec}; use core::{fmt::Debug, marker::PhantomData, slice}; use spin::Mutex; +use hvisor_pt::arch::aarch64::{vmsav8_4k_3level_arch, vmsav8_4k_4level_arch, Aarch64PageTable}; +use hvisor_pt::common::{ + addr::PAddrExec, + arch::PTArchExec, + frame::{FrameSize, MemAttr}, +}; +use hvisor_pt::spec::memory::{PageTableMem, PageTableMemExec, Table}; +use vstd::prelude::*; + #[derive(Debug)] pub enum PagingError { NoMemory, @@ -115,7 +124,6 @@ pub trait GenericPageTableImmut: Sized { fn level(&self) -> usize; fn starting_level(&self) -> usize; - unsafe fn from_root(root_paddr: PhysAddr, level: usize) -> Self; fn root_paddr(&self) -> PhysAddr; fn query(&self, vaddr: Self::VA) -> PagingResult<(PhysAddr, MemFlags, PageSize)>; } @@ -139,333 +147,143 @@ pub trait GenericPageTable: GenericPageTableImmut { fn flush(&self, vaddr: Option); } -/// A immutable level-3/4 page table implements `GenericPageTableImmut`. -pub struct HvPageTableImmut { - /// Root table frame. - root: Frame, - level: usize, - /// Phantom data. - _phantom: PhantomData<(VA, PTE)>, -} +verus! { -impl HvPageTableImmut -where - VA: From + Into + Copy, - PTE: GenericPTE, -{ - fn new(level: usize) -> Self { - assert!(level == 3 || level == 4); - Self { - root: Frame::new_zero().expect("failed to allocate root frame for host page table"), - level, - _phantom: PhantomData, - } - } - - fn get_entry_mut(&self, vaddr: VA) -> PagingResult<(&mut PTE, PageSize)> { - let vaddr = vaddr.into(); - let p3 = if self.level == 4 { - let p4 = table_of_mut::(self.root_paddr()); - let p4e = &mut p4[p4_index(vaddr)]; - next_table_mut(p4e)? - } else { - table_of_mut::(self.root_paddr()) - }; - let p3e = &mut p3[p3_index(vaddr)]; - if p3e.is_huge() { - return Ok((p3e, PageSize::Size1G)); - } - - let p2 = next_table_mut(p3e)?; - let p2e = &mut p2[p2_index(vaddr)]; - if p2e.is_huge() { - return Ok((p2e, PageSize::Size2M)); - } - - let p1 = next_table_mut(p2e)?; - let p1e = &mut p1[p1_index(vaddr)]; - Ok((p1e, PageSize::Size4K)) - } - - fn walk( - &self, - table: &[PTE], - level: usize, - start_vaddr: usize, - limit: usize, - func: &impl Fn(usize, usize, usize, &PTE), - ) { - let mut n = 0; - for (i, entry) in table.iter().enumerate() { - let vaddr = start_vaddr + (i << (12 + (3 - level) * 9)); - if entry.is_present() { - func(level, i, vaddr, entry); - if level < 3 { - match next_table_mut(entry) { - Ok(entry) => self.walk(entry, level + 1, vaddr, limit, func), - Err(PagingError::MappedToHugePage) => {} - _ => unreachable!(), - } - } - n += 1; - if n >= limit { - break; - } - } - } - } - - fn dump(&self, limit: usize) { - static LOCK: Mutex<()> = Mutex::new(()); - let _lock = LOCK.lock(); - - println!("Root: {:x?}", self.root_paddr()); - self.walk( - table_of(self.root_paddr()), - self.starting_level(), - 0, - limit, - &|level: usize, idx: usize, vaddr: usize, entry: &PTE| { - for _ in 0..level * 2 { - print!(" "); - } - println!( - "[ADDR:{:#x?} level:{} - idx:{:03}], vaddr:{:08x?}: {:x?}", - entry as *const _ as VirtAddr, level, idx, vaddr, entry - ); - }, - ); - } +/// Memory that stores page tables. +pub struct PageTableMemory { + arch: PTArchExec, + tables: Vec, } -impl GenericPageTableImmut for HvPageTableImmut -where - VA: From + Into + Copy, - PTE: GenericPTE, -{ - type VA = VA; - - unsafe fn from_root(root_paddr: PhysAddr, level: usize) -> Self { - Self { - root: Frame::from_paddr(root_paddr), - level, - _phantom: PhantomData, - } - } - - fn root_paddr(&self) -> PhysAddr { - self.root.start_paddr() - } - - fn query(&self, vaddr: VA) -> PagingResult<(PhysAddr, MemFlags, PageSize)> { - let (entry, size) = self.get_entry_mut(vaddr)?; - if entry.is_unused() { - return Err(PagingError::NotMapped); +impl PageTableMemExec for PageTableMemory { + spec fn view(self) -> PageTableMem { + PageTableMem { + tables: Seq::new(self.tables.len() as nat, |i| Table { + base: self.frames[i].start_paddr(), + size: FrameSize::Size4K, + level: 0, + }), + arch: self.arch@, } - let off = size.page_offset(vaddr.into()); - Ok((entry.addr() + off, entry.flags(), size)) - } - - fn level(&self) -> usize { - self.level } - fn starting_level(&self) -> usize { - if self.level == 4 { - 0 - } else { - 1 - } + fn root(&self) -> PAddrExec { + PAddrExec(self.tables[0].start_paddr()) } -} - -/// A extended level-3/4 page table that can change its mapping. It also tracks all intermediate -/// level tables. Locks need to be used if change the same page table concurrently. -struct HvPageTableUnlocked { - inner: HvPageTableImmut, - /// Intermediate level table frames. - intrm_tables: Vec, - /// Phantom data. - _phantom: PhantomData<(VA, PTE, I)>, -} -impl HvPageTableUnlocked -where - VA: From + Into + Copy, - PTE: GenericPTE, - I: PagingInstr, -{ - fn new(level: usize) -> Self { + fn new_init(arch: PTArchExec) -> Self { Self { - inner: HvPageTableImmut::new(level), - intrm_tables: Vec::new(), - _phantom: PhantomData, + arch, + tables: vec![Frame::new().unwrap()], } } - unsafe fn from_root(root_paddr: PhysAddr, level: usize) -> Self { - Self { - inner: HvPageTableImmut::from_root(root_paddr, level), - intrm_tables: Vec::new(), - _phantom: PhantomData, - } + fn is_table_empty(&self, base: PAddrExec) -> bool { + let table = self.tables.iter().find(|t| t.start_paddr() == base.0).unwrap(); + let contents = unsafe { + core::slice::from_raw_parts(base.0 as *const u8, table.size()) + }; + contents.iter().all(|&b| b == 0) } - fn alloc_intrm_table(&mut self) -> HvResult { - let frame = Frame::new_zero()?; - let paddr = frame.start_paddr(); - self.intrm_tables.push(frame); - Ok(paddr) + fn alloc_table(&mut self, level: usize) -> (PAddrExec, FrameSize) { + let frame = Frame::new().unwrap(); + let res = (PAddrExec(frame.start_paddr()), FrameSize::Size4K); + self.tables.push(frame); + res } - fn _dealloc_intrm_table(&mut self, _paddr: PhysAddr) {} - - fn get_entry_mut_or_create(&mut self, page: Page) -> PagingResult<&mut PTE> { - let vaddr: usize = page.vaddr.into(); - let p3 = if self.inner.level == 4 { - let p4 = table_of_mut::(self.inner.root_paddr()); - let p4e = &mut p4[p4_index(vaddr)]; - - next_table_mut_or_create(p4e, || self.alloc_intrm_table())? - } else { - table_of_mut::(self.inner.root_paddr()) - }; - let p3e = &mut p3[p3_index(vaddr)]; - if page.size == PageSize::Size1G { - return Ok(p3e); - } - - let p2 = next_table_mut_or_create(p3e, || self.alloc_intrm_table())?; - let p2e = &mut p2[p2_index(vaddr)]; - if page.size == PageSize::Size2M { - return Ok(p2e); - } - - let p1 = next_table_mut_or_create(p2e, || self.alloc_intrm_table())?; - let p1e = &mut p1[p1_index(vaddr)]; - Ok(p1e) + fn dealloc_table(&mut self, base: PAddrExec) { + let index = self.tables.iter().position(|f| f.start_paddr() == base.0).unwrap(); + self.tables.remove(index); } - fn map_page( - &mut self, - page: Page, - paddr: PhysAddr, - flags: MemFlags, - ) -> PagingResult<&mut PTE> { - let entry: &mut PTE = self.get_entry_mut_or_create(page)?; - if !entry.is_unused() { - return Err(PagingError::AlreadyMapped); - } - entry.set_addr(page.size.align_down(paddr)); - entry.set_flags(flags, page.size.is_huge()); - Ok(entry) + fn read(&self, base:PAddrExec, index:usize) -> u64 { + unsafe { (base.0 as *const u64).offset(index as isize).read_volatile() } } - fn unmap_page(&mut self, vaddr: VA) -> PagingResult<(PhysAddr, PageSize)> { - let (entry, size) = self.inner.get_entry_mut(vaddr)?; - if entry.is_unused() { - return Err(PagingError::NotMapped); - } - let paddr = entry.addr(); - entry.clear(); - Ok((paddr, size)) + fn write(&mut self, base:PAddrExec, index:usize, val:u64) { + unsafe { (base.0 as *mut u64).offset(index as isize).write_volatile(val) } } +} - fn update(&mut self, vaddr: VA, paddr: PhysAddr, flags: MemFlags) -> PagingResult { - let (entry, size) = self.inner.get_entry_mut(vaddr)?; - entry.set_addr(paddr); - entry.set_flags(flags, size.is_huge()); - Ok(size) - } } -/// A extended level-3/4 page table implements `GenericPageTable`. It use locks to avoid data -/// racing between it and its clonees. -pub struct HvPageTable { - inner: HvPageTableUnlocked, - /// Make sure all accesses to the page table and its clonees is exclusive. +/// Page table implementation for aarch64. +pub struct HvPageTable + Into + Copy, I: PagingInstr> { + inner: Aarch64PageTable, clonee_lock: Arc>, + _phantom: PhantomData<(VA, I)>, } -impl HvPageTable +impl HvPageTable where VA: From + Into + Copy, - PTE: GenericPTE, I: PagingInstr, { - #[allow(dead_code)] - pub fn dump(&self, limit: usize) { - self.inner.inner.dump(limit) - } - - /// Clone only the top level page table mapping from `src`. pub fn clone_from(src: &impl GenericPageTableImmut) -> Self { // XXX: The clonee won't track intermediate tables, must ensure it lives shorter than the // original page table. let pt = Self::new(src.level()); let dst_p4_table = - unsafe { slice::from_raw_parts_mut(pt.root_paddr() as *mut PTE, ENTRY_COUNT) }; + unsafe { slice::from_raw_parts_mut(pt.root_paddr() as *mut u64, ENTRY_COUNT) }; let src_p4_table = - unsafe { slice::from_raw_parts(src.root_paddr() as *const PTE, ENTRY_COUNT) }; + unsafe { slice::from_raw_parts(src.root_paddr() as *const u64, ENTRY_COUNT) }; dst_p4_table.clone_from_slice(src_p4_table); pt } } -impl GenericPageTableImmut for HvPageTable +impl GenericPageTableImmut for HvPageTable where VA: From + Into + Copy, - PTE: GenericPTE, I: PagingInstr, { type VA = VA; - unsafe fn from_root(root_paddr: PhysAddr, level: usize) -> Self { - Self { - inner: HvPageTableUnlocked::from_root(root_paddr, level), - clonee_lock: Arc::new(Mutex::new(())), - } - } - - fn root_paddr(&self) -> PhysAddr { - self.inner.inner.root_paddr() + fn level(&self) -> usize { + self.inner.arch().level_count() } - fn query(&self, vaddr: VA) -> PagingResult<(PhysAddr, MemFlags, PageSize)> { - let _lock = self.clonee_lock.lock(); - self.inner.inner.query(vaddr) + fn starting_level(&self) -> usize { + 0 } - fn level(&self) -> usize { - self.inner.inner.level() + fn root_paddr(&self) -> PhysAddr { + self.inner.root() } - fn starting_level(&self) -> usize { - self.inner.inner.starting_level() + fn query(&self, vaddr: Self::VA) -> PagingResult<(PhysAddr, MemFlags, PageSize)> { + let _lock = self.clonee_lock.lock(); + info!("query {:#x}", vaddr.into()); + self.inner + .query(vaddr.into()) + .map(|(vb, pb, sz, attr)| (pb, attr_to_flags(attr), frame_size_to_page_size(sz))) + .map_err(|_| PagingError::NotMapped) } } -impl GenericPageTable for HvPageTable +impl GenericPageTable for HvPageTable where VA: From + Into + Copy, - PTE: GenericPTE, I: PagingInstr, { fn new(level: usize) -> Self { + assert!(level == 3 || level == 4); + let arch = if level == 4 { + vmsav8_4k_4level_arch() + } else { + vmsav8_4k_3level_arch() + }; Self { - inner: HvPageTableUnlocked::new(level), + inner: Aarch64PageTable::new(arch, 0x0, 0x80000000), clonee_lock: Arc::new(Mutex::new(())), + _phantom: PhantomData, } } - fn map(&mut self, region: &MemoryRegion) -> HvResult { - assert!( - is_aligned(region.start.into()), - "region.start = {:#x?}", - region.start.into() - ); - assert!(is_aligned(region.size), "region.size = {:#x?}", region.size); - trace!( + fn map(&mut self, region: &MemoryRegion) -> HvResult { + info!( "create mapping in {}: {:#x?}", core::any::type_name::(), region @@ -475,40 +293,33 @@ where let mut size = region.size; while size > 0 { let paddr = region.mapper.map_fn(vaddr); - let page_size = if PageSize::Size1G.is_aligned(vaddr) + let frame_size = if PageSize::Size1G.is_aligned(vaddr) && PageSize::Size1G.is_aligned(paddr) && size >= PageSize::Size1G as usize && !region.flags.contains(MemFlags::NO_HUGEPAGES) { - PageSize::Size1G + FrameSize::Size1G } else if PageSize::Size2M.is_aligned(vaddr) && PageSize::Size2M.is_aligned(paddr) && size >= PageSize::Size2M as usize && !region.flags.contains(MemFlags::NO_HUGEPAGES) { - PageSize::Size2M + FrameSize::Size2M } else { - PageSize::Size4K + FrameSize::Size4K }; - let page = Page::new_aligned(vaddr.into(), page_size); self.inner - .map_page(page, paddr, region.flags) - .map_err(|e: PagingError| { - error!( - "failed to map page: {:#x?}({:?}) -> {:#x?}, {:?}", - vaddr, page_size, paddr, e - ); - e - })?; - vaddr += page_size as usize; - size -= page_size as usize; + .map(vaddr, paddr, frame_size, flags_to_attr(region.flags)) + .map_err(|_| PagingError::AlreadyMapped)?; + vaddr += frame_size.as_usize(); + size -= frame_size.as_usize(); } Ok(()) } - fn unmap(&mut self, region: &MemoryRegion) -> HvResult { - trace!( - "destroy mapping in {}: {:#x?}", + fn unmap(&mut self, region: &MemoryRegion) -> HvResult { + info!( + "remove mapping in {}: {:#x?}", core::any::type_name::(), region ); @@ -516,25 +327,40 @@ where let mut vaddr = region.start.into(); let mut size = region.size; while size > 0 { - let (_, page_size) = self.inner.unmap_page(vaddr.into()).map_err(|e| { - error!("failed to unmap page: {:#x?}, {:?}", vaddr, e); - e - })?; + let page_size = self + .inner + .query(vaddr.into()) + .map(|(_, _, sz, _)| frame_size_to_page_size(sz)) + .map_err(|_| PagingError::NotMapped)?; + self.inner + .unmap(vaddr.into()) + .map_err(|_| PagingError::NotMapped)?; if !page_size.is_aligned(vaddr) { error!("error vaddr={:#x?}", vaddr); loop {} } - assert!(page_size.is_aligned(vaddr)); - assert!(page_size as usize <= size); vaddr += page_size as usize; size -= page_size as usize; } Ok(()) } - fn update(&mut self, vaddr: VA, paddr: PhysAddr, flags: MemFlags) -> PagingResult { + fn update( + &mut self, + vaddr: Self::VA, + paddr: PhysAddr, + flags: MemFlags, + ) -> PagingResult { let _lock = self.clonee_lock.lock(); - self.inner.update(vaddr, paddr, flags) + let page_size = self + .inner + .query(vaddr.into()) + .map(|(_, _, sz, _)| frame_size_to_page_size(sz)) + .map_err(|_| PagingError::NotMapped)?; + self.inner + .protect(vaddr.into(), flags_to_attr(flags)) + .map(|_| page_size) + .map_err(|_| PagingError::NotMapped) } fn clone(&self) -> Self { @@ -553,51 +379,49 @@ where } } -const fn p4_index(vaddr: usize) -> usize { - (vaddr >> (12 + 27)) & (ENTRY_COUNT - 1) -} - -const fn p3_index(vaddr: usize) -> usize { - (vaddr >> (12 + 18)) & (ENTRY_COUNT - 1) -} - -const fn p2_index(vaddr: usize) -> usize { - (vaddr >> (12 + 9)) & (ENTRY_COUNT - 1) -} - -const fn p1_index(vaddr: usize) -> usize { - (vaddr >> 12) & (ENTRY_COUNT - 1) -} - -fn table_of<'a, E>(paddr: PhysAddr) -> &'a [E] { - let ptr = paddr as *const E; - unsafe { slice::from_raw_parts(ptr, ENTRY_COUNT) } +fn attr_to_flags(attr: MemAttr) -> MemFlags { + let mut flags = MemFlags::empty(); + if attr.readable { + flags |= MemFlags::READ; + } + if attr.writable { + flags |= MemFlags::WRITE; + } + if attr.executable { + flags |= MemFlags::EXECUTE; + } + if attr.device { + flags |= MemFlags::IO; + } + if attr.user_accessible { + flags |= MemFlags::USER; + } + flags } -fn table_of_mut<'a, E>(paddr: PhysAddr) -> &'a mut [E] { - let ptr = paddr as *mut E; - unsafe { slice::from_raw_parts_mut(ptr, ENTRY_COUNT) } +fn flags_to_attr(flags: MemFlags) -> MemAttr { + MemAttr { + readable: flags.contains(MemFlags::READ), + writable: flags.contains(MemFlags::WRITE), + executable: flags.contains(MemFlags::EXECUTE), + device: flags.contains(MemFlags::IO), + user_accessible: flags.contains(MemFlags::USER), + } } -fn next_table_mut<'a, E: GenericPTE>(entry: &E) -> PagingResult<&'a mut [E]> { - if !entry.is_present() { - Err(PagingError::NotMapped) - } else if entry.is_huge() { - Err(PagingError::MappedToHugePage) - } else { - Ok(table_of_mut(entry.addr())) +fn page_size_to_frame_size(size: PageSize) -> FrameSize { + match size { + PageSize::Size4K => FrameSize::Size4K, + PageSize::Size2M => FrameSize::Size2M, + PageSize::Size1G => FrameSize::Size1G, } } -fn next_table_mut_or_create<'a, E: GenericPTE>( - entry: &mut E, - mut allocator: impl FnMut() -> HvResult, -) -> PagingResult<&'a mut [E]> { - if entry.is_unused() { - let paddr = allocator().map_err(|_| PagingError::NoMemory)?; - entry.set_table(paddr); - Ok(table_of_mut(paddr)) - } else { - next_table_mut(entry) +fn frame_size_to_page_size(size: FrameSize) -> PageSize { + match size { + FrameSize::Size4K => PageSize::Size4K, + FrameSize::Size2M => PageSize::Size2M, + FrameSize::Size1G => PageSize::Size1G, + _ => panic!("Unsupported frame size"), } } diff --git a/src/arch/aarch64/s2pt.rs b/src/arch/aarch64/s2pt.rs index 9e947a11..36807853 100644 --- a/src/arch/aarch64/s2pt.rs +++ b/src/arch/aarch64/s2pt.rs @@ -14,202 +14,9 @@ // Authors: // #![allow(unused)] -use aarch64_cpu::registers::VTTBR_EL2; -use core::fmt; -use numeric_enum_macro::numeric_enum; - -use crate::consts::PAGE_SIZE; -use crate::memory::addr::{GuestPhysAddr, HostPhysAddr, PhysAddr}; -use crate::memory::MemFlags; - use super::paging::{GenericPTE, HvPageTable, PagingInstr}; - -bitflags::bitflags! { - /// Memory attribute fields in the VMSAv8-64 translation table format descriptors. - #[derive(Clone, Copy, Debug)] - pub struct DescriptorAttr: u64 { - // Attribute fields in stage 2 VMSAv8-64 Block and Page descriptors: - - /// Whether the descriptor is valid. - const VALID = 1 << 0; - /// The descriptor gives the address of the next level of translation table or 4KB page. - /// (not a 2M, 1G block) - const NON_BLOCK = 1 << 1; - /// Memory attributes index field. - const ATTR = 0b1111 << 2; - /// Access permission: accessable at EL0/1, Read / Write. - const S2AP_R = 1 << 6; - /// Access permission: accessable at EL0/1, Write. - const S2AP_W = 1 << 7; - /// Shareability: Inner Shareable (otherwise Outer Shareable). - const INNER = 1 << 8; - /// Shareability: Inner or Outer Shareable (otherwise Non-shareable). - const SHAREABLE = 1 << 9; - /// The Access flag. - const AF = 1 << 10; - } -} - -numeric_enum! { - #[repr(u64)] - #[derive(Debug, Clone, Copy, Eq, PartialEq)] - enum MemType { - Device = 1, - Normal = 15, - } -} - -impl DescriptorAttr { - const ATTR_INDEX_MASK: u64 = 0b1111_00; - - const fn from_mem_type(mem_type: MemType) -> Self { - let mut bits = (mem_type as u64) << 2; - if matches!(mem_type, MemType::Normal) { - bits |= Self::INNER.bits() | Self::SHAREABLE.bits(); - } - Self::from_bits_truncate(bits) - } - - fn mem_type(&self) -> MemType { - let idx = (self.bits() & Self::ATTR_INDEX_MASK) >> 2; - match idx { - 1 => MemType::Device, - 15 => MemType::Normal, - _ => panic!("Invalid memory attribute index"), - } - } -} - -impl MemType { - fn empty() -> Self { - Self::try_from(0).unwrap() - } -} - -impl From for MemFlags { - fn from(attr: DescriptorAttr) -> Self { - let mut flags = Self::empty(); - if attr.contains(DescriptorAttr::VALID) && attr.contains(DescriptorAttr::S2AP_R) { - flags |= Self::READ; - } - if attr.contains(DescriptorAttr::S2AP_W) { - flags |= Self::WRITE; - } - if attr.mem_type() == MemType::Device { - flags |= Self::IO; - } - flags - } -} - -impl From for DescriptorAttr { - fn from(flags: MemFlags) -> Self { - let mut attr = if flags.contains(MemFlags::IO) { - Self::from_mem_type(MemType::Device) - } else { - Self::from_mem_type(MemType::Normal) - }; - attr |= Self::VALID | Self::AF; - if flags.contains(MemFlags::READ) { - attr |= Self::S2AP_R; - } - if flags.contains(MemFlags::WRITE) { - attr |= Self::S2AP_W; - } - attr - } -} - -#[derive(Clone, Copy)] -#[repr(transparent)] -pub struct PageTableEntry(u64); - -impl PageTableEntry { - const PHYS_ADDR_MASK: usize = 0xffff_ffff_ffff & !(PAGE_SIZE - 1); - - pub const fn empty() -> Self { - Self(0) - } -} - -impl GenericPTE for PageTableEntry { - fn addr(&self) -> HostPhysAddr { - PhysAddr::from(self.0 as usize & Self::PHYS_ADDR_MASK) - } - - fn flags(&self) -> MemFlags { - DescriptorAttr::from_bits_truncate(self.0).into() - } - - fn is_unused(&self) -> bool { - self.0 == 0 - } - - fn is_present(&self) -> bool { - DescriptorAttr::from_bits_truncate(self.0).contains(DescriptorAttr::VALID) - } - - fn is_huge(&self) -> bool { - !DescriptorAttr::from_bits_truncate(self.0).contains(DescriptorAttr::NON_BLOCK) - } - - fn set_addr(&mut self, paddr: HostPhysAddr) { - self.0 = - (self.0 & !Self::PHYS_ADDR_MASK as u64) | (paddr as u64 & Self::PHYS_ADDR_MASK as u64); - } - - fn set_flags(&mut self, flags: MemFlags, is_huge: bool) { - let mut mem_type: MemType = MemType::Normal; - if flags.contains(MemFlags::IO) { - mem_type = MemType::Device; - } - let mut flags: DescriptorAttr = flags.into(); - if !is_huge { - flags |= DescriptorAttr::NON_BLOCK; - } - self.set_flags_and_mem_type(flags, mem_type); - } - - fn set_table(&mut self, paddr: HostPhysAddr) { - self.set_addr(paddr); - self.set_flags_and_mem_type( - DescriptorAttr::VALID | DescriptorAttr::NON_BLOCK, - MemType::Normal, - ); - } - - fn clear(&mut self) { - self.0 = 0 - } -} - -impl PageTableEntry { - fn pt_flags(&self) -> MemFlags { - DescriptorAttr::from_bits_truncate(self.0).into() - } - - fn memory_type(&self) -> MemType { - DescriptorAttr::from_bits_truncate(self.0).mem_type() - } - - fn set_flags_and_mem_type(&mut self, flags: DescriptorAttr, mem_type: MemType) { - let attr = flags | DescriptorAttr::from_mem_type(mem_type); - self.0 = (attr.bits() & !Self::PHYS_ADDR_MASK as u64) - | (self.0 as u64 & Self::PHYS_ADDR_MASK as u64); - } -} - -impl fmt::Debug for PageTableEntry { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Stage2PageTableEntry") - .field("raw", &self.0) - .field("paddr", &self.addr()) - .field("attr", &DescriptorAttr::from_bits_truncate(self.0)) - .field("flags", &self.pt_flags()) - .field("memory_type", &self.memory_type()) - .finish() - } -} +use crate::memory::addr::{GuestPhysAddr, HostPhysAddr, PhysAddr}; +use aarch64_cpu::registers::VTTBR_EL2; pub struct S2PTInstr; @@ -227,7 +34,7 @@ impl PagingInstr for S2PTInstr { } } -pub type Stage2PageTable = HvPageTable; +pub type Stage2PageTable = HvPageTable; pub fn stage2_mode_detect() { info!("Dynamical detection of stage-2 paging mode is not supported yet.");