From c617f261aca5c668994efe481f5791c54b1875c3 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 18 Mar 2025 22:01:18 +0800 Subject: [PATCH 01/10] [feat] bring host VM concept into axvm, do compile --- src/config.rs | 22 +++++++++ src/vm.rs | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/src/config.rs b/src/config.rs index 25af342..e514107 100644 --- a/src/config.rs +++ b/src/config.rs @@ -86,6 +86,28 @@ impl From for AxVMConfig { } } +impl AxVMConfig { + pub fn new_host(id: usize, name: String, cpu_num: usize) -> Self { + Self { + id, + name, + vm_type: VMType::VMTHostVM, + cpu_num, + phys_cpu_ids: None, + phys_cpu_sets: None, + cpu_config: AxVCpuConfig::default(), + image_config: VMImageConfig::default(), + memory_regions: Vec::new(), + emu_devices: Vec::new(), + pass_through_devices: Vec::new(), + } + } + + pub fn append_memory_region(&mut self, region: VmMemConfig) { + self.memory_regions.push(region); + } +} + impl AxVMConfig { /// Returns VM id. pub fn id(&self) -> usize { diff --git a/src/vm.rs b/src/vm.rs index 2d53ec6..1f0cee2 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -2,6 +2,7 @@ use alloc::boxed::Box; use alloc::format; use alloc::sync::Arc; use alloc::vec::Vec; +use alloc::string::String; use core::sync::atomic::{AtomicBool, Ordering}; use axerrno::{AxResult, ax_err, ax_err_type}; @@ -33,6 +34,7 @@ struct AxVMInnerConst { config: AxVMConfig, vcpu_list: Box<[AxVCpuRef]>, devices: AxVmDevices, + is_host_vm: bool, } unsafe impl Send for AxVMInnerConst {} @@ -180,6 +182,7 @@ impl AxVM { config, vcpu_list: vcpu_list.into_boxed_slice(), devices, + is_host_vm: false, }, inner_mut: AxVMInnerMut { address_space: Mutex::new(address_space), @@ -214,6 +217,17 @@ impl AxVM { self.inner_const.id } + /// Returns the VM name. + /// The name is used for logging and debugging purposes. + /// It is not required to be unique. + /// The name is set in the VM configuration. + /// If the name is not set in the configuration, the name is an empty string. + /// The name is immutable. + #[inline] + pub fn name(&self) -> String { + self.inner_const.config.name() + } + /// Retrieves the vCPU corresponding to the given vcpu_id for the VM. /// Returns None if the vCPU does not exist. #[inline] @@ -238,6 +252,12 @@ impl AxVM { self.inner_mut.address_space.lock().page_table_root() } + /// Returns if this VM is a host VM. + #[inline] + pub const fn is_host_vm(&self) -> bool { + self.inner_const.is_host_vm + } + /// Returns guest VM image load region in `Vec<&'static mut [u8]>`, /// according to the given `image_load_gpa` and `image_size. /// `Vec<&'static mut [u8]>` is a series of (HVA) address segments, @@ -339,3 +359,104 @@ impl AxVM { Ok(exit_reason) } } + +use x86_vcpu::LinuxContext; + +impl AxVM { + pub fn new_host(config: AxVMConfig, host_cpus: &[LinuxContext]) -> AxResult> { + let result = Arc::new({ + let vcpu_id_pcpu_sets = config.get_vcpu_affinities_pcpu_ids(); + + // Create VCpus. + let mut vcpu_list = Vec::with_capacity(vcpu_id_pcpu_sets.len()); + + for (vcpu_id, phys_cpu_set, _pcpu_id) in vcpu_id_pcpu_sets { + vcpu_list.push(Arc::new(VCpu::new_host( + vcpu_id, + host_cpus[vcpu_id].clone(), + phys_cpu_set, + )?)); + } + + // Set up Memory regions. + let mut address_space = + AddrSpace::new_empty(GuestPhysAddr::from(VM_ASPACE_BASE), VM_ASPACE_SIZE)?; + for mem_region in config.memory_regions() { + let mapping_flags = MappingFlags::from_bits(mem_region.flags).ok_or_else(|| { + ax_err_type!( + InvalidInput, + format!("Illegal flags {:?}", mem_region.flags) + ) + })?; + + // Check mapping flags. + if mapping_flags.contains(MappingFlags::DEVICE) { + warn!( + "Do not include DEVICE flag in memory region flags, it should be configured in pass_through_devices" + ); + continue; + } + + info!( + "Setting up memory region: [{:#x}~{:#x}] {:?}", + mem_region.gpa, + mem_region.gpa + mem_region.size, + mapping_flags + ); + + // Handle ram region. + match mem_region.map_type { + VmMemMappingType::MapIentical => { + address_space.map_linear( + GuestPhysAddr::from(mem_region.gpa), + HostPhysAddr::from(mem_region.gpa), + mem_region.size, + mapping_flags, + )?; + } + VmMemMappingType::MapAlloc => { + warn!("MapAlloc is not supported for host VM"); + } + } + } + + let devices = axdevice::AxVmDevices::new(AxVmDeviceConfig { + emu_configs: config.emu_devices().to_vec(), + }); + + Self { + running: AtomicBool::new(false), + inner_const: AxVMInnerConst { + id: 0, + config, + vcpu_list: vcpu_list.into_boxed_slice(), + devices, + is_host_vm: true, + }, + inner_mut: AxVMInnerMut { + address_space: Mutex::new(address_space), + _marker: core::marker::PhantomData, + }, + } + }); + + info!("VM created: id={}", result.id()); + + // Setup VCpus. + for vcpu in result.vcpu_list() { + let entry = if vcpu.id() == 0 { + result.inner_const.config.bsp_entry() + } else { + result.inner_const.config.ap_entry() + }; + vcpu.setup( + entry, + result.ept_root(), + as AxArchVCpu>::SetupConfig::default(), + )?; + } + info!("VM setup: id={}", result.id()); + + Ok(result) + } +} From 3d666d14bfded05f05583adb47096e97a6a58d41 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sat, 22 Mar 2025 18:16:15 +0800 Subject: [PATCH 02/10] [feat] support guest_phys_to_host_phys --- src/config.rs | 2 +- src/vm.rs | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/config.rs b/src/config.rs index e514107..12833bf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -94,7 +94,7 @@ impl AxVMConfig { vm_type: VMType::VMTHostVM, cpu_num, phys_cpu_ids: None, - phys_cpu_sets: None, + phys_cpu_sets: Some((0..cpu_num).map(|i| 1 << i).collect::>()), cpu_config: AxVCpuConfig::default(), image_config: VMImageConfig::default(), memory_regions: Vec::new(), diff --git a/src/vm.rs b/src/vm.rs index 1f0cee2..155f585 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -1,8 +1,8 @@ use alloc::boxed::Box; use alloc::format; +use alloc::string::String; use alloc::sync::Arc; use alloc::vec::Vec; -use alloc::string::String; use core::sync::atomic::{AtomicBool, Ordering}; use axerrno::{AxResult, ax_err, ax_err_type}; @@ -278,6 +278,12 @@ impl AxVM { Ok(image_load_hva) } + /// Translates a guest physical address to a host physical address. + /// Returns None if the translation fails or the address is not mapped. + pub fn guest_phys_to_host_phys(&self, gpa: GuestPhysAddr) -> Option { + self.inner_mut.address_space.lock().translate(gpa) + } + /// Returns if the VM is running. pub fn running(&self) -> bool { self.running.load(Ordering::Relaxed) @@ -371,6 +377,10 @@ impl AxVM { let mut vcpu_list = Vec::with_capacity(vcpu_id_pcpu_sets.len()); for (vcpu_id, phys_cpu_set, _pcpu_id) in vcpu_id_pcpu_sets { + debug!( + "Creating vCPU[{}] {:x?}\nLinuxContext: {:#x?}", + vcpu_id, phys_cpu_set, host_cpus[vcpu_id] + ); vcpu_list.push(Arc::new(VCpu::new_host( vcpu_id, host_cpus[vcpu_id].clone(), @@ -389,14 +399,6 @@ impl AxVM { ) })?; - // Check mapping flags. - if mapping_flags.contains(MappingFlags::DEVICE) { - warn!( - "Do not include DEVICE flag in memory region flags, it should be configured in pass_through_devices" - ); - continue; - } - info!( "Setting up memory region: [{:#x}~{:#x}] {:?}", mem_region.gpa, From ea62bac5ce01d81562e00be471de7ee1daff3001 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 26 Mar 2025 15:16:02 +0800 Subject: [PATCH 03/10] [feat] add FIXME notes in phys_cpu_sets of new_host --- src/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.rs b/src/config.rs index 12833bf..424f3e0 100644 --- a/src/config.rs +++ b/src/config.rs @@ -94,6 +94,8 @@ impl AxVMConfig { vm_type: VMType::VMTHostVM, cpu_num, phys_cpu_ids: None, + // Set the pCpu affinity for each vCpu to be the same as its vCpu id. + // FIXME: we cannot ensure that cpu_id reserved by Linux start from 0, this is just a temporary solution. phys_cpu_sets: Some((0..cpu_num).map(|i| 1 << i).collect::>()), cpu_config: AxVCpuConfig::default(), image_config: VMImageConfig::default(), From d385bf294c20076552bfb6294e88434d50d22d6a Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 27 Mar 2025 10:02:12 +0800 Subject: [PATCH 04/10] [fix] improve outputs msg --- src/vm.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/vm.rs b/src/vm.rs index 155f585..526570d 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -377,10 +377,7 @@ impl AxVM { let mut vcpu_list = Vec::with_capacity(vcpu_id_pcpu_sets.len()); for (vcpu_id, phys_cpu_set, _pcpu_id) in vcpu_id_pcpu_sets { - debug!( - "Creating vCPU[{}] {:x?}\nLinuxContext: {:#x?}", - vcpu_id, phys_cpu_set, host_cpus[vcpu_id] - ); + debug!("Creating host vCPU[{}] {:x?}", vcpu_id, phys_cpu_set,); vcpu_list.push(Arc::new(VCpu::new_host( vcpu_id, host_cpus[vcpu_id].clone(), @@ -400,7 +397,7 @@ impl AxVM { })?; info!( - "Setting up memory region: [{:#x}~{:#x}] {:?}", + "Setting up host VM memory region: [{:#x}~{:#x}] {:?}", mem_region.gpa, mem_region.gpa + mem_region.size, mapping_flags @@ -442,7 +439,7 @@ impl AxVM { } }); - info!("VM created: id={}", result.id()); + info!("Host VM created: id={}", result.id()); // Setup VCpus. for vcpu in result.vcpu_list() { @@ -457,7 +454,7 @@ impl AxVM { as AxArchVCpu>::SetupConfig::default(), )?; } - info!("VM setup: id={}", result.id()); + info!("Host VM setup: id={}", result.id()); Ok(result) } From 6798b5c1dea837e018300f53989b71d08ccb31dc Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Mon, 31 Mar 2025 15:20:17 +0800 Subject: [PATCH 05/10] [refactor] make axaddrspace more generic --- src/vm.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vm.rs b/src/vm.rs index 526570d..6c8d5f2 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -8,7 +8,9 @@ use core::sync::atomic::{AtomicBool, Ordering}; use axerrno::{AxResult, ax_err, ax_err_type}; use spin::Mutex; +use axaddrspace::npt::{EPTEntry, EPTMetadata, NestedPageTable}; use axaddrspace::{AddrSpace, GuestPhysAddr, HostPhysAddr, MappingFlags}; + use axdevice::{AxVmDeviceConfig, AxVmDevices}; use axvcpu::{AxArchVCpu, AxVCpu, AxVCpuExitReason, AxVCpuHal}; @@ -42,7 +44,7 @@ unsafe impl Sync for AxVMInnerConst {} struct AxVMInnerMut { // Todo: use more efficient lock. - address_space: Mutex>, + address_space: Mutex>, _marker: core::marker::PhantomData, } From 204ceb50eeb4e8231eac5ab7876d0926f46656cb Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 3 Apr 2025 00:09:57 +0800 Subject: [PATCH 06/10] [feat] modify some type visibility --- src/lib.rs | 3 +++ src/vm.rs | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1476e48..887c3fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,9 @@ pub use hal::AxVMHal; pub use vm::AxVCpuRef; pub use vm::AxVM; pub use vm::AxVMRef; +pub use vm::VCpu; + +pub use vcpu::AxVCpuCreateConfig; /// The architecture-independent per-CPU type. pub type AxVMPerCpu = axvcpu::AxPerCpu>; diff --git a/src/vm.rs b/src/vm.rs index 6c8d5f2..c789b57 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -8,7 +8,7 @@ use core::sync::atomic::{AtomicBool, Ordering}; use axerrno::{AxResult, ax_err, ax_err_type}; use spin::Mutex; -use axaddrspace::npt::{EPTEntry, EPTMetadata, NestedPageTable}; +use axaddrspace::npt::{EPTEntry, EPTMetadata}; use axaddrspace::{AddrSpace, GuestPhysAddr, HostPhysAddr, MappingFlags}; use axdevice::{AxVmDeviceConfig, AxVmDevices}; @@ -23,7 +23,7 @@ const VM_ASPACE_SIZE: usize = 0x7fff_ffff_f000; /// A vCPU with architecture-independent interface. #[allow(type_alias_bounds)] -type VCpu = AxVCpu>; +pub type VCpu = AxVCpu>; /// A reference to a vCPU. #[allow(type_alias_bounds)] pub type AxVCpuRef = Arc>; From 70d7d5ad7616d388936b64211c8c1b764166cc6d Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Mon, 7 Apr 2025 10:23:34 +0800 Subject: [PATCH 07/10] [feat] modify guest_phys_to_host_phys --- src/vm.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vm.rs b/src/vm.rs index c789b57..fc91c70 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -283,7 +283,11 @@ impl AxVM { /// Translates a guest physical address to a host physical address. /// Returns None if the translation fails or the address is not mapped. pub fn guest_phys_to_host_phys(&self, gpa: GuestPhysAddr) -> Option { - self.inner_mut.address_space.lock().translate(gpa) + self.inner_mut + .address_space + .lock() + .translate(gpa) + .map(|(hpa, _, _)| hpa) } /// Returns if the VM is running. From 10d357a3dcab815a6cf7b250aa5221ca05df816b Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 17 Apr 2025 21:16:59 +0800 Subject: [PATCH 08/10] [refactor] simply new_host --- src/lib.rs | 2 +- src/vcpu.rs | 2 ++ src/vm.rs | 54 +++++++++++++++++++++-------------------------------- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 887c3fc..4cf7e1b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,7 @@ pub use vm::AxVM; pub use vm::AxVMRef; pub use vm::VCpu; -pub use vcpu::AxVCpuCreateConfig; +pub use vcpu::{AxVCpuCreateConfig, HostContext}; /// The architecture-independent per-CPU type. pub type AxVMPerCpu = axvcpu::AxPerCpu>; diff --git a/src/vcpu.rs b/src/vcpu.rs index 27fa2cc..be88cd1 100644 --- a/src/vcpu.rs +++ b/src/vcpu.rs @@ -5,6 +5,8 @@ cfg_if::cfg_if! { pub use x86_vcpu::VmxArchVCpu as AxArchVCpuImpl; pub use x86_vcpu::VmxArchPerCpuState as AxVMArchPerCpuImpl; pub use x86_vcpu::has_hardware_support; + /// TODO: seperate it into a different crate. + pub use x86_vcpu::LinuxContext as HostContext; pub type AxVCpuCreateConfig = (); // Note: diff --git a/src/vm.rs b/src/vm.rs index fc91c70..eb897e4 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -15,7 +15,7 @@ use axdevice::{AxVmDeviceConfig, AxVmDevices}; use axvcpu::{AxArchVCpu, AxVCpu, AxVCpuExitReason, AxVCpuHal}; use crate::config::{AxVMConfig, VmMemMappingType}; -use crate::vcpu::{AxArchVCpuImpl, AxVCpuCreateConfig}; +use crate::vcpu::AxArchVCpuImpl; use crate::{AxVMHal, has_hardware_support}; const VM_ASPACE_BASE: usize = 0x0; @@ -85,8 +85,7 @@ impl AxVM { .unwrap_or(GuestPhysAddr::from_usize(0x9000_0000)), }; #[cfg(target_arch = "x86_64")] - let arch_config = AxVCpuCreateConfig::default(); - + let arch_config = vcpu_id; vcpu_list.push(Arc::new(VCpu::new( vcpu_id, 0, // Currently not used. @@ -375,22 +374,8 @@ impl AxVM { use x86_vcpu::LinuxContext; impl AxVM { - pub fn new_host(config: AxVMConfig, host_cpus: &[LinuxContext]) -> AxResult> { + pub fn new_host(config: AxVMConfig, host_ctxs: &[LinuxContext]) -> AxResult> { let result = Arc::new({ - let vcpu_id_pcpu_sets = config.get_vcpu_affinities_pcpu_ids(); - - // Create VCpus. - let mut vcpu_list = Vec::with_capacity(vcpu_id_pcpu_sets.len()); - - for (vcpu_id, phys_cpu_set, _pcpu_id) in vcpu_id_pcpu_sets { - debug!("Creating host vCPU[{}] {:x?}", vcpu_id, phys_cpu_set,); - vcpu_list.push(Arc::new(VCpu::new_host( - vcpu_id, - host_cpus[vcpu_id].clone(), - phys_cpu_set, - )?)); - } - // Set up Memory regions. let mut address_space = AddrSpace::new_empty(GuestPhysAddr::from(VM_ASPACE_BASE), VM_ASPACE_SIZE)?; @@ -429,6 +414,24 @@ impl AxVM { emu_configs: config.emu_devices().to_vec(), }); + let vcpu_id_pcpu_sets = config.get_vcpu_affinities_pcpu_ids(); + + // Create VCpus. + let mut vcpu_list = Vec::with_capacity(vcpu_id_pcpu_sets.len()); + + for (vcpu_id, phys_cpu_set, _pcpu_id) in vcpu_id_pcpu_sets { + debug!("Creating host vCPU[{}] {:x?}", vcpu_id, phys_cpu_set,); + let vcpu = VCpu::new(vcpu_id, 0, phys_cpu_set, vcpu_id)?; + + // Setup VCpus. + vcpu.setup_from_context( + address_space.page_table_root(), + host_ctxs[vcpu_id].clone(), + )?; + + vcpu_list.push(Arc::new(vcpu)); + } + Self { running: AtomicBool::new(false), inner_const: AxVMInnerConst { @@ -447,21 +450,6 @@ impl AxVM { info!("Host VM created: id={}", result.id()); - // Setup VCpus. - for vcpu in result.vcpu_list() { - let entry = if vcpu.id() == 0 { - result.inner_const.config.bsp_entry() - } else { - result.inner_const.config.ap_entry() - }; - vcpu.setup( - entry, - result.ept_root(), - as AxArchVCpu>::SetupConfig::default(), - )?; - } - info!("Host VM setup: id={}", result.id()); - Ok(result) } } From 2353f78fd4c705abb453aa3166fb97808f685c08 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Mon, 21 Apr 2025 11:07:27 +0800 Subject: [PATCH 09/10] [refactor] modify guest_phys_to_host_phys --- src/vm.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vm.rs b/src/vm.rs index eb897e4..ba1cd88 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -6,6 +6,7 @@ use alloc::vec::Vec; use core::sync::atomic::{AtomicBool, Ordering}; use axerrno::{AxResult, ax_err, ax_err_type}; +use page_table_multiarch::PageSize; use spin::Mutex; use axaddrspace::npt::{EPTEntry, EPTMetadata}; @@ -281,12 +282,11 @@ impl AxVM { /// Translates a guest physical address to a host physical address. /// Returns None if the translation fails or the address is not mapped. - pub fn guest_phys_to_host_phys(&self, gpa: GuestPhysAddr) -> Option { - self.inner_mut - .address_space - .lock() - .translate(gpa) - .map(|(hpa, _, _)| hpa) + pub fn guest_phys_to_host_phys( + &self, + gpa: GuestPhysAddr, + ) -> Option<(HostPhysAddr, MappingFlags, PageSize)> { + self.inner_mut.address_space.lock().translate(gpa) } /// Returns if the VM is running. From 8b5e282bf8fb668a3379db7d00552ab07670f732 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 23 Apr 2025 14:40:43 +0800 Subject: [PATCH 10/10] [feat] introduce allow_huge flag in linear mapping --- src/vm.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vm.rs b/src/vm.rs index ba1cd88..f005e1e 100644 --- a/src/vm.rs +++ b/src/vm.rs @@ -133,6 +133,7 @@ impl AxVM { HostPhysAddr::from(mem_region.gpa), mem_region.size, mapping_flags, + true, )?; } else { warn!( @@ -170,6 +171,7 @@ impl AxVM { HostPhysAddr::from(pt_device.base_hpa), pt_device.length, MappingFlags::DEVICE | MappingFlags::READ | MappingFlags::WRITE, + false, )?; } @@ -402,6 +404,7 @@ impl AxVM { HostPhysAddr::from(mem_region.gpa), mem_region.size, mapping_flags, + true, )?; } VmMemMappingType::MapAlloc => {