From c04c8080bf24dfee2886c5508a6bcb03bb9cbe26 Mon Sep 17 00:00:00 2001 From: hky1999 Date: Sat, 19 Oct 2024 19:15:06 +0800 Subject: [PATCH 01/19] [feat] functions essential to hypervisor implementation (#29) * update aarch64 hv part in arceos (#15) * Modify platform configuration of aarch64-qemu-virt-hv for GICv2 with virtualization (#20) * refactor: totally remove hv related trap handlers from axhal/arch/aarch64 (#22) * Add gic trait impl (#26) * [refactor] remove hv related scripts and configs from arceos --- Cargo.lock | 15 +- api/axfeat/Cargo.toml | 3 + modules/axhal/Cargo.toml | 5 +- modules/axhal/src/arch/aarch64/mod.rs | 49 +++++- .../axhal/src/platform/aarch64_common/boot.rs | 153 +++++++++++++++++- .../platform/aarch64_common/generic_timer.rs | 36 ++++- .../axhal/src/platform/aarch64_common/gic.rs | 143 ++++++++++++++++ .../src/platform/aarch64_qemu_virt/mod.rs | 2 + modules/axruntime/Cargo.toml | 2 +- modules/axtask/Cargo.toml | 2 +- ulib/axstd/Cargo.toml | 3 + 11 files changed, 396 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f713d8cdb4..272bbf376d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d25e73c949c69f75d1b9dba39c5475523403b31eb8c2fdc99da4dc33bc1aca" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2?rev=7acc0dd#7acc0dd1e1fefa90b4cd45342072d2350a74b071" dependencies = [ + "crate_interface", "tock-registers", ] @@ -446,6 +446,8 @@ dependencies = [ "axlog", "bitflags 2.6.0", "cfg-if", + "cortex-a", + "crate_interface", "dw_apb_uart", "handler_table", "int_ratio", @@ -814,6 +816,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "cortex-a" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8256fd5103e10027467cc7a97c9ff27fcc4547ea24864da0aff2e7aef6e18e28" +dependencies = [ + "tock-registers", +] + [[package]] name = "cpumask" version = "0.1.0" diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index ab86e937b5..804531f671 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -58,6 +58,9 @@ driver-ramdisk = ["axdriver?/ramdisk", "axfs?/use-ramdisk"] driver-ixgbe = ["axdriver?/ixgbe"] driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"] +#Hypervisor support +hv = ["axhal/hv"] + # Logging log-level-off = ["axlog/log-level-off"] log-level-error = ["axlog/log-level-error"] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index b08438ef73..d2a5b8d38e 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -18,6 +18,7 @@ irq = [] tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] default = [] +hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] [dependencies] log = "=0.4.21" @@ -37,6 +38,7 @@ page_table_multiarch = { version = "0.5", optional = true } axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } +cortex-a = { version = "8.1.1", optional = true } [target.'cfg(target_arch = "x86_64")'.dependencies] x86 = "0.52" @@ -53,7 +55,8 @@ riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4" tock-registers = "0.8" -arm_gicv2 = "0.1" +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", rev = "7acc0dd" } +crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } dw_apb_uart = "0.1" diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 19c0134bb8..9a850cb47f 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -5,6 +5,10 @@ mod trap; use core::arch::asm; +#[cfg(feature = "hv")] +use aarch64_cpu::registers::{TTBR0_EL2, VBAR_EL2}; +//Todo: remove this, when hv is enabled, `TTBR1_EL1` is not used. +#[cfg_attr(feature = "hv", allow(unused_imports))] use aarch64_cpu::registers::{DAIF, TPIDR_EL0, TTBR0_EL1, TTBR1_EL1, VBAR_EL1}; use memory_addr::{PhysAddr, VirtAddr}; use tock_registers::interfaces::{Readable, Writeable}; @@ -49,8 +53,13 @@ pub fn halt() { /// Returns the physical address of the page table root. #[inline] pub fn read_page_table_root() -> PhysAddr { + #[cfg(not(feature = "hv"))] let root = TTBR1_EL1.get(); - pa!(root as usize) + + #[cfg(feature = "hv")] + let root = TTBR0_EL2.get(); + + PhysAddr::from(root as usize) } /// Reads the `TTBR0_EL1` register. @@ -68,8 +77,17 @@ pub unsafe fn write_page_table_root(root_paddr: PhysAddr) { let old_root = read_page_table_root(); trace!("set page table root: {:#x} => {:#x}", old_root, root_paddr); if old_root != root_paddr { - // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) - TTBR1_EL1.set(root_paddr.as_usize() as _); + #[cfg(not(feature = "hv"))] + { + // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff) + TTBR1_EL1.set(root_paddr.as_usize() as _); + } + + #[cfg(feature = "hv")] + { + // kernel space page table at EL2 use TTBR0_EL2 (0x0000_0000_0000_0000..0x0000_ffff_ffff_ffff) + TTBR0_EL2.set(root_paddr.as_usize() as _); + } flush_tlb(None); } } @@ -92,10 +110,24 @@ pub unsafe fn write_page_table_root0(root_paddr: PhysAddr) { pub fn flush_tlb(vaddr: Option) { unsafe { if let Some(vaddr) = vaddr { - asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + #[cfg(not(feature = "hv"))] + { + asm!("tlbi vaae1is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + } + #[cfg(feature = "hv")] + { + asm!("tlbi vae2is, {}; dsb sy; isb", in(reg) vaddr.as_usize()) + } } else { // flush the entire TLB - asm!("tlbi vmalle1; dsb sy; isb") + #[cfg(not(feature = "hv"))] + { + asm!("tlbi vmalle1; dsb sy; isb") + } + #[cfg(feature = "hv")] + { + asm!("tlbi alle2is; dsb sy; isb") + } } } } @@ -108,8 +140,11 @@ pub fn flush_icache_all() { /// Sets the base address of the exception vector (writes `VBAR_EL1`). #[inline] -pub fn set_exception_vector_base(vbar_el1: usize) { - VBAR_EL1.set(vbar_el1 as _); +pub fn set_exception_vector_base(vbar: usize) { + #[cfg(not(feature = "hv"))] + VBAR_EL1.set(vbar as _); + #[cfg(feature = "hv")] + VBAR_EL2.set(vbar as _); } /// Flushes the data cache line (64 bytes) at the given virtual address diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 31a5e222db..e8dbab6f72 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,6 +1,9 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use page_table_entry::aarch64::{A64PTE, MemAttr}; +use memory_addr::PhysAddr; +//Todo: remove this, when hv is enabled, `MemAttr` is not used. +#[cfg_attr(feature = "hv", allow(unused_imports))] +use page_table_entry::aarch64::{MemAttr, A64PTE}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; @@ -14,6 +17,7 @@ static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; #[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; +#[cfg(not(feature = "hv"))] unsafe fn switch_to_el1() { SPSel.write(SPSel::SP::ELx); SP_EL0.set(0); @@ -57,6 +61,7 @@ unsafe fn switch_to_el1() { } } +#[cfg(not(feature = "hv"))] unsafe fn init_mmu() { MAIR_EL1.set(MemAttr::MAIR_VALUE); @@ -101,6 +106,7 @@ unsafe fn init_boot_page_table() { } /// The earliest entry point for the primary CPU. +#[cfg(not(feature = "hv"))] #[naked] #[unsafe(no_mangle)] #[unsafe(link_section = ".text.boot")] @@ -141,6 +147,7 @@ unsafe extern "C" fn _start() -> ! { } /// The earliest entry point for the secondary CPUs. +#[cfg(not(feature = "hv"))] #[cfg(feature = "smp")] #[naked] #[unsafe(no_mangle)] @@ -169,3 +176,147 @@ unsafe extern "C" fn _start_secondary() -> ! { entry = sym crate::platform::rust_entry_secondary, ) } + +#[cfg(feature = "hv")] +unsafe fn switch_to_el2() { + SPSel.write(SPSel::SP::ELx); + let current_el = CurrentEL.read(CurrentEL::EL); + + if current_el == 3 { + SCR_EL3.write( + SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, + ); + SPSR_EL3.write( + SPSR_EL3::M::EL2h + + SPSR_EL3::D::Masked + + SPSR_EL3::A::Masked + + SPSR_EL3::I::Masked + + SPSR_EL3::F::Masked, + ); + ELR_EL3.set(LR.get()); + SP_EL1.set(BOOT_STACK.as_ptr_range().end as u64); + // This should be SP_EL2. To + asm::eret(); + } +} + +#[cfg(feature = "hv")] +unsafe fn init_mmu_el2() { + // Set EL1 to 64bit. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Device-nGnRE memory + let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; + // Normal memory + let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; + MAIR_EL2.write(attr0 + attr1); // 0xff_04 + + // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. + let tcr_flags0 = TCR_EL2::TG0::KiB_4 + + TCR_EL2::SH0::Inner + + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::T0SZ.val(16); + TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); + barrier::isb(barrier::SY); + + let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + TTBR0_EL2.set(root_paddr); + + // Flush the entire TLB + crate::arch::flush_tlb(None); + + // Enable the MMU and turn on I-cache and D-cache + SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); + barrier::isb(barrier::SY); +} + +#[cfg(feature = "hv")] +unsafe fn cache_invalidate(cache_level: usize) { + core::arch::asm!( + r#" + msr csselr_el1, {0} + mrs x4, ccsidr_el1 // read cache size id. + and x1, x4, #0x7 + add x1, x1, #0x4 // x1 = cache line size. + ldr x3, =0x7fff + and x2, x3, x4, lsr #13 // x2 = cache set number – 1. + ldr x3, =0x3ff + and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. + clz w4, w3 // x4 = way position in the cisw instruction. + mov x5, #0 // x5 = way counter way_loop. + // way_loop: + 1: + mov x6, #0 // x6 = set counter set_loop. + // set_loop: + 2: + lsl x7, x5, x4 + orr x7, {0}, x7 // set way. + lsl x8, x6, x1 + orr x7, x7, x8 // set set. + dc cisw, x7 // clean and invalidate cache line. + add x6, x6, #1 // increment set counter. + cmp x6, x2 // last set reached yet? + ble 2b // if not, iterate set_loop, + add x5, x5, #1 // else, next way. + cmp x5, x3 // last way reached yet? + ble 1b // if not, iterate way_loop + "#, + in(reg) cache_level, + options(nostack) + ); +} + +#[cfg(feature = "hv")] +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start() -> ! { + // PC = 0x8_0000 + // X0 = dtb + core::arch::asm!(" + // disable cache and MMU + mrs x1, sctlr_el2 + bic x1, x1, #0xf + msr sctlr_el2, x1 + + // cache_invalidate(0): clear dl1$ + mov x0, #0 + bl {cache_invalidate} + mov x0, #2 + bl {cache_invalidate} + + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + mov x20, x0 // save DTB pointer + + adrp x8, {boot_stack} // setup boot stack + add x8, x8, {boot_stack_size} + mov sp, x8 + + bl {switch_to_el2} // switch to EL1 + bl {enable_fp} // enable fp/neon + bl {init_boot_page_table} + bl {init_mmu_el2} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry(cpu_id, dtb) + mov x1, x20 + ldr x8, ={entry} + blr x8 + b .", + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + options(noreturn), + ); +} diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 8b5bcb97b6..8af7ffc728 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -34,8 +34,8 @@ pub fn epochoffset_nanos() -> u64 { /// Set a one-shot timer. /// -/// A timer interrupt will be triggered at the specified monotonic time deadline (in nanoseconds). -#[cfg(feature = "irq")] +/// A timer interrupt will be triggered at the given deadline (in nanoseconds). +#[cfg(all(feature = "irq", not(feature = "hv")))] pub fn set_oneshot_timer(deadline_ns: u64) { let cnptct = CNTPCT_EL0.get(); let cnptct_deadline = nanos_to_ticks(deadline_ns); @@ -48,6 +48,23 @@ pub fn set_oneshot_timer(deadline_ns: u64) { } } +#[cfg(all(feature = "irq", feature = "hv"))] +pub fn set_oneshot_timer(deadline_ns: u64) { + let cnptct = CNTPCT_EL0.get(); + let cnptct_deadline = nanos_to_ticks(deadline_ns); + if cnptct < cnptct_deadline { + let interval = cnptct_deadline - cnptct; + debug_assert!(interval <= u32::MAX as u64); + unsafe { + core::arch::asm!("msr CNTHP_TVAL_EL2, {}", in(reg) interval); + } + } else { + unsafe { + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); + } + } +} + /// Early stage initialization: stores the timer frequency. pub(crate) fn init_early() { let freq = CNTFRQ_EL0.get(); @@ -77,10 +94,21 @@ pub(crate) fn init_early() { } pub(crate) fn init_percpu() { - #[cfg(feature = "irq")] + #[cfg(all(feature = "irq", not(feature = "hv")))] { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); CNTP_TVAL_EL0.set(0); - crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } + #[cfg(all(feature = "irq", feature = "hv"))] + { + // ENABLE, bit [0]， Enables the timer. + let ctl = 1; + let tval = 0; + unsafe { + core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) ctl); + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) tval); + } + } + #[cfg(feature = "irq")] + crate::platform::irq::set_enable(crate::platform::irq::TIMER_IRQ_NUM, true); } diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index f8b21c9113..5fdf37eb2b 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -7,9 +7,14 @@ use memory_addr::PhysAddr; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; +#[cfg(not(feature = "hv"))] /// The timer IRQ number. pub const TIMER_IRQ_NUM: usize = translate_irq(14, InterruptType::PPI).unwrap(); +#[cfg(feature = "hv")] +/// Non-secure EL2 Physical Timer irq number. +pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); + /// The UART IRQ number. pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); @@ -58,3 +63,141 @@ pub(crate) fn init_primary() { pub(crate) fn init_secondary() { GICC.init(); } + +#[cfg(feature = "hv")] +mod gic_if { + + use super::{GICC, GICD}; + use arm_gicv2::GicTrait; + struct GicIfImpl; + + #[crate_interface::impl_interface] + impl GicTrait for GicIfImpl { + /// Sets the enable status of a specific interrupt in the GICD (Distributor). + /// + /// # Parameters + /// - `vector`: The interrupt vector number, identifying the interrupt to be enabled or disabled. + /// - `enable`: A boolean value indicating whether to enable the interrupt. `true` enables the interrupt, `false` disables it. + /// + /// This function locks and accesses the GICD controller, then sets the enable status of the specified interrupt vector based on the `enable` parameter. + /// It provides a mechanism for controlling whether interrupts can trigger CPU responses, used for interrupt management. + fn set_enable(vector: usize, enable: bool) { + GICD.lock().set_enable(vector, enable); + } + + /// Retrieves the enable status of a specified interrupt vector from the GICD. + /// + /// # Parameters + /// - `vector`: The index of the interrupt vector, used to identify a specific interrupt source. + /// + /// # Returns + /// - `bool`: Indicates whether the specified interrupt vector is enabled. `true` means the interrupt vector is enabled, `false` means it is not enabled. + fn get_enable(vector: usize) -> bool { + GICD.lock().get_enable(vector) + } + + /// Get the type of the GICD register + /// + /// This function locks the GICD and calls its internal `get_typer` method to retrieve the type of the GICD register + /// + /// # Returns + /// * `u32` - The type of the GICD register + fn get_typer() -> u32 { + GICD.lock().get_typer() + } + + /// Get the Implementer ID Register (IIDR) of the interrupt controller + /// + /// This function locks the GICD (interrupt controller) and calls its `get_iidr` method to retrieve the value of the Implementer ID Register. + /// This register can be used to identify the specific hardware implementer and version. + fn get_iidr() -> u32 { + GICD.lock().get_iidr() + } + + /// Set the state of an interrupt source + /// + /// This function updates the state of a specific interrupt source in the GICD (Interrupt Controller). + /// It first locks the GICD and then updates the interrupt source state using the provided interrupt ID (`int_id`), + /// new state value (`state`), and current CPU ID (`current_cpu_id`). + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `state`: The new state value for the interrupt source. + /// - `current_cpu_id`: The ID of the current CPU. + fn set_state(int_id: usize, state: usize, current_cpu_id: usize) { + GICD.lock().set_state(int_id, state, current_cpu_id); + } + + /// Get the state of an interrupt source + /// + /// This function retrieves the current state of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The current state value. + fn get_state(int_id: usize) -> usize { + GICD.lock().get_state(int_id) + } + + /// Set the ICFGR (Interrupt Configuration and Control Register) + /// + /// This function sets the configuration of a specific interrupt source in the ICFGR. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `cfg`: The new configuration value. + fn set_icfgr(int_id: usize, cfg: u8) { + GICD.lock().set_icfgr(int_id, cfg); + } + + /// Get the target CPU for an interrupt source + /// + /// This function retrieves the target CPU for a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The target CPU ID. + fn get_target_cpu(int_id: usize) -> usize { + GICD.lock().get_target_cpu(int_id) + } + + /// Set the target CPU for an interrupt source + /// + /// This function sets the target CPU for a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `target`: The new target CPU value. + fn set_target_cpu(int_id: usize, target: u8) { + GICD.lock().set_target_cpu(int_id, target); + } + + /// Get the priority of an interrupt source + /// + /// This function retrieves the priority of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// + /// Returns: + /// - The priority value. + fn get_priority(int_id: usize) -> usize { + GICD.lock().get_priority(int_id) + } + + /// Set the priority of an interrupt source + /// + /// This function sets the priority of a specific interrupt source. + /// + /// Parameters: + /// - `int_id`: The ID of the interrupt source. + /// - `priority`: The new priority value. + fn set_priority(int_id: usize, priority: u8) { + GICD.lock().set_priority(int_id, priority); + } + } +} diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index 13a865384d..1d50d10d0b 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -30,6 +30,7 @@ unsafe extern "C" { pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { crate::mem::clear_bss(); crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_primary(cpu_id); super::aarch64_common::pl011::init_early(); @@ -38,6 +39,7 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { } #[cfg(feature = "smp")] +#[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { crate::arch::set_exception_vector_base(exception_vector_base as usize); crate::arch::write_page_table_root0(0.into()); // disable low address access diff --git a/modules/axruntime/Cargo.toml b/modules/axruntime/Cargo.toml index e6029da263..3f76289949 100644 --- a/modules/axruntime/Cargo.toml +++ b/modules/axruntime/Cargo.toml @@ -37,7 +37,7 @@ axdisplay = { workspace = true, optional = true } axtask = { workspace = true, optional = true } crate_interface = "0.1" -percpu = { version = "0.1", optional = true } +percpu = { version = "0.1.4", optional = true } kernel_guard = { version = "0.1", optional = true } chrono = { version = "0.4.38", default-features = false } diff --git a/modules/axtask/Cargo.toml b/modules/axtask/Cargo.toml index 6232c42f9d..f5d0ad73aa 100644 --- a/modules/axtask/Cargo.toml +++ b/modules/axtask/Cargo.toml @@ -40,7 +40,7 @@ cfg-if = "1.0" log = "=0.4.21" axhal = { workspace = true } axconfig = { workspace = true, optional = true } -percpu = { version = "0.1", optional = true } +percpu = { version = "0.1.4", optional = true } kspin = { version = "0.1", optional = true } lazyinit = { version = "0.2", optional = true } memory_addr = { version = "0.3", optional = true } diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 4adb9125b0..9801bc7c9c 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -66,6 +66,9 @@ driver-ramdisk = ["axfeat/driver-ramdisk"] driver-ixgbe = ["axfeat/driver-ixgbe"] driver-bcm2835-sdhci = ["axfeat/driver-bcm2835-sdhci"] +# Hypervisor support +hv = ["axfeat/hv"] + # Logging log-level-off = ["axfeat/log-level-off"] log-level-error = ["axfeat/log-level-error"] From abdea81fd2e4d057d640693b8be34126d9e4090e Mon Sep 17 00:00:00 2001 From: Debin Date: Tue, 22 Oct 2024 11:10:22 +0800 Subject: [PATCH 02/19] [feat]Adapting for rk3588. (#28) --- Makefile | 2 + doc/figures/RKDevTool3.3.png | Bin 0 -> 92129 bytes doc/platform_rk3588.md | 9 ++ modules/axhal/build.rs | 2 + .../axhal/src/platform/aarch64_common/boot.rs | 4 +- .../axhal/src/platform/aarch64_common/mod.rs | 5 +- .../platform/aarch64_rk3588j/dw_apb_uart.rs | 147 ++++++++++++++++++ .../axhal/src/platform/aarch64_rk3588j/mem.rs | 68 ++++++++ .../axhal/src/platform/aarch64_rk3588j/mod.rs | 66 ++++++++ .../axhal/src/platform/aarch64_rk3588j/mp.rs | 20 +++ modules/axhal/src/platform/mod.rs | 3 + platforms/aarch64-rk3588j.toml | 68 ++++++++ scripts/make/rk3588.mk | 13 ++ 13 files changed, 404 insertions(+), 3 deletions(-) create mode 100644 doc/figures/RKDevTool3.3.png create mode 100644 doc/platform_rk3588.md create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mem.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mod.rs create mode 100644 modules/axhal/src/platform/aarch64_rk3588j/mp.rs create mode 100644 platforms/aarch64-rk3588j.toml create mode 100644 scripts/make/rk3588.mk diff --git a/Makefile b/Makefile index f71378e3d8..c30c887f9d 100644 --- a/Makefile +++ b/Makefile @@ -142,6 +142,8 @@ ifeq ($(PLAT_NAME), aarch64-raspi4) include scripts/make/raspi4.mk else ifeq ($(PLAT_NAME), aarch64-bsta1000b) include scripts/make/bsta1000b-fada.mk +else ifeq ($(PLATFORM_NAME), aarch64-rk3588j) + include scripts/make/rk3588.mk endif defconfig: _axconfig-gen diff --git a/doc/figures/RKDevTool3.3.png b/doc/figures/RKDevTool3.3.png new file mode 100644 index 0000000000000000000000000000000000000000..d390f4219b81a1752fe22216aff5c200f51c0af5 GIT binary patch literal 92129 zcmeFY1yr2Nwl3NUPJje=cMqc98-qmln!3lC5K`V7EDe>93Ti1dgK@EG?I67Hj4-2m!`K|X$r^hfJ}fBT@Jqhg?7 zK0!i${6_`LX8-`oLpeG&>JwySObnDq0Av(Yw8u#3j|edch=_HQQ)ck+NyvmjI(nXs zi0N5ecdx{xR9=2TA?@1whNj6WQd&(9Ov|_gItC_Y76FB@x;{o8J`pJmGq-p2uVm${ zZ611|`j_beejmrfp#JCx9Ru^BTpbti2pRCuDHb;J13*Q35bt9oWE5OfG(3D-9u2d& z>JxMVLS89dh)c@&1cI1^j_;Kw=w0HcFO#QQ@wqib^bF>!7>xV^f)9RxDi*J<7_Z)(-Cj2{sTgW{@0~)#fr{#4z3QPAP|B~F8*m%%&5SZBx>EPi zjN_BP?YJ5cW2PYfHoGvF$R0KqvA0sUqt}QMSF_x-AFb7WTpFkwU8T*;qU2L=uT#*W zRixJw)^v*K(A>mOBA~N}!>C3P28pr;TJ8cJtiTrqS(wISM3oU{h#kH1{6+|oIp$K& z*ZHitQcDZV;De+_rf4}}AJ2#@h@@NxPEpcM811m!$206owS%a%ukPQ1(dE|nM?^%V zv2oq9IiHXXAA4U@h!3PMv8cVUNAn$PTHR8f-hJi0uMQ*@dd0}{Q`DR`mgJhOmRf@r zOVT5ioxrl>##&JP^Uz$I9R5r3x?7LWZ%V5(udFD+qQ(wRjJB|IZ?h)j4sVou`?Z=a zsD#~6R{U*-amd97o}TCnyD@Ln#ShiqJ#1qYEWuo+j+xO6eYcBSl}a{Lg;qM^My1M* z){D<=bG62Gj2y}sJC0+`1*g<2xNcP!`|c+?l5LkNb_c4nyIpeW?`+HhciUo?U4A6G zHqi%+33}t7ZTtevXw#bfu4un6A4&(b_60|}+j2@ue;Jl+z`B{W!9};z7yib@Dh#iC zZIEk`%tR2Ampe^&Bf*>+KDNO0KpTVY7PC7|KXiuAVBJ^>XGmk~EN6ZW8_=$yB0|cS zD1Hi#i;gciFCDStN&Y6wj8{qS@k}~Vg^i&+gU&=$U%b;F59n*$A#EqKz&Wdzp0z*H z6M?buLx}>a>ww#RCTs5`c$rcwBv1j?5yk~xILu9mWHyNC6p&{=^^h|8jMb(gjwqcWaj zXjU#2(SBM$CZg?D&OfZwLo%@W^jVFZi7tCPq2pX(wwbxo6{okVvQd7YP(CXq8NZxV zDw(UB^?IaUs3N|qhOo*e4Ly96e*rauxr~8|-`L3rx{r*|f*R*WqDAU@=j=7 z)BKD9RA!I}m{qpwj=z?vuY}Qja#vB)E=efQF*s~f!H+9%mjON8fK(LZl24<7(oZ;le;=BFN@nJy3;h8Z^;9tiQu}*EUzfv z#6y@(S4E)e*MG$fTn^+6)?zp2uB3(NA`v|_JCOxxg%kmOwYiBE-u4m^jOfc^Jmxl5 zY+XG5&-Rz5bqsBq()&46EC=Vb=6GMdNEsPA@W-y8yrJjmgM*$)KV=b#1QYk8l~xEx zFcwbj{?4b#pp3JPr$79i{KG6WVeV zEtcNcoPzLVR%;6%EUvC59Myqyup4dV8E25eegWb)=0-|WTvQo;z$Gt7`nCSV)4wkk z%GO(6_aT1eIUZb*Z+*&KHq;=C_>#tDH75Eb?^`G>L(>#`GZ%dy+tJp72FG%FOJ7F5 zgMVey(?;!y=;ra+3S$uIfX;?YYJ~uK19ik2%VDyZ@3X3nIbX6XHj!G9(hMWSvhl&0 zQ4%4lVt1CyC9@E1A&n4EsNrdH$!96|0Z>Sss2b zF>@4^5H9))fH7E165G#9a7YQz;pUS@#kNavG>}O7Ycc zv8YkgzMrCC35XQ97PQ*q|I zs)h3JQh52hl3|a%^>iS;GsOSl|MyISa=tfUy89P^`tO=(bxwV?WZ(SHzh$D9y*yC~ z;<|XCb;IeDl-t(8kAF+od%3SF7{BkEw)AIq{Vh`>>6mXySuYPbrgO#mJElr`G|Ka@ zUHm;FsEzV34G$3gyF_rXPHn$t8vS>PK)CN;NB{ST;GZM^mv@2RO+e~_2>vPaKYZ^0 z%WeEqTmBPmIsF1hfIl{5vL`zkc6gIw>+2vQn-MCdGg@IJB4ET|@eJ^y(WOOH!=!G% zz(A{_d9rfo?6Cz&CydU#DcK&*{PQxK^7t3PIriTOSuN|gH-tYEa@~ZcEG)u)m7E<2 zD+0Dqp>X}k;cE8lV7BTyPT9B#Y}q^rFip6-qrOK!Mg5PrOz=?Xg-1e zWpT$o%ZN>VG+%Q`Q7yQLXC$`RzElv0T2Y+^B4jchFz)_D_NXB1Ji+%%4S&Fe=krC| zB>XouM*D|^Ye}ZuAXyzl`@-?0Cm>L1qZ6n+sJn80x>OfzIAaiCz@@7;pwm7pJ>=?! z4EOck%+NWUt(7IP8dVsVB{0ZMXUR>A+g9P9RN>z(;GZl^w^I?dd-(e=zfYUKmYMu4 zS}eEkZqTQIDB&0R=Ep@;>a?}wA&j=*g1XFX2c@1hvM$_&#jV+@t&8$`a@#9!O>lqW z^3+aEawr`)+6_{S2x=gpKsqrHkjXYVV#UZrM?$$Fak>Peli!^;Rm2DiG*BSZfO~Yl zQb}Np;ZFSs9cri#6O#Dx_#|d09GYF2X1)}Y*aeyXmKx1Shfwlu*^0E9U{L5$%xp0{ z8IB&%^%m6Ps2vNEfZ4RA7v9>$!`O1!*b42wnV<-i(D!=_#LR7z2+sGRH4Y%FHls!~p$2XGY$h3`)ET2kw;;vEn@skzq?M&UA9ied1 zf#yK!i)?p7fdv7JsS!efDWk%t_~2NG!f8ufZkW5@yn%gZnv5pQfzcV3sw%Jj;x)A8 z8j~rDN4BQXk>#l_6Rm+LqS&l1Eo617K(TlndKixhcFn%gf6Tzc{@O1_z5#1m7`K!H z+Y&e}G+1k|N*#9aQMLw6L*;VXj7^wgXLsJ=5f$VPuu*GwTg%;dmOu=;72tKkS6 zN*31ib_lmZ`GvxoA&2n+{P}Zy;C$kQp$ol}xm1$i0TS2FU@Fx8XS9{d7WQQFB2ST0l=BLI8muDJrZ{b4;LVHt*;GBGX=o-fKr7BX}OX%aYfsFP ztzp9Gndl)jAMGMT;at2aDL4y}?Ad9J)Td9NZ;vYm(JOR&6;q3iF%+ z6{<07s2sel=q=sgBs~=~8ExsaYM?I6Q|yoF!+zYoaRWChd-YCD6aJFRTr6w!g-Zn% zP&&n-=~>&-(Hm57SBhgsG9TS$Ez2B8zcJnIVq1y7@{18Kj_;V_LEFM?^Je9ik86#8W=cwv#P|fba9`dY--6} zfViyAUr^z`I}|AELnYlKBhgYSr77e?KYKM~)yO5No?*XfC4y^6wzVB%U0cK+7sB6J zd=NSj(UL)zOA3@1h=cRtGuN0v1&z=Idx^|Qc0ZL$i6~Ajxi5=fWNp_G?X`ln_i;qQ zBF1)H{ascxVhf`y+77s$%&#KCnQZ&$=yGBGunU!+iE)+=uG$V7gGO=)0gyo+N_lG8CB}?x zqxI!S0F~I*5+TJAfu-ZJa=Wslqto4Zx$;bBap~$y06=xYr^uva0{}pBY7LCPeG35S zHpWQ20RXJG+2QzM007>2 z;Iuw37(Tir{uW`AHoUmRUrMErk=}o4tukaK2=Z_N4_c;KrZE|z^O;DrX^6Z9V7;Xv z&tls~h%bF!`@BsQ?&883PywGN$DefB9ds%mWm6?j z&IFTtgM%Eu^bE!OKx-SB#a`5hKWFhvui0Q3Gk^KGAYCzulD=U%Jih<%8qPN&xYT*+ zj0Bp4lP9>&IcR(oZ)%hhf5GRv>lq&aHwl_@^M-)md-+&zPaHG+bZJ9d*kS$!&=33t z*q8qWxD6T?0*E73MZ~ueSqc+mR~20Bx%GtrGvJfi{IUv?;PS8F;MK zQTg;gYCO7pboP&@yZ%uF;Q#Xfd-K!@AZ-3=m>Y|IMQ8INq9Q$UQiaQ~)PPIl7XTe0 zx?fJr3WK^&s@TmKxKBsGiVWa{t!RswWr??)zIH?$Y7vG=Gc|!Mr&HFDeU<4 zs`&wJe&pd|=VeZ|H9R`H`JRRI8^%t&&lAZxa3ApIv+e+`tRMb?hF+Q*=~$|9d~IA> zIzJVq_zjb@{*-f-oZXK%IX&^)3w&IU{%LkZkfP}YJOXqDJ z+rfectYywdp7<}VNaBC`9oOVc+3@|0 zhI2DH&g`}P@UM74-nM_a0@7BJ#iKTNln-K``{x4ua{)yEPgp>w>25-|c~tDaURmX5wzZ{-LtPen zTo5i50;DaYmeO9Dtk~YI_B2>SL~i2WEpF6;%H-x3V0Ar%S}7{-vj3dw6VlL%(1f0r3LD+ahHQw^X{}yOP_Yl%vd7fqtx~&sZ=U!MQccQ`(3xnV2Lz`)j|b>hgev`zT4%h zs<0G$oRQ9gnX!%JRkln66+7Rad1M8$e3lBT4ng-iGa~jCTVw(9s(hYmZzzR6gp5eyIoDC znZWZ#C7uB|vrO3W4qQ%S@k9jjdE1Wvj3;8%!S%*q-NCX>XVG;u^$=I^tQJ)W)W}Ec zTe{1fP=ve#pJcZ-*=p^q2&81T`JhjzaHYiCX+mK$gn*r6v@jQEumQ~ZOjDsRJp8;BTTJAQ3wWwghvw3 zk~NlVA}@Do^bRPalX_gopQN~Yx*Big>Km`V6BgdrVR}1dPoV9xwus)G3o7e!py!!A z%G;& z=(pr^K#Z@{@(aLdq8AyN398z$<18F(`!Ln{CN0Oykml77J=q!DtkV^$=&ruJ52@JF z?rjm{i7uC)Xu6(mObb5$tZx%ntX60;>vj6E=Zg(KC?VcO2UZ`KbGXUzDJm|fc8q7 zdCaCyVm>xTLeje&Qr#fLWo;l`-z3nWp)URGyBL$k3u;yo}{*y)36 z^a1<(iOIq-qZBS<$2EWs#tt$EzUBFo5ufk1vOLpbdJ}NW37|BNSRuSn8@+WWB2wBx+3Jn+Z!Q;23irzUk`#~&Icc3 zH%cd;LrP-RFUqE^L_z3-%KX-Kog*>%f?>G&P7`SbaQ^iGQ6(?&`zbZmKLI9 zddX+T)a1BlE{5@I=pnr|b!JhQouz({H6yYr4hr9lXsYs5TTHXDoizxFxnd|jk|I{A zstBAarG8-lRc_4e!A=ehhib?`@>axpc;XR@M{O+qmAyspao?2A+5-BK*pYDY=2jXq zx^#VIV@46#a1$k*Sa#e%Oh39aBvoU|jL=3eid5W7jol_y1ZLDx`1&zC)S_(42<+;Wt15cup1x508dVni z&9+H9hu~mQu|G4y9KJr2>u`gOQd&JzebVJW8?g~%+RCZmJ=Ypt)-O5-hxBQ_NpMGf zkz#>m>9nELVPVBF@OYhJ@%yK=ND)a=3s>D1+P2Onlqyv_zoP!Ha4O;wQaEzebkZ4C zAI15(N)Hw9gl7J^yY%7ys^rW5gb8V~^qk7#Pj>{8b_uzx9z0Dw?cbwTh-Z((%B`Gm z$+RpLPnAMH@F?dn%G)c}FxX!yu*F`fn@Fk22IST*W4C!UcnU9m;G+})Ry^eqlB6@T z`GRgu7Cd_{a*hA#XlL;zF=tXjoI2x~Y3j0%6i_PCl9U6sz%pf*x}i-xWtPZRL;#N~ zcrHSs@frW1rMdJ&mG7r-QvvN(Eg^cgym=)P>#xmqOonl5d`-|al0F7-iY=y8O@;5ns^jKs&pSbWf8vH&+> z8)9syBhmzoUO{q?=UM@qN*Kn$RyqnVlX%){S$lCHRAwgmNO`mfaT!Pi_%J;fedpc0|Cz++qrTTAsN`g41EwKcnFI9$nb8U{3Iwd6cL2j`U2PeeF9)wdB1bE79Qig$SNX_1&P97}@|(&CALM6xb)E zMT>#^)vDs!j>TAqN>%F2GW$GKU(LVbr^QL6Eq?R7XTDHoCfb_>YsE#K4x+F4dRp>$ z#|Va|WuLWo!{674)O6*dwp5WiK$C`r#VWpW8YL|0gb@b%#>tK*jI(-QNG2Xm99Mck z%-!*QU*<9oPJ?aZ2~!$4#8P#9({f-DGi3G}T}0x#@tOpcOdGF0jJHi5$7jEWj{flP z!^%l>NoTKrMPC6~f42Sw-jgk8ctgQE^&y76|ARN8EFtntCZxe!0JO$d$>9aUtSSFvj z*9k_%SGbE>Komz6doh@PjB&{ILaYt7(lR)`j;6#Ql{C$SvpA`#rx@(|R19$x*XrP& z8ixGt{4&3-7;R4G0i~g21gj5CUv!A#B(#T;)D&+M>MC9pQU0tu{sV8)Q~LQ(`_RR5 z?{K@!%|L2*omxB&rOQgTrrht|ui=GLP4jBX9rz6cMFlB}kb&K^hx_#sdlh(1`lCf> zNLz+Rjy$@NHB?)1r|jI?Q~EX4151riD;gm!uS9(R?&X0X&AWk*QKlq_iSP~VtCo^R z=6NVAQWu}f8onj5+x}ExWT#~GYlcUJBcfWkjl)GSMcs(GNrQRqLS}LmU-t7CpnH{< z`TmHqwvILAZN$u~pPA_Mt*zy$;3;Rx$Uxj9WIp{joCkp}samutLwR>h_E~HIW+iw_CZAjiED+IKVjp!;_A!iUIaMi+vo>d_> zqpVJOmK!-lb0-`|bTxpB5gG<-r@GR_uKLt2H?bSsnwiRWEoR;s&xB%`bYMM&OKjK& zsnYUscv$$3On5N|k34W|O9>Y#UMM zJ)_FVB|>HlAakRz2bPr#`~q;vb9flBb5>!+H?6yNy2S4TUn@bQn3OzcFM3Kbxbd4% zt&f~#;172&L#>!C3pixYkV#rc+~vVxn|##9gPbWc?hK7q$x3l)g$kd~Z*1edGRA8% zAP3dxdQsZUI&f`;J33{hxN|yDhuYfy40QU3aEx@zPd#=6Zo#lNcj-%?%urg(vx8{g z(NY^C+_q?Uw~y8B)0;r%h<|_Nqu?S|Rq`IAAq%^Jcre$(sy}{K&YDKxbBI<{%xt~hR;1-*NF zWG(r1sJLcX#U0^Tr1tYu&i#$_F95}B-6;O9*%|aal$}Bz-B0=`pMPq{LQ~g53C$$F z80)vrIHlHpF_P|Ub$}*{;i>T7Wzra$mr7t?jil!vh`*1Ymw#1lU;v*>ovEKW(|&WQ zy*~^3c`@zsQyWxt9q2EhRgt7sk+TF6-K|L9FKo64%w%F@V~2~HjLG>gIBEc9+e)ZcSQL(c8_DdD9)e40Xl18oaNp z_nARGO7Foc5dW+@fDz(ZO`f-zIGcgYq5K`nD~E&1uYnKt{Vei@MGl;g>}Ll}Nf!r5 z8D$ysU|@V{QngIYV+yt=V|V50b|?)wzY1BGW2py?HjR)6iezch;|tP@!@Mb_-=}l? zEn3P48QX(C3ves%DD;H9uf6l4(a3&-&Z?eS^;7w3QwwTrjEuNUnrjyBXSRhO<*^6i7f{NhBEQaq%Y9t z*8%m>lXqC9j(KwCN|9Ekm4`G7awNXpPTE^`2?4&wJY_#=L6HDQQ>V_v&FPYKVY(|# zh$yscy_O#5pZe_8k7mZosCbB7G3Y#dv}VajD>B?bHNtmu(FboK-q0qJBkgBpGRo44 z-;A&IvFzc=f9c|yB`VJ_rX8io8W)`X6}U{2D8)*^S^j*m`pFui0oNVHU9n=)Ix1CN zNXX<{{?B2(pZ6>UwA@c`*e zqAF)q9*^pc8HI&naUlBH5&s(r^6y?N9tKGnn&M=IdGX-lLeJ-WLA+95$fS=H0j(Jv zaw>9&M3QkfD^|MhvZ2+4uaB)Y*B8G;a|z|$iakV9va=%_yHOWbhQ4!_Vv+W-mohRu zbi535B~-)$%B6MMiBbG9?1c631`urExridPIM?#bi=iCZ!jT>Njt zX7390Jn3G{VKLRcy=%>*U^NY9bK}#bA#%X2?m*-?GjgwLEFR#Pp=0cFI z)7JwSN?QtFv8fwKA@Q-92R&=>Fchlpd8HcAwKJ-_5O+;k%?Kye&H!R$FO6J_1t7oS zPn$IGQ@C~P1(qoyt_dYpuv1s>h&rS7AFDVj_C@tZls_J;3r;@s=d(Hz<(tF0O=EE7 z<4>GOFn4?&N)keNuv!nq&9iXbb5rZ*P!^wVP7B4bN}9lJ6mU@ zDQE^M@Fu4VHPH1;Z*cwe7blAucr$4=+r}trcrwxQF}fPddOOq7HG$z^-GP=~XxxYk zv1?+it`SJ0n?b2L@d4y-);rXf_iV%D(1V8%l;&ig~8InbM`tepqO2b^*$fbQUi9U=T+Q&{G z<1IpK&L7$YzEjY2CWIk33Jfu?F=&N!pWSgfR?`wMONy;sPmgsw#5{I6`%p`-oV@uWTo1Te{M1xGVHuF(R_OfM1RU^mVjR}2>Z-FG| z8MEt&_Irol!w2t5yy-&%*%8nf#mE^5%hx>Dngg%V0#6H3TV6YjUf2@e5%+J@=rV5DV41h2!EUwahdeQ`iie^Gn~YG}ARR9jF|dohj#^DIoOA3DT(psrW~CFttIo5oZrIDmsoh zym%~9K6-^B&b>4}qt7$Gp6PZtrynKbe+FQ%{6%N~RW{~nqG@{mFMx{hzz<+|Zdb&g z)9W|=KdycOYA8G(-w=QNx&G%3>M0s;-|dH}F51zbi&uVze%KD}No)9oO!m``ix&jX zkv@xI+=qQ%+H*?fym{JJ>DOI(yYWT+FY+V*I9O(Ed?z0M)enH$u0zD@qBbya#1mE2 zZ~z<+H>|3G^V*-(FeFi#_xW$dfifrZ*iu zZE84A+*|uhR+a`iZ?Ctprt%qn?zc6~6!{)3Ke{?vY3Dv3DV@JlG+v*}e0=w|fI?+! zJ9~)!C~;-qO}T24%*BCKgj+eG)AF;)Eg>wUWG4=f!9e5LQgGrB5N&LcY14z+==tmF z7TJWuuigw1{^i;Zf+~6g`FA_tR0=wfe*s4M14bAmnySC`M4P-@qt+c+J>&e&d=JUH zvZKB1i9zLXxp?~Qw%r+1^>o?QhYEOocy2t==>>Si_+K&YUz9&G;f*}k&e<&@l?>NQ z)PwvfX%r|zoxO4;9sSLe?t;4Y|G~CCAD(ji{Q{hvj>Hu7D%@^(Lupn$<3tq_zFgmj z<`t6EgJzq2Bd~&RISh?$EVKy_1`P&RAxeh2&a#J=!E8@bi-2|==0P8hwd}7FCkqyQ zv`o_N3QTEgEhVYi&ha|-6Ht^sUn1?ziiJE}HBl)kDOpPvD`sV7A#Htg`|dmrUwoje zNR_l=HOP7-74CAAAarEZ7H~Ouv{cc&4=Z>2^bgKoucv^@_#D2##XVZSQI)w-YK5v? zj(DbrcZYwrg+;*YdF_wug{~7mXY6e*s|tyDA-vSy7Cc)C*>ZgN;wBIKb$~Ikk$Tv- zgt9HLbs(xy`PQ+6rFr!ILYX^r_K$;wv-Ydy7KvA82TRAZsfRyX?2kuRZ0e6G_f3B6 za5tgfnGd!W^ev$^-P0od2L#}u|6YQIvp5ZOZCFr!k0(Yp=v&b6c1Nwo1>3a2WzZK& z5kF4O%sJPeK*X6@r+t}WeK`Bvd<2yhE}KAnxT`Y%Ir*jh0ugAFOJe6m-zhrze?UO& zOPPMM0ab<(P`JW@gY1Q#d#(rSN5n1naf5H0cXE+yb#0u`;j!-?bg~eAWaGBs zTBqIP88N#h$Wc0IBuHr;)^rtYSBEcJ-` zx8!f|e|rf&jRT`_f`ASqeRVnUxY>00ELo85PDlIP(dCvw)@bjj%t~Y1F-%A`mEP3x zFh%Pqso8v7i*T9UB6YT=k`;0-btD<~68ZgOiQkgH!T;?gOfRQ_5GMOFpuB|vcWzfX zew6@f<#?cf?b7~my;v4BGrYsucSReV^<1ufZh@<4iQTOyw~)P6r;n%F%0zShC}=V8 z*;@=`WdHw2zWWdO&yxQI=>&ypOaqm*9P(Gn*B-*{>fSVryvB^A=Q`_0h4B@>eb2&lw$d{ zV%{zH_koVok^2Rwn2o@PJ$+K4bJC6lrz*71jo)%^dSmADLuZJ@Nvemwic9a^v`WbS z=RQqwS!~oGiAcbmsK_f#U)7fl1yMln436kKP_ig;cUzW`q^)PKL!(zx*tfAsP{ z&qw82HD$X;Z^ zKjb`Ib`6j}yz6=kk;z$TF`2GyZ>Z=x^sj0-o%c^0$=QRndLoaP@lFfndyNw2eHY;D zYvl2B#w`gpOA@p02C-iu?#`t&v6mN5K%wqQ8Rg-*!5Nwgy)6^FMDaUYq2FfsBbD5O zkjMS0yIM?}Nsi*o>z*r<#W>BshpH}ipNi~FlrHEx=@glIW@XfuKGTh{x3jRQKaKZR zjCX=A32QzDt5$XkTS?t~D2~9@z>**K6r+`*-Hy3~v2mSYswF2z=U~CVtLA4%7dA4b zINJ8B={m}(^=aC_2XEmhsBa%s31kK`SC@lMGMHdOr*S|9x!DXVDG5yr;qSn2gEHwW zV8C(Ol0itEF?W|>UqdZDF-jm~+nNyt^t`qU5lC2O^E3;%xCO zh-P9?TSS_cAj?lQ-S5*+%Ogn&Ug((5TGt$YbEoTof)GD%o2@L+^7vyS$eZ%X#JCu_ zs5SL1{2+P_vu5#fFQ7hfSom=j)P2uEfqfqScHcoqZwV-Pa!XupM&_N{l;q=CZEc!@ zSzzzuRW`2M7~T`S=3;ia<;v7FEPA)kD>qqH-8F0Vwux1sx8@*&ch!V(NK-mPTuAl1 zJk3DU`cp}&BW!51d{X(pUCP%@^3cT8{Fh7l5*A+hOtC8L^Wh2H)=SBYE%ps0O=ax^ zmW|K!hkHHj!0Gh{oJ25DU+*e}qfjHzWt{d=)8b17E)`a66l|6>nQk|?&S0%c{J|6k zg|_$8oCbwa&Gn1P{vwOcHVv4seS5Y}P4wKwvPR3;QWm*9wX!-tni>1|uAAyE?h3=2 zTAN^R=F>Yn(`td3_NY;mqUJXl*IAHd4ldC8)?ze{*FZYVzwxPUlPcv zpSJL9rZVq)-TC09thOaYR%1e8QwB8W0CJICmPqc+v;_$>)H*Bko0QucHVQ|JHXMfK;a~-tx_a{FQ9U6#pAqT(jIFgX&N7fn8dC1f6A9yIcEY%&i2@dO75R- zmjG66SqvSaKh|-ro6{$Ea(KeajpjsFNx52Z;2gE^6xlj*nFr{>qF7Tw2xmBW7q=>M zx@AJ1UNE~ZW*&gdkS*gc^F$$Xh?hQ#)zD$X9DG}~ow)L(x6B#eE3f^)F<1a;$GC2U z$U(B^DywF5D67^w>@b}1+lC>tc;W0IX=m7;O`Ml>FIH2xxq=g^$aUv7hLA&Th}zaY zaZz*T4`z4fSXW_>`Rc>ERXPTwQN!tpHjX~B0pqw%kG%gj=j zW9)EJcJ7}@OiHxqdqg8lh9u@@*FRKsg~o|r%Jrvqws%^eFXk>c{!*Uu(?-hUd=>#10KPub#X zK=vkIR7>k5Z8Fn=k%HlCt>m;QUpmOQK26_OMxruv1Z-!|^oy+T(z4Ti)t$!^ zV^UfOA;&3nEs;{#k0Ub?>{+}tYdt&kwya6I2#Viw6RU~#nA_tnZLnfzt2MeUVr!># zr(cf&1;p0LFR%YDvfj1RWNY1i7O4b|Cc2r8=dJ6jv?xt`@(yitM+EL3L2&=X{QS&F zrWy-Yrb6_I|@n%I+o_V;c%48(0zs%tJwfMgXPXApU{{FvX@Y63M{Bn@c3+bFRfGD zs6BH{pp_s%TAk695WdvieInz3B*B{%9UxiHx~nDS|HD+BfW4(x!!HWsPc#J>Q6(B6~3<{O!GN$ubaLPdWd|#pDHR)LAgG4qFd>2^QRdN=wQSx=?$@O z>1Y6J^|=qlboZM%g=9rxud(q=Eq9Pyt9Dh@K{BHd4IesHc)I~Z0jgD~8XGP$F@)I- z9#Y@#F?wPIbm4AuC}BNB`tUSm)OZZ-g^HcGOSMDXeIil!3MvQFiYpSKYLwzn>wM(B#ct-Sz+xB~eod%J}z6!yq z$f-h~Xz@!C$UO5fe5CHc%dGCMoQE5t*oE3aUiolMQkwhPbR)pxGYRZ&ut+5=oH$}; z(@!jt^gK@R9Y@aEdmgjdMhfwwXq-mUnOSNHOMbH9@Hbbaoja5-H{SnP3JBnz%n;$S z=lMWW;ODet5IZ}8^|~B9%57vqkJR|px$Vfe6E_IF8u4c_WHg3l>m!m!Htppw!iEZ?}iQ z8-G@BcKeSo7#*uK(&qKqtGJ6v&Pw7Q)uEh28Kk1V7`1$-@s92UB z_+8A|#JX8TtBH`WYX5<^ze?t4lB&hJD+|zm&kZk>d%eYnQGm47fMa5R@P=14utQ8V zBWgjfMcb$;O_(g);>X31yc=z-nm{J2*(XZ|!XDZ6@0nSWW0(^VhERv5J&N=L&w0bo z(>~|WP|A-VJCgIl(Vmg&C-x7aVX3h8>Ggk$TgI-N0OMGhyLAg;wIlikzMtc!F!!=y zgp@Kl>w$sxE_WhTE?rU84eL5Uu~6b=rwx4BY_7E}+zF!aqLr#W$b7R&sY8TYdbr_c zBVW2>I4ZCGTJpAqwNP>sb8ng7O0bc_aJbN0`|C_B@N)yZSqXiV?aw{yOhog3^fU!u z5U#|Dao1@>^~)u{ON^RSwNH|IKJlXDWBa|JMO4jM>oGd7mHcBvY!vU2P9=7y{?)zf z0HO53!#P!4N_6*m;Lq7$zS?#WL< zJtjl3OyJp`UVRc;-4q-BYQcc-?o%<*FF?1i#;JW{qKYhXVh({m+FwlriKkF7k4~fg zio!$&VskJb{Vi59RA@ZoHdWxusT`1TAAG)%!2#~d`%CTpu zyS$~ILUv125&|Sud+kyO`*hcIqn20u20Zb<85Xlc-fVO zPRZJ19&Q5XE7o8-*$d!%HyxBX2)4NM=X0lu(-;WnR(x`EjEnv$6yV zhb)w4*Dv;U#gP=3g%Mtv-}5c9v>BY1E#~~r0wio+ zF^xm44U!5-ANg|QZJ@&_rBb#8kjY7~#TeFgRFlH7rPak_*J5>(Ha9oo^`--CEY~ii z7zD9rwX&Fu^j(YTc#}?tdK&qn+9~7hqBIu|co-ywzpM@i`4B>k{|9?-9TnH}?TL~E zLI{%J?(Xgu+#Q-m6I>g2jo>uaxQ5`+Xd{gi+#$F_pm7ZvG#T!^w^s67>we$NyEAL% zy}RbWb55Oes!pA%z4vGDU7rdNyIxdu5!^B*5M}MWP0cd4&TS!Agka4}xQE$c**tZK zv%@veg4u}0Tegy=q!U;$1BkmK1q9jdU6id5dnl3AtqUldJLWAlU-uIZ>y~{?3qILh zZ<_{50G6?CUudrz==-+dCF!bWxw4rGr~VDty8lIeFTK&MJR#Z{^7)jNj`ky${4bm* z@lQIx_wtE!%U{6mf6ctGcPGxfIhFAU79bzJ5x$m3jX0Y!V^0p;3Q`zZW^`6;!~{=v zrHJslnyQt0p@yc}x%X_@&=@VCQHrB0jmEyr>TSrfU;D7qKf<_Ry=}!sqDZgf)ROni zYn-ivY)cJv|Dk5dQLG?58U`iL%L4N1k#Y?Uh6F9SZQ*AxEXhc3fiiRJOK!!oxv^;6 z2Q_SS#CmKycX&Cq_tVHH5N>Gx+EYvQJcXzCfF7i}nuSt* zt==*u#6Znv3xoQl(x~xAAFwyzwc>7Wc6p{ooi<}Lx~77C`M})9s9Udf-30KXtXgDl zim-)yzQ1wH{9*KAKq;+xeHBP;Hu(ntHgM=5DQ%W8Q#l*8C5ma8hA`e`pUOekE0hK9 zRPgEa?DW!-?JwW0XeSm1u&!_hex2~f*U8~9H#kg;%;M;Uqs>ZZDbVp-pC^Pgulb5l z@nW}D-am2?(4{6P4G86^m|>Jurcw&#wJs+UztaD7ENsQr#AxAUXwhyt;IsB@2}1iY zDn7Xb02V8_TZForND}1=Bc5&g*U3(1Z1eCC7KbDBP)vNi_(L~~T2U#ruOXi!JynjL zdSi*MxRtqwyU&o1v$R3Dm!3ja-C^ikP{EeOyt#Bjv4Q!-X(oM0CZz1fk+BX+ojYLP zP{G`;wbD0>yHZB5E7ETAW)yE_>H~w3yRhV^rEWa9=!+k6VQ0}K{$u#A1mAlsXnvf4 ziPqZ|U4mkwx&u!O8Ga1rX0~OH1#!Ri2MPT|&mxuK{L9vB3h&dY8SlR;6lbyJ?1-6u zKWCxOz0SaD);MUn{62l8&+VnCUfzU#RyOHq7}Tsa$K?OQ=sh3p6m6O8WQQ1N^Na9U za?)~in`q{eIBoLfMF8 z+h@o2RIEwM_G$;X4nc8Pg~_819?j(GNAdgdsD#zT&POg{narZZ3?^0BV8Z3G0{r9t z%P|gmLlMqKFwUB@ql}BagC!lWW1}3F8T;AUppbFv(ezD6P5mllAYv=Sx6q$9ifM6= zE@9MyWJeg=i|m<6tXO|Y4)fL|`5YCawif?3v?R|0yt&($qc_&;p5K{o-?=kY<6;RG0+mta*h8SZ zSE{V?*on4kr6(#`=e3D)IY2oxhAg24?H~$cyL5(U8w`hVBo61;~wZl0j~JuoXJ3VX8SQu$HUV_XSR7ZeBS%->4Z0n}!5dqE!R}_Ba@gkS9j1 zY@f$l2zonBb#bZjOf2i;!WU<@;1=~InmsmfiFIwXBef>khoM%Uk%u&df9COpx5;hF z6oz$9>~0FY{%s+NRK!)k`ww>VsRz8i)6A7pfdA~B21YlxgFeWsf~CO;xJ(s<0JL7w zErc;sECosC{YC<3-hkJR8s_^N$Zj^pq^3y6OU#C76M}JgzI2BO_z617aKo;d&J4Xy zaymh69p1m@%y*?lHxK_1TloKJygV-q$p_U;a)Oey zO|2HCgZldNBrW+jl5rZ#^M5Gd_I`PzU(oP_F+t?j`-2Zzrlg6IH#CF~YsTbpeUWE` zlv@TJRhZj@+x#qa%6j|~{)q?#YP zpI%18d6sJZE7T59BT87G{bdpN!^)%@W1r-D;7F&-Huq1CpOQd7#we3zBU*&{RXe5f zN|Qj&a^6Qd)fsF;&1iM_0!DcZ@InZe51b9W5fmBWuMn*GR;zRLfflhZCHUm0)g`Vz zxJYvi630jfPsGtYg76zoLiVG}@{cA*4xY2&6m5}h*0S&yeL5XnEO>5V1~!hkts9oFFnuwg{s~!wiY-a?kSuZl z%J|Y~a_Ghb!N)xHtM@Tq!{%Ec({x|5m$FI)n7_F-T)AE=+{{l@B4vZRWRK;0pEq)=j%ld4XiK5jIz z>uKQEj=1=)$^HFlZZa0O3brx@%-SLQp&^@0@3E57lI)Iv_46IhRX&%QgCY!*CciDu zgG#j(qCWm8uD#rqXp31f`!!L>>Bos4hG%L|vcMqA5u^A5RaRnEr{ZFq_Tm7Do3Cj9 z@qGErWC0ps?VGRCtq`DxYE_SA+i<&JQ8?{-CL4}&jSMyXT8r^QROLr z7puZ4qVk2o(57uP)4v8` zz4-hqsCG-1bZdA`D&zAI0|5q8N5pcy{W?wvJVC~D)+zeJ0A_R7p{+}rzc}~J(V4XO z;w@HJ^`RRpVHI!+6VDMp*O9m0GQkMkG%)R+kvXb2s^jw;>23W|;G#@fjBqEHDy=d% znLhT{tt$DLz*m!S4Qp?;XAq%@d0xXxK>cw0rfP!ssEU+iT3X^Zp+*JabrN#utMy|| z8aZ$nB#E5b(3KC3gHAY+je14W6JJekW788t z;9fH+cdR^^BSY8jz#PmJJg#8F0OijvY)dnN7^>xEG);{%Otg9%lYk<>XUOC)QWp@- zqJq~OPNQ*VdbrAsFuTjY^)M|&jzZd+Qy8jv@hQn-?ejjL*Hi}ew*_rv^!1&XWyH;~J>coGHpMhO!>)7NHt?^_#b&(`k4RP^X|}%$4=?Y0F+X%x@R^JTbe2z-;0N zIWvad4q&w0)Cc4@}6R@|@ePQg7?he@{T!#LD1{0l6tJywClsA0(dEuWYy*=EQll ztUUW4twws7^3q>_8<(>6n7g9D?xP_G!eT(5Ox>o;G7++%hID{ZL?=&Y6uGM@Ojg`H zp&gq%Jw+0`#A8(6Cn$gpC4&^v3+?aibT2|}NBM>;j1EDg8GU47+SM7VA^g;&eD9Sv zq!;`#YkzVSmab|^wAn^3GS^o&U@XfUDeC0S1k)QPYB2|^iK0(Cl%u%2-M*>lYuJOw zCPwn}kKv_#E3=4ttL!A8Z@Q_6n>}v3my>qDG#O0?$M*aw-fO4KFKahYG%iLlluAxg z+D0xv^T_|CrsqIVI3m|rS>X9O%4<8qHk2iWdd6B93?s$A^rj%R&dfHiXXaXPKnQT% zxvzGY^Ws)WpfZrngg2B+NG>O8`_1PE<2wn??^LQfz%S6>RPPHw_w|+d!bGmh_KU~} zwooNj#3USP{4%6$(A#XVyFmTmsoH$Eb#}wY{3}JzU2l5AZNI#t5tjr`X(esl)?|`$ zC5NN5>DsIhP@#3!vE$ipOy&W56$OrK@T%*iB`2Rzpn^TqXetQDIAE>O?rKuo*t&BY z_S{X&+R{C$C!;9d`;|Gd)UQlC;TrN~R^jr#sZ|@f(%zzd{Pd{Q3I-0rQ>9$jPU}}4 zXc?ter?f;eB4LW%R_x`6{*v-{ZF2!Em-Jml^GlVw(I(gr6*T~JOmZNYVDPDAfo zb-xtTII?NENXz_w(ZZ;L71xgRX;bHpWU|4q(bSH?&&vhwCCPHM!GwhebPyemyi#0s zon-Bo0}*{GdRfZMysjRp+^oE}=2IUAPE1^0XEDVqHv-TG*yUZ); zje5>dXj!WbI&gozE%Nq05i4vqPUYFDQ@{Up1I?^9-2OPBeWo1XsNDN=50jO9cTzWF ziR+L#X`#?Z%~A4ob|bsE?%)kIpL{v$^Ebs}S6AO4=O4K>B8=H*3$=_KW4T~Fr-5LE zJXc7_C-yXVP9_U<9sQh2Vguhszu~brb!$!n@NSMGb+>gt)fnMHR6=(3SD_G%LZPHz zwItov#FM7>$$kpHb>{|hAVYwhER<;E2zEwL7n7Wte36w{!uW-3#f3#xnPKUX$iQHE~JysJ5N49uDC z@F%fap=hb)2VCvkGom}Dc>x6UHM~I;49a%s_?{dNR5tVk)(S%9N@aGaE?3k^lzR<{ znb<2`wHYHI@of*oEJLR+25F^!ljewEybM$_f_Ha6(#ulb;&6xXmRN$hr{*8Yb|JX3 z_Fr8rU%HPwgU~pIXz{K?yH@P=@EZmY=(L=K^J9D#FBBDd7Q%VX!no@BefQUsQ!~TM zTQ||wt`H6ZJo}bMz+YRm*|KHKq}Oen26T{gPkhA;x2&>wfUG8&x}5`rtz>DBwP z?b&V2iIS5Fe=v&jc9)x}6+eHq6jNT?+q|*cEVW|I@3c<&5dA9FSa>X1Y;ONtkzhHI z(foPyY-WwB;iB0Zp#%1jnLsr_v+BLKRMPB-XoJX{oq{*5`34c?Z=@JASB+5dDSGY5 zq}8LFD4nou9@Y#!Rr|j9)~wj$K)PyN3IPqJzi_X;v9VS zt_!-R?)xPOfNJx{v^7|_SnN;HBB2&X6Nfjok@fr!pFw5=w}2!o!N@aa(%S-Em2m^{ zo*!Pj3o=|~4S|>yAp5Wlmi)-E@{|-UwUJhkyvkp?D@d^5Hhp%R5j^gWHr-H9RdAc8 z%k&uqc!l0#SQ;`>8ewz{APgH1pEx>{M%VW@Ew75nuIa9;l)ZnZ+Co`>0JcMkzNWMo zYd$bL$^ZB!<{D=bM7eT2@%6`|P{pr}?~QUXLmICrkVC~MLF|-C zyxsb@gMfl*<+oGnU6&*W*DZgPKdGH*LH)Q59+W;%%R$0tDc_Lz*=MZxd&ZAR2_BM9 zq4tErX(}0z*;;Ys%bpq3LHX!-ZtY}Jp ziLVY-zI3^Ar~gA!MNZ)WBh;E+7lO_;j9g(^(^gUS6e>RmC-pUmADvW3?fMibP2HbP zlEkPRcITf@(_(rip`YMEW?Lk-?9h{r78R1A2Jvf1pM9N)JoJTukSZ21Xryab`%$|zVUr}TyA^`H|M!JSpTMHvr3iXD#k;%Y|;@**q^UE}h_FN9SqXn$X1+`)eC zJA&zKp>`CnPA0JIviYm0XvLzBX{uHaze=U83HZCFb~hp)O3nZ`HI}aw4l(~Gifw?% zt%n{1JuV^Buh(+$8+Lf0i*Xhw^86@Oj*`!(S_Sc^Z7lJyCdo-E!q+v|o3o}!XBc`* z?HEvE=e$@N%(`Z?P5RMlwcrj?kqedZoa;Ts=>&+*g=1?BAT2KP`4H{*JW=u3v1Fa$ z&uVXAb_cHq;e_RKux93Ab%&#?O!7^rr=A9{831L`D{5>zipKJqJqVmH7G^28Usg@npm286Zm zNIK~^Sxy(MiQKo)0PgtuA;~I)u9Ar1V->JMbVF~de`I6cZoPDyt#b}fir(=G5~ zuuRFVEi$eCplkM>?4CQMK?HlzNy$u@>({w`9<+a!K!4Q|(E^+BIihdf2*?#VEG<(I z*mZrKKxnjEO<${j#5=u>ymOVhnBtyB;YJ(Jw%LT)L@@{=5KVwhdekoS(5Se!be(a8 zG*7D;DL5y2vbHF@SoV@Il&}ab?W*-lI@2J)fVB}%x*Y@PtA{jg&zG6D1=Ns1W$#Qe zLS3FvdQ05kkXa8Km#0hVdsf)La!k(YQ|%1Zl~_4i**TO$g(RjV^bl5RIkO@OS@Ml& z0zInoJ+|HN={L76xOqYVL+17H;l@dA{^(Rzx}r%BXO?Z=cof=uO>>(&PXdb?p&`5BHf;e(^^{!xQoIaU zsR&p|^yAv=-Eru^g%buUCS3YL*X70|d5(R)g#nZQ#aEQ@zoe|_A1Ouo(Ak1#j^3uJ2AwzdKV|z;ZZ=@WidHZnqm5mwh+Ij> zo?5*m>9%i}y@h))8XXmamABo>S3 z_WCV~Ey-K$lj=QjD^<98W%>OL&h%%(GSx^OC%=)C6uY&2rg*;H;Fn63Q<0Xet;p8R zhEoseyJ~1fs(u(W;2p3`9kt@zLUjYEo)OcZ8>rmKY`6b~l~4x)4DV>g-W*g-m}dKj zR3WiR{<&3ZIXO`*OetVi`oo_8~9ZxxKU2dnu3e{Zf)`Nz}lpY)dF z)R4FQNgYY#e^vT7`1?QA{Acbo;q9G=cNW`(`o~>gGyMOWX?0;-n>_S);rQ#NXhIbm z%1Q)aQqbpG`;ClS*8)6gK7Er4zUiU7p#0$`)Q$s1JQ{ZwE4dr_wM>*A#GoiemA?SZ}EqLX+4P!{kGWb zc#pa}sbk}5^(Gdb(!yJQ>lvs>^^`3upyD~dvcVxR#~3p~2AP}tOoc7h(Y0?*{Mf;; zEV=w2B?kZB5<^1fUp)+@2Ft1Ur%FLFd=Gmvcy@V16rvsd<ecMpA)}34W{KUzJrMr1(bhc+gdbE_z!sNkw zp0X6vV>j5!sq7+J#Z#sNlW^i*^q1=jhPPtfGvy!umU+Ivrph#ZG@^mqW&iKxp2GoL z`rZiob_?J`!#Cn@ohEDGl$#iftED%=aK;B>>~-f($@3V>TBTQ&e=S80hz>&gjg)2k zkf9jpYS9@@*7mGNqAIg1vDG_kH5}4*7cl2hor8p=^ya^{nBX7E^}%`fGmN)(eHt48 zdIGY4IVGeVQ`mtLFB2ij0i{`(j;>HZx?P3M6iasO@l;5mkuAO2?A{Bs?{q^n(T zkG&ObQ~lF-W?8}YrcB9Au=2xNs@xnNjgVWSv-P&kNge!S%Dts|oWu8gv>(?;qAMM& zSN^||KE%u%r7npEiq6?NtKBV*oJg#-VS43|3q?wJ49%vf@Grhl+fCUFLZqah9=t^7 zS~PSUn;MYUgaV9-{sP_X za7-`b5B7|Z@vJ0!OD8|r$e_c7cJw~IO5ZFgM9|n#HiCIT(N#+yFX5AO=_wSabh+^R zjzpi;Xhq$H;02s#fbB64UzmNQ!R?WMG;XuC;~O1f=KE7A?D2;WzmbkV{YFZBcF?CR z33(i^ZxZA5KmAAn*xN-7y6oTW#r6l6SbP*ETYR=Pma(Yi%vm94g>y*gMo-?<0P(Ir zZ*U^7_2j8-aP-pqrhOC`+WfU#sK4oF6of*+V(H=gpg*+bEtiM-4-cs8WxUdzR^PT6 z6+3&$338Jn#Ekv6$YUaVH%svg&I-IPiN&~$;7H5&-R@Jy{#vZQP1Pl#HC1ESs*&D| z-u#(A3z=l0Ll3*1;rY?mZPVNj=392t8FGcJcF15NO%ctQ#Hc8F%=PM)n$XwXHDYRg z(_HHt?R@46{=wiL%j~AFMKB!)N5{(P;#z^6k+l{{J0@q2wKn)^``ps8lHKI}+qxt? zT`Q)vh8vm`W6fn%8mVeXWC+}|BHN3U z0DJCsTYWn(TN$H2o+G?>SeLzGYox-8Zj5t z2WQuINq{xa6`swNvpi&4%jL7~9Q<{KcEf#vF~V1~S0ztX zo{+dsV4c;FxF1yQ7J`gHJs z=}d~0>OyLUZ1uLl-KARF8&OwP%WMOfJ_oaoRNgexYcctjEB+??#E_We**4CUHaYy4 zW1I+c_rs;8wv1PR;Xs@>ImJZs?5e>QXtNB0mVL_kBWRE@CAl&zAzv>wFGX}lH&&`; zrNe-*1RFMgSzsEWBZIWn(h*VVOYvt>uE)u+ov3x9J7h|=%l^7cF4>7l;S8hxg-ZYl zSf}t1BiUP#s1iNKN`~mb$;m~u=8WGVK|7XMCsB9?V=XJq*#H1x&{6C?S^Qeu?3}cN zC@%|!Yqfu=XcU|{ zs*e0qsy;IuhvHry%Bjwq&-%g@CT@f4!0#<(BfSqIe)55*JFa&yn#i<@h{sFzqjB0# zzR?lGx33cu3ijf8F$;brucqtBE>5Q`j0-PXrNGF+lQG+KUC?7#JE~dKCs$BWdWtik zJIFY0h1VI>1<44^BuJLsc@K+t`%>(i3W!1Mn)GW<#Cd= z@+YnK-cOJ`7n-I_&!>SOEaoZ=8Vz6Hol|m*kQTirC8U^$rvTd}x9=}+fX~-&-K4!N zC-$CP4EhRg=ZofO;GIpJNw8E;Td?AGTVF0<$u*&~Xz>nUfm=pj{G?vGCR_Kic$QDK z#fbdeDmqbd{*slSx`-BF9Vw`2RU0ujqA#9&yi*tPU5dRa;}k^TPPDd~VU9&4pO_<6}gM z3Q#XT6;LcmZl&|km>9SF8)=~>?p$^cvd8K=9kp@XutCWDQ81M|dl$xL%<^TS_@5j627ays44`SWE0j3XChK)unUj%#7A~i>H;%^s@^AdO zivV{}_bA@+@V!B_g+zq{)G7MSwG8mFub9rT!oYP|w-&fu12c2ZQkY5HjW=%ejI~;BT?Lqx<|&Q2lKb76b_f{7$JX#EqNW`Q?2tSe~#P3(#2Y3kasGX%o4)aXQM`Lnrj&$0O zvMnb=44nbITKP%V%QzVns++xuOwM4iZ-4p0d45`t%b@kLmGr&-2lLZDgU|M#ZB&g1 z`pG2e+VUcT$##004Dq6J;#Kstpb5^ZS#dA=%dn01od&l2$S^zu42<1K;#GzTrm`a^ zm37=kd}evR3T+0hoXWiu&}DzwT)@mcllHQSm`5q*`Ubq6x(zQ9rfseVZ+H@82Jyut zE&(rzzfLI0--%Cg4Of1IW;|{NO`lARuluR=uzksyOEdi{_%Dfyp)r)6KWV66I-jLv z-o&%jSOA3>wcD%3sQ|U(6V;1N+b=*1bphFpMLg=|3#_SlrMVOKcrPeuP3}mR>y^6X zhYn0rhfVV$(f@iWtFd6SV0bOW{l*N+&`4a0ofxF)C6>&CMVxi2Boqxm_arwTue9u5 zkR<6r$;Q0*^T(OUjvr<>f+~GuEmdn>O;xSMU%`agH{L~iK0_zoJ)InGmy0Jcwj03O z{ZO>#fpAiXQ+iqVCF1mC89w(@K<8R#e#$d|CLk#if7l;EqwH8$xSXrtT91K<=FwSq zuue8~_yio1iU2tHZS!4{yiI&N`P?TlYS9Jo)Nnyz^8)J!C z$ZV}HM%|nzukvBzJ4nRZ>ke&XQNg_zvSfcJECSSOT#Z22@YV;GZq@j9@`^{7^nUBs zpy%v4F7>mLb?rKB=FI_sIBOHl?)_I3I>mAnf`sy##(f15z+a_it1aApIt2XrNzv#v zL=Z@8N>Xvc#D%uf6lHxrvmtfD+gA*QmkkKaNM7l&os7q9;&(B+}M+T=DsEtF}Ukjm$XY8d?vT|wp)XV$X zC&|Q~iH&rV*I0N#c_bs?{NBRm*0=d0O{D-i%f{0WcM(pr(O00@Lq?JHQeMIzsMxEP zNlAzv`DcZ~U!i!%?lOX)DgiPk!l@MvC6JlOg!V9o$G|l@p^#IiG`74L&8CW3EtirK zDo-#)P2e~s&U={NbE}wa9S=AOPm)6t)#CS=$VFZkD)~ufG|!L233+`{;3G3XQfv9* zSk_Hl9^a{Vc(sfauUE8aAIaWb1hRQW14Fh!v`rg>p!&e9Y&NE4XIYfu7HGNd*?jBw z^qk&l^IfxINa}{SZW~XY(7MkG@i?nM>cU{1R?J5qZ_*8SC{EgfrWSb3)e>0?vgc-O zIJ9gaYIzGgV+3J0H$mmCI5$=BaqwnV*hy|y|2f27J`@SnQ=PnwXN~|%=s(~zV zVhPN2dnFJ-+8xDx`I%Iw>m9EeoQID4e!WzzECq?Y_G3~6CvsTe#EM9uOFLnjj_IE# zSS^HFx#~?9V4L-=f_hH&|hex=e_aGcKERzy?C4TBtGF*%z$1o8^Wl-soJWU2(p|xlWvC!+4A%c4?VTTR3y@IFQPA{G61!yNTb2Vfs& zviCABMK-(`tl3+-+GL79@=_KjwRZZZCIGN1Daou{NoF}Hs6jp}iX-o+%rIn8tv}a4 zfSZpF?LF=L=RZL!_jVi?ccP%?ZVLUc{wA7W%okqc<`ZH#bZi#zuww#YVO7yroRDAb zg~KcF7mC0qpz}8HNzmH7*U?FE{gp}Cjydx8e?s$5CJz4f=JVAe<}7|E>Son)arV6k40_DpSAG@iki4P-p$fwf$<3C5?9ihw=WHFUQ&!t;mH zqj$);T)JEM$Y27Ra){4{d7d@HV`NfWo!1BQ(eDFO&?mZaY$3xI`Ilr&<}P4Y+LNS{=5biBYJdhOBaB807qZ87Irl-vSh!$SWq_ zdzof1sFdk`VbgBo7kkm1qQkyCW0?~9;%eeG$$-*e(zJShDl06qJ~l(pEty`XU@$*P z_1EcT`EC@IqCC4RRl3Oc7;~bT_sy<&XVgg+4s6@zte&bvN}q-z`<;v4iev}vq4(ns zkTe5Y^wQ`6mwNMX_M%<4+vbFGw4$l%>PQ8Iu-|^um_8{i3Av{*s~34@*g=QcccK@WFdiHX#L zl6^_AEB;YD@!RMT@Ou~amW8WG=5>1sjS6-3!CB_F(V5sua&cQ|X4VRnAZ0sZPWr|X z+LTEA?9*tP;k}>sl3QG#cSx3wz9ii^ z#wA`+ymT**%8noHyI+Vq3V&{;z7Hx1{6K)Hi`EwQNkG$~8q2n9HP(*DLH^wQj6m03 zFwRwcE$&83Tong7=&NOoHjKM5q6c)E8q0I_wkD@mnreq61F_{$vfSV#O7W(>b^W=$S&Sd9uLg#Ee3C^O)p_t=Mp|Nh# zK{N&$pR%@Kft@l-PuxIS@c=8~(7iE0#V*y^8I;*nDgO03pLmliPZyh)J*^7U`nd)X z;9cYy1(GY1S$&9g8OBg{2-HU^o@4~W_MDo?+UiB%{h&|lvdX%=zBePeA#9t0)rM|q&yM)>tHIYrenci8@r+qG!^#IGwh82wUJ^z6O1L@_ z^r50SJKhe~HAs~v<=8R`WHh|{$U3N@=d<|6I1_1r4m8uwFtMr=lKk|lRZcaYH;mj; zVtz92C6DkR^KlS2Pw$N|zNf|Fy+=PdJ>m9-jbvbhTc(qic`a6u8d{y~bbTdROWnR7 zwslAReLs2%=E=xr2Drlk^rZz+X7ig_=r-mj3{IizQ?KaAT0QHXz`01o*?=aDC!Q4y zJ`p$X12gtshV1Z`BGB?`E%kMZlUi)uO^o%lbPRbulg@xx z(2#$wqiT)Wn0&Uuiw>H0TE?uP_bS3}LKm9jioIc^F%9<9^fK+z@ofEZ&DQFHb4?}R zvyv>8+?lLX?&g@Lmrg(OaWf#ggPFKC#KUyoHB=hp_Kb~^{J3&99uQRdxHYmm;9=kb zvo@^BfEimi%vsV4SX}nddmrRtJbN4r+I7ft1ASovg3|_UZDSZdD{qEF;B?1%%bxPD z38Z%BN)UfRnuAb&!SdH6?poenZ5 z8GSRBmD9suYni|q%@pc{}Q)kE4oYC zY7HKE0g;T{p3jjCv!)ICvXI^{-W!c=d4(HW>RSRv~t-sxk8Nir+H>Y zkJr}xjU9osduh`$sE@t_{~1NizK*%F+v*!jWH{w8u44^lH`iWw^&t=915J+)4B2(; z72t_>IcoV@!q}AdlVT?YU1?E+>$KTaY}Zk#pbT`@1Ig8!U&#*OdIW<^C4dK*dO=Z#~#`j4o(`ehvaS9cgGlh z);>sYmz%DGSXp)8!)z-MwB^NyJX!trQ#zqzR_rv{>ULZ&I}@u(%j|2=oAV9OI*3Qo zcDUYVY@qUPOK7Fgij%9np6-lO*S%rjdlSFDqte;##GGRZ(j2Pan%|Qnk%kR=QtG)h zd?N&MBC+0^WgY2r_;#Fr2=ePg=~GPy6Mab{;*l$A$ckf7Ju>iXEn#HST+kKe5XIq< zYF|?9w`8^>-h0ixFgO}UBo_d)6P#RGa~j6BX!}XiTOrIL>7i8p8wuHGDK@i; zVPol-1M!67E;6)_RgPbA_(Nj1^^FB<1qCi+kxK$u+a(|9R(WRlV*feOa$*t!)_6GK zZ)F*D0nSqmUS9gRDeKk1-8j$LgnP&2CZacpq2U2r-Cq!;5kz3Ne)r|hY{H6YB3HgQ zr>!IRMp*gC9-LBadUIw!?nT~wvdIdNN;7`xvRwb_I8qw?d|0?BYci2`yK;YIC@#J! zKKBP?cJ?j19sAeMtb!1aJ`2SfJi}~Ug%&v|bD^bZ{DRGy55ZH*6|$~t4GJP*!}S5% z!0tFmKqp}r*uXB6U5;hqAbEy<*fNhd>P1}lcu0!!a1bl6Oz$06hJtGeQw>6>2)!Yaywcg<4(ox(lv?Q}Y9f=f`~NW_7Ya`WAY9Sq5gU+m4M|BUy^3&XomZ z3?*P#ip!*1o@-uzM0&g71GL`Z!|?mb%rdl$%pqb_%7wwwHO|vd{MnOxVnMD;A9V`F z)OrSv%C#Mod;R?khVa-g3w}I0*7hv3swMT@NhKF|q_Z{K136HKU z#f%hMY1fOkcx%Q1f0wVX*6Q`dEQ1fiB-5yMUSQVF$64#?+t)4g$dy}Boitv0)EkWz zqwVlmDjMJLCtDGc*cWc)r;H|XdTlA+X6KHjl-5~C&(@c%!sa`55|i+M0Hc(ZaHH&_ zd@>A#`T2$F5oIr5qMF^J2c+jFD|xGY3B45zS}t_;te80XEDu{N3T?V~V+o_?mc z+Vx^g@z1NdZ0ifL?MV)pc}<718vS{%7@Zh6u9FR=Fz83Xx7_Z(9kWrWok+WEU*@-vt++5Z?n4c zm1f$5D^rTpWLXfSD4vsYAHDw}Z>;E5le?Rpv!^s~KSavjw_rv^Q&H@E0aMVyI-%#0 zk9Mv@6b+;wnKXXMM>&9dH5GF8g%ixE8&>-*_(9@_wOsbyK6*QLyhD6E+Gp3y1y&5V zl`Z65>gzSz=jANPXUr@(oY^9?3`f!=sw+8lmJEsWnh8j&(PdlTvBUi)o_f=y4a&np zDwfsCJ{CeXC44Tu*zBBU6tFV+D@fLO{hBv9=+{@35bEaqZApK7mEXOXiuCb@_)Mk6 zI9s9O{5YXIE#@X!9l`-3(W?J>giNKoHL)+N$HAXT=^avxP-?g@V~lXb?3Jb~C)Sw; zi92@1;g?4R%eP@lzL~?fO>49u)zd0EC}uYz4~K=MAidoDkGK5*SCIM?w8xVtk5c2R zM-I78V<7X4gt1!-eXI*BTn_I)!0Fw~A-4NcATe`>+5FvXxql>r#cGYElY2yQqXVs} z%xYjvd;En@PrCQ~9~3lWcKGVET1kO?C4m5!vd;~c<*%S^2ZHeX*o#sHhR01)X2vWL zX%(W~j5VX*2gAQySO_1VIy75{_>$7=nv4Qt*dCT$*xgKZ9p&DesD>DSaD5;T*8|RK z8^zN%Jxyz%2+y0v6EOM7Br#k=>DIM~gR@+{9Zdt+L8Sr0)DxzsH+_X;V?pGxxogfP z>;veIvkn=bp*yie#c+a^$h5Recld_T!xXdi<05&JIQo|Gc`%x~M<|`=!Z&D&mh)pj zpfGH->j>ygarHC+*w|f>_6#hGo0m>XgaN<`pQ$E)NaHl5(+6%|PSwKK>HeH=l>OOJ zslBtqd`@E8QU>SI_*Sc9GTjatDxG-VK=7+!6{i-U`44qx4XjcD66?BzGtBB}$Ryiu zq-2QGo8qQ~q)poLvUh^`H4hediZ#D%Kc1NwILRd#wGD~|T0MptRjNeZ!H}s1CA~PG zNYD{4dq zyFx}|koQ$at;#UHg>@z~_`utWSOFOr4qo8)8A2e)77`Bn?g_Rz$;5N?DXKe2h@U!X zT3E#r^A!Ye`gJ#9t!zwv1ZJGQL!iUF(tQpQ?^^kLg&XC~QmN^KD44(UZoM>SNH+{n zO{4YDzlF69ENj z*c%8*)LCKxtXWUHh?)W#-1cdDJ@_o!_eI97&So$SMg9JU`FtPqfHsNxt$5;A+&3*A{7QTDVch%3I1Qd%fWndw(3QdhLJq({@be1V8+b3*NE*WnNtRp{ODj~wOOU109Fmhngr&H`0e zUO#Q(*-$K)f*4iX^TesntHmcOF&eZFgI4R_WDgYaXqA6q^~SZ&Ew!J1l_PEvO=489 z)m=2|AP*l|t!zU7Bd7n$jlco^qs(P}?$i}qp44^U(l5aBDUkk1(gOMJC2ul$SdwDB zj)``G_MtgEiL5_iGBtS+J_xN$yF{Jsj*>+Qm2f(o zN}v5{H|BHWW#^=Y`N-x^Q4^VruUX^Hc_bkTR3-2BGh;%L`(T-O0=)a0AoW=JmY^qI zG;qro-o(};v@%w#wQ0~;UJWwwcd`ZKYy-cTBI&vaK6$h}FEXCCM?C9AC$9b|J0KL; zek~G^qp6fa_W~{Idk~{MOKe1Y7+Xx!c>+rDK;Zt>J1Rpz$xO9F=AoymjrUPvQamzO zTYF5AR073+)YYdyEq+P3%vK%jA=*5#Jko!I1fl!Tw`?TdTG>9fB}NwKjnk5p#I?$( zq)HRNrr3Y!{j@bYsZ4zi54LM=l3C+S}SzH1O6pUsip7&ICll zV0~kBVO-<4g%6zg3tYYVYX`+Bef6QE_DFQ&Ho1v{^5Co*z(68Nwe`V|NGSLURR42pLUkI^=N z+!E-~8({Q9;~t@KI4N@wO@RU2+NGer3|OV4S^F5gLn3Jnp)>~mFz@GlbNULp{iEQ7 zd`IxhW>wG{{|X!9dn6pqzi{LKn?rmAQwP7&uMF43>*Pqe@P9q+KTo(h`i*2Mbc}Mw zDRtMM+8%uEv)w<5pbK;p>o0k8_H+C<(krt0TVu<(N` zd_s~uP?I+l{-@wyI2FENcT45$kB*G5TL|YItzv++R>@fH;mb>sfr49sm#qT(t`A%7 z@Fvh@n?K-YwYMeMarMd{{rF#!GZnD1&<1~^MXMc?`7)xHH(7@HpJoz^Rn-WJ?HJ&2 zTZ^y75YXPVMx15Or+we}L=qTHqh>Bj#-dJ4%uk)R>Y51frIcc{-P)L)ied^*nE#a5 zfZCF;uhD9oy5yU63;FWmoIc*vU0uZ@>05n#!P$QAA-7TlF-Eyt#2J=2MKb z%Y4F+(@yduD`E0WDqiASD2<#IzH$o3$h+pnm>yaGFf_xwGb7uN)b@@u-qOtkEpI`S z8h>SLBE5_zN*=#4<&U#sI0z;|6WlBkRr?j9=D>9B0vnXIiF)cE$px!%6G&15>QpPq z_vT+Il~A(U&Jfyic+6zjcDMmMiz~8&Y_X@$4upwTPDhx~@uX^D*8+fQCCo-bKDlv7E z+`OcbXoiuv)-EXh+KFF54fu-GfE~C^5CuF+fs&_;l|cuYdMe6V@UT_{f^PEnqhtAd zWp=V+=X4^K^yW`J*jQUkqR~0{>-lxw2N1g^Iu1jOYF(A3i$3Du*r|}rr)-(SoeV3P z!Ep`-OIxNX&ZWeYH%rr+p7QJYDTCj%M4~T><7ss3LVHDx`e1L>a0xr!_1u?ch|?GP zWk>d;Jfg8;2cKD$y*%@uhkqUZSDf}gv!qhBaj?N^kbR-CkXlCy4UMlb#!D-U~R|XA;SGnHaX*Wy+Sj)fvEMF{E)E6UsSBSM6ltVVgBOlKC({ zqGT*mky`R?YEk+f(14q&*ELA4!v1p{WTcbQ$X?PeSe82}=O~d_aHA<6`lSS7O_Fq2 z$9DH8Rz>1CN5Vrf8BIWC+HdzkD={{`quDA;dGgSPM;45{w;?{lP`SsEa6t`@Eiev^ zbHn_QxaDM%i1q?eqz1CkweItjpUk*>OM8?eA5E^!M9P~n_q{y8zPQD5PIc}b4DlYS zR<>v%;s!BIq>Rm18u~Bny#-X9U6(Eh2}yuJ5wgf;$ur!QBci z+_fOVt$^T8fVs;$Yu@sApcQ}zo{Juq(MlxtSE#n4y!C0x|CC2U!{Lno5ZnhgP^_H&; z9^(U?&C2>|q-~wa1u%;tG^e!b^w`7Cp%!Z4+6=r9B{rj@q0euoR_msi-dJxUJz-I# zmfvps^po{5H#_SkEa`NF*@X@>Ze30?r1pY{0!qnKETCC%a?!P;u`Dxy78ddfjZEsj zS@5BNt0E+~r9D1O2H)Ltl%6{(&7mHjzo6Dj2R2KZIOHrSu3)U zRklL|Rnc-z-Mue5YP{3i##IpT;hAM%tu3`f{R))!ZRVBM=s``OGyV*k!qJXrlDG%J z05CGdym^fE%C|MuCQ%nO^;Pn9BktzhXeZsg$GVj0Ow<^(NbdP567 zp$h%v&A_Y)25E&7SE^~p>e2{xp6+A&INiutUYwqUiK!s+DI&DWaNo(E)XHr)UKz`w zQN?^8ZGc0s{s!0#3Dq6Eg_w7`c-t>l+7L_Vk&(*3xz)>MdT8AYCe7cJ=UQVCXVm}uqkhk4zGHs`ccLg+< zNodpWwPkOU$I>uUVmV?-=>hai%e2==fUbn*w()98jNMGPpXE>?ytHuU5S98Rd#npTb5t z)~8|$LA2ac6;lh%tG2B~4C@PzX;xjyIQ1i#`rMq;%-XHrmFh8e9BLQ|B`ncXhtDhM zE19(-7p8HTHP>2ZzkS$KP4vuw?bGwiAAC$3fB;2^3B=+x>u^ybD<@l_xG)LDFdH?0 zdY>sSv^_dfOKyg3$DxePl*b~X-h6;#6yrc$$F*%MczxnLY%6IqM7}zJLg9gne=rp8 zSN(rLl13&#s%qGJ!HOI(^q#RnL{n4OkVHnEz0cDJN9v;=4{D38+P3vH0IPtiwvv7} zt#i<*tLnZPBnHbi-89`oKrcW;dRc*SFlvr_Il`?e+O#7b z!0gwD7_TsNOPCSgTRSj6)Ku;#GKBn+WW0n0;4Zkrtj@-zV%rn-Y-757*3C4F{S7El zA6hgTQ-o+Sb;KWnG%H<;y|fp@sVYSuDL4G{%Q1-^?c;DJ7Ov0FJ?1H3S1>zRa#zRp zi=AV9hnRz32uII;Ar#EL$rN=s-uAz#J|0@gIoHBQn5DN5d^32)cqWe);J7)pLCQ^Y zI9$EgM5BX`9z$ggl^#8h0*ydVB3KK%;}~LSfQUCWQ~TCEaRsq2WWh?RvmD0BL7E>U ziFbc`d}xv@bCaCUh+&tIgJj_2QE_|&8}8{Kk6u21BbsHC`i zVTGgRQCfuW@k&!wQsljYRdMrjQQQ(eJVWn6Ek!>yk0IKY-EfXhi$-gFYB0K7{feCn z$#;NrCmxUJ@En@bqWxHri1aEUW#`lZn2ldk-R|{M71AQ^_Z3~6(;m;Iy%AO(Y zN3u3{0p(jzsO&=>_r>_*&~({TDejYfqsE-MFnEf>vYL}?VDGWPmoZHei@3&z0d`w6 zOjh#Tq_xcG;77Kys~GvbSuF%I=$h^l6R!xdvZmKz9#Jz?gEI>iQ&i1g4FPRxuc-DU z?mtqf?U8Y&R^i(ndm6ugCuJ-4y+Gw4ZDI`^8FO!<(;7Ig!$H_`}GrQQ=j;x zLzv2YTT?A}BsfI6z9@(3F)!@OxZlOYPdt1h94B-^GplO2fb!|y!T_VA8yzqnK;8WpE<6R$Ktn`OYp$9$m`nI+hwDaK}ta@_A;x6ns>f>(k3&B$O9?*QZj^*#p{tIEu-%SGg3n6c)0M)}g+i7K|X!j72 zQ-y`6Kspo|RZ@Q7EEYh|mCs`zQNUupRCjlQ+@?fT_arxybOaClktgrn{PY%ZV^cqH z#$u%M*-t!?iKUa3Seg&*e|$8IAJ+6*5-OhJZEuFaL@#1DB!3~iE>mXcQ)h4mCT)Op z+EC3~n+L1yrO-&|xDG+Eg)I*{SVEqw;1Bw)b3c3k9UF;ow*@yOSUYbw+Z_Pn6}Cw zQo?cH5E#d#T17B}LM0^#$(vJ86+~oHK&HQ~Jbahhw=P)FTPHP|SqVE)#a zGqm|cWMXM5!uf-@DCT_WX4K|-oNRApcJZbjG;8kz2{%xY|34V%^9BsagXQ~{W>pL5 z4yr`MQEhS|_UghmvXWcoT7(x9hG$>KBF*jQ*`2_WmQB)qQ%Unm3LR=9r7G&SmeEwP z!=}~Jj-6EiQp5+&|1k;1vIt044AA$3;RuKuaZ zA1Txbk|BTJOzr=tfT#IfFNtb$Lp%a~z(&~0gSBo5JbJ`Tg{V`Qw)o+T8kGQQ^$K4l zbucC4oV0Q>ZD~n~s{Ty7LJp6G&POKggg3GJH{k{s!-1a+p9Q#K-D8}zPA{?Ooc^!B z2T_+n#+{ROw|;-CIhFnX+Ab?=-G~2f-PiHJ8p?Xs^*(H^B4P{f;amLC?6M*Ucktb! zoc}cY$+nt&>-OU-e{Ms4=Ou2d#Pn(_+zUCoqO;^ncyt9*4s+;r=JFqYoqXHL6wW8B zfDXJUx{l4KR*iS9qmKW1ZKJCf~K*134 zNhIr(!)u0-So|OsT?UK%7nH|}e%#c73jGET25)hnZo zineq){+Lg}#hBJ)IF__eqW~7kE*QeD0wEiHhPTH|a1Hx?%Q zC=%WqG6f@})|oX69P-e^DC8Q4_KfH=@$9ILCm{7#!^H+Qhqf#A=H)iSt^}!nSfvw2 z5`Dp-d<-#w){QWUU~Qy}ziSSsSsLw!=mZl8Z%8cutC?jJVKkYzcTv^PrFm`=*dIl= z4G+5FHuCqYDo!pl6_e*1?LE$6Cy--wOC?N9N=c7%#&Abq^ty0&ET7c%86cnCB&{BQ zS|A=Hw13|pT(ukX%2CJg1g~>J{Ca80hsVq|2Ej4N%CWn$?baO`(|sj<4MM?0916_y z8TrBPd+Uq~_9rRSqRQUta&5G`6yT(`7dDmSNMz`bKYxiR;{WBv`~0vA zURg~wZG#{&!k_`{Re(l0*vI$VG+*~P;#zeSn_3lP%#ao7lx$rNE8C@*W{Vg5CcqpC z*v>8R7@}&Lk;cK`Dk7L zWmH3Ykp%?br54TrEE2(WaDpxd6GR<=?|PJSTdO#Ga7I(bp%t30sYt3Y#;`m9Sy-k|G5E{FlzUkgwd69|(M$b*Wg(SA*s-;fs4u6c5 z@GL~O@8vovRVq%ZPEucC5+=eV-v(~*S%Dstm;$BbWWQN(f%&y^?Mc(=V?|bzXsN_K zX=dL0nq=D+DngS|5R20D2e7?2cM&bIs)f`mf+L9%#X*|K$uIxNclRMy1gD?HcB z48qJy1;?d;AXl21QZC_ro0pHUZYgvtDPrTyooXS7ikfdUUk>6@Daqv82KqWg@P_gup@G{UU$XK!!Fn|s{P<&nd=)8iifpp&Lh^MWc8$$8t8 zNE6)v`s4y)R(IPh?sw!L*RH9`Ym`}BC8YetAShIbbwUrH;>80=U>iw!KcS*|!Z!@B z-Bv7>86doNOs1E#_mPWSLYa>R(2d$WxspfyN?hN3BqR;R{VCAR6s`9f`e{GL8CNio zSSiX;lsx_HWb!JRJeYj&vP3wZRp3_CIhEpF_)Issh{6n3oGF9!_ug2|1;<48w>PZ- zE|S;Hhh;daY|ouhUuSG+Nf!e+mnJg!xxmlxDH5*35$H>V4kEi?IS*T4=mQxoA@lw^ zmg}U1`G`mL&C*G1pNCLN*BtgUZhUWz6;`mf?7c~3*h=b(73R*k?vKOg#E^A-%4f91 z_*8Ik%Z6YF=_Rl{{(Yp<)EgeE)IuSN=0b*HXI*Ffs?U{f*zM@U&1qrGPb5lNxTzoH z;17{lf8ITnSeK5{L~BYBBf@u-xrsstG#gYlq0J&`1FoPX>eIzAcnngDiC1))l;%lc zP*NI&g|HWOVpGmV*dFPAN|UQPv%<7CpmpfN83v6y>BGIgN25*{+9N~Kjm)bx8Q3$f zGDk{srZ!^5E}Z!IwYFoePi>2pL|qe*E&w8hu1S@mb%QZOx=1dk6IA!fFD?}RDdqBE z>GeL4e_2Wom;XJoQu=T60B+xbVIdjeP0X@R7(eLZ@?;4#xiYXPte2;h>liA7>a8W% zsQguEo4)BNQHi^kQ#-EdVACWCi@kdm;L5-#v_M5Ksn|gtKyRB~C!Ut+E#6 zF7mun%ha)OOOX>?;?6-@wDbGyd%5JjVd-!@=&iZZx&&tLp#DG9zRUC5`Gs)x)eXz$ z7s6+Z+TYfh?xX)@odxC``+GcHW1b-VtKCJ{znOv2gVEmxgWm6+dT=z=QDA#pJfSt2 zspoh0(>Ub*g@86+t(fTLR@PKCw$RCR!`{igbEo083X~!1M29Z?LZCF(IKFG)aRI}S zG`C{l9x2ynuCIyypgE6{zhGnWTdbfok;GNV6q9njHxsDoFhMnVpJv(e z(t20}BSJcU@kJU2hyg8xrTMdOadCDn3eQ@T4&&;ZO|%a(ev<(hSBSX0CMj?W>Fu(_ z%dOXoi|Xv$dQTMVkNT}Nua|W?MqLI6HmT9K16_onxIBe3uFv@wf}#H{ylc#6*P~+c?U^fbHreo2L`9pUi}Rl6QnDqjS;{SW zY?ExcI+BSv393V*SHm<#Gzp!CYmYa-tWkhk~6}DZnr~b3}Rbp`dm3j3$r1q(^uxm;yIyU?vmS! z1qCv)hsQ#ANWvJ@on4Ayuuy>RID7s=h&_?m<<}6t(tvs9(h3hZVOvH&qPlZCVo z;ZxiP-tm3iLB5_k{;?c%?D%si`<4Hm5DQPjPiQu!ceMZ38=R~jU8G+K33{KuzF5)u zyV-hNZrio3WH#^L29&Gw**;WJq`tDwoSfj{zKhlQv9ydRDymqwD> z?K`C5>@~~Dv2}TRNV-u~!6a!!L5IbQ8s8b4qYVe>ouMU1pvZ`{d#j(sx z1n&bb>(Txx^NyvC&^{hJen<4=r`skHkd);U9c5H!{I;%i7Fdz;)W%ugK0Q^c{JjhP zE2xMYctHg(1z7y-<8XYBmG&@b(DbqtrL%+^Jfz;V*-Ihx(o02DIhG?mF8(gYBNkOosINVyt;Y_zZ2y zxuY;a z>Ze2+5OIm`iqt+d$7l1o8*_?xVxF9c+i~KtdmTP+H+eUGA{#&iZ}4hBxG%<}68~*6 z4L|VSUi#>a3_g*~jD?h2{6sGX{#bUgtKZ{q3_j>uQEwC^_g4hKKonJPHY)#x>-S&b zfx|WIs(<0SDi6=8=wDoSH>br})wmT@mtM+CJk&~yz7z8szkZ9|VtTh4f3i*TE-`N~ z4=MOKpy2-Ve(|075ZCQn3l<$zHbUR>bQGI;4T@&%%`mo8bGceoAU^FzMMVOz+1?Nq z@i|SA-*}X152ri!^yBT|oo4zp6@$J5&l0N%s>;yQ&46NBBZ^u1r zd5OD-1z@VZDUyj%{}b`s3061B$L8LDr&!?Iz!N94qLCS1s&>?NiY4x&A?bP=SSLO& z+9HRuW)WLew+fCt5a^gE#O-?#kF-3j9`J%j!|On&-oQjpz53zaf2(u{(djJ`e-LeI9V(5ZJY zx$uLYK}|HTC?;vLzqNRHp+lm3dTHCPB)K*ZUR_F+&`#C!vpF^3kA0o+Vg3`k5D6ak z+{XlzYU|qYBgm(QdPv@{h5S7-ELT{$x+t*p`>TJJF{PBsz<{ zVmN4Ae`^M!>jy_{Z>L_}Y3ZbeX}ZdCF1D4S3Qj3Z5$5-M45~ zO-+}e8&yDAZxW25ks>QI*?#)u^JU_*h=N5G=eMBesXUsH zvKT_r48MzYTvg=M_YKA&ls};6ng(Cp!&fCzsb`#36~%uc(5iKqIQVbCz9RF23h9+M zK^JtRu?_|hGQe!U{ayMdqh;wBdFn_617^EDC8yTu4E2|HGv;s)To+a#i$iQd&*+7MK1N@mQ?Gi;FLoeUV+* zYYo*GmB?b?^Gf+$DEhyD3wxfB!;LL5aj)|!k6kC56W=^A+UD#UZ# zOsoBheY1g;47IJtX{(!-pg#imipAdoysUbs#m*QM*_MuHf|m*b+je*?qof=C z(o>`nIcIc%G22givD&(UjnOTgEyZE>Tt>z6A*Org1OeKum1rIaNwY^fvMR}i8b~6u z{kzZ><+F}=N}Z%FCGV5BZ;YDM<*K^f!G0CdX$a&wkcs1*Qo60N% zK5>#6(#`G-eY`>aNxDN3{G%UB5oxcCAb@2AC2l@_?2mb>#>GF+mYZ?+AfmXswtfgg? zMM%xZL-s_1ghh|~8*L}!4fJl}D#LB|Qhfi~wZ_BX+m2ypD~cAq%dMNdaO4*we=Hfh zf0d5)mjEf`2;UJSVp z3Fl!cJ*@72*=+WlaNA&+P+M^rtra(a49|C8J#X7*gkraEos4$EUf1@^qJE56H?5wR zufW5o2v~CoK?UmJ>WX_v>Uy^-l0+w1KMrz3SvUL4*F=8E8AoiInuf4A5SAbw$22~H z4Y*7?Uq%Y&=2Zv>5<}PA>D{d53y%rdpf-w zJ{dVtwr#<{t{pr{9DM;A$sNcRm64-XQr()~ptXo)0u~L-nL>K^`Nv z&+_M|e4DlJk)t`x@bLS#BUJ<0-_~eRRRl)Fh|dNZ=ph{F_GrGPp;+qB$G z`0{<@{H)nXqt^#Pv2cXv6Zzp&Q7vZNWUnKhi{kDDx$^t$kY3zG*H+4Q2Y4ai1FQd0 zk^bjv#dTS@hvY#c{2{i#ohlEC?c;;!{N+^n?_2)o-WY2-L#y4mZXcQk+_NE&QlfIv zr3Jt%%}cf*S}EGD5r-?`$VzWbWU!OzTvk$!^+**i&Sp_)4H)fsW}-0)YHt{l_n7c5 z`uY$aHdDuk`Txu3|FAk=Pf#dgv15sagfS*5^l&6``9IT(h>1R1;4?b9=;TUd%x&{l zG}iIHjLz@yw)?rz&fVUA#)JG{3Y?|aD*C&)4IMnv$|3E$;uvx^tlSS9_xS1x|Faf^-b zr2nS3R+E1<`}3%iHtV9JQUBt|tX1OYDqd68)jy%M$ekFY$vBQ4=>{1Lv`OgZyeCM8k>XS~>xE(yX*zIv zO4-f%*z*ZB&LLyoUkEkA_h0-^4H3iHc2bQ6EI8)TE;dlm0qkZd*|X9cvDnjGIzTW; z4F`kYzf9Ibb^IG;kGQNu16I*zd^iHiLjF^Fh#LeL$x+w}t;jUzURK5rY{ZPD*L~|_ zlJ|4Cwz6NRG>I^`mVUxE`wpU=rB27IXl~=JKRKp7slA;vrz_o;sp4du&DNZ$IT{;z z{N&E(BW{X@dT!X7l9`p6GLK!hin_Xrz*2{+xcs}L+ntv=ynZM7AFK6PjE_d==&N-i zi6mGn-UEGGaR)0rW;$HJXI8jC1DpTPGDjm|i0$CeZw(8b<%1DuOZsk5Yb*QNvVbzl1mRiI{pG(n_o+zu?Y`zsZt7=)vcXH~5C8QiN_9gf2 zI%N$N4**fecSJd1908oXxiBF7K71R3qrluB^bHcopdl%0UpA1C*gQ3R5GWEmy>#R@ zf*Hg)|EUVb7qcuX$ClxeAnF@<_Gb-Wl(iX&9Hws{R>%(Y10(kiJWY5RzbCzBuVZ1b z{v$aBKib5VB$@>Mca^NJmFR*=2qI-xSL%6Wvbx`hI})amu>t6SX*!y4n<6Og8c%tf zX}x{6{36ob*@?b+IGON-LP60&H>8%Nu_9(BKqXP>D`@nhU8PPnG{_G}N0<5{7Aupy~3 zDo1uIb|(2)qmN5B68<)rsjNTL1LIiv8Em1B+6U^wiHx7WN0RnL4dYh}keB)8XQTx~ zuy-PNv)}R%qTYgaBI|#p$Ib_LX1;T62`8)shzEvRlSq@ zuu|jlfiuhsJL#wu=Rp&Xh`62}PULG=LfBf_Slfe#rL>;jJAi)Y3YklA6hf;dO8LwD64_*IgsgxyCZYOB(}Iq*cJ25v#Qs={UYdY$}t4UOc#Z zv9FWohXzuJ%nx08HYwOPLZI756~Zx`DVfi~wY>>WBO`fAg*ej46DheH+_046QZ+I1 zsE8p~s}z!gi@mh2mf_*7iTf{Tuha*{Dv{#~IA2RP$C|h(Kz*tD55j6k)ZZSk@KzZH zAx~XzX7EeLr`?_=4a@2?TY@-o@gi()_Dt9bK_sBp3%-eGo9?vR_Z}us^lIPO`(r|> z^I}+fwM0%vpD{5rFmJdC7|Tj`FJoag=ej$~axZWbG?GHuhwvqyBol)HvL^H%C4{Q~ z-g(db!C`~b1kf;-;I2udeNMVsc{;{DeHb+~kOWEIouM`Yt7e!jP{;dgdh;x8$+9i< zk5Vu&5h>If6)n_RHgJ7W-lH$*)q} zP`N9TQ@rYM*@s}aC^PZk4-;lb;bPzTBkH}rCn<+k-KFUdYJKQMuQ16d4UIG9^4f}e zjM9E)_dIBBRZXO6d7BNy-MgJ6@#9ov*MdUR`AG#pO#&O%gk?MrXOx6bxH5GCQ?*K2sBEl7^wy<(vD8!) z+0HcFsnwg?qPD?B!eP#v@x%@a(KllYEK}C+b{!T#%pFD1-ZblNzEW;%YPwxAF_gIZ?N`Jy3Q#$&-9)3GctC0L z=D6BDyDL0kK&(;A{rVoPkRr{I&BJz;K?`NMI~i#llOm-iwXRcIyU92c;SdXx&mEke`Xv<=SHWR}eVf|4l$7nL1uVfnPKzQz-kk=ZoX!mJtXOc^ zDyWDl>oFqRgi6`;$1fwt=yIwc3$PeSQCZ1jLjYg@K zf5D_o!*6GuB^U{sRAp1$OiKhZ*bP3GaT^RJ;0tQU*q{=TJvrp&T(*=Nr8_?KRgD*E ziJyF~{6?~4Ih>URrn}1Cy?h!%OlC4gMKp-?I{I}(@zN9_8=nz<8(viYCAQ1*5r}R+ zN)v)-baTcIvjq*;?p0(0>#bXDw!sSjJ}ViaMDiPb zEURvsa?H9_W(7IfIst8KyUK1p$s4rQ5UuN;Ik~vnb#>n!#n(UN5^|^SEco^;HkU>| zue|_rpQV*udo+-t1QJOZ@nEGznr`DMFqiQqZqg-|sk-+&02mPC&*~O&Q^^wsGG+_I zLP?hB&{tK$9pm#X}VViJdhvYX~qAqiZk4|{)cq`LH_o- z|8e;T`G4$4tm#~S+$fN1c{f3nOq>l{+Fy2iIxqc5w?$puEWJi18hNTD3jB;0m~F?kHe$k9}Phsx4lh$*;{qu1H(Vazu+1kMoiwNB;#%aK2DlJ(g_zi zlK3T&quHa5NML)Rt>$N6GioR19Ta99eC*IS7927N=X@>!?rmTHMDY)xto;L-bNnOf z@ttWC)aFg&A_DUFzlGZu694-$6X_NHaR;$s{re7^7EMJ21jcArh^IbI9-;-B{KFBBoPn zX-p5HUkG2$)nPxsqxTr(adCTPm_H8?J2_ZorT+Lccz9z0t@wFw6PQ)f$Ck<*Re$_Q{>GsGTNRO@ z)_te%-V&-q`Tqs$HGTmS)+`zT5~VW%n*8cW8+45&xK>w3O|5L-q2xK zyMC?FaD0kA2Ty(!*H}dky5BQr4;0(Q*dSYnI&VxFmE=Y=*?inAc1}=^R|b=LzNKf@ zK`Kp7Kg@Jc1mt2f(;nauGE!3)NRZl6hg1uRQg4bNfEjnO4+N29pif zD4=U4VMl@a3IFIsI8u62f-YKtnnZp@n*4okq((k(uQ){;Pop78!u}`l$ zG8@LJQ^RH42|tyg3K~?*VJ6LJv(0_vGO2)^j;!5a8L~w=2$4<6TP|Z}AVzq1eQ7ig3(v41+=~S1j9RypwUQ{BiG`k5 zdW2Nc>-#nnjZVK}(5>W4l&UYZBaIJ2eI|ne1}aH)*9c=V$=iY824fXqNbS9!s64lc zECywT^--&;IBPtyCBI=g)5ZA|-zt??3kuB=ozoiM2+f3X!P%xfeq8gsjYpa6uDY9BvE&y$#_!tjTX11$rTd$p4L z?5c$@n&fL}Vi2wR)&+iUT4<{Evf+rzlj4eowLROVII2WMnM7?YE%VSTe$oCa3yH+h<%11TM(KOo5c1y0Lk-vrt~SR|x~}qLUwOH3 z=m(3Jj!Le)MKMmq=_3VP8Qa=6dzPx(#M2z@TAnKsU1 zHZ(175$Si-wV``GsN?MXVL3LwTsI=gaU?Tsyu|!&v)5~+KAtUIg4r+Cvt0PXCS zM)IaDo@e`~%i0SGLDZ6VOjQ?GqaY2w$+*>51twm}>tjWCu*=>jlIvy8`?_8wJ$7oQ z*GX9_5v{>}(}Ry(ZM)~Qzof~H53x?q>K*BPm_L&i<4a5_XPE+s`p;`Ila9pu@iAKF z#>R7WlPRb=dMr1sGo&?A4+{QkoC;%Y|lf%ISHGJ)@v6gu(N0_y6`s18hrSEGNRcTD7 zq7yC`*`{<{oRF`@7p=?eCX-NqCFj@0@u$&-|HK1t&bC-^kwt&O}Vr zwm3s2-afHhA~aEZolkRS*Hsc4g)IN1&m$nyeqTbtvmJahN(}zAt1J;aDUyZjc=ZHN zCEap|Y@wn|%&Nnzr~?$DVBLsSEKwF&3jLvHyS0Ejah|YHv+5?80}kpxj!u~^I^P*B zv&e!mosL?c)pr07;tU2D07AF!!gyrTj0?SftPYg}ilC2boveLjeSI%<_#Z<5VmBRMGkz4BLAhD&>7Dn~n3*gXhYc)L}0K zI&*v8a^rl!``$)Hs2CH@J9P7!hwuSUkx#dW?m<01l1z%6m`_4f6@ppR9H-is{^Cutg zC7e2pNw^DS({AN&`luO|45(9o96JCLMLOZC=2<*8j?;CGPY$+F=Z4_<3;CMK4}?ytyqEq&1XchOC506F@&>GYnXARIfG2??F|aY zax}xcgxd1PBPHyCcdnm~0xMeQrsNKMlJ|Eh^KDB>rXz?>SQiLODp$LZ;qW$QPRLSP7=I8*|Kak zg15p*Gv`$w#=XFWMS;!bB-ltn$7iCPlP|+*n9nV&FSg^xzFyW_i0b4bh3;Gd^>^JO z4E%wO7?&U_7zPmb1#grRpJuN|84o7ZLJAPu{ z5}F{ClGTnw+a^Zb(#bYZm?g4OYm}zkw6Rnz7>$-H!AiOfqoz!|@uOOnrg{E=EFSDj zzQjv68XS(J!PmbKi1dnL1Z$1?24{wb!y~@c73c6AE=Oe*%QLJ9)E2+Y!7zp653_(7 zJ0^oEpU@g}Xhh?jrIRF0P;ozYlC=>iXrP^>)-;o^+Iu7=mu42E+&jEQ+DzoM6eHm& zK-oa8)6iAWXmU|?oc}dLFD#dqw^-=KfHWqOcDGu*h6!yrSy-wBP*cUeC($VgA}YkR zN)?9n_ydL+D&sgi9d6B{h^CZ^oYnZqjub(yHNrCG=f+JN$RO7k%XCWItAa&H?Acvf zjQYsBZn5s++rd~X3ta3hOkD199@vT7K=V(v#kU26sYRYGL>A&v+b&g~)vDebmJYLu z!M@C#i+B3PkI_<@(&-uL2xd{f1kl$t1pfhrxH;vq8WXvhB~YE@3t zp@NEmdls9JX&@3(w5=%VR!jMmqJbEgNIs}&%5{6p>)QmAvlw;lIC7}S$B0Atj~rZv zbRU-NKe_3nKQ=`abcAd(10O$-f4S)&F8_Pv|BgFI-hkoCy2`#KE)`Qc`aIEa^n=po z1a&eSop)5`S_kJ!K4%OKkss|f8Jz;(l|YpGDUvl66l$zQCX|~h{t=pCpjW3~*+BFR zuld6R8TdQ2{cssXKzS(ZA_s?NP=o4t4EzPnz_|Y9@M`p5pc%qwG9+tPuHeuNtNc@7 zI5dO84dar?PxHCSoqqvV^*H^$jQIU@th#Vq>c^&MH0hdqJfBoz_tnqP zyh|il)$#K+*QJIzr!A=bAb$#zZn^dWM9=Y-uk4_zrD3+LysXj{j-)WBXAL*4ep^>M zf&W_Dllzr;@tYwCM;|hO=?TiR-&qIlj?8D!ANZP$uvLg-4u79i6^e~5Q+xM9y|1VK zvy-r5OS#qnPPkXYmL#^(%L+V>CD4-eUdlRXjpH1=r@+fA_zU5~#yE#K&;E{8bw2af z=sg{IxLWNMK^hwo$zEw~RNSNeU?*4oH__2G!2ZB9AJQzw=RH<9{A`@dd@_Yo>q&$K zo8)~0#H5V9Oa z^N~1w-JUBq^d-(oJ|_RtojC>N(fAY2gbW=MGq*y)wE9c)`-NL+F;BPqqt58dj#VdF zg_QOCf=_w*wD0=0aSkPUJmE{%88t7T7E6F5 zyrw|62|F2X){aTn?%AL!V_Hog%&kPXa>XV#VGl^pWKcP+Cvy-mihg+9XVYn-11jZ_ zTF|qZys?>9B70~9s`ML7;cwf3zWbeN{kLsE|LRWhw{1ZGa-%);m;cAN`44nZjtH^9 zJE`Tpw1cgTJu5fs2q#rkiK4(Q60QwCz#;^jP$_LA4O>v=%{(I#Xa%%VK1qb&#Mx6` z3W-V8fdumyUxSAQg;F7FZw==KWW!Tk|&nIO8XacVxtR~NA zHl8XB#mOd;Y^OmYW!EvCII`HD8rA1awocJmW@slY@$T@;foH{P)0Ns5uho3#NYx^* z=T#th|AW2vj%upg`h~F<1f@$?dRKZEmEH{y0-*_^g(@AXD$<(?gc6F0(i`w~PAz{kK~$vWna!{kMV!GKC{Er$^%{S8l{iZKmCfO7U_?WYA}zSk}LgQz5vY zOhy5&5)EZ$oV>r+Q$+8mXy+qs{7_~m{<=VD3p!j*=xjyaAIKY3l9&PnxF|C9svok( zsJefM@9dA|Q%dj5h~c$LSpHt>n)AT$ZeooGGrQ@t5w53ZO5CcEHSE00EKO%{9d_#x z$Ngy9zBkgO*il&ml1WV=AF>$z{%75r=eLY{Z=f!A6PU3)U6MMu@;sqK|8@uLJ2L3F z)rG2#NA=iIojz%lb&H7lR$ATtq+xEB=i;*;+CzK2@`;=|dJ&muuh$#Uwn=FLvGo3m zq6i)jU2BHK11)jxN5!9PKK1|}Jp9rcbSIRr>DE$QBrtSwG3O)PJ-JEGG$mKU1RBX> zPi_pgUn+x-2B4AgEO}C&lZSR3c!biGgKQxKiVjv;ES;PY+ER-G3=1u;56N+m&kN38O|v6D0^%VV zBhv${+X{1o!Z6k1CwKhg{WhFE$9b2P{WHmT#t$9n<5Nwwbf0VfMn(;} z=&q=+cyM90w+4Rw*2na?TIzJz_+1TGnP#jeyv-H!7fvG=uM>Et#Tz$AMSCgrdi_d| zSO0DNJzlf$Thda3nJ7Mw1#^G z1n5KbE|FqGk@efL&+Es<=Dh8g%A_hvZVvdby+01;@)t-ojARqDmPM*Pk1)E{@1hwG z*KOfUkB4xF>Fv^cN?-{9Yi_W$23{h(uFPwFf(nUx#XgaGywzKM|&l~b} zv7DoG_ZYUTGF6kG2y{9sS`3})c2h5|Poh>H{M$&Q+STMxtvjyvy+4X0@GE-wnySqp81- zi9DXWkMSq(#B~^``c$Xu&iKcCjyk!lRCh;ab|7WZvrXj7B^e{1;#s*Y_MGY|;lR-U zzgSHz^A`~F@35Z#5-j?qfuH&7#|$za;xBr$oWp>o}DzDB4DcsormQo;AM5$aeXM#NiH%*IL! z7IyxN-_JP5q~Xl3X-F0tUnvV_xwGuT$SWW?;UM9MciS{E#AQ^?4|_D%>DS`EA@6(8 zGWA1b+zXvaMs`~ew+;U!S{5?xcw^;v$Zuqc+~|N;Pp4M^IE@)4E;NY=bP*pmP?dI^~6G{(R@`4e>kl_^;A-9 z*@$HQ@LqLA=^sqp#3$RN`Nj+Lqzo(VOxRxcccw?JUpc&CykTEYZmPQed9dh`(m#nV z*=*76RBF0@*5~kMKljJFbCLRjTc0A9UCt_HMFkH+ELh|$o{^DV5l;+FlTjMJ2>p=~ zBV*ma_-UE)=I$e=5K`_7?r;AN3I2_?=l)*_0)jti11CQU3Dj6wPfoR$d0?!Z8{9To zv(DBYx)$NnXUKG$>>|j0XkP8#(@93 zgoJ;O!ZnJ+@kW(HRsL(R+%T%;S`jtre*>$WB?9Q|?D7k>S9 zg5I<@aiN!IugfX@CYn9vb71ys~=qH4s{X!@s4cZ#nUDc@nbReY4VVjE97yfagqD2pBl+- zto*0Lf2j0-F>CzJ)@1p+08r8ajSJ*3&4EVKa+A7JdgHnFV+j_`-V1QXawn9b1~jGI z3~!**Vji#}3|*LQdPJ!=5`YioZp?Tb0sr zE^Le`&UR(Dw5tVzMtbkjXc#FQNxXSF&n%TVp>L`!>x}{Ni3F9;LYviPZgOCr3r|)W z)2gQQZ-b&)aKffMFf^Lw>v>mMajWz0DYh3S@9DbMKVSo;vDt#?qVi#GraTi4Z$$Dp zo;O))U)@ww23(vuduJOr!}Hl%*7%J_ z5ibJ|k5Q=sW57bM+(nfFi5hk-3Zo+`xkyM^=3wCM>2gWFW@~C?eHdN!o&2C@uNK6` z+Oiw0buyp(mB!X7IK~_hjkn6;D*|EcHrH{w&H(d$v5rO$y?xFMiv!1k@Yp)1t@pxy zSiBQ<;V9N+I{K?InyLhFfUpP2^HB+O2~B$m-d=8d(3V-%9gq9KM`!k~{F0Q4Wn=qG zPY!Lk?cQoui=JYwLC(U+k6WEGjRRG@dkj61nFGU7o4M^f6t4+ltx0MiKd`UN4Bm4) zA+b7EbzqgOBLj6cQuO#rQhpj-(^uBg4Z%dKMK z{*RIOQ}P(5Uf*$t-^eiN2B)$B&dg;+T!x(~)3Rb^^tYKr%9N%uPtJwLna7TN!dEbL z#+yitLF1rPljr8fB_(Ils^11)v<Ew6l@E@#UIptzjhyv)9vJgF0 z6*98Ve>15lO*0410{mucYnw}U;i}2r<6&RMM>R{3jjlR~u+);=?RC(4HF(?)Q)^US zT3lt>LB9z;%L@CAEJXVQJuggG#O(|hTkXcpZZ*)6X2TDbH+>Ipr;?dut4cJ~C!X9c z7PGf=vwu8@NNa`kHql1sV@a)2nW9VSvJY=a_tK4vU7|B|EvwBEd0nuJi@D~|T|)J4 zCpaeljog#@U=ap!dy-$7Pg{t^V9NJAU&ZaHb^pUJjyZN-X6hSEC!KX0m10fPQ}dVE z0emIxZo6d#F(z92958g!Wr#m)k%3J+-~rWUj)=y!9ygSC&O$ZHt~v&mZ}iPMt+o+Gtyo^(cUM-dQ6u1K#9U6F0rH9fKP8rscuuEpFOArJWPgjT$5* zvlq@IG7!tFW@OxE=JV1a5Eh59Gg(Njvcqy--fR?^vai{i!l_CN&tT_ND`xwAMhlEU zs@_QiUV-0iV} z(I%S?UN5b+Aad}XBnx!zWi#xo-oztWq3NKV*Gn(pYImjYTP9?b=<{UfPc6@ul(|hAuud77Z{3oT?5sXLHBj12Nmz}oqbxgkaOIOLSDl33%S~^jS=dSd7h>G9 z41Jk1ek^4Vh~O%@;cJ2`h+2%Wx*&~31m2vPwF#hX?B)z`ku}XtlR^kc=@xOQO32g{ z6SJtc-;b-KGAGPeAsg(!;~th@bPztbj=v$)@7vvCvzei5OZ%fvcw9HH2g*fbq-9RX z@_zSGyuEKy417}VGlKIIHZQUe7}b?Lf)c?#X9Y}O*&nGW*jMy_Y=1rKw|*p1RMYen zkwJ9YfR1W;Z=fp5L$tg{bstzI=}HXgFW_ohxXqMEyY7=6FIeJtu*nEBzGa+T(m0`# z9#Es&yXjlFeYh*rx|IQy?EUqdEE!dfwyXQr!^KJBj*Pn0S;V(D?NVG7E zIz9s^`C_5@v+?YgX!Vt9JG3`N>FA~Uf$#*+=gZu`8}J<-iBGS^UKf4~^B3A2TYf0om!zjHA$bnAs zh=$f@uIOgaYu)vxq)Ic9cV4IWyiNtwP8L;!CS@RJppeFUqwSm1Ef#bH$7YH3fQ0HQ zjv1L{#R!?ATedp-isE@;dfi)+FOsNOszTPbzR0SkT?g1Sygi~_4(DZ#?$Bw&2ns%S z-v#Z9wO-n6v_S8SY^3cWGw>LGB`(;yaMUFEh9;|jmSx%XG30oxpp#6VI?nl-GxsYl zQ;Q2LdG>G}L#hv`kF2ObHy5qJtgqg>ko=&UGB@~bK|&>W`mCT?b%ocW%@#mi;8Qvy zhqb?(q;)kgvfApG{zx#}5`y}k?Qji7RYB4}l&ie-;J^01_@k+bVI2)lv&v3xbd;tt zUCuXZcsDn@?~Q=44M~UyU2yGUD?wc@qp2)6yU8mq^~9&jFy$@v0+glE(80|yRag0W z%ahG6#tYHc(bOsCI_T}{OVops=FY_ob&n*(g=j$%c75)wHUd#7(Ki;@uRaJSfr$WxsvsN|3dz3;o{&qO@)b!;&nP~tTB)h+>_iolI z$F-SJJds`H!DH$~e6<3`)^S1z>z|(|0ag`vFy*WuMPB_NdLxuF3tzR&?e9@4^wHhe zry!hBn=wIlmg1OD+6EvXRt{~|2=&v4!W08grb9>ul|3ix=|r%1`gU8Gjfs?e_@}Hm z#P;MzmmXHr(m}ci-xM$>IW~DJDnZO6;C$bvOCK}=UTaaDT)5+r^w88NZ>C@M&P>%soA&} zGsE6;<`bA4kaNk{p3H}Coz_SIOJb_n_sM0SL~^^#Q9AX4*?S;x46bCUKDpDeZp4Vy z4dLS;9c>8BNTn+%MIMQYxgqVR3L zH;sALkv&%jZEh^GO5{<0l%BKYS=DbG^(RPm^4muti6T}Kj}}>S9D4` z+pGN}ks?XWJqcG}7dMyN`^ec2)#9{VP+QiC4ZL0ns>taT{f$f@%Zjw`x5-d-f%h{x zO?P3cG}UipdAHXa9^6ix8sP2wmi)BGPJ${q)2%D4z6O9LIM=~?Egn=fEj$w%(&VN) zD<6yPqD9cgr*NngWE4_ZVeDspR(az?wyJ=-%xTl_=!ifGfEv3kHxOvtXs9bOdRtcV zFPRBTXBu-<_b<5#XW0d#7Cxu*nH3W;pv`Kc-%h3uRg2-5;ATkl?Bw$+_ozC;<=-ewtsplV3<7?cw zY?(52a%i@aR~~3aa);q_1^nU|bmcztwSlef#{nLGR5rev4#0wYa2;{rdltPLGldy6 zzf+`=G8nl3d`dd^3&OV%hlpx9^~g_OzZf;{y|Cq>Eqlz><>L|(inXLj*IeolTjLqh zLfCmdL;H@NgPNYxdgLQ^U2a&rI5jF&@9x_4Es{bnBD#A4;J2rX`njga!w1 z6nCc!GYH*X9Zx%T>KrU)B%Q0t!4JZCf;HcA7- ztIIrFFnKX7di?}@F9Ye3)^xY=CN&hu+M(bQ*w|^p#6k$;;lEgJhX$k{BCISxJ{YZL ztYihe@4hf{rFg8|i?vSp`k_V1+|op2Px__IPZ#euBGyo~(;_kCtSFhkes=Kfp(`Ew zL3Y^XT?o(Z@cc|u$OA?Qld)vrGzPt2i!-z{0fX21PdgD>8%X)Vb+1hvW zBng-rGlTY5({|HozV%x4erza&b8ci-jJe{TIn*kpJDLH*QM5>K2(Nbb5jH?Y7F>=hBjk%(1Y2yaZZcHXP2Uv47D?& zej^)>v;JU71!Z$7lq*wpMVvvNv|n}#d^M_i`#Z)*wyAdOiHb`yARn_w@)TB^jvnxk z6qu|D&*&!C25k%>ll}1txQ4gzk|zXf45PhSNU8VykTHS+-bv|BV)f|+HODuzVyjqom-KZWlGr6{`b^S-=RnG{= z%xb>TH-rt50CgsrY{qC5B)d>|k5%MSS)HiK$rx5l1yS?si(pDGHt zde6kk2swu)q})YR?egwn?|I9fb$Jc&?xg(_V+H8i$KXSfd_RHi-|fH$=(0&oZB zY^dq!w$o8Vd--=LrQYg;G-6Um^l5z6E=QN%)k76B4ZWK3KwVj{^5KAZ(Mrw4B*%ih zGb0*}T9wsy8pC4?R?~m6Z@1wK-B=e9TNgSSCv zkp&a7x5|~dE!7&Mwlu`Db9>#p*X+z9wepz}vrGbMcZ!s{rLAz(4mtzkjNTTK4?>e@ zzYFbKkz#qdJuIUec2^u!4~51cI?GQr8FZc+_?aZ$wST!wzt5|X!N@~Q7`AXfx~wZU zA$MR|s{KsL@w-7X2APc-XGaSZf&Fh2@<^i=^74Gwd_{(Lek1!)K38oV zi>)v`+cP9K1(i#pgBBP8&%!$uuvZTd(*m~TAa|=_S!jEb{c6k}E~L76>ykuuz$YB& zry#LbKyM(`9aK>#LM#A@0D(S0M1)W4CPLVivioNQaLF*+Qg3w%-pz!`NeBkyWw zHJg3$KH!!|z{vA9rg)dxJg}Q-bBS_lEhb>rXr{&s>;^DKjOsON)^=@Z@fT)amPBnp zN3sWYu3P-T_{u?x>^%1t>h4)oS|yQ6NmJ_$mMy}-aUgiT0*x&CjjZiAvJ6x9_k527 z$!yI3zACYiLCmBLUl23OZ|DgikAS>}TFoxX52masW4M`1}?Y!wc#k-k+;!(%+0p7yOhwpz0yQn=gC5H|0x z#9{d*r?srh_SF%1Q@9@QxWg#-7t8pF|@O`WRr=5_6X%m&(dUj5yEY4DbQS- zKIus^)-!z5I2QLK50c1TVUusA{Ixg}#IcTXGh4aYnU{Q|hC@s<_w2NS^@)=UY1UfAXkC-Y_Ag(AEp&~l^2ugple!dN~O1I>i0W#-c~(`gJbor zU!$YTM;Ru*o{UCI0=vZSSe7pF!aNP!ma%|4ja(~6s}3BoJ~OW^YjbZnyiOTXR(5+} z<*NDvOV7qTV#H8d?9ZJfzS%2w?kk_J0Ob{J$c-K9NBwEJ=ldEyiB%)Ty9?XO(D`ym z6Pzr>z)kYA3bs$u)7-k&qg+fyt{qmvi>2Vja2eaGRUH@3Ea7_^-d`zXbfz&W^emM` zJd0e|kxVqnMULBdU94(cqLa3l8=z5)dr$fZ8Y2xcY|v&OW0gLo38Uy*${DMaK#da5wdr=3&tb z3Mpm;K}(o4r>ms-klLEjm$!W#LVBhX7|Ovtx?briPr*+9kor8NNqIC6bt3D4#&)d# zF!w&PqwOA0++N;8BI8w z_bW5WUC2wH`fTPY!dkimAE|y7opiZk3p0bTa0Q*hKc(J$2@s#nM@7!pxpkwEZZ0u8 z2G`M&fRRQMSE(`Y!k5mtc2j1YkdT*U`K#t=mT`Q!-2?{)5O0KM=XLHm+Xo)LC8{?Y zQ)5}N4E9_6-N0`r;Y(*9Yl&37j!TOODxLOzUkB($=nf47?Ut6j!u6(ko$xij0zx^y zq{e%rR#0&_{13Td&pE_3$EVdwOPJ8*%T{0Wg8WHoQGfGL{q^^sRn`7?0&n{_GK8^z zzGLXWxUK(-o!na}?(M?BE+i|Hm_SCZ^iMPG)_g_tG0a+!l#yBEJ)_xS>*mts=zzCF z%OzluytP1W`Ds-Si<%&_w`9swiOy-8W!d9S8x8%wdaA0EfpnFSYKt$DcrQ?JX-rh% z-j|fJZ>BG&X^2SK`H(&o`A6*PQ77MqRf9k zr~J!{@;bM@4MD_rAR>wA9aD|vXQn7-DNE2T9!}cd#7Qj}pTSx&Q!k`UN^y~D4Fq8+ zoa(Y=*c9qM@E1wmP1N>+r%;AKhQ8~H4H8ErjcH6$fRry7aFvQC$Tr;V5>c1C?Y@t% zd4AryJajJ>OV}@ws$Hb$d@71Wb1B%ESMq3@&Vp!zMH+3Xf@`mnPhV zIdTiRin(PN;IPEW`VC^KT&pVdOytrN3-*m&1<}FQ#79J0{5aXFX@587nJ*yb?1hDZ_x8W z<4?7n5@a`0<0sxGAAZj+Z-GWKqm}r#zbKl8jNTq16BC4QSWf_(KH)Dxtaut3bX7g_ z1xs0Rl?P@*s6xMc?yzu&z)YFlygdqde)a?fQ$=E5g zO<$+t(5GZKUunax<^egPy*CuF}_@QQeS zy2W#@umtO%Ek^{#td4H|@22U0(&DeGBE;^-KpQ7UpG&6P_HA{6K$?#n*Z7eA=U)S{IKSQ0-bYTWUPsYLT zzdpPdG7Ap%t+hl4Kb~Ee<|mR0aUj=W&zM@mS?X4~N4kdYnH|D^eiOlJ6p#GB)Vt}uGpfGbhKR??B(f4YYUzkJto{K|8|Pjz@d?Q5UO zE++rO?f%u%Oa8t0@CKoUnYO3gHUR66Kb(5}TnY&courh185BDd@()RsY&na4f7rxP zA}oAPWtBt!%BN$tSeYxHqz$Up45AyzcXLLwi#*kBrnGd+yun-!Uy4f=L|HghG_DcB zrGG6m!u+kqXyorTM*kXq5E&)we;MhW{&ybbHQRX{tmOqXd|Ohr48rw8bUo&T;dV@| z8u?3IewBi$(0Ma4l_RsuF6N^@R&Y)+wWlW+m+$ifU+V#5m!g_gwqMsC-F^Mdn$3XU zEw(c$x{0}0cu_sp6bGc=O5?XytD$$DDMbo4xFMZUT4z2;vc$6aY_91Vr!bSj3uWoK zvN5n!-MtqXra1o>)L&Myg&kwf>>BLSjwL%5wwlmVwAAiTt^U%T!=O)Hd;q_ zZ$362$(3Lpx_E!ro*VN7E`*t+h8%_RfSvTYczjy@%(yG&YS_^6Y#P|qTme`y@VaWYcsq))sw+s+s(>5q`8e8N!mldyG>^>ed5D)N{tjNOIKa2Nz z?i-w^YB&XWFG%8)vVEz1`Y@GiXY-=E;cPkU{oa}h)Vj$2k6<(te|KMWgqu7O8K+7& zjzC1Kb_E@D52)~EYLo()nNK%su2e$o*(NJ(-meSEH;J|39&zSovl67meIkrYmjP!wmzz<~Mxc2Y23Vo0Y9txg zik~dU-(3)TwfJn>=_BafVYr|`Mi*cjNvD_frI7YL)}(a}FSNey>cdyQNnXcfPYq`& zCu|-&P#=IsD93;e{)LpG-ts(8JWdVnS2B*2H85-i-x?^fG_MmDgQQ=IQ}fa72HEN} zs1BdZxTNPf$ajS4JWaz(EQL^iZsu2A_pPVSUJrN2%%&ogy#g*jdA%cdWjqAHG1ik_ z;^gBi#3N&t&l*?vxG$&aD>m|WsfnYbYSoa6vAbavT(3Yli^sw`B%`;mlHWOm;LiVf zLsi%?Ns36%9@Vi^#F#!&_>*tS(2Yi9CVt{rhc>qu%?EHo8KrFyqRWR}CP`~E#dUnT zZYvh=%?x6|sOM$r+?(kx9W?C1Mdn48Wr4|bcbbh)rqg^7Q$bjEedhKQ0Rv@0K|X)0Kuw6Ha?(<2L&sEY*7CqKu@h`}j0$@<3U2-A3FzV#y7z12pv$ znQHNXt(U8N6Rch!zKd$#(S{65_0s!xntY6np^ng`3SX4q7`$PVz*C@26NmL;xlR=~ z)w6M&%2m$FkW_euqc(ZR;PtSDXP3$To(*K;YwGw1lTeJ*2MFRW}A`4IB(`Wr}zNHrao?7dfZYz*l^rt=Ao5% zdACbWZL5LFGH&Nw)4Ux^lsjJ=jkyP1(WAm8zB*8o!gt}IF~rS+l!PVU{6b6pL?g3< zghqmPmna&58)+@sxFV{uNDL+R`P$BOwfQXi*MRde^w; zk{K~|>;^|}-4tK{BM*L+kxhyMpy*AOtqYpyXz6SJ~jKN)LTbch}4J;89{+OHQ z%kJcB63vp}6BlqpYd2L$GSzoutQM|lVSn|u*J9+Y4_~Usd>o4+g6#R3rl~Zuen<%^ zv)ptNS?D#e#(dT#7(Z}2920+7r5O(P;fV9{mEI>! zICm_IVtdnp{Jo+_-*o~8LvdfC&%4K_PmGsECnYW!U6S!*W3Ef-dsm=t|9L_chb7nq zR@7ZMD)H%0x6w-aL_fUIV@d|@)eiP(2-y0#;atII>YH(SgZ701LlUPwCZ2}RIX@HZ z0<%r=SkpG@O*DHZHEB#l>egIaQ@NU?9Dvhi)>C@67QEYH7wU;!N6}0&J9K`MXTPXl z9N%h-!D)tfS;|6!`59lUtI;pey;XoFEToN=boVP=Z$l}6256Q_rA(DoD@Uj@VL|G4 zaqR;07>H{hC7Ld|g>}YP#{fjgg7|sd@4qYjurIBA(&8~^{18!R@YaGyrVFIwIgO-# z|25UmO;0vfEG43;VcX4imnE_@R}3IhGMP}X^)AkI>{vWC6%vuFpdn)J!R#kBMAgsI z0vz#Et~nQagyDD5Z?&UvO{uU~+aKL5mL+s3@h2&k9R*tg0y>FW21dd+f zHtYx1Y@7R=poC@6#GI+D4srn6FXVEKaaINYrFaRwZ*ec`I27NmR`Q{>_@{KvQbP2U zT>Cw~TS%!zEva>p*&+OU&cY8`GH+N!jE+>wzhvAU8j|-MKeJrllO!JbGPrzJKyC_4&-+k2p3RRiWal)xTL-Rtsi7_z1k=t6K3u2vtDm*F!j=T!B1#@f13$5 zoZIU!*L<8NqcM5o0ObVQ_!U6C)^B^r}B^MD3vtY(Q3Sa%tR5uAPB#SVh&t z@Xm;~#>mL%?zrd?*C;tZdND^NZ;T_zP}(Acd}5*D{yScqslMxiS%u3H&A_Un+D;U&GRt@g=0^8uem4H$gjcNsr}$ z#&<_Nw!wYU>((r|s|5e*s4|_zCVPu!93(X5R1LXFAhe7uC(!6eU8cz6=TyJ$0BmoG61T6C9d{7X*#h z1R9~`V3+thAzCWy>>9j$q0_L!-FVNeWj0Cage7v-xwVo(8T%h%d>t1TDb7dvMN6bL zyqbu$TfP|eHFdM%+go;nu?*N=I<=Hrb5?>lY++hcH??pC*$2vuk64WAebjjrzC;_2 zS`oPNCV4Q|(nwz;`ztF!BppEQgO^y|$@Ynq|wY2bNk zw9a%SHlvUGNKLp?U_lRCWh=_Tp3_jUaB)Y^L$fSB;6`#z7ORNEWJf9hbkUDNz4S)! zjPs`d@w(>g0yC}HNT#m%+X?bEsyY{YhfUBZ#oAW;yqO2@N<8?>hShmqy~qN?(RsZw zba3rEM$PI*ddv#WW&xk1?Dvk>8;k@#);yO2u+QFDMbWFH`B7hG`<=64@*WAi^ZgOd zROL8vsYSa=HMutvoiezy3>smTK^}wJ>93w>cFBU^OC=JpJoD~l#9}QGdun<9PFv^$VE~EZO74_Goqs+{XQ>#zPqAdR=e?*&>-| zAjpcEE7^&g1+AAvnqSs z#!DSBPVo<{;2FLvg|oCjlZSOBRtu3jdV8rw6ilv{Uu9S@yI?+1s?srVTAL0ZiIjcN z=F@CjT@;yytxAdR_ZI)o^BX@a+mo%E;LIMtx6ik_U8eYCEs z-qo&yYygZ@X8J~MsV%PkPbokb!II>ux|re>CfdclI2y}lQcS_XOeYGvqI7Gifa{3+ zo*Sl8ps#4hZl%jy;C0rS3-&_yw<*cUkitH`A_GE|Wo{&Q_a`y*dE3z94ywz>5Em3b zml>$3oT>AB=F_9lf+_d(eE8lty_fY+@dCH&Es{4asI1qg)KG`08`e09GfsBab@TQ%==Q2u{t~8kXz<<5|Ds+WiRJ3D^Am`yT#zB`XnIIw0=$mKIb{~3SsNZdZp7^DFBKuLBMnUR z=o5=(-Y3}^oH(b+B;|MLDL5ymt>*tmW{_Q+{1J&q7bekZ%Tbl+8^u6`&1WRX^;M>v zo{U9AA(3+uW9}IDV99!knf(j{4a+{NxtQmra+kca#GR&VBB$LJmaE%6fz5j3+Elu< zp6Wt3fli$(bEoo7(z6N1a=h~+%B=WuL;~exiRFnYy|3&yIYA@W@w$s;i2fk)8cFrM ziT4%SOhkqAeBNg49W9|aY^Qp%1I>tDgPw}hAiyjETc*H$sf$U!%^=kClgQx$q4f*%^0m!XOyg^=`VATa&M;1*aEaAS*7(dNZDlsF-MCC1fTh2LPDatrV}6K4^1DPjH@x2%7kS z23EnPogb=mIW70x%v_v@;R0x`--+4pRPQlK_7!&+;}xM3x)Its#!jK>I8TDAw}9wFH?Cb^gDEkW^iD@(I` z)E(^GfvXi~iSFrQuz>2dj=n}@OiVYZvLLO_vo#*&uvV|NM>i#|Te@1Bbys>?a5T7z zgp-w?J_||K`54V5Lh0zF#>G`t^{5dQZlv0Ax2@6%aaG+({2&zV@I7WW)jkeyXy^6W zaIxC{?Y)K5D2rgfh#q86OYHGopS?MbLMhLAh3pf8V7wPOeZH)cawH8Veq!|q8(VWG1IURMRR-^kO9iZxzHQu-76V4W7@brb6*p59 z`&=`P1Rjh^fxf*?gLG9)FPw3T+IQGVLw!heCkJ|(!vHc@TmDo;oUizdN>pc%3(e<7 z$Y#Hgo284=P8k&18bvDrhZ5em^(*zcdh-qt zaB?33VW)Al*QhqtujXRPSO2W<&UT4v+Ls2t2Y2`|zmZ+j`Q{R)1@KAR8a-ADCB5FU zT%CE8`#<=fNy13e2cf5`zP^#@9`pd3&bihiCG9`5`(aXJ<_@ z(V+*7+|6cxDn5$nbpIDlRw%YUJA0|Lw~nSwGo?4tvYiYC$L8EIgrHUmKtY|VzAn8B zRoyevQZv9?(XpH^t)fE;TG}jjo>4JgtFnhuNT=ITQcW-F&7dqPYUy?LN12~-%uwIx zPlnIBUC*x{u-wR(SijDHkSHiilXR7~)<3K5vha|72XJ82mr-QYPxcP7j|(rw4#rVy z)89Vb8+ycyJStE%S+L2=Ud8|(=VG?vyGuCjVi-gLfKB9cR$3}q)DGOV0J0b(sp1Wn z?6;A{8|lRU>D;RK$RSeH>;}Hf2BG=f1DVu+?m`o&1rl#gc;D zPJbg0#7dEZiUvYk9@*)iFo$4lRQanzPkFt-I-$ol;}z;#PUGqun;QZmbDboXAAt(; z@^NTgOo~}DtS7r|3e_@ZVOWR!j0t=uv5k9#=pdd z`PV!WE^l033FD>qaJU1F{w~dYH?t#QvF09D<;k&c`xd|EI1sw*q^_b8#hNfezMN3+ zjB{LI3Kzz#gZAzMrC6oc;u#NqEwLv~9?o zmAR~Kw4m`dc&(l8!|`>y>!#Hv7`j#a4wP?kZoI6Oyy&MC35&j)sT47$4BWOA9n3L} zjUV;}ktg_P&SllaGEbG04quiY2@TyWyiX1-6f*3IyB#+uKL*LTPY z%ErTGO>v=P&UwZax__!MxXI2({ki;}?c)-=|92zx+pmv*`#Z9KBJzLivdzmsA*nS? zD=|$Vp96s9QXS_P4k)LE9g&{ZS)d9!5xWO@41%?Y@)&6Tu#z|e1_P%CwEDl^|x$1TFBn(e?GpIeE>}uUh%20t@&vSFWraikm zNP2uz!btHu^&2vdxr?$tiytk=M83EF5UGT?sxbWI3_|W9S*-u9FFQF`kOFK+k>^=X zjqZF`RsJN(Pj+7TKacoDC|m0DH6JhKmsPhr(FZjcO7J|ZQ9GXW-8?wQe5IFw&3sel ziy!Nwl-_F|{#Vv7(m6|&A)h2?g?L|(Q9t=FOVNMr0^K5Ci=^f$ z(j)2cEJ_;r$J4Iwu%pd06F_F`($2j{KmGyfFd*_DFfc|a8J^SRM=P+0t^^2wzW1V# z?u2@~YiwOYC+1Uj%9Y%sz=Migx8}|tUbnUyzQ+=*SEI7w!)m@!-)q2?obM_V=fww; z3o4;eyGA2dxE$HfOC#YlUb?{`kz>tou+4E$0q~y9A3YEPc)qe1sQVpKqQeKJkY1y0 z=x<~tX{RPG1^V=FkJA%L39L$FH$PP)6u_Zt_DU(t{DvJxr-hKCr#C-bIN11GtB7Ac zFMK0A`SG__)_?Cv_VYp|!Dq5VCv4XE<^9uVutnx2QuAQkQ1~2Re)8Nq*xod8(b$jz6U^Y{ z&bKXP!y#|9(4{qFh`m?&bndRGc`6*5G}XbbjFwG5T15p)b(Vn$sfTG(>SL~SWVXWnCtok zSD_B7s>s({bH0WST9v0{c-iYx2Xki@hH9!{{5B1(99WY}7Dj&~GZg(bU-jC- zXU}^l`;5`^IL9U3p872E+d)Y#D3Gg7On)Lo+(sy>!wU<*`vH?sg3JX!YOcykMnt;H zZXaB}k9CiO?ZmTntiu$dlMK(x)niHaEC#B2Vz0bmBu^%R zItl@&X;z!%c3eYL-v`DbaZwP%jzl}*@#5S?)H!Yf7-QJqLN|s%Ai3T6=q#b$Bem#lLL+@;V_q0&o`s)3!ez#G-w`~gfC(Z zT6MmFR3H`>cef}R@`jZ?8L8E$O;+Yb^Rig>IFlMpmbA}5it3`%tHjks7G|DYkjYztPGBMN_nM!w=l1j){EAWz($tqUyRxA9PBgZrAiW7uIhS{t!k!U(Bj%BZQu?_P=~-;W~2=j~w+@A;HXl6v$@X!=~|3YfIX# z%|{!vY?4E%W0*~49eWpQy-S|;1VRUGF5ZwzG`7KWCvT1W&LIrrlsrDuLMXS276|LZ z*zbm%`7?QZ~nk zYmDfV`vsTuXENL~)VVxvgRV@VxL%$E$J$rhBAbm@+H9IS6>+iLM#EsJ5RIx6Z$XBu zs*RjPqDl73QZ~@L9Ci$XXnppkeviwkv3ChD6=>d=f@`E9mAyPt_yx6Dsthxvg$$(J zikS24GRuSVW~{RlValB>qVWs-=eUp+uB{0rTRa`<(bjQ9?wii=a<%10)mldjKSb5#27@z zcvCor3;0zf$3O^rEuu?N?C1mJ5c%S^3Q4jz&w*ngvtrc}J0PsvFw_Eu5-TVwy59bn zo+*A<@X(+@?6_|bC_*=^IW;QkW6#sEe7sPb<|BIkMBgUx zRPCl!Er$xkeU9f1J*@BXQ+eVv95827gfo6$+Z!jOCQmW}b}1w8OE)ubs2Yj*Rv!Fh zj(>c}xXdBRL`O6AzKHgM>+NEv2j7X`>hm8wYEK-N*8r~B@}(Q^wuAQox>8`G5M>u%$S|o3F zFvHPWe(*isO+(}rDr?evQnaYVF!a){`JWCr<_Z2pH9rcg@`ZbyM%e{S}tr?t9#c+CuS%)L>qET`G8m6uI|PtbYV)cA_qFW*ls%u|!m zI##UG)qS3)!ZTr^5p~He3w7|q06Nm5T-sk2OLl*}5-S;sYfX#o)43r_Fz}S4$n+$% znTbj1eWahuft7*M1}Pb5%Vud`5~1Hx9Ozh)ibq>W1!W>;qyZ)np;^CMbdU&mXO<)pUZ0bKn8*x#>l8nvKAVW*Q}RbeWCx^qC${7uy>goy4O_4trI+K2$ty)U%|2 z3EW}ogSZL*+Bvw4KXtr5B{b5)YlZG%5S0zdi7C52Xj6XdU@@quyIkm{P@V&7AXi^A zEZa<5wOZ@P_MluH}T6I?a%+N^aPbbNpq?6F$y zQv1Xg*cv+>Nq*X2Zm|;Yl)%ULhGZ)X&eRPTx}sa?WFM!mJ}@QG*+q1TEsb`E(ZtdY zW&ezdXFE+jf9oP3Pnk0`){#mz?YJ4?fq{xeQuykFC5?4s;K)@PPo@#_TocUv+#TM4 z;h!y~uEvMj2ZzNQnCnB&cHGr^F_5x$eH$>s(k`W&_I@pz#T>GV^_i+hs4(`up(vPT_(As3D8;LVgOAx(`yX@EQo;ZUM&FKhVw+$gZYSlS~ zP-yU)F?;2DRLZ5-6&9>4_t9c#ve;5WayjbHQto%js{q{QmzernUIoAbzUrxJfVa5W zU%_9YV_D0t-Y_1tI{e%+X%ce*K z;-NLDn`v(T&eXiQvwJ+(HrnuM9u9$CJM+@G{ub9GIb3U9LX=B-YEQ^9d10mVrc(4> zJ_I)*&m+!;ru1=mGWFvOu? zeD!B6!?!RI0DztSYq#n@j52?ivOx#0yEYoWHo9t%g45DfM$Ia6r$c!YJiquXdDs@xxd~Y&;MbRZ` z2Vviw8ha!Aw;t;b+35pVwf&EPZ<=-s;>AD6x*|hbR`kRv;eIh`^`TG9b3Dq#^xxip z61a)!h!ty5uvz36W6fNMBIU#3b8H1yY8ZK7(qfsEjc2DEmGfjPEZ{Qj3nkDR6YYFs z1O@+KzNKY}MW;Ormk7}M2kV~?;OEgpOUag*Km^-LBCH1emYtxahPNS2h^sMrIp6gdl`5@&RIg`;6+sR;ubK ztD1Gb$cw*9a}l@48%D6Hdr-B(zxI+#U8*hL?mPmwr@r!~>e9pR$o!Q&{)U=UQ!5>- zA}t_<505BoPGe$&cPv_gbe|8sD#rhQqq|4N;PlSwG=u@H)M2#|XQ?=Jy9~0RhI4W^ zjqpW7i;k1)?0`2jHeC6U*x}nmydy&?IsYXW)wAX`OqmsH#IU_xIIGy9>F^EWHx~c9 z&mu6Tb6u|&h+~pVFEBmRrP{ijW@D7zbwF~$B80#hPd{}9F!YfhGxF&9F-U`F_aQ7M z((FmMiQRy0d_bdX6XRx2u>DmD-_@8Y)W_;5u=o+pQY$!@>p?J&%g1x;LGEXi18>UX%(w&!eW)^(s zv%XpS7nFbF>8~BlZ#?~P@w5%_-3GwOToA@sm-J2=SdYMQx2?a}YTisVNQ8)}MGWmr zn>DxspS)s%wGoKHPEI7cjhfewco~>3CdK=gMA+`Tdw!E6JLX?2;qsSmOSG#G41DCS z5t_I!83^8jX1}JSg67G!pPsd}wGvsao_9Gs9sk}|s?Va~2~_U8IhP=+Tg?#6dop$> zm9-l_an)0b1|kdi?jKal-jaq&?`aj2l@$un;u*3s1_P zViX{3fJoUPFk_VyO~Wy5gGoqfox<9>jo%TlI`ef~-ni6_BkJnLT46I<0neF6CwlLT zu|6G44y49m>7@;6tO+-(N^29}y$#M>F^6hK0r}tb5LF zIe_L1R(_>>#td4v$gB&wc}nHNG~}E%k9eDWxGKCM_bvVUnvVcivks2&0uz(j53bd)i4W^n9`$c-5?Q!Y>UJuD9S6k-W#z<_NTDei^d*#This tool has only been tested on [Pji's](https://www.pji.net.cn/) Electronic Control Unit of RK3588. Other RK3588 development boards require independent testing. +3. Set the path of `boot.img` in **boot** and connect the RK3588 board. +4. Press the `Run` button to flash the image to the RK3588 board. + +![RKDevTool](./figures/RKDevTool3.3.png) \ No newline at end of file diff --git a/modules/axhal/build.rs b/modules/axhal/build.rs index 56061e6434..f2804abd18 100644 --- a/modules/axhal/build.rs +++ b/modules/axhal/build.rs @@ -5,6 +5,7 @@ const BUILTIN_PLATFORMS: &[&str] = &[ "aarch64-bsta1000b", "aarch64-qemu-virt", "aarch64-raspi4", + "aarch64-rk3588j", "riscv64-qemu-virt", "x86_64-pc-oslab", "x86_64-qemu-q35", @@ -15,6 +16,7 @@ const BUILTIN_PLATFORM_FAMILIES: &[&str] = &[ "aarch64-phytium-pi", "aarch64-qemu-virt", "aarch64-raspi", + "aarch64-rk3588j", "riscv64-qemu-virt", "x86-pc", ]; diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index e8dbab6f72..9f9039ae3a 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -276,6 +276,7 @@ unsafe extern "C" fn _start() -> ! { // PC = 0x8_0000 // X0 = dtb core::arch::asm!(" + mov x20, x0 // save DTB pointer // disable cache and MMU mrs x1, sctlr_el2 bic x1, x1, #0xf @@ -289,13 +290,12 @@ unsafe extern "C" fn _start() -> ! { mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id - mov x20, x0 // save DTB pointer adrp x8, {boot_stack} // setup boot stack add x8, x8, {boot_stack_size} mov sp, x8 - bl {switch_to_el2} // switch to EL1 + bl {switch_to_el2} // switch to EL2 bl {enable_fp} // enable fp/neon bl {init_boot_page_table} bl {init_mmu_el2} diff --git a/modules/axhal/src/platform/aarch64_common/mod.rs b/modules/axhal/src/platform/aarch64_common/mod.rs index c585541fe8..3cc33611fb 100644 --- a/modules/axhal/src/platform/aarch64_common/mod.rs +++ b/modules/axhal/src/platform/aarch64_common/mod.rs @@ -7,5 +7,8 @@ pub mod psci; #[cfg(feature = "irq")] pub mod gic; -#[cfg(not(platform_family = "aarch64-bsta1000b"))] +#[cfg(not(any( + platform_family = "aarch64-bsta1000b", + platform_family = "aarch64-rk3588j" +)))] pub mod pl011; diff --git a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs new file mode 100644 index 0000000000..6d9de293be --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs @@ -0,0 +1,147 @@ +//! Definitions for snps,dw-apb-uart serial driver. +//! Uart snps,dw-apb-uart driver in Rust for BST A1000b FADA board. +use crate::mem::phys_to_virt; +use kspin::SpinNoIrq; +use memory_addr::PhysAddr; + +use tock_registers::{ + interfaces::{Readable, Writeable}, + register_structs, + registers::{ReadOnly, ReadWrite}, +}; + +register_structs! { + DW8250Regs { + /// Get or Put Register. + (0x00 => rbr: ReadWrite), + (0x04 => ier: ReadWrite), + (0x08 => fcr: ReadWrite), + (0x0c => lcr: ReadWrite), + (0x10 => mcr: ReadWrite), + (0x14 => lsr: ReadOnly), + (0x18 => msr: ReadOnly), + (0x1c => scr: ReadWrite), + (0x20 => lpdll: ReadWrite), + (0x24 => _reserved0), + /// Uart Status Register. + (0x7c => usr: ReadOnly), + (0x80 => _reserved1), + (0xc0 => dlf: ReadWrite), + (0xc4 => @END), + } +} + +/// dw-apb-uart serial driver: DW8250 +pub struct DW8250 { + base_vaddr: usize, +} + +impl DW8250 { + /// New a DW8250 + pub const fn new(base_vaddr: usize) -> Self { + Self { base_vaddr } + } + + const fn regs(&self) -> &DW8250Regs { + unsafe { &*(self.base_vaddr as *const _) } + } + + /// DW8250 initialize + pub fn init(&mut self) { + const UART_SRC_CLK: u32 = 25000000; + const BST_UART_DLF_LEN: u32 = 6; + const BAUDRATE: u32 = 115200; + //const BAUDRATE: u32 = 38400; + + let get_baud_divider = |baudrate| (UART_SRC_CLK << (BST_UART_DLF_LEN - 4)) / baudrate; + let divider = get_baud_divider(BAUDRATE); + + // Waiting to be no USR_BUSY. + while self.regs().usr.get() & 0b1 != 0 {} + + // bst_serial_hw_init_clk_rst + + /* Disable interrupts and Enable FIFOs */ + self.regs().ier.set(0); + self.regs().fcr.set(1); + + /* Disable flow ctrl */ + self.regs().mcr.set(0); + + /* Clear MCR_RTS */ + self.regs().mcr.set(self.regs().mcr.get() | (1 << 1)); + + /* Enable access DLL & DLH. Set LCR_DLAB */ + self.regs().lcr.set(self.regs().lcr.get() | (1 << 7)); + + /* Set baud rate. Set DLL, DLH, DLF */ + self.regs().rbr.set((divider >> BST_UART_DLF_LEN) & 0xff); + self.regs() + .ier + .set((divider >> (BST_UART_DLF_LEN + 8)) & 0xff); + self.regs().dlf.set(divider & ((1 << BST_UART_DLF_LEN) - 1)); + + /* Clear DLAB bit */ + self.regs().lcr.set(self.regs().lcr.get() & !(1 << 7)); + + /* Set data length to 8 bit, 1 stop bit, no parity. Set LCR_WLS1 | LCR_WLS0 */ + self.regs().lcr.set(self.regs().lcr.get() | 0b11); + } + + /// DW8250 serial output + pub fn putchar(&mut self, c: u8) { + // Check LSR_TEMT + // Wait for last character to go. + while self.regs().lsr.get() & (1 << 6) == 0 {} + self.regs().rbr.set(c as u32); + } + + /// DW8250 serial input + pub fn getchar(&mut self) -> Option { + // Check LSR_DR + // Wait for a character to arrive. + if self.regs().lsr.get() & 0b1 != 0 { + Some((self.regs().rbr.get() & 0xff) as u8) + } else { + None + } + } + + /// DW8250 serial interrupt enable or disable + pub fn set_ier(&mut self, enable: bool) { + if enable { + // Enable interrupts + self.regs().ier.set(1); + } else { + // Disable interrupts + self.regs().ier.set(0); + } + } +} + +const UART_BASE: PhysAddr = pa!(axconfig::UART_PADDR); + +pub static UART: SpinNoIrq = + SpinNoIrq::new(DW8250::new(phys_to_virt(UART_BASE).as_usize())); + +/// Writes a byte to the console. +pub fn putchar(c: u8) { + let mut uart = UART.lock(); + match c { + b'\r' | b'\n' => { + uart.putchar(b'\r'); + uart.putchar(b'\n'); + } + c => uart.putchar(c), + } +} + +/// Reads a byte from the console, or returns [`None`] if no input is available. +pub fn getchar() -> Option { + UART.lock().getchar() +} + +/// UART simply initialize +pub fn init_early() { + UART.lock().init(); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs new file mode 100644 index 0000000000..d7cf3415ee --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -0,0 +1,68 @@ +use crate::mem::{MemRegion, MemRegionFlags, PhysAddr}; +use page_table_entry::{aarch64::A64PTE, GenericPTE, MappingFlags}; + +/// Returns (rk3588j only) memory regions. +pub(crate) fn default_rk3588j_regions() -> impl Iterator { + [ + MemRegion { + paddr: PhysAddr::from(0x9400000), + size: 0xe6c00000, + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "free memory", + }, + MemRegion { + paddr: PhysAddr::from(0x1f0000000), + size: 0x10000000, + flags: MemRegionFlags::FREE | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "free memory", + }, + ] + .into_iter() +} + +/// Returns platform-specific memory regions. +pub(crate) fn platform_regions() -> impl Iterator { + crate::mem::default_free_regions() + .chain(default_rk3588j_regions()) + .chain(crate::mem::default_mmio_regions()) +} + +pub(crate) unsafe fn init_boot_page_table( + boot_pt_l0: *mut [A64PTE; 512], + boot_pt_l1: *mut [A64PTE; 512], +) { + let boot_pt_l0 = &mut *boot_pt_l0; + let boot_pt_l1 = &mut *boot_pt_l1; + boot_pt_l0[0] = A64PTE::new_table(PhysAddr::from(boot_pt_l1.as_ptr() as usize)); + + boot_pt_l1[0] = A64PTE::new_page( + PhysAddr::from(0), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[1] = A64PTE::new_page( + PhysAddr::from(0x4000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[2] = A64PTE::new_page( + PhysAddr::from(0x8000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[3] = A64PTE::new_page( + PhysAddr::from(0xC000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::DEVICE, + true, + ); + boot_pt_l1[7] = A64PTE::new_page( + PhysAddr::from(0x1_C000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[8] = A64PTE::new_page( + PhysAddr::from(0x1_F000_0000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs new file mode 100644 index 0000000000..c59047bd78 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs @@ -0,0 +1,66 @@ +pub mod dw_apb_uart; +pub mod mem; + +#[cfg(feature = "smp")] +pub mod mp; + +#[cfg(feature = "irq")] +pub mod irq { + pub use crate::platform::aarch64_common::gic::*; +} + +pub mod console { + pub use super::dw_apb_uart::*; +} + +pub mod time { + pub use crate::platform::aarch64_common::generic_timer::*; +} + +pub mod misc { + pub use crate::platform::aarch64_common::psci::system_off as terminate; +} + +extern "C" { + fn exception_vector_base(); + fn rust_main(cpu_id: usize, dtb: usize); + #[cfg(feature = "smp")] + fn rust_main_secondary(cpu_id: usize); +} + +pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { + crate::mem::clear_bss(); + crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] + crate::arch::write_page_table_root0(0.into()); // disable low address access + crate::cpu::init_primary(cpu_id); + super::dw_apb_uart::init_early(); + super::aarch64_common::generic_timer::init_early(); + rust_main(cpu_id, dtb); +} + +#[cfg(feature = "smp")] +#[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. +pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { + crate::arch::set_exception_vector_base(exception_vector_base as usize); + crate::arch::write_page_table_root0(0.into()); // disable low address access + crate::cpu::init_secondary(cpu_id); + rust_main_secondary(cpu_id); +} + +/// Initializes the platform devices for the primary CPU. +/// +/// For example, the interrupt controller and the timer. +pub fn platform_init() { + #[cfg(feature = "irq")] + super::aarch64_common::gic::init_primary(); + super::aarch64_common::generic_timer::init_percpu(); +} + +/// Initializes the platform devices for secondary CPUs. +#[cfg(feature = "smp")] +pub fn platform_init_secondary() { + #[cfg(feature = "irq")] + super::aarch64_common::gic::init_secondary(); + super::aarch64_common::generic_timer::init_percpu(); +} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs new file mode 100644 index 0000000000..01f84fc4c1 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs @@ -0,0 +1,20 @@ +use crate::mem::{virt_to_phys, PhysAddr, VirtAddr}; + +/// Hart number of rk3588 board +pub const MAX_HARTS: usize = 8; +/// CPU HWID from cpu device tree nodes with "reg" property +pub const CPU_HWID: [usize; MAX_HARTS] = [0x00, 0x100, 0x200, 0x300, 0x400, 0x500, 0x600, 0x700]; + +/// Starts the given secondary CPU with its boot stack. +pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) { + assert!(cpu_id < MAX_HARTS, "No support for rk3588 core {}", cpu_id); + extern "C" { + fn _start_secondary(); + } + let entry = virt_to_phys(VirtAddr::from(_start_secondary as usize)); + crate::platform::aarch64_common::psci::cpu_on( + CPU_HWID[cpu_id], + entry.as_usize(), + stack_top.as_usize(), + ); +} diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 349aac9ae0..57431a0e05 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -25,6 +25,9 @@ cfg_if::cfg_if! { } else if #[cfg(all(target_arch = "aarch64", platform_family = "aarch64-phytium-pi"))] { mod aarch64_phytium_pi; pub use self::aarch64_phytium_pi::*; + } else if #[cfg(all(target_arch = "aarch64", platform_family = "aarch64-rk3588j"))] { + mod aarch64_rk3588j; + pub use self::aarch64_rk3588j::*; } else { mod dummy; pub use self::dummy::*; diff --git a/platforms/aarch64-rk3588j.toml b/platforms/aarch64-rk3588j.toml new file mode 100644 index 0000000000..61d0793234 --- /dev/null +++ b/platforms/aarch64-rk3588j.toml @@ -0,0 +1,68 @@ +# Architecture identifier. +arch = "aarch64" +# Platform identifier. +platform = "aarch64-rk3588j" +# Platform family. +family = "aarch64-rk3588j" + +# Base address of the whole physical memory. +phys-memory-base = "0x20_0000" +# Size of the whole physical memory. +phys-memory-size = "0x800_0000" # 128M +# Base physical address of the kernel image. +kernel-base-paddr = "0x48_0000" +# Base virtual address of the kernel image. +kernel-base-vaddr = "0x0000_0000_0048_0000" +# Linear mapping offset, for quick conversions between physical and virtual +# addresses. +phys-virt-offset = "0xffff_0000_0000_0000" +# Kernel address space base. +kernel-aspace-base = "0xffff_0000_0000_0000" +# Kernel address space size. +kernel-aspace-size = "0x0000_ffff_ffff_f000" +# MMIO regions with format (`base_paddr`, `size`). +mmio-regions = [ + # ["0x0900_0000", "0x1000"], # PL011 UART + # ["0x0800_0000", "0x5_0000"], # GICv2 with Virtualization (GICV@0x0803_0000, GICH@0x0804_0000) + # ["0x0a00_0000", "0x4000"], # VirtIO + # ["0x1000_0000", "0x2eff_0000"], # PCI memory ranges (ranges 1: 32-bit MMIO space) + # ["0x40_1000_0000", "0x1000_0000"], # PCI config space + ["0xfeb50000", "0x1000"], # uart8250 UART0 + ["0xfe600000", "0x10000"], # gic-v3 gicd + ["0xfe680000", "0x100000"], # gic-v3 gicr + ["0xa41000000", "0x400000"], + ["0xa40c00000", "0x400000"], + ["0xf4000000","0x1000000"], + ["0xf3000000","0x1000000"], +] +# VirtIO MMIO regions with format (`base_paddr`, `size`). +virtio-mmio-regions = [] + +# Base physical address of the PCIe ECAM space. +pci-ecam-base = "0xf4000000" +# End PCI bus number (`bus-range` property in device tree). +pci-bus-end = "0xff" +# PCI device memory ranges (`ranges` property in device tree). +pci-ranges = [] +# UART Address +uart-paddr = "0xfeb5_0000" +uart-irq = "0x14d" + +# GICC Address +gicd-paddr = "0xfe600000" +# GICR Address +gicc-paddr = "0xfe680000" +gicr-paddr = "0xfe680000" + +# PSCI +psci-method = "smc" + +# pl031@9010000 { +# clock-names = "apb_pclk"; +# clocks = <0x8000>; +# interrupts = <0x00 0x02 0x04>; +# reg = <0x00 0x9010000 0x00 0x1000>; +# compatible = "arm,pl031\0arm,primecell"; +# }; +# RTC (PL031) Address +# rtc-paddr = "0x901_0000" \ No newline at end of file diff --git a/scripts/make/rk3588.mk b/scripts/make/rk3588.mk new file mode 100644 index 0000000000..49c0a44434 --- /dev/null +++ b/scripts/make/rk3588.mk @@ -0,0 +1,13 @@ +RK3588_GITHUB_URL = https://github.com/arceos-hypervisor/platform_tools/releases/download/latest/rk3588.zip +RK3588_MKIMG_FILE = ./tools/rk3588/mkimg +check-download: +ifeq ("$(wildcard $(RK3588_MKIMG_FILE))","") + @echo "file not found, downloading from $(RK3588_GITHUB_URL)..."; + wget $(RK3588_GITHUB_URL); + unzip -o rk3588.zip -d tools; + rm rk3588.zip; +endif + +kernel: check-download build + $(RK3588_MKIMG_FILE) --dtb rk3588-firefly-itx-3588j.dtb --img $(OUT_BIN) + @echo 'Built the FIT-uImage boot.img' From f95d7e733ad29e714455ee70c965ae09e8096aea Mon Sep 17 00:00:00 2001 From: hky1999 Date: Wed, 23 Oct 2024 16:19:02 +0800 Subject: [PATCH 03/19] [refactor] aarch64 el2 support (#31) --- .../axhal/src/platform/aarch64_common/boot.rs | 151 ------------- .../src/platform/aarch64_common/boot_el2.rs | 198 ++++++++++++++++++ .../axhal/src/platform/aarch64_common/mod.rs | 4 + 3 files changed, 202 insertions(+), 151 deletions(-) create mode 100644 modules/axhal/src/platform/aarch64_common/boot_el2.rs diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 9f9039ae3a..1140c17b67 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,8 +1,5 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use memory_addr::PhysAddr; -//Todo: remove this, when hv is enabled, `MemAttr` is not used. -#[cfg_attr(feature = "hv", allow(unused_imports))] use page_table_entry::aarch64::{MemAttr, A64PTE}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; @@ -17,7 +14,6 @@ static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; #[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; -#[cfg(not(feature = "hv"))] unsafe fn switch_to_el1() { SPSel.write(SPSel::SP::ELx); SP_EL0.set(0); @@ -61,7 +57,6 @@ unsafe fn switch_to_el1() { } } -#[cfg(not(feature = "hv"))] unsafe fn init_mmu() { MAIR_EL1.set(MemAttr::MAIR_VALUE); @@ -106,7 +101,6 @@ unsafe fn init_boot_page_table() { } /// The earliest entry point for the primary CPU. -#[cfg(not(feature = "hv"))] #[naked] #[unsafe(no_mangle)] #[unsafe(link_section = ".text.boot")] @@ -147,7 +141,6 @@ unsafe extern "C" fn _start() -> ! { } /// The earliest entry point for the secondary CPUs. -#[cfg(not(feature = "hv"))] #[cfg(feature = "smp")] #[naked] #[unsafe(no_mangle)] @@ -176,147 +169,3 @@ unsafe extern "C" fn _start_secondary() -> ! { entry = sym crate::platform::rust_entry_secondary, ) } - -#[cfg(feature = "hv")] -unsafe fn switch_to_el2() { - SPSel.write(SPSel::SP::ELx); - let current_el = CurrentEL.read(CurrentEL::EL); - - if current_el == 3 { - SCR_EL3.write( - SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, - ); - SPSR_EL3.write( - SPSR_EL3::M::EL2h - + SPSR_EL3::D::Masked - + SPSR_EL3::A::Masked - + SPSR_EL3::I::Masked - + SPSR_EL3::F::Masked, - ); - ELR_EL3.set(LR.get()); - SP_EL1.set(BOOT_STACK.as_ptr_range().end as u64); - // This should be SP_EL2. To - asm::eret(); - } -} - -#[cfg(feature = "hv")] -unsafe fn init_mmu_el2() { - // Set EL1 to 64bit. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); - - // Device-nGnRE memory - let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; - // Normal memory - let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc - + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; - MAIR_EL2.write(attr0 + attr1); // 0xff_04 - - // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. - let tcr_flags0 = TCR_EL2::TG0::KiB_4 - + TCR_EL2::SH0::Inner - + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable - + TCR_EL2::T0SZ.val(16); - TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); - barrier::isb(barrier::SY); - - let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; - TTBR0_EL2.set(root_paddr); - - // Flush the entire TLB - crate::arch::flush_tlb(None); - - // Enable the MMU and turn on I-cache and D-cache - SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); - barrier::isb(barrier::SY); -} - -#[cfg(feature = "hv")] -unsafe fn cache_invalidate(cache_level: usize) { - core::arch::asm!( - r#" - msr csselr_el1, {0} - mrs x4, ccsidr_el1 // read cache size id. - and x1, x4, #0x7 - add x1, x1, #0x4 // x1 = cache line size. - ldr x3, =0x7fff - and x2, x3, x4, lsr #13 // x2 = cache set number – 1. - ldr x3, =0x3ff - and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. - clz w4, w3 // x4 = way position in the cisw instruction. - mov x5, #0 // x5 = way counter way_loop. - // way_loop: - 1: - mov x6, #0 // x6 = set counter set_loop. - // set_loop: - 2: - lsl x7, x5, x4 - orr x7, {0}, x7 // set way. - lsl x8, x6, x1 - orr x7, x7, x8 // set set. - dc cisw, x7 // clean and invalidate cache line. - add x6, x6, #1 // increment set counter. - cmp x6, x2 // last set reached yet? - ble 2b // if not, iterate set_loop, - add x5, x5, #1 // else, next way. - cmp x5, x3 // last way reached yet? - ble 1b // if not, iterate way_loop - "#, - in(reg) cache_level, - options(nostack) - ); -} - -#[cfg(feature = "hv")] -#[naked] -#[no_mangle] -#[link_section = ".text.boot"] -unsafe extern "C" fn _start() -> ! { - // PC = 0x8_0000 - // X0 = dtb - core::arch::asm!(" - mov x20, x0 // save DTB pointer - // disable cache and MMU - mrs x1, sctlr_el2 - bic x1, x1, #0xf - msr sctlr_el2, x1 - - // cache_invalidate(0): clear dl1$ - mov x0, #0 - bl {cache_invalidate} - mov x0, #2 - bl {cache_invalidate} - - mrs x19, mpidr_el1 - and x19, x19, #0xffffff // get current CPU id - - adrp x8, {boot_stack} // setup boot stack - add x8, x8, {boot_stack_size} - mov sp, x8 - - bl {switch_to_el2} // switch to EL2 - bl {enable_fp} // enable fp/neon - bl {init_boot_page_table} - bl {init_mmu_el2} - - mov x8, {phys_virt_offset} // set SP to the high address - add sp, sp, x8 - - mov x0, x19 // call rust_entry(cpu_id, dtb) - mov x1, x20 - ldr x8, ={entry} - blr x8 - b .", - cache_invalidate = sym cache_invalidate, - init_boot_page_table = sym init_boot_page_table, - init_mmu_el2 = sym init_mmu_el2, - switch_to_el2 = sym switch_to_el2, - enable_fp = sym enable_fp, - boot_stack = sym BOOT_STACK, - boot_stack_size = const TASK_STACK_SIZE, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry, - options(noreturn), - ); -} diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs new file mode 100644 index 0000000000..d8b6bc0cf1 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -0,0 +1,198 @@ +use core::ptr::addr_of_mut; + +use aarch64_cpu::{asm, asm::barrier, registers::*}; +use memory_addr::PhysAddr; +use page_table_entry::aarch64::A64PTE; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + +use axconfig::TASK_STACK_SIZE; + +#[link_section = ".bss.stack"] +static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; + +#[link_section = ".data.boot_page_table"] +static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; + +#[link_section = ".data.boot_page_table"] +static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; + +unsafe fn switch_to_el2() { + SPSel.write(SPSel::SP::ELx); + let current_el = CurrentEL.read(CurrentEL::EL); + + if current_el == 3 { + SCR_EL3.write( + SCR_EL3::NS::NonSecure + SCR_EL3::HCE::HvcEnabled + SCR_EL3::RW::NextELIsAarch64, + ); + SPSR_EL3.write( + SPSR_EL3::M::EL2h + + SPSR_EL3::D::Masked + + SPSR_EL3::A::Masked + + SPSR_EL3::I::Masked + + SPSR_EL3::F::Masked, + ); + ELR_EL3.set(LR.get()); + asm::eret(); + } +} + +unsafe fn init_mmu_el2() { + // Set EL1 to 64bit. + HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + + // Device-nGnRE memory + let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; + // Normal memory + let attr1 = MAIR_EL2::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + MAIR_EL2::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc; + MAIR_EL2.write(attr0 + attr1); // 0xff_04 + + // Enable TTBR0 and TTBR1 walks, page size = 4K, vaddr size = 48 bits, paddr size = 40 bits. + let tcr_flags0 = TCR_EL2::TG0::KiB_4 + + TCR_EL2::SH0::Inner + + TCR_EL2::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable + + TCR_EL2::T0SZ.val(16); + TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); + barrier::isb(barrier::SY); + + let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + TTBR0_EL2.set(root_paddr); + + // Flush the entire TLB + crate::arch::flush_tlb(None); + + // Enable the MMU and turn on I-cache and D-cache + SCTLR_EL2.modify(SCTLR_EL2::M::Enable + SCTLR_EL2::C::Cacheable + SCTLR_EL2::I::Cacheable); + barrier::isb(barrier::SY); +} + +unsafe fn enable_fp() { + if cfg!(feature = "fp_simd") { + CPACR_EL1.write(CPACR_EL1::FPEN::TrapNothing); + barrier::isb(barrier::SY); + } +} + +unsafe fn init_boot_page_table() { + crate::platform::mem::init_boot_page_table(addr_of_mut!(BOOT_PT_L0), addr_of_mut!(BOOT_PT_L1)); +} + +unsafe fn cache_invalidate(cache_level: usize) { + core::arch::asm!( + r#" + msr csselr_el1, {0} + mrs x4, ccsidr_el1 // read cache size id. + and x1, x4, #0x7 + add x1, x1, #0x4 // x1 = cache line size. + ldr x3, =0x7fff + and x2, x3, x4, lsr #13 // x2 = cache set number – 1. + ldr x3, =0x3ff + and x3, x3, x4, lsr #3 // x3 = cache associativity number – 1. + clz w4, w3 // x4 = way position in the cisw instruction. + mov x5, #0 // x5 = way counter way_loop. + // way_loop: + 1: + mov x6, #0 // x6 = set counter set_loop. + // set_loop: + 2: + lsl x7, x5, x4 + orr x7, {0}, x7 // set way. + lsl x8, x6, x1 + orr x7, x7, x8 // set set. + dc cisw, x7 // clean and invalidate cache line. + add x6, x6, #1 // increment set counter. + cmp x6, x2 // last set reached yet? + ble 2b // if not, iterate set_loop, + add x5, x5, #1 // else, next way. + cmp x5, x3 // last way reached yet? + ble 1b // if not, iterate way_loop + "#, + in(reg) cache_level, + options(nostack) + ); +} + +/// The earliest entry point for the primary CPU. +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start() -> ! { + // PC = 0x8_0000 + // X0 = dtb + core::arch::asm!(" + // disable cache and MMU + mrs x1, sctlr_el2 + bic x1, x1, #0xf + msr sctlr_el2, x1 + + // cache_invalidate(0): clear dl1$ + mov x0, #0 + bl {cache_invalidate} + mov x0, #2 + bl {cache_invalidate} + + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + mov x20, x0 // save DTB pointer + + adrp x8, {boot_stack} // setup boot stack + add x8, x8, {boot_stack_size} + mov sp, x8 + + bl {switch_to_el2} // switch to EL2 + bl {enable_fp} // enable fp/neon + bl {init_boot_page_table} + bl {init_mmu_el2} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry(cpu_id, dtb) + mov x1, x20 + ldr x8, ={entry} + blr x8 + b .", + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + options(noreturn), + ); +} + +/// The earliest entry point for the secondary CPUs. +#[cfg(feature = "smp")] +#[naked] +#[no_mangle] +#[link_section = ".text.boot"] +unsafe extern "C" fn _start_secondary() -> ! { + core::arch::asm!(" + mrs x19, mpidr_el1 + and x19, x19, #0xffffff // get current CPU id + + mov sp, x0 + bl {switch_to_el2} + bl {init_mmu_el2} + bl {enable_fp} + + mov x8, {phys_virt_offset} // set SP to the high address + add sp, sp, x8 + + mov x0, x19 // call rust_entry_secondary(cpu_id) + ldr x8, ={entry} + blr x8 + b .", + switch_to_el2 = sym switch_to_el2, + init_mmu_el2 = sym init_mmu_el2, + enable_fp = sym enable_fp, + phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry_secondary, + options(noreturn), + ) +} diff --git a/modules/axhal/src/platform/aarch64_common/mod.rs b/modules/axhal/src/platform/aarch64_common/mod.rs index 3cc33611fb..8a2a450930 100644 --- a/modules/axhal/src/platform/aarch64_common/mod.rs +++ b/modules/axhal/src/platform/aarch64_common/mod.rs @@ -1,4 +1,8 @@ +#[cfg(not(feature = "hv"))] mod boot; +#[cfg(feature = "hv")] +// Todo: maybe we can enter el2 in arm_vcpu? +mod boot_el2; pub mod generic_timer; #[cfg(not(platform_family = "aarch64-raspi"))] From 230e4319efbc66375225fbbad0d1dc90fe7df2bd Mon Sep 17 00:00:00 2001 From: hky1999 Date: Wed, 20 Nov 2024 22:32:30 +0800 Subject: [PATCH 04/19] [feat] support irq handling routine from arceos-umhv (#33) --- Cargo.lock | 3 +- modules/axhal/Cargo.toml | 2 +- modules/axhal/src/irq.rs | 10 +- .../src/platform/aarch64_common/boot_el2.rs | 13 +- .../platform/aarch64_common/generic_timer.rs | 10 +- .../axhal/src/platform/aarch64_common/gic.rs | 153 ++---------------- .../src/platform/aarch64_qemu_virt/mod.rs | 1 + modules/axhal/src/platform/dummy/mod.rs | 5 + 8 files changed, 47 insertions(+), 150 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 272bbf376d..bb680752a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,8 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2?rev=7acc0dd#7acc0dd1e1fefa90b4cd45342072d2350a74b071" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2#dfe5f164b94cdd07081c2fe74a0cfe4bef2852c9" dependencies = [ - "crate_interface", "tock-registers", ] diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index d2a5b8d38e..39a01bb433 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -55,7 +55,7 @@ riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "9.4" tock-registers = "0.8" -arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", rev = "7acc0dd" } +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2" } crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 9498bcb002..2d6b1247c6 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -7,6 +7,9 @@ use crate::trap::{IRQ, register_trap_handler}; pub use crate::platform::irq::{register_handler, set_enable}; +#[cfg(target_arch = "aarch64")] +pub use crate::platform::irq::fetch_irq; + /// The type if an IRQ handler. pub type IrqHandler = handler_table::Handler; @@ -35,8 +38,13 @@ pub(crate) fn register_handler_common(irq_num: usize, handler: IrqHandler) -> bo false } +/// Core IRQ handling routine, registered at `axhal::trap::IRQ`, +/// which dispatches IRQs to registered handlers. +/// +/// Note: this function is denoted as public here because it'll be called by the +/// hypervisor for hypervisor reserved IRQ handling. #[register_trap_handler(IRQ)] -fn handler_irq(irq_num: usize) -> bool { +pub fn handler_irq(irq_num: usize) -> bool { let guard = kernel_guard::NoPreempt::new(); dispatch_irq(irq_num); drop(guard); // rescheduling may occur when preemption is re-enabled. diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index d8b6bc0cf1..fd15e6ac36 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -38,7 +38,18 @@ unsafe fn switch_to_el2() { unsafe fn init_mmu_el2() { // Set EL1 to 64bit. - HCR_EL2.write(HCR_EL2::RW::EL1IsAarch64); + // Enable `IMO` and `FMO` to make sure that: + // * Physical IRQ interrupts are taken to EL2; + // * Virtual IRQ interrupts are enabled; + // * Physical FIQ interrupts are taken to EL2; + // * Virtual FIQ interrupts are enabled. + HCR_EL2.modify( + HCR_EL2::VM::Enable + + HCR_EL2::RW::EL1IsAarch64 + + HCR_EL2::IMO::EnableVirtualIRQ // Physical IRQ Routing. + + HCR_EL2::FMO::EnableVirtualFIQ // Physical FIQ Routing. + + HCR_EL2::TSC::EnableTrapEl1SmcToEl2, + ); // Device-nGnRE memory let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 8af7ffc728..127db32c39 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -101,12 +101,12 @@ pub(crate) fn init_percpu() { } #[cfg(all(feature = "irq", feature = "hv"))] { - // ENABLE, bit [0]， Enables the timer. - let ctl = 1; - let tval = 0; unsafe { - core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) ctl); - core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) tval); + // ENABLE, bit [0], Enables the timer. + // * 0b0: Timer disabled. + // * 0b1: Timer enabled. + core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); + core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); } } #[cfg(feature = "irq")] diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs index 5fdf37eb2b..dad3172cd3 100644 --- a/modules/axhal/src/platform/aarch64_common/gic.rs +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -42,13 +42,24 @@ pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { crate::irq::register_handler_common(irq_num, handler) } +/// Fetches the IRQ number. +pub fn fetch_irq() -> usize { + GICC.iar() as usize +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. -pub fn dispatch_irq(_unused: usize) { - GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); +pub fn dispatch_irq(irq_no: usize) { + if irq_no == 0 { + GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); + } else { + crate::irq::dispatch_irq_common(irq_no as _); + GICC.eoi(irq_no as _); + GICC.dir(irq_no as _); + } } /// Initializes GICD, GICC on the primary CPU. @@ -63,141 +74,3 @@ pub(crate) fn init_primary() { pub(crate) fn init_secondary() { GICC.init(); } - -#[cfg(feature = "hv")] -mod gic_if { - - use super::{GICC, GICD}; - use arm_gicv2::GicTrait; - struct GicIfImpl; - - #[crate_interface::impl_interface] - impl GicTrait for GicIfImpl { - /// Sets the enable status of a specific interrupt in the GICD (Distributor). - /// - /// # Parameters - /// - `vector`: The interrupt vector number, identifying the interrupt to be enabled or disabled. - /// - `enable`: A boolean value indicating whether to enable the interrupt. `true` enables the interrupt, `false` disables it. - /// - /// This function locks and accesses the GICD controller, then sets the enable status of the specified interrupt vector based on the `enable` parameter. - /// It provides a mechanism for controlling whether interrupts can trigger CPU responses, used for interrupt management. - fn set_enable(vector: usize, enable: bool) { - GICD.lock().set_enable(vector, enable); - } - - /// Retrieves the enable status of a specified interrupt vector from the GICD. - /// - /// # Parameters - /// - `vector`: The index of the interrupt vector, used to identify a specific interrupt source. - /// - /// # Returns - /// - `bool`: Indicates whether the specified interrupt vector is enabled. `true` means the interrupt vector is enabled, `false` means it is not enabled. - fn get_enable(vector: usize) -> bool { - GICD.lock().get_enable(vector) - } - - /// Get the type of the GICD register - /// - /// This function locks the GICD and calls its internal `get_typer` method to retrieve the type of the GICD register - /// - /// # Returns - /// * `u32` - The type of the GICD register - fn get_typer() -> u32 { - GICD.lock().get_typer() - } - - /// Get the Implementer ID Register (IIDR) of the interrupt controller - /// - /// This function locks the GICD (interrupt controller) and calls its `get_iidr` method to retrieve the value of the Implementer ID Register. - /// This register can be used to identify the specific hardware implementer and version. - fn get_iidr() -> u32 { - GICD.lock().get_iidr() - } - - /// Set the state of an interrupt source - /// - /// This function updates the state of a specific interrupt source in the GICD (Interrupt Controller). - /// It first locks the GICD and then updates the interrupt source state using the provided interrupt ID (`int_id`), - /// new state value (`state`), and current CPU ID (`current_cpu_id`). - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `state`: The new state value for the interrupt source. - /// - `current_cpu_id`: The ID of the current CPU. - fn set_state(int_id: usize, state: usize, current_cpu_id: usize) { - GICD.lock().set_state(int_id, state, current_cpu_id); - } - - /// Get the state of an interrupt source - /// - /// This function retrieves the current state of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The current state value. - fn get_state(int_id: usize) -> usize { - GICD.lock().get_state(int_id) - } - - /// Set the ICFGR (Interrupt Configuration and Control Register) - /// - /// This function sets the configuration of a specific interrupt source in the ICFGR. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `cfg`: The new configuration value. - fn set_icfgr(int_id: usize, cfg: u8) { - GICD.lock().set_icfgr(int_id, cfg); - } - - /// Get the target CPU for an interrupt source - /// - /// This function retrieves the target CPU for a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The target CPU ID. - fn get_target_cpu(int_id: usize) -> usize { - GICD.lock().get_target_cpu(int_id) - } - - /// Set the target CPU for an interrupt source - /// - /// This function sets the target CPU for a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `target`: The new target CPU value. - fn set_target_cpu(int_id: usize, target: u8) { - GICD.lock().set_target_cpu(int_id, target); - } - - /// Get the priority of an interrupt source - /// - /// This function retrieves the priority of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - /// Returns: - /// - The priority value. - fn get_priority(int_id: usize) -> usize { - GICD.lock().get_priority(int_id) - } - - /// Set the priority of an interrupt source - /// - /// This function sets the priority of a specific interrupt source. - /// - /// Parameters: - /// - `int_id`: The ID of the interrupt source. - /// - `priority`: The new priority value. - fn set_priority(int_id: usize, priority: u8) { - GICD.lock().set_priority(int_id, priority); - } - } -} diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index 1d50d10d0b..4bb16d8450 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -42,6 +42,7 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { #[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { crate::arch::set_exception_vector_base(exception_vector_base as usize); + #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_secondary(cpu_id); rust_main_secondary(cpu_id); diff --git a/modules/axhal/src/platform/dummy/mod.rs b/modules/axhal/src/platform/dummy/mod.rs index e0d229ff3a..616ca47e7d 100644 --- a/modules/axhal/src/platform/dummy/mod.rs +++ b/modules/axhal/src/platform/dummy/mod.rs @@ -83,6 +83,11 @@ pub mod irq { /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. pub fn dispatch_irq(irq_num: usize) {} + + /// Fetches the IRQ number. + pub fn fetch_irq() -> usize { + 0 + } } /// Initializes the platform devices for the primary CPU. From 46c3a88e6edd86f06ac0f58080716a0d37346e4c Mon Sep 17 00:00:00 2001 From: hky1999 Date: Tue, 17 Dec 2024 19:51:13 +0800 Subject: [PATCH 05/19] [feat] update allocator version to support alloc_pages_at (#34) * [feat] update allocator version to support alloc_pages_at * [feat] set phys-virt-offset as 0xffff_8000_0000_0000 for x86_64 qemu --- Cargo.toml | 2 ++ modules/axalloc/src/lib.rs | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 2924a174af..87e7f85b09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,5 +63,7 @@ axsync = { path = "modules/axsync" } axtask = { path = "modules/axtask" } axdma = { path = "modules/axdma" } +allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } + [profile.release] lto = true diff --git a/modules/axalloc/src/lib.rs b/modules/axalloc/src/lib.rs index ef6f0f6c94..c9a2db6dea 100644 --- a/modules/axalloc/src/lib.rs +++ b/modules/axalloc/src/lib.rs @@ -146,6 +146,24 @@ impl GlobalAllocator { self.palloc.lock().alloc_pages(num_pages, align_pow2) } + /// Allocates contiguous pages starting from the given address. + /// + /// It allocates `num_pages` pages from the page allocator starting from the + /// given address. + /// + /// `align_pow2` must be a power of 2, and the returned region bound will be + /// aligned to it. + pub fn alloc_pages_at( + &self, + start: usize, + num_pages: usize, + align_pow2: usize, + ) -> AllocResult { + self.palloc + .lock() + .alloc_pages_at(start, num_pages, align_pow2) + } + /// Gives back the allocated pages starts from `pos` to the page allocator. /// /// The pages should be allocated by [`alloc_pages`], and `align_pow2` From 1ce8e38555ce834385ccc615da78029cbb9781f0 Mon Sep 17 00:00:00 2001 From: bhxh <32200913+buhenxihuan@users.noreply.github.com> Date: Tue, 31 Dec 2024 16:12:23 +0800 Subject: [PATCH 06/19] adapt hypervisor for a1000b and fix the bug of smp startup on a1000 (#37) * [FIX] Fix the bug of smp startup on a1000 * [feat] adapt hypervisor for a1000b --- .../src/platform/aarch64_bsta1000b/mem.rs | 31 +++++++++++++++++-- .../src/platform/aarch64_bsta1000b/mod.rs | 19 ++++++++++-- .../src/platform/aarch64_common/boot_el2.rs | 4 ++- .../axhal/src/platform/aarch64_rk3588j/mem.rs | 2 +- 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs index 63b0349b91..f122e28f77 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mem.rs @@ -1,9 +1,21 @@ -use crate::mem::MemRegion; +use crate::mem::{MemRegion, MemRegionFlags}; use page_table_entry::{GenericPTE, MappingFlags, aarch64::A64PTE}; +/// Returns (a1000b only) memory regions. +pub(crate) fn default_a1000b_regions() -> impl Iterator { + [MemRegion { + paddr: pa!(0x80000000), + size: 0x70000000, + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "reserved memory", + }] + .into_iter() +} /// Returns platform-specific memory regions. pub(crate) fn platform_regions() -> impl Iterator { - crate::mem::default_free_regions().chain(crate::mem::default_mmio_regions()) + crate::mem::default_free_regions() + .chain(default_a1000b_regions()) + .chain(crate::mem::default_mmio_regions()) } pub(crate) unsafe fn init_boot_page_table( @@ -32,4 +44,19 @@ pub(crate) unsafe fn init_boot_page_table( MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, true, ); + boot_pt_l1[3] = A64PTE::new_page( + pa!(0xc0000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[6] = A64PTE::new_page( + pa!(0x180000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); + boot_pt_l1[7] = A64PTE::new_page( + pa!(0x1C0000000), + MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE, + true, + ); } diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs index cc9cbed108..ad0d0df152 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs @@ -19,7 +19,10 @@ pub mod time { pub use crate::platform::aarch64_common::generic_timer::*; } -unsafe extern "C" { +use crate::mp::CPU_HWID; +use crate::mp::MAX_HARTS; + +extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] @@ -36,7 +39,19 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { } #[cfg(feature = "smp")] -pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { +pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_hwid: usize) { + let mut cpu_id = cpu_hwid; + let mut map_success = false; + for index in 0..MAX_HARTS { + if cpu_id == CPU_HWID[index] { + cpu_id = index; + map_success = true; + break; + } + } + if !map_success { + panic!("CPU{} not found", cpu_id); + } crate::arch::set_exception_vector_base(exception_vector_base as usize); crate::cpu::init_secondary(cpu_id); rust_main_secondary(cpu_id); diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index fd15e6ac36..25045b1920 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -132,6 +132,9 @@ unsafe extern "C" fn _start() -> ! { // PC = 0x8_0000 // X0 = dtb core::arch::asm!(" + // save DTB pointer + mov x20, x0 + // disable cache and MMU mrs x1, sctlr_el2 bic x1, x1, #0xf @@ -145,7 +148,6 @@ unsafe extern "C" fn _start() -> ! { mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id - mov x20, x0 // save DTB pointer adrp x8, {boot_stack} // setup boot stack add x8, x8, {boot_stack_size} diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs index d7cf3415ee..4c0a970aba 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -8,7 +8,7 @@ pub(crate) fn default_rk3588j_regions() -> impl Iterator { paddr: PhysAddr::from(0x9400000), size: 0xe6c00000, flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::WRITE, - name: "free memory", + name: "reserved memory", }, MemRegion { paddr: PhysAddr::from(0x1f0000000), From aa2dacf8038cfdbc1ce98b43f067c9865f08d44a Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 1 Jan 2025 19:34:43 +0800 Subject: [PATCH 07/19] Rebase to arceos-org/arceos main --- .../platforms}/aarch64-rk3588j.toml | 0 configs/platforms/x86_64-qemu-q35.toml | 1 + .../src/platform/aarch64_bsta1000b/mod.rs | 2 +- .../axhal/src/platform/aarch64_common/boot.rs | 2 +- .../src/platform/aarch64_common/boot_el2.rs | 60 ++++++++++--------- .../platform/aarch64_rk3588j/dw_apb_uart.rs | 24 +++++++- .../axhal/src/platform/aarch64_rk3588j/mem.rs | 2 +- .../axhal/src/platform/aarch64_rk3588j/mod.rs | 2 +- .../axhal/src/platform/aarch64_rk3588j/mp.rs | 4 +- 9 files changed, 61 insertions(+), 36 deletions(-) rename {platforms => configs/platforms}/aarch64-rk3588j.toml (100%) diff --git a/platforms/aarch64-rk3588j.toml b/configs/platforms/aarch64-rk3588j.toml similarity index 100% rename from platforms/aarch64-rk3588j.toml rename to configs/platforms/aarch64-rk3588j.toml diff --git a/configs/platforms/x86_64-qemu-q35.toml b/configs/platforms/x86_64-qemu-q35.toml index a44e98cf93..b68a10873a 100644 --- a/configs/platforms/x86_64-qemu-q35.toml +++ b/configs/platforms/x86_64-qemu-q35.toml @@ -40,6 +40,7 @@ mmio-regions = [ [0xfec0_0000, 0x1000], # IO APIC [0xfed0_0000, 0x1000], # HPET [0xfee0_0000, 0x1000], # Local APIC + [0x380000000000, 0x4000] # PCI devices ] # [(uint, uint)] # VirtIO MMIO regions with format (`base_paddr`, `size`). virtio-mmio-regions = [] # [(uint, uint)] diff --git a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs index ad0d0df152..8f1a582580 100644 --- a/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs +++ b/modules/axhal/src/platform/aarch64_bsta1000b/mod.rs @@ -22,7 +22,7 @@ pub mod time { use crate::mp::CPU_HWID; use crate::mp::MAX_HARTS; -extern "C" { +unsafe extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] diff --git a/modules/axhal/src/platform/aarch64_common/boot.rs b/modules/axhal/src/platform/aarch64_common/boot.rs index 1140c17b67..31a5e222db 100644 --- a/modules/axhal/src/platform/aarch64_common/boot.rs +++ b/modules/axhal/src/platform/aarch64_common/boot.rs @@ -1,6 +1,6 @@ use aarch64_cpu::{asm, asm::barrier, registers::*}; use core::ptr::addr_of_mut; -use page_table_entry::aarch64::{MemAttr, A64PTE}; +use page_table_entry::aarch64::{A64PTE, MemAttr}; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index 25045b1920..ca06e24e0d 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -7,13 +7,13 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use axconfig::TASK_STACK_SIZE; -#[link_section = ".bss.stack"] +#[unsafe(link_section = ".bss.stack")] static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L0: [A64PTE; 512] = [A64PTE::empty(); 512]; -#[link_section = ".data.boot_page_table"] +#[unsafe(link_section = ".data.boot_page_table")] static mut BOOT_PT_L1: [A64PTE; 512] = [A64PTE::empty(); 512]; unsafe fn switch_to_el2() { @@ -126,12 +126,13 @@ unsafe fn cache_invalidate(cache_level: usize) { /// The earliest entry point for the primary CPU. #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start() -> ! { - // PC = 0x8_0000 - // X0 = dtb - core::arch::asm!(" + unsafe { + // PC = 0x8_0000 + // X0 = dtb + core::arch::naked_asm!(" // save DTB pointer mov x20, x0 @@ -166,26 +167,27 @@ unsafe extern "C" fn _start() -> ! { ldr x8, ={entry} blr x8 b .", - cache_invalidate = sym cache_invalidate, - init_boot_page_table = sym init_boot_page_table, - init_mmu_el2 = sym init_mmu_el2, - switch_to_el2 = sym switch_to_el2, - enable_fp = sym enable_fp, - boot_stack = sym BOOT_STACK, - boot_stack_size = const TASK_STACK_SIZE, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry, - options(noreturn), - ); + cache_invalidate = sym cache_invalidate, + init_boot_page_table = sym init_boot_page_table, + init_mmu_el2 = sym init_mmu_el2, + switch_to_el2 = sym switch_to_el2, + enable_fp = sym enable_fp, + boot_stack = sym BOOT_STACK, + boot_stack_size = const TASK_STACK_SIZE, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry, + ); + } } /// The earliest entry point for the secondary CPUs. #[cfg(feature = "smp")] #[naked] -#[no_mangle] -#[link_section = ".text.boot"] +#[unsafe(no_mangle)] +#[unsafe(link_section = ".text.boot")] unsafe extern "C" fn _start_secondary() -> ! { - core::arch::asm!(" + unsafe { + core::arch::naked_asm!(" mrs x19, mpidr_el1 and x19, x19, #0xffffff // get current CPU id @@ -201,11 +203,11 @@ unsafe extern "C" fn _start_secondary() -> ! { ldr x8, ={entry} blr x8 b .", - switch_to_el2 = sym switch_to_el2, - init_mmu_el2 = sym init_mmu_el2, - enable_fp = sym enable_fp, - phys_virt_offset = const axconfig::PHYS_VIRT_OFFSET, - entry = sym crate::platform::rust_entry_secondary, - options(noreturn), - ) + switch_to_el2 = sym switch_to_el2, + init_mmu_el2 = sym init_mmu_el2, + enable_fp = sym enable_fp, + phys_virt_offset = const axconfig::plat::PHYS_VIRT_OFFSET, + entry = sym crate::platform::rust_entry_secondary, + ) + } } diff --git a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs index 6d9de293be..5d34bc9b0f 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/dw_apb_uart.rs @@ -119,11 +119,33 @@ impl DW8250 { } } -const UART_BASE: PhysAddr = pa!(axconfig::UART_PADDR); +const UART_BASE: PhysAddr = pa!(axconfig::devices::UART_PADDR); pub static UART: SpinNoIrq = SpinNoIrq::new(DW8250::new(phys_to_virt(UART_BASE).as_usize())); +/// Write a slice of bytes to the console. +pub fn write_bytes(bytes: &[u8]) { + for c in bytes { + putchar(*c); + } +} + +/// Reads bytes from the console into the given mutable slice. +/// Returns the number of bytes read. +pub fn read_bytes(bytes: &mut [u8]) -> usize { + let mut read_len = 0; + while read_len < bytes.len() { + if let Some(c) = getchar() { + bytes[read_len] = c; + } else { + break; + } + read_len += 1; + } + read_len +} + /// Writes a byte to the console. pub fn putchar(c: u8) { let mut uart = UART.lock(); diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs index 4c0a970aba..5c2fd2d609 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mem.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mem.rs @@ -1,5 +1,5 @@ use crate::mem::{MemRegion, MemRegionFlags, PhysAddr}; -use page_table_entry::{aarch64::A64PTE, GenericPTE, MappingFlags}; +use page_table_entry::{GenericPTE, MappingFlags, aarch64::A64PTE}; /// Returns (rk3588j only) memory regions. pub(crate) fn default_rk3588j_regions() -> impl Iterator { diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs index c59047bd78..873cde06d3 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mod.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mod.rs @@ -21,7 +21,7 @@ pub mod misc { pub use crate::platform::aarch64_common::psci::system_off as terminate; } -extern "C" { +unsafe extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] diff --git a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs index 01f84fc4c1..70d8cebecb 100644 --- a/modules/axhal/src/platform/aarch64_rk3588j/mp.rs +++ b/modules/axhal/src/platform/aarch64_rk3588j/mp.rs @@ -1,4 +1,4 @@ -use crate::mem::{virt_to_phys, PhysAddr, VirtAddr}; +use crate::mem::{PhysAddr, VirtAddr, virt_to_phys}; /// Hart number of rk3588 board pub const MAX_HARTS: usize = 8; @@ -8,7 +8,7 @@ pub const CPU_HWID: [usize; MAX_HARTS] = [0x00, 0x100, 0x200, 0x300, 0x400, 0x50 /// Starts the given secondary CPU with its boot stack. pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) { assert!(cpu_id < MAX_HARTS, "No support for rk3588 core {}", cpu_id); - extern "C" { + unsafe extern "C" { fn _start_secondary(); } let entry = virt_to_phys(VirtAddr::from(_start_secondary as usize)); From 39848356f3bb43c1c0a5b02a6188bd3235e789a5 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 1 Jan 2025 22:36:01 +0800 Subject: [PATCH 08/19] [fix] clippy error in aarch64 boot_el2 --- modules/axhal/src/platform/aarch64_common/boot_el2.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index ca06e24e0d..931a00e991 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -67,7 +67,7 @@ unsafe fn init_mmu_el2() { TCR_EL2.write(TCR_EL2::PS::Bits_40 + tcr_flags0); barrier::isb(barrier::SY); - let root_paddr = PhysAddr::from(BOOT_PT_L0.as_ptr() as usize).as_usize() as _; + let root_paddr = PhysAddr::from(&raw const BOOT_PT_L0 as usize).as_usize() as _; TTBR0_EL2.set(root_paddr); // Flush the entire TLB From adf6a454bdb1b4f32935f5f099a3f76f5349129a Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 11 Mar 2025 16:30:34 +0800 Subject: [PATCH 09/19] [wip] add platform x86_64-qemu-linux, do compile --- Cargo.lock | 2 + Makefile | 3 +- configs/platforms/x86_64-qemu-linux.toml | 48 +++++ modules/axhal/Cargo.toml | 1 + modules/axhal/build.rs | 6 +- modules/axhal/linker_linux.lds.S | 101 ++++++++++ modules/axhal/src/lib.rs | 3 + modules/axhal/src/platform/mod.rs | 6 +- .../axhal/src/platform/x86_linux/ap_start.S | 70 +++++++ modules/axhal/src/platform/x86_linux/apic.rs | 127 ++++++++++++ modules/axhal/src/platform/x86_linux/boot.rs | 52 +++++ .../axhal/src/platform/x86_linux/config.rs | 118 ++++++++++++ .../axhal/src/platform/x86_linux/consts.rs | 42 ++++ .../axhal/src/platform/x86_linux/dtables.rs | 39 ++++ modules/axhal/src/platform/x86_linux/entry.rs | 69 +++++++ .../axhal/src/platform/x86_linux/header.rs | 77 ++++++++ modules/axhal/src/platform/x86_linux/lapic.rs | 117 ++++++++++++ modules/axhal/src/platform/x86_linux/mem.rs | 86 +++++++++ modules/axhal/src/platform/x86_linux/misc.rs | 20 ++ modules/axhal/src/platform/x86_linux/mod.rs | 180 ++++++++++++++++++ modules/axhal/src/platform/x86_linux/mp.rs | 94 +++++++++ .../axhal/src/platform/x86_linux/multiboot.S | 147 ++++++++++++++ .../axhal/src/platform/x86_linux/percpu.rs | 146 ++++++++++++++ modules/axhal/src/platform/x86_linux/time.rs | 105 ++++++++++ .../axhal/src/platform/x86_linux/uart16550.rs | 127 ++++++++++++ modules/axruntime/Cargo.toml | 1 + modules/axruntime/src/lib.rs | 4 + modules/axruntime/src/vmm.rs | 166 ++++++++++++++++ scripts/make/build.mk | 3 + scripts/vmm/guest/disable-arceos-vmm.sh | 6 + scripts/vmm/guest/enable-arceos-vmm.sh | 10 + scripts/vmm/guest/setup.sh | 25 +++ scripts/vmm/guest/test_hypercall.c | 40 ++++ scripts/vmm/guest/update-cmdline.sh | 4 + scripts/vmm/host/.gitignore | 2 + scripts/vmm/host/Makefile | 58 ++++++ scripts/vmm/scp.mk | 6 + 37 files changed, 2108 insertions(+), 3 deletions(-) create mode 100644 configs/platforms/x86_64-qemu-linux.toml create mode 100644 modules/axhal/linker_linux.lds.S create mode 100644 modules/axhal/src/platform/x86_linux/ap_start.S create mode 100644 modules/axhal/src/platform/x86_linux/apic.rs create mode 100644 modules/axhal/src/platform/x86_linux/boot.rs create mode 100644 modules/axhal/src/platform/x86_linux/config.rs create mode 100644 modules/axhal/src/platform/x86_linux/consts.rs create mode 100644 modules/axhal/src/platform/x86_linux/dtables.rs create mode 100644 modules/axhal/src/platform/x86_linux/entry.rs create mode 100644 modules/axhal/src/platform/x86_linux/header.rs create mode 100644 modules/axhal/src/platform/x86_linux/lapic.rs create mode 100644 modules/axhal/src/platform/x86_linux/mem.rs create mode 100644 modules/axhal/src/platform/x86_linux/misc.rs create mode 100644 modules/axhal/src/platform/x86_linux/mod.rs create mode 100644 modules/axhal/src/platform/x86_linux/mp.rs create mode 100644 modules/axhal/src/platform/x86_linux/multiboot.S create mode 100644 modules/axhal/src/platform/x86_linux/percpu.rs create mode 100644 modules/axhal/src/platform/x86_linux/time.rs create mode 100644 modules/axhal/src/platform/x86_linux/uart16550.rs create mode 100644 modules/axruntime/src/vmm.rs create mode 100755 scripts/vmm/guest/disable-arceos-vmm.sh create mode 100755 scripts/vmm/guest/enable-arceos-vmm.sh create mode 100755 scripts/vmm/guest/setup.sh create mode 100644 scripts/vmm/guest/test_hypercall.c create mode 100755 scripts/vmm/guest/update-cmdline.sh create mode 100644 scripts/vmm/host/.gitignore create mode 100644 scripts/vmm/host/Makefile create mode 100644 scripts/vmm/scp.mk diff --git a/Cargo.lock b/Cargo.lock index bb680752a9..6ef2802b37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -463,6 +463,7 @@ dependencies = [ "riscv", "riscv_goldfish", "sbi-rt", + "spin", "static_assertions", "tock-registers", "x2apic", @@ -551,6 +552,7 @@ dependencies = [ "chrono", "crate_interface", "kernel_guard", + "lazyinit", "percpu", ] diff --git a/Makefile b/Makefile index c30c887f9d..322945da56 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,7 @@ APP_NAME := $(shell basename $(APP)) LD_SCRIPT := $(TARGET_DIR)/$(TARGET)/$(MODE)/linker_$(PLAT_NAME).lds OUT_ELF := $(OUT_DIR)/$(APP_NAME)_$(PLAT_NAME).elf OUT_BIN := $(OUT_DIR)/$(APP_NAME)_$(PLAT_NAME).bin +OUT_ASM := $(OUT_DIR)/$(APP_NAME)_$(PLAT_NAME).asm all: build @@ -152,7 +153,7 @@ defconfig: _axconfig-gen oldconfig: _axconfig-gen $(call oldconfig) -build: $(OUT_DIR) $(OUT_BIN) +build: $(OUT_DIR) $(OUT_BIN) $(OUT_ASM) disasm: $(OBJDUMP) $(OUT_ELF) | less diff --git a/configs/platforms/x86_64-qemu-linux.toml b/configs/platforms/x86_64-qemu-linux.toml new file mode 100644 index 0000000000..4abca925dc --- /dev/null +++ b/configs/platforms/x86_64-qemu-linux.toml @@ -0,0 +1,48 @@ +# Architecture identifier. +arch = "x86_64" # str +# Platform identifier. +platform = "x86_64-qemu-linux" # str + +# +# Platform configs +# +[plat] +# Platform family. +family = "x86-linux" # str + +# Base address of the whole physical memory. +phys-memory-base = "0x4000_0000" # uint +# Size of the whole physical memory. (# 1G) +phys-memory-size = "0x4000_0000" # uint +# Base physical address of the kernel image. +kernel-base-paddr = "0x4000_0000" # uint +# Base virtual address of the kernel image. +kernel-base-vaddr = "0xffff_ff00_0000_0000" # uint +# Linear mapping offset, for quick conversions between physical and virtual +# addresses. (# kernel-base-vaddr - phys-memory-base) +phys-virt-offset = "0xffff_feff_c000_0000" # uint + +# +# Device specifications +# +[devices] +# MMIO regions with format (`base_paddr`, `size`). +mmio-regions = [ + [0xb000_0000, 0x1000_0000], # PCI config space + [0xfe00_0000, 0xc0_0000], # PCI devices + [0xfec0_0000, 0x1000], # IO APIC + [0xfed0_0000, 0x1000], # HPET + [0xfee0_0000, 0x1000], # Local APIC + [0x380000000000, 0x4000] # PCI devices +] # [(uint, uint)] +# VirtIO MMIO regions with format (`base_paddr`, `size`). +virtio-mmio-regions = [] # [(uint, uint)] +# Base physical address of the PCIe ECAM space (should read from ACPI 'MCFG' table). +pci-ecam-base = 0xb000_0000 # uint +# End PCI bus number. +pci-bus-end = 0xff # uint +# PCI device memory ranges (not used on x86). +pci-ranges = [] # [(uint, uint)] + +# Timer interrupt frequencyin Hz. (4.0GHz) +timer-frequency = 4_000_000_000 # uint diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 39a01bb433..7e60928188 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -39,6 +39,7 @@ axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } cortex-a = { version = "8.1.1", optional = true } +spin = "0.9" [target.'cfg(target_arch = "x86_64")'.dependencies] x86 = "0.52" diff --git a/modules/axhal/build.rs b/modules/axhal/build.rs index f2804abd18..774cb2c974 100644 --- a/modules/axhal/build.rs +++ b/modules/axhal/build.rs @@ -60,7 +60,11 @@ fn gen_linker_script(arch: &str, platform: &str) -> Result<()> { } else { arch }; - let ld_content = std::fs::read_to_string("linker.lds.S")?; + let ld_content = if platform.contains("linux") { + std::fs::read_to_string("linker_linux.lds.S") + } else { + std::fs::read_to_string("linker.lds.S") + }?; let ld_content = ld_content.replace("%ARCH%", output_arch); let ld_content = ld_content.replace( "%KERNEL_BASE%", diff --git a/modules/axhal/linker_linux.lds.S b/modules/axhal/linker_linux.lds.S new file mode 100644 index 0000000000..c3daa62abc --- /dev/null +++ b/modules/axhal/linker_linux.lds.S @@ -0,0 +1,101 @@ +OUTPUT_ARCH(%ARCH%) + +BASE_ADDRESS = %KERNEL_BASE%; + +ENTRY(_start) +SECTIONS +{ + . = BASE_ADDRESS; + _skernel = .; + + .header : { + __header_start = .; + KEEP(*(.header)) + . = ALIGN(4K); + __header_end = .; + } + + .text : ALIGN(4K) { + _stext = .; + *(.text.boot) + *(.text .text.*) + . = ALIGN(4K); + _etext = .; + } + + .rodata : ALIGN(4K) { + _srodata = .; + *(.rodata .rodata.*) + *(.srodata .srodata.*) + *(.sdata2 .sdata2.*) + . = ALIGN(4K); + _erodata = .; + } + + .data : ALIGN(4K) { + _sdata = .; + *(.data.boot_page_table) + . = ALIGN(4K); + *(.data .data.*) + *(.sdata .sdata.*) + *(.got .got.*) + } + + .tdata : ALIGN(0x10) { + _stdata = .; + *(.tdata .tdata.*) + _etdata = .; + } + + .tbss : ALIGN(0x10) { + _stbss = .; + *(.tbss .tbss.*) + *(.tcommon) + _etbss = .; + } + + . = ALIGN(4K); + _percpu_start = .; + _percpu_end = _percpu_start + SIZEOF(.percpu); + .percpu 0x0 : AT(_percpu_start) { + _percpu_load_start = .; + *(.percpu .percpu.*) + _percpu_load_end = .; + . = _percpu_load_start + ALIGN(64) * %SMP%; + } + . = _percpu_end; + + . = ALIGN(4K); + _edata = .; + + .bss : ALIGN(4K) { + boot_stack = .; + *(.bss.stack) + . = ALIGN(4K); + boot_stack_top = .; + + _sbss = .; + *(.bss .bss.*) + *(.sbss .sbss.*) + *(COMMON) + . = ALIGN(4K); + _ebss = .; + } + + _ekernel = .; + + __entry_offset = _start - BASE_ADDRESS; + __kernel_size = _ekernel - BASE_ADDRESS; + + /DISCARD/ : { + *(.comment) *(.gnu*) *(.note*) *(.eh_frame*) + } +} + +SECTIONS { + linkme_IRQ : { *(linkme_IRQ) } + linkm2_IRQ : { *(linkm2_IRQ) } + linkme_PAGE_FAULT : { *(linkme_PAGE_FAULT) } + linkm2_PAGE_FAULT : { *(linkm2_PAGE_FAULT) } +} +INSERT AFTER .tbss; diff --git a/modules/axhal/src/lib.rs b/modules/axhal/src/lib.rs index 87d61b0747..caf4ca51a4 100644 --- a/modules/axhal/src/lib.rs +++ b/modules/axhal/src/lib.rs @@ -76,3 +76,6 @@ pub use self::platform::platform_init; #[cfg(feature = "smp")] pub use self::platform::platform_init_secondary; + +pub use self::platform::mem::host_memory_regions; +pub use self::platform::vmm_platform_init; diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 57431a0e05..d6a652e715 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -28,7 +28,11 @@ cfg_if::cfg_if! { } else if #[cfg(all(target_arch = "aarch64", platform_family = "aarch64-rk3588j"))] { mod aarch64_rk3588j; pub use self::aarch64_rk3588j::*; - } else { + // } else if #[cfg(all(target_arch = "x86_64", platform_family = "x86-linux"))] { + } else if #[cfg(all(target_arch = "x86_64"))] { + mod x86_linux; + pub use self::x86_linux::*; + }else { mod dummy; pub use self::dummy::*; } diff --git a/modules/axhal/src/platform/x86_linux/ap_start.S b/modules/axhal/src/platform/x86_linux/ap_start.S new file mode 100644 index 0000000000..0b53a6cd6e --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/ap_start.S @@ -0,0 +1,70 @@ +# Boot application processors into the protected mode. + +# Each non-boot CPU ("AP") is started up in response to a STARTUP +# IPI from the boot CPU. Section B.4.2 of the Multi-Processor +# Specification says that the AP will start in real mode with CS:IP +# set to XY00:0000, where XY is an 8-bit value sent with the +# STARTUP. Thus this code must start at a 4096-byte boundary. +# +# Because this code sets DS to zero, it must sit +# at an address in the low 2^16 bytes. + +.equ pa_ap_start32, ap_start32 - ap_start + {start_page_paddr} +.equ pa_ap_gdt, .Lap_tmp_gdt - ap_start + {start_page_paddr} +.equ pa_ap_gdt_desc, .Lap_tmp_gdt_desc - ap_start + {start_page_paddr} + +.equ stack_ptr, {start_page_paddr} + 0xff0 +.equ entry_ptr, {start_page_paddr} + 0xff8 + +.global ap_start +.global ap_end + +# 0x6000 +.section .text +.code16 +.p2align 12 +ap_start: + cli + wbinvd + + xor ax, ax + mov ds, ax + mov es, ax + mov ss, ax + mov fs, ax + mov gs, ax + + # load the 64-bit GDT + lgdt [pa_ap_gdt_desc] + + # switch to protected-mode + mov eax, cr0 + or eax, (1 << 0) + mov cr0, eax + + # far jump to 32-bit code. 0x8 is code32 segment selector + ljmp 0x8, offset pa_ap_start32 + +.code32 +ap_start32: + mov esp, [stack_ptr] + mov eax, [entry_ptr] + jmp eax + +.balign 8 +# .type multiboot_header, STT_OBJECT +.Lap_tmp_gdt_desc: + .short .Lap_tmp_gdt_end - .Lap_tmp_gdt - 1 # limit + .long pa_ap_gdt # base + +.balign 16 +.Lap_tmp_gdt: + .quad 0x0000000000000000 # 0x00: null + .quad 0x00cf9b000000ffff # 0x08: code segment (base=0, limit=0xfffff, type=32bit code exec/read, DPL=0, 4k) + .quad 0x00af9b000000ffff # 0x10: code segment (base=0, limit=0xfffff, type=64bit code exec/read, DPL=0, 4k) + .quad 0x00cf93000000ffff # 0x18: data segment (base=0, limit=0xfffff, type=32bit data read/write, DPL=0, 4k) +.Lap_tmp_gdt_end: + +# 0x7000 +.p2align 12 +ap_end: diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs new file mode 100644 index 0000000000..3a6f93ad7a --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -0,0 +1,127 @@ +#![allow(dead_code)] + +use core::{cell::SyncUnsafeCell, mem::MaybeUninit}; + +use kspin::SpinNoIrq; +use lazyinit::LazyInit; +use memory_addr::PhysAddr; +use x2apic::ioapic::IoApic; +use x2apic::lapic::{LocalApic, LocalApicBuilder, xapic_base}; +use x86_64::instructions::port::Port; + +use self::vectors::*; +use crate::mem::phys_to_virt; + +pub(super) mod vectors { + pub const APIC_TIMER_VECTOR: u8 = 0xf0; + pub const APIC_SPURIOUS_VECTOR: u8 = 0xf1; + pub const APIC_ERROR_VECTOR: u8 = 0xf2; +} + +/// The maximum number of IRQs. +pub const MAX_IRQ_COUNT: usize = 256; + +/// The timer IRQ number. +pub const TIMER_IRQ_NUM: usize = APIC_TIMER_VECTOR as usize; + +const IO_APIC_BASE: PhysAddr = pa!(0xFEC0_0000); + +static LOCAL_APIC: SyncUnsafeCell> = + SyncUnsafeCell::new(MaybeUninit::uninit()); +static mut IS_X2APIC: bool = false; +static IO_APIC: LazyInit> = LazyInit::new(); + +/// Enables or disables the given IRQ. +#[cfg(feature = "irq")] +pub fn set_enable(vector: usize, enabled: bool) { + // should not affect LAPIC interrupts + if vector < APIC_TIMER_VECTOR as _ { + unsafe { + if enabled { + IO_APIC.lock().enable_irq(vector as u8); + } else { + IO_APIC.lock().disable_irq(vector as u8); + } + } + } +} + +/// Registers an IRQ handler for the given IRQ. +/// +/// It also enables the IRQ if the registration succeeds. It returns `false` if +/// the registration failed. +#[cfg(feature = "irq")] +pub fn register_handler(vector: usize, handler: crate::irq::IrqHandler) -> bool { + crate::irq::register_handler_common(vector, handler) +} + +/// Dispatches the IRQ. +/// +/// This function is called by the common interrupt handler. It looks +/// up in the IRQ handler table and calls the corresponding handler. If +/// necessary, it also acknowledges the interrupt controller after handling. +#[cfg(feature = "irq")] +pub fn dispatch_irq(vector: usize) { + crate::irq::dispatch_irq_common(vector); + unsafe { local_apic().end_of_interrupt() }; +} + +pub(super) fn local_apic<'a>() -> &'a mut LocalApic { + // It's safe as `LOCAL_APIC` is initialized in `init_primary`. + unsafe { LOCAL_APIC.get().as_mut().unwrap().assume_init_mut() } +} + +pub(super) fn raw_apic_id(id_u8: u8) -> u32 { + if unsafe { IS_X2APIC } { + id_u8 as u32 + } else { + (id_u8 as u32) << 24 + } +} + +fn cpu_has_x2apic() -> bool { + match raw_cpuid::CpuId::new().get_feature_info() { + Some(finfo) => finfo.has_x2apic(), + None => false, + } +} + +pub(super) fn init_primary() { + info!("Initialize Local APIC..."); + + unsafe { + // Disable 8259A interrupt controllers + Port::::new(0x21).write(0xff); + Port::::new(0xA1).write(0xff); + } + + let mut builder = LocalApicBuilder::new(); + builder + .timer_vector(APIC_TIMER_VECTOR as _) + .error_vector(APIC_ERROR_VECTOR as _) + .spurious_vector(APIC_SPURIOUS_VECTOR as _); + + if cpu_has_x2apic() { + info!("Using x2APIC."); + unsafe { IS_X2APIC = true }; + } else { + info!("Using xAPIC."); + let base_vaddr = phys_to_virt(PhysAddr::from(unsafe { xapic_base() } as usize)); + builder.set_xapic_base(base_vaddr.as_usize() as u64); + } + + let mut lapic = builder.build().unwrap(); + unsafe { + lapic.enable(); + LOCAL_APIC.get().as_mut().unwrap().write(lapic); + } + + info!("Initialize IO APIC..."); + let io_apic = unsafe { IoApic::new(phys_to_virt(IO_APIC_BASE).as_usize() as u64) }; + IO_APIC.init_once(SpinNoIrq::new(io_apic)); +} + +#[cfg(feature = "smp")] +pub(super) fn init_secondary() { + unsafe { local_apic().enable() }; +} diff --git a/modules/axhal/src/platform/x86_linux/boot.rs b/modules/axhal/src/platform/x86_linux/boot.rs new file mode 100644 index 0000000000..8ece1cd754 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/boot.rs @@ -0,0 +1,52 @@ +use core::arch::global_asm; + +use x86_64::registers::control::{Cr0Flags, Cr4Flags}; +use x86_64::registers::model_specific::EferFlags; + +use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; + +/// Flags set in the ’flags’ member of the multiboot header. +/// +/// (bits 1, 16: memory information, address fields in header) +const MULTIBOOT_HEADER_FLAGS: usize = 0x0001_0002; + +/// The magic field should contain this. +const MULTIBOOT_HEADER_MAGIC: usize = 0x1BADB002; + +/// This should be in EAX. +pub(super) const MULTIBOOT_BOOTLOADER_MAGIC: usize = 0x2BADB002; + +const CR0: u64 = Cr0Flags::PROTECTED_MODE_ENABLE.bits() + | Cr0Flags::MONITOR_COPROCESSOR.bits() + | Cr0Flags::NUMERIC_ERROR.bits() + | Cr0Flags::WRITE_PROTECT.bits() + | Cr0Flags::PAGING.bits(); +const CR4: u64 = Cr4Flags::PHYSICAL_ADDRESS_EXTENSION.bits() + | Cr4Flags::PAGE_GLOBAL.bits() + | if cfg!(feature = "fp_simd") { + Cr4Flags::OSFXSR.bits() | Cr4Flags::OSXMMEXCPT_ENABLE.bits() + } else { + 0 + }; +const EFER: u64 = EferFlags::LONG_MODE_ENABLE.bits() | EferFlags::NO_EXECUTE_ENABLE.bits(); + +#[unsafe(link_section = ".bss.stack")] +static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; + +global_asm!( + include_str!("multiboot.S"), + mb_magic = const MULTIBOOT_BOOTLOADER_MAGIC, + mb_hdr_magic = const MULTIBOOT_HEADER_MAGIC, + mb_hdr_flags = const MULTIBOOT_HEADER_FLAGS, + entry = sym super::rust_entry, + entry_secondary = sym super::rust_entry_from_vmm, + + offset = const PHYS_VIRT_OFFSET, + boot_stack_size = const TASK_STACK_SIZE, + boot_stack = sym BOOT_STACK, + + cr0 = const CR0, + cr4 = const CR4, + efer_msr = const x86::msr::IA32_EFER, + efer = const EFER, +); diff --git a/modules/axhal/src/platform/x86_linux/config.rs b/modules/axhal/src/platform/x86_linux/config.rs new file mode 100644 index 0000000000..7817701686 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/config.rs @@ -0,0 +1,118 @@ +use core::fmt::{Debug, Formatter, Result}; +use core::{mem::size_of, slice}; + +const CONFIG_SIGNATURE: [u8; 6] = *b"ARCEOS"; +const CONFIG_REVISION: u16 = 13; + +const HV_CELL_NAME_MAXLEN: usize = 31; + +/// The jailhouse cell configuration. +/// +/// @note Keep Config._HEADER_FORMAT in jailhouse-cell-linux in sync with this +/// structure. +#[derive(Debug)] +#[repr(C, packed)] +pub struct HvCellDesc { + signature: [u8; 6], + revision: u16, + name: [u8; HV_CELL_NAME_MAXLEN + 1], + id: u32, // set by the driver + num_memory_regions: u32, +} + +#[derive(Debug)] +#[repr(C, packed)] +pub struct HvMemoryRegion { + pub phys_start: u64, + pub virt_start: u64, + pub size: u64, + pub flags: u64, +} + +/// General descriptor of the system. +#[derive(Debug)] +#[repr(C)] +pub struct HvSystemConfig { + pub signature: [u8; 6], + pub revision: u16, + /// AxVisor location in memory + pub hypervisor_memory: HvMemoryRegion, + /// RTOS location in memory + pub rtos_memory: HvMemoryRegion, + pub root_cell: HvCellDesc, + // CellConfigLayout placed here. +} + +// /// A dummy layout with all variant-size fields empty. +// #[derive(Debug)] +// #[repr(C, packed)] +// struct CellConfigLayout { +// mem_regions: [HvMemoryRegion; 0], +// } + +pub struct CellConfig<'a> { + desc: &'a HvCellDesc, +} + +impl HvCellDesc { + pub const fn config(&self) -> CellConfig { + CellConfig::from(self) + } + + pub const fn config_size(&self) -> usize { + self.num_memory_regions as usize * size_of::() + } +} + +impl HvSystemConfig { + pub fn get<'a>() -> &'a Self { + unsafe { &*super::consts::hv_config_ptr() } + } + + pub const fn size(&self) -> usize { + size_of::() + self.root_cell.config_size() + } + + pub fn check(&self) { + assert_eq!(self.signature, CONFIG_SIGNATURE); + assert_eq!(self.revision, CONFIG_REVISION); + } +} + +impl<'a> CellConfig<'a> { + const fn from(desc: &'a HvCellDesc) -> Self { + Self { desc } + } + + fn config_ptr(&self) -> *const T { + unsafe { (self.desc as *const HvCellDesc).add(1) as _ } + } + + pub const fn size(&self) -> usize { + self.desc.config_size() + } + + pub fn mem_regions(&self) -> &'static [HvMemoryRegion] { + // XXX: data may unaligned, which cause panic on debug mode. Same below. + // See: https://doc.rust-lang.org/src/core/slice/mod.rs.html#6435-6443 + unsafe { + let ptr = self.config_ptr() as _; + slice::from_raw_parts(ptr, self.desc.num_memory_regions as usize) + } + } +} + +impl Debug for CellConfig<'_> { + fn fmt(&self, f: &mut Formatter) -> Result { + let name = self.desc.name; + let mut len = 0; + while name[len] != 0 { + len += 1; + } + f.debug_struct("CellConfig") + .field("name", &core::str::from_utf8(&name[..len])) + .field("size", &self.size()) + .field("mem regions", &self.mem_regions()) + .finish() + } +} diff --git a/modules/axhal/src/platform/x86_linux/consts.rs b/modules/axhal/src/platform/x86_linux/consts.rs new file mode 100644 index 0000000000..5776694253 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/consts.rs @@ -0,0 +1,42 @@ +use memory_addr::MemoryAddr; + +use super::config::HvSystemConfig; +use super::header::HvHeader; +use super::percpu::PerCpu; +use crate::mem::VirtAddr; + +/// Size of the hypervisor heap. +pub const HV_HEAP_SIZE: usize = 32 * 1024 * 1024; // 32 MB + +/// Size of the per-CPU data (stack and other CPU-local data). +pub const PER_CPU_SIZE: usize = 512 * 1024; // 512 KB + +/// Start virtual address of the hypervisor memory. +pub const HV_BASE: usize = 0xffff_ff00_0000_0000; + +/// Pointer of the `HvHeader` structure. +pub const HV_HEADER_PTR: *const HvHeader = __header_start as _; + +/// Pointer of the per-CPU data array. +pub const PER_CPU_ARRAY_PTR: *mut PerCpu = _ekernel as _; + +/// Pointer of the `HvSystemConfig` structure. +pub fn hv_config_ptr() -> *const HvSystemConfig { + (PER_CPU_ARRAY_PTR as usize + HvHeader::get().max_cpus as usize * PER_CPU_SIZE) as _ +} + +/// Pointer of the free memory pool. +pub fn free_memory_start() -> VirtAddr { + VirtAddr::from(hv_config_ptr() as usize + HvSystemConfig::get().size()).align_up_4k() + // align_up(hv_config_ptr() as usize + HvSystemConfig::get().size()) +} + +/// End virtual address of the hypervisor memory. +pub fn hv_end() -> VirtAddr { + VirtAddr::from(HV_BASE + HvSystemConfig::get().hypervisor_memory.size as usize) +} + +unsafe extern "C" { + unsafe fn __header_start(); + unsafe fn _ekernel(); +} diff --git a/modules/axhal/src/platform/x86_linux/dtables.rs b/modules/axhal/src/platform/x86_linux/dtables.rs new file mode 100644 index 0000000000..2352a717e2 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/dtables.rs @@ -0,0 +1,39 @@ +//! Description tables (per-CPU GDT, per-CPU ISS, IDT) + +use lazyinit::LazyInit; + +use crate::arch::{GdtStruct, IdtStruct, TaskStateSegment}; + + +static IDT: LazyInit = LazyInit::new(); + +#[percpu::def_percpu] +static TSS: LazyInit = LazyInit::new(); + +#[percpu::def_percpu] +static GDT: LazyInit = LazyInit::new(); + +fn init_percpu() { + unsafe { + IDT.load(); + let tss = TSS.current_ref_mut_raw(); + let gdt = GDT.current_ref_mut_raw(); + tss.init_once(TaskStateSegment::new()); + gdt.init_once(GdtStruct::new(tss)); + gdt.load(); + gdt.load_tss(); + } +} + +/// Initializes IDT, GDT on the primary CPU. +pub(super) fn init_primary() { + axlog::ax_println!("\nInitialize IDT & GDT..."); + IDT.init_once(IdtStruct::new()); + init_percpu(); +} + +/// Initializes IDT, GDT on secondary CPUs. +#[cfg(feature = "smp")] +pub(super) fn init_secondary() { + init_percpu(); +} diff --git a/modules/axhal/src/platform/x86_linux/entry.rs b/modules/axhal/src/platform/x86_linux/entry.rs new file mode 100644 index 0000000000..eee338a1d0 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/entry.rs @@ -0,0 +1,69 @@ +use super::percpu::PerCpu; + +unsafe extern "sysv64" fn switch_stack(linux_sp: usize) -> i32 { + unsafe { + let linux_cr3 = x86::controlregs::cr3(); + let linux_tp = x86::msr::rdmsr(x86::msr::IA32_GS_BASE); + + let vmm_entry = |linux_sp: usize| -> i32 { + let cpu_data = PerCpu::new(); + let hv_sp = cpu_data.stack_top(); + let ret; + core::arch::asm!(" + mov [rsi], {linux_tp} // save gs_base to stack + mov rcx, rsp + mov rsp, {hv_sp} + push rcx + call {entry} + pop rsp", + entry = sym super::vmm_cpu_entry, + linux_tp = in(reg) linux_tp, + hv_sp = in(reg) hv_sp, + in("rdi") cpu_data, + in("rsi") linux_sp, + lateout("rax") ret, + out("rcx") _, + clobber_abi("sysv64"), + ); + ret + }; + + let ret = vmm_entry(linux_sp); + + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, linux_tp); + x86::controlregs::cr3_write(linux_cr3); + ret + } +} + +#[naked] +#[unsafe(no_mangle)] +pub unsafe extern "C" fn _start() -> i32 { + unsafe { + core::arch::naked_asm!(" + // rip is pushed + cli + push rbp + push rbx + push r12 + push r13 + push r14 + push r15 + push 0 // skip gs_base + + mov rdi, rsp + call {0} + + pop r15 // skip gs_base + pop r15 + pop r14 + pop r13 + pop r12 + pop rbx + pop rbp + ret + // rip will pop when return", + sym switch_stack, + ); + } +} diff --git a/modules/axhal/src/platform/x86_linux/header.rs b/modules/axhal/src/platform/x86_linux/header.rs new file mode 100644 index 0000000000..0417acfe05 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/header.rs @@ -0,0 +1,77 @@ +use core::fmt::{Debug, Formatter, Result}; + +use super::consts::{HV_HEADER_PTR, PER_CPU_SIZE}; + +const HEADER_SIGNATURE: [u8; 8] = *b"ARCEOSIM"; + +#[repr(C)] +pub struct HvHeader { + pub signature: [u8; 8], + pub core_size: usize, + pub percpu_size: usize, + pub entry: usize, + /// Available CPU numbers provided by current physical platform. + pub max_cpus: u32, + /// CPU numbers reserved for ArceOS. + /// The Rest of Available CPUs will be reserved for host Linux. + pub arceos_cpus: u32, +} + +impl HvHeader { + pub fn get<'a>() -> &'a Self { + unsafe { &*HV_HEADER_PTR } + } + + pub fn reserved_cpus(&self) -> u32 { + if self.arceos_cpus < self.max_cpus { + self.max_cpus - self.arceos_cpus + } else { + warn!( + "Invalid HvHeader: arceos_cpus ({}) >= max_cpus ({})", + self.arceos_cpus, self.max_cpus + ); + self.max_cpus + } + } +} + +#[repr(C)] +struct HvHeaderStuff { + signature: [u8; 8], + core_size: unsafe extern "C" fn(), + percpu_size: usize, + entry: unsafe extern "C" fn(), + max_cpus: u32, + rt_cpus: u32, +} + +unsafe extern "C" { + unsafe fn __entry_offset(); + unsafe fn __kernel_size(); + +} + +#[used] +#[unsafe(link_section = ".header")] +static HEADER_STUFF: HvHeaderStuff = HvHeaderStuff { + signature: HEADER_SIGNATURE, + core_size: __kernel_size, + percpu_size: PER_CPU_SIZE, + entry: __entry_offset, + max_cpus: 0, + rt_cpus: 0, +}; + +impl Debug for HvHeader { + fn fmt(&self, f: &mut Formatter) -> Result { + f.debug_struct("HvHeader") + .field("signature", &core::str::from_utf8(&self.signature)) + .field("core_size", &self.core_size) + .field("percpu_size", &self.percpu_size) + .field("entry", &self.entry) + .field("max_cpus", &self.max_cpus) + .field("arceos_cpus", &self.arceos_cpus) + .field("reserved_cpus", &self.reserved_cpus()) + .finish() + } +} diff --git a/modules/axhal/src/platform/x86_linux/lapic.rs b/modules/axhal/src/platform/x86_linux/lapic.rs new file mode 100644 index 0000000000..7d4797866e --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/lapic.rs @@ -0,0 +1,117 @@ +/// This is just a simplified LocalApic used for send sipi request to other cores. +/// Refer to `apic.rs` for detailed implementation for APIC. +// Because `LocalApicBuilder` from `x2apic` can not be used here. +use spin::{Once, RwLock}; +use x86::apic::{ApicControl, ApicId, x2apic::X2APIC, xapic::XAPIC}; + +extern crate alloc; + +/// TODO: remove this Arc. +use alloc::sync::Arc; + +use crate::mem::{PhysAddr, phys_to_virt}; + +const APIC_BASE: PhysAddr = pa!(0xFEE0_0000); + +bitflags::bitflags! { + /// IA32_APIC_BASE MSR. + struct ApicBase: u64 { + /// Processor is BSP. + const BSP = 1 << 8; + /// Enable x2APIC mode. + const EXTD = 1 << 10; + /// xAPIC global enable/disable. + const EN = 1 << 11; + } +} + +impl ApicBase { + pub fn read() -> Self { + unsafe { Self::from_bits_retain(x86::msr::rdmsr(x86::msr::IA32_APIC_BASE)) } + } +} + +pub(super) struct LocalApic { + inner: Arc>, + is_x2apic: bool, +} + +unsafe impl Send for LocalApic {} +unsafe impl Sync for LocalApic {} + +impl LocalApic { + pub fn new() -> Result { + let base = ApicBase::read(); + + debug!("ApicBase {:#x}", base.bits()); + + if base.contains(ApicBase::EXTD) { + info!("Using x2APIC."); + Ok(Self { + inner: Arc::new(RwLock::new(X2APIC::new())), + is_x2apic: true, + }) + } else if base.contains(ApicBase::EN) { + info!("Using xAPIC."); + let base_vaddr = phys_to_virt(APIC_BASE); + let apic_region = + unsafe { core::slice::from_raw_parts_mut(base_vaddr.as_usize() as _, 0x1000 / 4) }; + Ok(Self { + inner: Arc::new(RwLock::new(XAPIC::new(apic_region))), + is_x2apic: false, + }) + } else { + Err("Local Apic init failed") + } + } +} + +static LOCAL_APIC: Once = Once::new(); + +pub(super) fn lapic<'a>() -> &'a LocalApic { + LOCAL_APIC.get().expect("Uninitialized Local APIC!") +} + +pub(super) fn init() { + let lapic = LocalApic::new().expect("LocalApic init failed"); + LOCAL_APIC.call_once(|| lapic); +} + +use crate::time::{Duration, busy_wait}; + +pub(super) unsafe fn start_ap(apic_id: u32, start_page_idx: u8) { + info!("Starting RT cpu {}...", apic_id); + let apic_id = if lapic().is_x2apic { + ApicId::X2Apic(apic_id) + } else { + ApicId::XApic(apic_id as u8) + }; + + // INIT-SIPI-SIPI Sequence + let mut lapic = lapic().inner.write(); + unsafe { + lapic.ipi_init(apic_id); + } + // delay_us(10 * 1000); // 10ms + busy_wait(Duration::from_millis(10)); // 10ms + unsafe { + lapic.ipi_startup(apic_id, start_page_idx); + } + // delay_us(200); // 200 us + busy_wait(Duration::from_micros(200)); // 200us + unsafe { + lapic.ipi_startup(apic_id, start_page_idx); + } +} + +pub(super) unsafe fn shutdown_ap(apic_id: u32) { + info!("Shutting down RT cpu {}...", apic_id); + let apic_id = if lapic().is_x2apic { + ApicId::X2Apic(apic_id) + } else { + ApicId::XApic(apic_id as u8) + }; + unsafe { + lapic().inner.write().ipi_init(apic_id); + } +} diff --git a/modules/axhal/src/platform/x86_linux/mem.rs b/modules/axhal/src/platform/x86_linux/mem.rs new file mode 100644 index 0000000000..4d0c7649d0 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/mem.rs @@ -0,0 +1,86 @@ +// TODO: get memory regions from multiboot info. + +use crate::mem::{virt_to_phys, MemRegion, MemRegionFlags, PhysAddr, VirtAddr, MemoryAddr}; + +use super::config::HvSystemConfig; +use super::consts::{HV_BASE, PER_CPU_ARRAY_PTR}; +use super::header::HvHeader; + +// MemRegion { +// paddr: virt_to_phys((_stext as usize).into()), +// size: _etext as usize - _stext as usize, +// flags: MemRegionFlags::RESERVED | MemRegionFlags::READ | MemRegionFlags::EXECUTE, +// name: ".text", +// }, + +unsafe extern "C" { + unsafe fn __header_start(); + unsafe fn __header_end(); +} + +/// Returns the vmm free memory regions (kernel image end to physical memory end). +fn vmm_free_regions() -> impl Iterator { + let mem_pool_start = super::consts::free_memory_start(); + let mem_pool_end = super::consts::hv_end().align_down_4k(); + let mem_pool_size = mem_pool_end.as_usize() - mem_pool_start.as_usize(); + core::iter::once(MemRegion { + paddr: virt_to_phys(mem_pool_start), + size: mem_pool_size, + flags: MemRegionFlags::FREE | MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "free memory", + }) +} + +fn vmm_per_cpu_data_regions() -> impl Iterator { + let start_paddr = virt_to_phys(VirtAddr::from(PER_CPU_ARRAY_PTR as usize)); + let end_paddr = virt_to_phys(super::consts::free_memory_start()); + + let size = end_paddr.as_usize() - start_paddr.as_usize(); + + core::iter::once(MemRegion { + paddr: start_paddr, + size, + flags: MemRegionFlags::READ | MemRegionFlags::WRITE, + name: "per-CPU data, configurations", + }) +} + +/// Returns platform-specific memory regions. +pub(crate) fn platform_regions() -> impl Iterator { + // Add region for HvHeader. + // See modules/axhal/linker_hv.lds.S for details. + core::iter::once(MemRegion { + paddr: virt_to_phys((__header_start as usize).into()), + size: __header_end as usize - __header_start as usize, + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ, + name: ".header", + }) + .chain(vmm_per_cpu_data_regions()) + .chain(vmm_free_regions()) + // Here we do not use the `default_free_regions`` from `crate::mem`. + // Cause we need to reserved regions for + // per-CPU data and `HvSystemConfig` + .chain(crate::mem::default_mmio_regions()) +} + +pub fn host_memory_regions() -> impl Iterator { + use crate::mem::MemRegionFlags; + + let sys_config = HvSystemConfig::get(); + let cell_config = &sys_config.root_cell.config(); + // Map all guest RAM to directly access in hypervisor. + cell_config + .mem_regions() + .iter() + .filter(|region| { + MemRegionFlags::from_bits(region.flags as _) + .unwrap() + .contains(MemRegionFlags::DEVICE) + }) + .map(|region| MemRegion { + paddr: PhysAddr::from(region.phys_start as usize), + size: region.size as usize, + flags: MemRegionFlags::from_bits(region.flags as _).unwrap(), + name: "Linux mem", + }) +} \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/misc.rs b/modules/axhal/src/platform/x86_linux/misc.rs new file mode 100644 index 0000000000..00bb63dd96 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/misc.rs @@ -0,0 +1,20 @@ +// use x86_64::instructions::port::PortWriteOnly; + +/// Shutdown the whole system (in QEMU), including all CPUs. +/// +/// See for more information. +pub fn terminate() -> ! { + info!("Shutting down..."); + axlog::ax_println!("\nBut now we need to find a way to return this core to Linux!!!\n"); + + // #[cfg(platform = "x86_64-qemu-q35")] + // unsafe { + // PortWriteOnly::new(0x604).write(0x2000u16) + // }; + + crate::arch::halt(); + warn!("It should shutdown!"); + loop { + crate::arch::halt(); + } +} diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs new file mode 100644 index 0000000000..dfdffff3c7 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -0,0 +1,180 @@ +mod apic; +// It's a simplied version of LocalApic, just use for sendsipi. +mod lapic; + +mod boot; +mod dtables; +mod entry; +mod uart16550; + +pub mod mem; +pub mod misc; +pub mod time; + +// mods for vmm usage. +mod percpu; + +mod config; +mod consts; +mod header; + +// #[cfg(feature = "smp")] +pub mod mp; + +#[cfg(feature = "irq")] +pub mod irq { + pub use super::apic::*; +} + +pub mod console { + pub use super::uart16550::*; +} + +use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; + +use axlog::ax_println as println; + +use config::HvSystemConfig; +// use error::HvResult; +use header::HvHeader; +use percpu::PerCpu; + +static VMM_PRIMARY_INIT_OK: AtomicU32 = AtomicU32::new(0); +static ERROR_NUM: AtomicI32 = AtomicI32::new(0); + +fn has_err() -> bool { + ERROR_NUM.load(Ordering::Acquire) != 0 +} + +fn wait_for(condition: impl Fn() -> bool) { + while !has_err() && condition() { + core::hint::spin_loop(); + } + if has_err() { + println!("[Error] Other cpu init failed!") + } +} + +unsafe extern "C" { + unsafe fn rust_vmm_main(cpu_id: usize); + unsafe fn rust_arceos_main(cpu_id: usize) -> !; +} + +fn current_cpu_id() -> usize { + match raw_cpuid::CpuId::new().get_feature_info() { + Some(finfo) => finfo.initial_local_apic_id() as usize, + None => 0, + } +} + +fn vmm_primary_init_early(cpu_id: usize) { + // We do not clear bss here. + // Because currently the image was loaded by Linux. + // crate::mem::clear_bss(); + + crate::cpu::init_primary(cpu_id); + self::time::init_early(); + + // println!("HvHeader\n{:#?}", HvHeader::get()); + + let system_config = HvSystemConfig::get(); + + println!( + "\n\ + Initializing ARCEOS on Core [{}]...\n\ + config_signature = {:?}\n\ + config_revision = {}\n\ + ", + cpu_id, + core::str::from_utf8(&system_config.signature), + system_config.revision, + ); + + VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); + + unsafe { + rust_vmm_main(cpu_id); + } +} + +extern "sysv64" fn vmm_cpu_entry(cpu_data: &mut PerCpu, _linux_sp: usize) -> i32 { + // Currently we set core 0 as Linux. + let is_primary = cpu_data.id == 0; + + let vm_cpus = HvHeader::get().reserved_cpus(); + + wait_for(|| PerCpu::entered_cpus() < vm_cpus); + + // println!( + // "{} CPU {} entered.", + // if is_primary { "Primary" } else { "Secondary" }, + // cpu_data.id + // ); + + // First, we init primary core for VMM. + if is_primary { + vmm_primary_init_early(cpu_data.id as usize); + } else { + // wait_for_counter(&VMM_PRIMARY_INIT_OK, 1); + + // wait_for_counter(&VMM_MAIN_INIT_OK, 1); + + // vmm_secondary_init_early(cpu_data.id as usize); + } + + let code = 0; + // println!( + // "{} CPU {} return back to driver with code {}.", + // if is_primary { "Primary" } else { "Secondary" }, + // cpu_data.id, + // code + // ); + code +} + +unsafe extern "C" fn rust_entry(_magic: usize, _mbi: usize) { + // TODO: handle multiboot info + // if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { + // crate::mem::clear_bss(); + // crate::cpu::init_primary(current_cpu_id()); + // self::uart16550::init(); + // self::dtables::init_primary(); + // self::time::init_early(); + // rust_main(current_cpu_id(), 0); + // } +} + +#[allow(unused_variables)] +unsafe extern "C" fn rust_entry_from_vmm(magic: usize) { + let cpu_id = current_cpu_id(); + info!("ARCEOS CPU entered on Core {}.", cpu_id); + + if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { + crate::cpu::init_secondary(cpu_id); + self::dtables::init_primary(); + self::time::init_early(); + rust_arceos_main(cpu_id); + } else { + panic!("Something is wrong during booting RT cores..."); + } +} + +/// Initializes the platform devices for the primary CPU. +/// Boot arceos cpus through sendsipi. +pub fn vmm_platform_init() { + self::lapic::init(); + self::mp::start_arceos_cpus(); +} + +/// Initializes the platform devices for the primary CPU. +pub fn platform_init() { + self::apic::init_primary(); + self::time::init_primary(); +} + +/// Initializes the platform devices for secondary CPUs. +#[cfg(feature = "smp")] +pub fn platform_init_secondary() { + self::apic::init_secondary(); + self::time::init_secondary(); +} diff --git a/modules/axhal/src/platform/x86_linux/mp.rs b/modules/axhal/src/platform/x86_linux/mp.rs new file mode 100644 index 0000000000..12bac5d6b5 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/mp.rs @@ -0,0 +1,94 @@ +use crate::mem::{phys_to_virt, virt_to_phys, PhysAddr, VirtAddr, PAGE_SIZE_4K}; +use crate::time::{busy_wait, Duration}; +use core::sync::atomic::Ordering; + +use super::header::HvHeader; +use super::percpu::PerCpu; +use axconfig::{SMP, TASK_STACK_SIZE}; + +const START_PAGE_IDX: u8 = 6; +const START_PAGE_COUNT: usize = 1; +const START_PAGE_PADDR: PhysAddr = pa!(START_PAGE_IDX as usize * PAGE_SIZE_4K); + +/// Starts the given secondary CPU with its boot stack. +pub fn start_secondary_cpu(_apic_id: usize, _stack_top: crate::mem::PhysAddr) { + // No need + // For CPU reserved for Linux, this step is completed by Linux. + // For CPU reserved for RT-task, this step is completed by `start_arceos_cpus`. +} + +pub fn continue_secondary_cpus() { + // super::VMM_MAIN_INIT_OK.fetch_add(1, Ordering::Release); + // super::VMM_MAIN_INIT_OK.store(1, Ordering::Release); +} + +core::arch::global_asm!( + include_str!("ap_start.S"), + start_page_paddr = const START_PAGE_PADDR.as_usize(), +); + +#[unsafe(link_section = ".bss.stack")] +static mut SECONDARY_BOOT_STACK: [[u8; TASK_STACK_SIZE]; SMP] = [[0; TASK_STACK_SIZE]; SMP]; + +/// Starts the given secondary CPU with its boot stack. +#[allow(clippy::uninit_assumed_init)] +pub fn start_arceos_cpus() { + unsafe extern "C" { + unsafe fn ap_entry32(); + unsafe fn ap_start(); + unsafe fn ap_end(); + } + const U64_PER_PAGE: usize = PAGE_SIZE_4K / 8; + + let start_page_ptr = phys_to_virt(START_PAGE_PADDR).as_mut_ptr() as *mut u64; + unsafe { + let start_page = + core::slice::from_raw_parts_mut(start_page_ptr, U64_PER_PAGE * START_PAGE_COUNT); + + // Since start page located at START_PAGE_PADDR belongs to Linux's physical address space. + // We construct a backup space for start page, after `start_ap`, we just copy the backup space back. + let mut backup: [u64; U64_PER_PAGE * START_PAGE_COUNT] = + core::mem::MaybeUninit::uninit().assume_init(); + backup.copy_from_slice(start_page); + core::ptr::copy_nonoverlapping( + ap_start as *const u64, + start_page_ptr, + (ap_end as usize - ap_start as usize) / 8, + ); + + // We need to use physical address here. + // Since current physical to virtual address is not identical mapped with offset 0xffff_ff80_0000_0000. + let ap_entry_virt = VirtAddr::from(ap_entry32 as usize); + let ap_entry_phys = virt_to_phys(ap_entry_virt); + + start_page[U64_PER_PAGE - 1] = ap_entry_phys.as_usize() as _; // entry + + let max_cpus = super::header::HvHeader::get().max_cpus; + let mut arceos_cpu_num = 0; + + for apic_id in 0..max_cpus { + if PerCpu::cpu_is_booted(apic_id as usize) { + continue; + } + let stack_top = virt_to_phys(VirtAddr::from( + SECONDARY_BOOT_STACK[arceos_cpu_num].as_ptr_range().end as usize, + )) + .as_usize(); + + start_page[U64_PER_PAGE - 2] = stack_top as u64; // stack_top + + super::lapic::start_ap(apic_id, START_PAGE_IDX); + // info!( + // "starting ArceOS {} CPU, apic id {}", + // arceos_cpu_num, apic_id + // ); + + arceos_cpu_num += 1; + // wait for max 100ms + busy_wait(Duration::from_millis(100)); // 100ms + } + // info!("starting {} CPUs for ArceOS ", arceos_cpu_num); + + start_page.copy_from_slice(&backup); + } +} diff --git a/modules/axhal/src/platform/x86_linux/multiboot.S b/modules/axhal/src/platform/x86_linux/multiboot.S new file mode 100644 index 0000000000..dfa16a12bf --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/multiboot.S @@ -0,0 +1,147 @@ +# Bootstrapping from 32-bit with the Multiboot specification. +# See https://www.gnu.org/software/grub/manual/multiboot/multiboot.html + +.section .text.boot +.code32 +# .global _start +# _start: +# mov edi, eax # arg1: magic: 0x2BADB002 +# mov esi, ebx # arg2: multiboot info +# jmp bsp_entry32 + +.balign 4 +.type multiboot_header, STT_OBJECT +multiboot_header: + .int {mb_hdr_magic} # magic: 0x1BADB002 + .int {mb_hdr_flags} # flags + .int -({mb_hdr_magic} + {mb_hdr_flags}) # checksum + .int multiboot_header - {offset} # header_addr + .int _skernel - {offset} # load_addr + .int _edata - {offset} # load_end + .int _ebss - {offset} # bss_end_addr + .int _start - {offset} # entry_addr + +# Common code in 32-bit, prepare states to enter 64-bit. +.macro ENTRY32_COMMON + # set data segment selectors + mov ax, 0x18 + mov ss, ax + mov ds, ax + mov es, ax + mov fs, ax + mov gs, ax + + # set PAE, PGE bit in CR4 + mov eax, {cr4} + mov cr4, eax + + # load the temporary page table + lea eax, [.Ltmp_pml4 - {offset}] + mov cr3, eax + + # set LME, NXE bit in IA32_EFER + mov ecx, {efer_msr} + mov edx, 0 + mov eax, {efer} + wrmsr + + # set protected mode, write protect, paging bit in CR0 + mov eax, {cr0} + mov cr0, eax +.endm + +# Common code in 64-bit +.macro ENTRY64_COMMON + # clear segment selectors + xor ax, ax + mov ss, ax + mov ds, ax + mov es, ax + mov fs, ax + mov gs, ax +.endm + +.code32 +bsp_entry32: + lgdt [.Ltmp_gdt_desc - {offset}] # load the temporary GDT + ENTRY32_COMMON + ljmp 0x10, offset bsp_entry64 - {offset} # 0x10 is code64 segment + +.code32 +.global ap_entry32 +ap_entry32: + ENTRY32_COMMON + ljmp 0x10, offset ap_entry64 - {offset} # 0x10 is code64 segment + +.code64 +bsp_entry64: + ENTRY64_COMMON + + # set RSP to boot stack + movabs rsp, offset {boot_stack} + add rsp, {boot_stack_size} + + # call rust_entry(magic, mbi) + movabs rax, offset {entry} + call rax + jmp .Lhlt + +.code64 +ap_entry64: + ENTRY64_COMMON + + # set RSP to high address (already set in ap_start.S) + mov rax, {offset} + add rsp, rax + + # call rust_entry_from_vmm(magic) + mov rdi, {mb_magic} + movabs rax, offset {entry_secondary} + + call rax + jmp .Lhlt + +.Lhlt: + hlt + jmp .Lhlt + +.section .rodata +.balign 8 +.Ltmp_gdt_desc: + .short .Ltmp_gdt_end - .Ltmp_gdt - 1 # limit + .long .Ltmp_gdt - {offset} # base + +.section .data +.balign 16 +.Ltmp_gdt: + .quad 0x0000000000000000 # 0x00: null + .quad 0x00cf9b000000ffff # 0x08: code segment (base=0, limit=0xfffff, type=32bit code exec/read, DPL=0, 4k) + .quad 0x00af9b000000ffff # 0x10: code segment (base=0, limit=0xfffff, type=64bit code exec/read, DPL=0, 4k) + .quad 0x00cf93000000ffff # 0x18: data segment (base=0, limit=0xfffff, type=32bit data read/write, DPL=0, 4k) +.Ltmp_gdt_end: + +.balign 4096 +.Ltmp_pml4: + # 0x0000_0000 ~ 0xffff_ffff + .quad .Ltmp_pdpt_low - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) + .zero 8 * 509 + # 0xffff_ff00_0000_0000 ~ 0xffff_ff00_ffff_ffff + # 0xffff_ff00_0000_0000 ~ 0xffff_ff00_0800_0000 + # 0x0000_0000_4000_0000 ~ 0x0000_0000_4800_0000 + .quad .Ltmp_pdpt_high - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) + .zero 8 * 1 + +# FIXME: may not work on macOS using hvf as the CPU does not support 1GB page (pdpe1gb) +.Ltmp_pdpt_low: + .quad 0x0000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x0) + .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) + .quad 0x80000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x8000_0000) + .quad 0xc0000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0xc000_0000) + .zero 8 * 508 + +.Ltmp_pdpt_high: + .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) + # .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) + # .quad 0x80000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x8000_0000) + # .quad 0xc0000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0xc000_0000) + .zero 8 * 511 diff --git a/modules/axhal/src/platform/x86_linux/percpu.rs b/modules/axhal/src/platform/x86_linux/percpu.rs new file mode 100644 index 0000000000..1a01b27184 --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/percpu.rs @@ -0,0 +1,146 @@ +use core::fmt::{Debug, Formatter, Result}; +use core::sync::atomic::{AtomicU32, Ordering}; + +// use crate::arch::vmm::{Vcpu, VcpuAccessGuestState}; +// use crate::arch::{cpu, ArchPerCpu, LinuxContext}; +// use crate::cell::Cell; +use super::consts::{PER_CPU_ARRAY_PTR, PER_CPU_SIZE}; +use super::current_cpu_id; +// use crate::error::HvResult; +use super::header::HvHeader; + +static ENTERED_CPUS: AtomicU32 = AtomicU32::new(0); +// static ACTIVATED_CPUS: AtomicU32 = AtomicU32::new(0); + +#[derive(Debug, Eq, PartialEq)] +pub enum CpuState { + HvDisabled, + HvEnabled, +} + +static mut BOOTED_CPU_APIC_ID: [u32; 255] = [u32::MAX; 255]; + +// Todo: this can be placed into per_cpu. +#[repr(C, align(4096))] +pub struct PerCpu { + /// Referenced by arch::cpu::thread_pointer() for x86_64. + self_vaddr: usize, + /// Current CPU's apic id, read from raw_cpuid. + pub id: u32, + pub state: CpuState, + // pub vcpu: Vcpu, + // arch: ArchPerCpu, + // linux: LinuxContext, + // Stack will be placed here. +} + +impl PerCpu { + pub fn new<'a>() -> &'a mut Self { + if Self::entered_cpus() >= HvHeader::get().max_cpus { + panic!("enter cpus exceed {}", HvHeader::get().max_cpus); + } + + let cpu_sequence = ENTERED_CPUS.fetch_add(1, Ordering::SeqCst); + let cpu_id = current_cpu_id(); + + unsafe { + BOOTED_CPU_APIC_ID[cpu_id] = cpu_sequence; + } + + let ret = unsafe { Self::from_id_mut(cpu_id as u32) }; + let vaddr = ret as *const _ as usize; + ret.id = cpu_id as u32; + ret.self_vaddr = vaddr; + + // unsafe { crate::arch::write_thread_pointer(vaddr.into()) }; + ret + } + + pub unsafe fn from_id_mut<'a>(cpu_id: u32) -> &'a mut Self { + let vaddr = PER_CPU_ARRAY_PTR as usize + cpu_id as usize * PER_CPU_SIZE; + &mut *(vaddr as *mut Self) + } + + pub fn current<'a>() -> &'a Self { + Self::current_mut() + } + + pub fn current_mut<'a>() -> &'a mut Self { + unsafe { &mut *(crate::arch::read_thread_pointer() as *mut Self) } + } + + pub fn stack_top(&self) -> usize { + self as *const _ as usize + PER_CPU_SIZE - 8 + } + + pub fn entered_cpus() -> u32 { + ENTERED_CPUS.load(Ordering::Acquire) + } + + // pub fn activated_cpus() -> u32 { + // ACTIVATED_CPUS.load(Ordering::Acquire) + // } + + pub fn cpu_is_booted(apic_id: usize) -> bool { + unsafe { BOOTED_CPU_APIC_ID[apic_id] != u32::MAX } + } + + // pub fn init(&mut self, linux_sp: usize, cell: &Cell) -> HvResult { + // info!("CPU {} init...", self.id); + + // // Save CPU state used for linux. + // self.state = CpuState::HvDisabled; + // self.linux = LinuxContext::load_from(linux_sp); + + // // Activate hypervisor page table on each cpu. + // unsafe { crate::memory::hv_page_table().read().activate() }; + + // self.arch.init(self.id)?; + + // // Initialize vCPU. Use `ptr::write()` to avoid dropping + // unsafe { core::ptr::write(&mut self.vcpu, Vcpu::new(&self.linux, cell)?) }; + + // self.state = CpuState::HvEnabled; + // Ok(()) + // } + + // pub fn activate_vmm(&mut self) -> HvResult { + // println!("Activating hypervisor on CPU {}...", self.id); + // ACTIVATED_CPUS.fetch_add(1, Ordering::SeqCst); + + // self.vcpu.enter(&self.linux)?; + // unreachable!() + // } + + // pub fn deactivate_vmm(&mut self, ret_code: usize) -> HvResult { + // println!("Deactivating hypervisor on CPU {}...", self.id); + // ACTIVATED_CPUS.fetch_sub(1, Ordering::SeqCst); + + // self.vcpu.set_return_val(ret_code); + // self.vcpu.exit(&mut self.linux)?; + // self.linux.restore(); + // self.state = CpuState::HvDisabled; + // self.linux.return_to_linux(self.vcpu.regs()); + // } + + // pub fn fault(&mut self) -> HvResult { + // warn!("VCPU fault: {:#x?}", self); + // self.vcpu.inject_fault()?; + // Ok(()) + // } +} + +impl Debug for PerCpu { + fn fmt(&self, f: &mut Formatter) -> Result { + let mut res = f.debug_struct("PerCpu"); + res.field("id", &self.id) + .field("self_vaddr", &self.self_vaddr) + .field("state", &self.state); + // if self.state != CpuState::HvDisabled { + // res.field("vcpu", &self.vcpu); + // } else { + // res.field("linux", &self.linux); + // } + res.finish() + } +} diff --git a/modules/axhal/src/platform/x86_linux/time.rs b/modules/axhal/src/platform/x86_linux/time.rs new file mode 100644 index 0000000000..f146eaccdb --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/time.rs @@ -0,0 +1,105 @@ +use raw_cpuid::CpuId; + +#[cfg(feature = "irq")] +use int_ratio::Ratio; + +#[cfg(feature = "irq")] +const LAPIC_TICKS_PER_SEC: u64 = 1_000_000_000; // TODO: need to calibrate + +#[cfg(feature = "irq")] +static mut NANOS_TO_LAPIC_TICKS_RATIO: Ratio = Ratio::zero(); + +static mut INIT_TICK: u64 = 0; +static mut CPU_FREQ_MHZ: u64 = axconfig::devices::TIMER_FREQUENCY as u64 / 1_000_000; + +/// RTC wall time offset in nanoseconds at monotonic time base. +static mut RTC_EPOCHOFFSET_NANOS: u64 = 0; + +/// Returns the current clock time in hardware ticks. +pub fn current_ticks() -> u64 { + unsafe { core::arch::x86_64::_rdtsc() - INIT_TICK } +} + +/// Converts hardware ticks to nanoseconds. +pub fn ticks_to_nanos(ticks: u64) -> u64 { + ticks * 1_000 / unsafe { CPU_FREQ_MHZ } +} + +/// Converts nanoseconds to hardware ticks. +pub fn nanos_to_ticks(nanos: u64) -> u64 { + nanos * unsafe { CPU_FREQ_MHZ } / 1_000 +} + +/// Return epoch offset in nanoseconds (wall time offset to monotonic clock start). +pub fn epochoffset_nanos() -> u64 { + unsafe { RTC_EPOCHOFFSET_NANOS } +} + +/// Set a one-shot timer. +/// +/// A timer interrupt will be triggered at the given deadline (in nanoseconds). +#[cfg(feature = "irq")] +pub fn set_oneshot_timer(deadline_ns: u64) { + let lapic = super::apic::local_apic(); + let now_ns = crate::time::monotonic_time_nanos(); + unsafe { + if now_ns < deadline_ns { + let apic_ticks = NANOS_TO_LAPIC_TICKS_RATIO.mul_trunc(deadline_ns - now_ns); + assert!(apic_ticks <= u32::MAX as u64); + lapic.set_timer_initial(apic_ticks.max(1) as u32); + } else { + lapic.set_timer_initial(1); + } + } +} + +pub(super) fn init_early() { + if let Some(freq) = CpuId::new() + .get_processor_frequency_info() + .map(|info| info.processor_base_frequency()) + { + if freq > 0 { + axlog::ax_println!("Got TSC frequency by CPUID: {} MHz", freq); + unsafe { CPU_FREQ_MHZ = freq as u64 } + } + } + + unsafe { INIT_TICK = core::arch::x86_64::_rdtsc() }; + + #[cfg(feature = "rtc")] + { + use x86_rtc::Rtc; + + // Get the current time in microseconds since the epoch (1970-01-01) from the x86 RTC. + // Subtract the timer ticks to get the actual time when ArceOS was booted. + let eopch_time_nanos = Rtc::new().get_unix_timestamp() * 1_000_000_000; + unsafe { + RTC_EPOCHOFFSET_NANOS = eopch_time_nanos - ticks_to_nanos(INIT_TICK); + } + } +} + +pub(super) fn init_primary() { + #[cfg(feature = "irq")] + unsafe { + use x2apic::lapic::{TimerDivide, TimerMode}; + let lapic = super::apic::local_apic(); + lapic.set_timer_mode(TimerMode::OneShot); + lapic.set_timer_divide(TimerDivide::Div256); // indeed it is Div1, the name is confusing. + lapic.enable_timer(); + + // TODO: calibrate with HPET + NANOS_TO_LAPIC_TICKS_RATIO = Ratio::new( + LAPIC_TICKS_PER_SEC as u32, + crate::time::NANOS_PER_SEC as u32, + ); + } +} + +#[cfg(feature = "smp")] +pub(super) fn init_secondary() { + #[cfg(feature = "irq")] + unsafe { + super::apic::local_apic().enable_timer(); + } +} diff --git a/modules/axhal/src/platform/x86_linux/uart16550.rs b/modules/axhal/src/platform/x86_linux/uart16550.rs new file mode 100644 index 0000000000..66ee56f01c --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/uart16550.rs @@ -0,0 +1,127 @@ +//! Uart 16550. + +use kspin::SpinNoIrq; +use x86_64::instructions::port::{Port, PortReadOnly, PortWriteOnly}; + +const UART_CLOCK_FACTOR: usize = 16; +const OSC_FREQ: usize = 1_843_200; + +static COM1: SpinNoIrq = SpinNoIrq::new(Uart16550::new(0x3f8)); + +bitflags::bitflags! { + /// Line status flags + struct LineStsFlags: u8 { + const INPUT_FULL = 1; + // 1 to 4 unknown + const OUTPUT_EMPTY = 1 << 5; + // 6 and 7 unknown + } +} + +struct Uart16550 { + data: Port, + int_en: PortWriteOnly, + fifo_ctrl: PortWriteOnly, + line_ctrl: PortWriteOnly, + modem_ctrl: PortWriteOnly, + line_sts: PortReadOnly, +} + +impl Uart16550 { + const fn new(port: u16) -> Self { + Self { + data: Port::new(port), + int_en: PortWriteOnly::new(port + 1), + fifo_ctrl: PortWriteOnly::new(port + 2), + line_ctrl: PortWriteOnly::new(port + 3), + modem_ctrl: PortWriteOnly::new(port + 4), + line_sts: PortReadOnly::new(port + 5), + } + } + + fn init(&mut self, baud_rate: usize) { + unsafe { + // Disable interrupts + self.int_en.write(0x00); + + // Enable DLAB + self.line_ctrl.write(0x80); + + // Set maximum speed according the input baud rate by configuring DLL and DLM + let divisor = OSC_FREQ / (baud_rate * UART_CLOCK_FACTOR); + self.data.write((divisor & 0xff) as u8); + self.int_en.write((divisor >> 8) as u8); + + // Disable DLAB and set data word length to 8 bits + self.line_ctrl.write(0x03); + + // Enable FIFO, clear TX/RX queues and + // set interrupt watermark at 14 bytes + self.fifo_ctrl.write(0xC7); + + // Mark data terminal ready, signal request to send + // and enable auxilliary output #2 (used as interrupt line for CPU) + self.modem_ctrl.write(0x0B); + } + } + + fn line_sts(&mut self) -> LineStsFlags { + unsafe { LineStsFlags::from_bits_truncate(self.line_sts.read()) } + } + + fn putchar(&mut self, c: u8) { + while !self.line_sts().contains(LineStsFlags::OUTPUT_EMPTY) {} + unsafe { self.data.write(c) }; + } + + fn getchar(&mut self) -> Option { + if self.line_sts().contains(LineStsFlags::INPUT_FULL) { + unsafe { Some(self.data.read()) } + } else { + None + } + } +} + +/// Writes a byte to the console. +fn putchar(c: u8) { + let mut uart = COM1.lock(); + match c { + b'\n' => { + uart.putchar(b'\r'); + uart.putchar(b'\n'); + } + c => uart.putchar(c), + } +} + +/// Reads a byte from the console, or returns [`None`] if no input is available. +fn getchar() -> Option { + COM1.lock().getchar() +} + +/// Write a slice of bytes to the console. +pub fn write_bytes(bytes: &[u8]) { + for c in bytes { + putchar(*c); + } +} + +/// Reads bytes from the console into the given mutable slice. +/// Returns the number of bytes read. +pub fn read_bytes(bytes: &mut [u8]) -> usize { + let mut read_len = 0; + while read_len < bytes.len() { + if let Some(c) = getchar() { + bytes[read_len] = c; + } else { + break; + } + read_len += 1; + } + read_len +} + +pub(super) fn init() { + COM1.lock().init(115200); +} diff --git a/modules/axruntime/Cargo.toml b/modules/axruntime/Cargo.toml index 3f76289949..c9e7388d6d 100644 --- a/modules/axruntime/Cargo.toml +++ b/modules/axruntime/Cargo.toml @@ -41,3 +41,4 @@ percpu = { version = "0.1.4", optional = true } kernel_guard = { version = "0.1", optional = true } chrono = { version = "0.4.38", default-features = false } +lazyinit = "0.2" diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 114db35cc0..91c2c84671 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -31,6 +31,10 @@ mod mp; #[cfg(feature = "smp")] pub use self::mp::rust_main_secondary; +mod vmm; + +pub use self::vmm::{rust_vmm_main, rust_arceos_main}; + const LOGO: &str = r#" d8888 .d88888b. .d8888b. d88888 d88P" "Y88b d88P Y88b diff --git a/modules/axruntime/src/vmm.rs b/modules/axruntime/src/vmm.rs new file mode 100644 index 0000000000..47cacf761b --- /dev/null +++ b/modules/axruntime/src/vmm.rs @@ -0,0 +1,166 @@ +/// Some problems remain: +/// 1. How to return rt core back to Linux so we can "dynamically" control these rt-cores. +/// 2. How to co-schedual process from Linux within process inside arceos. +/// 3. How to handle memory mapping, currently both sides have a complete memory view of the other's address space. +use core::sync::atomic::Ordering; + +fn is_init_ok() -> bool { + super::INITED_CPUS.load(Ordering::Acquire) == (axconfig::SMP + 1) +} + +unsafe extern "C" { + unsafe fn main(); +} + +/// The main entry point of the ArceOS runtime for RT CPUs. +/// When booting from Linux and reserving CPU for RT tasks! +/// +/// It is called from the `rust_entry_from_vmm` code in [axhal]. +#[unsafe(no_mangle)] +pub extern "C" fn rust_arceos_main(cpu_id: usize) { + info!("ARCEOS CPU {:x} started.", cpu_id); + + #[cfg(feature = "paging")] + { + info!("Initialize kernel page table..."); + vmm_remap_kernel_memory().expect("remap kernel memoy failed"); + } + + info!("Initialize platform devices..."); + axhal::platform_init(); + + #[cfg(feature = "multitask")] + axtask::init_scheduler(); + + #[cfg(feature = "irq")] + { + info!("Initialize interrupt handlers..."); + super::init_interrupt(); + } + + info!("ARCEOS Primary CPU {:x} init OK.", cpu_id); + super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); + + while !is_init_ok() { + core::hint::spin_loop(); + } + + info!("ARCEOS Primary CPU {:x} enter main ...", cpu_id); + unsafe { main() }; + info!("ARCEOS main task exited: exit_code={}", 0); + + #[cfg(feature = "multitask")] + axtask::exit(0); + #[cfg(not(feature = "multitask"))] + { + debug!("main task exited: exit_code={}", 0); + axhal::misc::terminate(); + } +} + +/// The main entry point of the ArceOS-VMM **when booting from Linux and set self as VMM**! +/// +/// It is called from the `vmm_cpu_entry` code in [axhal]. `cpu_id` is the ID of +/// the current CPU, and `dtb` is the address of the device tree blob. It +/// finally calls the application's `main` function after all initialization +/// work is done. +/// +/// In multi-core environment, this function is called on the primary CPU. +#[cfg_attr(not(test), unsafe(no_mangle))] +pub extern "C" fn rust_vmm_main(cpu_id: usize) { + ax_println!("{}", super::LOGO); + ax_println!( + "\ + arch = {}\n\ + platform = {}\n\ + target = {}\n\ + smp = {}\n\ + build_mode = {}\n\ + log_level = {}\n\ + ", + option_env!("AX_ARCH").unwrap_or(""), + option_env!("AX_PLATFORM").unwrap_or(""), + option_env!("AX_TARGET").unwrap_or(""), + option_env!("AX_SMP").unwrap_or(""), + option_env!("AX_MODE").unwrap_or(""), + option_env!("AX_LOG").unwrap_or(""), + ); + + axlog::init(); + axlog::set_max_level(option_env!("AX_LOG").unwrap_or("")); // no effect if set `log-level-*` features + info!("Logging is enabled."); + info!("VMM Primary CPU {} started", cpu_id); + + info!("Found physcial memory regions:"); + for r in axhal::mem::memory_regions() { + info!( + " [{:x?}, {:x?}) {} ({:?})", + r.paddr, + r.paddr + r.size, + r.name, + r.flags + ); + } + + super::init_allocator(); + + #[cfg(feature = "paging")] + { + info!("Initialize kernel page table..."); + vmm_remap_kernel_memory().expect("remap kernel memory failed"); + } + + info!("Initialize VMM platform..."); + axhal::vmm_platform_init(); + + // info!("VMM Primary CPU {} init OK.", cpu_id); + + super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); + + while !is_init_ok() { + core::hint::spin_loop(); + } +} + +#[cfg(feature = "paging")] +fn vmm_remap_kernel_memory() -> Result<(), axhal::paging::PagingError> { + use axhal::host_memory_regions; + use axhal::mem::{memory_regions, phys_to_virt}; + use axhal::paging::PageTable; + use lazyinit::LazyInit; + + static KERNEL_PAGE_TABLE: LazyInit = LazyInit::new(); + + if axhal::cpu::this_cpu_is_bsp() { + info!("BSP CPU init KERNEL_PAGE_TABLE..."); + let mut kernel_page_table = PageTable::try_new()?; + for r in memory_regions() { + kernel_page_table.map_region( + phys_to_virt(r.paddr), + |_| r.paddr, + r.size, + r.flags.into(), + false, + false, + )?; + } + + for r in host_memory_regions() { + kernel_page_table.map_region( + phys_to_virt(r.paddr), + |_| r.paddr, + r.size, + r.flags.into(), + false, + false, + )?; + } + + KERNEL_PAGE_TABLE.init_once(kernel_page_table); + + info!("KERNEL_PAGE_TABLE init success"); + } + + unsafe { axhal::arch::write_page_table_root(KERNEL_PAGE_TABLE.root_paddr()) }; + Ok(()) +} diff --git a/scripts/make/build.mk b/scripts/make/build.mk index d346b03375..2be9a11cc6 100644 --- a/scripts/make/build.mk +++ b/scripts/make/build.mk @@ -52,4 +52,7 @@ $(OUT_DIR): $(OUT_BIN): _cargo_build $(OUT_ELF) $(call run_cmd,$(OBJCOPY),$(OUT_ELF) --strip-all -O binary $@) +$(OUT_ASM): $(OUT_BIN) + $(call run_cmd,$(OBJDUMP),$(OUT_ELF) > $(OUT_ASM)) + .PHONY: _cargo_build diff --git a/scripts/vmm/guest/disable-arceos-vmm.sh b/scripts/vmm/guest/disable-arceos-vmm.sh new file mode 100755 index 0000000000..2b46fa4494 --- /dev/null +++ b/scripts/vmm/guest/disable-arceos-vmm.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable diff --git a/scripts/vmm/guest/enable-arceos-vmm.sh b/scripts/vmm/guest/enable-arceos-vmm.sh new file mode 100755 index 0000000000..0b4c1a2061 --- /dev/null +++ b/scripts/vmm/guest/enable-arceos-vmm.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +JH_DIR=~/jailhouse-equation +JH=$JH_DIR/tools/jailhouse + +sudo $JH disable +sudo rmmod jailhouse +sudo insmod $JH_DIR/driver/jailhouse.ko +sudo chown $(whoami) /dev/jailhouse +sudo $JH enable diff --git a/scripts/vmm/guest/setup.sh b/scripts/vmm/guest/setup.sh new file mode 100755 index 0000000000..7d141f4c98 --- /dev/null +++ b/scripts/vmm/guest/setup.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Install packages +sudo sed -i "s/http:\/\/archive.ubuntu.com/http:\/\/mirrors.tuna.tsinghua.edu.cn/g" /etc/apt/sources.list +sudo apt-get update +sudo apt-get install -y build-essential python3-mako + +# Create a hypervisor image link to /lib/firmware/evm-intel.bin +sudo mkdir -p /lib/firmware +sudo ln -sf ~/evm-intel.bin /lib/firmware + +# Clone jailhouse-equation, apply patches and build +# git clone https://github.com/EquationOS/jailhouse-equation +cd jailhouse-equation +make + +# Update grub config file to update kernel cmdline +./update-cmdline.sh +sudo update-grub + +echo +echo "Setup OK!" +echo "Press ENTER to reboot..." +read +sudo reboot diff --git a/scripts/vmm/guest/test_hypercall.c b/scripts/vmm/guest/test_hypercall.c new file mode 100644 index 0000000000..ceeec157f9 --- /dev/null +++ b/scripts/vmm/guest/test_hypercall.c @@ -0,0 +1,40 @@ +#include +#include +#include + +#define HYPERCALL "vmcall" + +static void in_guest() { + printf("Execute VMCALL OK.\n"); + printf("You are in the Guest mode.\n"); + exit(0); +} + +static void in_host() { + printf("Execute VMCALL failed.\n"); + printf("You are in the Host mode.\n"); + exit(1); +} + +static void sig_handler(int signum) { + printf("Caught signal %d\n", signum); + in_host(); +} + +static inline long hypercall(int num) { + long ret; + asm volatile(HYPERCALL : "=a"(ret) : "a"(num) : "memory"); + return ret; +} + +int main () { + signal(SIGSEGV, sig_handler); + signal(SIGILL, sig_handler); + int ret = hypercall(2333); + if (ret == 2333) { + in_guest(); + } else { + in_host(); + } + return 0; +} diff --git a/scripts/vmm/guest/update-cmdline.sh b/scripts/vmm/guest/update-cmdline.sh new file mode 100755 index 0000000000..67c72af656 --- /dev/null +++ b/scripts/vmm/guest/update-cmdline.sh @@ -0,0 +1,4 @@ +# Update grub config +cmdline='memmap=0x8000000\\\\\\$0x40000000' +sudo sed -i "s/GRUB_CMDLINE_LINUX=.*/GRUB_CMDLINE_LINUX=$cmdline/" /etc/default/grub +echo "Appended kernel cmdline: $cmdline, see '/etc/default/grub'" diff --git a/scripts/vmm/host/.gitignore b/scripts/vmm/host/.gitignore new file mode 100644 index 0000000000..37c1c8464d --- /dev/null +++ b/scripts/vmm/host/.gitignore @@ -0,0 +1,2 @@ +*.img +user-data \ No newline at end of file diff --git a/scripts/vmm/host/Makefile b/scripts/vmm/host/Makefile new file mode 100644 index 0000000000..23a03f6e4a --- /dev/null +++ b/scripts/vmm/host/Makefile @@ -0,0 +1,58 @@ +QEMU ?= qemu-system-x86_64 +PORT ?= 2334 +SMP ?= 4 +OUT_ELF ?= ../../../apps/vmm/vmm_x86_64-linux.elf + +TELNET_PORT := 4321 + +UV ?= 24.04 + +ifeq ($(UV),22.04) +RELEASE_NAME = focal +endif + +ifeq ($(UV),24.04) +RELEASE_NAME = noble +endif + +qemu_image := ubuntu-$(UV)-server-cloudimg-amd64.img +qemu_args := \ + -smp $(SMP) -m 4G -accel kvm -nographic \ + -machine q35,kernel_irqchip=split \ + -cpu host,-kvm-asyncpf,-kvm-pv-eoi,-kvm-pv-ipi,-kvm-pv-sched-yield,-kvm-pv-unhalt,-kvm-steal-time,-kvmclock \ + -drive file=$(qemu_image) \ + -net user,id=net,hostfwd=tcp::$(PORT)-:22 -net nic,model=e1000e \ + -serial mon:stdio \ + -serial telnet:localhost:$(TELNET_PORT),server + +$(qemu_image): + wget -nc https://cloud-images.ubuntu.com/releases/$(RELEASE_NAME)/release/$(qemu_image) + +.ONESHELL: +image: $(qemu_image) + cat >user-data < Date: Mon, 17 Mar 2025 14:59:36 +0800 Subject: [PATCH 10/19] [wip] on type15 boot --- Cargo.lock | 66 ++++++++++ modules/axhal/Cargo.toml | 11 +- .../axhal/src/platform/x86_linux/config.rs | 51 +++++++- .../axhal/src/platform/x86_linux/consts.rs | 19 +-- .../axhal/src/platform/x86_linux/context.rs | 16 +++ modules/axhal/src/platform/x86_linux/entry.rs | 50 +++++-- .../axhal/src/platform/x86_linux/header.rs | 11 +- modules/axhal/src/platform/x86_linux/mem.rs | 56 +++----- modules/axhal/src/platform/x86_linux/misc.rs | 10 +- modules/axhal/src/platform/x86_linux/mod.rs | 122 ++++++++++++------ modules/axhal/src/platform/x86_linux/mp.rs | 8 +- .../axhal/src/platform/x86_linux/multiboot.S | 8 +- .../axhal/src/platform/x86_linux/percpu.rs | 5 - modules/axhal/src/platform/x86_linux/time.rs | 2 +- .../axhal/src/platform/x86_linux/uart16550.rs | 4 +- modules/axmm/src/aspace.rs | 1 + modules/axruntime/src/vmm.rs | 82 +++++++----- 17 files changed, 358 insertions(+), 164 deletions(-) create mode 100644 modules/axhal/src/platform/x86_linux/context.rs diff --git a/Cargo.lock b/Cargo.lock index 6ef2802b37..7098b7c922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,6 +213,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axaddrspace" +version = "0.1.0" +source = "git+https://github.com/arceos-hypervisor/axaddrspace.git#f1ab1108c1477f6f4f56035b74ea76c8932d6b8d" +dependencies = [ + "axerrno", + "bit_field", + "bitflags 2.6.0", + "cfg-if", + "lazyinit", + "log", + "memory_addr", + "memory_set", + "numeric-enum-macro", + "page_table_entry", + "page_table_multiarch", +] + [[package]] name = "axalloc" version = "0.1.0" @@ -470,6 +488,7 @@ dependencies = [ "x86", "x86_64 0.15.2", "x86_rtc", + "x86_vcpu", ] [[package]] @@ -598,6 +617,17 @@ dependencies = [ "timer_list", ] +[[package]] +name = "axvcpu" +version = "0.1.0" +source = "git+https://github.com/arceos-hypervisor/axvcpu.git#8414a575723f929d5fd24010ce16446d90ecf268" +dependencies = [ + "axaddrspace", + "axerrno", + "memory_addr", + "percpu", +] + [[package]] name = "base64" version = "0.13.1" @@ -1177,6 +1207,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f769efcf10b9dfb4c913bebb409cda77b1a3f072b249bf5465e250bcb30eb49" +[[package]] +name = "memory_set" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4552d02c866c57e8b06b919ea8c2f8f398cad245b8f6aac726657bc972d663d" +dependencies = [ + "memory_addr", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1202,6 +1241,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "numeric-enum-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e4bdb6b46b592948e700ea1ef24a4296491f6a0ee722b258040abd15a3714" + [[package]] name = "once_cell" version = "1.18.0" @@ -1916,6 +1961,27 @@ dependencies = [ "x86_64 0.15.2", ] +[[package]] +name = "x86_vcpu" +version = "0.1.0" +source = "git+https://github.com/arceos-hypervisor/x86_vcpu.git#5f6eaf4157ee8385ba1a2bfe5060098be18b0245" +dependencies = [ + "axaddrspace", + "axerrno", + "axvcpu", + "bit_field", + "bitflags 2.6.0", + "cfg-if", + "crate_interface", + "log", + "memory_addr", + "numeric-enum-macro", + "page_table_entry", + "raw-cpuid 11.1.0", + "x86", + "x86_64 0.15.2", +] + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index 7e60928188..65a4783265 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -18,7 +18,15 @@ irq = [] tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] default = [] -hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "arm_gicv2/el2", "dep:crate_interface"] +hv = [ + "paging", + "cortex-a", + "percpu/arm-el2", + "page_table_entry/arm-el2", + "arm_gicv2/el2", + "dep:crate_interface", + "dep:x86_vcpu", +] [dependencies] log = "=0.4.21" @@ -47,6 +55,7 @@ x86_64 = "0.15" x2apic = "0.4" raw-cpuid = "11.1" x86_rtc = { version = "0.1", optional = true } +x86_vcpu = { git = "https://github.com/arceos-hypervisor/x86_vcpu.git", optional = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] riscv = "0.11" diff --git a/modules/axhal/src/platform/x86_linux/config.rs b/modules/axhal/src/platform/x86_linux/config.rs index 7817701686..5765802808 100644 --- a/modules/axhal/src/platform/x86_linux/config.rs +++ b/modules/axhal/src/platform/x86_linux/config.rs @@ -1,8 +1,12 @@ use core::fmt::{Debug, Formatter, Result}; use core::{mem::size_of, slice}; -const CONFIG_SIGNATURE: [u8; 6] = *b"ARCEOS"; -const CONFIG_REVISION: u16 = 13; +use bitflags::bitflags; + +use crate::mem::{MemRegion, MemRegionFlags}; + +const CONFIG_SIGNATURE: [u8; 6] = *b"EVMSYS"; +const CONFIG_REVISION: u16 = 314; const HV_CELL_NAME_MAXLEN: usize = 31; @@ -21,15 +25,52 @@ pub struct HvCellDesc { } #[derive(Debug)] -#[repr(C, packed)] +#[repr(C)] pub struct HvMemoryRegion { pub phys_start: u64, pub virt_start: u64, pub size: u64, - pub flags: u64, + pub flags: MemFlags, +} + +bitflags! { + #[derive(Debug, Clone)] + pub struct MemFlags: u64 { + const READ = 1 << 0; + const WRITE = 1 << 1; + const EXECUTE = 1 << 2; + const DMA = 1 << 3; + const IO = 1 << 4; + const NO_HUGEPAGES = 1 << 8; + const USER = 1 << 9; + } +} + +impl Into for MemFlags { + fn into(self) -> MemRegionFlags { + let mut flags = MemRegionFlags::empty(); + if self.contains(MemFlags::READ) { + flags |= MemRegionFlags::READ; + } + if self.contains(MemFlags::WRITE) { + flags |= MemRegionFlags::WRITE; + } + if self.contains(MemFlags::EXECUTE) { + flags |= MemRegionFlags::EXECUTE; + } + if self.contains(MemFlags::DMA) { + flags |= MemRegionFlags::DEVICE; + } + if self.contains(MemFlags::IO) { + flags |= MemRegionFlags::DEVICE; + } + flags + } } /// General descriptor of the system. +/// +/// See jailhouse dir `driver/cell-config.h` for details. #[derive(Debug)] #[repr(C)] pub struct HvSystemConfig { @@ -37,8 +78,6 @@ pub struct HvSystemConfig { pub revision: u16, /// AxVisor location in memory pub hypervisor_memory: HvMemoryRegion, - /// RTOS location in memory - pub rtos_memory: HvMemoryRegion, pub root_cell: HvCellDesc, // CellConfigLayout placed here. } diff --git a/modules/axhal/src/platform/x86_linux/consts.rs b/modules/axhal/src/platform/x86_linux/consts.rs index 5776694253..a7ed9bdf7a 100644 --- a/modules/axhal/src/platform/x86_linux/consts.rs +++ b/modules/axhal/src/platform/x86_linux/consts.rs @@ -2,38 +2,25 @@ use memory_addr::MemoryAddr; use super::config::HvSystemConfig; use super::header::HvHeader; -use super::percpu::PerCpu; use crate::mem::VirtAddr; - -/// Size of the hypervisor heap. -pub const HV_HEAP_SIZE: usize = 32 * 1024 * 1024; // 32 MB - -/// Size of the per-CPU data (stack and other CPU-local data). -pub const PER_CPU_SIZE: usize = 512 * 1024; // 512 KB - -/// Start virtual address of the hypervisor memory. -pub const HV_BASE: usize = 0xffff_ff00_0000_0000; +use axconfig::plat::KERNEL_BASE_VADDR; /// Pointer of the `HvHeader` structure. pub const HV_HEADER_PTR: *const HvHeader = __header_start as _; -/// Pointer of the per-CPU data array. -pub const PER_CPU_ARRAY_PTR: *mut PerCpu = _ekernel as _; - /// Pointer of the `HvSystemConfig` structure. pub fn hv_config_ptr() -> *const HvSystemConfig { - (PER_CPU_ARRAY_PTR as usize + HvHeader::get().max_cpus as usize * PER_CPU_SIZE) as _ + _ekernel as _ } /// Pointer of the free memory pool. pub fn free_memory_start() -> VirtAddr { VirtAddr::from(hv_config_ptr() as usize + HvSystemConfig::get().size()).align_up_4k() - // align_up(hv_config_ptr() as usize + HvSystemConfig::get().size()) } /// End virtual address of the hypervisor memory. pub fn hv_end() -> VirtAddr { - VirtAddr::from(HV_BASE + HvSystemConfig::get().hypervisor_memory.size as usize) + VirtAddr::from(KERNEL_BASE_VADDR + HvSystemConfig::get().hypervisor_memory.size as usize) } unsafe extern "C" { diff --git a/modules/axhal/src/platform/x86_linux/context.rs b/modules/axhal/src/platform/x86_linux/context.rs new file mode 100644 index 0000000000..e4e5362c0b --- /dev/null +++ b/modules/axhal/src/platform/x86_linux/context.rs @@ -0,0 +1,16 @@ +use lazyinit::LazyInit; + +use x86_vcpu::LinuxContext; + +#[percpu::def_percpu] +static LINUX_CTX: LazyInit = LazyInit::new(); + +pub fn set_linux_context(linux_sp: usize) { + let linux_ctx = unsafe { LINUX_CTX.current_ref_mut_raw() }; + + linux_ctx.init_once(LinuxContext::load_from(linux_sp)); +} + +pub fn get_linux_context() -> &'static LinuxContext { + unsafe { LINUX_CTX.current_ref_raw() } +} \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/entry.rs b/modules/axhal/src/platform/x86_linux/entry.rs index eee338a1d0..36dfe8823b 100644 --- a/modules/axhal/src/platform/x86_linux/entry.rs +++ b/modules/axhal/src/platform/x86_linux/entry.rs @@ -1,25 +1,53 @@ -use super::percpu::PerCpu; +use core::sync::atomic::{AtomicU32, Ordering}; +use super::header::HvHeader; +use axconfig::{SMP, TASK_STACK_SIZE}; + +static ENTERED_CPUS: AtomicU32 = AtomicU32::new(0); + +#[unsafe(link_section = ".bss.stack")] +static mut VMM_BOOT_STACK: [[u8; TASK_STACK_SIZE]; SMP] = [[0; TASK_STACK_SIZE]; SMP]; + +pub fn entered_cpus() -> u32 { + ENTERED_CPUS.load(Ordering::Acquire) +} + +#[unsafe(link_section = ".text.boot")] unsafe extern "sysv64" fn switch_stack(linux_sp: usize) -> i32 { unsafe { let linux_cr3 = x86::controlregs::cr3(); let linux_tp = x86::msr::rdmsr(x86::msr::IA32_GS_BASE); let vmm_entry = |linux_sp: usize| -> i32 { - let cpu_data = PerCpu::new(); - let hv_sp = cpu_data.stack_top(); + if entered_cpus() >= HvHeader::get().max_cpus { + panic!( + "enter cpus exceed Linux max cpus {}", + HvHeader::get().max_cpus + ); + } + if entered_cpus() >= SMP as u32 { + panic!("enter cpus exceed configured SMP {}", SMP); + } + + // Note: cpu_id here is not Local APIC ID, it is the index of entered CPUs. + // We just use it here to choose VMM_BOOT_STACK. + let core_id = ENTERED_CPUS.fetch_add(1, Ordering::SeqCst); + // let _cpu_id = current_cpu_id(); + + // let cpu_data = PerCpu::new(); + let hv_sp = VMM_BOOT_STACK[core_id as usize].as_ptr_range().end as usize; let ret; core::arch::asm!(" - mov [rsi], {linux_tp} // save gs_base to stack - mov rcx, rsp - mov rsp, {hv_sp} - push rcx - call {entry} - pop rsp", + mov [rsi], {linux_tp} // save gs_base to stack + mov rcx, rsp + mov rsp, {hv_sp} + push rcx + call {entry} + pop rsp", entry = sym super::vmm_cpu_entry, linux_tp = in(reg) linux_tp, hv_sp = in(reg) hv_sp, - in("rdi") cpu_data, + in("rdi") core_id, in("rsi") linux_sp, lateout("rax") ret, out("rcx") _, @@ -37,10 +65,12 @@ unsafe extern "sysv64" fn switch_stack(linux_sp: usize) -> i32 { } #[naked] +#[unsafe(link_section = ".text.boot")] #[unsafe(no_mangle)] pub unsafe extern "C" fn _start() -> i32 { unsafe { core::arch::naked_asm!(" + .code64 // rip is pushed cli push rbp diff --git a/modules/axhal/src/platform/x86_linux/header.rs b/modules/axhal/src/platform/x86_linux/header.rs index 0417acfe05..fd84b95957 100644 --- a/modules/axhal/src/platform/x86_linux/header.rs +++ b/modules/axhal/src/platform/x86_linux/header.rs @@ -1,13 +1,18 @@ use core::fmt::{Debug, Formatter, Result}; -use super::consts::{HV_HEADER_PTR, PER_CPU_SIZE}; +use super::consts::HV_HEADER_PTR; -const HEADER_SIGNATURE: [u8; 8] = *b"ARCEOSIM"; +const HEADER_SIGNATURE: [u8; 8] = *b"EVMIMAGE"; +/// Hypervisor description. +/// Located at the beginning of the hypervisor binary image and loaded by +/// the driver (which also initializes some fields). +/// See jailhouse dir `driver/jailhouse.h` for details. #[repr(C)] pub struct HvHeader { pub signature: [u8; 8], pub core_size: usize, + /// Not used, always 0. pub percpu_size: usize, pub entry: usize, /// Available CPU numbers provided by current physical platform. @@ -56,7 +61,7 @@ unsafe extern "C" { static HEADER_STUFF: HvHeaderStuff = HvHeaderStuff { signature: HEADER_SIGNATURE, core_size: __kernel_size, - percpu_size: PER_CPU_SIZE, + percpu_size: 0, entry: __entry_offset, max_cpus: 0, rt_cpus: 0, diff --git a/modules/axhal/src/platform/x86_linux/mem.rs b/modules/axhal/src/platform/x86_linux/mem.rs index 4d0c7649d0..99b7ebb8f9 100644 --- a/modules/axhal/src/platform/x86_linux/mem.rs +++ b/modules/axhal/src/platform/x86_linux/mem.rs @@ -1,10 +1,8 @@ // TODO: get memory regions from multiboot info. -use crate::mem::{virt_to_phys, MemRegion, MemRegionFlags, PhysAddr, VirtAddr, MemoryAddr}; +use crate::mem::{MemRegion, MemRegionFlags, MemoryAddr, PhysAddr, virt_to_phys}; use super::config::HvSystemConfig; -use super::consts::{HV_BASE, PER_CPU_ARRAY_PTR}; -use super::header::HvHeader; // MemRegion { // paddr: virt_to_phys((_stext as usize).into()), @@ -31,18 +29,24 @@ fn vmm_free_regions() -> impl Iterator { }) } -fn vmm_per_cpu_data_regions() -> impl Iterator { - let start_paddr = virt_to_phys(VirtAddr::from(PER_CPU_ARRAY_PTR as usize)); - let end_paddr = virt_to_phys(super::consts::free_memory_start()); - - let size = end_paddr.as_usize() - start_paddr.as_usize(); +pub fn host_memory_regions() -> impl Iterator { + use crate::mem::MemRegionFlags; - core::iter::once(MemRegion { - paddr: start_paddr, - size, - flags: MemRegionFlags::READ | MemRegionFlags::WRITE, - name: "per-CPU data, configurations", - }) + let sys_config = HvSystemConfig::get(); + let cell_config = &sys_config.root_cell.config(); + // Map all guest RAM to directly access in hypervisor. + cell_config + .mem_regions() + .iter() + .filter(|region| { + Into::::into(region.flags.clone()).contains(MemRegionFlags::DEVICE) + }) + .map(|region| MemRegion { + paddr: PhysAddr::from(region.phys_start as usize), + size: region.size as usize, + flags: region.flags.clone().into(), + name: "Linux mem", + }) } /// Returns platform-specific memory regions. @@ -55,32 +59,10 @@ pub(crate) fn platform_regions() -> impl Iterator { flags: MemRegionFlags::RESERVED | MemRegionFlags::READ, name: ".header", }) - .chain(vmm_per_cpu_data_regions()) .chain(vmm_free_regions()) // Here we do not use the `default_free_regions`` from `crate::mem`. // Cause we need to reserved regions for // per-CPU data and `HvSystemConfig` .chain(crate::mem::default_mmio_regions()) + .chain(host_memory_regions()) } - -pub fn host_memory_regions() -> impl Iterator { - use crate::mem::MemRegionFlags; - - let sys_config = HvSystemConfig::get(); - let cell_config = &sys_config.root_cell.config(); - // Map all guest RAM to directly access in hypervisor. - cell_config - .mem_regions() - .iter() - .filter(|region| { - MemRegionFlags::from_bits(region.flags as _) - .unwrap() - .contains(MemRegionFlags::DEVICE) - }) - .map(|region| MemRegion { - paddr: PhysAddr::from(region.phys_start as usize), - size: region.size as usize, - flags: MemRegionFlags::from_bits(region.flags as _).unwrap(), - name: "Linux mem", - }) -} \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/misc.rs b/modules/axhal/src/platform/x86_linux/misc.rs index 00bb63dd96..bc38365354 100644 --- a/modules/axhal/src/platform/x86_linux/misc.rs +++ b/modules/axhal/src/platform/x86_linux/misc.rs @@ -1,16 +1,16 @@ -// use x86_64::instructions::port::PortWriteOnly; +use x86_64::instructions::port::PortWriteOnly; /// Shutdown the whole system (in QEMU), including all CPUs. /// /// See for more information. pub fn terminate() -> ! { info!("Shutting down..."); - axlog::ax_println!("\nBut now we need to find a way to return this core to Linux!!!\n"); + axlog::ax_println!("\nMaybe we need to find a way to return this core to Linux?\n"); // #[cfg(platform = "x86_64-qemu-q35")] - // unsafe { - // PortWriteOnly::new(0x604).write(0x2000u16) - // }; + unsafe { + PortWriteOnly::new(0x604).write(0x2000u16) + }; crate::arch::halt(); warn!("It should shutdown!"); diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index dfdffff3c7..352785ae49 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -12,10 +12,11 @@ pub mod misc; pub mod time; // mods for vmm usage. -mod percpu; +// mod percpu; mod config; mod consts; +mod context; mod header; // #[cfg(feature = "smp")] @@ -37,7 +38,8 @@ use axlog::ax_println as println; use config::HvSystemConfig; // use error::HvResult; use header::HvHeader; -use percpu::PerCpu; + +use crate::cpu; static VMM_PRIMARY_INIT_OK: AtomicU32 = AtomicU32::new(0); static ERROR_NUM: AtomicI32 = AtomicI32::new(0); @@ -56,10 +58,18 @@ fn wait_for(condition: impl Fn() -> bool) { } unsafe extern "C" { - unsafe fn rust_vmm_main(cpu_id: usize); + unsafe fn rust_vmm_main(cpu_id: usize) -> isize; + #[cfg(feature = "smp")] + // unsafe fn rust_main_secondary(cpu_id: usize) -> !; unsafe fn rust_arceos_main(cpu_id: usize) -> !; } +unsafe extern "C" { + fn rust_main(cpu_id: usize, dtb: usize) -> !; + #[cfg(feature = "smp")] + fn rust_main_secondary(cpu_id: usize) -> !; +} + fn current_cpu_id() -> usize { match raw_cpuid::CpuId::new().get_feature_info() { Some(finfo) => finfo.initial_local_apic_id() as usize, @@ -67,18 +77,37 @@ fn current_cpu_id() -> usize { } } -fn vmm_primary_init_early(cpu_id: usize) { +fn vmm_primary_init_early() { + let cpu_id = current_cpu_id(); + // We do not clear bss here. // Because currently the image was loaded by Linux. - // crate::mem::clear_bss(); - + crate::mem::clear_bss(); crate::cpu::init_primary(cpu_id); + self::uart16550::init(); + VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); +} + +fn vmm_secondary_init_early() { + #[cfg(feature = "smp")] + { + let cpu_id = current_cpu_id(); + println!("Secondary CPU {} entered.", cpu_id); + crate::cpu::init_secondary(cpu_id); + } +} + +fn vmm_primary_init() { + let cpu_id = current_cpu_id(); + self::dtables::init_primary(); self::time::init_early(); - // println!("HvHeader\n{:#?}", HvHeader::get()); + println!("HvHeader\n{:#?}", HvHeader::get()); let system_config = HvSystemConfig::get(); + system_config.check(); + println!( "\n\ Initializing ARCEOS on Core [{}]...\n\ @@ -89,46 +118,65 @@ fn vmm_primary_init_early(cpu_id: usize) { core::str::from_utf8(&system_config.signature), system_config.revision, ); +} - VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); - - unsafe { - rust_vmm_main(cpu_id); +fn vmm_secondary_init() { + #[cfg(feature = "smp")] + { + self::dtables::init_secondary(); } } -extern "sysv64" fn vmm_cpu_entry(cpu_data: &mut PerCpu, _linux_sp: usize) -> i32 { - // Currently we set core 0 as Linux. - let is_primary = cpu_data.id == 0; +extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { + let cpu_id = current_cpu_id(); - let vm_cpus = HvHeader::get().reserved_cpus(); + // Use Cpu ID 0 as primary core. + // TODO: on some platform Local Apic ID may not start from Zero. + let is_primary = core_id == 0; + + println!( + "{} Core {} CPU {} entered.", + if is_primary { "Primary" } else { "Secondary" }, + core_id, + cpu_id, + ); - wait_for(|| PerCpu::entered_cpus() < vm_cpus); + let vm_cpus = HvHeader::get().reserved_cpus(); - // println!( - // "{} CPU {} entered.", - // if is_primary { "Primary" } else { "Secondary" }, - // cpu_data.id - // ); + wait_for(|| entry::entered_cpus() < vm_cpus); // First, we init primary core for VMM. if is_primary { - vmm_primary_init_early(cpu_data.id as usize); + vmm_primary_init_early(); } else { - // wait_for_counter(&VMM_PRIMARY_INIT_OK, 1); + vmm_secondary_init_early(); + } - // wait_for_counter(&VMM_MAIN_INIT_OK, 1); + // Note: this has to be done after `cpu::init_primary`. + // Because LinuxContext will be stored in percpu area. + context::set_linux_context(linux_sp); - // vmm_secondary_init_early(cpu_data.id as usize); + if is_primary { + vmm_primary_init(); + } else { + vmm_secondary_init(); + } + + unsafe { + if is_primary { + rust_main(cpu_id, 0); + } else { + rust_main_secondary(cpu_id); + } } let code = 0; - // println!( - // "{} CPU {} return back to driver with code {}.", - // if is_primary { "Primary" } else { "Secondary" }, - // cpu_data.id, - // code - // ); + println!( + "{} CPU {} return back to driver with code {}.", + if is_primary { "Primary" } else { "Secondary" }, + cpu_id, + code + ); code } @@ -163,18 +211,20 @@ unsafe extern "C" fn rust_entry_from_vmm(magic: usize) { /// Boot arceos cpus through sendsipi. pub fn vmm_platform_init() { self::lapic::init(); - self::mp::start_arceos_cpus(); + // self::mp::start_arceos_cpus(); } /// Initializes the platform devices for the primary CPU. pub fn platform_init() { - self::apic::init_primary(); - self::time::init_primary(); + self::lapic::init(); + + // self::apic::init_primary(); + // self::time::init_primary(); } /// Initializes the platform devices for secondary CPUs. #[cfg(feature = "smp")] pub fn platform_init_secondary() { - self::apic::init_secondary(); - self::time::init_secondary(); + // self::apic::init_secondary(); + // self::time::init_secondary(); } diff --git a/modules/axhal/src/platform/x86_linux/mp.rs b/modules/axhal/src/platform/x86_linux/mp.rs index 12bac5d6b5..2959f75d86 100644 --- a/modules/axhal/src/platform/x86_linux/mp.rs +++ b/modules/axhal/src/platform/x86_linux/mp.rs @@ -3,7 +3,7 @@ use crate::time::{busy_wait, Duration}; use core::sync::atomic::Ordering; use super::header::HvHeader; -use super::percpu::PerCpu; +// use super::percpu::PerCpu; use axconfig::{SMP, TASK_STACK_SIZE}; const START_PAGE_IDX: u8 = 6; @@ -67,9 +67,9 @@ pub fn start_arceos_cpus() { let mut arceos_cpu_num = 0; for apic_id in 0..max_cpus { - if PerCpu::cpu_is_booted(apic_id as usize) { - continue; - } + // if PerCpu::cpu_is_booted(apic_id as usize) { + // continue; + // } let stack_top = virt_to_phys(VirtAddr::from( SECONDARY_BOOT_STACK[arceos_cpu_num].as_ptr_range().end as usize, )) diff --git a/modules/axhal/src/platform/x86_linux/multiboot.S b/modules/axhal/src/platform/x86_linux/multiboot.S index dfa16a12bf..ec6cc7ce5d 100644 --- a/modules/axhal/src/platform/x86_linux/multiboot.S +++ b/modules/axhal/src/platform/x86_linux/multiboot.S @@ -124,12 +124,10 @@ ap_entry64: .Ltmp_pml4: # 0x0000_0000 ~ 0xffff_ffff .quad .Ltmp_pdpt_low - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) - .zero 8 * 509 - # 0xffff_ff00_0000_0000 ~ 0xffff_ff00_ffff_ffff - # 0xffff_ff00_0000_0000 ~ 0xffff_ff00_0800_0000 - # 0x0000_0000_4000_0000 ~ 0x0000_0000_4800_0000 + .zero 8 * 255 + # 0xffff_8000_0000_0000 ~ 0xffff_8000_ffff_ffff .quad .Ltmp_pdpt_high - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) - .zero 8 * 1 + .zero 8 * 255 # FIXME: may not work on macOS using hvf as the CPU does not support 1GB page (pdpe1gb) .Ltmp_pdpt_low: diff --git a/modules/axhal/src/platform/x86_linux/percpu.rs b/modules/axhal/src/platform/x86_linux/percpu.rs index 1a01b27184..7206cdc394 100644 --- a/modules/axhal/src/platform/x86_linux/percpu.rs +++ b/modules/axhal/src/platform/x86_linux/percpu.rs @@ -4,7 +4,6 @@ use core::sync::atomic::{AtomicU32, Ordering}; // use crate::arch::vmm::{Vcpu, VcpuAccessGuestState}; // use crate::arch::{cpu, ArchPerCpu, LinuxContext}; // use crate::cell::Cell; -use super::consts::{PER_CPU_ARRAY_PTR, PER_CPU_SIZE}; use super::current_cpu_id; // use crate::error::HvResult; use super::header::HvHeader; @@ -56,10 +55,6 @@ impl PerCpu { ret } - pub unsafe fn from_id_mut<'a>(cpu_id: u32) -> &'a mut Self { - let vaddr = PER_CPU_ARRAY_PTR as usize + cpu_id as usize * PER_CPU_SIZE; - &mut *(vaddr as *mut Self) - } pub fn current<'a>() -> &'a Self { Self::current_mut() diff --git a/modules/axhal/src/platform/x86_linux/time.rs b/modules/axhal/src/platform/x86_linux/time.rs index f146eaccdb..13524f8e76 100644 --- a/modules/axhal/src/platform/x86_linux/time.rs +++ b/modules/axhal/src/platform/x86_linux/time.rs @@ -10,7 +10,7 @@ const LAPIC_TICKS_PER_SEC: u64 = 1_000_000_000; // TODO: need to calibrate static mut NANOS_TO_LAPIC_TICKS_RATIO: Ratio = Ratio::zero(); static mut INIT_TICK: u64 = 0; -static mut CPU_FREQ_MHZ: u64 = axconfig::devices::TIMER_FREQUENCY as u64 / 1_000_000; +static mut CPU_FREQ_MHZ: u64 = 1 as u64 / 1_000_000; /// RTC wall time offset in nanoseconds at monotonic time base. static mut RTC_EPOCHOFFSET_NANOS: u64 = 0; diff --git a/modules/axhal/src/platform/x86_linux/uart16550.rs b/modules/axhal/src/platform/x86_linux/uart16550.rs index 66ee56f01c..5909b9524e 100644 --- a/modules/axhal/src/platform/x86_linux/uart16550.rs +++ b/modules/axhal/src/platform/x86_linux/uart16550.rs @@ -6,7 +6,9 @@ use x86_64::instructions::port::{Port, PortReadOnly, PortWriteOnly}; const UART_CLOCK_FACTOR: usize = 16; const OSC_FREQ: usize = 1_843_200; -static COM1: SpinNoIrq = SpinNoIrq::new(Uart16550::new(0x3f8)); +const PORT_COM1: u16 = 0x2f8; + +static COM1: SpinNoIrq = SpinNoIrq::new(Uart16550::new(PORT_COM1)); bitflags::bitflags! { /// Line status flags diff --git a/modules/axmm/src/aspace.rs b/modules/axmm/src/aspace.rs index 23165d996f..d4b675a09e 100644 --- a/modules/axmm/src/aspace.rs +++ b/modules/axmm/src/aspace.rs @@ -74,6 +74,7 @@ impl AddrSpace { flags: MappingFlags, ) -> AxResult { if !self.contains_range(start_vaddr, size) { + warn!("address out of range: {:?} + {}, paddr {:?} {:?}", start_vaddr, size, start_paddr, flags); return ax_err!(InvalidInput, "address out of range"); } if !start_vaddr.is_aligned_4k() || !start_paddr.is_aligned_4k() || !is_aligned_4k(size) { diff --git a/modules/axruntime/src/vmm.rs b/modules/axruntime/src/vmm.rs index 47cacf761b..d3a85ea0bc 100644 --- a/modules/axruntime/src/vmm.rs +++ b/modules/axruntime/src/vmm.rs @@ -5,7 +5,7 @@ use core::sync::atomic::Ordering; fn is_init_ok() -> bool { - super::INITED_CPUS.load(Ordering::Acquire) == (axconfig::SMP + 1) + super::INITED_CPUS.load(Ordering::Acquire) == axconfig::SMP } unsafe extern "C" { @@ -61,13 +61,12 @@ pub extern "C" fn rust_arceos_main(cpu_id: usize) { /// The main entry point of the ArceOS-VMM **when booting from Linux and set self as VMM**! /// /// It is called from the `vmm_cpu_entry` code in [axhal]. `cpu_id` is the ID of -/// the current CPU, and `dtb` is the address of the device tree blob. It -/// finally calls the application's `main` function after all initialization -/// work is done. +/// the current CPU. It finally calls the application's `main` function after all +/// initialization work is done. /// /// In multi-core environment, this function is called on the primary CPU. #[cfg_attr(not(test), unsafe(no_mangle))] -pub extern "C" fn rust_vmm_main(cpu_id: usize) { +pub extern "C" fn rust_vmm_main(cpu_id: usize) -> isize { ax_println!("{}", super::LOGO); ax_println!( "\ @@ -105,13 +104,13 @@ pub extern "C" fn rust_vmm_main(cpu_id: usize) { super::init_allocator(); #[cfg(feature = "paging")] - { - info!("Initialize kernel page table..."); - vmm_remap_kernel_memory().expect("remap kernel memory failed"); - } + axmm::init_memory_management(); + + info!("Initialize platform devices..."); + axhal::platform_init(); - info!("Initialize VMM platform..."); - axhal::vmm_platform_init(); + #[cfg(feature = "multitask")] + axtask::init_scheduler(); // info!("VMM Primary CPU {} init OK.", cpu_id); @@ -120,40 +119,50 @@ pub extern "C" fn rust_vmm_main(cpu_id: usize) { while !is_init_ok() { core::hint::spin_loop(); } + + unsafe { main() }; + + info!("VMM main task exited: exit_code={}", -1); + -1 } -#[cfg(feature = "paging")] -fn vmm_remap_kernel_memory() -> Result<(), axhal::paging::PagingError> { - use axhal::host_memory_regions; - use axhal::mem::{memory_regions, phys_to_virt}; - use axhal::paging::PageTable; - use lazyinit::LazyInit; +use lazyinit::LazyInit; - static KERNEL_PAGE_TABLE: LazyInit = LazyInit::new(); +use axhal::host_memory_regions; +use axhal::mem::{memory_regions, phys_to_virt}; +use axhal::paging::PageTable; +static KERNEL_PAGE_TABLE: LazyInit = LazyInit::new(); + +#[cfg(feature = "paging")] +fn vmm_remap_kernel_memory() -> Result<(), axhal::paging::PagingError> { if axhal::cpu::this_cpu_is_bsp() { info!("BSP CPU init KERNEL_PAGE_TABLE..."); let mut kernel_page_table = PageTable::try_new()?; for r in memory_regions() { - kernel_page_table.map_region( - phys_to_virt(r.paddr), - |_| r.paddr, - r.size, - r.flags.into(), - false, - false, - )?; + kernel_page_table + .map_region( + phys_to_virt(r.paddr), + |_| r.paddr, + r.size, + r.flags.into(), + false, + false, + )? + .flush_all(); } for r in host_memory_regions() { - kernel_page_table.map_region( - phys_to_virt(r.paddr), - |_| r.paddr, - r.size, - r.flags.into(), - false, - false, - )?; + kernel_page_table + .map_region( + phys_to_virt(r.paddr), + |_| r.paddr, + r.size, + r.flags.into(), + false, + false, + )? + .flush_all(); } KERNEL_PAGE_TABLE.init_once(kernel_page_table); @@ -164,3 +173,8 @@ fn vmm_remap_kernel_memory() -> Result<(), axhal::paging::PagingError> { unsafe { axhal::arch::write_page_table_root(KERNEL_PAGE_TABLE.root_paddr()) }; Ok(()) } + +/// Initializes kernel paging for secondary CPUs. +pub fn init_memory_management_secondary() { + unsafe { axhal::arch::write_page_table_root(KERNEL_PAGE_TABLE.root_paddr()) }; +} From 5f34ace34831dc795eb66587d315b67669aed089 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 18 Mar 2025 11:56:13 +0800 Subject: [PATCH 11/19] [wip] boot and enter main --- modules/axhal/src/platform/x86_linux/boot.rs | 34 ++-- modules/axhal/src/platform/x86_linux/mod.rs | 54 ++---- modules/axhal/src/platform/x86_linux/time.rs | 2 +- modules/axmm/src/aspace.rs | 5 +- modules/axruntime/src/lang_items.rs | 2 +- modules/axruntime/src/lib.rs | 4 - modules/axruntime/src/vmm.rs | 180 ------------------- 7 files changed, 41 insertions(+), 240 deletions(-) delete mode 100644 modules/axruntime/src/vmm.rs diff --git a/modules/axhal/src/platform/x86_linux/boot.rs b/modules/axhal/src/platform/x86_linux/boot.rs index 8ece1cd754..1488072f50 100644 --- a/modules/axhal/src/platform/x86_linux/boot.rs +++ b/modules/axhal/src/platform/x86_linux/boot.rs @@ -33,20 +33,20 @@ const EFER: u64 = EferFlags::LONG_MODE_ENABLE.bits() | EferFlags::NO_EXECUTE_ENA #[unsafe(link_section = ".bss.stack")] static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; -global_asm!( - include_str!("multiboot.S"), - mb_magic = const MULTIBOOT_BOOTLOADER_MAGIC, - mb_hdr_magic = const MULTIBOOT_HEADER_MAGIC, - mb_hdr_flags = const MULTIBOOT_HEADER_FLAGS, - entry = sym super::rust_entry, - entry_secondary = sym super::rust_entry_from_vmm, - - offset = const PHYS_VIRT_OFFSET, - boot_stack_size = const TASK_STACK_SIZE, - boot_stack = sym BOOT_STACK, - - cr0 = const CR0, - cr4 = const CR4, - efer_msr = const x86::msr::IA32_EFER, - efer = const EFER, -); +// global_asm!( +// include_str!("multiboot.S"), +// mb_magic = const MULTIBOOT_BOOTLOADER_MAGIC, +// mb_hdr_magic = const MULTIBOOT_HEADER_MAGIC, +// mb_hdr_flags = const MULTIBOOT_HEADER_FLAGS, +// entry = sym super::rust_entry, +// entry_secondary = sym super::rust_entry_from_vmm, + +// offset = const PHYS_VIRT_OFFSET, +// boot_stack_size = const TASK_STACK_SIZE, +// boot_stack = sym BOOT_STACK, + +// cr0 = const CR0, +// cr4 = const CR4, +// efer_msr = const x86::msr::IA32_EFER, +// efer = const EFER, +// ); diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 352785ae49..5fd1f7bbb4 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -48,7 +48,7 @@ fn has_err() -> bool { ERROR_NUM.load(Ordering::Acquire) != 0 } -fn wait_for(condition: impl Fn() -> bool) { +fn wait_while(condition: impl Fn() -> bool) { while !has_err() && condition() { core::hint::spin_loop(); } @@ -57,13 +57,6 @@ fn wait_for(condition: impl Fn() -> bool) { } } -unsafe extern "C" { - unsafe fn rust_vmm_main(cpu_id: usize) -> isize; - #[cfg(feature = "smp")] - // unsafe fn rust_main_secondary(cpu_id: usize) -> !; - unsafe fn rust_arceos_main(cpu_id: usize) -> !; -} - unsafe extern "C" { fn rust_main(cpu_id: usize, dtb: usize) -> !; #[cfg(feature = "smp")] @@ -80,19 +73,19 @@ fn current_cpu_id() -> usize { fn vmm_primary_init_early() { let cpu_id = current_cpu_id(); + println!("Primary CPU {} init early", cpu_id); // We do not clear bss here. // Because currently the image was loaded by Linux. - crate::mem::clear_bss(); + // crate::mem::clear_bss(); crate::cpu::init_primary(cpu_id); self::uart16550::init(); - VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); } fn vmm_secondary_init_early() { #[cfg(feature = "smp")] { let cpu_id = current_cpu_id(); - println!("Secondary CPU {} entered.", cpu_id); + println!("Secondary CPU {} init early.", cpu_id); crate::cpu::init_secondary(cpu_id); } } @@ -134,21 +127,31 @@ extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { // TODO: on some platform Local Apic ID may not start from Zero. let is_primary = core_id == 0; + let vm_cpus = HvHeader::get().reserved_cpus(); + println!( - "{} Core {} CPU {} entered.", + "{} Core {} CPU {} entered. {} of {}", if is_primary { "Primary" } else { "Secondary" }, core_id, cpu_id, + entry::entered_cpus(), + vm_cpus ); - let vm_cpus = HvHeader::get().reserved_cpus(); + wait_while(|| entry::entered_cpus() < vm_cpus); - wait_for(|| entry::entered_cpus() < vm_cpus); + println!( + "{} Core {} CPU {} start to initialize.", + if is_primary { "Primary" } else { "Secondary" }, + core_id, + cpu_id, + ); // First, we init primary core for VMM. if is_primary { vmm_primary_init_early(); } else { + wait_while(|| VMM_PRIMARY_INIT_OK.load(Ordering::Acquire) == 0); vmm_secondary_init_early(); } @@ -192,32 +195,11 @@ unsafe extern "C" fn rust_entry(_magic: usize, _mbi: usize) { // } } -#[allow(unused_variables)] -unsafe extern "C" fn rust_entry_from_vmm(magic: usize) { - let cpu_id = current_cpu_id(); - info!("ARCEOS CPU entered on Core {}.", cpu_id); - - if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { - crate::cpu::init_secondary(cpu_id); - self::dtables::init_primary(); - self::time::init_early(); - rust_arceos_main(cpu_id); - } else { - panic!("Something is wrong during booting RT cores..."); - } -} - -/// Initializes the platform devices for the primary CPU. -/// Boot arceos cpus through sendsipi. -pub fn vmm_platform_init() { - self::lapic::init(); - // self::mp::start_arceos_cpus(); -} - /// Initializes the platform devices for the primary CPU. pub fn platform_init() { self::lapic::init(); + VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); // self::apic::init_primary(); // self::time::init_primary(); } diff --git a/modules/axhal/src/platform/x86_linux/time.rs b/modules/axhal/src/platform/x86_linux/time.rs index 13524f8e76..f146eaccdb 100644 --- a/modules/axhal/src/platform/x86_linux/time.rs +++ b/modules/axhal/src/platform/x86_linux/time.rs @@ -10,7 +10,7 @@ const LAPIC_TICKS_PER_SEC: u64 = 1_000_000_000; // TODO: need to calibrate static mut NANOS_TO_LAPIC_TICKS_RATIO: Ratio = Ratio::zero(); static mut INIT_TICK: u64 = 0; -static mut CPU_FREQ_MHZ: u64 = 1 as u64 / 1_000_000; +static mut CPU_FREQ_MHZ: u64 = axconfig::devices::TIMER_FREQUENCY as u64 / 1_000_000; /// RTC wall time offset in nanoseconds at monotonic time base. static mut RTC_EPOCHOFFSET_NANOS: u64 = 0; diff --git a/modules/axmm/src/aspace.rs b/modules/axmm/src/aspace.rs index d4b675a09e..e7cc036beb 100644 --- a/modules/axmm/src/aspace.rs +++ b/modules/axmm/src/aspace.rs @@ -74,7 +74,10 @@ impl AddrSpace { flags: MappingFlags, ) -> AxResult { if !self.contains_range(start_vaddr, size) { - warn!("address out of range: {:?} + {}, paddr {:?} {:?}", start_vaddr, size, start_paddr, flags); + warn!( + "address out of range: {:?} + {:#x}, paddr {:?} {:?}", + start_vaddr, size, start_paddr, flags + ); return ax_err!(InvalidInput, "address out of range"); } if !start_vaddr.is_aligned_4k() || !start_paddr.is_aligned_4k() || !is_aligned_4k(size) { diff --git a/modules/axruntime/src/lang_items.rs b/modules/axruntime/src/lang_items.rs index 6600cdd746..61a5869ee2 100644 --- a/modules/axruntime/src/lang_items.rs +++ b/modules/axruntime/src/lang_items.rs @@ -2,6 +2,6 @@ use core::panic::PanicInfo; #[panic_handler] fn panic(info: &PanicInfo) -> ! { - error!("{}", info); + axlog::ax_println!("[{}]:{}", axhal::cpu::this_cpu_id(), info); axhal::misc::terminate() } diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 91c2c84671..114db35cc0 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -31,10 +31,6 @@ mod mp; #[cfg(feature = "smp")] pub use self::mp::rust_main_secondary; -mod vmm; - -pub use self::vmm::{rust_vmm_main, rust_arceos_main}; - const LOGO: &str = r#" d8888 .d88888b. .d8888b. d88888 d88P" "Y88b d88P Y88b diff --git a/modules/axruntime/src/vmm.rs b/modules/axruntime/src/vmm.rs deleted file mode 100644 index d3a85ea0bc..0000000000 --- a/modules/axruntime/src/vmm.rs +++ /dev/null @@ -1,180 +0,0 @@ -/// Some problems remain: -/// 1. How to return rt core back to Linux so we can "dynamically" control these rt-cores. -/// 2. How to co-schedual process from Linux within process inside arceos. -/// 3. How to handle memory mapping, currently both sides have a complete memory view of the other's address space. -use core::sync::atomic::Ordering; - -fn is_init_ok() -> bool { - super::INITED_CPUS.load(Ordering::Acquire) == axconfig::SMP -} - -unsafe extern "C" { - unsafe fn main(); -} - -/// The main entry point of the ArceOS runtime for RT CPUs. -/// When booting from Linux and reserving CPU for RT tasks! -/// -/// It is called from the `rust_entry_from_vmm` code in [axhal]. -#[unsafe(no_mangle)] -pub extern "C" fn rust_arceos_main(cpu_id: usize) { - info!("ARCEOS CPU {:x} started.", cpu_id); - - #[cfg(feature = "paging")] - { - info!("Initialize kernel page table..."); - vmm_remap_kernel_memory().expect("remap kernel memoy failed"); - } - - info!("Initialize platform devices..."); - axhal::platform_init(); - - #[cfg(feature = "multitask")] - axtask::init_scheduler(); - - #[cfg(feature = "irq")] - { - info!("Initialize interrupt handlers..."); - super::init_interrupt(); - } - - info!("ARCEOS Primary CPU {:x} init OK.", cpu_id); - super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); - - while !is_init_ok() { - core::hint::spin_loop(); - } - - info!("ARCEOS Primary CPU {:x} enter main ...", cpu_id); - unsafe { main() }; - info!("ARCEOS main task exited: exit_code={}", 0); - - #[cfg(feature = "multitask")] - axtask::exit(0); - #[cfg(not(feature = "multitask"))] - { - debug!("main task exited: exit_code={}", 0); - axhal::misc::terminate(); - } -} - -/// The main entry point of the ArceOS-VMM **when booting from Linux and set self as VMM**! -/// -/// It is called from the `vmm_cpu_entry` code in [axhal]. `cpu_id` is the ID of -/// the current CPU. It finally calls the application's `main` function after all -/// initialization work is done. -/// -/// In multi-core environment, this function is called on the primary CPU. -#[cfg_attr(not(test), unsafe(no_mangle))] -pub extern "C" fn rust_vmm_main(cpu_id: usize) -> isize { - ax_println!("{}", super::LOGO); - ax_println!( - "\ - arch = {}\n\ - platform = {}\n\ - target = {}\n\ - smp = {}\n\ - build_mode = {}\n\ - log_level = {}\n\ - ", - option_env!("AX_ARCH").unwrap_or(""), - option_env!("AX_PLATFORM").unwrap_or(""), - option_env!("AX_TARGET").unwrap_or(""), - option_env!("AX_SMP").unwrap_or(""), - option_env!("AX_MODE").unwrap_or(""), - option_env!("AX_LOG").unwrap_or(""), - ); - - axlog::init(); - axlog::set_max_level(option_env!("AX_LOG").unwrap_or("")); // no effect if set `log-level-*` features - info!("Logging is enabled."); - info!("VMM Primary CPU {} started", cpu_id); - - info!("Found physcial memory regions:"); - for r in axhal::mem::memory_regions() { - info!( - " [{:x?}, {:x?}) {} ({:?})", - r.paddr, - r.paddr + r.size, - r.name, - r.flags - ); - } - - super::init_allocator(); - - #[cfg(feature = "paging")] - axmm::init_memory_management(); - - info!("Initialize platform devices..."); - axhal::platform_init(); - - #[cfg(feature = "multitask")] - axtask::init_scheduler(); - - // info!("VMM Primary CPU {} init OK.", cpu_id); - - super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); - - while !is_init_ok() { - core::hint::spin_loop(); - } - - unsafe { main() }; - - info!("VMM main task exited: exit_code={}", -1); - -1 -} - -use lazyinit::LazyInit; - -use axhal::host_memory_regions; -use axhal::mem::{memory_regions, phys_to_virt}; -use axhal::paging::PageTable; - -static KERNEL_PAGE_TABLE: LazyInit = LazyInit::new(); - -#[cfg(feature = "paging")] -fn vmm_remap_kernel_memory() -> Result<(), axhal::paging::PagingError> { - if axhal::cpu::this_cpu_is_bsp() { - info!("BSP CPU init KERNEL_PAGE_TABLE..."); - let mut kernel_page_table = PageTable::try_new()?; - for r in memory_regions() { - kernel_page_table - .map_region( - phys_to_virt(r.paddr), - |_| r.paddr, - r.size, - r.flags.into(), - false, - false, - )? - .flush_all(); - } - - for r in host_memory_regions() { - kernel_page_table - .map_region( - phys_to_virt(r.paddr), - |_| r.paddr, - r.size, - r.flags.into(), - false, - false, - )? - .flush_all(); - } - - KERNEL_PAGE_TABLE.init_once(kernel_page_table); - - info!("KERNEL_PAGE_TABLE init success"); - } - - unsafe { axhal::arch::write_page_table_root(KERNEL_PAGE_TABLE.root_paddr()) }; - Ok(()) -} - -/// Initializes kernel paging for secondary CPUs. -pub fn init_memory_management_secondary() { - unsafe { axhal::arch::write_page_table_root(KERNEL_PAGE_TABLE.root_paddr()) }; -} From 30ec987a474585a5756762aba9e9d02750878309 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 18 Mar 2025 22:49:14 +0800 Subject: [PATCH 12/19] [wip] how to store LinuxContext for each cpu elegantly --- modules/axhal/src/lib.rs | 4 ++- .../axhal/src/platform/x86_linux/config.rs | 4 ++- .../axhal/src/platform/x86_linux/context.rs | 34 +++++++++++++++++-- modules/axhal/src/platform/x86_linux/mem.rs | 27 ++++++--------- modules/axhal/src/platform/x86_linux/mod.rs | 4 ++- 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/modules/axhal/src/lib.rs b/modules/axhal/src/lib.rs index caf4ca51a4..3951a65ffe 100644 --- a/modules/axhal/src/lib.rs +++ b/modules/axhal/src/lib.rs @@ -28,6 +28,7 @@ #![feature(naked_functions)] #![feature(doc_auto_cfg)] #![feature(sync_unsafe_cell)] +#![feature(const_trait_impl)] #[allow(unused_imports)] #[macro_use] @@ -78,4 +79,5 @@ pub use self::platform::platform_init; pub use self::platform::platform_init_secondary; pub use self::platform::mem::host_memory_regions; -pub use self::platform::vmm_platform_init; +pub use self::platform::get_linux_context_by_cpu_id; +pub use self::platform::get_linux_context_list; \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/config.rs b/modules/axhal/src/platform/x86_linux/config.rs index 5765802808..af8a2774e4 100644 --- a/modules/axhal/src/platform/x86_linux/config.rs +++ b/modules/axhal/src/platform/x86_linux/config.rs @@ -1,9 +1,11 @@ +//! Cell configuration structures inherited from Jailhouse. + use core::fmt::{Debug, Formatter, Result}; use core::{mem::size_of, slice}; use bitflags::bitflags; -use crate::mem::{MemRegion, MemRegionFlags}; +use crate::mem::MemRegionFlags; const CONFIG_SIGNATURE: [u8; 6] = *b"EVMSYS"; const CONFIG_REVISION: u16 = 314; diff --git a/modules/axhal/src/platform/x86_linux/context.rs b/modules/axhal/src/platform/x86_linux/context.rs index e4e5362c0b..112b049e07 100644 --- a/modules/axhal/src/platform/x86_linux/context.rs +++ b/modules/axhal/src/platform/x86_linux/context.rs @@ -1,16 +1,46 @@ +use core::mem::MaybeUninit; + use lazyinit::LazyInit; use x86_vcpu::LinuxContext; +use axconfig::SMP; + +use crate::cpu::this_cpu_id; + #[percpu::def_percpu] static LINUX_CTX: LazyInit = LazyInit::new(); +static mut LINUX_CTX_LIST: LazyInit<[MaybeUninit; SMP]> = LazyInit::new(); + +/// Set Linux context for current CPU. pub fn set_linux_context(linux_sp: usize) { let linux_ctx = unsafe { LINUX_CTX.current_ref_mut_raw() }; - linux_ctx.init_once(LinuxContext::load_from(linux_sp)); + + unsafe { + if LINUX_CTX_LIST.is_inited() { + LINUX_CTX_LIST[this_cpu_id()].write(LinuxContext::load_from(linux_sp)); + } else { + let mut list = [const { MaybeUninit::uninit() }; SMP]; + list[this_cpu_id()].write(LinuxContext::load_from(linux_sp)); + LINUX_CTX_LIST.init_once(list); + } + } } +/// Get Linux context for current CPU. +#[allow(unused)] pub fn get_linux_context() -> &'static LinuxContext { unsafe { LINUX_CTX.current_ref_raw() } -} \ No newline at end of file +} + +/// Get Linux context for the given CPU ID. +pub fn get_linux_context_by_cpu_id(cpu_id: usize) -> &'static LinuxContext { + unsafe { LINUX_CTX.remote_ref_raw(cpu_id) } +} + +pub fn get_linux_context_list() -> &'static [LinuxContext; SMP] { + // LINUX_CTX_LIST.as_ref() + unsafe { &*(&raw const LINUX_CTX_LIST as *const [LinuxContext; SMP]) } +} diff --git a/modules/axhal/src/platform/x86_linux/mem.rs b/modules/axhal/src/platform/x86_linux/mem.rs index 99b7ebb8f9..a54f780f1a 100644 --- a/modules/axhal/src/platform/x86_linux/mem.rs +++ b/modules/axhal/src/platform/x86_linux/mem.rs @@ -30,23 +30,15 @@ fn vmm_free_regions() -> impl Iterator { } pub fn host_memory_regions() -> impl Iterator { - use crate::mem::MemRegionFlags; - let sys_config = HvSystemConfig::get(); let cell_config = &sys_config.root_cell.config(); - // Map all guest RAM to directly access in hypervisor. - cell_config - .mem_regions() - .iter() - .filter(|region| { - Into::::into(region.flags.clone()).contains(MemRegionFlags::DEVICE) - }) - .map(|region| MemRegion { - paddr: PhysAddr::from(region.phys_start as usize), - size: region.size as usize, - flags: region.flags.clone().into(), - name: "Linux mem", - }) + + cell_config.mem_regions().iter().map(|region| MemRegion { + paddr: PhysAddr::from(region.phys_start as usize), + size: region.size as usize, + flags: region.flags.clone().into(), + name: "Linux mem", + }) } /// Returns platform-specific memory regions. @@ -64,5 +56,8 @@ pub(crate) fn platform_regions() -> impl Iterator { // Cause we need to reserved regions for // per-CPU data and `HvSystemConfig` .chain(crate::mem::default_mmio_regions()) - .chain(host_memory_regions()) + .chain(host_memory_regions().filter(|region| { + // Map all guest RAM to directly access in hypervisor. + region.flags.contains(MemRegionFlags::DEVICE) + })) } diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 5fd1f7bbb4..7c1efbab52 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -31,6 +31,9 @@ pub mod console { pub use super::uart16550::*; } +pub use context::get_linux_context_by_cpu_id; +pub use context::get_linux_context_list; + use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; use axlog::ax_println as println; @@ -39,7 +42,6 @@ use config::HvSystemConfig; // use error::HvResult; use header::HvHeader; -use crate::cpu; static VMM_PRIMARY_INIT_OK: AtomicU32 = AtomicU32::new(0); static ERROR_NUM: AtomicI32 = AtomicI32::new(0); From d2f45f12f41f930db26ffd3e955672a77640a9f9 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 20 Mar 2025 22:18:47 +0800 Subject: [PATCH 13/19] [feat] support type15 in axvisor, wait for improvement --- modules/axhal/src/platform/x86_linux/apic.rs | 22 +++++++---- .../axhal/src/platform/x86_linux/consts.rs | 9 ++++- .../axhal/src/platform/x86_linux/context.rs | 17 ++++++--- modules/axhal/src/platform/x86_linux/mem.rs | 17 ++++++++- modules/axhal/src/platform/x86_linux/mod.rs | 37 +++++++++---------- modules/axtask/src/api.rs | 7 ++-- 6 files changed, 72 insertions(+), 37 deletions(-) diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index 3a6f93ad7a..5b946d79dd 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -86,13 +86,15 @@ fn cpu_has_x2apic() -> bool { } } -pub(super) fn init_primary() { +pub(super) fn init_primary(enabled: bool) { info!("Initialize Local APIC..."); - unsafe { - // Disable 8259A interrupt controllers - Port::::new(0x21).write(0xff); - Port::::new(0xA1).write(0xff); + if enabled { + unsafe { + // Disable 8259A interrupt controllers + Port::::new(0x21).write(0xff); + Port::::new(0xA1).write(0xff); + } } let mut builder = LocalApicBuilder::new(); @@ -112,7 +114,9 @@ pub(super) fn init_primary() { let mut lapic = builder.build().unwrap(); unsafe { - lapic.enable(); + if enabled { + lapic.enable(); + } LOCAL_APIC.get().as_mut().unwrap().write(lapic); } @@ -122,6 +126,8 @@ pub(super) fn init_primary() { } #[cfg(feature = "smp")] -pub(super) fn init_secondary() { - unsafe { local_apic().enable() }; +pub(super) fn init_secondary(enabled: bool) { + if enabled { + unsafe { local_apic().enable() }; + } } diff --git a/modules/axhal/src/platform/x86_linux/consts.rs b/modules/axhal/src/platform/x86_linux/consts.rs index a7ed9bdf7a..e6d4c9aab3 100644 --- a/modules/axhal/src/platform/x86_linux/consts.rs +++ b/modules/axhal/src/platform/x86_linux/consts.rs @@ -1,4 +1,4 @@ -use memory_addr::MemoryAddr; +use memory_addr::{MemoryAddr, is_aligned_4k}; use super::config::HvSystemConfig; use super::header::HvHeader; @@ -13,6 +13,13 @@ pub fn hv_config_ptr() -> *const HvSystemConfig { _ekernel as _ } +pub fn cfg_region_start() -> VirtAddr { + let cfg_ptr = hv_config_ptr() as usize; + // The linker script ensures that `ekernel` is aligned to 4KB. + assert!(is_aligned_4k(cfg_ptr), "_ekernel is not aligned to 4KB"); + VirtAddr::from(cfg_ptr) +} + /// Pointer of the free memory pool. pub fn free_memory_start() -> VirtAddr { VirtAddr::from(hv_config_ptr() as usize + HvSystemConfig::get().size()).align_up_4k() diff --git a/modules/axhal/src/platform/x86_linux/context.rs b/modules/axhal/src/platform/x86_linux/context.rs index 112b049e07..0a7534abc1 100644 --- a/modules/axhal/src/platform/x86_linux/context.rs +++ b/modules/axhal/src/platform/x86_linux/context.rs @@ -14,18 +14,25 @@ static LINUX_CTX: LazyInit = LazyInit::new(); static mut LINUX_CTX_LIST: LazyInit<[MaybeUninit; SMP]> = LazyInit::new(); /// Set Linux context for current CPU. -pub fn set_linux_context(linux_sp: usize) { - let linux_ctx = unsafe { LINUX_CTX.current_ref_mut_raw() }; - linux_ctx.init_once(LinuxContext::load_from(linux_sp)); +pub fn set_linux_context(linux_sp: usize, cpu_id: usize) { + // let linux_ctx = unsafe { LINUX_CTX.current_ref_mut_raw() }; + // linux_ctx.init_once(LinuxContext::load_from(linux_sp)); unsafe { if LINUX_CTX_LIST.is_inited() { - LINUX_CTX_LIST[this_cpu_id()].write(LinuxContext::load_from(linux_sp)); + LINUX_CTX_LIST[cpu_id].write(LinuxContext::load_from(linux_sp)); } else { let mut list = [const { MaybeUninit::uninit() }; SMP]; - list[this_cpu_id()].write(LinuxContext::load_from(linux_sp)); + list[cpu_id].write(LinuxContext::load_from(linux_sp)); LINUX_CTX_LIST.init_once(list); } + + axlog::ax_println!( + "Core {} {} Linux context set {:#x?}", + this_cpu_id(), + cpu_id, + &LINUX_CTX_LIST[cpu_id].assume_init() + ); } } diff --git a/modules/axhal/src/platform/x86_linux/mem.rs b/modules/axhal/src/platform/x86_linux/mem.rs index a54f780f1a..48bf91314e 100644 --- a/modules/axhal/src/platform/x86_linux/mem.rs +++ b/modules/axhal/src/platform/x86_linux/mem.rs @@ -29,6 +29,20 @@ fn vmm_free_regions() -> impl Iterator { }) } +fn vmm_cfg_regions() -> impl Iterator { + let vmm_cfg_start = super::consts::cfg_region_start(); + let vmm_cfg_end = super::consts::free_memory_start(); + let vmm_cfg_size = vmm_cfg_end.as_usize() - vmm_cfg_start.as_usize(); + + core::iter::once(MemRegion { + paddr: virt_to_phys(vmm_cfg_start), + size: vmm_cfg_size, + // Provided by host, read-only. + flags: MemRegionFlags::RESERVED | MemRegionFlags::READ, + name: "System config (for VMM)", + }) +} + pub fn host_memory_regions() -> impl Iterator { let sys_config = HvSystemConfig::get(); let cell_config = &sys_config.root_cell.config(); @@ -49,8 +63,9 @@ pub(crate) fn platform_regions() -> impl Iterator { paddr: virt_to_phys((__header_start as usize).into()), size: __header_end as usize - __header_start as usize, flags: MemRegionFlags::RESERVED | MemRegionFlags::READ, - name: ".header", + name: ".header (for VMM)", }) + .chain(vmm_cfg_regions()) .chain(vmm_free_regions()) // Here we do not use the `default_free_regions`` from `crate::mem`. // Cause we need to reserved regions for diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 7c1efbab52..8d537e9f65 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -42,7 +42,6 @@ use config::HvSystemConfig; // use error::HvResult; use header::HvHeader; - static VMM_PRIMARY_INIT_OK: AtomicU32 = AtomicU32::new(0); static ERROR_NUM: AtomicI32 = AtomicI32::new(0); @@ -72,9 +71,7 @@ fn current_cpu_id() -> usize { } } -fn vmm_primary_init_early() { - let cpu_id = current_cpu_id(); - +fn vmm_primary_init_early(cpu_id: usize) { println!("Primary CPU {} init early", cpu_id); // We do not clear bss here. // Because currently the image was loaded by Linux. @@ -83,17 +80,15 @@ fn vmm_primary_init_early() { self::uart16550::init(); } -fn vmm_secondary_init_early() { +fn vmm_secondary_init_early(cpu_id: usize) { #[cfg(feature = "smp")] { - let cpu_id = current_cpu_id(); println!("Secondary CPU {} init early.", cpu_id); crate::cpu::init_secondary(cpu_id); } } -fn vmm_primary_init() { - let cpu_id = current_cpu_id(); +fn vmm_primary_init(cpu_id: usize) { self::dtables::init_primary(); self::time::init_early(); @@ -115,7 +110,7 @@ fn vmm_primary_init() { ); } -fn vmm_secondary_init() { +fn vmm_secondary_init(_cpu_id: usize) { #[cfg(feature = "smp")] { self::dtables::init_secondary(); @@ -132,11 +127,11 @@ extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { let vm_cpus = HvHeader::get().reserved_cpus(); println!( - "{} Core {} CPU {} entered. {} of {}", + "{} Core {} (LAPIC_ID {}) entered. [{}/{}]", if is_primary { "Primary" } else { "Secondary" }, core_id, cpu_id, - entry::entered_cpus(), + core_id, vm_cpus ); @@ -151,20 +146,20 @@ extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { // First, we init primary core for VMM. if is_primary { - vmm_primary_init_early(); + vmm_primary_init_early(cpu_id); } else { wait_while(|| VMM_PRIMARY_INIT_OK.load(Ordering::Acquire) == 0); - vmm_secondary_init_early(); + vmm_secondary_init_early(cpu_id); } // Note: this has to be done after `cpu::init_primary`. // Because LinuxContext will be stored in percpu area. - context::set_linux_context(linux_sp); + context::set_linux_context(linux_sp, cpu_id); if is_primary { - vmm_primary_init(); + vmm_primary_init(cpu_id); } else { - vmm_secondary_init(); + vmm_secondary_init(cpu_id); } unsafe { @@ -201,14 +196,18 @@ unsafe extern "C" fn rust_entry(_magic: usize, _mbi: usize) { pub fn platform_init() { self::lapic::init(); - VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); - // self::apic::init_primary(); + // Consruct LAPIC but DO NOT operate the LAPIC. + // because the LAPIC belongs to Linux, we should not touch it. + // self::apic::init_primary(false); // self::time::init_primary(); + + VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); + // Secondary CPUs continue to initialize. } /// Initializes the platform devices for secondary CPUs. #[cfg(feature = "smp")] pub fn platform_init_secondary() { - // self::apic::init_secondary(); + // self::apic::init_secondary(false); // self::time::init_secondary(); } diff --git a/modules/axtask/src/api.rs b/modules/axtask/src/api.rs index 7cf96a043c..5226116634 100644 --- a/modules/axtask/src/api.rs +++ b/modules/axtask/src/api.rs @@ -210,8 +210,9 @@ pub fn exit(exit_code: i32) -> ! { pub fn run_idle() -> ! { loop { yield_now(); - debug!("idle task: waiting for IRQs..."); - #[cfg(feature = "irq")] - axhal::arch::wait_for_irqs(); + // debug!("idle task: waiting for IRQs..."); + // #[cfg(feature = "irq")] + // axhal::arch::wait_for_irqs(); + core::hint::spin_loop(); } } From 814ff45c4608632b041fbbc065c01f87aea49462 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sat, 22 Mar 2025 18:19:26 +0800 Subject: [PATCH 14/19] [refactor] remote redundant lapic.rs, wfi in arceos's core boot --- modules/axhal/src/lib.rs | 1 - modules/axhal/src/platform/x86_linux/apic.rs | 6 +- .../axhal/src/platform/x86_linux/context.rs | 26 ---- modules/axhal/src/platform/x86_linux/lapic.rs | 117 ------------------ modules/axhal/src/platform/x86_linux/mod.rs | 7 +- 5 files changed, 5 insertions(+), 152 deletions(-) delete mode 100644 modules/axhal/src/platform/x86_linux/lapic.rs diff --git a/modules/axhal/src/lib.rs b/modules/axhal/src/lib.rs index 3951a65ffe..d3f6f32e5f 100644 --- a/modules/axhal/src/lib.rs +++ b/modules/axhal/src/lib.rs @@ -79,5 +79,4 @@ pub use self::platform::platform_init; pub use self::platform::platform_init_secondary; pub use self::platform::mem::host_memory_regions; -pub use self::platform::get_linux_context_by_cpu_id; pub use self::platform::get_linux_context_list; \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index 5b946d79dd..b9d58f8f89 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -120,9 +120,9 @@ pub(super) fn init_primary(enabled: bool) { LOCAL_APIC.get().as_mut().unwrap().write(lapic); } - info!("Initialize IO APIC..."); - let io_apic = unsafe { IoApic::new(phys_to_virt(IO_APIC_BASE).as_usize() as u64) }; - IO_APIC.init_once(SpinNoIrq::new(io_apic)); + // info!("Initialize IO APIC..."); + // let io_apic = unsafe { IoApic::new(phys_to_virt(IO_APIC_BASE).as_usize() as u64) }; + // IO_APIC.init_once(SpinNoIrq::new(io_apic)); } #[cfg(feature = "smp")] diff --git a/modules/axhal/src/platform/x86_linux/context.rs b/modules/axhal/src/platform/x86_linux/context.rs index 0a7534abc1..ca1c6b77c3 100644 --- a/modules/axhal/src/platform/x86_linux/context.rs +++ b/modules/axhal/src/platform/x86_linux/context.rs @@ -6,18 +6,10 @@ use x86_vcpu::LinuxContext; use axconfig::SMP; -use crate::cpu::this_cpu_id; - -#[percpu::def_percpu] -static LINUX_CTX: LazyInit = LazyInit::new(); - static mut LINUX_CTX_LIST: LazyInit<[MaybeUninit; SMP]> = LazyInit::new(); /// Set Linux context for current CPU. pub fn set_linux_context(linux_sp: usize, cpu_id: usize) { - // let linux_ctx = unsafe { LINUX_CTX.current_ref_mut_raw() }; - // linux_ctx.init_once(LinuxContext::load_from(linux_sp)); - unsafe { if LINUX_CTX_LIST.is_inited() { LINUX_CTX_LIST[cpu_id].write(LinuxContext::load_from(linux_sp)); @@ -26,27 +18,9 @@ pub fn set_linux_context(linux_sp: usize, cpu_id: usize) { list[cpu_id].write(LinuxContext::load_from(linux_sp)); LINUX_CTX_LIST.init_once(list); } - - axlog::ax_println!( - "Core {} {} Linux context set {:#x?}", - this_cpu_id(), - cpu_id, - &LINUX_CTX_LIST[cpu_id].assume_init() - ); } } -/// Get Linux context for current CPU. -#[allow(unused)] -pub fn get_linux_context() -> &'static LinuxContext { - unsafe { LINUX_CTX.current_ref_raw() } -} - -/// Get Linux context for the given CPU ID. -pub fn get_linux_context_by_cpu_id(cpu_id: usize) -> &'static LinuxContext { - unsafe { LINUX_CTX.remote_ref_raw(cpu_id) } -} - pub fn get_linux_context_list() -> &'static [LinuxContext; SMP] { // LINUX_CTX_LIST.as_ref() unsafe { &*(&raw const LINUX_CTX_LIST as *const [LinuxContext; SMP]) } diff --git a/modules/axhal/src/platform/x86_linux/lapic.rs b/modules/axhal/src/platform/x86_linux/lapic.rs deleted file mode 100644 index 7d4797866e..0000000000 --- a/modules/axhal/src/platform/x86_linux/lapic.rs +++ /dev/null @@ -1,117 +0,0 @@ -/// This is just a simplified LocalApic used for send sipi request to other cores. -/// Refer to `apic.rs` for detailed implementation for APIC. -// Because `LocalApicBuilder` from `x2apic` can not be used here. -use spin::{Once, RwLock}; -use x86::apic::{ApicControl, ApicId, x2apic::X2APIC, xapic::XAPIC}; - -extern crate alloc; - -/// TODO: remove this Arc. -use alloc::sync::Arc; - -use crate::mem::{PhysAddr, phys_to_virt}; - -const APIC_BASE: PhysAddr = pa!(0xFEE0_0000); - -bitflags::bitflags! { - /// IA32_APIC_BASE MSR. - struct ApicBase: u64 { - /// Processor is BSP. - const BSP = 1 << 8; - /// Enable x2APIC mode. - const EXTD = 1 << 10; - /// xAPIC global enable/disable. - const EN = 1 << 11; - } -} - -impl ApicBase { - pub fn read() -> Self { - unsafe { Self::from_bits_retain(x86::msr::rdmsr(x86::msr::IA32_APIC_BASE)) } - } -} - -pub(super) struct LocalApic { - inner: Arc>, - is_x2apic: bool, -} - -unsafe impl Send for LocalApic {} -unsafe impl Sync for LocalApic {} - -impl LocalApic { - pub fn new() -> Result { - let base = ApicBase::read(); - - debug!("ApicBase {:#x}", base.bits()); - - if base.contains(ApicBase::EXTD) { - info!("Using x2APIC."); - Ok(Self { - inner: Arc::new(RwLock::new(X2APIC::new())), - is_x2apic: true, - }) - } else if base.contains(ApicBase::EN) { - info!("Using xAPIC."); - let base_vaddr = phys_to_virt(APIC_BASE); - let apic_region = - unsafe { core::slice::from_raw_parts_mut(base_vaddr.as_usize() as _, 0x1000 / 4) }; - Ok(Self { - inner: Arc::new(RwLock::new(XAPIC::new(apic_region))), - is_x2apic: false, - }) - } else { - Err("Local Apic init failed") - } - } -} - -static LOCAL_APIC: Once = Once::new(); - -pub(super) fn lapic<'a>() -> &'a LocalApic { - LOCAL_APIC.get().expect("Uninitialized Local APIC!") -} - -pub(super) fn init() { - let lapic = LocalApic::new().expect("LocalApic init failed"); - LOCAL_APIC.call_once(|| lapic); -} - -use crate::time::{Duration, busy_wait}; - -pub(super) unsafe fn start_ap(apic_id: u32, start_page_idx: u8) { - info!("Starting RT cpu {}...", apic_id); - let apic_id = if lapic().is_x2apic { - ApicId::X2Apic(apic_id) - } else { - ApicId::XApic(apic_id as u8) - }; - - // INIT-SIPI-SIPI Sequence - let mut lapic = lapic().inner.write(); - unsafe { - lapic.ipi_init(apic_id); - } - // delay_us(10 * 1000); // 10ms - busy_wait(Duration::from_millis(10)); // 10ms - unsafe { - lapic.ipi_startup(apic_id, start_page_idx); - } - // delay_us(200); // 200 us - busy_wait(Duration::from_micros(200)); // 200us - unsafe { - lapic.ipi_startup(apic_id, start_page_idx); - } -} - -pub(super) unsafe fn shutdown_ap(apic_id: u32) { - info!("Shutting down RT cpu {}...", apic_id); - let apic_id = if lapic().is_x2apic { - ApicId::X2Apic(apic_id) - } else { - ApicId::XApic(apic_id as u8) - }; - unsafe { - lapic().inner.write().ipi_init(apic_id); - } -} diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 8d537e9f65..66ba329258 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -1,6 +1,4 @@ mod apic; -// It's a simplied version of LocalApic, just use for sendsipi. -mod lapic; mod boot; mod dtables; @@ -31,7 +29,6 @@ pub mod console { pub use super::uart16550::*; } -pub use context::get_linux_context_by_cpu_id; pub use context::get_linux_context_list; use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; @@ -198,7 +195,7 @@ pub fn platform_init() { // Consruct LAPIC but DO NOT operate the LAPIC. // because the LAPIC belongs to Linux, we should not touch it. - // self::apic::init_primary(false); + self::apic::init_primary(false); // self::time::init_primary(); VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); @@ -208,6 +205,6 @@ pub fn platform_init() { /// Initializes the platform devices for secondary CPUs. #[cfg(feature = "smp")] pub fn platform_init_secondary() { - // self::apic::init_secondary(false); + self::apic::init_secondary(false); // self::time::init_secondary(); } From cca8647369a6c51c94375486fd2d2217e576d56e Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Wed, 26 Mar 2025 15:18:14 +0800 Subject: [PATCH 15/19] [feat] boot secondard cores belongs to arceos itself --- modules/axhal/src/lib.rs | 4 +- modules/axhal/src/platform/x86_linux/apic.rs | 22 ++- modules/axhal/src/platform/x86_linux/boot.rs | 39 +++--- modules/axhal/src/platform/x86_linux/entry.rs | 10 +- .../axhal/src/platform/x86_linux/header.rs | 3 +- modules/axhal/src/platform/x86_linux/mod.rs | 43 +++--- modules/axhal/src/platform/x86_linux/mp.rs | 125 ++++++++---------- .../axhal/src/platform/x86_linux/multiboot.S | 19 +-- modules/axruntime/src/mp.rs | 5 +- 9 files changed, 133 insertions(+), 137 deletions(-) diff --git a/modules/axhal/src/lib.rs b/modules/axhal/src/lib.rs index d3f6f32e5f..5d906bafbd 100644 --- a/modules/axhal/src/lib.rs +++ b/modules/axhal/src/lib.rs @@ -78,5 +78,7 @@ pub use self::platform::platform_init; #[cfg(feature = "smp")] pub use self::platform::platform_init_secondary; +pub use self::platform::config as hvconfig; +pub use self::platform::context::get_linux_context_list; +pub use self::platform::header as hvheader; pub use self::platform::mem::host_memory_regions; -pub use self::platform::get_linux_context_list; \ No newline at end of file diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index b9d58f8f89..7fd4249eb5 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -31,6 +31,9 @@ static LOCAL_APIC: SyncUnsafeCell> = static mut IS_X2APIC: bool = false; static IO_APIC: LazyInit> = LazyInit::new(); +const MAX_APIC_ID: u32 = 254; +static mut APIC_TO_CPU_ID: [u32; MAX_APIC_ID as usize + 1] = [u32::MAX; MAX_APIC_ID as usize + 1]; + /// Enables or disables the given IRQ. #[cfg(feature = "irq")] pub fn set_enable(vector: usize, enabled: bool) { @@ -117,6 +120,9 @@ pub(super) fn init_primary(enabled: bool) { if enabled { lapic.enable(); } + + APIC_TO_CPU_ID[lapic.id() as usize] = 0; + LOCAL_APIC.get().as_mut().unwrap().write(lapic); } @@ -127,7 +133,21 @@ pub(super) fn init_primary(enabled: bool) { #[cfg(feature = "smp")] pub(super) fn init_secondary(enabled: bool) { + let lapic = local_apic(); + if enabled { - unsafe { local_apic().enable() }; + unsafe { + lapic.enable(); + } } + unsafe { + APIC_TO_CPU_ID[lapic.id() as usize] = 0; + }; +} + +/// Returns if the given APIC ID is reserved. +/// The APIC ID is reserved if it entered Linux, which has set the corresponding +/// entry in `APIC_TO_CPU_ID` to 0. +pub(super) fn apic_id_is_reserved(apic_id: usize) -> bool { + unsafe { APIC_TO_CPU_ID[apic_id] == 0 } } diff --git a/modules/axhal/src/platform/x86_linux/boot.rs b/modules/axhal/src/platform/x86_linux/boot.rs index 1488072f50..59efca0473 100644 --- a/modules/axhal/src/platform/x86_linux/boot.rs +++ b/modules/axhal/src/platform/x86_linux/boot.rs @@ -3,7 +3,7 @@ use core::arch::global_asm; use x86_64::registers::control::{Cr0Flags, Cr4Flags}; use x86_64::registers::model_specific::EferFlags; -use axconfig::{TASK_STACK_SIZE, plat::PHYS_VIRT_OFFSET}; +use axconfig::plat::PHYS_VIRT_OFFSET; /// Flags set in the ’flags’ member of the multiboot header. /// @@ -30,23 +30,20 @@ const CR4: u64 = Cr4Flags::PHYSICAL_ADDRESS_EXTENSION.bits() }; const EFER: u64 = EferFlags::LONG_MODE_ENABLE.bits() | EferFlags::NO_EXECUTE_ENABLE.bits(); -#[unsafe(link_section = ".bss.stack")] -static mut BOOT_STACK: [u8; TASK_STACK_SIZE] = [0; TASK_STACK_SIZE]; - -// global_asm!( -// include_str!("multiboot.S"), -// mb_magic = const MULTIBOOT_BOOTLOADER_MAGIC, -// mb_hdr_magic = const MULTIBOOT_HEADER_MAGIC, -// mb_hdr_flags = const MULTIBOOT_HEADER_FLAGS, -// entry = sym super::rust_entry, -// entry_secondary = sym super::rust_entry_from_vmm, - -// offset = const PHYS_VIRT_OFFSET, -// boot_stack_size = const TASK_STACK_SIZE, -// boot_stack = sym BOOT_STACK, - -// cr0 = const CR0, -// cr4 = const CR4, -// efer_msr = const x86::msr::IA32_EFER, -// efer = const EFER, -// ); +global_asm!( + include_str!("multiboot.S"), + mb_magic = const MULTIBOOT_BOOTLOADER_MAGIC, + mb_hdr_magic = const MULTIBOOT_HEADER_MAGIC, + mb_hdr_flags = const MULTIBOOT_HEADER_FLAGS, + // entry = sym super::rust_entry, + entry_secondary = sym super::rust_entry_secondary, + + offset = const PHYS_VIRT_OFFSET, + // boot_stack_size = const TASK_STACK_SIZE, + // boot_stack = sym BOOT_STACK, + + cr0 = const CR0, + cr4 = const CR4, + efer_msr = const x86::msr::IA32_EFER, + efer = const EFER, +); diff --git a/modules/axhal/src/platform/x86_linux/entry.rs b/modules/axhal/src/platform/x86_linux/entry.rs index 36dfe8823b..856a121693 100644 --- a/modules/axhal/src/platform/x86_linux/entry.rs +++ b/modules/axhal/src/platform/x86_linux/entry.rs @@ -32,9 +32,7 @@ unsafe extern "sysv64" fn switch_stack(linux_sp: usize) -> i32 { // Note: cpu_id here is not Local APIC ID, it is the index of entered CPUs. // We just use it here to choose VMM_BOOT_STACK. let core_id = ENTERED_CPUS.fetch_add(1, Ordering::SeqCst); - // let _cpu_id = current_cpu_id(); - // let cpu_data = PerCpu::new(); let hv_sp = VMM_BOOT_STACK[core_id as usize].as_ptr_range().end as usize; let ret; core::arch::asm!(" @@ -58,12 +56,20 @@ unsafe extern "sysv64" fn switch_stack(linux_sp: usize) -> i32 { let ret = vmm_entry(linux_sp); + axlog::ax_println!( + "CPU {} return back to driver with code {}.", + ret, + super::current_cpu_id() + ); + x86::msr::wrmsr(x86::msr::IA32_GS_BASE, linux_tp); x86::controlregs::cr3_write(linux_cr3); ret } } +/// Cores entered from Linux will call this function. +/// This function will switch to VMM stack and call `super::vmm_cpu_entry`. #[naked] #[unsafe(link_section = ".text.boot")] #[unsafe(no_mangle)] diff --git a/modules/axhal/src/platform/x86_linux/header.rs b/modules/axhal/src/platform/x86_linux/header.rs index fd84b95957..56b9e25286 100644 --- a/modules/axhal/src/platform/x86_linux/header.rs +++ b/modules/axhal/src/platform/x86_linux/header.rs @@ -6,7 +6,7 @@ const HEADER_SIGNATURE: [u8; 8] = *b"EVMIMAGE"; /// Hypervisor description. /// Located at the beginning of the hypervisor binary image and loaded by -/// the driver (which also initializes some fields). +/// the driver (which also initializes some fields). /// See jailhouse dir `driver/jailhouse.h` for details. #[repr(C)] pub struct HvHeader { @@ -27,6 +27,7 @@ impl HvHeader { unsafe { &*HV_HEADER_PTR } } + /// Get the number of CPUs reserved for Linux. pub fn reserved_cpus(&self) -> u32 { if self.arceos_cpus < self.max_cpus { self.max_cpus - self.arceos_cpus diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 66ba329258..5e6e2d8785 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -12,10 +12,10 @@ pub mod time; // mods for vmm usage. // mod percpu; -mod config; +pub mod config; mod consts; -mod context; -mod header; +pub mod context; +pub mod header; // #[cfg(feature = "smp")] pub mod mp; @@ -29,8 +29,6 @@ pub mod console { pub use super::uart16550::*; } -pub use context::get_linux_context_list; - use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; use axlog::ax_println as println; @@ -114,6 +112,9 @@ fn vmm_secondary_init(_cpu_id: usize) { } } +/// Cores entered from Linux will call this function. +/// Cores reserved for ArceOS for other purposed will be shutdown by Linux +/// before entry and restarted through SIPI by ArceOS's `start_secondary_cpu` in mp.rs. extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { let cpu_id = current_cpu_id(); @@ -166,33 +167,23 @@ extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { rust_main_secondary(cpu_id); } } - - let code = 0; - println!( - "{} CPU {} return back to driver with code {}.", - if is_primary { "Primary" } else { "Secondary" }, - cpu_id, - code - ); - code } -unsafe extern "C" fn rust_entry(_magic: usize, _mbi: usize) { - // TODO: handle multiboot info - // if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { - // crate::mem::clear_bss(); - // crate::cpu::init_primary(current_cpu_id()); - // self::uart16550::init(); - // self::dtables::init_primary(); - // self::time::init_early(); - // rust_main(current_cpu_id(), 0); - // } +/// Core reserved for ArceOS will entered through this function. +unsafe extern "C" fn rust_entry_secondary(magic: usize) { + // #[cfg(feature = "smp")] + if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { + // Note: DO not call log related functions before percpu area is initialized. + crate::cpu::init_secondary(current_cpu_id()); + self::dtables::init_secondary(); + unsafe { + rust_main_secondary(current_cpu_id()); + } + } } /// Initializes the platform devices for the primary CPU. pub fn platform_init() { - self::lapic::init(); - // Consruct LAPIC but DO NOT operate the LAPIC. // because the LAPIC belongs to Linux, we should not touch it. self::apic::init_primary(false); diff --git a/modules/axhal/src/platform/x86_linux/mp.rs b/modules/axhal/src/platform/x86_linux/mp.rs index 2959f75d86..ad17205fa7 100644 --- a/modules/axhal/src/platform/x86_linux/mp.rs +++ b/modules/axhal/src/platform/x86_linux/mp.rs @@ -1,94 +1,83 @@ -use crate::mem::{phys_to_virt, virt_to_phys, PhysAddr, VirtAddr, PAGE_SIZE_4K}; -use crate::time::{busy_wait, Duration}; -use core::sync::atomic::Ordering; +use memory_addr::VirtAddr; -use super::header::HvHeader; -// use super::percpu::PerCpu; -use axconfig::{SMP, TASK_STACK_SIZE}; +use crate::mem::{PAGE_SIZE_4K, PhysAddr, phys_to_virt, virt_to_phys}; +use crate::time::{Duration, busy_wait}; const START_PAGE_IDX: u8 = 6; -const START_PAGE_COUNT: usize = 1; const START_PAGE_PADDR: PhysAddr = pa!(START_PAGE_IDX as usize * PAGE_SIZE_4K); - -/// Starts the given secondary CPU with its boot stack. -pub fn start_secondary_cpu(_apic_id: usize, _stack_top: crate::mem::PhysAddr) { - // No need - // For CPU reserved for Linux, this step is completed by Linux. - // For CPU reserved for RT-task, this step is completed by `start_arceos_cpus`. -} - -pub fn continue_secondary_cpus() { - // super::VMM_MAIN_INIT_OK.fetch_add(1, Ordering::Release); - // super::VMM_MAIN_INIT_OK.store(1, Ordering::Release); -} +const U64_PER_PAGE: usize = PAGE_SIZE_4K / 8; core::arch::global_asm!( - include_str!("ap_start.S"), - start_page_paddr = const START_PAGE_PADDR.as_usize(), + include_str!("ap_start.S"), + start_page_paddr = const START_PAGE_PADDR.as_usize(), ); -#[unsafe(link_section = ".bss.stack")] -static mut SECONDARY_BOOT_STACK: [[u8; TASK_STACK_SIZE]; SMP] = [[0; TASK_STACK_SIZE]; SMP]; +unsafe extern "C" { + unsafe fn ap_entry32(); + unsafe fn ap_start(); + unsafe fn ap_end(); +} -/// Starts the given secondary CPU with its boot stack. -#[allow(clippy::uninit_assumed_init)] -pub fn start_arceos_cpus() { - unsafe extern "C" { - unsafe fn ap_entry32(); - unsafe fn ap_start(); - unsafe fn ap_end(); - } - const U64_PER_PAGE: usize = PAGE_SIZE_4K / 8; +static mut BACKUP_PAGE: [u64; U64_PER_PAGE] = [0; U64_PER_PAGE]; +unsafe fn setup_startup_page(stack_top: PhysAddr, boot_fn: F) +where + F: FnOnce(), +{ let start_page_ptr = phys_to_virt(START_PAGE_PADDR).as_mut_ptr() as *mut u64; + let start_page = unsafe { core::slice::from_raw_parts_mut(start_page_ptr, U64_PER_PAGE) }; + + // Since start page located at START_PAGE_PADDR belongs to Linux's physical address space. + // We construct a backup space for start page, after `start_ap`, we just copy the backup space back. + unsafe { BACKUP_PAGE.copy_from_slice(&start_page) }; + unsafe { - let start_page = - core::slice::from_raw_parts_mut(start_page_ptr, U64_PER_PAGE * START_PAGE_COUNT); - - // Since start page located at START_PAGE_PADDR belongs to Linux's physical address space. - // We construct a backup space for start page, after `start_ap`, we just copy the backup space back. - let mut backup: [u64; U64_PER_PAGE * START_PAGE_COUNT] = - core::mem::MaybeUninit::uninit().assume_init(); - backup.copy_from_slice(start_page); core::ptr::copy_nonoverlapping( ap_start as *const u64, start_page_ptr, (ap_end as usize - ap_start as usize) / 8, ); + } + + // We need to use physical address here. + // Since current physical to virtual address is not identical mapped with offset 0xffff_ff80_0000_0000. + let ap_entry_virt = VirtAddr::from_usize(ap_entry32 as usize); + let ap_entry_phys = virt_to_phys(ap_entry_virt); + + start_page[U64_PER_PAGE - 2] = stack_top.as_usize() as u64; // stack_top + start_page[U64_PER_PAGE - 1] = ap_entry_phys.as_usize() as _; // entry - // We need to use physical address here. - // Since current physical to virtual address is not identical mapped with offset 0xffff_ff80_0000_0000. - let ap_entry_virt = VirtAddr::from(ap_entry32 as usize); - let ap_entry_phys = virt_to_phys(ap_entry_virt); + boot_fn(); - start_page[U64_PER_PAGE - 1] = ap_entry_phys.as_usize() as _; // entry + // Restore the start page. + unsafe { start_page.copy_from_slice(&BACKUP_PAGE) }; +} - let max_cpus = super::header::HvHeader::get().max_cpus; - let mut arceos_cpu_num = 0; +/// Starts the given secondary CPU with its boot stack. +/// Returns true if the caller should wait for the CPU to be ready. +pub fn start_secondary_cpu(apic_id: usize, stack_top: PhysAddr) -> bool { + // DO not boot CPUs that are reserved for host Linux. + if super::apic::apic_id_is_reserved(apic_id) { + info!("CPU {} is reserved for Linux, skip", apic_id); + return false; + } - for apic_id in 0..max_cpus { - // if PerCpu::cpu_is_booted(apic_id as usize) { - // continue; - // } - let stack_top = virt_to_phys(VirtAddr::from( - SECONDARY_BOOT_STACK[arceos_cpu_num].as_ptr_range().end as usize, - )) - .as_usize(); + let boot_fn = || { + let apic_id = super::apic::raw_apic_id(apic_id as u8); + let lapic = super::apic::local_apic(); - start_page[U64_PER_PAGE - 2] = stack_top as u64; // stack_top + info!("Starting secondary CPU {}", apic_id); - super::lapic::start_ap(apic_id, START_PAGE_IDX); - // info!( - // "starting ArceOS {} CPU, apic id {}", - // arceos_cpu_num, apic_id - // ); + // INIT-SIPI-SIPI Sequence + // Ref: Intel SDM Vol 3C, Section 8.4.4, MP Initialization Example + unsafe { lapic.send_init_ipi(apic_id) }; + busy_wait(Duration::from_millis(10)); // 10ms + unsafe { lapic.send_sipi(START_PAGE_IDX, apic_id) }; + busy_wait(Duration::from_micros(200)); // 200us + unsafe { lapic.send_sipi(START_PAGE_IDX, apic_id) }; + }; - arceos_cpu_num += 1; - // wait for max 100ms - busy_wait(Duration::from_millis(100)); // 100ms - } - // info!("starting {} CPUs for ArceOS ", arceos_cpu_num); + unsafe { setup_startup_page(stack_top, boot_fn) }; - start_page.copy_from_slice(&backup); - } + return true; } diff --git a/modules/axhal/src/platform/x86_linux/multiboot.S b/modules/axhal/src/platform/x86_linux/multiboot.S index ec6cc7ce5d..fdf48d3f0c 100644 --- a/modules/axhal/src/platform/x86_linux/multiboot.S +++ b/modules/axhal/src/platform/x86_linux/multiboot.S @@ -76,14 +76,6 @@ ap_entry32: .code64 bsp_entry64: ENTRY64_COMMON - - # set RSP to boot stack - movabs rsp, offset {boot_stack} - add rsp, {boot_stack_size} - - # call rust_entry(magic, mbi) - movabs rax, offset {entry} - call rax jmp .Lhlt .code64 @@ -124,10 +116,9 @@ ap_entry64: .Ltmp_pml4: # 0x0000_0000 ~ 0xffff_ffff .quad .Ltmp_pdpt_low - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) - .zero 8 * 255 - # 0xffff_8000_0000_0000 ~ 0xffff_8000_ffff_ffff + .zero 8 * 510 + # 0xffff_ff80_0000_0000 ~ 0xffff_ff80_ffff_ffff .quad .Ltmp_pdpt_high - {offset} + 0x3 # PRESENT | WRITABLE | paddr(tmp_pdpt) - .zero 8 * 255 # FIXME: may not work on macOS using hvf as the CPU does not support 1GB page (pdpe1gb) .Ltmp_pdpt_low: @@ -138,8 +129,6 @@ ap_entry64: .zero 8 * 508 .Ltmp_pdpt_high: - .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) - # .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) - # .quad 0x80000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x8000_0000) - # .quad 0xc0000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0xc000_0000) + # 0xffff_ff80_0000_0000 to 0x4000_0000 + .quad 0x40000000 | 0x83 # PRESENT | WRITABLE | HUGE_PAGE | paddr(0x4000_0000) .zero 8 * 511 diff --git a/modules/axruntime/src/mp.rs b/modules/axruntime/src/mp.rs index 5e1baab4b9..241f450418 100644 --- a/modules/axruntime/src/mp.rs +++ b/modules/axruntime/src/mp.rs @@ -18,10 +18,11 @@ pub fn start_secondary_cpus(primary_cpu_id: usize) { })); debug!("starting CPU {}...", i); - axhal::mp::start_secondary_cpu(i, stack_top); + let current_entered_cpus = ENTERED_CPUS.load(Ordering::Acquire); + let should_wait = axhal::mp::start_secondary_cpu(i, stack_top); logic_cpu_id += 1; - while ENTERED_CPUS.load(Ordering::Acquire) <= logic_cpu_id { + while should_wait && ENTERED_CPUS.load(Ordering::Acquire) <= logic_cpu_id { core::hint::spin_loop(); } } From 04896fe37060a363e5b95e833d0a29589d0c85a2 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 27 Mar 2025 10:03:25 +0800 Subject: [PATCH 16/19] [fix] delete redundant files, boot from Linux and reserved cores for ArceOS --- .../axhal/src/platform/x86_linux/context.rs | 1 - .../axhal/src/platform/x86_linux/dtables.rs | 1 - modules/axhal/src/platform/x86_linux/misc.rs | 4 +- modules/axhal/src/platform/x86_linux/mod.rs | 2 - .../axhal/src/platform/x86_linux/percpu.rs | 141 ------------------ modules/axhal/src/platform/x86_linux/time.rs | 4 + modules/axruntime/src/mp.rs | 1 - 7 files changed, 5 insertions(+), 149 deletions(-) delete mode 100644 modules/axhal/src/platform/x86_linux/percpu.rs diff --git a/modules/axhal/src/platform/x86_linux/context.rs b/modules/axhal/src/platform/x86_linux/context.rs index ca1c6b77c3..8be903d43f 100644 --- a/modules/axhal/src/platform/x86_linux/context.rs +++ b/modules/axhal/src/platform/x86_linux/context.rs @@ -22,6 +22,5 @@ pub fn set_linux_context(linux_sp: usize, cpu_id: usize) { } pub fn get_linux_context_list() -> &'static [LinuxContext; SMP] { - // LINUX_CTX_LIST.as_ref() unsafe { &*(&raw const LINUX_CTX_LIST as *const [LinuxContext; SMP]) } } diff --git a/modules/axhal/src/platform/x86_linux/dtables.rs b/modules/axhal/src/platform/x86_linux/dtables.rs index 2352a717e2..0d3415130c 100644 --- a/modules/axhal/src/platform/x86_linux/dtables.rs +++ b/modules/axhal/src/platform/x86_linux/dtables.rs @@ -4,7 +4,6 @@ use lazyinit::LazyInit; use crate::arch::{GdtStruct, IdtStruct, TaskStateSegment}; - static IDT: LazyInit = LazyInit::new(); #[percpu::def_percpu] diff --git a/modules/axhal/src/platform/x86_linux/misc.rs b/modules/axhal/src/platform/x86_linux/misc.rs index bc38365354..52d54ca921 100644 --- a/modules/axhal/src/platform/x86_linux/misc.rs +++ b/modules/axhal/src/platform/x86_linux/misc.rs @@ -8,9 +8,7 @@ pub fn terminate() -> ! { axlog::ax_println!("\nMaybe we need to find a way to return this core to Linux?\n"); // #[cfg(platform = "x86_64-qemu-q35")] - unsafe { - PortWriteOnly::new(0x604).write(0x2000u16) - }; + unsafe { PortWriteOnly::new(0x604).write(0x2000u16) }; crate::arch::halt(); warn!("It should shutdown!"); diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 5e6e2d8785..5e32c5601f 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -10,8 +10,6 @@ pub mod misc; pub mod time; // mods for vmm usage. -// mod percpu; - pub mod config; mod consts; pub mod context; diff --git a/modules/axhal/src/platform/x86_linux/percpu.rs b/modules/axhal/src/platform/x86_linux/percpu.rs deleted file mode 100644 index 7206cdc394..0000000000 --- a/modules/axhal/src/platform/x86_linux/percpu.rs +++ /dev/null @@ -1,141 +0,0 @@ -use core::fmt::{Debug, Formatter, Result}; -use core::sync::atomic::{AtomicU32, Ordering}; - -// use crate::arch::vmm::{Vcpu, VcpuAccessGuestState}; -// use crate::arch::{cpu, ArchPerCpu, LinuxContext}; -// use crate::cell::Cell; -use super::current_cpu_id; -// use crate::error::HvResult; -use super::header::HvHeader; - -static ENTERED_CPUS: AtomicU32 = AtomicU32::new(0); -// static ACTIVATED_CPUS: AtomicU32 = AtomicU32::new(0); - -#[derive(Debug, Eq, PartialEq)] -pub enum CpuState { - HvDisabled, - HvEnabled, -} - -static mut BOOTED_CPU_APIC_ID: [u32; 255] = [u32::MAX; 255]; - -// Todo: this can be placed into per_cpu. -#[repr(C, align(4096))] -pub struct PerCpu { - /// Referenced by arch::cpu::thread_pointer() for x86_64. - self_vaddr: usize, - /// Current CPU's apic id, read from raw_cpuid. - pub id: u32, - pub state: CpuState, - // pub vcpu: Vcpu, - // arch: ArchPerCpu, - // linux: LinuxContext, - // Stack will be placed here. -} - -impl PerCpu { - pub fn new<'a>() -> &'a mut Self { - if Self::entered_cpus() >= HvHeader::get().max_cpus { - panic!("enter cpus exceed {}", HvHeader::get().max_cpus); - } - - let cpu_sequence = ENTERED_CPUS.fetch_add(1, Ordering::SeqCst); - let cpu_id = current_cpu_id(); - - unsafe { - BOOTED_CPU_APIC_ID[cpu_id] = cpu_sequence; - } - - let ret = unsafe { Self::from_id_mut(cpu_id as u32) }; - let vaddr = ret as *const _ as usize; - ret.id = cpu_id as u32; - ret.self_vaddr = vaddr; - - // unsafe { crate::arch::write_thread_pointer(vaddr.into()) }; - ret - } - - - pub fn current<'a>() -> &'a Self { - Self::current_mut() - } - - pub fn current_mut<'a>() -> &'a mut Self { - unsafe { &mut *(crate::arch::read_thread_pointer() as *mut Self) } - } - - pub fn stack_top(&self) -> usize { - self as *const _ as usize + PER_CPU_SIZE - 8 - } - - pub fn entered_cpus() -> u32 { - ENTERED_CPUS.load(Ordering::Acquire) - } - - // pub fn activated_cpus() -> u32 { - // ACTIVATED_CPUS.load(Ordering::Acquire) - // } - - pub fn cpu_is_booted(apic_id: usize) -> bool { - unsafe { BOOTED_CPU_APIC_ID[apic_id] != u32::MAX } - } - - // pub fn init(&mut self, linux_sp: usize, cell: &Cell) -> HvResult { - // info!("CPU {} init...", self.id); - - // // Save CPU state used for linux. - // self.state = CpuState::HvDisabled; - // self.linux = LinuxContext::load_from(linux_sp); - - // // Activate hypervisor page table on each cpu. - // unsafe { crate::memory::hv_page_table().read().activate() }; - - // self.arch.init(self.id)?; - - // // Initialize vCPU. Use `ptr::write()` to avoid dropping - // unsafe { core::ptr::write(&mut self.vcpu, Vcpu::new(&self.linux, cell)?) }; - - // self.state = CpuState::HvEnabled; - // Ok(()) - // } - - // pub fn activate_vmm(&mut self) -> HvResult { - // println!("Activating hypervisor on CPU {}...", self.id); - // ACTIVATED_CPUS.fetch_add(1, Ordering::SeqCst); - - // self.vcpu.enter(&self.linux)?; - // unreachable!() - // } - - // pub fn deactivate_vmm(&mut self, ret_code: usize) -> HvResult { - // println!("Deactivating hypervisor on CPU {}...", self.id); - // ACTIVATED_CPUS.fetch_sub(1, Ordering::SeqCst); - - // self.vcpu.set_return_val(ret_code); - // self.vcpu.exit(&mut self.linux)?; - // self.linux.restore(); - // self.state = CpuState::HvDisabled; - // self.linux.return_to_linux(self.vcpu.regs()); - // } - - // pub fn fault(&mut self) -> HvResult { - // warn!("VCPU fault: {:#x?}", self); - // self.vcpu.inject_fault()?; - // Ok(()) - // } -} - -impl Debug for PerCpu { - fn fmt(&self, f: &mut Formatter) -> Result { - let mut res = f.debug_struct("PerCpu"); - res.field("id", &self.id) - .field("self_vaddr", &self.self_vaddr) - .field("state", &self.state); - // if self.state != CpuState::HvDisabled { - // res.field("vcpu", &self.vcpu); - // } else { - // res.field("linux", &self.linux); - // } - res.finish() - } -} diff --git a/modules/axhal/src/platform/x86_linux/time.rs b/modules/axhal/src/platform/x86_linux/time.rs index f146eaccdb..ee7046ca25 100644 --- a/modules/axhal/src/platform/x86_linux/time.rs +++ b/modules/axhal/src/platform/x86_linux/time.rs @@ -79,6 +79,9 @@ pub(super) fn init_early() { } } +/// Currently we just passthrough the Lapic to host Linux. +/// We can just enable APIC timer on CPUs that are reserved for ArceOS itself. +#[allow(unused)] pub(super) fn init_primary() { #[cfg(feature = "irq")] unsafe { @@ -96,6 +99,7 @@ pub(super) fn init_primary() { } } +#[allow(unused)] #[cfg(feature = "smp")] pub(super) fn init_secondary() { #[cfg(feature = "irq")] diff --git a/modules/axruntime/src/mp.rs b/modules/axruntime/src/mp.rs index 241f450418..3acc4966aa 100644 --- a/modules/axruntime/src/mp.rs +++ b/modules/axruntime/src/mp.rs @@ -18,7 +18,6 @@ pub fn start_secondary_cpus(primary_cpu_id: usize) { })); debug!("starting CPU {}...", i); - let current_entered_cpus = ENTERED_CPUS.load(Ordering::Acquire); let should_wait = axhal::mp::start_secondary_cpu(i, stack_top); logic_cpu_id += 1; From fd509e6730d134f0752806c1a91ef6c0645e4876 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Sun, 6 Apr 2025 15:51:49 +0800 Subject: [PATCH 17/19] [feat] introduce alloc_frames --- modules/axalloc/src/lib.rs | 8 +++++++- modules/axhal/src/paging.rs | 11 +++++++++++ modules/axhal/src/platform/x86_linux/apic.rs | 18 +++++++++++++----- modules/axhal/src/platform/x86_linux/mod.rs | 13 +++++++++---- modules/axhal/src/platform/x86_linux/mp.rs | 6 +++++- 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/modules/axalloc/src/lib.rs b/modules/axalloc/src/lib.rs index c9a2db6dea..979c3e67cf 100644 --- a/modules/axalloc/src/lib.rs +++ b/modules/axalloc/src/lib.rs @@ -143,7 +143,13 @@ impl GlobalAllocator { /// `align_pow2` must be a power of 2, and the returned region bound will be /// aligned to it. pub fn alloc_pages(&self, num_pages: usize, align_pow2: usize) -> AllocResult { - self.palloc.lock().alloc_pages(num_pages, align_pow2) + let res = self.palloc.lock().alloc_pages(num_pages, align_pow2); + + if res.is_err() { + warn!("alloc_pages res {:?}", res); + } + + res } /// Allocates contiguous pages starting from the given address. diff --git a/modules/axhal/src/paging.rs b/modules/axhal/src/paging.rs index 3d390304dc..ec3e02cb5e 100644 --- a/modules/axhal/src/paging.rs +++ b/modules/axhal/src/paging.rs @@ -42,10 +42,21 @@ impl PagingHandler for PagingHandlerImpl { .ok() } + fn alloc_frames(count: usize, align_pow2: usize) -> Option { + global_allocator() + .alloc_pages(count, align_pow2) + .map(|vaddr| virt_to_phys(vaddr.into())) + .ok() + } + fn dealloc_frame(paddr: PhysAddr) { global_allocator().dealloc_pages(phys_to_virt(paddr).as_usize(), 1) } + fn dealloc_frames(paddr: PhysAddr, count: usize) { + global_allocator().dealloc_pages(phys_to_virt(paddr).as_usize(), count) + } + #[inline] fn phys_to_virt(paddr: PhysAddr) -> VirtAddr { phys_to_virt(paddr) diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index 7fd4249eb5..b181ef909d 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -89,7 +89,7 @@ fn cpu_has_x2apic() -> bool { } } -pub(super) fn init_primary(enabled: bool) { +pub(super) fn init_primary(enabled: bool, cpu_id: usize) { info!("Initialize Local APIC..."); if enabled { @@ -121,7 +121,7 @@ pub(super) fn init_primary(enabled: bool) { lapic.enable(); } - APIC_TO_CPU_ID[lapic.id() as usize] = 0; + APIC_TO_CPU_ID[lapic.id() as usize] = cpu_id as u32; LOCAL_APIC.get().as_mut().unwrap().write(lapic); } @@ -132,7 +132,7 @@ pub(super) fn init_primary(enabled: bool) { } #[cfg(feature = "smp")] -pub(super) fn init_secondary(enabled: bool) { +pub(super) fn init_secondary(enabled: bool, cpu_id: usize) { let lapic = local_apic(); if enabled { @@ -141,7 +141,7 @@ pub(super) fn init_secondary(enabled: bool) { } } unsafe { - APIC_TO_CPU_ID[lapic.id() as usize] = 0; + APIC_TO_CPU_ID[lapic.id() as usize] = cpu_id as u32; }; } @@ -149,5 +149,13 @@ pub(super) fn init_secondary(enabled: bool) { /// The APIC ID is reserved if it entered Linux, which has set the corresponding /// entry in `APIC_TO_CPU_ID` to 0. pub(super) fn apic_id_is_reserved(apic_id: usize) -> bool { - unsafe { APIC_TO_CPU_ID[apic_id] == 0 } + unsafe { APIC_TO_CPU_ID[apic_id] != u32::MAX } +} + +pub(super) fn apic_to_cpu_id(apic_id: u32) -> u32 { + if apic_id <= MAX_APIC_ID { + unsafe { APIC_TO_CPU_ID[apic_id as usize] } + } else { + u32::MAX + } } diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 5e32c5601f..4273a251b4 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -171,20 +171,23 @@ extern "sysv64" fn vmm_cpu_entry(core_id: usize, linux_sp: usize) -> i32 { unsafe extern "C" fn rust_entry_secondary(magic: usize) { // #[cfg(feature = "smp")] if magic == self::boot::MULTIBOOT_BOOTLOADER_MAGIC { + let cpu_id = current_cpu_id(); // Note: DO not call log related functions before percpu area is initialized. - crate::cpu::init_secondary(current_cpu_id()); + crate::cpu::init_secondary(cpu_id); self::dtables::init_secondary(); unsafe { - rust_main_secondary(current_cpu_id()); + rust_main_secondary(cpu_id); } } } /// Initializes the platform devices for the primary CPU. pub fn platform_init() { + let cpu_id = current_cpu_id(); + // Consruct LAPIC but DO NOT operate the LAPIC. // because the LAPIC belongs to Linux, we should not touch it. - self::apic::init_primary(false); + self::apic::init_primary(false, cpu_id); // self::time::init_primary(); VMM_PRIMARY_INIT_OK.store(1, Ordering::Release); @@ -194,6 +197,8 @@ pub fn platform_init() { /// Initializes the platform devices for secondary CPUs. #[cfg(feature = "smp")] pub fn platform_init_secondary() { - self::apic::init_secondary(false); + let cpu_id = current_cpu_id(); + + self::apic::init_secondary(false, cpu_id); // self::time::init_secondary(); } diff --git a/modules/axhal/src/platform/x86_linux/mp.rs b/modules/axhal/src/platform/x86_linux/mp.rs index ad17205fa7..84386f4203 100644 --- a/modules/axhal/src/platform/x86_linux/mp.rs +++ b/modules/axhal/src/platform/x86_linux/mp.rs @@ -58,7 +58,11 @@ where pub fn start_secondary_cpu(apic_id: usize, stack_top: PhysAddr) -> bool { // DO not boot CPUs that are reserved for host Linux. if super::apic::apic_id_is_reserved(apic_id) { - info!("CPU {} is reserved for Linux, skip", apic_id); + info!( + "CPU {} APIC id {} is reserved for Linux, skip", + super::apic::apic_to_cpu_id(apic_id as u32), + apic_id + ); return false; } From e00480d5422f6838e2d158dfb0d5a87434cf2df7 Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Thu, 10 Apr 2025 10:27:36 +0800 Subject: [PATCH 18/19] [feat] support shutdown ap --- modules/axhal/src/cpu.rs | 16 +++++++++++++ modules/axhal/src/lib.rs | 1 + modules/axhal/src/platform/x86_linux/apic.rs | 25 +++++++++++++++++--- modules/axhal/src/platform/x86_linux/mod.rs | 3 ++- modules/axhal/src/platform/x86_linux/mp.rs | 24 +++++++++++++++++++ 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/modules/axhal/src/cpu.rs b/modules/axhal/src/cpu.rs index ae35d3654b..07aa4818e5 100644 --- a/modules/axhal/src/cpu.rs +++ b/modules/axhal/src/cpu.rs @@ -9,6 +9,9 @@ static IS_BSP: bool = false; #[percpu::def_percpu] static CURRENT_TASK_PTR: usize = 0; +#[percpu::def_percpu] +static IS_RESERVED: bool = false; + /// Returns the ID of the current CPU. #[inline] pub fn this_cpu_id() -> usize { @@ -22,6 +25,19 @@ pub fn this_cpu_is_bsp() -> bool { IS_BSP.read_current() } +/// Returns whether the current CPU is reserved for Linux. +#[inline] +pub fn this_cpu_is_reserved() -> bool { + IS_RESERVED.read_current() +} + +/// Sets the current CPU as reserved for Linux. +/// Only called from initialization code related to boot for Linux. +#[inline] +pub fn set_this_cpu_is_reserved() { + IS_RESERVED.write_current(true); +} + /// Gets the pointer to the current task with preemption-safety. /// /// Preemption may be enabled when calling this function. This function will diff --git a/modules/axhal/src/lib.rs b/modules/axhal/src/lib.rs index 5d906bafbd..aeec8fa4f2 100644 --- a/modules/axhal/src/lib.rs +++ b/modules/axhal/src/lib.rs @@ -82,3 +82,4 @@ pub use self::platform::config as hvconfig; pub use self::platform::context::get_linux_context_list; pub use self::platform::header as hvheader; pub use self::platform::mem::host_memory_regions; +pub use self::platform::mp::shutdown_secondary_cpus; diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index b181ef909d..2248a57411 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -33,6 +33,8 @@ static IO_APIC: LazyInit> = LazyInit::new(); const MAX_APIC_ID: u32 = 254; static mut APIC_TO_CPU_ID: [u32; MAX_APIC_ID as usize + 1] = [u32::MAX; MAX_APIC_ID as usize + 1]; +static mut APIC_ID_IS_RESERVED: [bool; MAX_APIC_ID as usize + 1] = + [false; MAX_APIC_ID as usize + 1]; /// Enables or disables the given IRQ. #[cfg(feature = "irq")] @@ -121,7 +123,11 @@ pub(super) fn init_primary(enabled: bool, cpu_id: usize) { lapic.enable(); } - APIC_TO_CPU_ID[lapic.id() as usize] = cpu_id as u32; + let apic_id = lapic.id(); + APIC_TO_CPU_ID[apic_id as usize] = cpu_id as u32; + if crate::cpu::this_cpu_is_reserved() { + APIC_ID_IS_RESERVED[apic_id as usize] = true; + } LOCAL_APIC.get().as_mut().unwrap().write(lapic); } @@ -141,7 +147,11 @@ pub(super) fn init_secondary(enabled: bool, cpu_id: usize) { } } unsafe { - APIC_TO_CPU_ID[lapic.id() as usize] = cpu_id as u32; + let apic_id = lapic.id(); + APIC_TO_CPU_ID[apic_id as usize] = cpu_id as u32; + if crate::cpu::this_cpu_is_reserved() { + APIC_ID_IS_RESERVED[apic_id as usize] = true; + } }; } @@ -149,7 +159,7 @@ pub(super) fn init_secondary(enabled: bool, cpu_id: usize) { /// The APIC ID is reserved if it entered Linux, which has set the corresponding /// entry in `APIC_TO_CPU_ID` to 0. pub(super) fn apic_id_is_reserved(apic_id: usize) -> bool { - unsafe { APIC_TO_CPU_ID[apic_id] != u32::MAX } + unsafe { APIC_ID_IS_RESERVED[apic_id as usize] } } pub(super) fn apic_to_cpu_id(apic_id: u32) -> u32 { @@ -159,3 +169,12 @@ pub(super) fn apic_to_cpu_id(apic_id: u32) -> u32 { u32::MAX } } + +/// Shuts down the target CPU by sending an INIT IPI to it. +pub(super) fn shutdown_ap(apic_id: u32) { + info!("Shutting down ArceOS cpu {apic_id}..."); + let apic_id = raw_apic_id(apic_id as u8); + unsafe { + local_apic().send_init_ipi(apic_id); + } +} diff --git a/modules/axhal/src/platform/x86_linux/mod.rs b/modules/axhal/src/platform/x86_linux/mod.rs index 4273a251b4..7ca192edab 100644 --- a/modules/axhal/src/platform/x86_linux/mod.rs +++ b/modules/axhal/src/platform/x86_linux/mod.rs @@ -1,5 +1,4 @@ mod apic; - mod boot; mod dtables; mod entry; @@ -70,6 +69,7 @@ fn vmm_primary_init_early(cpu_id: usize) { // Because currently the image was loaded by Linux. // crate::mem::clear_bss(); crate::cpu::init_primary(cpu_id); + crate::cpu::set_this_cpu_is_reserved(); self::uart16550::init(); } @@ -78,6 +78,7 @@ fn vmm_secondary_init_early(cpu_id: usize) { { println!("Secondary CPU {} init early.", cpu_id); crate::cpu::init_secondary(cpu_id); + crate::cpu::set_this_cpu_is_reserved(); } } diff --git a/modules/axhal/src/platform/x86_linux/mp.rs b/modules/axhal/src/platform/x86_linux/mp.rs index 84386f4203..5e3a146aa7 100644 --- a/modules/axhal/src/platform/x86_linux/mp.rs +++ b/modules/axhal/src/platform/x86_linux/mp.rs @@ -1,3 +1,5 @@ +use core::sync::atomic::AtomicBool; + use memory_addr::VirtAddr; use crate::mem::{PAGE_SIZE_4K, PhysAddr, phys_to_virt, virt_to_phys}; @@ -85,3 +87,25 @@ pub fn start_secondary_cpu(apic_id: usize, stack_top: PhysAddr) -> bool { return true; } + +static SHUTDOWN_SECONDARY_CPUS: AtomicBool = AtomicBool::new(false); + +pub fn shutdown_secondary_cpus() { + // Only need to shutdown secondary CPUs once. + if SHUTDOWN_SECONDARY_CPUS.load(core::sync::atomic::Ordering::SeqCst) { + return; + } + + SHUTDOWN_SECONDARY_CPUS.store(true, core::sync::atomic::Ordering::SeqCst); + + info!("Shutting down secondary CPUs..."); + + for cpuid in 0..axconfig::SMP { + // DO not shutdown CPUs that are reserved for host Linux. + if super::apic::apic_id_is_reserved(cpuid) { + continue; + } + debug!("Trying to shut down CPU {}", cpuid); + super::apic::shutdown_ap(cpuid as u32); + } +} From 35bf0f3c3c844dc9704065bb8f5dbb01da00073e Mon Sep 17 00:00:00 2001 From: hky1999 <976929993@qq.com> Date: Tue, 15 Apr 2025 21:24:01 +0800 Subject: [PATCH 19/19] [refactor] improve terminate, disable virt --- modules/axhal/src/platform/x86_linux/apic.rs | 11 +++++++++- .../axhal/src/platform/x86_linux/config.rs | 15 ++++++++++++++ modules/axhal/src/platform/x86_linux/misc.rs | 20 ++++++++++++++++--- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/modules/axhal/src/platform/x86_linux/apic.rs b/modules/axhal/src/platform/x86_linux/apic.rs index 2248a57411..e7efca59d6 100644 --- a/modules/axhal/src/platform/x86_linux/apic.rs +++ b/modules/axhal/src/platform/x86_linux/apic.rs @@ -159,7 +159,7 @@ pub(super) fn init_secondary(enabled: bool, cpu_id: usize) { /// The APIC ID is reserved if it entered Linux, which has set the corresponding /// entry in `APIC_TO_CPU_ID` to 0. pub(super) fn apic_id_is_reserved(apic_id: usize) -> bool { - unsafe { APIC_ID_IS_RESERVED[apic_id as usize] } + unsafe { APIC_ID_IS_RESERVED[apic_id] } } pub(super) fn apic_to_cpu_id(apic_id: u32) -> u32 { @@ -170,6 +170,15 @@ pub(super) fn apic_to_cpu_id(apic_id: u32) -> u32 { } } +pub(super) fn cpu_id_to_apic_id(cpu_id: usize) -> Option { + for (apic_id, id) in unsafe { APIC_TO_CPU_ID.iter().enumerate() } { + if *id == cpu_id as u32 { + return Some(apic_id as u32); + } + } + None +} + /// Shuts down the target CPU by sending an INIT IPI to it. pub(super) fn shutdown_ap(apic_id: u32) { info!("Shutting down ArceOS cpu {apic_id}..."); diff --git a/modules/axhal/src/platform/x86_linux/config.rs b/modules/axhal/src/platform/x86_linux/config.rs index af8a2774e4..7f9e716508 100644 --- a/modules/axhal/src/platform/x86_linux/config.rs +++ b/modules/axhal/src/platform/x86_linux/config.rs @@ -157,3 +157,18 @@ impl Debug for CellConfig<'_> { .finish() } } + +pub fn cpu_is_reserved(cpu_id: usize) -> bool { + use super::apic::{apic_id_is_reserved, cpu_id_to_apic_id}; + if cpu_id >= axconfig::SMP { + warn!("cpu_id {} is out of range", cpu_id); + return false; + } + + if let Some(apic_id) = cpu_id_to_apic_id(cpu_id) { + return apic_id_is_reserved(apic_id as usize); + } else { + warn!("cpu_id {} is not mapped to apic id", cpu_id); + return false; + } +} diff --git a/modules/axhal/src/platform/x86_linux/misc.rs b/modules/axhal/src/platform/x86_linux/misc.rs index 52d54ca921..ba54713474 100644 --- a/modules/axhal/src/platform/x86_linux/misc.rs +++ b/modules/axhal/src/platform/x86_linux/misc.rs @@ -1,14 +1,28 @@ +use x86::bits64::vmx; use x86_64::instructions::port::PortWriteOnly; +use x86_64::registers::control::{Cr4, Cr4Flags}; + +use crate::cpu::{this_cpu_id, this_cpu_is_reserved}; /// Shutdown the whole system (in QEMU), including all CPUs. /// /// See for more information. pub fn terminate() -> ! { info!("Shutting down..."); - axlog::ax_println!("\nMaybe we need to find a way to return this core to Linux?\n"); - // #[cfg(platform = "x86_64-qemu-q35")] - unsafe { PortWriteOnly::new(0x604).write(0x2000u16) }; + if this_cpu_is_reserved() { + // #[cfg(platform = "x86_64-qemu-q35")] + unsafe { PortWriteOnly::new(0x604).write(0x2000u16) }; + } else { + warn!("Instance {} CPU terminated", this_cpu_id()); + + // Execute VMXOFF. + unsafe { + let _ = vmx::vmxoff(); + // Remove VMXE bit in CR4. + Cr4::update(|cr4| cr4.remove(Cr4Flags::VIRTUAL_MACHINE_EXTENSIONS)); + } + } crate::arch::halt(); warn!("It should shutdown!");