diff --git a/.gitignore b/.gitignore index 2a2c5a3dd2..9e0184934b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ qemu.log rusty-tags.vi /.project* /.axconfig.* +zignore-* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 3afcf98b3d..891377d017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,6 +137,13 @@ dependencies = [ "crate_interface", ] +[[package]] +name = "arceos-ticker" +version = "0.1.0" +dependencies = [ + "axstd", +] + [[package]] name = "arceos_api" version = "0.1.0" @@ -151,6 +158,7 @@ dependencies = [ "axfs", "axhal", "axio", + "axipi", "axlog", "axmm", "axnet", @@ -200,7 +208,7 @@ dependencies = [ [[package]] name = "arm_gicv2" version = "0.1.0" -source = "git+https://github.com/arceos-hypervisor/arm_gicv2#dfe5f164b94cdd07081c2fe74a0cfe4bef2852c9" +source = "git+https://github.com/arceos-hypervisor/arm_gicv2?branch=debin%2F2vm_timer#0ad077649e030b9836a5dc55a5a01b1fd0246e11" dependencies = [ "tock-registers 0.8.1", ] @@ -408,6 +416,7 @@ dependencies = [ "axdriver", "axfs", "axhal", + "axipi", "axlog", "axnet", "axruntime", @@ -486,6 +495,7 @@ dependencies = [ "bitflags 2.9.1", "cfg-if", "cortex-a", + "cpumask", "crate_interface", "dw_apb_uart", "handler_table", @@ -504,6 +514,7 @@ dependencies = [ "raw-cpuid 11.4.0", "riscv", "riscv_goldfish", + "riscv_plic", "sbi-rt", "static_assertions", "tock-registers 0.9.0", @@ -522,6 +533,18 @@ dependencies = [ "axerrno", ] +[[package]] +name = "axipi" +version = "0.1.0" +dependencies = [ + "axconfig", + "axhal", + "kspin", + "lazyinit", + "log", + "percpu", +] + [[package]] name = "axlibc" version = "0.1.0" @@ -597,6 +620,7 @@ dependencies = [ "axdriver", "axfs", "axhal", + "axipi", "axlog", "axmm", "axnet", @@ -1303,15 +1327,15 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memory_addr" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f769efcf10b9dfb4c913bebb409cda77b1a3f072b249bf5465e250bcb30eb49" +checksum = "3d4054cba279515fa87761b101d857333ce06391dbe8f18a11347204a7111416" [[package]] name = "memory_set" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335675b7ab07460f532d3b2b557313be73037fdf81b65464d8c0d8bd90d2fbf9" +checksum = "e7d47cbc25a4d00427f9070fd768eaf907f19c903fb72b547b19db2d56b9408e" dependencies = [ "memory_addr", ] @@ -1355,9 +1379,9 @@ checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "page_table_entry" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c097d641745a066856a26eed6e486d4430bb3e32c94f1203ea09c63239b360a0" +checksum = "ba811ef8ca8fb33d776e128624cb4fe25c9804cab96f83b822d4322431e6dd5a" dependencies = [ "aarch64-cpu 10.0.0", "bitflags 2.9.1", @@ -1367,10 +1391,11 @@ dependencies = [ [[package]] name = "page_table_multiarch" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4647889585d29762d747be0916d6d28db72967a697d142be86f187a6b496832a" +checksum = "98cb76e21ce462270afd83b331599d5b83f876c2a98c0a70382b20d73e1da6be" dependencies = [ + "bitmaps", "log", "memory_addr", "page_table_entry", @@ -1577,9 +1602,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "riscv" -version = "0.12.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea8ff73d3720bdd0a97925f0bf79ad2744b6da8ff36be3840c48ac81191d7a7" +checksum = "0f1671c79a01a149fe000af2429ce9ccc8e58cdecda72672355d50e5536b363c" dependencies = [ "critical-section", "embedded-hal", @@ -1590,9 +1615,9 @@ dependencies = [ [[package]] name = "riscv-macros" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f265be5d634272320a7de94cea15c22a3bfdd4eb42eb43edc528415f066a1f25" +checksum = "e8c4aa1ea1af6dcc83a61be12e8189f9b293c3ba5a487778a4cd89fb060fdbbc" dependencies = [ "proc-macro2", "quote", @@ -1611,6 +1636,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07aac72f95e774476db82916d79f2d303191310393830573c1ab5c821b21660a" +[[package]] +name = "riscv_plic" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daae4c8e29d4d8c36fbda4c318ffa1755645766fefab8fcff61a8ecd577bb822" +dependencies = [ + "tock-registers 0.8.1", +] + [[package]] name = "rlsf" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 79482084de..28da4705af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "modules/axruntime", "modules/axsync", "modules/axtask", + "modules/axipi", "api/axfeat", "api/arceos_api", @@ -29,6 +30,8 @@ members = [ "examples/httpserver", "examples/httpserver", "examples/shell", + + "examples/ticker", ] [workspace.package] @@ -64,6 +67,7 @@ axruntime = { path = "modules/axruntime" } axsync = { path = "modules/axsync" } axtask = { path = "modules/axtask" } axdma = { path = "modules/axdma" } +axipi = { path = "modules/axipi" } allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } diff --git a/api/arceos_api/Cargo.toml b/api/arceos_api/Cargo.toml index 9f679d830f..e7db29ee11 100644 --- a/api/arceos_api/Cargo.toml +++ b/api/arceos_api/Cargo.toml @@ -13,6 +13,7 @@ documentation = "https://arceos-org.github.io/arceos/arceos_api/index.html" default = [] irq = ["axfeat/irq"] +ipi = ["dep:axipi", "axfeat/ipi"] alloc = ["dep:axalloc", "axfeat/alloc"] paging = ["dep:axmm", "axfeat/paging"] dma = ["dep:axdma", "axfeat/dma"] @@ -43,3 +44,4 @@ axdriver = { workspace = true, optional = true } axfs = { workspace = true, optional = true } axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } +axipi = { workspace = true, optional = true } diff --git a/api/arceos_api/src/lib.rs b/api/arceos_api/src/lib.rs index 3e6d291870..95658a17e6 100644 --- a/api/arceos_api/src/lib.rs +++ b/api/arceos_api/src/lib.rs @@ -402,6 +402,8 @@ pub mod modules { pub use axdriver; #[cfg(feature = "fs")] pub use axfs; + #[cfg(feature = "ipi")] + pub use axipi; #[cfg(feature = "paging")] pub use axmm; #[cfg(feature = "net")] diff --git a/api/arceos_posix_api/build.rs b/api/arceos_posix_api/build.rs index bb8368511e..09adb71ac9 100644 --- a/api/arceos_posix_api/build.rs +++ b/api/arceos_posix_api/build.rs @@ -76,11 +76,12 @@ typedef struct {{ impl bindgen::callbacks::ParseCallbacks for MyCallbacks { fn include_file(&self, fname: &str) { if !fname.contains("ax_pthread_mutex.h") { - println!("cargo:rerun-if-changed={}", fname); + println!("cargo:rerun-if-changed={fname}"); } } } + let target = std::env::var("TARGET").unwrap(); let mut builder = bindgen::Builder::default() .header(in_file) .clang_arg("-I./../../ulib/axlibc/include") @@ -88,6 +89,10 @@ typedef struct {{ .derive_default(true) .size_t_is_usize(false) .use_core(); + if let Some(llvm_target) = target.strip_suffix("-softfloat") { + // remove "-softfloat" suffix for some targets + builder = builder.clang_arg(format!("--target={llvm_target}")); + } for ty in allow_types { builder = builder.allowlist_type(ty); } diff --git a/api/axfeat/Cargo.toml b/api/axfeat/Cargo.toml index 5163d97c15..da8819dcb6 100644 --- a/api/axfeat/Cargo.toml +++ b/api/axfeat/Cargo.toml @@ -20,6 +20,7 @@ fp_simd = ["axhal/fp_simd"] # Interrupts irq = ["axhal/irq", "axruntime/irq", "axtask?/irq"] +ipi = ["irq", "dep:axipi", "axhal/ipi", "axruntime/ipi"] gicv3 = ["axhal/gicv3"] # Memory @@ -61,7 +62,7 @@ driver-fxmac = ["axdriver?/fxmac"] # fxmac ethernet driver for PhytiumPi driver-bcm2835-sdhci = ["axdriver?/bcm2835-sdhci"] #Hypervisor support -hv = ["axhal/hv"] +hv = ["axhal/hv", "ipi"] # Logging log-level-off = ["axlog/log-level-off"] @@ -82,4 +83,5 @@ axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } axsync = { workspace = true, optional = true } axtask = { workspace = true, optional = true } +axipi = { workspace = true, optional = true } kspin = { version = "0.1", optional = true } diff --git a/api/axfeat/src/lib.rs b/api/axfeat/src/lib.rs index 85fc2abb26..44a1ba57dc 100644 --- a/api/axfeat/src/lib.rs +++ b/api/axfeat/src/lib.rs @@ -7,6 +7,7 @@ //! - `fp_simd`: Enable floating point and SIMD support. //! - Interrupts: //! - `irq`: Enable interrupt handling support. +//! - `ipi`: Enable Inter-Processor Interrupts (IPIs). //! - Memory //! - `alloc`: Enable dynamic memory allocation. //! - `alloc-tlsf`: Use the TLSF allocator. diff --git a/configs/platforms/aarch64-qemu-virt.toml b/configs/platforms/aarch64-qemu-virt.toml index add3f14f06..71260bb840 100644 --- a/configs/platforms/aarch64-qemu-virt.toml +++ b/configs/platforms/aarch64-qemu-virt.toml @@ -97,7 +97,7 @@ uart-irq = 1 # uint gicc-paddr = 0x0801_0000 # uint # GIC Distributor base address gicd-paddr = 0x0800_0000 # uint -# GIC Rdistributor base address +# GIC Redistributor base address gicr-paddr = 0x080a_0000 # PSCI diff --git a/configs/platforms/aarch64-rk3588j.toml b/configs/platforms/aarch64-rk3588j.toml index 6624cc307e..9a769f3279 100644 --- a/configs/platforms/aarch64-rk3588j.toml +++ b/configs/platforms/aarch64-rk3588j.toml @@ -72,4 +72,5 @@ psci-method = "smc" # compatible = "arm,pl031\0arm,primecell"; # }; # RTC (PL031) Address -# rtc-paddr = "0x901_0000" \ No newline at end of file +# rtc-paddr = "0x901_0000" +nothing = "nothing" # str diff --git a/examples/helloworld/Cargo.toml b/examples/helloworld/Cargo.toml index 31806aa9bb..30ccb889d3 100644 --- a/examples/helloworld/Cargo.toml +++ b/examples/helloworld/Cargo.toml @@ -7,4 +7,4 @@ authors = ["Yuekai Jia "] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -axstd = { workspace = true, optional = true } +axstd = { workspace = true, features = ["irq"], optional = true } diff --git a/examples/ticker/Cargo.toml b/examples/ticker/Cargo.toml new file mode 100644 index 0000000000..5acf582b68 --- /dev/null +++ b/examples/ticker/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "arceos-ticker" +version = "0.1.0" +edition.workspace = true +authors = ["aarkegz "] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axstd = { workspace = true, features = ["irq", "multitask", "paging"], optional = true } diff --git a/examples/ticker/src/main.rs b/examples/ticker/src/main.rs new file mode 100644 index 0000000000..f99bf94043 --- /dev/null +++ b/examples/ticker/src/main.rs @@ -0,0 +1,26 @@ +#![cfg_attr(feature = "axstd", no_std)] +#![cfg_attr(feature = "axstd", no_main)] + +use core::time::Duration; + +#[cfg(feature = "axstd")] +use axstd::os::arceos::api::time::ax_monotonic_time; +#[cfg(feature = "axstd")] +use axstd::os::arceos::modules; +#[cfg(feature = "axstd")] +use axstd::println; + +#[cfg_attr(feature = "axstd", unsafe(no_mangle))] +fn main() { + #[cfg(feature = "axstd")] + { + println!("Ticking!"); + + println!("Current monotonic time: {:?}", ax_monotonic_time()); + + loop { + modules::axtask::sleep(Duration::from_millis(500)); + println!("Current monotonic time: {:?}", ax_monotonic_time()); + } + } +} diff --git a/modules/axalloc/Cargo.toml b/modules/axalloc/Cargo.toml index c0314aa1ef..139012efdc 100644 --- a/modules/axalloc/Cargo.toml +++ b/modules/axalloc/Cargo.toml @@ -21,6 +21,6 @@ page-alloc-4g = ["allocator/page-alloc-4g"] # Support up to 4G memory capacity log = "=0.4.21" cfg-if = "1.0" kspin = "0.1" -memory_addr = "0.3" +memory_addr = "0.4" axerrno = "0.1" allocator = { git = "https://github.com/arceos-org/allocator.git", tag ="v0.1.1", features = ["bitmap"] } diff --git a/modules/axdma/Cargo.toml b/modules/axdma/Cargo.toml index 12095f703a..1dd462595c 100644 --- a/modules/axdma/Cargo.toml +++ b/modules/axdma/Cargo.toml @@ -13,7 +13,7 @@ documentation = "https://arceos-org.github.io/arceos/axdma/index.html" [dependencies] log = "=0.4.21" kspin = "0.1" -memory_addr = "0.3" +memory_addr = "0.4" axerrno = "0.1" allocator = { git = "https://github.com/arceos-org/allocator.git", tag = "v0.1.1" } axalloc = { workspace = true } diff --git a/modules/axhal/Cargo.toml b/modules/axhal/Cargo.toml index bad3d708a5..1788402665 100644 --- a/modules/axhal/Cargo.toml +++ b/modules/axhal/Cargo.toml @@ -20,7 +20,16 @@ tls = ["alloc"] rtc = ["x86_rtc", "riscv_goldfish", "arm_pl031"] uspace = ["paging"] default = [] -hv = ["paging", "cortex-a", "percpu/arm-el2", "page_table_entry/arm-el2", "dep:crate_interface"] +hv = [ + "paging", + "cortex-a", + "percpu/arm-el2", + "page_table_entry/arm-el2", + "arm_gicv2/el2", + "arm_gicv2/hv", + "dep:crate_interface", +] +ipi = ["irq"] [dependencies] log = "=0.4.21" @@ -33,14 +42,15 @@ kspin = "0.1" int_ratio = "0.1.1" lazyinit = "0.2" percpu = "0.2" -memory_addr = "0.3" +memory_addr = "0.4" handler_table = "0.1" page_table_entry = "0.5" -page_table_multiarch = "0.5" +page_table_multiarch = { version = "0.5", features = ["copy-from"] } axlog = { workspace = true } axconfig = { workspace = true } axalloc = { workspace = true, optional = true } cortex-a = { version = "8.1.1", optional = true } +cpumask = { version = "0.1", optional = true } [target.'cfg(target_arch = "x86_64")'.dependencies] x86 = "0.52" @@ -50,14 +60,16 @@ raw-cpuid = "11.3" x86_rtc = { version = "0.1", optional = true } [target.'cfg(any(target_arch = "riscv32", target_arch = "riscv64"))'.dependencies] -riscv = "0.12" +riscv = "0.14" sbi-rt = { version = "0.0.3", features = ["legacy"] } +riscv_plic = { version = "0.1.0" } riscv_goldfish = { version = "0.1", optional = true } [target.'cfg(target_arch = "aarch64")'.dependencies] aarch64-cpu = "10.0" tock-registers = "0.9" arm-gic-driver = { git = "https://github.com/rcore-os/arm-gic-driver", rev = "ad1fddc194f1cf5b7602e7f35a7136f823465cb0" } +arm_gicv2 = { git = "https://github.com/arceos-hypervisor/arm_gicv2", branch = "debin/2vm_timer" } crate_interface = { version = "0.1.3", optional = true } arm_pl011 = "0.1" arm_pl031 = { version = "0.2", optional = true } diff --git a/modules/axhal/src/arch/riscv/mod.rs b/modules/axhal/src/arch/riscv/mod.rs index 07caf58643..226d1b694f 100644 --- a/modules/axhal/src/arch/riscv/mod.rs +++ b/modules/axhal/src/arch/riscv/mod.rs @@ -5,8 +5,13 @@ mod context; mod trap; use memory_addr::{PhysAddr, VirtAddr}; -use riscv::asm; -use riscv::register::{satp, sstatus, stvec}; +use riscv::{ + asm, + register::{ + satp, sstatus, + stvec::{self, Stvec, TrapMode}, + }, +}; #[cfg(feature = "uspace")] pub use self::context::UspaceContext; @@ -85,7 +90,10 @@ pub fn flush_tlb(vaddr: Option) { /// Writes Supervisor Trap Vector Base Address Register (`stvec`). #[inline] pub fn set_trap_vector_base(stvec: usize) { - unsafe { stvec::write(stvec, stvec::TrapMode::Direct) } + let mut stvec = Stvec::from_bits(stvec); + stvec.set_trap_mode(TrapMode::Direct); + + unsafe { stvec::write(stvec) } } /// Reads the thread pointer of the current CPU. diff --git a/modules/axhal/src/irq.rs b/modules/axhal/src/irq.rs index 2d6b1247c6..b71c84ecf7 100644 --- a/modules/axhal/src/irq.rs +++ b/modules/axhal/src/irq.rs @@ -7,6 +7,12 @@ use crate::trap::{IRQ, register_trap_handler}; pub use crate::platform::irq::{register_handler, set_enable}; +#[cfg(all(target_arch = "aarch64", feature = "hv"))] +pub use crate::platform::irq::{MyVgic, inject_interrupt}; + +#[cfg(feature = "ipi")] +pub use crate::platform::irq::{IPI_IRQ_NUM, send_ipi_all_others, send_ipi_one}; + #[cfg(target_arch = "aarch64")] pub use crate::platform::irq::fetch_irq; diff --git a/modules/axhal/src/platform/aarch64_common/boot_el2.rs b/modules/axhal/src/platform/aarch64_common/boot_el2.rs index 931a00e991..ebe600e675 100644 --- a/modules/axhal/src/platform/aarch64_common/boot_el2.rs +++ b/modules/axhal/src/platform/aarch64_common/boot_el2.rs @@ -43,6 +43,10 @@ unsafe fn init_mmu_el2() { // * Virtual IRQ interrupts are enabled; // * Physical FIQ interrupts are taken to EL2; // * Virtual FIQ interrupts are enabled. + // + // To passthrough IRQs to EL1, disable `HCR_EL2::IMO::EnableVirtualIRQ` and + // `HCR_EL2::FMO::EnableVirtualFIQ`, which can be done in `arm_vcpu`. + // `HCR_EL2` here is used only outside of guest VMs. HCR_EL2.modify( HCR_EL2::VM::Enable + HCR_EL2::RW::EL1IsAarch64 @@ -51,6 +55,9 @@ unsafe fn init_mmu_el2() { + HCR_EL2::TSC::EnableTrapEl1SmcToEl2, ); + // By default we enable the virtual GIC CPU interface. `arm_vcpu` may override this. + ICH_HCR_EL2.modify(ICH_HCR_EL2::En.val(1)); + // Device-nGnRE memory let attr0 = MAIR_EL2::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck; // Normal memory diff --git a/modules/axhal/src/platform/aarch64_common/generic_timer.rs b/modules/axhal/src/platform/aarch64_common/generic_timer.rs index 127db32c39..14119bb2b3 100644 --- a/modules/axhal/src/platform/aarch64_common/generic_timer.rs +++ b/modules/axhal/src/platform/aarch64_common/generic_timer.rs @@ -94,6 +94,15 @@ pub(crate) fn init_early() { } pub(crate) fn init_percpu() { + #[cfg(all(not(feature = "irq"), feature = "hv"))] + { + use aarch64_cpu::registers::{CNTHCTL_EL2, CNTVOFF_EL2}; + use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; + // Disable EL1 timer traps and the timer offset. + CNTHCTL_EL2.modify(CNTHCTL_EL2::EL1PCEN::SET + CNTHCTL_EL2::EL1PCTEN::SET); + CNTVOFF_EL2.set(0); + } + #[cfg(all(feature = "irq", not(feature = "hv")))] { CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET); @@ -101,12 +110,14 @@ pub(crate) fn init_percpu() { } #[cfg(all(feature = "irq", feature = "hv"))] { + // `arm_vcpu` will set `CNTVOFF_EL2` and `CNTHCTL_EL2` later. + unsafe { // ENABLE, bit [0], Enables the timer. // * 0b0: Timer disabled. // * 0b1: Timer enabled. - core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); - core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); + // core::arch::asm!("msr CNTHP_CTL_EL2, {0:x}", in(reg) 0b1); + // core::arch::asm!("msr CNTHP_TVAL_EL2, {0:x}", in(reg) 0); } } #[cfg(feature = "irq")] diff --git a/modules/axhal/src/platform/aarch64_common/gic.rs b/modules/axhal/src/platform/aarch64_common/gic.rs new file mode 100644 index 0000000000..5d63ebe1a8 --- /dev/null +++ b/modules/axhal/src/platform/aarch64_common/gic.rs @@ -0,0 +1,145 @@ +use crate::{irq::IrqHandler, mem::phys_to_virt}; +use arm_gicv2::{GicCpuInterface, GicDistributor, InterruptType, translate_irq}; +use axconfig::devices::{GICC_PADDR, GICD_PADDR, UART_IRQ}; +use kspin::SpinNoIrq; +use memory_addr::PhysAddr; + +#[cfg(feature = "hv")] +use arm_gicv2::GicHypervisorInterface; +#[cfg(feature = "hv")] +use axconfig::devices::{GICH_PADDR, GICV_PADDR}; + +/// The maximum number of IRQs. +pub const MAX_IRQ_COUNT: usize = 1024; + +#[cfg(not(feature = "hv"))] +/// The timer IRQ number. +pub const TIMER_IRQ_NUM: usize = translate_irq(14, InterruptType::PPI).unwrap(); + +#[cfg(feature = "hv")] +/// Non-secure EL2 Physical Timer irq number. +pub const TIMER_IRQ_NUM: usize = translate_irq(10, InterruptType::PPI).unwrap(); + +/// The UART IRQ number. +pub const UART_IRQ_NUM: usize = translate_irq(UART_IRQ, InterruptType::SPI).unwrap(); +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); + +const GICD_BASE: PhysAddr = pa!(GICD_PADDR); +const GICC_BASE: PhysAddr = pa!(GICC_PADDR); + +#[cfg(feature = "hv")] +const GICV_BASE: PhysAddr = pa!(GICV_PADDR); +#[cfg(feature = "hv")] +const GICH_BASE: PhysAddr = pa!(GICH_PADDR); + +static GICD: SpinNoIrq = + SpinNoIrq::new(GicDistributor::new(phys_to_virt(GICD_BASE).as_mut_ptr())); + +// per-CPU, no lock +static GICC: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICC_BASE).as_mut_ptr()); +#[cfg(feature = "hv")] +static GICV: GicCpuInterface = GicCpuInterface::new(phys_to_virt(GICV_BASE).as_mut_ptr()); +#[cfg(feature = "hv")] +static GICH: GicHypervisorInterface = + GicHypervisorInterface::new(phys_to_virt(GICH_BASE).as_mut_ptr()); + +/// Enables or disables the given IRQ. +pub fn set_enable(irq_num: usize, enabled: bool) { + trace!("GICD set enable: {} {}", irq_num, enabled); + GICD.lock().set_enable(irq_num as _, enabled); +} + +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_ipi_one(dest_cpu_id: usize, irq_num: usize) { + GICD.lock().send_sgi(dest_cpu_id, irq_num); +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_ipi_all_others(irq_num: usize) { + GICD.lock().send_sgi_all_except_self(irq_num); +} + +/// Registers an IRQ handler for the given IRQ. +/// +/// It also enables the IRQ if the registration succeeds. It returns `false` if +/// the registration failed. +pub fn register_handler(irq_num: usize, handler: IrqHandler) -> bool { + trace!("register handler irq {}", irq_num); + crate::irq::register_handler_common(irq_num, handler) +} + +/// Fetches the IRQ number. +pub fn fetch_irq() -> usize { + GICC.iar() as usize +} + +/// Dispatches the IRQ. +/// +/// This function is called by the common interrupt handler. It looks +/// up in the IRQ handler table and calls the corresponding handler. If +/// necessary, it also acknowledges the interrupt controller after handling. +pub fn dispatch_irq(irq_no: usize) { + // I know, `irq_no == 0` seems very strange here. + // The truth is, the previous design of ArceOS in aarch64 DO NOT fetch the IRQ number from GICC, + // until the function closure passed to `GICC.handle_irq`. + // So, the `irq_no` is always 0 when `dispatch_irq` is called by `handler_irq` from inside ArceOS. + // + // However, when `handler_irq` is called by the arceos-vmm app, the `irq_no` has been fetched from GICC, so we can not use the `handler_irq` directly. + // Instead, we call `GICC.eoi` and `GICC.dir` manually. + if irq_no == 0 { + GICC.handle_irq(|irq_num| crate::irq::dispatch_irq_common(irq_num as _)); + } else { + crate::irq::dispatch_irq_common(irq_no as _); + GICC.eoi(irq_no as _); + GICC.dir(irq_no as _); + } +} + +/// Initializes GICD, GICC on the primary CPU. +pub(crate) fn init_primary() { + info!("Initialize GICv2..."); + GICD.lock().init(); + GICC.init(); + + #[cfg(feature = "hv")] + { + GICV.init(); + } +} + +#[cfg(feature = "hv")] +pub fn inject_interrupt(vector: usize) { + let hcr = GICH.get_hcr(); + GICH.set_hcr(hcr | 1 << 0); + let mut lr = 0; + lr |= vector << 0; + lr |= 1 << 19; + lr |= 1 << 28; + GICH.set_lr(0, lr as u32); +} + +#[cfg(feature = "hv")] +pub struct MyVgic {} + +#[cfg(feature = "hv")] +impl MyVgic { + pub fn get_gich() -> &'static GicHypervisorInterface { + &GICH + } + pub fn get_gicd() -> &'static SpinNoIrq { + &GICD + } + pub fn get_gicc() -> &'static GicCpuInterface { + &GICC + } + pub fn get_gicv() -> &'static GicCpuInterface { + &GICV + } +} + +/// Initializes GICC on secondary CPUs. +#[cfg(feature = "smp")] +pub(crate) fn init_secondary() { + GICC.init(); +} diff --git a/modules/axhal/src/platform/aarch64_common/gicv3.rs b/modules/axhal/src/platform/aarch64_common/gicv3.rs index febd096c1c..447ba1141f 100644 --- a/modules/axhal/src/platform/aarch64_common/gicv3.rs +++ b/modules/axhal/src/platform/aarch64_common/gicv3.rs @@ -1,12 +1,18 @@ extern crate alloc; -use crate::{arch::disable_irqs, irq::IrqHandler, mem::phys_to_virt}; +use crate::{arch::disable_irqs, cpu::this_cpu_id, irq::IrqHandler, mem::phys_to_virt}; use alloc::boxed::Box; use arm_gic_driver::*; +#[cfg(feature = "hv")] +use arm_gicv2::GicHypervisorInterface; +use arm_gicv2::{InterruptType, translate_irq}; use axconfig::devices::{GICD_PADDR, GICR_PADDR, UART_IRQ}; -use core::ptr::NonNull; +use core::{panic, ptr::NonNull}; use kspin::SpinNoIrq; -use memory_addr::PhysAddr; +use memory_addr::{MemoryAddr, PhysAddr}; + +use aarch64_cpu::registers::{ICC_SRE_EL2, SCTLR_EL3::I}; +use tock_registers::interfaces::{Readable, Writeable}; /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; @@ -21,6 +27,8 @@ pub const TIMER_IRQ_NUM: usize = arm_gic_driver::IntId::ppi(10).to_u32() as usiz /// The UART IRQ number. pub const UART_IRQ_NUM: usize = arm_gic_driver::IntId::spi(UART_IRQ as u32).to_u32() as usize; +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = translate_irq(1, InterruptType::SGI).unwrap(); const GICD_BASE: PhysAddr = pa!(GICD_PADDR); const GICR_BASE: PhysAddr = pa!(GICR_PADDR); @@ -79,6 +87,7 @@ pub fn fetch_irq() -> usize { /// up in the IRQ handler table and calls the corresponding handler. If /// necessary, it also acknowledges the interrupt controller after handling. pub fn dispatch_irq(irq_num: usize) { + trace!("dispatch_irq: {}", irq_num); let intid: Option; if irq_num == 0 { intid = GICR.lock().as_mut().unwrap().ack(); @@ -92,7 +101,159 @@ pub fn dispatch_irq(irq_num: usize) { } } -/// Initializes GICD, GICR on the primary CPU. +/// Reads and returns the value of the given aarch64 system register. +macro_rules! read_sysreg { + ($name:ident) => { + { + let mut value: u64; + unsafe{::core::arch::asm!( + concat!("mrs {value:x}, ", ::core::stringify!($name)), + value = out(reg) value, + options(nomem, nostack), + );} + value + } + } +} + +/// Writes the given value to the given aarch64 system register. +macro_rules! write_sysreg { + ($name:ident, $value:expr) => { + { + let v: u64 = $value; + unsafe{::core::arch::asm!( + concat!("msr ", ::core::stringify!($name), ", {value:x}"), + value = in(reg) v, + options(nomem, nostack), + )} + } + } +} + +#[cfg(feature = "hv")] +pub fn inject_interrupt(vector: usize) { + // mask + const LR_VIRTIRQ_MASK: usize = (1 << 32) - 1; + + let elsr: u64 = read_sysreg!(ich_elrsr_el2); + let vtr = read_sysreg!(ich_vtr_el2) as usize; + let lr_num: usize = (vtr & 0xf) + 1; + let mut free_lr = -1 as isize; + for i in 0..lr_num { + // find a free list register + if (1 << i) & elsr > 0 { + if free_lr == -1 { + free_lr = i as isize; + } + continue; + } + let lr_val = read_lr(i) as usize; + // if a virtual interrupt is enabled and equals to the physical interrupt irq_id + if (lr_val & LR_VIRTIRQ_MASK) == vector { + trace!("virtual irq {} enables again", vector); + } + } + trace!("use free lr {} to inject irq {}", free_lr, vector); + + if free_lr == -1 { + panic!("No free list register to inject IRQ {}", vector); + } else { + let mut val = vector as u64; // vector + val |= 1 << 60; // group 1 + val |= 1 << 62; // state pending + // hardware interrupt not supported + write_lr(free_lr as usize, val); + } +} + +fn read_lr(id: usize) -> u64 { + let id = id as u64; + match id { + //TODO get lr size from gic reg + 0 => read_sysreg!(ich_lr0_el2), + 1 => read_sysreg!(ich_lr1_el2), + 2 => read_sysreg!(ich_lr2_el2), + 3 => read_sysreg!(ich_lr3_el2), + 4 => read_sysreg!(ich_lr4_el2), + 5 => read_sysreg!(ich_lr5_el2), + 6 => read_sysreg!(ich_lr6_el2), + 7 => read_sysreg!(ich_lr7_el2), + 8 => read_sysreg!(ich_lr8_el2), + 9 => read_sysreg!(ich_lr9_el2), + 10 => read_sysreg!(ich_lr10_el2), + 11 => read_sysreg!(ich_lr11_el2), + 12 => read_sysreg!(ich_lr12_el2), + 13 => read_sysreg!(ich_lr13_el2), + 14 => read_sysreg!(ich_lr14_el2), + 15 => read_sysreg!(ich_lr15_el2), + _ => { + panic!("invalid lr id {}", id); + } + } +} + +fn write_lr(id: usize, val: u64) { + let id = id as u64; + match id { + 0 => write_sysreg!(ich_lr0_el2, val), + 1 => write_sysreg!(ich_lr1_el2, val), + 2 => write_sysreg!(ich_lr2_el2, val), + 3 => write_sysreg!(ich_lr3_el2, val), + 4 => write_sysreg!(ich_lr4_el2, val), + 5 => write_sysreg!(ich_lr5_el2, val), + 6 => write_sysreg!(ich_lr6_el2, val), + 7 => write_sysreg!(ich_lr7_el2, val), + 8 => write_sysreg!(ich_lr8_el2, val), + 9 => write_sysreg!(ich_lr9_el2, val), + 10 => write_sysreg!(ich_lr10_el2, val), + 11 => write_sysreg!(ich_lr11_el2, val), + 12 => write_sysreg!(ich_lr12_el2, val), + 13 => write_sysreg!(ich_lr13_el2, val), + 14 => write_sysreg!(ich_lr14_el2, val), + 15 => write_sysreg!(ich_lr15_el2, val), + _ => { + panic!("invalid lr id {}", id); + } + } +} + +fn send_sgi_inner(aff3: u8, aff2: u8, aff1: u8, target: u8, vector: usize, to_all: bool) { + let value = ((vector & 0xF) << 24) | // vector + (1 << target) | // target bitmap + ((aff1 as usize) << 16) | // affinity level 1 + ((aff2 as usize) << 32) | // affinity level 2 + ((aff3 as usize) << 48) | // affinity level 3 + ((to_all as usize) << 40); // interrupt routing mode + + write_sysreg!(icc_sgi1r_el1, value as _); +} + +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_ipi_one(dest: usize, vector: usize) { + #[cfg(platform_family = "aarch64-rk3588j")] + { + // learnt from hVisor, that rockchip socs follow the 0.0.x.0 affinity scheme + // while other socs follow 0.0.0.x + // + // the best and standard way is reading + send_sgi_inner(0, 0, dest as _, 0, vector, false); + } + #[cfg(not(platform_family = "aarch64-rk3588j"))] + { + // the default affinity scheme is 0.0.0.x + send_sgi_inner(0, 0, 0, dest as _, vector, false); + } +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_ipi_all_others(vector: usize) { + send_sgi_inner(0, 0, 0, 0, vector, true); +} + +// dummy implementation +pub struct MyVgic {} + +/// Initializes GICD, GICC on the primary CPU. pub(crate) fn init_primary() { info!("Initialize GICv3..."); let mut gicd = arm_gic_driver::v3::Gic::new( @@ -113,8 +274,6 @@ pub(crate) fn init_primary() { GICD.lock().replace(gicd); GICR.lock().replace(interface); - - disable_irqs(); } /// Initializes GICR on secondary CPUs. diff --git a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs index 6ab67a04ec..4bb16d8450 100644 --- a/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/axhal/src/platform/aarch64_qemu_virt/mod.rs @@ -21,6 +21,7 @@ pub mod misc { } unsafe extern "C" { + fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); #[cfg(feature = "smp")] fn rust_main_secondary(cpu_id: usize); @@ -28,6 +29,7 @@ unsafe extern "C" { pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { crate::mem::clear_bss(); + crate::arch::set_exception_vector_base(exception_vector_base as usize); #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_primary(cpu_id); @@ -39,6 +41,7 @@ pub(crate) unsafe extern "C" fn rust_entry(cpu_id: usize, dtb: usize) { #[cfg(feature = "smp")] #[allow(dead_code)] // FIXME: temporariy allowd to bypass clippy warnings. pub(crate) unsafe extern "C" fn rust_entry_secondary(cpu_id: usize) { + crate::arch::set_exception_vector_base(exception_vector_base as usize); #[cfg(not(feature = "hv"))] crate::arch::write_page_table_root0(0.into()); // disable low address access crate::cpu::init_secondary(cpu_id); diff --git a/modules/axhal/src/platform/dummy/mod.rs b/modules/axhal/src/platform/dummy/mod.rs index 616ca47e7d..cedd20a359 100644 --- a/modules/axhal/src/platform/dummy/mod.rs +++ b/modules/axhal/src/platform/dummy/mod.rs @@ -88,6 +88,15 @@ pub mod irq { pub fn fetch_irq() -> usize { 0 } + + #[cfg(feature = "ipi")] + pub const IPI_IRQ_NUM: usize = 0; + + #[cfg(feature = "ipi")] + pub fn send_ipi_one(dest_cpu: usize, irq_num: usize) {} + + #[cfg(feature = "ipi")] + pub fn send_ipi_all_others(irq_num: usize) {} } /// Initializes the platform devices for the primary CPU. diff --git a/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs b/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs index 0d83f090a6..209a3effaa 100644 --- a/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/loongarch64_qemu_virt/irq.rs @@ -37,3 +37,15 @@ pub fn dispatch_irq(irq_num: usize) { } crate::irq::dispatch_irq_common(irq_num) } + +/// The IPI IRQ number. An placeholder now. +#[cfg(feature = "ipi")] +pub const IPI_IRQ_NUM: usize = 0; + +/// Send an IPI to the specified CPU. +#[cfg(feature = "ipi")] +pub fn send_ipi_one(dest_cpu: usize, irq_num: usize) {} + +/// Send a broadcast IPI to all CPUs. +#[cfg(feature = "ipi")] +pub fn send_ipi_all_others(irq_num: usize) {} diff --git a/modules/axhal/src/platform/mod.rs b/modules/axhal/src/platform/mod.rs index 622633315f..382f32d7cf 100644 --- a/modules/axhal/src/platform/mod.rs +++ b/modules/axhal/src/platform/mod.rs @@ -34,5 +34,7 @@ cfg_if::cfg_if! { } else { mod dummy; pub use self::dummy::*; + // mod riscv64_qemu_virt; + // pub use self::riscv64_qemu_virt::*; } } diff --git a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs index d46dfb3015..a9cb37671a 100644 --- a/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs +++ b/modules/axhal/src/platform/riscv64_qemu_virt/irq.rs @@ -1,8 +1,43 @@ -//! TODO: PLIC +//! Minimal PLIC support temporarily disabled. -use crate::irq::IrqHandler; +use kspin::SpinNoIrq; use lazyinit::LazyInit; + use riscv::register::sie; +// use riscv_plic::{HartContext, InterruptSource, Plic}; +use sbi_rt::{HartMask, send_ipi}; + +use memory_addr::{PhysAddr, pa}; + +use crate::cpu::this_cpu_id; +use crate::irq::IrqHandler; +use crate::mem::phys_to_virt; + +// struct Context { +// hart_id: usize, +// } + +// impl HartContext for Context { +// fn index(self) -> usize { +// self.hart_id * 2 + 1 +// } +// } + +// impl Context { +// const fn new(hart_id: usize) -> Self { +// Self { hart_id } +// } +// } + +// struct Interrupt { +// irq_num: usize, +// } + +// impl InterruptSource for Interrupt { +// fn id(self) -> core::num::NonZeroU32 { +// core::num::NonZeroU32::new(self.irq_num as u32).unwrap() +// } +// } /// `Interrupt` bit in `scause` pub(super) const INTC_IRQ_BASE: usize = 1 << (usize::BITS - 1); @@ -19,27 +54,46 @@ pub(super) const S_EXT: usize = INTC_IRQ_BASE + 9; static TIMER_HANDLER: LazyInit = LazyInit::new(); +static IPI_HANDLER: LazyInit = LazyInit::new(); + /// The maximum number of IRQs. pub const MAX_IRQ_COUNT: usize = 1024; /// The timer IRQ number (supervisor timer interrupt in `scause`). pub const TIMER_IRQ_NUM: usize = S_TIMER; +/// The IPI IRQ number (supervisor software interrupt in `scause`). +pub const IPI_IRQ_NUM: usize = S_SOFT; + macro_rules! with_cause { - ($cause: expr, @TIMER => $timer_op: expr, @EXT => $ext_op: expr $(,)?) => { + ($cause: expr, + @TIMER => $timer_op: expr, + @IPI => $ipi_op: expr, + @EXT => $ext_op: expr $(,)?) => { match $cause { S_TIMER => $timer_op, + S_SOFT => $ipi_op, S_EXT => $ext_op, _ => panic!("invalid trap cause: {:#x}", $cause), } }; } +// const PLIC_BASE: PhysAddr = pa!(axconfig::PLIC_PADDR); + +// static PLIC: SpinNoIrq = SpinNoIrq::new(Plic::new(phys_to_virt(PLIC_BASE).as_mut_ptr())); + /// Enables or disables the given IRQ. -pub fn set_enable(scause: usize, _enabled: bool) { - if scause == S_EXT { - // TODO: set enable in PLIC - } +pub fn set_enable(irq_num: usize, enabled: bool) { + // TODO: set enable in PLIC + // let source = Interrupt { irq_num }; + // let context = Context::new(this_cpu_id()); + + // if enabled { + // PLIC.lock().enable(source, context); + // } else { + // PLIC.lock().disable(source, context); + // } } /// Registers an IRQ handler for the given IRQ. @@ -55,10 +109,37 @@ pub fn register_handler(scause: usize, handler: IrqHandler) -> bool { } else { false }, + @IPI => if !IPI_HANDLER.is_inited() { + IPI_HANDLER.init_once(handler); + true + } else { + false + }, @EXT => crate::irq::register_handler_common(scause & !INTC_IRQ_BASE, handler), ) } +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_ipi_one(dest_cpu_id: usize, _irq_num: usize) { + let res = send_ipi(HartMask::from_mask_base(1, dest_cpu_id)); + if res.is_err() { + warn!("send_ipi_one failed: {:?}", res); + } +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_ipi_all_others(_irq_num: usize) { + for i in 0..axconfig::SMP { + if i != this_cpu_id() { + let res = send_ipi(HartMask::from_mask_base(1, i)); + if res.is_err() { + warn!("send_ipi_all_others failed: {:?}", res); + break; + } + } + } +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks @@ -71,11 +152,25 @@ pub fn dispatch_irq(scause: usize) { trace!("IRQ: timer"); TIMER_HANDLER(); }, - @EXT => crate::irq::dispatch_irq_common(0), // TODO: get IRQ number from PLIC + @IPI => { + trace!("IRQ: IPI"); + IPI_HANDLER(); + }, + @EXT => { + // if let Some(irq_num) = PLIC.lock().claim(Context::new(this_cpu_id())) { + // let irq_num = irq_num.get() as usize; + // trace!("IRQ: external {}", irq_num); + // crate::irq::dispatch_irq_common(irq_num); + // PLIC.lock().complete(Context::new(this_cpu_id()), Interrupt { irq_num }); + // } + crate::irq::dispatch_irq_common(0); + }, ); } pub(super) fn init_percpu() { + // PLIC.lock().init_by_context(Context::new(this_cpu_id())); + // enable soft interrupts, timer interrupts, and external interrupts unsafe { sie::set_ssoft(); diff --git a/modules/axhal/src/platform/x86_pc/apic.rs b/modules/axhal/src/platform/x86_pc/apic.rs index ca2a86c36b..7f0184836c 100644 --- a/modules/axhal/src/platform/x86_pc/apic.rs +++ b/modules/axhal/src/platform/x86_pc/apic.rs @@ -16,6 +16,7 @@ pub(super) mod vectors { pub const APIC_TIMER_VECTOR: u8 = 0xf0; pub const APIC_SPURIOUS_VECTOR: u8 = 0xf1; pub const APIC_ERROR_VECTOR: u8 = 0xf2; + pub const APIC_IPI_VECTOR: u8 = 0xf3; } /// The maximum number of IRQs. @@ -24,6 +25,9 @@ pub const MAX_IRQ_COUNT: usize = 256; /// The timer IRQ number. pub const TIMER_IRQ_NUM: usize = APIC_TIMER_VECTOR as usize; +/// The IPI IRQ number. +pub const IPI_IRQ_NUM: usize = APIC_IPI_VECTOR as usize; + const IO_APIC_BASE: PhysAddr = pa!(0xFEC0_0000); static LOCAL_APIC: SyncUnsafeCell> = @@ -55,6 +59,21 @@ pub fn register_handler(vector: usize, handler: crate::irq::IrqHandler) -> bool crate::irq::register_handler_common(vector, handler) } +/// Sends Software Generated Interrupt (SGI)(s) (usually IPI) to the given dest CPU. +pub fn send_ipi_one(dest_cpu_id: usize, irq_num: usize) { + unsafe { + local_apic().send_ipi(irq_num as _, dest_cpu_id as _); + }; +} + +/// Sends a broadcast IPI to all CPUs. +pub fn send_ipi_all_others(irq_num: usize) { + use x2apic::lapic::IpiAllShorthand; + unsafe { + local_apic().send_ipi_all(irq_num as _, IpiAllShorthand::AllExcludingSelf); + }; +} + /// Dispatches the IRQ. /// /// This function is called by the common interrupt handler. It looks diff --git a/modules/axipi/Cargo.toml b/modules/axipi/Cargo.toml new file mode 100644 index 0000000000..e1ee1318f4 --- /dev/null +++ b/modules/axipi/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "axipi" +edition = "2021" +version.workspace = true +authors = ["Keyang Hu "] +description = "ArceOS IPI management module" +license.workspace = true +homepage.workspace = true +# repository = "https://github.com/arceos-org/arceos/tree/main/modules/axipi" +# documentation = "https://arceos-org.github.io/arceos/axipi/index.html" + +[features] +default = [] + +[dependencies] +log = "=0.4.21" +lazyinit = { version = "0.2" } + +percpu = { version = "0.2" } +kspin = { version = "0.1" } + +axhal = { workspace = true, features = ["ipi"] } +axconfig = { workspace = true } \ No newline at end of file diff --git a/modules/axipi/src/event.rs b/modules/axipi/src/event.rs new file mode 100644 index 0000000000..a517999f73 --- /dev/null +++ b/modules/axipi/src/event.rs @@ -0,0 +1,52 @@ +use alloc::{boxed::Box, sync::Arc}; + +/// A callback function that will be called when an [`IPIEvent`] is received and handled. +pub struct Callback(Box); + +impl Callback { + /// Create a new [`Callback`] with the given function. + pub fn new(callback: F) -> Self { + Self(Box::new(callback)) + } + + /// Call the callback function. + pub fn call(self) { + (self.0)() + } +} + +impl From for Callback { + fn from(callback: T) -> Self { + Self::new(callback) + } +} + +/// A [`Callback`] that can be called multiple times. It's used for multicast IPI events. +#[derive(Clone)] +pub struct MulticastCallback(Arc); + +impl MulticastCallback { + /// Create a new [`MulticastCallback`] with the given function. + pub fn new(callback: F) -> Self { + Self(Arc::new(callback)) + } + + /// Convert the [`MulticastCallback`] into a [`Callback`]. + pub fn into_unicast(self) -> Callback { + Callback(Box::new(move || (self.0)())) + } +} + +impl From for MulticastCallback { + fn from(callback: T) -> Self { + Self::new(callback) + } +} + +/// An IPI event that is sent from a source CPU to the target CPU. +pub struct IPIEvent { + /// The source CPU ID that sent the IPI event. + pub src_cpu_id: usize, + /// The callback function that will be called when the IPI event is handled. + pub callback: Callback, +} diff --git a/modules/axipi/src/lib.rs b/modules/axipi/src/lib.rs new file mode 100644 index 0000000000..fddf9b04f7 --- /dev/null +++ b/modules/axipi/src/lib.rs @@ -0,0 +1,65 @@ +//! [ArceOS](https://github.com/arceos-org/arceos) Inter-Processor Interrupt (IPI) primitives. + +#![cfg_attr(not(test), no_std)] + +#[macro_use] +extern crate log; +extern crate alloc; + +use lazyinit::LazyInit; + +use kspin::SpinNoIrq; + +use axhal::cpu::this_cpu_id; +use axhal::irq::IPI_IRQ_NUM; + +mod event; +mod queue; + +pub use event::*; +use queue::IPIEventQueue; + +#[percpu::def_percpu] +static IPI_EVENT_QUEUE: LazyInit> = LazyInit::new(); + +/// Initialize the per-CPU IPI event queue. +pub fn init() { + IPI_EVENT_QUEUE.with_current(|ipi_queue| { + ipi_queue.init_once(SpinNoIrq::new(IPIEventQueue::default())); + }); +} + +/// Sends an IPI event to the processor(s) specified by `dest_cpu`. +pub fn send_ipi_event_to_one>(dest_cpu: usize, callback: T) { + warn!("Send IPI event to CPU {}", dest_cpu); + + unsafe { IPI_EVENT_QUEUE.remote_ref_raw(dest_cpu) } + .lock() + .push(this_cpu_id(), callback.into()); + axhal::irq::send_ipi_one(dest_cpu, IPI_IRQ_NUM); +} + +/// Sends an IPI event to all processors except the current one. +pub fn send_ipi_event_to_all>(callback: T) { + let current_cpu_id = this_cpu_id(); + let callback = callback.into(); + for cpu_id in 0..axconfig::SMP { + if cpu_id != current_cpu_id { + unsafe { IPI_EVENT_QUEUE.remote_ref_raw(cpu_id) } + .lock() + .push(current_cpu_id, callback.clone().into_unicast()); + } + } + axhal::irq::send_ipi_all_others(IPI_IRQ_NUM); +} + +/// The handler for IPI events. It retrieves the events from the queue and calls the corresponding callbacks. +pub fn ipi_handler() { + while let Some((src_cpu_id, callback)) = unsafe { IPI_EVENT_QUEUE.current_ref_mut_raw() } + .lock() + .pop_one() + { + warn!("Received IPI event from CPU {}", src_cpu_id); + callback.call(); + } +} diff --git a/modules/axipi/src/queue.rs b/modules/axipi/src/queue.rs new file mode 100644 index 0000000000..ddd300d618 --- /dev/null +++ b/modules/axipi/src/queue.rs @@ -0,0 +1,53 @@ +use alloc::collections::VecDeque; + +use crate::{Callback, IPIEvent}; + +/// A queue of IPI events. +/// +/// It internally uses a `VecDeque` to store the events, make it +/// possible to pop these events using FIFO order. +pub struct IPIEventQueue { + events: VecDeque, +} + +impl IPIEventQueue { + /// Create a new empty timer list. + pub fn new() -> Self { + Self { + events: VecDeque::new(), + } + } + + /// Whether there is no event. + #[allow(dead_code)] + #[inline] + pub fn is_empty(&self) -> bool { + self.events.is_empty() + } + + /// Push a new event into the queue. + pub fn push(&mut self, src_cpu_id: usize, callback: Callback) { + self.events.push_back(IPIEvent { + src_cpu_id, + callback, + }); + } + + /// Try to pop the latest event that exists in the queue. + /// + /// Return `None` if no event is available. + #[must_use] + pub fn pop_one(&mut self) -> Option<(usize, Callback)> { + if let Some(e) = self.events.pop_front() { + Some((e.src_cpu_id, e.callback)) + } else { + None + } + } +} + +impl Default for IPIEventQueue { + fn default() -> Self { + Self::new() + } +} diff --git a/modules/axmm/Cargo.toml b/modules/axmm/Cargo.toml index faac117267..b53e27ecb4 100644 --- a/modules/axmm/Cargo.toml +++ b/modules/axmm/Cargo.toml @@ -17,6 +17,6 @@ axconfig = { workspace = true } log = "=0.4.21" axerrno = "0.1" lazyinit = "0.2" -memory_addr = "0.3" +memory_addr = "0.4" kspin = "0.1" -memory_set = "0.3" +memory_set = "0.4" diff --git a/modules/axmm/src/aspace.rs b/modules/axmm/src/aspace.rs index 39e377e065..0c52a2ffc7 100644 --- a/modules/axmm/src/aspace.rs +++ b/modules/axmm/src/aspace.rs @@ -85,7 +85,7 @@ impl AddrSpace { size: usize, limit: VirtAddrRange, ) -> Option { - self.areas.find_free_area(hint, size, limit) + self.areas.find_free_area(hint, size, limit, PAGE_SIZE_4K) } /// Add a new linear mapping. diff --git a/modules/axruntime/Cargo.toml b/modules/axruntime/Cargo.toml index 8209722d1d..f2e7bb9872 100644 --- a/modules/axruntime/Cargo.toml +++ b/modules/axruntime/Cargo.toml @@ -17,6 +17,7 @@ irq = ["axhal/irq", "axtask?/irq", "percpu", "kernel_guard"] tls = ["axhal/tls", "axtask?/tls"] alloc = ["axalloc"] paging = ["axhal/paging", "axmm"] +ipi = ["dep:axipi"] multitask = ["axtask/multitask"] fs = ["axdriver", "axfs"] @@ -35,6 +36,7 @@ axfs = { workspace = true, optional = true } axnet = { workspace = true, optional = true } axdisplay = { workspace = true, optional = true } axtask = { workspace = true, optional = true } +axipi = { workspace = true, optional = true } crate_interface = "0.1" percpu = { version = "0.2", optional = true } diff --git a/modules/axruntime/src/lib.rs b/modules/axruntime/src/lib.rs index 1261f95116..bffb5c07eb 100644 --- a/modules/axruntime/src/lib.rs +++ b/modules/axruntime/src/lib.rs @@ -260,6 +260,12 @@ fn init_interrupt() { axtask::on_timer_tick(); }); + #[cfg(feature = "ipi")] + { + axipi::init(); + axhal::irq::register_handler(axhal::irq::IPI_IRQ_NUM, axipi::ipi_handler); + } + // Enable IRQs before starting app axhal::arch::enable_irqs(); } diff --git a/modules/axruntime/src/mp.rs b/modules/axruntime/src/mp.rs index 5e1baab4b9..b97ba0a665 100644 --- a/modules/axruntime/src/mp.rs +++ b/modules/axruntime/src/mp.rs @@ -44,6 +44,9 @@ pub extern "C" fn rust_main_secondary(cpu_id: usize) -> ! { #[cfg(feature = "multitask")] axtask::init_scheduler_secondary(); + #[cfg(feature = "ipi")] + axipi::init(); + info!("Secondary CPU {:x} init OK.", cpu_id); super::INITED_CPUS.fetch_add(1, Ordering::Relaxed); diff --git a/modules/axtask/Cargo.toml b/modules/axtask/Cargo.toml index 830d678f66..3a06503b19 100644 --- a/modules/axtask/Cargo.toml +++ b/modules/axtask/Cargo.toml @@ -43,7 +43,7 @@ axconfig = { workspace = true, optional = true } percpu = { version = "0.2", optional = true } kspin = { version = "0.1", optional = true } lazyinit = { version = "0.2", optional = true } -memory_addr = { version = "0.3", optional = true } +memory_addr = { version = "0.4", optional = true } timer_list = { version = "0.1", optional = true } kernel_guard = { version = "0.1", optional = true } crate_interface = { version = "0.1", optional = true } diff --git a/modules/axtask/src/api.rs b/modules/axtask/src/api.rs index 7cf96a043c..d0c8f21bd1 100644 --- a/modules/axtask/src/api.rs +++ b/modules/axtask/src/api.rs @@ -210,8 +210,8 @@ pub fn exit(exit_code: i32) -> ! { pub fn run_idle() -> ! { loop { yield_now(); - debug!("idle task: waiting for IRQs..."); - #[cfg(feature = "irq")] - axhal::arch::wait_for_irqs(); + // debug!("idle task: waiting for IRQs..."); + // #[cfg(feature = "irq")] + // axhal::arch::wait_for_irqs(); } } diff --git a/modules/axtask/src/run_queue.rs b/modules/axtask/src/run_queue.rs index 2e68bc1559..4ccacc73a4 100644 --- a/modules/axtask/src/run_queue.rs +++ b/modules/axtask/src/run_queue.rs @@ -494,6 +494,8 @@ impl AxRunQueue { } } // TODO: priority + #[cfg(feature = "smp")] + task.set_cpu_id(self.cpu_id as _); self.scheduler.lock().put_prev_task(task, preempt); true } else { diff --git a/modules/axtask/src/task.rs b/modules/axtask/src/task.rs index dfb2a844c0..c45304b372 100644 --- a/modules/axtask/src/task.rs +++ b/modules/axtask/src/task.rs @@ -6,6 +6,9 @@ use core::{alloc::Layout, cell::UnsafeCell, fmt, ptr::NonNull}; #[cfg(feature = "preempt")] use core::sync::atomic::AtomicUsize; +#[cfg(feature = "smp")] +use core::sync::atomic::AtomicU32; + use kspin::SpinNoIrq; use memory_addr::{VirtAddr, align_up_4k}; @@ -51,6 +54,9 @@ pub struct TaskInner { /// Mark whether the task is in the wait queue. in_wait_queue: AtomicBool, + /// Used to indicate the CPU ID where the task is running or will run. + #[cfg(feature = "smp")] + cpu_id: AtomicU32, /// Used to indicate whether the task is running on a CPU. #[cfg(feature = "smp")] on_cpu: AtomicBool, @@ -227,6 +233,8 @@ impl TaskInner { #[cfg(feature = "irq")] timer_ticket_id: AtomicU64::new(0), #[cfg(feature = "smp")] + cpu_id: AtomicU32::new(0), + #[cfg(feature = "smp")] on_cpu: AtomicBool::new(false), #[cfg(feature = "preempt")] need_resched: AtomicBool::new(false), @@ -398,6 +406,20 @@ impl TaskInner { self.ctx.get() } + /// Set the CPU ID where the task is running or will run. + #[cfg(feature = "smp")] + pub fn set_cpu_id(&self, cpu_id: u32) { + self.cpu_id.store(cpu_id, Ordering::Release); + } + + /// Returns the CPU ID where the task is running or will run. + /// + /// Note: the task may not be running on the CPU, it just exists in the run queue. + #[cfg(feature = "smp")] + pub fn cpu_id(&self) -> u32 { + self.cpu_id.load(Ordering::Acquire) + } + /// Returns whether the task is running on a CPU. /// /// It is used to protect the task from being moved to a different run queue @@ -405,8 +427,7 @@ impl TaskInner { /// The `on_cpu field is set to `true` when the task is preparing to run on a CPU, /// and it is set to `false` when the task has finished its scheduling process in `clear_prev_task_on_cpu()`. #[cfg(feature = "smp")] - #[inline] - pub(crate) fn on_cpu(&self) -> bool { + pub fn on_cpu(&self) -> bool { self.on_cpu.load(Ordering::Acquire) } diff --git a/ulib/axlibc/build.rs b/ulib/axlibc/build.rs index 6c33c929c7..d8d7ce0dbe 100644 --- a/ulib/axlibc/build.rs +++ b/ulib/axlibc/build.rs @@ -2,6 +2,7 @@ fn main() { fn gen_c_to_rust_bindings(in_file: &str, out_file: &str) { println!("cargo:rerun-if-changed={in_file}"); + let target = std::env::var("TARGET").unwrap(); let allow_types = ["tm", "jmp_buf"]; let mut builder = bindgen::Builder::default() .header(in_file) @@ -9,6 +10,10 @@ fn main() { .derive_default(true) .size_t_is_usize(false) .use_core(); + if let Some(llvm_target) = target.strip_suffix("-softfloat") { + // remove "-softfloat" suffix for some targets + builder = builder.clang_arg(format!("--target={llvm_target}")); + } for ty in allow_types { builder = builder.allowlist_type(ty); } diff --git a/ulib/axstd/Cargo.toml b/ulib/axstd/Cargo.toml index 50baad283e..064bc0784f 100644 --- a/ulib/axstd/Cargo.toml +++ b/ulib/axstd/Cargo.toml @@ -27,6 +27,7 @@ fp_simd = ["axfeat/fp_simd"] # Interrupts irq = ["arceos_api/irq", "axfeat/irq"] +ipi = ["arceos_api/ipi", "axfeat/ipi"] gicv3 = ["axfeat/gicv3","alloc"] # Memory @@ -69,7 +70,7 @@ driver-fxmac = ["axfeat/driver-fxmac"] driver-bcm2835-sdhci = ["axfeat/driver-bcm2835-sdhci"] # Hypervisor support -hv = ["axfeat/hv"] +hv = ["axfeat/hv", "axfeat/ipi", "arceos_api/ipi"] # Logging log-level-off = ["axfeat/log-level-off"]