diff --git a/bins/msvisor/src/main.rs b/bins/msvisor/src/main.rs index a90e3494..f7b3181e 100644 --- a/bins/msvisor/src/main.rs +++ b/bins/msvisor/src/main.rs @@ -83,11 +83,10 @@ fn msvisor_start(isol: &Arc) { let isol = get_isol(isol.id).expect("isol don't exist?"); if let Err(e) = isol.run() { + log::error!("isol{} run failed. err={e:?}", isol.id); // if in debug mod, error will lead to exit. #[cfg(debug_assertions)] panic!("isol{} run failed. err={e:?}", isol.id); - #[cfg(not(debug_assertions))] - log::error!("isol{} run failed. err={e:?}", isol.id) } } diff --git a/common_service/mm/src/faas_buffer/mod.rs b/common_service/mm/src/faas_buffer/mod.rs index 64b35848..4b32a3d9 100644 --- a/common_service/mm/src/faas_buffer/mod.rs +++ b/common_service/mm/src/faas_buffer/mod.rs @@ -23,7 +23,11 @@ lazy_static! { #[no_mangle] pub fn buffer_alloc(slot: &str, l: Layout, fingerprint: u64) -> MMResult { - let addr = BUFFER_ALLOCATOR.lock().allocate_first_fit(l).unwrap(); + let addr = BUFFER_ALLOCATOR + .lock() + .allocate_first_fit(l) + .expect("alloc mem failed"); + let addr = addr.as_ptr() as usize; BUFFER_REGISTER diff --git a/common_service/ruxfdtab/Cargo.lock b/common_service/ruxfdtab/Cargo.lock index aa7f94b2..d977fbba 100644 --- a/common_service/ruxfdtab/Cargo.lock +++ b/common_service/ruxfdtab/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ [[package]] name = "arm_gic" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "tock-registers", ] @@ -22,7 +22,7 @@ dependencies = [ [[package]] name = "arm_pl011" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "tock-registers", ] @@ -36,7 +36,7 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axerrno" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "log", ] @@ -44,7 +44,7 @@ dependencies = [ [[package]] name = "axfs_devfs" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axfs_vfs", "log", @@ -54,7 +54,7 @@ dependencies = [ [[package]] name = "axfs_ramfs" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axfs_vfs", "log", @@ -64,7 +64,7 @@ dependencies = [ [[package]] name = "axfs_vfs" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", "bitflags 2.6.0", @@ -74,7 +74,7 @@ dependencies = [ [[package]] name = "axio" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", ] @@ -82,7 +82,7 @@ dependencies = [ [[package]] name = "axlog" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "cfg-if", "crate_interface", @@ -93,7 +93,7 @@ dependencies = [ [[package]] name = "axsync" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "ruxtask", "spinlock", @@ -138,7 +138,7 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "capability" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", "bitflags 2.6.0", @@ -189,7 +189,7 @@ dependencies = [ [[package]] name = "driver_block" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "driver_common", "fscommon", @@ -199,12 +199,12 @@ dependencies = [ [[package]] name = "driver_common" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" [[package]] name = "dtb" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "fdt-rs", "lazy_init", @@ -213,7 +213,7 @@ dependencies = [ [[package]] name = "dw_apb_uart" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "tock-registers", ] @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "flatten_objects" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "bitmaps", ] @@ -349,7 +349,7 @@ dependencies = [ [[package]] name = "lazy_init" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" [[package]] name = "lazy_static" @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "page_table_entry" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "aarch64-cpu", "bitflags 2.6.0", @@ -495,7 +495,7 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percpu" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "cfg-if", "percpu_macros", @@ -506,7 +506,7 @@ dependencies = [ [[package]] name = "percpu_macros" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "proc-macro2", "quote", @@ -534,7 +534,7 @@ dependencies = [ [[package]] name = "ratio" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" [[package]] name = "raw-cpuid" @@ -592,7 +592,7 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ruxconfig" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "serde", "toml_edit", @@ -601,7 +601,7 @@ dependencies = [ [[package]] name = "ruxdriver" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "cfg-if", "driver_block", @@ -627,7 +627,7 @@ dependencies = [ [[package]] name = "ruxfdtable" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", "axio", @@ -640,7 +640,7 @@ dependencies = [ [[package]] name = "ruxfs" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", "axfs_devfs", @@ -661,7 +661,7 @@ dependencies = [ [[package]] name = "ruxhal" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "aarch64-cpu", "arm_gic", @@ -696,7 +696,7 @@ dependencies = [ [[package]] name = "ruxtask" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "axerrno", "cfg-if", @@ -797,7 +797,7 @@ dependencies = [ [[package]] name = "spinlock" version = "0.1.0" -source = "git+https://github.com/anti-entropy123/ruxos/#69217283ca94c10d6b084be7a71a35b63b29f2b5" +source = "git+https://github.com/anti-entropy123/ruxos/#ca09e9478c5f78bdc408bf28e12c1f437ea1d83e" dependencies = [ "cfg-if", "kernel_guard", diff --git a/common_service/ruxfdtab/Cargo.toml b/common_service/ruxfdtab/Cargo.toml index c120626b..9a650a54 100644 --- a/common_service/ruxfdtab/Cargo.toml +++ b/common_service/ruxfdtab/Cargo.toml @@ -6,11 +6,6 @@ edition = "2021" [lib] crate-type = ["dylib"] -[features] -mpk = ["ms_std/mpk"] -log = [] - -default = [] [dependencies] ms_std = { path = "../../ms_std" } @@ -25,18 +20,21 @@ axio = { git = "https://github.com/anti-entropy123/ruxos/" } ruxfdtable = { git = "https://github.com/anti-entropy123/ruxos/" } # ruxfdtable = { path = "/home/yjn/rust_project/ruxos/modules/ruxfdtable" } -ruxfs = { git = "https://github.com/anti-entropy123/ruxos/", features = [ - "use-ramdisk", -] } -# ruxfs = { path = "/home/yjn/rust_project/ruxos/modules/ruxfs", features = [ -# "use-ramdisk", -# ] } +ruxfs = { git = "https://github.com/anti-entropy123/ruxos/", features = [] } +# ruxfs = { path = "/home/yjn/rust_project/ruxos/modules/ruxfs", features = [] } ruxdriver = { git = "https://github.com/anti-entropy123/ruxos/", features = [ "ramdisk", -] } +], optional = true } # ruxdriver = { path = "/home/yjn/rust_project/ruxos/modules/ruxdriver", features = [ # "ramdisk", -# ] } +# ], optional = true } spin = "0.9.8" -lazy_static = { version = "1.5.0", features = ["spin_no_std"] } \ No newline at end of file +lazy_static = { version = "1.5.0", features = ["spin_no_std"] } + +[features] +mpk = ["ms_std/mpk"] +use-ramdisk = ["ruxfs/use-ramdisk", "ruxfs/full", "ruxdriver"] +log = [] + +default = ["use-ramdisk"] diff --git a/common_service/ruxfdtab/src/as_apis.rs b/common_service/ruxfdtab/src/as_apis.rs index 7e9cdac8..71dcf198 100644 --- a/common_service/ruxfdtab/src/as_apis.rs +++ b/common_service/ruxfdtab/src/as_apis.rs @@ -3,7 +3,7 @@ extern crate alloc; use std::path::PathBuf; use alloc::vec; -use ms_std::libos::libos; +use ms_std::{libos::libos, sync::UPSafeCell}; use spin::Mutex; use crate::{ @@ -14,9 +14,14 @@ use ms_hostcall::{ fdtab::{FdtabError, FdtabResult}, types::{DirEntry, Fd, OpenFlags, OpenMode, Size, Stat, TimeSpec}, }; + +#[cfg(feature = "use-ramdisk")] use ruxdriver::init_drivers; +#[cfg(feature = "use-ramdisk")] +use ruxfs::init_blkfs; + use ruxfdtable::{FileLike, RuxStat}; -use ruxfs::{fops::OpenOptions, init_blkfs, init_filesystems, prepare_commonfs}; +use ruxfs::{fops::OpenOptions, init_filesystems, init_tempfs, prepare_commonfs}; fn convert(ruxstat: RuxStat) -> Stat { return Stat { @@ -62,28 +67,34 @@ fn get_fs_image_path() -> PathBuf { .join(image_path) } -fn init() { +fn init() -> bool { + #[cfg(feature = "use-ramdisk")] let all_devices = init_drivers(get_fs_image_path().to_str().unwrap()); #[cfg(feature = "log")] println!("block devices nums: {}", all_devices.block.len()); + + #[cfg(feature = "use-ramdisk")] let mount_point = init_blkfs(all_devices.block); + #[cfg(not(feature = "use-ramdisk"))] + let mount_point = init_tempfs(); let mut mount_points = vec![mount_point]; prepare_commonfs(&mut mount_points); init_filesystems(mount_points); #[cfg(feature = "log")] println!("ruxfs init ok"); + + true } lazy_static::lazy_static! { - static ref MUST_EXIC: Mutex = { - init(); - Mutex::new(true) + static ref MUST_EXIC: bool = { + init() }; } #[no_mangle] pub fn open(path: &str, flags: OpenFlags, mode: OpenMode) -> FdtabResult { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; #[cfg(feature = "log")] { println!("ruxfdtab: open path={:?}", path); @@ -115,13 +126,15 @@ pub fn open(path: &str, flags: OpenFlags, mode: OpenMode) -> FdtabResult { #[no_mangle] pub fn write(fd: Fd, buf: &[u8]) -> FdtabResult { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; // libos!(stdout(format!("fd: {}, buf: {:?}", fd, buf).as_bytes())); let f = get_file_like(fd)?; let result = f .write(buf) .map_err(|e| FdtabError::RuxfsError(e.to_string())); + + #[cfg(feature = "use-ramdisk")] f.flush().unwrap(); result @@ -129,7 +142,7 @@ pub fn write(fd: Fd, buf: &[u8]) -> FdtabResult { #[no_mangle] pub fn read(fd: Fd, buf: &mut [u8]) -> FdtabResult { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; get_file_like(fd)? .read(buf) .map_err(|e| FdtabError::RuxfsError(e.to_string())) @@ -137,13 +150,13 @@ pub fn read(fd: Fd, buf: &mut [u8]) -> FdtabResult { #[no_mangle] pub fn close(fd: Fd) -> FdtabResult<()> { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; close_file_like(fd) } #[no_mangle] pub fn lseek(fd: Fd, pos: u32) -> FdtabResult<()> { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; fs::File::from_fd(fd)? .inner .lock() @@ -155,7 +168,7 @@ pub fn lseek(fd: Fd, pos: u32) -> FdtabResult<()> { #[no_mangle] pub fn stat(fd: Fd) -> FdtabResult { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; let stat = fs::File::from_fd(fd)? .stat() .map_err(|e| FdtabError::RuxfsError(e.to_string()))?; @@ -168,7 +181,7 @@ pub fn stat(fd: Fd) -> FdtabResult { #[no_mangle] pub fn readdir(path: &str) -> FdtabResult> { - let _exec = MUST_EXIC.lock(); + let _exec = *MUST_EXIC; #[cfg(feature = "log")] println!("[DEBUG] ruxfs read_dir: {:?}", path); diff --git a/isol_config/map_reduce.json b/isol_config/map_reduce.json index 95a22279..cf18f0ec 100644 --- a/isol_config/map_reduce.json +++ b/isol_config/map_reduce.json @@ -2,7 +2,7 @@ "services": [ [ "fdtab", - "libruxfdtab.so" + "libfdtab.so" ], [ "stdio", diff --git a/isol_config/map_reduce_large_c1_ramfs.json b/isol_config/map_reduce_large_c1_ramfs.json new file mode 100644 index 00000000..98dfb600 --- /dev/null +++ b/isol_config/map_reduce_large_c1_ramfs.json @@ -0,0 +1,63 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "mapper", + "libmapper.so" + ], + [ + "reducer", + "libreducer.so" + ], + [ + "simple_file", + "libsimple_file.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + "mapper" + ], + "args": { + "reducer_num": "1" + } + }, + { + "list": [ + "reducer" + ], + "args": { + "mapper_num": "1" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/map_reduce_large_c3_ramfs.json b/isol_config/map_reduce_large_c3_ramfs.json new file mode 100644 index 00000000..e8202e19 --- /dev/null +++ b/isol_config/map_reduce_large_c3_ramfs.json @@ -0,0 +1,67 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "mapper", + "libmapper.so" + ], + [ + "reducer", + "libreducer.so" + ], + [ + "simple_file", + "libsimple_file.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + "mapper", + "mapper", + "mapper" + ], + "args": { + "reducer_num": "3" + } + }, + { + "list": [ + "reducer", + "reducer", + "reducer" + ], + "args": { + "mapper_num": "3" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/map_reduce_large_c5_ramfs.json b/isol_config/map_reduce_large_c5_ramfs.json new file mode 100644 index 00000000..e93cab69 --- /dev/null +++ b/isol_config/map_reduce_large_c5_ramfs.json @@ -0,0 +1,71 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "time", + "libtime.so" + ], + [ + "fatfs", + "libfatfs.so" + ] + ], + "apps": [ + [ + "mapper", + "libmapper.so" + ], + [ + "reducer", + "libreducer.so" + ], + [ + "simple_file", + "libsimple_file.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + "mapper", + "mapper", + "mapper", + "mapper", + "mapper" + ], + "args": { + "reducer_num": "5" + } + }, + { + "list": [ + "reducer", + "reducer", + "reducer", + "reducer", + "reducer" + ], + "args": { + "mapper_num": "5" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/parallel_sort_c1_ramfs.json b/isol_config/parallel_sort_c1_ramfs.json new file mode 100644 index 00000000..a41c7ef1 --- /dev/null +++ b/isol_config/parallel_sort_c1_ramfs.json @@ -0,0 +1,94 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "fatfs", + "libfatfs.so" + ], + [ + "time", + "libtime.so" + ] + ], + "apps": [ + [ + "simple_file", + "libsimple_file.so" + ], + [ + "file_reader", + "libfile_reader.so" + ], + [ + "sorter", + "libsorter.so" + ], + [ + "splitter", + "libsplitter.so" + ], + [ + "merger", + "libmerger.so" + ], + [ + "checker", + "libchecker.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + { + "name": "file_reader", + "args": { + "slot_name": "input-part-0", + "input_file": "sort_data_0.txt" + } + } + ], + "args": {} + }, + { + "list": [ + "sorter" + ], + "args": { + "sorter_num": "1", + "merger_num": "1" + } + }, + { + "list": [ + "splitter" + ], + "args": {} + }, + { + "list": [ + "merger" + ], + "args": { + "sorter_num": "1" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/parallel_sort_c3_ramfs.json b/isol_config/parallel_sort_c3_ramfs.json new file mode 100644 index 00000000..2424a5cb --- /dev/null +++ b/isol_config/parallel_sort_c3_ramfs.json @@ -0,0 +1,114 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "fatfs", + "libfatfs.so" + ], + [ + "time", + "libtime.so" + ] + ], + "apps": [ + [ + "simple_file", + "libsimple_file.so" + ], + [ + "file_reader", + "libfile_reader.so" + ], + [ + "sorter", + "libsorter.so" + ], + [ + "splitter", + "libsplitter.so" + ], + [ + "merger", + "libmerger.so" + ], + [ + "checker", + "libchecker.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + { + "name": "file_reader", + "args": { + "slot_name": "input-part-0", + "input_file": "sort_data_0.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-1", + "input_file": "sort_data_1.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-2", + "input_file": "sort_data_2.txt" + } + } + ], + "args": {} + }, + { + "list": [ + "sorter", + "sorter", + "sorter" + ], + "args": { + "sorter_num": "3", + "merger_num": "3" + } + }, + { + "list": [ + "splitter", + "splitter", + "splitter" + ], + "args": {} + }, + { + "list": [ + "merger", + "merger", + "merger" + ], + "args": { + "sorter_num": "3" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/parallel_sort_c5_ramfs.json b/isol_config/parallel_sort_c5_ramfs.json new file mode 100644 index 00000000..a10f6171 --- /dev/null +++ b/isol_config/parallel_sort_c5_ramfs.json @@ -0,0 +1,134 @@ +{ + "services": [ + [ + "fdtab", + "libruxfdtab.so" + ], + [ + "stdio", + "libstdio.so" + ], + [ + "mm", + "libmm.so" + ], + [ + "fatfs", + "libfatfs.so" + ], + [ + "time", + "libtime.so" + ] + ], + "apps": [ + [ + "simple_file", + "libsimple_file.so" + ], + [ + "file_reader", + "libfile_reader.so" + ], + [ + "sorter", + "libsorter.so" + ], + [ + "splitter", + "libsplitter.so" + ], + [ + "merger", + "libmerger.so" + ], + [ + "checker", + "libchecker.so" + ] + ], + "fs_image": "fs_images/fatfs.img", + "groups": [ + { + "list": [ + "simple_file" + ], + "args": {} + }, + { + "list": [ + { + "name": "file_reader", + "args": { + "slot_name": "input-part-0", + "input_file": "sort_data_0.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-1", + "input_file": "sort_data_1.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-2", + "input_file": "sort_data_2.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-3", + "input_file": "sort_data_3.txt" + } + }, + { + "name": "file_reader", + "args": { + "slot_name": "input-part-4", + "input_file": "sort_data_4.txt" + } + } + ], + "args": {} + }, + { + "list": [ + "sorter", + "sorter", + "sorter", + "sorter", + "sorter" + ], + "args": { + "sorter_num": "5", + "merger_num": "5" + } + }, + { + "list": [ + "splitter", + "splitter", + "splitter", + "splitter", + "splitter" + ], + "args": {} + }, + { + "list": [ + "merger", + "merger", + "merger", + "merger", + "merger" + ], + "args": { + "sorter_num": "5" + } + } + ] +} \ No newline at end of file diff --git a/isol_config/ruxos.json b/isol_config/ruxos.json deleted file mode 100644 index 264f982d..00000000 --- a/isol_config/ruxos.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "services": [ - [ - "fdtab", - "libruxfdtab.so" - ], - [ - "stdio", - "libstdio.so" - ], - [ - "mm", - "libmm.so" - ] - ], - "apps": [ - [ - "hello1", - "libsimple_file.so" - ] - ] -} diff --git a/isol_config/simple_file.json b/isol_config/simple_file.json index b01bf7fa..3d84dcfe 100644 --- a/isol_config/simple_file.json +++ b/isol_config/simple_file.json @@ -15,6 +15,10 @@ [ "fatfs", "libfatfs.so" + ], + [ + "time", + "libtime.so" ] ], "apps": [ diff --git a/justfile b/justfile index e4e6cad6..e4752e06 100644 --- a/justfile +++ b/justfile @@ -15,7 +15,7 @@ mpk_flag := if enable_mpk == "1" { } } else { "" } -mpk_feature_flag := if mpk_flag == "" { "" } else { "--features {{mpk_flag}}" } +mpk_feature_flag := if mpk_flag == "" { "" } else { "--features " + mpk_flag } buffer_feature_flag := if enable_file_buffer == "1" { "--features file-based" } else { "" } @@ -59,6 +59,9 @@ long_chain: just rust_func array_sum +simple_file: + just rust_func simple_file + all_libos: ./scripts/build_all_common{{ if enable_mpk == "1" { "_mpk" } else { "" } }}.sh {{ release_flag }} @@ -190,4 +193,7 @@ measure_avg isol_file: total_dur=$(echo "$output" | grep -o '"total_dur(ms)": [0-9.]*' | awk -F': ' '{print $2}'); \ total_dur_rounded=$(printf "%.3f\n" "$total_dur") ;\ echo "$total_dur_rounded ," ;\ - done ; \ No newline at end of file + done ; + +gen_data: + sudo ./scripts/gen_data.py \ No newline at end of file diff --git a/scripts/gen_data.py b/scripts/gen_data.py index f0670bf8..fe30e85b 100755 --- a/scripts/gen_data.py +++ b/scripts/gen_data.py @@ -41,5 +41,5 @@ def gen_parallel_sort(file_num: int, total_size: int): if __name__ == "__main__": - # gen_parallel_sort(5, 1 * 1024 * 1024) - gen_word_count(3, 100 * 1024 * 1024) + gen_parallel_sort(3, 25 * 1024 * 1024) + # gen_word_count(3, 100 * 1024 * 1024) diff --git a/user/checker/Cargo.lock b/user/checker/Cargo.lock index f1f84cd2..3c676b6a 100644 --- a/user/checker/Cargo.lock +++ b/user/checker/Cargo.lock @@ -30,9 +30,11 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "checker" version = "0.1.0" dependencies = [ + "heapless", "ms_hostcall", "ms_std", "ms_std_proc_macro", + "serde", ] [[package]] diff --git a/user/checker/Cargo.toml b/user/checker/Cargo.toml index f719cf92..d3dbebb8 100644 --- a/user/checker/Cargo.toml +++ b/user/checker/Cargo.toml @@ -12,8 +12,10 @@ crate-type = ["dylib"] ms_std = { path = "../../ms_std", features = ["alloc_def", "unwinding"] } ms_hostcall = { path = "../../ms_hostcall" } ms_std_proc_macro = { path = "../../ms_std_proc_macro" } +heapless = "0.8.0" +serde = { version = "1.0.217", default-features = false, features = ["derive"] } [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] -default = [] \ No newline at end of file +default = [] diff --git a/user/checker/src/lib.rs b/user/checker/src/lib.rs index 25b87b57..68014e23 100644 --- a/user/checker/src/lib.rs +++ b/user/checker/src/lib.rs @@ -3,17 +3,28 @@ use alloc::{format, vec::Vec}; -use ms_std::{args, prelude::*,time::{SystemTime, UNIX_EPOCH}}; +use ms_std::{ + args, + prelude::*, + time::{SystemTime, UNIX_EPOCH}, +}; use ms_std_proc_macro::FaasData; +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] #[derive(Default, FaasData)] struct VecArg { + #[cfg(feature = "pkey_per_func")] + array: heapless::Vec, + #[cfg(not(feature = "pkey_per_func"))] array: Vec, } #[no_mangle] pub fn main() -> Result<()> { - println!("com_start4: {}", SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64); + println!( + "com_start4: {}", + SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 + ); let merger_num: u32 = { let m = args::get("merger_num").unwrap(); m.parse().unwrap() @@ -38,7 +49,10 @@ pub fn main() -> Result<()> { last_max = *first } } - println!("com_end4: {}", SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64); + println!( + "com_end4: {}", + SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 + ); println!("sort result is ok, total sort {} numbers", counter); Ok(().into()) } diff --git a/user/file_reader/src/lib.rs b/user/file_reader/src/lib.rs index 9875fb45..eee684ce 100644 --- a/user/file_reader/src/lib.rs +++ b/user/file_reader/src/lib.rs @@ -13,7 +13,8 @@ use ms_std_proc_macro::FaasData; use serde::{Deserialize, Serialize}; // use ms_std_proc_macro::FaasData; -#[derive(Default, FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(Default, FaasData)] struct VecArg { #[cfg(feature = "pkey_per_func")] content: heapless::String<{ 110 * 1024 * 1024 }>, diff --git a/user/mapper/Cargo.lock b/user/mapper/Cargo.lock index d9288ac8..853d7493 100644 --- a/user/mapper/Cargo.lock +++ b/user/mapper/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -57,6 +45,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "gimli" version = "0.26.2" @@ -74,11 +68,11 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "ahash", + "foldhash", "serde", ] @@ -172,15 +166,9 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.96", ] -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - [[package]] name = "proc-macro2" version = "1.0.93" @@ -339,29 +327,3 @@ dependencies = [ "gimli", "libc", ] - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] diff --git a/user/mapper/Cargo.toml b/user/mapper/Cargo.toml index f3d780c0..df8e5ba9 100644 --- a/user/mapper/Cargo.toml +++ b/user/mapper/Cargo.toml @@ -13,8 +13,8 @@ ms_std = { path = "../../ms_std", features = ["alloc_def", "unwinding"] } ms_hostcall = { path = "../../ms_hostcall" } ms_std_proc_macro = { path = "../../ms_std_proc_macro" } -hashbrown = { version = "=0.14.3", default-features = false, features = [ - "ahash", +hashbrown = { version = "0.15.0", default-features = false, features = [ + "default-hasher", "serde", ] } heapless = { version = "0.8.0", default-features = false } diff --git a/user/mapper/src/lib.rs b/user/mapper/src/lib.rs index 05555275..33835071 100644 --- a/user/mapper/src/lib.rs +++ b/user/mapper/src/lib.rs @@ -32,7 +32,8 @@ extern crate alloc; // content: String, // } -#[derive(FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(FaasData)] struct Mapper2Reducer { #[cfg(feature = "pkey_per_func")] shuffle: heapless::FnvIndexMap, u32, 1024>, @@ -65,39 +66,41 @@ fn mapper_func(my_id: &str, reducer_num: u64) -> Result<()> { // DataBuffer::from_buffer_slot(format!("part-{}", my_id)).expect("missing input data."); // println!("access_buffer_long={}", reader.content.len()); + let start = SystemTime::now(); let file_name = format!("fake_data_{}.txt", my_id); let mut f = File::open(&file_name)?; let mut content = String::new(); + // let content = get_input_from_mem(&file_name); + // f.read_to_string(&mut content).expect("read file failed."); + println!( - "read_start: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 - ); - f.read_to_string(&mut content).expect("read file failed."); - println!( - "read_end: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 + "read_end, cost: {}ms", + SystemTime::elapsed(&start).as_millis() ); + let start = SystemTime::now(); let mut counter = HashMap::new(); - println!( - "comput_start: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 - ); for line in content.lines() { - let words = line.trim().split(' '); - // .filter(|word| word.chars().all(char::is_alphanumeric)); + let words = line + .trim() + .split(' ') + .filter(|word| word.chars().all(char::is_alphanumeric)); for word in words { - for word in word.split('.') { - let old_count = *counter.entry(word).or_insert(0u32); - counter.insert(word, old_count + 1); - } + let old_count = *counter.entry(word).or_insert(0u32); + // for word in word.split('.') { + // let old_count = *counter.entry(word).or_insert(0u32); + // counter.insert(word, old_count + 1); + // } + counter.insert(word, old_count + 1); } } println!( - "comput_end: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 + "split word end, cost: {}ms", + SystemTime::elapsed(&start).as_millis() ); + let start = SystemTime::now(); + let total: u32 = counter.values().sum(); // 打印总和 println!("The sum of all values is: {}", total); @@ -112,11 +115,6 @@ fn mapper_func(my_id: &str, reducer_num: u64) -> Result<()> { data_buffers.push(buffer); } - ms_std::println!("the counter nums is {}", counter.len()); - println!( - "register_start: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 - ); for (word, count) in counter { let shuffle_idx = getidx(word, reducer_num); @@ -133,8 +131,8 @@ fn mapper_func(my_id: &str, reducer_num: u64) -> Result<()> { } println!( - "register_end: {}", - SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 + "shuffle _end, cost {}ms", + SystemTime::elapsed(&start).as_millis() ); Ok(().into()) @@ -151,3 +149,26 @@ pub fn main() -> Result<()> { mapper_func(my_id, reducer_num) } + +// fn get_input_from_mem(filename: &str) -> &'static str { +// match filename { +// "fake_data_0.txt" => { +// include_str!("../../../image_content/fake_data_0.txt") +// } +// "fake_data_1.txt" => { +// include_str!("../../../image_content/fake_data_1.txt") +// } +// "fake_data_2.txt" => { +// include_str!("../../../image_content/fake_data_2.txt") +// } +// "fake_data_3.txt" => { +// include_str!("../../../image_content/fake_data_3.txt") +// } +// "fake_data_4.txt" => { +// include_str!("../../../image_content/fake_data_4.txt") +// } +// _ => { +// panic!("unknown filename") +// } +// } +// } diff --git a/user/merger/src/lib.rs b/user/merger/src/lib.rs index d20809b5..59e7c2a6 100644 --- a/user/merger/src/lib.rs +++ b/user/merger/src/lib.rs @@ -14,7 +14,8 @@ type NumberArray = heapless::Vec; #[cfg(not(feature = "pkey_per_func"))] type NumberArray = Vec; -#[derive(Default, FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(Default, FaasData)] struct VecArg { #[cfg(feature = "pkey_per_func")] array: heapless::Vec, diff --git a/user/reducer/Cargo.lock b/user/reducer/Cargo.lock index c4df4b64..87f7f4f8 100644 --- a/user/reducer/Cargo.lock +++ b/user/reducer/Cargo.lock @@ -2,18 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "ahash" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -57,6 +45,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "gimli" version = "0.26.2" @@ -74,11 +68,11 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" dependencies = [ - "ahash", + "foldhash", "serde", ] @@ -160,15 +154,9 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.96", ] -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - [[package]] name = "proc-macro2" version = "1.0.93" @@ -339,29 +327,3 @@ dependencies = [ "gimli", "libc", ] - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "zerocopy" -version = "0.7.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.96", -] diff --git a/user/reducer/Cargo.toml b/user/reducer/Cargo.toml index 786cf1d4..52005665 100644 --- a/user/reducer/Cargo.toml +++ b/user/reducer/Cargo.toml @@ -14,8 +14,8 @@ ms_hostcall = { path = "../../ms_hostcall" } ms_std_proc_macro = { path = "../../ms_std_proc_macro" } heapless = { version = "0.8.0", default-features = false } -hashbrown = { version = "0.14", default-features = false, features = [ - "ahash", +hashbrown = { version = "0.15", default-features = false, features = [ + "default-hasher", "serde", ] } serde = { version = "1.0.217", default-features = false, features = ["derive"] } diff --git a/user/reducer/src/lib.rs b/user/reducer/src/lib.rs index 57a4705d..8195ce80 100644 --- a/user/reducer/src/lib.rs +++ b/user/reducer/src/lib.rs @@ -18,7 +18,8 @@ use serde::{Deserialize, Serialize}; extern crate alloc; -#[derive(FaasData, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(Default, FaasData)] struct Mapper2Reducer { #[cfg(feature = "pkey_per_func")] shuffle: heapless::FnvIndexMap, u32, 1024>, diff --git a/user/simple_file/Cargo.lock b/user/simple_file/Cargo.lock index 90894a2d..39a3004d 100644 --- a/user/simple_file/Cargo.lock +++ b/user/simple_file/Cargo.lock @@ -138,7 +138,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.96", ] [[package]] diff --git a/user/simple_file/Cargo.toml b/user/simple_file/Cargo.toml index 8c10abef..ac3d22f8 100644 --- a/user/simple_file/Cargo.toml +++ b/user/simple_file/Cargo.toml @@ -13,7 +13,8 @@ ms_std = { path = "../../ms_std", features = ["alloc_def", "unwinding"] } ms_hostcall = { path = "../../ms_hostcall" } ms_std_proc_macro = { path = "../../ms_std_proc_macro" } + [features] mpk = ["ms_std/mpk"] pkey_per_func = ["mpk"] -default = [] \ No newline at end of file +default = [] diff --git a/user/simple_file/src/lib.rs b/user/simple_file/src/lib.rs index b4f84139..295ca39e 100644 --- a/user/simple_file/src/lib.rs +++ b/user/simple_file/src/lib.rs @@ -11,10 +11,81 @@ use ms_std::{ fs::File, io::{Read, Write}, println, + time::SystemTime, }; +// fn init_input_file() { +// File::create("fake_data_0.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/fake_data_0.txt")) +// .unwrap(); + +// File::create("fake_data_1.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/fake_data_1.txt")) +// .unwrap(); + +// File::create("fake_data_2.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/fake_data_2.txt")) +// .unwrap(); + +// File::create("fake_data_3.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/fake_data_3.txt")) +// .unwrap(); + +// File::create("fake_data_4.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/fake_data_4.txt")) +// .unwrap(); + +// let content = include_str!("../../../image_content/sort_data_0.txt"); +// File::create("sort_data_0.txt") +// .unwrap() +// .write_str(content) +// .unwrap(); + +// let start = SystemTime::now(); +// let mut array: Vec = Vec::new(); +// for num in content.split(',') { +// let num = num.trim(); +// if num.is_empty() { +// continue; +// } +// let num = num.parse().unwrap(); +// array.push(num); +// } +// println!( +// "split {} numbers cost {}ms", +// array.len(), +// SystemTime::elapsed(&start).as_millis() +// ); + +// File::create("sort_data_1.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/sort_data_1.txt")) +// .unwrap(); + +// File::create("sort_data_2.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/sort_data_2.txt")) +// .unwrap(); + +// File::create("sort_data_3.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/sort_data_3.txt")) +// .unwrap(); + +// File::create("sort_data_4.txt") +// .unwrap() +// .write_str(include_str!("../../../image_content/sort_data_4.txt")) +// .unwrap(); +// } + #[no_mangle] pub fn main() -> Result<()> { + let start_time = SystemTime::now(); let path = "lines.txt"; /////////////////// test create/write/read. /////////////////// @@ -52,5 +123,11 @@ pub fn main() -> Result<()> { Err("seek failed")? } + // init_input_file(); + + // println!( + // "simple_file exec: {}ms", + // SystemTime::elapsed(&start_time).as_millis() + // ); Ok(().into()) } diff --git a/user/sorter/src/lib.rs b/user/sorter/src/lib.rs index 79fa11c5..ccf7837b 100644 --- a/user/sorter/src/lib.rs +++ b/user/sorter/src/lib.rs @@ -4,13 +4,16 @@ use alloc::{format, string::String, vec::Vec}; use ms_std::{ args, + fs::File, + io::Read, prelude::*, time::{SystemTime, UNIX_EPOCH}, }; use ms_std_proc_macro::FaasData; use serde::{Deserialize, Serialize}; -#[derive(Default, FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(Default, FaasData)] struct Reader2Sorter { #[cfg(feature = "pkey_per_func")] content: heapless::String<{ 110 * 1024 * 1024 }>, @@ -18,7 +21,8 @@ struct Reader2Sorter { content: String, } -#[derive(Default, FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, , Deserialize))] +#[derive(Default, FaasData)] struct VecArg { #[cfg(feature = "pkey_per_func")] array: heapless::Vec, @@ -48,18 +52,27 @@ pub fn main() -> Result<()> { n.parse().unwrap() }; + ///////////// file_reader //////////////// + // let mut input_file = File::open(&format!("sort_data_{}.txt", my_id)).unwrap(); + // let mut content = String::new(); + // input_file.read_to_string(&mut content).unwrap(); + + // let mut input: DataBuffer = + // DataBuffer::with_slot(format!("input-part-{}", my_id)); + + // input.content = content; + // drop(input); + ///////////////////////////////////////// + let input: DataBuffer = DataBuffer::from_buffer_slot(format!("input-part-{}", my_id)).unwrap(); - - println!( - "sorter: input-part-{}, input length={}", - my_id, - input.content.len() - ); + let content = input.content.as_str(); let mut buffer: DataBuffer = DataBuffer::with_slot(format!("sorter-resp-part-{}", my_id)); - for num in input.content.split(',') { + + let start = SystemTime::now(); + for num in content.split(',') { let num = num.trim(); if num.is_empty() { continue; @@ -70,7 +83,20 @@ pub fn main() -> Result<()> { buffer.array.push(num); } + println!( + "sorter: input-part-{}, input length={}. split word cost: {}ms", + my_id, + input.content.len(), + SystemTime::elapsed(&start).as_millis(), + ); + + let start = SystemTime::now(); buffer.array.sort(); + println!( + "numbers array length is {}, sort cost {}ms", + buffer.array.len(), + SystemTime::elapsed(&start).as_millis() + ); if my_id.eq("0") { // let mut pivots: DataBuffer = ; diff --git a/user/splitter/src/lib.rs b/user/splitter/src/lib.rs index 17f27958..2fd8e134 100644 --- a/user/splitter/src/lib.rs +++ b/user/splitter/src/lib.rs @@ -10,7 +10,8 @@ use ms_std::{ use ms_std_proc_macro::FaasData; use serde::{Deserialize, Serialize}; -#[derive(Default, FaasData, Serialize, Deserialize)] +#[cfg_attr(feature = "file-based", derive(Serialize, Deserialize))] +#[derive(Default, FaasData)] struct VecArg { #[cfg(feature = "pkey_per_func")] array: heapless::Vec, @@ -26,11 +27,12 @@ struct Pivots { #[no_mangle] pub fn main() -> Result<()> { + let my_id = args::get("id").unwrap(); println!( - "com_start2: {}", + "splitter id: {}, com_start2: {}", + my_id, SystemTime::now().duration_since(UNIX_EPOCH).as_micros() as f64 / 1000000f64 ); - let my_id = args::get("id").unwrap(); let numbers: DataBuffer = DataBuffer::from_buffer_slot(format!("sorter-resp-part-{}", my_id)).unwrap(); @@ -48,6 +50,7 @@ pub fn main() -> Result<()> { DataBuffer::with_slot(format!("splitter-{}-resp-part-{}", my_id, idx)); #[cfg(feature = "pkey_per_func")] { + part.array.resize(partition.len(), 0).unwrap(); for (idx, item) in part.array.iter_mut().enumerate() { *item = partition[idx]; }