Skip to content

Expr/p99 #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anti-entropy123 merged 2 commits into from
Jan 22, 2025
Merged

Expr/p99 #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8dc35f9
prepare test map reduce.
anti-entropy123 Jan 16, 2025
330a944
finish.
anti-entropy123 Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 41 additions & 57 deletions isol_config/long_chain_n10.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,106 +23,90 @@
],
"apps": [
[
"array_sum0",
"libarray_sum.so"
],
[
"array_sum1",
"libarray_sum.so"
],
[
"array_sum2",
"libarray_sum.so"
],
[
"array_sum3",
"libarray_sum.so"
],
[
"array_sum4",
"libarray_sum.so"
],
[
"array_sum5",
"libarray_sum.so"
],
[
"array_sum6",
"libarray_sum.so"
],
[
"array_sum7",
"libarray_sum.so"
],
[
"array_sum8",
"libarray_sum.so"
],
[
"array_sum9",
"array_sum",
"libarray_sum.so"
]
],
"groups": [
{
"list": [
"array_sum0"
"array_sum"
],
"args": {"array_num": "0"}
"args": {
"n": "0"
}
},
{
"list": [
"array_sum1"
"array_sum"
],
"args": {"array_num": "1"}
"args": {
"n": "1"
}
},
{
"list": [
"array_sum2"
"array_sum"
],
"args": {"array_num": "2"}
"args": {
"n": "2"
}
},
{
"list": [
"array_sum3"
"array_sum"
],
"args": {"array_num": "3"}
"args": {
"n": "3"
}
},
{
"list": [
"array_sum4"
"array_sum"
],
"args": {"array_num": "4"}
"args": {
"n": "4"
}
},
{
"list": [
"array_sum5"
"array_sum"
],
"args": {"array_num": "5"}
"args": {
"n": "5"
}
},
{
"list": [
"array_sum6"
"array_sum"
],
"args": {"array_num": "6"}
"args": {
"n": "6"
}
},
{
"list": [
"array_sum7"
"array_sum"
],
"args": {"array_num": "7"}
"args": {
"n": "7"
}
},
{
"list": [
"array_sum8"
"array_sum"
],
"args": {"array_num": "8"}
"args": {
"n": "8"
}
},
{
"list": [
"array_sum9"
"array_sum"
],
"args": {"array_num": "9"}
"args": {
"n": "9"
}
}
]
}
2 changes: 1 addition & 1 deletion isol_config/map_reduce_large_c3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"libreducer.so"
]
],
"fs_image": "fs_images/new_image.img",
"fs_image": "fs_images/fatfs.img",
"groups": [
{
"list": [
Expand Down
41 changes: 19 additions & 22 deletions justfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,31 +7,29 @@ enable_file_buffer := "0"

enable_release := "1"

cmd_flag := if enable_mpk == "1" {
mpk_flag := if enable_mpk == "1" {
if enable_pkey_per_func == "1" {
"pkey_per_func"
} else {
"mpk"
}
} else { "" }

feature_flag := if enable_mpk == "1" {
if enable_pkey_per_func == "1" {
"--features pkey_per_func"
} else {
"--features mpk"
}
} else { "" }
mpk_feature_flag := if mpk_flag == "" { "" } else { "--features {{mpk_flag}}" }

buffer_feature_flag := if enable_file_buffer == "1" { "--features file-based" } else { "" }

release_flag := if enable_release == "1" {
"--release"
} else { "" }

rust_func func_name:
cargo build {{ release_flag }} {{ feature_flag }} {{ if enable_file_buffer == "1" { "--features file-based" } else { "" } }} --manifest-path user/{{ func_name }}/Cargo.toml
cargo build {{ release_flag }} {{ mpk_feature_flag }} {{ buffer_feature_flag }} \
--manifest-path user/{{ func_name }}/Cargo.toml

libos lib_name:
cargo build {{ release_flag }} {{ if enable_mpk == "1" { "--features mpk" } else { "" } }} --manifest-path common_service/{{ lib_name }}/Cargo.toml
cargo build {{ release_flag }} {{ if enable_mpk == "1" { "--features mpk" } else { "" } }} \
--manifest-path common_service/{{ lib_name }}/Cargo.toml

pass_args:
just rust_func func_a
Expand Down Expand Up @@ -60,19 +58,18 @@ long_chain:
done

just rust_func array_sum


all_libos:
./scripts/build_all_common{{ if enable_mpk == "1" { "_mpk" } else { "" } }}.sh {{ release_flag }}

all_rust:
just all_libos
./scripts/build_user.sh {{ release_flag }} {{ cmd_flag }}
./scripts/build_user.sh {{ release_flag }} {{ mpk_flag }}

run_rust_test:
just all_libos
just all_rust
./scripts/run_tests.sh {{ cmd_flag }}
./scripts/run_tests.sh {{ mpk_flag }}

cc_flags_p1 := "-Wl,--gc-sections -nostdlib -Wl,--whole-archive"
cc_flags_p2 := "-Wl,--no-whole-archive -shared"
Expand All @@ -82,7 +79,7 @@ c_mapper_so:
@echo "c_mapper.so"
cd user/wasmtime_mapper \
&& cargo build \
--target {{target}} {{feature_flag}} \
--target {{target}} {{mpk_feature_flag}} \
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_mapper.a \
{{cc_flags_p2}} \
Expand All @@ -91,7 +88,7 @@ c_mapper_so:
c_reducer_so:
@echo "c_reducer.so"
cd user/wasmtime_reducer \
&& cargo build --target {{target}} {{feature_flag}} \
&& cargo build --target {{target}} {{mpk_feature_flag}} \
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_reducer.a \
{{cc_flags_p2}} \
Expand All @@ -107,7 +104,7 @@ wasmtime_wordcount: c_mapper_so c_reducer_so
c_sorter_so:
@echo "c_sorter.so"
cd user/wasmtime_sorter \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_sorter.a \
{{cc_flags_p2}} \
Expand All @@ -116,7 +113,7 @@ c_sorter_so:
c_spliter_so:
@echo "c_spliter.so"
cd user/wasmtime_spliter \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_spliter.a \
{{cc_flags_p2}} \
Expand All @@ -125,7 +122,7 @@ c_spliter_so:
c_merger_so:
@echo "c_merger.so"
cd user/wasmtime_merger \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_merger.a \
{{cc_flags_p2}} \
Expand All @@ -134,7 +131,7 @@ c_merger_so:
c_checker_so:
@echo "c_checker.so"
cd user/wasmtime_checker \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_checker.a \
{{cc_flags_p2}} \
Expand All @@ -157,7 +154,7 @@ all_c_wasm: wasmtime_wordcount wasmtime_parallel_sort
cpython_wordcount_so:
@echo "cpython_wordcount.so"
cd user/wasmtime_cpython_wordcount \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_cpython_wordcount.a \
{{cc_flags_p2}} \
Expand All @@ -171,7 +168,7 @@ cpython_wordcount: cpython_wordcount_so
cpython_parallel_sort_so:
@echo "cpython_parallel_sort.so"
cd user/wasmtime_cpython_parallel_sort \
&& cargo build --target {{target}} {{feature_flag}}\
&& cargo build --target {{target}} {{mpk_feature_flag}}\
&& cc {{cc_flags_p1}} \
target/{{target}}/debug/libwasmtime_cpython_parallel_sort.a \
{{cc_flags_p2}} \
Expand All @@ -189,7 +186,7 @@ all_wasm: all_c_wasm all_py_wasm
measure_avg isol_file:
#!/bin/bash
for (( i=1; i<=10; i++ )); do \
output=$(cargo run {{ release_flag }} {{feature_flag}} -- --files {{ isol_file }} --metrics total-dur 2>&1); \
output=$(cargo run {{ release_flag }} {{mpk_feature_flag}} -- --files {{ isol_file }} --metrics total-dur 2>&1); \
total_dur=$(echo "$output" | grep -o '"total_dur(ms)": [0-9.]*' | awk -F': ' '{print $2}'); \
total_dur_rounded=$(printf "%.3f\n" "$total_dur") ;\
echo "$total_dur_rounded ," ;\
Expand Down
6 changes: 5 additions & 1 deletion libmsvisor/src/isolation/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use std::{
use anyhow::{anyhow, Ok};

use lazy_static::lazy_static;
use log::info;
use log::{info, warn};
use ms_hostcall::types::{
IsolationID as IsolID,
MetricEvent::{IsolBegin, IsolEnd, Mem},
Expand Down Expand Up @@ -99,6 +99,10 @@ impl Isolation {
)
.register(config);

if let Some(image) = &config.fs_image {
warn!("will use a non default file system image: {}", image)
}

let isol = Arc::from(Self {
id: new_id,
loader,
Expand Down
4 changes: 2 additions & 2 deletions scripts/gen_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,5 @@ def gen_parallel_sort(file_num: int, total_size: int):


if __name__ == "__main__":
gen_parallel_sort(5, 1 * 1024 * 1024)
# gen_word_count(5, 1 * 1024 * 1024)
# gen_parallel_sort(5, 1 * 1024 * 1024)
gen_word_count(3, 100 * 1024 * 1024)
21 changes: 13 additions & 8 deletions scripts/map_reduce_parallel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,28 +5,29 @@

# 并行执行 ./scripts/rust_end_to_end.sh map_reduce c3

concur_num = 10


def test_map_reduce_parallel():
import concurrent.futures

def run_script(instance):
command = f"./scripts/rust_end_to_end.sh map_reduce {instance}"
def run_script():
command = f"./target/release/msvisor --files isol_config/map_reduce_large_c3.json --metrics total-dur 2>&1"
result = subprocess.run(command, shell=True,
capture_output=True, text=True)
return result.stdout, result.stderr

instances = ['c3', 'c3', 'c3', 'c3', 'c3', 'c3', 'c3',
'c3', 'c3', 'c3'] # Add more instances as needed
latencies = []

with concurrent.futures.ThreadPoolExecutor() as executor:
futures = [executor.submit(run_script, instance)
for instance in instances]
futures = [executor.submit(run_script)
for _ in range(concur_num)]

for future in concurrent.futures.as_completed(futures):
stdout, stderr = future.result()
# 从 stdout 中提取 'Total Dur (ms): 37.825' 中的数字
for line in stdout.splitlines():
match = re.search(r'Total Dur \(ms\): (\d+\.\d+)', line)
match = re.search(r'"total_dur\(ms\)": ([\d.]+)', line)
if match:
duration = float(match.group(1))
# print("Total Duration (ms):", duration)
Expand All @@ -35,8 +36,12 @@ def run_script(instance):
if stderr:
print("Error:", stderr)

print("Latencies:", sorted(latencies))
latencies = sorted(latencies)
print("Latencies:", latencies)
print("p99:", latencies[-2])


if __name__ == '__main__':
import sys
concur_num = int(sys.argv[1])
test_map_reduce_parallel()
Loading