test: save test results

Signed-off-by: Henry Gressmann <[email protected]>

Author: explodingcamera

crates/tinywasm/tests/mvp.csv

@@ -0,0 +1 @@
+0.0.4,9258,7567,[{"name":"0","passed":0,"failed":54},{"name":"0","passed":0,"failed":109},{"name":"66","passed":66,"failed":25},{"name":"104","passed":104,"failed":8},{"name":"0","passed":0,"failed":171},{"name":"0","passed":0,"failed":21},{"name":"0","passed":0,"failed":30},{"name":"0","passed":0,"failed":25},{"name":"0","passed":0,"failed":22},{"name":"0","passed":0,"failed":56},{"name":"4","passed":4,"failed":4},{"name":"702","passed":702,"failed":76},{"name":"0","passed":0,"failed":93},{"name":"10","passed":10,"failed":1},{"name":"0","passed":0,"failed":61},{"name":"0","passed":0,"failed":76},{"name":"0","passed":0,"failed":1},{"name":"21","passed":21,"failed":73},{"name":"1005","passed":1005,"failed":1509},{"name":"1","passed":1,"failed":363},{"name":"2401","passed":2401,"failed":6},{"name":"1005","passed":1005,"failed":1509},{"name":"1","passed":1,"failed":363},{"name":"2401","passed":2401,"failed":6},{"name":"0","passed":0,"failed":2},{"name":"269","passed":269,"failed":591},{"name":"34","passed":34,"failed":129},{"name":"0","passed":0,"failed":6},{"name":"138","passed":138,"failed":303},{"name":"1","passed":1,"failed":4},{"name":"4","passed":4,"failed":75},{"name":"0","passed":0,"failed":16},{"name":"4","passed":4,"failed":49},{"name":"0","passed":0,"failed":96},{"name":"0","passed":0,"failed":42},{"name":"0","passed":0,"failed":118},{"name":"1","passed":1,"failed":156},{"name":"0","passed":0,"failed":1},{"name":"38","passed":38,"failed":70},{"name":"5","passed":5,"failed":46},{"name":"1","passed":1,"failed":28},{"name":"0","passed":0,"failed":1},{"name":"1","passed":1,"failed":66},{"name":"0","passed":0,"failed":60},{"name":"2","passed":2,"failed":34},{"name":"5","passed":5,"failed":48},{"name":"0","passed":0,"failed":42},{"name":"0","passed":0,"failed":43},{"name":"0","passed":0,"failed":34},{"name":"0","passed":0,"failed":19},{"name":"0","passed":0,"failed":1},{"name":"0","passed":0,"failed":6},{"name":"0","passed":0,"failed":172},{"name":"484","passed":484,"failed":1},{"name":"0","passed":0,"failed":5},{"name":"0","passed":0,"failed":21},{"name":"0","passed":0,"failed":32},{"name":"0","passed":0,"failed":11},{"name":"0","passed":0,"failed":2},{"name":"0","passed":0,"failed":10},{"name":"0","passed":0,"failed":59},{"name":"1","passed":1,"failed":27},{"name":"16","passed":16,"failed":42},{"name":"3","passed":3,"failed":33},{"name":"1","passed":1,"failed":2},{"name":"0","passed":0,"failed":59},{"name":"0","passed":0,"failed":118},{"name":"1","passed":1,"failed":49},{"name":"176","passed":176,"failed":0},{"name":"176","passed":176,"failed":0},{"name":"176","passed":176,"failed":0},{"name":"0","passed":0,"failed":176}]

crates/tinywasm/tests/testsuite/mod.rs

@@ -0,0 +1,149 @@
+use eyre::Result;
+use std::io::{BufRead, Seek, SeekFrom};
+use std::{
+    collections::BTreeMap,
+    fmt::{Debug, Formatter},
+    io::BufReader,
+};
+
+mod run;
+mod util;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+struct TestGroupResult {
+    name: String,
+    passed: usize,
+    failed: usize,
+}
+
+pub struct TestSuite(BTreeMap<String, TestGroup>);
+
+impl TestSuite {
+    pub fn new() -> Self {
+        Self(BTreeMap::new())
+    }
+
+    pub fn failed(&self) -> bool {
+        self.0.values().any(|group| group.stats().1 > 0)
+    }
+
+    fn test_group(&mut self, name: &str) -> &mut TestGroup {
+        self.0.entry(name.to_string()).or_insert_with(TestGroup::new)
+    }
+
+    // create or add to a test result file
+    pub fn save_csv(&self, path: &str, version: &str) -> Result<()> {
+        use std::fs::OpenOptions;
+        use std::io::Write;
+
+        let mut file = OpenOptions::new().create(true).append(true).read(true).open(path)?;
+        let last_line = BufReader::new(&file).lines().last().transpose()?;
+
+        // Check if the last line starts with the current commit
+        if let Some(last) = last_line {
+            if last.starts_with(version) {
+                // Truncate the file size to remove the last line
+                let len_to_truncate = last.len() as i64;
+                file.set_len(file.metadata()?.len() - len_to_truncate as u64)?;
+            }
+        }
+
+        // Seek to the end of the file for appending
+        file.seek(SeekFrom::End(0))?;
+
+        let mut passed = 0;
+        let mut failed = 0;
+
+        let mut groups = Vec::new();
+        for group in self.0.values() {
+            let (group_passed, group_failed) = group.stats();
+            passed += group_passed;
+            failed += group_failed;
+
+            groups.push(TestGroupResult {
+                name: group_passed.to_string(),
+                passed: group_passed,
+                failed: group_failed,
+            });
+        }
+
+        let groups = serde_json::to_string(&groups)?;
+        let line = format!("{},{},{},{}\n", version, passed, failed, groups);
+        file.write_all(line.as_bytes()).expect("failed to write to csv file");
+
+        Ok(())
+    }
+}
+
+impl Debug for TestSuite {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        use owo_colors::OwoColorize;
+        let mut total_passed = 0;
+        let mut total_failed = 0;
+
+        for (group_name, group) in &self.0 {
+            let (group_passed, group_failed) = group.stats();
+            total_passed += group_passed;
+            total_failed += group_failed;
+
+            writeln!(f, "{}", group_name.bold().underline())?;
+            writeln!(f, "  Tests Passed: {}", group_passed.to_string().green())?;
+            writeln!(f, "  Tests Failed: {}", group_failed.to_string().red())?;
+
+            // for (test_name, test) in &group.tests {
+            //     write!(f, "    {}: ", test_name.bold())?;
+            //     match &test.result {
+            //         Ok(()) => {
+            //             writeln!(f, "{}", "Passed".green())?;
+            //         }
+            //         Err(e) => {
+            //             writeln!(f, "{}", "Failed".red())?;
+            //             // writeln!(f, "Error: {:?}", e)?;
+            //         }
+            //     }
+            //     writeln!(f, "      Span: {:?}", test.span)?;
+            // }
+        }
+
+        writeln!(f, "\n{}", "Total Test Summary:".bold().underline())?;
+        writeln!(f, "  Total Tests: {}", (total_passed + total_failed))?;
+        writeln!(f, "  Total Passed: {}", total_passed.to_string().green())?;
+        writeln!(f, "  Total Failed: {}", total_failed.to_string().red())?;
+        Ok(())
+    }
+}
+
+struct TestGroup {
+    tests: BTreeMap<String, TestCase>,
+}
+
+impl TestGroup {
+    fn new() -> Self {
+        Self { tests: BTreeMap::new() }
+    }
+
+    fn stats(&self) -> (usize, usize) {
+        let mut passed_count = 0;
+        let mut failed_count = 0;
+
+        for test in self.tests.values() {
+            match test.result {
+                Ok(()) => passed_count += 1,
+                Err(_) => failed_count += 1,
+            }
+        }
+
+        (passed_count, failed_count)
+    }
+
+    fn add_result(&mut self, name: &str, span: wast::token::Span, result: Result<()>) {
+        self.tests.insert(name.to_string(), TestCase { result, _span: span });
+    }
+}
+
+struct TestCase {
+    result: Result<()>,
+    _span: wast::token::Span,
+}

crates/tinywasm/tests/testsuite/run.rs

@@ -0,0 +1,128 @@
+use crate::testsuite::util::{parse_module, wastarg2tinywasmvalue, wastret2tinywasmvalue};
+
+use super::TestSuite;
+use eyre::{eyre, Result};
+use log::debug;
+use tinywasm_types::TinyWasmModule;
+use wast::{lexer::Lexer, parser::ParseBuffer, QuoteWat, Wast};
+
+impl TestSuite {
+    pub fn run(&mut self, tests: &[&str]) -> Result<()> {
+        tests.iter().for_each(|group| {
+            let test_group = self.test_group(group);
+
+            let wast = wasm_testsuite::get_test_wast(group).expect("failed to get test wast");
+            let wast = std::str::from_utf8(&wast).expect("failed to convert wast to utf8");
+
+            let mut lexer = Lexer::new(wast);
+            // we need to allow confusing unicode characters since they are technically valid wasm
+            lexer.allow_confusing_unicode(true);
+
+            let buf = ParseBuffer::new_with_lexer(lexer).expect("failed to create parse buffer");
+            let wast_data = wast::parser::parse::<Wast>(&buf).expect("failed to parse wat");
+
+            let mut last_module: Option<TinyWasmModule> = None;
+            for (i, directive) in wast_data.directives.into_iter().enumerate() {
+                let span = directive.span();
+                use wast::WastDirective::*;
+                let name = format!("{}-{}", group, i);
+
+                match directive {
+                    // TODO: needs to support more binary sections
+                    Wat(QuoteWat::Wat(wast::Wat::Module(module))) => {
+                        let result = std::panic::catch_unwind(|| parse_module(module))
+                            .map_err(|e| eyre!("failed to parse module: {:?}", e))
+                            .and_then(|res| res);
+
+                        match &result {
+                            Err(_) => last_module = None,
+                            Ok(m) => last_module = Some(m.clone()),
+                        }
+
+                        test_group.add_result(&format!("{}-parse", name), span, result.map(|_| ()));
+                    }
+
+                    // these all pass already :)
+                    AssertMalformed {
+                        span,
+                        module: QuoteWat::Wat(wast::Wat::Module(module)),
+                        message: _,
+                    } => {
+                        let res = std::panic::catch_unwind(|| parse_module(module).map(|_| ()));
+                        test_group.add_result(
+                            &format!("{}-malformed", name),
+                            span,
+                            match res {
+                                Ok(Ok(_)) => Err(eyre!("expected module to be malformed")),
+                                Err(_) | Ok(Err(_)) => Ok(()),
+                            },
+                        );
+                    }
+
+                    AssertReturn { span, exec, results } => {
+                        let Some(module) = last_module.as_ref() else {
+                            // we skip tests for modules that failed to parse
+                            println!("no module found for assert_return: {:?}", exec);
+                            continue;
+                        };
+
+                        let res: Result<Result<()>, _> = std::panic::catch_unwind(|| {
+                            let mut store = tinywasm::Store::new();
+                            let module = tinywasm::Module::from(module);
+                            let instance = module.instantiate(&mut store)?;
+
+                            use wast::WastExecute::*;
+                            let invoke = match exec {
+                                Wat(_) => return Result::Ok(()), // not used by the testsuite
+                                Get { module: _, global: _ } => return Result::Ok(()),
+                                Invoke(invoke) => invoke,
+                            };
+
+                            let args = invoke
+                                .args
+                                .into_iter()
+                                .map(wastarg2tinywasmvalue)
+                                .collect::<Result<Vec<_>>>()?;
+                            let res = instance.get_func(&store, invoke.name)?.call(&mut store, &args)?;
+                            let expected = results
+                                .into_iter()
+                                .map(wastret2tinywasmvalue)
+                                .collect::<Result<Vec<_>>>()?;
+
+                            if res.len() != expected.len() {
+                                return Result::Err(eyre!("expected {} results, got {}", expected.len(), res.len()));
+                            }
+
+                            for (i, (res, expected)) in res.iter().zip(expected).enumerate() {
+                                if res != &expected {
+                                    return Result::Err(eyre!(
+                                        "result {} did not match: {:?} != {:?}",
+                                        i,
+                                        res,
+                                        expected
+                                    ));
+                                }
+                            }
+
+                            Ok(())
+                        });
+
+                        let res = match res {
+                            Err(e) => Err(eyre!("test panicked: {:?}", e)),
+                            Ok(Err(e)) => Err(e),
+                            Ok(Ok(())) => Ok(()),
+                        };
+
+                        test_group.add_result(&format!("{}-return", name), span, res);
+                    }
+                    Invoke(m) => {
+                        debug!("invoke: {:?}", m);
+                    }
+                    _ => test_group.add_result(&format!("{}-unknown", name), span, Err(eyre!("unsupported directive"))),
+                }
+            }
+        });
+
+        Ok(())
+    }
+}

crates/tinywasm/tests/testsuite/util.rs

@@ -0,0 +1,72 @@
+use eyre::{eyre, Result};
+use tinywasm_types::TinyWasmModule;
+
+pub fn parse_module(mut module: wast::core::Module) -> Result<TinyWasmModule> {
+    let parser = tinywasm_parser::Parser::new();
+    Ok(parser.parse_module_bytes(module.encode().expect("failed to encode module"))?)
+}
+
+pub fn wastarg2tinywasmvalue(arg: wast::WastArg) -> Result<tinywasm_types::WasmValue> {
+    let wast::WastArg::Core(arg) = arg else {
+        return Err(eyre!("unsupported arg type"));
+    };
+
+    use tinywasm_types::WasmValue;
+    use wast::core::WastArgCore::*;
+    Ok(match arg {
+        F32(f) => WasmValue::F32(f32::from_bits(f.bits)),
+        F64(f) => WasmValue::F64(f64::from_bits(f.bits)),
+        I32(i) => WasmValue::I32(i),
+        I64(i) => WasmValue::I64(i),
+        _ => return Err(eyre!("unsupported arg type")),
+    })
+}
+
+pub fn wastret2tinywasmvalue(arg: wast::WastRet) -> Result<tinywasm_types::WasmValue> {
+    let wast::WastRet::Core(arg) = arg else {
+        return Err(eyre!("unsupported arg type"));
+    };
+
+    use tinywasm_types::WasmValue;
+    use wast::core::WastRetCore::*;
+    Ok(match arg {
+        F32(f) => nanpattern2tinywasmvalue(f)?,
+        F64(f) => nanpattern2tinywasmvalue(f)?,
+        I32(i) => WasmValue::I32(i),
+        I64(i) => WasmValue::I64(i),
+        _ => return Err(eyre!("unsupported arg type")),
+    })
+}
+
+enum Bits {
+    U32(u32),
+    U64(u64),
+}
+trait FloatToken {
+    fn bits(&self) -> Bits;
+}
+impl FloatToken for wast::token::Float32 {
+    fn bits(&self) -> Bits {
+        Bits::U32(self.bits)
+    }
+}
+impl FloatToken for wast::token::Float64 {
+    fn bits(&self) -> Bits {
+        Bits::U64(self.bits)
+    }
+}
+
+fn nanpattern2tinywasmvalue<T>(arg: wast::core::NanPattern<T>) -> Result<tinywasm_types::WasmValue>
+where
+    T: FloatToken,
+{
+    use wast::core::NanPattern::*;
+    Ok(match arg {
+        CanonicalNan => tinywasm_types::WasmValue::F32(f32::NAN),
+        ArithmeticNan => tinywasm_types::WasmValue::F32(f32::NAN),
+        Value(v) => match v.bits() {
+            Bits::U32(v) => tinywasm_types::WasmValue::F32(f32::from_bits(v)),
+            Bits::U64(v) => tinywasm_types::WasmValue::F64(f64::from_bits(v)),
+        },
+    })
+}