diff --git a/scripts/wasmtestreport.py b/scripts/wasmtestreport.py index 7c6a73438..ea2328566 100755 --- a/scripts/wasmtestreport.py +++ b/scripts/wasmtestreport.py @@ -51,6 +51,7 @@ TESTFILES_DST = LIND_ROOT / "testfiles" DETERMINISTIC_PARENT_NAME = "deterministic" NON_DETERMINISTIC_PARENT_NAME = "non-deterministic" +FAIL_PARENT_NAME = "fail" EXPECTED_DIRECTORY = Path("./expected") SKIP_TESTS_FILE = "skip_test_cases.txt" @@ -65,7 +66,12 @@ "Lind_wasm_Segmentation_Fault": "Lind Wasm Segmentation Failure", "Lind_wasm_Timeout": "Timeout During Lind Wasm run", "Unknown_Failure": "Unknown Failure", - "Output_mismatch": "C Compiler and Wasm Output mismatch" + "Output_mismatch": "C Compiler and Wasm Output mismatch", + "Fail_native_succeeded": "Fail Test: Native Succeeded (Should Fail)", + "Fail_wasm_succeeded": "Fail Test: Wasm Succeeded (Should Fail)", + "Fail_both_succeeded": "Fail Test: Both Native and Wasm Succeeded (Should Fail)", + "Fail_native_compiling": "Fail Test: Native Compilation Failure (Should Succeed)", + "Fail_wasm_compiling": "Fail Test: Wasm Compilation Failure (Should Succeed)" } # ---------------------------------------------------------------------- @@ -440,10 +446,82 @@ def run_compiled_wasm(wasm_file, timeout_sec=DEFAULT_TIMEOUT): # TODO: Currently for non deterministic cases, we are only compiling and running the test case, success means the compiled test case ran, need to add more specific tests # def test_single_file_unified(source_file, result, timeout_sec=DEFAULT_TIMEOUT, test_mode="deterministic"): - """Unified test function for both deterministic and non-deterministic tests""" + """Unified test function for both deterministic, non-deterministic and failing tests""" source_file = Path(source_file) handler = TestResultHandler(result, source_file) + # For fail tests, we need to run both native and wasm + if test_mode == "fail": + # Run native version + native_success, native_output, native_retcode, native_error = compile_and_run_native(source_file, timeout_sec) + + # NOTE: We explicitly early-abort here and report the native compilation failure + # rather than treating it as a successful "fail-test". + if native_error == "Failure_native_compiling": + # Record this specifically as a fail-test native-compilation error so it is + # counted alongside other `Fail_*` test categories instead of the generic + # compilation error bucket used elsewhere. + failure_info = ( + "=== FAILURE: Native compilation failed during fail-test (expected runtime failure) ===\n" + f"Native output:\n{native_output}" + ) + add_test_result(result, str(source_file), "Failure", "Fail_native_compiling", failure_info) + return + + # Compile and run WASM + wasm_file, wasm_compile_error = compile_c_to_wasm(source_file) + if wasm_file is None: + # Record this specifically as a fail-test WASM-compilation error so it is + # counted alongside other `Fail_*` test categories instead of the generic + # Lind_wasm_compiling bucket used elsewhere. + failure_info = ( + "=== FAILURE: Wasm compilation failed during fail-test (expected runtime failure) ===\n" + f"Wasm compile output:\n{wasm_compile_error}" + ) + add_test_result(result, str(source_file), "Failure", "Fail_wasm_compiling", failure_info) + return + + try: + wasm_retcode, wasm_output = run_compiled_wasm(wasm_file, timeout_sec) + + # Normalize return codes for comparison + native_failed = native_retcode != 0 + + # Check if wasm_retcode is an integer or string + if isinstance(wasm_retcode, str): + wasm_failed = wasm_retcode in ["timeout", "unknown_error"] # Explicitly check for failure strings + else: + wasm_failed = wasm_retcode != 0 + + # Both should fail for this test to pass + if native_failed and wasm_failed: + # Success: both failed as expected + output_info = ( + f"Native exit code: {native_retcode}\n" + f"Wasm exit code: {wasm_retcode}\n" + "Both failed as expected." + ) + handler.add_success(output_info) + elif not native_failed and not wasm_failed: + # Both succeeded when they should have failed + failure_info = build_fail_message("both", native_output, wasm_output, native_retcode, wasm_retcode) + add_test_result(result, str(source_file), "Failure", "Fail_both_succeeded", failure_info) + elif not native_failed: + # Only native succeeded + failure_info = build_fail_message("native_only", native_output, wasm_output, native_retcode, wasm_retcode) + add_test_result(result, str(source_file), "Failure", "Fail_native_succeeded", failure_info) + else: + # Only wasm succeeded + failure_info = build_fail_message("wasm_only", native_output, wasm_output, native_retcode, wasm_retcode) + add_test_result(result, str(source_file), "Failure", "Fail_wasm_succeeded", failure_info) + + finally: + # Always clean up WASM file + if wasm_file and wasm_file.exists(): + wasm_file.unlink() + + return # Exit early for fail tests + # For deterministic tests, get expected output expected_output = None if test_mode == "deterministic": @@ -453,9 +531,9 @@ def test_single_file_unified(source_file, result, timeout_sec=DEFAULT_TIMEOUT, t return # Compile and run WASM - wasm_file, compile_err = compile_c_to_wasm(source_file) + wasm_file, wasm_compile_error = compile_c_to_wasm(source_file) if wasm_file is None: - handler.add_compile_failure(compile_err) + handler.add_compile_failure(wasm_compile_error) return try: @@ -491,6 +569,9 @@ def test_single_file_deterministic(source_file, result, timeout_sec=DEFAULT_TIME def test_single_file_non_deterministic(source_file, result, timeout_sec=DEFAULT_TIMEOUT): test_single_file_unified(source_file, result, timeout_sec, "non_deterministic") +def test_single_file_fail(source_file, result, timeout_sec=DEFAULT_TIMEOUT): + test_single_file_unified(source_file, result, timeout_sec, "fail") + # ---------------------------------------------------------------------- # Function: analyze_testfile_dependencies # @@ -1141,9 +1222,50 @@ def run_tests(config, artifacts_root, results, timeout_sec): test_single_file_deterministic(dest_source, results["deterministic"], timeout_sec) elif parent_name == NON_DETERMINISTIC_PARENT_NAME: test_single_file_non_deterministic(dest_source, results["non_deterministic"], timeout_sec) + elif parent_name == FAIL_PARENT_NAME: + test_single_file_fail(dest_source, results["fail"], timeout_sec) else: - # Log warning for tests not in deterministic/non-deterministic folders - logger.warning(f"Test file {original_source} is not in a deterministic or non-deterministic folder - skipping") + # Log warning for tests not in deterministic/non-deterministic/fail folders + logger.warning(f"Test file {original_source} is not in a deterministic, non-deterministic, or fail folder - skipping") + +def build_fail_message(case: str, native_output: str, wasm_output: str, native_retcode=None, wasm_retcode=None) -> str: + """ + Build a consistent failure message for fail-tests. + + Args: + case: One of "both", "native_only", "wasm_only" describing which succeeded. + native_output: Captured native stdout/stderr text. + wasm_output: Captured wasm stdout/stderr text. + native_retcode: Native return code (optional, included where helpful). + wasm_retcode: Wasm return code (optional, included where helpful). + + Returns: + A formatted failure string. + """ + if case == "both": + return ( + "=== FAILURE: Both Native and Wasm succeeded when they should fail ===\n" + f"Native output:\n{native_output}\n\n" + f"Wasm output:\n{wasm_output}" + ) + elif case == "native_only": + return ( + "=== FAILURE: Native succeeded when it should fail ===\n" + f"Native output:\n{native_output}\n\n" + f"Wasm failed with exit code {wasm_retcode}:\n{wasm_output}" + ) + elif case == "wasm_only": + return ( + "=== FAILURE: Wasm succeeded when it should fail ===\n" + f"Wasm output:\n{wasm_output}\n\n" + f"Native failed with exit code {native_retcode}:\n{native_output}" + ) + else: + return ( + "=== FAILURE: Unexpected fail-test result ===\n" + f"Native (rc={native_retcode}) output:\n{native_output}\n\n" + f"Wasm (rc={wasm_retcode}) output:\n{wasm_output}" + ) def main(): os.chdir(LIND_WASM_BASE) @@ -1184,7 +1306,8 @@ def main(): results = { "deterministic": get_empty_result(), - "non_deterministic": get_empty_result() + "non_deterministic": get_empty_result(), + "fail": get_empty_result() } # Prepare artifacts root diff --git a/skip_test_cases.txt b/skip_test_cases.txt index 0bc1afbf6..516a74da8 100644 --- a/skip_test_cases.txt +++ b/skip_test_cases.txt @@ -6,8 +6,6 @@ file_tests/deterministic/popen.c file_tests/deterministic/creat_access.c file_tests/deterministic/readlinkat.c file_tests/non-deterministic/sc-writev.c -memory_tests/deterministic/mmap-negative1.c -memory_tests/deterministic/mmap-negative2.c memory_tests/deterministic/mmaptest.c networking_tests/deterministic/uds-getsockname.c networking_tests/deterministic/uds-nb-select.c @@ -23,5 +21,5 @@ process_tests/non-deterministic/fork.c process_tests/non-deterministic/template.c process_tests/non-deterministic/tls_test.c signal_tests/deterministic/sigpipe.c -signal_tests/deterministic/signal_resethand.c +signal_tests/fail/signal_resethand.c signal_tests/non-deterministic/signal_recursive.c diff --git a/tests/unit-tests/memory_tests/deterministic/mmap-negative1.c b/tests/unit-tests/memory_tests/fail/mmap-negative1.c similarity index 100% rename from tests/unit-tests/memory_tests/deterministic/mmap-negative1.c rename to tests/unit-tests/memory_tests/fail/mmap-negative1.c diff --git a/tests/unit-tests/memory_tests/deterministic/mmap-negative2.c b/tests/unit-tests/memory_tests/fail/mmap-negative2.c similarity index 100% rename from tests/unit-tests/memory_tests/deterministic/mmap-negative2.c rename to tests/unit-tests/memory_tests/fail/mmap-negative2.c diff --git a/tests/unit-tests/signal_tests/deterministic/signal_resethand.c b/tests/unit-tests/signal_tests/fail/signal_resethand.c similarity index 100% rename from tests/unit-tests/signal_tests/deterministic/signal_resethand.c rename to tests/unit-tests/signal_tests/fail/signal_resethand.c