Added support for purposeful failing test cases

scripts/wasmtestreport.py

@@ -51,6 +51,7 @@
 TESTFILES_DST = LIND_ROOT / "testfiles"
 DETERMINISTIC_PARENT_NAME = "deterministic"
 NON_DETERMINISTIC_PARENT_NAME = "non-deterministic"
+FAIL_PARENT_NAME = "fail"
 EXPECTED_DIRECTORY = Path("./expected")
 SKIP_TESTS_FILE = "skip_test_cases.txt"
 
@@ -65,7 +66,12 @@
     "Lind_wasm_Segmentation_Fault": "Lind Wasm Segmentation Failure",
     "Lind_wasm_Timeout": "Timeout During Lind Wasm run",
     "Unknown_Failure": "Unknown Failure",
-    "Output_mismatch": "C Compiler and Wasm Output mismatch"
+    "Output_mismatch": "C Compiler and Wasm Output mismatch",
+    "Fail_native_succeeded": "Fail Test: Native Succeeded (Should Fail)",
+    "Fail_wasm_succeeded": "Fail Test: Wasm Succeeded (Should Fail)",
+    "Fail_both_succeeded": "Fail Test: Both Native and Wasm Succeeded (Should Fail)",
+    "Fail_native_compiling": "Fail Test: Native Compilation Failure (Should Succeed)",
+    "Fail_wasm_compiling": "Fail Test: Wasm Compilation Failure (Should Succeed)"
     }
 
 # ----------------------------------------------------------------------
@@ -440,10 +446,82 @@ def run_compiled_wasm(wasm_file, timeout_sec=DEFAULT_TIMEOUT):
 # TODO: Currently for non deterministic cases, we are only compiling and running the test case, success means the compiled test case ran, need to add more specific tests
 # 
 def test_single_file_unified(source_file, result, timeout_sec=DEFAULT_TIMEOUT, test_mode="deterministic"):
-    """Unified test function for both deterministic and non-deterministic tests"""
+    """Unified test function for both deterministic, non-deterministic and failing tests"""
     source_file = Path(source_file)
     handler = TestResultHandler(result, source_file)
 
+    # For fail tests, we need to run both native and wasm
+    if test_mode == "fail":
+        # Run native version
+        native_success, native_output, native_retcode, native_error = compile_and_run_native(source_file, timeout_sec)
+
+        # NOTE: We explicitly early-abort here and report the native compilation failure
+        # rather than treating it as a successful "fail-test".
+        if native_error == "Failure_native_compiling":
+            # Record this specifically as a fail-test native-compilation error so it is
+            # counted alongside other `Fail_*` test categories instead of the generic
+            # compilation error bucket used elsewhere.
+            failure_info = (
+                "=== FAILURE: Native compilation failed during fail-test (expected runtime failure) ===\n"
+                f"Native output:\n{native_output}"
+            )
+            add_test_result(result, str(source_file), "Failure", "Fail_native_compiling", failure_info)
+            return
+
+        # Compile and run WASM
+        wasm_file, wasm_compile_error = compile_c_to_wasm(source_file)
+        if wasm_file is None:
+            # Record this specifically as a fail-test WASM-compilation error so it is
+            # counted alongside other `Fail_*` test categories instead of the generic
+            # Lind_wasm_compiling bucket used elsewhere.
+            failure_info = (
+                "=== FAILURE: Wasm compilation failed during fail-test (expected runtime failure) ===\n"
+                f"Wasm compile output:\n{wasm_compile_error}"
+            )
+            add_test_result(result, str(source_file), "Failure", "Fail_wasm_compiling", failure_info)
+            return
+
+        try:
+            wasm_retcode, wasm_output = run_compiled_wasm(wasm_file, timeout_sec)
+
+            # Normalize return codes for comparison
+            native_failed = native_retcode != 0
+
+            # Check if wasm_retcode is an integer or string
+            if isinstance(wasm_retcode, str):
+                wasm_failed = wasm_retcode in ["timeout", "unknown_error"]  # Explicitly check for failure strings
+            else:
+                wasm_failed = wasm_retcode != 0
+
+            # Both should fail for this test to pass
+            if native_failed and wasm_failed:
+                # Success: both failed as expected
+                output_info = (
+                    f"Native exit code: {native_retcode}\n"
+                    f"Wasm exit code: {wasm_retcode}\n"
+                    "Both failed as expected."
+                )
+                handler.add_success(output_info)
+            elif not native_failed and not wasm_failed:
+                # Both succeeded when they should have failed
+                failure_info = build_fail_message("both", native_output, wasm_output, native_retcode, wasm_retcode)
+                add_test_result(result, str(source_file), "Failure", "Fail_both_succeeded", failure_info)
+            elif not native_failed:
+                # Only native succeeded
+                failure_info = build_fail_message("native_only", native_output, wasm_output, native_retcode, wasm_retcode)
+                add_test_result(result, str(source_file), "Failure", "Fail_native_succeeded", failure_info)
+            else:
+                # Only wasm succeeded
+                failure_info = build_fail_message("wasm_only", native_output, wasm_output, native_retcode, wasm_retcode)
+                add_test_result(result, str(source_file), "Failure", "Fail_wasm_succeeded", failure_info)
+
+        finally:
+            # Always clean up WASM file
+            if wasm_file and wasm_file.exists():
+                wasm_file.unlink()
+
+        return  # Exit early for fail tests
+
     # For deterministic tests, get expected output
     expected_output = None
     if test_mode == "deterministic":
@@ -453,9 +531,9 @@ def test_single_file_unified(source_file, result, timeout_sec=DEFAULT_TIMEOUT, t
             return
 
     # Compile and run WASM
-    wasm_file, compile_err = compile_c_to_wasm(source_file)
+    wasm_file, wasm_compile_error = compile_c_to_wasm(source_file)
     if wasm_file is None:
-        handler.add_compile_failure(compile_err)
+        handler.add_compile_failure(wasm_compile_error)
         return
 
     try:
@@ -491,6 +569,9 @@ def test_single_file_deterministic(source_file, result, timeout_sec=DEFAULT_TIME
 def test_single_file_non_deterministic(source_file, result, timeout_sec=DEFAULT_TIMEOUT):
     test_single_file_unified(source_file, result, timeout_sec, "non_deterministic")
 
+def test_single_file_fail(source_file, result, timeout_sec=DEFAULT_TIMEOUT):
+    test_single_file_unified(source_file, result, timeout_sec, "fail")
+
 # ----------------------------------------------------------------------
 # Function: analyze_testfile_dependencies
 #
@@ -1141,9 +1222,50 @@ def run_tests(config, artifacts_root, results, timeout_sec):
             test_single_file_deterministic(dest_source, results["deterministic"], timeout_sec)
         elif parent_name == NON_DETERMINISTIC_PARENT_NAME:
             test_single_file_non_deterministic(dest_source, results["non_deterministic"], timeout_sec)
+        elif parent_name == FAIL_PARENT_NAME:
+            test_single_file_fail(dest_source, results["fail"], timeout_sec)
         else:
-            # Log warning for tests not in deterministic/non-deterministic folders
-            logger.warning(f"Test file {original_source} is not in a deterministic or non-deterministic folder - skipping")
+            # Log warning for tests not in deterministic/non-deterministic/fail folders
+            logger.warning(f"Test file {original_source} is not in a deterministic, non-deterministic, or fail folder - skipping")
+
+def build_fail_message(case: str, native_output: str, wasm_output: str, native_retcode=None, wasm_retcode=None) -> str:
+    """
+    Build a consistent failure message for fail-tests.
+
+    Args:
+        case: One of "both", "native_only", "wasm_only" describing which succeeded.
+        native_output: Captured native stdout/stderr text.
+        wasm_output: Captured wasm stdout/stderr text.
+        native_retcode: Native return code (optional, included where helpful).
+        wasm_retcode: Wasm return code (optional, included where helpful).
+
+    Returns:
+        A formatted failure string.
+    """
+    if case == "both":
+        return (
+            "=== FAILURE: Both Native and Wasm succeeded when they should fail ===\n"
+            f"Native output:\n{native_output}\n\n"
+            f"Wasm output:\n{wasm_output}"
+        )
+    elif case == "native_only":
+        return (
+            "=== FAILURE: Native succeeded when it should fail ===\n"
+            f"Native output:\n{native_output}\n\n"
+            f"Wasm failed with exit code {wasm_retcode}:\n{wasm_output}"
+        )
+    elif case == "wasm_only":
+        return (
+            "=== FAILURE: Wasm succeeded when it should fail ===\n"
+            f"Wasm output:\n{wasm_output}\n\n"
+            f"Native failed with exit code {native_retcode}:\n{native_output}"
+        )
+    else:
+        return (
+            "=== FAILURE: Unexpected fail-test result ===\n"
+            f"Native (rc={native_retcode}) output:\n{native_output}\n\n"
+            f"Wasm (rc={wasm_retcode}) output:\n{wasm_output}"
+        )
 
 def main():
     os.chdir(LIND_WASM_BASE)
@@ -1184,7 +1306,8 @@ def main():
 
     results = {
         "deterministic": get_empty_result(),
-        "non_deterministic": get_empty_result()
+        "non_deterministic": get_empty_result(),
+        "fail": get_empty_result()
     }
 
     # Prepare artifacts root

skip_test_cases.txt

@@ -6,8 +6,6 @@ file_tests/deterministic/popen.c
 file_tests/deterministic/creat_access.c
 file_tests/deterministic/readlinkat.c
 file_tests/non-deterministic/sc-writev.c
-memory_tests/deterministic/mmap-negative1.c
-memory_tests/deterministic/mmap-negative2.c
 memory_tests/deterministic/mmaptest.c
 networking_tests/deterministic/uds-getsockname.c
 networking_tests/deterministic/uds-nb-select.c
@@ -23,5 +21,5 @@ process_tests/non-deterministic/fork.c
 process_tests/non-deterministic/template.c
 process_tests/non-deterministic/tls_test.c
 signal_tests/deterministic/sigpipe.c
-signal_tests/deterministic/signal_resethand.c
+signal_tests/fail/signal_resethand.c
 signal_tests/non-deterministic/signal_recursive.c