add initial --harness-validation flag. todo: verify behavior when to …

…make flaky.

Signed-off-by: Pineda <[email protected]>

test/common.py

@@ -67,6 +67,10 @@ def pytest_addoption(parser):
     parser.addoption(
         "--password", default="analog", help="SSH login password",
     )
+    parser.addoption(
+        "--harness-validation",action="store_true",
+        help="Enable results validation for failing tests.",
+    )
 
 
 def pytest_runtest_setup(item):
@@ -93,6 +97,10 @@ def pytest_runtest_setup(item):
         pytest.skip("CMOS testing disabled. Use --cmos flag to enable")
     elif not lvds and "lvds_test" in marks:
         pytest.skip("LVDS testing disabled. Use --lvds flag to enable")
+
+    # Test harness validation
+    if item.config.getoption("--harness-validation"):
+        item.config.option.reruns = 1
 
 
 def pytest_generate_tests(metafunc):

test/conftest.py

@@ -22,6 +22,20 @@
 
 import adi
 
+import yaml
+
+KNOWN_FAILING_FILE = os.path.join(os.path.dirname(__file__), "test-harness-failures.yaml")
+
+def load_known_failing():
+    """Load known failing tests from YAML."""
+    with open(KNOWN_FAILING_FILE, "r") as f:
+        return yaml.safe_load(f) or {}
+
+def is_known_failing(test_name, hardware):
+    """Check if a test is in the known failing list."""
+    known_failures = load_known_failing()
+    return hardware in known_failures.get(test_name, [])
+
 try:
     from test.scpi import dcxo_calibrate
 
@@ -37,6 +51,21 @@ def pytest_runtest_makereport(item, call):
     """
 
     if call.when == "call" and call.excinfo is not None:
+        if item.config.getoption("--harness-validation"):  # Only run if flag is enabled
+            hardware = item.config.getoption("--hardware", default="default_hardware")
+            test_name = item.name
+
+        # Check if the test is known failing
+        print(f"Test reruns left:'{item.config.option.reruns}'")
+        if is_known_failing(test_name, hardware):
+            item.add_marker("flaky")
+            print(f"Test '{test_name}' on '{hardware}' is known failing. Do not reboot or rerun.")
+            item.add_marker(pytest.mark.skip(reason=f"Test '{test_name}' on '{hardware}' is known failing. Do not reboot or rerun."))
+        else:
+            print(f"Test '{test_name}' on '{hardware}' is a new failure. Reboot then rerun to validate.")
+
+
+
         # Extract error type and message
         exception_type_and_message_formatted = call.excinfo.exconly() or "N/A"
         item.user_properties.append(
@@ -317,3 +346,13 @@ def test_verify_links(request):
 @pytest.fixture()
 def test_verify_links_errors_stable(request):
     yield verify_links_errors_stable
+
+#########################################
+# Test Harness Fixtures
+
+
+@pytest.fixture
+def validation_enabled(request):
+    """Fixture to check if validation is enabled."""
+    return request.config.getoption("--harness-validation")
+

test/test-harness-failures.yaml

@@ -0,0 +1,4 @@
+# Pyadi-iio failing tests on specific platforms
+test_ad9361_known_failing[classname=adi.ad9361]:
+  - zynq_zc702_adv7511_ad9361_fmcomms2_3
+  - default_hardware