Add a "random" flag to randomize the test order (defaults to False).

Author: ntoll

README.md

@@ -64,19 +64,21 @@ those who use PyTest, when using PyScript.
 5. If a named `pattern` argument is provided, it will be used to match test
    modules in the specification for target directories. The default pattern is
    "test_*.py".
-6. If there is a `conftest.py` file in any of the specified directories
+6. If a named `random` boolean argument is provided (default: `False`), then
+   the order in which modules and tests are run will be randomized.
+7. If there is a `conftest.py` file in any of the specified directories
    containing a test module, it will be imported for any global `setup` and
    `teardown` functions to use for modules found within that directory. These
    `setup` and `teardown` functions can be overridden in the individual test
    modules.
-7. The `result` of awaiting `upytest.run` is a Python dictionary containing 
+8. The `result` of awaiting `upytest.run` is a Python dictionary containing 
    lists of tests bucketed under the keys: `"passes"`, `"fails"` and 
    `"skipped"`. The result also provides information about the Python
    interpreter used to run the tests, long with a boolean flag to indicate if
    the tests were running in a web worker. These results are JSON serializable
    and can be used for further processing and analysis (again, see `main.py`
    for an example of this in action.)
-8. In your `index.html` make sure you use the `terminal` attribute
+9. In your `index.html` make sure you use the `terminal` attribute
    when referencing your Python script (as in the `index.html` file in
    this repository):
    ```html

main.py

@@ -34,6 +34,11 @@
         "fails": 9,
         "skipped": 6,
     },
+    "result_random": {
+        "passes": 11,
+        "fails": 9,
+        "skipped": 6,
+    },
     "result_module": {
         "passes": 10,
         "fails": 9,
@@ -55,6 +60,9 @@
 # Run all tests in the tests directory.
 print("\033[1mRunning all tests in directory...\033[0m")
 actual_results["result_all"] = await upytest.run("./tests")
+# Run all tests in the tests directory in random order.
+print("\n\n\033[1mRunning all tests in directory in random order...\033[0m")
+actual_results["result_random"] = await upytest.run("./tests", random=True)
 # Run all tests in a specific module.
 print("\n\n\033[1mRunning all tests in a specific module...\033[0m")
 actual_results["result_module"] = await upytest.run(
@@ -90,6 +98,13 @@
                     test_status
                 ), f"Test {test["test_name"]} does not end with {test_status}"
 
+# Ensure the randomized tests are different from the non-randomized tests.
+for test_status in ["passes", "fails", "skipped"]:
+    assert (
+        actual_results["result_all"][test_status]
+        != actual_results["result_random"][test_status]
+    ), f"Randomized tests are the same as non-randomized tests for {test_status}"
+
 # Ensure the results are JSON serializable.
 import json
 check = json.dumps(actual_results)
@@ -106,6 +121,15 @@
                 f" Skipped: {len(actual_results['result_all']['skipped'])}.",
             ),
         ),
+        div(
+            p(
+                b("Randomized Tests: "),
+                f"Passes: {len(actual_results['result_all']['passes'])},"
+                f" Fails: {len(actual_results['result_all']['fails'])},"
+                f" Skipped: {len(actual_results['result_all']['skipped'])}.",
+                f" (Different order to the non-randomized 'All Tests').",
+            ),
+        ),
         div(
             p(
                 b("Tests in a Specified Module: "),

upytest.py

@@ -30,6 +30,7 @@
 import io
 import inspect
 import time
+import random
 from pathlib import Path
 import asyncio
 from pyscript import RUNNING_IN_WORKER
@@ -120,6 +121,28 @@ def parse_traceback_from_exception(ex):
     return "\n".join(result)
 
 
+def shuffle(a_list):
+    """
+    Shuffle a list, in place. 
+    
+    This function is needed because MicroPython does not have a random.shuffle
+    function.
+
+    It falls back to random.shuffle if using CPython, otherwise it uses a
+    simple implementation of the Fisher-Yates in-place shuffle algorithm.
+
+    Context: 
+    
+    https://stackoverflow.com/questions/73143243/are-there-any-alternatives-for-the-python-module-randoms-shuffle-function-in
+    """
+    if hasattr(random, "shuffle"):
+        random.shuffle(a_list)
+    else:
+        for i in range(len(a_list) - 1, 0, -1):
+            j = random.randrange(i+1)
+            a_list[i], a_list[j] = a_list[j], a_list[i]
+
+
 class TestCase:
     """
     Represents an individual test to run.
@@ -274,7 +297,7 @@ async def print(self, text):
                 await asyncio.sleep(0)
             print(text, end="", flush=True)
 
-    async def run(self):
+    async def run(self, randomize=False):
         """
         Run each TestCase instance for this module. If a setup or teardown
         exists, these will be evaluated immediately before and after the
@@ -284,6 +307,8 @@ async def run(self):
         for each skipped test.
         """
         print(f"\n{self.path}: ", end="")
+        if randomize:
+            shuffle(self._tests)
         for test_case in self.tests:
             if self.setup:
                 if is_awaitable(self.setup):
@@ -466,12 +491,16 @@ async def run(*args, **kwargs):
     print("Running in worker: \033[1m", RUNNING_IN_WORKER, "\033[0m")
     targets = []
     pattern = kwargs.get("pattern", "test_*.py")
+    randomize = kwargs.get("random", False)
+    print("Randomize test order: \033[1m", randomize, "\033[0m")
     for arg in args:
         if isinstance(arg, str):
             targets.append(arg)
         else:
             raise ValueError(f"Unexpected argument: {arg}")
     test_modules = discover(targets, pattern)
+    if randomize:
+        shuffle(test_modules)
     module_count = len(test_modules)
     test_count = sum([len(module.tests) for module in test_modules])
     print(
@@ -483,7 +512,7 @@ async def run(*args, **kwargs):
     passed_tests = []
     start = time.time()
     for module in test_modules:
-        await module.run()
+        await module.run(randomize)
         for test in module.tests:
             if test.status == FAIL:
                 failed_tests.append(test)
@@ -542,6 +571,7 @@ async def run(*args, **kwargs):
         "platform": sys.platform,
         "version": sys.version,
         "running_in_worker": RUNNING_IN_WORKER,
+        "randomize": randomize,
         "passes": [test.as_dict for test in passed_tests],
         "fails": [test.as_dict for test in failed_tests],
         "skipped": [test.as_dict for test in skipped_tests],