Changed sort order for concept exercises with failed subtests and rerecoreded golden output for them, (#122)

Author: BethanyG

Diff for: runner/data.py

@@ -153,15 +153,31 @@ def _factory(items):
 
     def as_json(self):
         """
-        Trim off the TestClass name and test_ prefix from each test_name.
-        Replace underscores with spaces for more human-readable strings.
-        Sort the current tests array by task_id and then Dump all results to formatted JSON.
+        -  Trim off the TestClass name and test_ prefix from each test_name.
+        - Replace underscores with spaces for more human-readable strings.
+        - Add a sort order signifier (~) to parent tests with subtests.
+         (~) ensures parent tests sort to last position.
+
+        - If it is a concept exercise, sort the current tests array by
+          task_id and variation# then Dump all results to formatted JSON.
         """
+
+
         trim_name = compile(r'^(.+)(Test\.test_)')
         results = asdict(self, dict_factory=self._factory)
+        concept_exercise = False
 
         for item in results["tests"]:
-            item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ')
 
-        results["tests"] = sorted(results["tests"], key= lambda item: item["task_id"])
+            if "[variation" not in item["name"] and item["task_id"] > 0:
+                item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ') + "~"
+                concept_exercise = True
+            else:
+                item["name"] = sub(trim_name, '\\1 > ', item["name"]).replace('_', ' ')
+
+        if concept_exercise:
+          results["tests"] = sorted(results["tests"], key= lambda item: (item["task_id"], item["name"]))
+        else:
+          results["tests"] = sorted(results["tests"], key=lambda item: (item["task_id"]))
+
         return dumps(results, indent=2)

Diff for: test/example-all-fail-tasks-and-subtests/results.json

@@ -2,48 +2,6 @@
   "version": 3,
   "status": "fail",
   "tests": [
-    {
-      "name": "ExampleAllFail > abc",
-      "status": "fail",
-      "message": "One or more variations of this test failed. Details can be found under each [variant#].",
-      "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
-    {
-      "name": "ExampleAllFail > hello",
-      "status": "fail",
-      "message": "One or more variations of this test failed. Details can be found under each [variant#].",
-      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
-    {
-      "name": "ExampleAllFail > hello [variation #1]",
-      "status": "fail",
-      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.",
-      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
-    {
-      "name": "ExampleAllFail > hello [variation #2]",
-      "status": "fail",
-      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.",
-      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
-    {
-      "name": "ExampleAllFail > hello [variation #3]",
-      "status": "fail",
-      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.",
-      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
-    {
-      "name": "ExampleAllFail > hello [variation #4]",
-      "status": "fail",
-      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.",
-      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 1
-    },
     {
       "name": "ExampleAllFail > abc [variation #1]",
       "status": "fail",
@@ -101,18 +59,46 @@
       "task_id": 1
     },
     {
-      "name": "ExampleAllFailOther > dummy",
+      "name": "ExampleAllFail > abc~",
       "status": "fail",
       "message": "One or more variations of this test failed. Details can be found under each [variant#].",
       "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 2
+      "task_id": 1
+    },
+    {
+      "name": "ExampleAllFail > hello [variation #1]",
+      "status": "fail",
+      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 15) : Expected: ('Hello, World!', 15) but got something else instead.",
+      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 1
+    },
+    {
+      "name": "ExampleAllFail > hello [variation #2]",
+      "status": "fail",
+      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 23) : Expected: ('Hello, World!', 23) but got something else instead.",
+      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 1
     },
     {
-      "name": "ExampleAllFailOther > hello",
+      "name": "ExampleAllFail > hello [variation #3]",
+      "status": "fail",
+      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 33) : Expected: ('Hello, World!', 33) but got something else instead.",
+      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 1
+    },
+    {
+      "name": "ExampleAllFail > hello [variation #4]",
+      "status": "fail",
+      "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 39) : Expected: ('Hello, World!', 39) but got something else instead.",
+      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 1
+    },
+    {
+      "name": "ExampleAllFail > hello~",
       "status": "fail",
       "message": "One or more variations of this test failed. Details can be found under each [variant#].",
-      "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
-      "task_id": 2
+      "test_code": "input_data = [15, 23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 1
     },
     {
       "name": "ExampleAllFailOther > dummy [variation #1]",
@@ -170,6 +156,13 @@
       "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
       "task_id": 2
     },
+    {
+      "name": "ExampleAllFailOther > dummy~",
+      "status": "fail",
+      "message": "One or more variations of this test failed. Details can be found under each [variant#].",
+      "test_code": "input_data = ['frog', 'fish', 'coconut', 'pineapple', 'carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 2
+    },
     {
       "name": "ExampleAllFailOther > hello [variation #1]",
       "status": "fail",
@@ -197,6 +190,13 @@
       "message": "AssertionError: 'Goodbye!' != ('Hello, World!', 10) : Expected: ('Hello, World!', 10) but got something else instead.",
       "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
       "task_id": 2
+    },
+    {
+      "name": "ExampleAllFailOther > hello~",
+      "status": "fail",
+      "message": "One or more variations of this test failed. Details can be found under each [variant#].",
+      "test_code": "input_data = [1, 2, 5, 10]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\n\nfor variant, (param, result) in enumerate(zip(input_data, result_data), start=1):\n    failure_msg=f'Expected: {result} but got something else instead.'\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result, msg=failure_msg)",
+      "task_id": 2
     }
   ]
 }

Diff for: test/example-config-multiple-files-subtests-and-tasks/results.json

@@ -3,56 +3,56 @@
   "status": "pass",
   "tests": [
     {
-      "name": "ExampleFirst > abc",
+      "name": "ExampleFirst > abc~",
       "status": "pass",
       "test_code": "self.assertEqual(hello(13), (\"Hello, World!\", 13))",
       "task_id": 1,
       "output": "User output is captured! 13"
     },
     {
-      "name": "ExampleFirst > hello",
+      "name": "ExampleFirst > hello~",
       "status": "pass",
       "test_code": "self.assertEqual(hello('Hi'), (\"Hello, World!\", 'Hi'))",
       "task_id": 1,
       "output": "User output is captured! Hi"
     },
     {
-      "name": "ExampleFirstOther > dummy",
+      "name": "ExampleFirstOther > dummy~",
       "status": "pass",
       "test_code": "self.assertEqual(hello('Banana'), (\"Hello, World!\", \"Banana\"))",
       "task_id": 2,
       "output": "User output is captured! Banana"
     },
     {
-      "name": "ExampleFirstOther > hello",
+      "name": "ExampleFirstOther > hello~",
       "status": "pass",
       "test_code": "self.assertEqual(hello(42), (\"Hello, World!\", 42))",
       "task_id": 2,
       "output": "User output is captured! 42"
     },
     {
-      "name": "ExampleSecond > abc",
+      "name": "ExampleSecond > abc~",
       "status": "pass",
       "test_code": "input_data = ['carrot', 'cucumber', 'grass', 'tree']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result,\n                         msg=f'Expected: {result} but got something else instead.')",
       "task_id": 3,
       "output": "User output is captured! carrot\nUser output is captured! cucumber\nUser output is captured! grass\nUser output is captured! tree"
     },
     {
-      "name": "ExampleSecond > hello",
+      "name": "ExampleSecond > hello~",
       "status": "pass",
       "test_code": "input_data = [1, 2, 5]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result,\n                         msg=f'Expected: {result} but got something else instead.')",
       "task_id": 3,
       "output": "User output is captured! 1\nUser output is captured! 2\nUser output is captured! 5"
     },
     {
-      "name": "ExampleSecondOther > dummy",
+      "name": "ExampleSecondOther > dummy~",
       "status": "pass",
       "test_code": "input_data = ['frog', 'fish', 'coconut']\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result,\n                         msg=f'Expected: {result} but got something else instead.')",
       "task_id": 4,
       "output": "User output is captured! frog\nUser output is captured! fish\nUser output is captured! coconut"
     },
     {
-      "name": "ExampleSecondOther > hello",
+      "name": "ExampleSecondOther > hello~",
       "status": "pass",
       "test_code": "input_data = [23, 33, 39]\nresult_data = [(\"Hello, World!\", param) for param in input_data]\nnumber_of_variants = range(1, len(input_data) + 1)\n\nfor variant, param, result in zip(number_of_variants, input_data, result_data):\n    with self.subTest(f\"variation #{variant}\", param=param, result=result):\n        self.assertEqual(hello(param), result,\n                         msg=f'Expected: {result} but got something else instead.')",
       "task_id": 4,

Diff for: test/example-has-stdout-and-tasks/results.json

@@ -3,39 +3,39 @@
   "status": "fail",
   "tests": [
     {
-      "name": "ExampleHasStdoutAndTasks > hello",
+      "name": "ExampleHasStdoutAndTasks > hello~",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
       "task_id": 1,
       "output": "Hello, World!"
     },
     {
-      "name": "ExampleHasStdoutAndTasks > abc",
+      "name": "ExampleHasStdoutAndTasks > abc~",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
       "task_id": 2,
       "output": "Hello, World!"
     },
     {
-      "name": "ExampleHasStdoutAndTasks > truncation",
+      "name": "ExampleHasStdoutAndTasks > truncation~",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
       "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")",
       "task_id": 3,
       "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]"
     },
     {
-      "name": "ExampleHasStdoutAndTasksOther > dummy",
+      "name": "ExampleHasStdoutAndTasksOther > dummy~",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
       "task_id": 4,
       "output": "Hello, World!"
     },
     {
-      "name": "ExampleHasStdoutAndTasksOther > hello",
+      "name": "ExampleHasStdoutAndTasksOther > hello~",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",