diff --git a/source/autograder_platform/Executors/common.py b/source/autograder_platform/Executors/common.py index ddc0a86..dbbdab9 100644 --- a/source/autograder_platform/Executors/common.py +++ b/source/autograder_platform/Executors/common.py @@ -26,7 +26,7 @@ def filterStdOut(stdOut: Optional[List[str]]) -> Optional[List[str]]: filteredOutput: List[str] = [] for line in stdOut: if "output " in line.lower(): - filteredOutput.append(line[line.lower().find("output ") + 7:]) + filteredOutput.append(line[line.lower().find("output ") + 7:].strip()) return filteredOutput diff --git a/source/autograder_platform/TestingFramework/Assertions.py b/source/autograder_platform/TestingFramework/Assertions.py index fd37e0e..0b591fb 100644 --- a/source/autograder_platform/TestingFramework/Assertions.py +++ b/source/autograder_platform/TestingFramework/Assertions.py @@ -1,3 +1,4 @@ +import difflib import math import re import unittest @@ -11,6 +12,13 @@ class Assertions(unittest.TestCase): The primary differentiation factor of this is that it formats the outputs in a nicer way for both gradescope and the local autograder """ + RED_BG: str = u"\u001b[41m" + RED_COLOR: str = u"\u001b[31m" + GREEN_BG: str = u"\u001b[42m" + RESET_COLOR: str = u"\u001b[0m" + + DIFF_MAX_CHARACTERS = 200 + def __init__(self, testResults): super().__init__(testResults) self.addTypeEqualityFunc(str, self.assertMultiLineEqual) @@ -51,14 +59,65 @@ def _convertStringToList(outputLine: str) -> list[str]: @staticmethod def _raiseFailure(shortDescription: str, expectedObject: object, actualObject: object, msg: Optional[str]): - errorMsg = f"Incorrect {shortDescription}.\n" + \ - f"Expected {shortDescription}: {expectedObject}\n" + \ - f"Your {shortDescription} : {actualObject}" + errorMsg = f"Incorrect {shortDescription}.\n" + if expectedObject is not None and actualObject is not None and isinstance(expectedObject, str) and isinstance(actualObject, str): + diffLog = Assertions._highlightStringDifferences(expectedObject, actualObject) + errorMsg += f"Expected {shortDescription}: {expectedObject}\n" + errorMsg += f"Your {shortDescription} : {actualObject}\n" + errorMsg += f"Diff Log {shortDescription}: {diffLog}{Assertions.RED_COLOR}" + else: + errorMsg += f"Expected {shortDescription}: {expectedObject}\n" + \ + f"Your {shortDescription} : {actualObject}" if msg: errorMsg += "\n\n" + str(msg) raise AssertionError(errorMsg) + @staticmethod + def _highlightStringDifferences(expected: str, actual: str) -> str: + """Return diff log strings with differences highlighted.""" + RED_BG = Assertions.RED_BG + GREEN_BG = Assertions.GREEN_BG + RESET_COLOR = Assertions.RESET_COLOR + + matcher = difflib.SequenceMatcher(None, expected, actual) + + diffLog = [] + # maxChars = min(Assertions.DIFF_MAX_CHARACTERS, len(actual), len(expected)) + maxChars = Assertions.DIFF_MAX_CHARACTERS + visibleCount = 0 + + for tag, i1, i2, j1, j2 in matcher.get_opcodes(): + if visibleCount >= maxChars: + break + + if tag == 'equal': + for ch in expected[i1:i2]: + if visibleCount >= maxChars: + break + diffLog.append(f"{RESET_COLOR}{GREEN_BG}{ch}{RESET_COLOR}") + visibleCount += 1 + elif tag == 'replace': + for ch in actual[j1:j2]: + if visibleCount >= maxChars: + break + diffLog.append(f"{RESET_COLOR}{RED_BG}{ch}{RESET_COLOR}") + visibleCount += 1 + elif tag == 'delete': + for ch in expected[i1:i2]: + if visibleCount >= maxChars: + break + diffLog.append(f"{RESET_COLOR}{RED_BG}{ch}{RESET_COLOR}") + visibleCount += 1 + elif tag == 'insert': + for ch in actual[j1:j2]: + if visibleCount >= maxChars: + break + diffLog.append(f"{RESET_COLOR}{RED_BG}{ch}{RESET_COLOR}") + visibleCount += 1 + + return ''.join(diffLog) + @staticmethod def _convertIterableFromString(expected, actual): for i in range(len(expected)): @@ -82,9 +141,16 @@ def _convertIterableFromString(expected, actual): return actual def _assertIterableEqual(self, expected, actual, msg: Optional[str] = None): + errorMsg = msg if msg else None + for i in range(len(expected)): if expected[i] != actual[i]: - self._raiseFailure("output", expected[i], actual[i], msg) + if isinstance(expected[i], str): + errorMsg = f"Expected output line {i+1} does not match your output line {i+1}" + if msg: + errorMsg += f"\n\n" + str(msg) + + self._raiseFailure("output", expected[i], actual[i], errorMsg) @staticmethod def findPrecision(x: float): diff --git a/tests/platform_tests/testAssertions.py b/tests/platform_tests/testAssertions.py index 4d53acc..a1baa6f 100644 --- a/tests/platform_tests/testAssertions.py +++ b/tests/platform_tests/testAssertions.py @@ -1,3 +1,5 @@ +import re + from autograder_platform.TestingFramework.Assertions import Assertions @@ -49,6 +51,42 @@ def testAssertMultilineEqualFailure(self): with self.assertRaises(AssertionError): self.assertMultiLineEqual("this\nis\na\nof\nlines", "this\nis\na\nof\nline") + def testAssertMultilineEqualFailureDiffLog(self): + expectedMsg= "Diff Log output: \x1b[0m\x1b[42ma\x1b[0m\x1b[0m\x1b[42mb\x1b[0m\x1b[0m\x1b[41mX\x1b[0m\x1b[0m\x1b[42md\x1b[0m\x1b[0m\x1b[42me\x1b[0m\x1b[31m" + with self.assertRaises(AssertionError) as ex: + self.assertMultiLineEqual("abcde", "abXde") + + actualMsg = str(ex.exception) + self.assertIn(expectedMsg, actualMsg) + + def testAssertMultilineEqualFailureDiffLogTruncatesAt200Chars(self): + expected = "b" * 205 + actual = "a" * 205 + + with self.assertRaises(AssertionError) as ex: + self.assertMultiLineEqual(expected, actual) + + actualMsg = str(ex.exception) + match = re.search(r"Diff Log output: (.*)$", actualMsg, re.DOTALL) + self.assertIsNotNone(match) + + diff_log = re.sub(r"\x1b\[[0-9;]*m", "", match.group(1)) + self.assertEqual(len(diff_log), 200) + + def testAssertMultilineEqualFailureDiffLogTruncatesToActualLength(self): + expected = "b" * 100 + actual = "a" * 50 + + with self.assertRaises(AssertionError) as ex: + self.assertMultiLineEqual(expected, actual) + + actualMsg = str(ex.exception) + match = re.search(r"Diff Log output: (.*)$", actualMsg, re.DOTALL) + self.assertIsNotNone(match) + + diff_log = re.sub(r"\x1b\[[0-9;]*m", "", match.group(1)) + self.assertEqual(len(diff_log), 50) + def testAssertFailureWithMsg(self): expectedMsg = "doubles aren't ints" with self.assertRaises(AssertionError) as ex: