Skip to content

Fix tests under capa/tests #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
irtazaakram merged 2 commits into from
Aug 19, 2025
Merged

Fix tests under capa/tests #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
697ab72
fix: fix quality tests
salman2013 Aug 18, 2025
73e9a8f
fix: fix the comments as per review
salman2013 Aug 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ allowlist_externals =
commands =
pip install -e .
mkdir -p var
pytest {posargs}
# pytest {posargs}
pytest xblocks_contrib/problem/capa/tests {posargs}

[testenv:docs]
setenv =
Expand Down Expand Up @@ -78,8 +79,8 @@ allowlist_externals =
deps =
-r{toxinidir}/requirements/quality.txt
commands =
pylint xblocks_contrib
pycodestyle xblocks_contrib
pydocstyle xblocks_contrib
isort --check-only --diff xblocks_contrib
pylint xblocks_contrib/problem/capa/tests
pycodestyle xblocks_contrib/problem/capa/tests
pydocstyle xblocks_contrib/problem/capa/tests
isort --check-only --diff xblocks_contrib/problem/capa/tests
make selfcheck
494 changes: 246 additions & 248 deletions xblocks_contrib/problem/capa/capa_problem.py
View file
Open in desktop

Large diffs are not rendered by default.

115 changes: 63 additions & 52 deletions xblocks_contrib/problem/capa/checker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@
from xblocks_contrib.problem.capa.capa_problem import LoncapaProblem

logging.basicConfig(format="%(levelname)s %(message)s")
log = logging.getLogger('capa.checker')
log = logging.getLogger("capa.checker")


class DemoSystem(object): # lint-amnesty, pylint: disable=missing-class-docstring
class DemoSystem: # pylint: disable=missing-class-docstring
def __init__(self):
self.lookup = TemplateLookup(directories=[path(__file__).dirname() / 'templates'])
self.lookup = TemplateLookup(directories=[path(__file__).dirname() / "templates"])
self.DEBUG = True

def render_template(self, template_filename, dictionary):
Expand All @@ -32,32 +32,39 @@ def render_template(self, template_filename, dictionary):


def main(): # lint-amnesty, pylint: disable=missing-function-docstring
parser = argparse.ArgumentParser(description='Check Problem Files')
parser.add_argument("command", choices=['test', 'show']) # Watch? Render? Open?
parser.add_argument("files", nargs="+", type=argparse.FileType('r'))
parser = argparse.ArgumentParser(description="Check Problem Files")
parser.add_argument("command", choices=["test", "show"]) # Watch? Render? Open?
parser.add_argument("files", nargs="+", type=argparse.FileType("r"))
parser.add_argument("--seed", required=False, type=int)
parser.add_argument("--log-level", required=False, default="INFO",
choices=['info', 'debug', 'warn', 'error',
'INFO', 'DEBUG', 'WARN', 'ERROR'])
parser.add_argument(
"--log-level",
required=False,
default="INFO",
choices=["info", "debug", "warn", "error", "INFO", "DEBUG", "WARN", "ERROR"],
)

args = parser.parse_args()
log.setLevel(args.log_level.upper())

system = DemoSystem()

for problem_file in args.files:
log.info("Opening {0}".format(problem_file.name))
log.info("Opening {0}".format(problem_file.name)) # pylint: disable=logging-format-interpolation

try:
problem = LoncapaProblem(problem_file, "fakeid", seed=args.seed, system=system) # lint-amnesty, pylint: disable=no-value-for-parameter, unexpected-keyword-arg
problem = LoncapaProblem(
problem_file, "fakeid", seed=args.seed, system=system
) # lint-amnesty, pylint: disable=no-value-for-parameter, unexpected-keyword-arg
except Exception as ex: # lint-amnesty, pylint: disable=broad-except
log.error("Could not parse file {0}".format(problem_file.name))
log.error(
"Could not parse file {0}".format(problem_file.name)
) # pylint: disable=logging-format-interpolation
log.exception(ex)
continue

if args.command == 'test':
if args.command == "test":
command_test(problem)
elif args.command == 'show':
elif args.command == "show":
command_show(problem)

problem_file.close()
Expand All @@ -80,10 +87,8 @@ def command_test(problem): # lint-amnesty, pylint: disable=missing-function-doc
check_that_suggested_answers_work(problem)
check_that_blanks_fail(problem)

log_captured_output(sys.stdout,
"captured stdout from {0}".format(problem))
log_captured_output(sys.stderr,
"captured stderr from {0}".format(problem))
log_captured_output(sys.stdout, "captured stdout from {0}".format(problem))
log_captured_output(sys.stderr, "captured stderr from {0}".format(problem))
except Exception as e: # lint-amnesty, pylint: disable=broad-except
log.exception(e)
finally:
Expand All @@ -92,69 +97,75 @@ def command_test(problem): # lint-amnesty, pylint: disable=missing-function-doc

def check_that_blanks_fail(problem):
"""Leaving it blank should never work. Neither should a space."""
blank_answers = dict((answer_id, "")
for answer_id in problem.get_question_answers())
blank_answers = dict((answer_id, "") for answer_id in problem.get_question_answers())
grading_results = problem.grade_answers(blank_answers)
try:
assert all(result == 'incorrect' for result in grading_results.values())
assert all(result == "incorrect" for result in grading_results.values())
except AssertionError:
log.error("Blank accepted as correct answer in {0} for {1}"
.format(problem,
[answer_id for answer_id, result
in sorted(grading_results.items())
if result != 'incorrect']))
# pylint: disable=logging-format-interpolation
log.error(
"Blank accepted as correct answer in {0} for {1}".format(
problem, [answer_id for answer_id, result in sorted(grading_results.items()) if result != "incorrect"]
)
)


def check_that_suggested_answers_work(problem):
"""Split this up so that we're only used for formula/numeric answers.

Examples of where this fails:
* Displayed answers use units but acceptable ones do not.
- L1e0.xml
- Presents itself as UndefinedVariable (when it tries to pass to calc)
* "a or d" is what's displayed, but only "a" or "d" is accepted, not the
string "a or d".
- L1-e00.xml
"""
Split this up so that we're only used for formula/numeric answers.

Examples of where this fails::

* Displayed answers use units but acceptable ones do not.
- L1e0.xml
- Presents itself as UndefinedVariable (when it tries to pass to calc)

* "a or d" is what's displayed, but only "a" or "d" is accepted, not the
string "a or d".
- L1-e00.xml
"""
# These are actual answers we get from the responsetypes
real_answers = problem.get_question_answers()

# all_answers is real_answers + blanks for other answer_ids for which the
# responsetypes can't provide us pre-canned answers (customresponse)
all_answer_ids = problem.get_answer_ids()
all_answers = dict((answer_id, real_answers.get(answer_id, ""))
for answer_id in all_answer_ids)
all_answers = dict((answer_id, real_answers.get(answer_id, "")) for answer_id in all_answer_ids)

log.debug("Real answers: {0}".format(real_answers))
log.debug("Real answers: {0}".format(real_answers)) # pylint: disable=logging-format-interpolation
if real_answers:
try:
real_results = dict((answer_id, result) for answer_id, result
in problem.grade_answers(all_answers).items()
if answer_id in real_answers)
real_results = dict(
(answer_id, result)
for answer_id, result in problem.grade_answers(all_answers).items()
if answer_id in real_answers
)
log.debug(real_results)
assert(all(result == 'correct'
for answer_id, result in real_results.items()))
assert all(result == "correct" for answer_id, result in real_results.items())
except UndefinedVariable as uv_exc:
log.error("The variable \"{0}\" specified in the ".format(uv_exc) + # lint-amnesty, pylint: disable=logging-not-lazy
"solution isn't recognized (is it a units measure?).")
log.error(
'The variable "{0}" specified in the '.format(uv_exc) # lint-amnesty, pylint: disable=logging-not-lazy
+ "solution isn't recognized (is it a units measure?)."
)
except AssertionError:
log.error("The following generated answers were not accepted for {0}:"
.format(problem))
# pylint: disable=logging-format-interpolation
log.error("The following generated answers were not accepted for {0}:".format(problem))
for question_id, result in sorted(real_results.items()):
if result != 'correct':
if result != "correct":
# pylint: disable=logging-format-interpolation
log.error(" {0} = {1}".format(question_id, real_answers[question_id]))
except Exception as ex: # lint-amnesty, pylint: disable=broad-except
log.error("Uncaught error in {0}".format(problem))
log.error("Uncaught error in {0}".format(problem)) # pylint: disable=logging-format-interpolation
log.exception(ex)


def log_captured_output(output_stream, stream_name): # lint-amnesty, pylint: disable=missing-function-docstring
output_stream.seek(0)
output_text = output_stream.read()
if output_text:
log.info("##### Begin {0} #####\n".format(stream_name) + output_text) # lint-amnesty, pylint: disable=logging-not-lazy
log.info("##### End {0} #####".format(stream_name))
log.info("##### Begin {0} #####\n".format(stream_name) + output_text)
log.info("##### End {0} #####".format(stream_name)) # pylint: disable=logging-format-interpolation


if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())
62 changes: 31 additions & 31 deletions xblocks_contrib/problem/capa/correctmap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
# Used by responsetypes and capa_problem


class CorrectMap(object):
class CorrectMap:
"""
Stores map between answer_id and response evaluation result for each question
in a capa problem. The response evaluation result for each answer_id includes
Expand Down Expand Up @@ -39,13 +39,13 @@ def __iter__(self):
return self.cmap.__iter__()

# See the documentation for 'set_dict' for the use of kwargs
def set( # lint-amnesty, pylint: disable=missing-function-docstring
def set( # lint-amnesty, pylint: disable=missing-function-docstring,disable=too-many-positional-arguments
self, # lint-amnesty, pylint: disable=unused-argument
answer_id=None,
correctness=None,
npoints=None,
msg='',
hint='',
msg="",
hint="",
hintmode=None,
queuestate=None,
answervariable=None,
Expand All @@ -54,13 +54,13 @@ def set( # lint-amnesty, pylint: disable=missing-function-docstring

if answer_id is not None:
self.cmap[answer_id] = {
'correctness': correctness,
'npoints': npoints,
'msg': msg,
'hint': hint,
'hintmode': hintmode,
'queuestate': queuestate,
'answervariable': answervariable,
"correctness": correctness,
"npoints": npoints,
"msg": msg,
"hint": hint,
"hintmode": hintmode,
"queuestate": queuestate,
"answervariable": answervariable,
}

def __repr__(self):
Expand Down Expand Up @@ -111,7 +111,7 @@ def is_correct(self, answer_id):
Returns true if the problem is correct OR partially correct.
"""
if answer_id in self.cmap:
return self.cmap[answer_id]['correctness'] in ['correct', 'partially-correct']
return self.cmap[answer_id]["correctness"] in ["correct", "partially-correct"]
return None

def is_partially_correct(self, answer_id):
Expand All @@ -120,24 +120,24 @@ def is_partially_correct(self, answer_id):
Returns true if the problem is partially correct.
"""
if answer_id in self.cmap:
return self.cmap[answer_id]['correctness'] == 'partially-correct'
return self.cmap[answer_id]["correctness"] == "partially-correct"
return None

def is_queued(self, answer_id):
return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
return answer_id in self.cmap and self.cmap[answer_id]["queuestate"] is not None

def is_right_queuekey(self, answer_id, test_key):
return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
return self.is_queued(answer_id) and self.cmap[answer_id]["queuestate"]["key"] == test_key

def get_queuetime_str(self, answer_id):
if self.cmap[answer_id]['queuestate']:
return self.cmap[answer_id]['queuestate']['time']
if self.cmap[answer_id]["queuestate"]:
return self.cmap[answer_id]["queuestate"]["time"]
else:
return None

def get_npoints(self, answer_id):
"""Return the number of points for an answer, used for partial credit."""
npoints = self.get_property(answer_id, 'npoints')
npoints = self.get_property(answer_id, "npoints")
if npoints is not None:
return npoints
elif self.is_correct(answer_id):
Expand All @@ -157,40 +157,40 @@ def get_property(self, answer_id, property, default=None): # lint-amnesty, pyli
return default

def get_correctness(self, answer_id):
return self.get_property(answer_id, 'correctness')
return self.get_property(answer_id, "correctness")

def get_msg(self, answer_id):
return self.get_property(answer_id, 'msg', '')
return self.get_property(answer_id, "msg", "")

def get_hint(self, answer_id):
return self.get_property(answer_id, 'hint', '')
return self.get_property(answer_id, "hint", "")

def get_hintmode(self, answer_id):
return self.get_property(answer_id, 'hintmode', None)
return self.get_property(answer_id, "hintmode", None)

def set_hint_and_mode(self, answer_id, hint, hintmode):
"""
- hint : (string) HTML text for hint
- hintmode : (string) mode for hint display ('always' or 'on_request')
- hint : (string) HTML text for hint
- hintmode : (string) mode for hint display ('always' or 'on_request')
"""
self.set_property(answer_id, 'hint', hint)
self.set_property(answer_id, 'hintmode', hintmode)
self.set_property(answer_id, "hint", hint)
self.set_property(answer_id, "hintmode", hintmode)

def update(self, other_cmap):
"""
Update this CorrectMap with the contents of another CorrectMap
"""
if not isinstance(other_cmap, CorrectMap):
raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
raise Exception("CorrectMap.update called with invalid argument %s" % other_cmap)
self.cmap.update(other_cmap.get_dict())
self.set_overall_message(other_cmap.get_overall_message())

def set_overall_message(self, message_str):
""" Set a message that applies to the question as a whole,
rather than to individual inputs. """
"""Set a message that applies to the question as a whole,
rather than to individual inputs."""
self.overall_message = str(message_str) if message_str else ""

def get_overall_message(self):
""" Retrieve a message that applies to the question as a whole.
If no message is available, returns the empty string """
"""Retrieve a message that applies to the question as a whole.
If no message is available, returns the empty string"""
return self.overall_message
Loading