import os, sys, json, csv, re, math
import matplotlib.pyplot as plt
from collections import namedtuple
Answer = namedtuple(“Answer”, [“question”, “type”, “value”, “notes”])
def read_code_cells(ipynb, default_notes={}):
answers = []
with open(ipynb) as f:
nb = json.load(f)
cells = nb[“cells”]
expected_exec_count = 1
for cell in cells:
if “execution_count” in cell and cell[“execution_count”]:
exec_count = cell[“execution_count”]
if exec_count != expected_exec_count:
raise Exception(f”Expected execution count {expected_exec_count} but found {exec_count}. Please do Restart & Run all then save before running the tester.”)
expected_exec_count = exec_count + 1
if cell[“cell_type”] != “code”:
continue
if not cell[“source”]:
continue
m = re.match(r”#[qQ](\d+)(.*)”, cell[“source”][0].strip())
if not m:
continue
qnum = int(m.group(1))
notes = m.group(2).strip()
config = parse_question_config(notes)
if “run” in config:
exec(config[“run”])
print(f”Reading Question {qnum}”)
if qnum in [a.question for a in answers]:
raise Exception(f”Answer {qnum} repeated!”)
expected = max([0] + [a.question for a in answers]) + 1
if qnum != expected:
print(f”Warning: Expected question {expected} next but found {qnum}!”)
# plots are display_data, so try to find those before looking for regular cell outputs
outputs = [o for o in cell[“outputs”]
if o.get(“output_type”) == “display_data”]
if len(outputs) == 0:
outputs = [o for o in cell[“outputs”]
if o.get(“output_type”) == “execute_result”]
assert len(outputs) < 2
if len(outputs) > 0:
output_str = “”.join(outputs[0][“data”][“text/plain”]).strip()
if output_str.startswith(“
print(“Usage: python3 tester.py
return
# dump results from this notebook to a summary .csv file
ipynb = sys.argv[1]
assert ipynb.endswith(“.ipynb”)
actual_path = ipynb.replace(“.ipynb”, “.csv”).replace(“.json”, “.csv”)
dump_results(ipynb, actual_path)
# compare to the answer key .csv file
expected_path = sys.argv[2] if len(sys.argv) > 2 else ipynb.replace(“.ipynb”, “-key.csv”)
result = compare(expected_path, actual_path)
result_path = os.path.join(os.path.dirname(ipynb), “test.json”)
with open(result_path, “w”) as f:
json.dump(result, f, indent=2)
# show user-friendly summary of test.json
print(“=”*40)
if len(result[“errors”]):
print(f”There were {len(result[‘errors’])} errors:\n”)
for err in result[“errors”]:
print(err)
print()
if len(result[“missing”]):
print(f'{len(result[“missing”])} answers not found, for question(s):’,
“, “.join([str(m) for m in result[“missing”]]))
print()
print(result[“summary”])
if __name__ == ‘__main__’:
main()