initial import

Makefile

+.PHONY: howto
+
+flake8:
+	flake8 --doctest assign_reviews.py
+
+pylint:
+	pylint assign_reviews.py
+
+pycodestyle:
+	pycodestyle assign_reviews.py
+
+pydocstyle:
+	pydocstyle assign_reviews.py
+
+doctest:
+	python -m doctest -v assign_reviews.py
+
+howto:
+	cd howto; pandoc howto.md -o howto.pdf
+
+lint: flake8 pylint pycodestyle pydocstyle
+test: doctest

assign_reviews.py

+# -*- coding: utf-8 -*-
+"""Unpack, obfuscate, shuffle, test, export and pack submissions for review.
+
+Usage:
+    Download combined zip from olat, then
+    > python3 assign_reviews.py <extracted folder|zipfile> [outfolder]
+    This will produce a folder for each student containing the to-be-reviewed
+    submission from some other random student, ready to be uploaded to olat.
+
+Run tests:
+    - python3 -m doctest -v assign_reviews.py
+    - flake8 --doctest assign_reviews.py
+    - pylint assign_reviews.py
+    - pydocstyle assign_reviews.py
+"""
+
+# I don't really like to assemble all imports at the top of the file, but PEP8
+# says:
+# > Imports are always put at the top of the file, just after any module
+# > comments and docstrings, and before module globals and constants.
+# so what do I know ¯\_(ツ)_/¯
+import re
+import random
+import os
+import tempfile
+import subprocess
+import shutil
+from argparse import ArgumentParser, RawDescriptionHelpFormatter, FileType
+from pyunpack import Archive
+import pandas as pd
+from glob2 import glob
+import nbformat
+from nbconvert import PDFExporter, HTMLExporter
+from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
+
+PARTICIPANTS_FILE = "participants.md"
+SENDER_EMAIL = "example.example@uni-hamburg.de"
+
+
+def find_submissions(ifolder):
+    """Find submitted ipynb files in folder."""
+    return glob(ifolder + "/**/2_submissions/")
+
+
+def parse(ifile):
+    """Parse filename/Path to extract name and bnumber.
+
+    >>> parse('ita_Exercise_4_/LASTNAME_FIRSTN_BAD0000/2_submissions/e.ipynb')
+    ('FIRSTN LASTNAME', 'BAD0000')
+    >>> parse('ita_Exe4_/Last_First_BAW0001/2_submissions/e.ipynb')
+    ('First Last', 'BAW0001')
+    """
+    regexp = re.compile('.*/(.*)_(.*)_([^_]*)/2_submissions/.*')
+    lastname, firstname, bnumber = regexp.findall(ifile)[0]
+    return firstname + " " + lastname, bnumber
+
+
+def smoke_test(ifile):
+    """Execute notebook file to check if it actually works."""
+    path = os.path.dirname(ifile)
+    notebook = nbformat.read(ifile, as_version=4)
+    execute = ExecutePreprocessor(timeout=600, kernel_name='python3')
+    try:
+        execute.preprocess(notebook, {'metadata': {'path': path}})
+    except (CellExecutionError, ModuleNotFoundError) as msg:
+        print('Failed to run "{}".'.format(ifile))
+
+        with open(os.path.join(path, "error.txt"), "w") as fhandle:
+            fhandle.write(remove_bash_colors(str(msg)))
+        return False
+    return True
+
+
+def remove_bash_colors(istring):
+    r"""Clean up bash color tags from string (eg. command output).
+
+    >>> remove_bash_colors('\x1B[0;31m--------------------------------\x1B[0m')
+    '--------------------------------'
+    >>> remove_bash_colors('\x1B[0;31mFileNotFoundError\x1B[0m')
+    'FileNotFoundError'
+    """
+    return re.sub(r'\x1B\[[0-9]*?;?[0-9]*?m', '', istring)
+
+
+def create_pdf(ifile):
+    """Create pdf file, unless it exists."""
+    pdf_filename = ifile.replace('ipynb', 'pdf')
+
+    notebook = nbformat.read(ifile, as_version=4)
+    if not os.path.exists(pdf_filename):
+        pdf_exporter = PDFExporter()
+        (pdf, _) = pdf_exporter.from_notebook_node(notebook)
+        with open(pdf_filename, "ab") as file:
+            file.write(pdf)
+
+
+def create_html(ifile):
+    """Create html file, unless it exists."""
+    html_filename = ifile.replace('ipynb', 'html')
+
+    notebook = nbformat.read(ifile, as_version=4)
+    if not os.path.exists(html_filename):
+        html_exporter = HTMLExporter()
+        (html, _) = html_exporter.from_notebook_node(notebook)
+        with open(html_filename, "a") as file:
+            file.write(html)
+
+
+def shuffle_wo_fixed(ilist):
+    """Shuffle input list while avoiding fixed points.
+
+    Return shuffled copy of input list.
+
+    >>> shuffle_wo_fixed([1, 2])
+    [2, 1]
+    >>> a = shuffle_wo_fixed([1, 2, 3])
+    >>> a == [2, 3, 1] or a == [3, 1, 2]
+    True
+    >>> input = [1, 2, 3]
+    >>> input == shuffle_wo_fixed(input)
+    False
+    >>> shuffle_wo_fixed([[1, 2], [2, 3]])
+    [[2, 3], [1, 2]]
+    >>> shuffle_wo_fixed([['a', 'b'], ['b', 'a']])
+    [['b', 'a'], ['a', 'b']]
+    >>> shuffle_wo_fixed(zip([1, 2], ['a', 'b']))
+    [(2, 'b'), (1, 'a')]
+    """
+    ilist = list(ilist)  # convert iterable to list for fixpoint check
+    olist = list(ilist)[:]
+    while True:
+        random.shuffle(olist)
+
+        # check for fixed points (identical items on some index in two lists)
+        # ugly, but I don't know better.
+        # See https://en.wikipedia.org/wiki/Derangement
+        if not [i for i in zip(olist, ilist) if i[0] == i[1]]:
+            break
+
+    return olist
+
+
+def move_file_or_unpack(file, target_dir):
+    """Move file to directory, unless it's a zip file, than unpack."""
+    ext = os.path.splitext(file)[1]
+    if ext in ('.zip', '.rar'):
+        Archive(file).extractall(target_dir)
+    else:
+        shutil.copy2(file, target_dir)
+
+
+def hoist_files(target_dir):
+    """Flat folder-containing-folder."""
+    files = os.listdir(target_dir)
+    if len(files) == 1:
+        target_dir_sub = os.path.join(target_dir, files[0])
+        if os.path.isdir(target_dir_sub):
+            for file in os.listdir(target_dir_sub):
+                shutil.move(os.path.join(target_dir_sub, file), target_dir)
+            os.rmdir(target_dir_sub)
+
+
+def rename_file(file, bnumber):
+    """Rename files to obfuscate owner, but try to leave data files alone."""
+    ext = os.path.splitext(file)[1]
+    if ext in ('.ipynb', '.pdf', '.html'):
+        dirname = os.path.dirname(file)
+        os.rename(file, os.path.join(dirname, bnumber + ext))
+
+
+def pack_folder(zipname, folder):
+    """Pack folder and move resulting zip into that folder."""
+    tmpfile = os.path.join(tempfile.gettempdir(), zipname)
+    zipfile = shutil.make_archive(tmpfile, 'zip', folder)
+    shutil.move(zipfile, folder)
+
+
+def unpack_zip(path):
+    """Make sure path is a folder and unzip if archive."""
+    if os.path.splitext(path)[1] == '.zip':
+        folder = tempfile.mkdtemp()
+        subprocess.run(["unzip", path, "-d", folder], check=True)
+        return folder
+    return path
+
+
+def find_participant(bnumber):
+    """Find participant in participants file."""
+    # the fact that i can misuse pandas to parse markdown is so fucking
+    # awesome it should be illegal
+    dataframe = (pd.read_csv(PARTICIPANTS_FILE, sep='|', comment=':')
+                 .dropna(axis="columns", how='all')
+                 .dropna(axis="rows", how='all')
+                 .rename(str.strip, axis='columns')
+                 .applymap(str.strip))
+
+    return dataframe[dataframe['B-nr'] == bnumber]
+
+
+def email_participant(participant, errfile=None):
+    """Open thunderbird and prepare an email."""
+    # shit this is so much fun i hope everyone fucks up their notebooks
+
+    print('preparing email...')
+    to1 = str(participant['email'].values[0])
+    to2 = str(participant['email (uni)'].values[0])
+    sender = SENDER_EMAIL
+    subject = "Your submission"
+    body = """Hey,
+it seems your submission failed to run, please see attached error log.
+Regards
+"""
+    attach = ',attachment=\'file:///{}\''.format(errfile) if errfile else ""
+    cmd = ('thunderbird -compose "to=\'{},{}\',from={},subject={},'
+           'body=\'{}\'{}"'
+           .format(to1, to2, sender, subject, body, attach))
+
+    # i want to use bash aliases for some thunderbird options
+    subprocess.call(['/bin/bash', '-i', '-c', cmd])
+
+
+def assign_reviews(ifolder, ofolder):
+    """Find and reassign submissions for review."""
+    ifolder = unpack_zip(ifolder)
+    files = find_submissions(ifolder)
+
+    assert files, 'no files found in {}'.format(ifolder)
+
+    # parse filenames
+    names = []
+    bnumbers = []
+    for file in files:
+        name, bnumber = parse(file)
+        names.append(name)
+        bnumbers.append(bnumber)
+
+    # synchronously shuffle reviewee file list and bnumbers for reassignment
+    reviewees, reviewee_bnumbers = zip(*shuffle_wo_fixed(zip(files, bnumbers)))
+
+    # move and rename files
+    # three things might happen:
+    # 1. we find a zip file              -> unpack, rename and repack
+    # 2. we find a number of files       -> rename and pack
+    #
+    # Always generate pdf & html file to smoke-test, but discard if html/pdf
+    # exists.
+    for name, bnumber, reviewee in zip(names, reviewee_bnumbers, reviewees):
+        new_path = os.path.join(ofolder, name)
+        print('Proccessing {}...'.format(name))
+
+        # create folders - this will crash if the folder exists, which is a
+        # good thing, b/c then something is wrong anyway
+        print('Create folder...')
+        os.mkdir(new_path)
+
+        # move and unpack reviewee files to reviewer folder
+        print('Copy and unpack files...')
+        for file in os.listdir(reviewee):
+            move_file_or_unpack(os.path.join(reviewee, file), new_path)
+            hoist_files(new_path)
+
+        # due to unpacking, files might have changed
+        print('Rename files...')
+        for file in os.listdir(new_path):
+            rename_file(os.path.join(new_path, file), bnumber)
+
+        # lets go find
+        try:
+            nbfile = glob(new_path + "/*.ipynb")[0]
+        except IndexError:
+            print("Couldn't find notebook...")
+            participant = find_participant(bnumber)
+            print(participant)
+            email_participant(participant)
+            continue
+
+        print('Test notebook...')
+        if not smoke_test(nbfile):
+            participant = find_participant(bnumber)
+            print("Failed for {} in folder {}".format(bnumber, name))
+            print(participant)
+
+            errfile = os.path.abspath(os.path.join(new_path, "error.txt"))
+            email_participant(participant, errfile)
+
+        print('Create PDF...')
+        create_pdf(nbfile)
+
+        print('Create HTML...')
+        create_html(nbfile)
+
+        print('Pack folder...')
+        pack_folder(bnumber, new_path)
+
+        print("")
+
+
+def main():
+    """Parse cli arguments and kick of processing."""
+    parser = ArgumentParser(description=__doc__,
+                            formatter_class=RawDescriptionHelpFormatter)
+    parser.add_argument("infile", type=FileType('r'),
+                        help="zipfile or folder to read from")
+    parser.add_argument("outdir", default="./", type=FileType('w'),
+                        help="target dir")
+
+    options = parser.parse_args()
+
+    assign_reviews(options.infile, options.outdir)
+
+
+if __name__ == "__main__":
+    main()

environment.yml

+channels:
+  - conda-forge
+
+dependencies:
+  - python>=3.6
+  - pip
+  - pip:
+      - pyunpack
+      - argparse
+      - pandas
+      - glob2
+      - nbformat
+      - nbconvert
+      - flake8
+      - pylint
+      - pycodestyle
+
+

howto/howto.md

+# Exercise-Review workflow using an automated reassignment script
+
+This documents details a workflow for OLAT submissions for review
+originally developed for the project "JUNOSOL - Jupyter-notebooks for
+self-organized learning" using a helper script to unpack, obfuscate,
+shuffle, test, export and pack OLAT submissions for review.
+
+Students submit jupyter notebooks to OLAT exercises, which get
+redistributed randomly to their peers and reviewed by them. Read more
+of the general workflow below.
+
+## Workflow:
+
+In general, we are trying to implement a submit-review-feedback cycle
+in OLAT, which is not supported natively, so we are using the revise
+submission function to assign reviewees to reviewers.
+
+0. **Prepare participants file.**
+
+    See example `participants.md`. This file contains email and names
+    to contact students with faulty submissions.
+
+1. **Create Exercise for students in OLAT and assign**
+
+    Students will be able to accept assignments and download/work on
+    the assigned jupyter notebooks exercise sheets.
+
+1. **Download OLAT generated zip of solutions.**
+
+    Once the exercise deadline expired, OLAT lets you download all
+    submissions of an exercise as one (broken…) zip file.
+ 
+    ![](screenshot_download_submissions.png)
+
+2. **Run zip through assign_reviews.py**
+ 
+    Run
+
+        python assign_reviews.py submissions.zip target_folder
+
+    and the script will try to automatically unpack the zipfile, rename
+    folders and files to use b-numbers instead of clear names, shuffle
+    participants, run the submitted jupyter notebooks to make sure they
+    aren't obviously broken and then pack each submission as zip file
+    ready for re-upload.
+
+    To make reviews easier, notebook files are also (additionally)
+    converted to pdf and html and added to the upload zip.
+
+3. **Deal with issues**
+
+    Each submission is smoke-tested to catch obvious problems. In case
+    issues are found an email is prepared to contact the student in
+    question with an error log attached. Typical problems are missing
+    files, missing notebooks, and errors rising from out-of-order
+    execution which can be quickly addressed if caught early. It is
+    recommended to test-run this script before the deadline to allow
+    students to fix their submissions.
+
+3. **Upload for review**
+
+	Log back into OLAT and upload each solution for review. Since there
+	is no review step in OLAT, reject students solution and upload the
+	corresponding zip file. This needs to be clearly communicated to
+	students and will still lead to confusion. Students can then
+	download the uploaded zip and conduct and upload a review as an
+	revision in OLAT.

participants.md

+| Name              | B-nr    | email                          | email (uni)                             |
+|:-----------------:|:-------:|:------------------------------:|:---------------------------------------:|
+| First Lastname    | BAR0000 | BAR0000@studsum.uni-hamburg.de | first.lastname@studium.uni-hamburg.de   |

readme.md

+# Reassign students solutions for review
+
+This is a helper script to unpack, obfuscate, shuffle, test, export and pack OLAT submissions for review
+originally developed for the project "JUNOSOL - Jupyter-notebooks for self-organized learning".
+
+Requirements were:
+
+- mostly automated
+- should capture obvious problems early
+- should try to obfuscate participants for semi-blind reviews
+- handle malformed zips produced by OLAT and mangled submissions
+
+More on the general workflow see [howto/howto.md](howto/howto.md)
+
+Setup:
+
+    conda env create -f environment.yml -p ./env
+    conda activate ./env
+    
+Usage:
+
+    python assign_reviews.py -h
+