#! /bin/bash
# Occasionally, students use UTF-8 characters in their source code (usually
# without knowing it). We might as well support it.
export LC_ALL=”C.UTF-8″
# Java figures out the proper student .java files that are required – but we
# need a little help for C and assembly. If you leave either of these blank,
# then this type of testing simply won’t work.
C_SRC=
S_SRC=asm4.s
if [[ $(which timeout 2>/dev/null) = “” ]]
then
echo “ERROR: The command ‘timeout’ is not installed on this system, the grading script will not work.” 1>&2
exit 1
fi
JAVA_SRCS=$(ls -1 *.java 2>/dev/null | grep -v -i -E “^Test_” | grep -v -E “XOR.java$” | grep -v -E “Base.java$”)
if [[ $JAVA_SRCS != “” ]]
then
echo “Compiling all of the Java sources – your code, plus all of the testcases…”
# BUGFIX:
#
# It appears that Java does *NOT* remove old .class files when compilation
# fails. So we had a student who had an old, buggy implementation of the
# code (which compiled). When they changed it to a new (non-compiling)
# version, javac (correctly) fails to compile the code but (maddeningly)
# leaves the old .class files around. So, when the code runs, the student
# sees the old behavior, for reasons that they don’t understand.
rm *.class 2>/dev/null
# BUGFIX:
#
# When there is an error which breaks some (but not all) of the files, javac
# will abort *all* of the builds. So we had a student who lost *all* of his
# testcase points because *some* testcases were broken. So we’d like to
# build all of the files individually.
#
# However, doing it that way (by default) is likely to be quite slow. So
# we will instead only fall back on that plan when the original compile
# fails.
javac *.java || {
echo “— JAVA COMPILE FAILURE —”
echo “javac reported some errors while building your code. (See the output above.)”
echo “This script will now re-run javac on each file, one at a time, in hopes of”
echo “succeeding in building some of the testcases.”
echo
echo “In order to not clutter up the script output, this rebuild will *NOT* print”
echo “out any error messages; refer to the output above to see why javac failed.”
echo
echo “This process is likely to be slow; if you want, you can use Ctrl-C to kill”
echo “this rebuild operation.”
echo “—————————-”
ls -1 *.java 2>&1 | xargs -r -n1 javac 1>/dev/null 2>&1
}
fi
if [[ ${S_SRC} != “” ]]
then
if [[ ! -f Mars4.5la.jar ]]
then
echo “Cannot find the Mars JAR file ‘Mars4.5la.jar’. Please copy the JAR file into the same directoy as the grading script – and name it ‘Mars4.5la.jar’ .”
exit 1
fi
fi
# running the MARS simulator once, since the first run on any computer creates
# some spurious output.
java -jar Mars4.5la.jar sm nc /dev/null 1>/dev/null 2>/dev/null
attempts=0
pass=0
# this variable can be increased, to apply various penalties for breaking
# the rules!
penaltyDivisor=1
failList=””
# if we have C or Java sources, check for any + operators in it – but make
# sure to exclude testcases.
if [[ ${C_SRC} != “” || ${JAVA_SRCS} != “” ]]
then
if [[ $(grep -E “[^+][+][^+]” ${C_SRC} ${JAVA_SRCS}) != “” ]]
then
echo
echo “ERROR: The grading script found that you used + or += in your C or Java code – your grade will be cut in half.”
echo ” —– BEGIN MATCHED LINES —–”
grep -E “[^+][+][^+]” ${C_SRC} ${JAVA_SRCS}
echo ” —– END MATCHED LINES —–”
penaltyDivisor=$(( penaltyDivisor*2 ))
fi
fi
# for most of the Java sources, we will also search for certain other
# banned constucts. For Project 3, this included all logical operators
# and if() statements. For Project 5, this included only if() statements –
# with an exception for MUX.
if [[ ${JAVA_SRCS} != “” ]]
then
JAVA_MORE_RESTRICTIONS=$(echo ${JAVA_SRCS} | xargs -n1 | grep -v -E “^(HwProj03_MUX_4by1|RussWire)[.]java$”)
# if [[ $(grep -E “[~!&|]” ${JAVA_MORE_RESTRICTIONS}) != “” ]]
# then
# echo
# echo “ERROR: The grading script found that you used some logical operators in Java code (other than the AND/OR/NOT classes) – your grade will be cut in half.”
#
# echo ” —– BEGIN MATCHED LINES —–”
# grep -E “[~!&|]” ${JAVA_MORE_RESTRICTIONS}
# echo ” —– END MATCHED LINES —–”
#
# penaltyDivisor=$(( penaltyDivisor*2 ))
# fi
# if [[ $(grep -E “if[ \t]*\(” ${JAVA_MORE_RESTRICTIONS}) != “” ]]
# then
# echo
# echo “ERROR: The grading script found that you used if() in Java code (other than the MUX class) – your grade will be cut in half.”
#
# echo ” —– BEGIN MATCHED LINES —–”
# grep -E “if[ \t]*\(” ${JAVA_MORE_RESTRICTIONS}
# echo ” —– END MATCHED LINES —–”
#
# penaltyDivisor=$(( penaltyDivisor*2 ))
# fi
fi
# run the MIPS checking script, which hunts down pseudoinstructions.
if [[ ${S_SRC} != “” ]]
then
if [[ ! -x mips_checker.pl ]]
then
echo “ERROR: The script mips_checker.pl either is not in the current directory, or is not executable. Please fix this. Until you do, this script will cut your score in half.”
penaltyDivisor=$(( penaltyDivisor*2 ))
elif [[ $(./mips_checker.pl < ${S_SRC} 2>&1) != “” ]]
then
echo “ERROR: mips_checker.pl reported some invalid instructions in your program – your grade will be cut in half.”
echo ” —– BEGIN mips_checker.pl —–”
./mips_checker.pl < ${S_SRC} 2>&1
echo ” —– END mips_checker.pl —–”
penaltyDivisor=$(( penaltyDivisor*2 ))
fi
fi
echo
echo “************* Running the testcases *************”
echo
for TESTCASE in $(ls -1 Test_*.java test_*.[cs] 2>/dev/null)
do
attempts=$(( attempts+1 ))
BASE=$(echo $TESTCASE | rev | cut -f2- -d’.’ | rev)
TYPE=$(echo $TESTCASE | rev | cut -f1 -d’.’ | rev)
if [[ ! -f $BASE.out ]]
then
echo “******************************”
echo “* TESTCASE ‘$TESTCASE’ FAILED”
echo “******************************”
echo “ERROR: The testcase file ‘$TESTCASE’ was found, but could not find a matching output file ‘$BASE.out'”
failList=”$failList
* $TESTCASE”
continue
fi
# how long is the output example? We’ll use this to calculate exactly how
# much output we’ll save.
lines=$(wc -l $BASE.out | awk ‘{print $1}’)
lines=$(( lines*2 + 10 ))
# run the testcase. Save the file into a temporary file. Of course, each
# different type of testcase is run in a different way.
if [[ $TYPE = “s” ]]
then
# we use head and tail to remove the stuff that Mars adds to the program
# (2 lines at the head, 1 at the tail). We then, later, use *ANOTHER*
# head operation to limit user data if the user generates far too much
# data.
#
# BUGFIX: grep out lines which are printed by Mars on Windows. Irritating.
timeout 15s java -jar Mars4.5la.jar sm ${S_SRC} $BASE.s 2>$BASE.stderr.unfiltered | cut -c-1000 | tail -n+3 | head -n-1 | head -n$lines > $BASE.student_output
# BUGFIX ON THE BUGFIX: Not all ‘bash’ installs support the redirection
# syntax I used. Do something simpler.
cat $BASE.stderr.unfiltered | grep -v java.util.prefs.WindowsPreferences | grep -v “Could not open/create prefs” > $BASE.stderr
rm $BASE.stderr.unfiltered
elif [[ $TYPE = “java” ]]
then
timeout 15s java -ea $BASE 2>$BASE.stderr | cut -c-1000 | head -n$lines > $BASE.student_output
elif [[ $TYPE = “c” ]]
then
echo “Compiling the testcase $TESTCASE, and linking it to ${C_SRC}…”
gcc -g -std=gnu99 ${C_SRC} $BASE.c -lm -o $BASE
RC=$?
if [[ $RC != 0 ]]
then
echo
echo “******************************”
echo “* TESTCASE ‘$TESTCASE’ FAILED”
echo “******************************”
echo “ERROR: The compilation process had a non-zero return code $RC.”
continue
fi
{
timeout 15s ./$BASE > >(cut -c-1000 | head -n$lines) 2>$BASE.stderr
RC=$?
if [[ $RC != 0 ]]
then
echo
echo “ERROR return code was: $RC”
fi
} > $BASE.student_output
else
echo “ERROR: The file extension $TYPE is not a recognized testcase type.” | tee $BASE.student_output
failList=”$failList
* $TESTCASE”
continue
fi
if [[ -s $BASE.stderr || $(diff $BASE.student_output $BASE.out 2>&1) != “” ]]
then
echo “******************************”
echo “* TESTCASE ‘$TESTCASE’ FAILED”
echo “******************************”
echo
if [[ -s $BASE.stderr ]]
then
echo ” —– stderr —–”
cat $BASE.stderr
echo ” —– END stderr —–”
echo
else
rm $BASE.stderr
echo ” —– diff OUTPUT —–”
diff $BASE.out $BASE.student_output
echo ” —– END diff —–”
# sometimes, the difference is only in whitespace. In intro-level
# courses, I use ‘diff -wB’ to ignore these differences, but in 252,
# it’s time for students to be more precise.
if [[ $(diff -wB $BASE.student_output $BASE.out 2>&1) == “” ]]
then
echo “WARNING: Your code did not exactly match the expected output, but it appears that all of”
echo “the differences are whitespace character or blank lines. You can use the command”
echo ” hexdump -C filename”
echo “to get a byte-by-byte dump of the file. I will now compare the hexdump of the two output”
echo “files (the expected output, and the actual).”
hexdump -C $BASE.out > $BASE.out.hexdump
hexdump -C $BASE.student_output > $BASE.student_output.hexdump
echo ” —– diff OUTPUT —–”
diff $BASE.out.hexdump $BASE.student_output.hexdump
echo ” —– END diff —–”
fi
echo
fi
failList=”$failList
* $TESTCASE”
else
echo “******************************”
echo “* Testcase ‘$TESTCASE’ passed”
echo “******************************”
rm $BASE.student_output $BASE.stderr
pass=$(( pass+1 ))
echo
fi
done
MAX_AUTO_SCORE=70
echo
echo “*******************************************”
echo “* OVERALL REPORT”
echo “* attempts: $attempts”
echo “* passed: $pass”
echo “*”
if [[ $penaltyDivisor != 1 ]]
then
echo “* penaltyDivisor: $penaltyDivisor (see above)”
fi
echo “* score: $(( MAX_AUTO_SCORE * pass / attempts / penaltyDivisor ))”
echo “* (out of $MAX_AUTO_SCORE possible)”
if [[ $failList != “” ]]
then
echo “*”
echo “* failed: $failList”
echo “*”
fi
echo “*******************************************”