[Refactor:Plagiarism] Do Lichen processing outside course dir (#45)

Author: williamjallen

bin/process_all.sh

@@ -19,6 +19,7 @@ if [ ! -f "${basepath}/config.json" ]; then
 		exit 1
 fi
 
+
 # delete any previous run results
 # TODO: determine if any caching should occur
 rm -rf "${basepath}/logs"
@@ -36,16 +37,55 @@ mkdir -p "${basepath}/provided_code/files"
 mkdir -p "${basepath}/other_gradeables"
 mkdir -p "${basepath}/users"
 
-# the default is r-x and we need PHP to be able to write if edits are made to the provided code
-chmod g=rwxs "${basepath}/provided_code/files"
+# Run Lichen and exit if an error occurs
+{
+    ############################################################################
+    # Finish setting up Lichen run
+
+    # The default is r-x and we need PHP to be able to write if edits are made to the provided code
+    chmod g=rwxs "${basepath}/provided_code/files" || exit 1
+
+    cd "$(dirname "${0}")" || exit 1
+
+    ############################################################################
+    # Do some preprocessing
+    echo "Beginning Lichen run: $(date +"%Y-%m-%d %H:%M:%S")"
+    ./concatenate_all.py "$basepath" "$datapath" || exit 1
+
+    ############################################################################
+    # Move the file somewhere to be processed (eventually this will be a worker machine)
+
+    # Tar+zip the file structure and save it to /tmp
+    cd $basepath || exit 1
+    archive_name=$(sha1sum "${basepath}/config.json" | awk '{ print $1 }') || exit 1
+    tar -czf "/tmp/LICHEN_JOB_${archive_name}.tar.gz" "config.json" "other_gradeables" "users" "provided_code" || exit 1
+    cd "$(dirname "${0}")" || exit 1
+
+    # TODO: move the archive to worker machine for processing
+
+    # Extract archive
+    tmp_location="/tmp/LICHEN_JOB_${archive_name}"
+    mkdir $tmp_location || exit 1
+    tar -xzf "/tmp/LICHEN_JOB_${archive_name}.tar.gz" -C "$tmp_location"
+    rm "/tmp/LICHEN_JOB_${archive_name}.tar.gz" || exit 1
+
+    ############################################################################
+    # Run Lichen
+    ./tokenize_all.py    "$tmp_location" || exit 1
+    ./hash_all.py        "$tmp_location" || exit 1
+    ./compare_hashes.out "$tmp_location" || exit 1
+
+    ############################################################################
+    # Zip the results back up and send them back to the course's lichen directory
+    cd $tmp_location || exit 1
+    tar -czf "/tmp/LICHEN_JOB_${archive_name}.tar.gz" "."
+    rm -rf $tmp_location || exit 1
 
-log_file="${basepath}/logs/lichen_job_output.txt"
+    # TODO: Move the archive back from worker machine
 
-cd $(dirname "${0}")
+    # Extract archive and restore Lichen file structure
+    cd $basepath || exit 1
+    tar --skip-old-files -xzf "/tmp/LICHEN_JOB_${archive_name}.tar.gz" -C "$basepath"
+    rm "/tmp/LICHEN_JOB_${archive_name}.tar.gz" || exit 1
 
-# run all of the modules and exit if an error occurs
-echo "Beginning Lichen run: $(date +"%Y-%m-%d %H:%M:%S")" >> $log_file 2>&1
-./concatenate_all.py  $basepath $datapath >> $log_file 2>&1 || exit 1
-./tokenize_all.py     $basepath           >> $log_file 2>&1 || exit 1
-./hash_all.py         $basepath           >> $log_file 2>&1 || exit 1
-./compare_hashes.out  $basepath           >> $log_file 2>&1 || exit 1
+} >> "${basepath}/logs/lichen_job_output.txt" 2>&1