Merge pull request #6894 from BOINC/dpa_submit60

batch acceleration: make it work for small batches

Author: AenBleidd

html/ops/batch_accel.php

@@ -18,6 +18,8 @@
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
 // identify batches that are almost complete, and accelerate their completion
+// We first run 'batch_stats.php', which computes needed data:
+// LTT hosts, accelerable batches, median TT of batches
 
 require_once('../inc/util.inc');
 require_once('../inc/boinc_db.inc');
@@ -26,16 +28,30 @@
 
 ini_set('display_errors', true);
 
-// accelerate batches if at least this frac done
-// can be set in project.inc
+// the following can be set in project.inc
 //
-if (!defined('BATCH_ACCEL_MIN_FRAC_DONE')){
-    define('BATCH_ACCEL_MIN_FRAC_DONE', .85);
+// accelerate a batch if at least this frac of jobs are done (success or fail)
+// This value is a guess; it may not be optimal.
+// If too low, extra instances are created, reducing throughput.
+// If too high, batches take longer to finish.
+//
+if (!defined('BATCH_ACCEL_MIN_FRAC_DONE')) {
+    define('BATCH_ACCEL_MIN_FRAC_DONE', 0.85);
+}
+
+// accelerate a batch if the number of 'not done' jobs
+// (i.e. unsent or in progress) is less than this.
+// This helps small batches get done quicker
+//
+if (!defined('BATCH_ACCEL_MAX_NOT_DONE')) {
+    define('BATCH_ACCEL_MAX_NOT_DONE', 20);
 }
 
 $apps = [];
 
-// batch is in-progress and at least 90% done; accelerate its remaining jobs
+// batch is in-progress and qualifies for acceleration (see above);
+// accelerate its remaining jobs by marking them as high-priority
+// and possibly creating a new instance
 //
 function do_batch($batch, $wus) {
     global $apps;
@@ -78,7 +94,7 @@ function do_batch($batch, $wus) {
                         break;
                     case RESULT_SERVER_STATE_IN_PROGRESS:
                         $age = $now - $r->sent_time;
-                        if ($age<$batch->expire_time) {
+                        if ($age < $batch->expire_time) {
                             echo "   have recent in-progress result $r->id\n";
                             $make_another_result = false;
                         }
@@ -145,12 +161,16 @@ function main() {
             echo "batch $batch->id not in progress\n";
             continue;
         }
-        if ($batch->fraction_done < BATCH_ACCEL_MIN_FRAC_DONE) {
-            echo "batch $batch->id only $batch->fraction_done done\n";
-            continue;
+        $n_done = $batch->njobs_success + $batch->nerror_jobs;
+        $n_not_done = count($wus) - $n_done;
+        if ($batch->fraction_done > BATCH_ACCEL_MIN_FRAC_DONE
+            || $n_not_done < BATCH_ACCEL_MAX_NOT_DONE
+        ) {
+            echo "doing batch $batch->id\n";
+            do_batch($batch, $wus);
+        } else {
+            echo "not doing batch $batch->id: $batch->fraction_done done\n";
         }
-        echo "doing batch $batch->id\n";
-        do_batch($batch, $wus);
     }
     echo sprintf("finished batch_accel: %s\n", time_str(time()));
 }

html/ops/batch_stats.php

@@ -18,7 +18,7 @@
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 // process turnaround time stats files
 
-// compute stats for batch acceleration
+// compute stats for batch acceleration, and update the DB accordingly
 //
 // - classify hosts as low turnaround time (LTT)
 //      host.error_rate = 1

html/user/get_output3.php

@@ -38,7 +38,7 @@
 // user is allowed to download output files from batch only if
 // they own batch or have manage-all permissions
 //
-function check_auth($batch_id){
+function check_auth($batch_id) {
     $user = get_logged_in_user();
     if (has_manage_access($user, 0)) {
         return;
@@ -86,7 +86,7 @@ function get_batch_zip() {
     check_auth($batch_id);
     $dir = "../../results/$batch_id";
     if (!is_dir($dir)) {
-        die('no batch dir');
+        error_page('no batch dir');
     }
     $name = "batch_$batch_id.zip";
     $cmd = "cd $dir; rm -f $name; zip -q $name *";
@@ -103,7 +103,17 @@ function get_batch_tar() {
     check_auth($batch_id);
     $dir = "../../results/$batch_id";
     if (!is_dir($dir)) {
-        die('no batch dir');
+        error_page('no batch dir');
+    }
+
+    $d = fopen($dir, 'r');
+    if (!$d) {
+        error_page('fopen() failed');
+    }
+    if (!flock($d, LOCK_EX|LOCK_NB)) {
+        error_page(
+            "A download of this batch is already in progress."
+        );
     }
 
     // get the size of the tar file (fast - doesn't read files)
@@ -124,7 +134,7 @@ function get_batch_tar() {
         }
     }
     if ($nbytes<0) {
-        error_page('tar --totals didn't produce result');
+        error_page("tar --totals didn't produce result");
     }
     pclose($f);
 
@@ -146,6 +156,7 @@ function get_batch_tar() {
         flush();
     }
     pclose($f);
+    fclose($d);
 }
 
 $action = get_str('action');