db_purge: remove --retired_wus option

Never purge WUs/results of a non-retired batch.

Author: davidpanderson

html/inc/submit_util.inc

@@ -393,19 +393,19 @@ function abort_batch($batch) {
 }
 
 // retire a batch:
-// - mark the batch as retired (don't delete it; might be useful later)
+// - mark the batch as retired (don't delete it)
 // - delete output files (results/batchid/)
 // - mark unsent WUs as cancelled
 //
 // Don't delete the batch; it may have in-progress jobs that
-// we still need to file-delete and purge,
+// we'll need to file-delete and purge,
 // and the batch needs to be present for this.
 // Use ops/delete_batches.php to delete retired batches with no WUs
 //
 function retire_batch($batch) {
     system("rm -rf ../../results/$batch->id");
 
-    // get list of unsent WUs in this batch
+    // we need to call get_batch_params() to set WU.status
     //
     $wus = BoincWorkunit::enum_fields(
         'id, rsc_fpops_est, canonical_credit, canonical_resultid, error_mask',
@@ -414,6 +414,9 @@ function retire_batch($batch) {
     get_batch_params($batch, $wus);
     $batch->update("state=".BATCH_STATE_RETIRED);
         // do this AFTER get_batch_params()
+
+    // get list of unsent WUs in this batch
+    //
     $wu_ids = [];
     foreach ($wus as $wu) {
         if ($wu->status == WU_UNSENT) {

html/user/submit.php

@@ -1038,6 +1038,7 @@ function handle_retire_batch($user) {
     } else {
         page_head("Confirm retire batch");
         echo "
+            <p>
             Retiring a batch will remove all of its output files.
             Are you sure you want to do this?
             <p>

sched/assimilator.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 '''
 Generic Assimilator framework
+
+NOT SUPPORTED.  Use script_assimilator instead
 '''
 
 import os, re, signal, sys, time, hashlib

sched/db_purge.cpp

@@ -23,11 +23,10 @@
 // to XML archive files,
 // then deleting it and the results from the DB.
 //
-// if --retired_wus:
-//      purge WUs for which file_delete_state=FILE_DELETE_DONE
-//      and the WU is in a retired batch
-// otherwise
-//      purge WUs for which file_delete_state=FILE_DELETE_DONE.
+// Purge WUs for which file_delete_state == FILE_DELETE_DONE
+// and the WU is either
+//    - in a retired batch
+//    - not in a batch
 //
 // The XML files have names of the form
 // wu_archive_TIME and result_archive_TIME
@@ -70,7 +69,6 @@ void usage() {
         "Purge workunit and result records that are no longer needed.\n\n"
         "Usage: db_purge [options]\n"
         "   -d N or --debug_level N     Set verbosity level (1-4; 3=normal, 4=debug)\n"
-        "   --retired_wus               purge WUs only from retired batches\n"
         "   --min_age_days N            Purge Wus w/ mod time at least N days ago\n"
         "   --max N                     Purge at most N WUs\n"
         "   --zip                       Compress output files by piping through zip\n"
@@ -236,7 +234,6 @@ double min_age_days = 0;
 bool no_archive = false;
 bool dont_delete = false;
 bool daily_dir = false;
-bool retired_wus = false;
 int max_number_workunits_to_purge = 0;
     // If nonzero, maximum number of workunits to purge.
     // Since all results associated with a purged workunit are also purged,
@@ -255,8 +252,6 @@ int id_modulus=0, id_remainder=0;
     // allow more than one to run - doesn't work if archiving is enabled
 char app_name[256];
 DB_APP app;
-string retired_batch_ids;
-    // if --retired_wus, a list of retired batch IDs as "id1,id2,..."
 
 bool time_to_quit() {
     if (max_number_workunits_to_purge) {
@@ -611,38 +606,37 @@ int purge_and_archive_results(DB_WORKUNIT& wu, int& number_results) {
     return 0;
 }
 
-// get list of IDs of retired batches
+// get list of IDs of retired batches (and 0, = no batch)
 //
 int get_retired_batch_ids(string &out) {
-    out.clear();
+    out = "0";
     char query[256];
     sprintf(query, "select id from batch where state=%d", BATCH_STATE_RETIRED);
     int retval = boinc_db.do_query(query);
-    if (retval) return mysql_errno(boinc_db.mysql);
+    if (retval) {
+        return mysql_errno(boinc_db.mysql);
+    }
 
     MYSQL_RES *rp;
     rp = mysql_store_result(boinc_db.mysql);
-    if (!rp) return mysql_errno(boinc_db.mysql);
-    bool first = true;
+    if (!rp) {
+        return mysql_errno(boinc_db.mysql);
+    }
     while (1) {
         MYSQL_ROW row = mysql_fetch_row(rp);
         if (!row) {
             mysql_free_result(rp);
             break;
         }
-        if (first) {
-            first = false;
-        } else {
-            out += ",";
-        }
+        out += ",";
         out += row[0];
     }
     return 0;
 }
 
 // return true if did anything
 //
-bool do_pass() {
+bool do_pass(string &retired_batch_ids) {
     int retval = 0;
 
     // The number of workunits/results purged in a single pass of do_pass().
@@ -683,13 +677,9 @@ bool do_pass() {
         sprintf(buf, " and appid=%lu", app.id);
         clause += buf;
     }
-    if (retired_wus) {
-        clause += " and batch in (";
-        clause += retired_batch_ids;
-        clause += ")";
-    } else {
-        clause += " and batch=0";
-    }
+    clause += " and batch in (";
+    clause += retired_batch_ids;
+    clause += ")";
 
     sprintf(buf, " limit %d", DB_QUERY_LIMIT);
     clause += buf;
@@ -862,8 +852,6 @@ int main(int argc, char** argv) {
             max_wu_per_file = atoi(argv[i]);
         } else if (is_arg(argv[i], "no_archive")) {
             no_archive = true;
-        } else if (is_arg(argv[i], "retired_wus")) {
-            retired_wus = true;
         } else if (is_arg(argv[i], "sleep")) {
             if (!argv[++i]) {
                 log_messages.printf(MSG_CRITICAL, "%s requires an argument\n\n", argv[--i]);
@@ -957,25 +945,14 @@ int main(int argc, char** argv) {
         if (time_to_quit()) {
             break;
         }
-        if (retired_wus) {
-            retval = get_retired_batch_ids(retired_batch_ids);
-            if (retval) {
-                log_messages.printf(MSG_CRITICAL,
-                    "Can't get retired batch IDs"
-                );
-                exit(1);
-            }
-            if (retired_batch_ids.empty()) {
-                log_messages.printf(MSG_NORMAL,
-                    "no retired batches; sleeping %d\n",
-                    sleep_sec
-                );
-                daemon_sleep(sleep_sec);
-                continue;
-            }
+        string retired_batch_ids;
+        retval = get_retired_batch_ids(retired_batch_ids);
+        if (retval) {
+            log_messages.printf(MSG_CRITICAL, "Can't get retired batch IDs");
+            exit(1);
         }
 
-        bool did_something = do_pass();
+        bool did_something = do_pass(retired_batch_ids);
         if (one_pass) {
             break;
         }

sched/file_deleter.cpp

@@ -15,10 +15,11 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
-
-// file deleter.  See usage() below for usage.
-// skips WUs with 'nodelete' in the name
-// skips files with <no_delete/> in the <file_info>
+// file deleter.
+// enumerate WUs and results with file_delete_state == FILE_DELETE_READY,
+// and delete the associated files.
+// skip WUs with 'nodelete' in the name
+// skip files with <no_delete/> in the <file_info>
 
 // enum sizes.  RESULT_PER_ENUM is three times larger on the
 // assumption of 3-fold average redundancy.

tools/process_input_template.cpp

@@ -20,6 +20,9 @@
 // by scanning the input template, macro-substituting the input files,
 // and putting in the command line element and additional XML
 //
+// Input files must already be staged.
+// If the .md5 file is not present, create it.
+//
 // Called (only) in create_work.cpp
 
 #include <stdio.h>