added comments and added RESCHEDULED to detect_killed

Author: Grutschus

src/seml/commands/manage.py

@@ -93,6 +93,8 @@ def cancel_empty_pending_jobs(db_collection_name: str, *sacred_ids: int):
             {'execution'},
         )
     )
+
+    # We exclude all SLURM jobs that are needed for continuing a RESCHEDULED experiment
     rescheduled_ids = []
     for exp in rescheduled_exps:
         execution = exp['execution']
@@ -672,7 +674,7 @@ def detect_killed(db_collection_name: str, print_detected: bool = True):
     cluster = get_cluster_name()
     exps = collection.find(
         {
-            'status': {'$in': [*States.PENDING, *States.RUNNING]},
+            'status': {'$in': [*States.PENDING, *States.RUNNING, *States.RESCHEDULED]},
             'execution.cluster': cluster,  # only check experiments that are running on the current cluster
             # Previously we only checked for started experiments by including the following line:
             # 'host': {'$exists': True},  # only check experiments that have been started

src/seml/commands/start.py

@@ -1133,6 +1133,8 @@ def claim_experiment(db_collection_name: str, exp_ids: Sequence[int]):
             'execution.task_id': task_id,
             'execution.cluster': cluster_name,
         }
+        # First, we check whether this SLURM job is responsible for a RESCHEDULED experiment
+        # If so, we claim this first
         exp = collection.find_one_and_update(
             {
                 '_id': {'$in': list(exp_ids)},
@@ -1144,6 +1146,8 @@ def claim_experiment(db_collection_name: str, exp_ids: Sequence[int]):
             {'$set': {'status': States.RUNNING[0], **update}},
             {'_id': 1, 'slurm': 1},
         )
+        # Only after we have checked that this job is not responsible for a RESCHEDULED experiment
+        # can we pick up a pending one.
         if exp is None:
             exp = collection.find_one_and_update(
                 {