Merge pull request #147 from CliMA/ne/hotfix

Fix broken worker polling

Author: nefrathenrici

.buildkite/derecho_pipeline.yml

@@ -1,7 +1,7 @@
 agents:
   queue: derecho
   modules: climacommon
-  pbs_q: preempt
+  pbs_q: main
   pbs_l_select: "1:ngpus=1:ncpus=1" # We are out of CPU-only hours...
 
 env:

Project.toml

@@ -1,7 +1,7 @@
 name = "ClimaCalibrate"
 uuid = "4347a170-ebd6-470c-89d3-5c705c0cacc2"
 authors = ["Climate Modeling Alliance"]
-version = "0.0.11"
+version = "0.0.12"
 
 [deps]
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"

src/workers.jl

@@ -202,7 +202,7 @@ function poll_file_for_worker_startup(
     launched_workers = 0
 
     for retry_delay in push!(collect(retry_delays), 0)
-        if process_exited(pid)
+        if process_exited(pid) && pid.exitcode != 0
             error(
                 """Worker launch process exited with code $(pid.exitcode).
           Please check the terminal for error messages from the job scheduler.""",