Merge pull request #624 from ratt-ru/pullfreeze

fix pull freeze if image exists

Author: SpheMakh

examples/simulation_pipeline.py

@@ -25,8 +25,6 @@
                           log_dir=os.path.join(OUTPUT, "logs"),
                           )
 
-pipeline.JOB_TYPE = "singularity"
-
 # 1: Make empty MS
 pipeline.add("cab/simms",                   # Executor image to start container from
              "simms_example",               # Container name

stimela/cargo/base/cubical/Dockerfile

@@ -9,5 +9,5 @@ RUN docker-apt-install casacore-dev \
     libcfitsio-dev
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 10
 RUN pip3 install --upgrade --force pip
-RUN pip install "cubical[lsm-support]@git+https://github.com/ratt-ru/CubiCal.git@v1.5.2"
+RUN pip install "cubical[lsm-support]@git+https://github.com/ratt-ru/CubiCal.git@v1.5.3"
 RUN gocubical --help

stimela/cargo/cab/cubical/parameters.json

@@ -1,10 +1,10 @@
 {
     "task": "cubical", 
     "base": "stimela/cubical", 
-    "tag": "1.5.00", 
+    "tag": "1.5.4", 
     "description": "CubiCal is a suite of fast radio interferometric calibration routines exploiting complex optimisation.", 
     "prefix": "--",
-    "version":"1.5.2",
+    "version":"1.5.3",
     "binary": "gocubical",
     "junk":["cubical.last", "montblanc.log"],
     "msdir": true, 

stimela/main.py

@@ -179,7 +179,7 @@ def run(argv):
     add("-g", "--globals", metavar="KEY=VALUE[:TYPE]", action="append", default=[],
         help="Global variables to pass to script. The type is assumed to string unless specified")
 
-    add("-jt", "--job-type", default="docker",
+    add("-jt", "--job-type", default="docker", choices = ["docker", "singularity", "podman"],
         help="Container technology to use when running jobs")
 
     add("-ll", "--log-level", default="INFO", choices=loglevels.upper().split() + loglevels.split(),
@@ -189,6 +189,7 @@ def run(argv):
 
     _globals = dict(_STIMELA_INPUT=args.input, _STIMELA_OUTPUT=args.output,
                     _STIMELA_MSDIR=args.msdir,
+                    _STIMELA_JOB_TYPE=args.job_type.lower(),
                     _STIMELA_LOG_LEVEL=args.log_level.upper())
 
     nargs = len(args.globals)

stimela/singularity.py

@@ -13,6 +13,8 @@
 from shutil import which
 
 
+log = stimela.logger()
+
 binary = which("singularity")
 if binary:
     __version_string = subprocess.check_output([binary, "--version"]).decode("utf8")
@@ -27,7 +29,7 @@
 class SingularityError(Exception):
     pass
 
-def pull(image, store_path, docker=True, directory=".", force=False):
+def pull(image, name, docker=True, directory=".", force=False):
     """ 
         pull an image
     """
@@ -38,13 +40,16 @@ def pull(image, store_path, docker=True, directory=".", force=False):
     if not os.path.exists(directory):
         os.mkdir(directory)
 
-    utils.xrun("singularity", ["build", 
+    image_path = os.path.abspath(os.path.join(directory, name))
+    if os.path.exists(image_path) and not force:
+        log.info(f"Singularity image already exists at '{image_path}'. To replace it, please re-run with the 'force' option")
+    else:
+        utils.xrun("singularity", ["build", 
         	"--force" if force else "", 
-         	os.path.join(directory,store_path), fp])
+         	image_path, fp])
 
     return 0
 
-
 class Container(object):
     def __init__(self, image, name,
                  volumes=None,