Merge pull request #72 from SpheMakh/prepare-0.2.5

Prepare release 0.2.5

Author: SpheMakh

CHANGES.md

@@ -13,3 +13,11 @@
 * Add lwimager, wslcean, casa imagers (instead of a single imager image)
 * Add owlcat base image
 * Add container, docker image, and job loging
+
+# 0.2.5 
+- Improve hnadling of ctrl+c. Send terminate
+  signal to container.   
+- Improve handling of FITS images when predicting
+  visibilities; allow for minimal FITS images.  
+- Add clean fuction to help clean up after stimela  
+- Improve/Fx logging of images and containers

Dockerfile

@@ -0,0 +1,4 @@
+FROM stimela/base
+ADD . /Stimela
+WORKDIR /Stimela
+RUN pip install /Stimela

bin/stimela

@@ -10,21 +10,4 @@ import stimela
 if not os.path.exists(stimela.LOG_HOME):
     os.mkdir(stimela.LOG_HOME)
 
-if not os.path.exists(stimela.LOG_IMAGES):
-    with open(stimela.LOG_IMAGES, "w") as std:
-        pass
-
-
-if not os.path.exists(stimela.LOG_CONTAINERS):
-    with open(stimela.LOG_CONTAINERS, "w") as std:
-        pass
-
-if not os.path.exists(stimela.LOG_PROCESS):
-    with open(stimela.LOG_PROCESS, "w") as std:
-        pass
-
-if not os.path.exists(stimela.LOG_CABS):
-    with open(stimela.LOG_CABS, "w") as std:
-        pass
-
 stimela.main([a for a in sys.argv[1:]])

examples/simulation_pipeline.py

@@ -24,7 +24,7 @@
     "telescope" :   "meerkat",              # Telescope name
     "direction" :   "J2000,0deg,-30deg",   # Phase tracking centre of observation
     "synthesis" :   0.125,                      # Synthesis time of observation
-    "dtime"     :   30,                      # Exposure time
+    "dtime"     :   4,                      # Integration time in seconds
     "freq0"     :   "750MHz",               # Start frequency of observation
     "dfreq"     :   "1MHz",                 # Channel width
     "nchan"     :   1                      # Number of channels

setup.py

@@ -7,15 +7,15 @@
 except ImportError as e:
   from distutils.core import setup
 
-import stimela_version
+from stimela_misc import version
 
 setup(name = "stimela",
-    version = stimela_version.version,
+    version = version.version,
     description = "Dockerized radio interferometry scripting framework",
     author = "Sphesihle Makhathini",
     author_email = "Sphesihle Makhathini <sphemakh@gmail.com>",
     url = "https://github.com/sphemakh/Stimela",
-    packages = ["stimela","stimela_version", "stimela/cargo","stimela/utils", "stimela/cargo/cab"],
+    packages = ["stimela","stimela_misc", "stimela/cargo","stimela/utils", "stimela/cargo/cab"],
     package_data = { "stimela/cargo" : ["cab/*/Dockerfile",
                                    "base/*/Dockerfile",
                                    "cab/*/src/*.py",

stimela/__init__.py

@@ -16,13 +16,10 @@
 from recipe import Recipe, PipelineException
 from stimela import docker
 
-from stimela.utils import stimela_logger
+from stimela.utils import logger
 
 LOG_HOME = os.path.expanduser("~/.stimela")
-LOG_IMAGES = LOG_HOME + "/stimela_images.log"
-LOG_CONTAINERS = LOG_HOME + "/stimela_containers.log"
-LOG_PROCESS = LOG_HOME + "/stimela_process.log"
-LOG_CABS = LOG_HOME + "/stimela_cab.log"
+LOG_FILE = LOG_HOME + "/stimela_logfile.json"
 
 
 BASE = os.listdir(cargo.BASE_PATH)
@@ -83,13 +80,18 @@ def build(argv):
             help="Do not use cache when building the image")
 
     args = parser.parse_args(argv)
+    log = logger.StimelaLogger(LOG_FILE)
 
     if args.base:
         for image in BASE:
             dockerfile = "{:s}/{:s}".format(cargo.BASE_PATH, image)
             image = "stimela/{0}:{1}".format(image, __version__)
             docker.build(image,
                          dockerfile)
+
+        log.log_image(image, replace=True)
+        log.write()
+
         return 0
 
     workdir = "/home/{}/output/".format(USER)
@@ -101,6 +103,7 @@ def build(argv):
 
     no_cache = ["--no-cache"] if args.no_cache else []
 
+
     if args.cab:
         cab_args = args.cab.split(",")
 
@@ -115,15 +118,10 @@ def build(argv):
                      path,
                      build_args=build_args, args=no_cache)
 
-        img = stimela_logger.Image(LOG_CABS)
-        img.add(dict(name=cab))
-        img.write()
+        log.log_image(image, replace=True, cab=True)
+        log.write()
         return
 
-    # clear old cabs
-    img = stimela_logger.Image(LOG_CABS)
-    img.clear()
-
     for image in CAB:
         IGNORE = args.ignore_cabs.split(",")
         if image in NOT_PUBLIC+IGNORE:
@@ -135,9 +133,9 @@ def build(argv):
                      dockerfile,
                      build_args=build_args, args=no_cache)
 
-        img.add(dict(name=image))
+        log.log_image(image, replace=True, cab=True)
+        log.write()
 
-    img.write()
 
 
 def info(cabname, header=False):
@@ -206,7 +204,6 @@ def run(argv):
     args = parser.parse_args(argv)
     tag =  None
 
-
     _globals = dict(STIMELA_INPUT=args.input, STIMELA_OUTPUT=args.output,
                     STIMELA_MSDIR=args.msdir,
                     CAB_TAG=tag)
@@ -248,14 +245,14 @@ def pull(argv):
             help="Tag")
 
     args = parser.parse_args(argv)
+    log = logger.StimelaLogger(LOG_FILE)
 
     if args.image:
         for image in args.image:
-            img = stimela_logger.Image(LOG_IMAGES)
 
             if not img.find(image):
                 docker.pull(image)
-                img.add(dict(name=image, tag=tagargs.tag))
+                log.log_image(image)
     else:
 
         base = []
@@ -266,11 +263,11 @@ def pull(argv):
         base = set(base)
 
         for image in base:
-            img = stimela_logger.Image(LOG_IMAGES)
-
-            if not img.find(image) and image not in ["stimela/ddfacet", "radioastro/ddfacet"]:
+            if image not in ["stimela/ddfacet", "radioastro/ddfacet"]:
                 docker.pull(image)
-                img.add(dict(name=image, tag=args.tag))
+                log.log_image(image)
+
+    log.write()
 
 
 def images(argv):
@@ -282,15 +279,16 @@ def images(argv):
     add = parser.add_argument
 
     add("-c", "--clear", action="store_true",
-            help="Clear images log file")
+            help="Clear the logfile that keeps track of stimela images. This does not do anythig to the images.")
 
     args = parser.parse_args(argv)
 
-    img = stimela_logger.Image(stimela.LOG_IMAGES)
-    img.display()
+    log = logger.StimelaLogger(LOG_FILE)
+    log.display('images')
 
     if args.clear:
-        img.clear()
+        log.clear('images')
+        log.write()
 
 
 def containers(argv):
@@ -302,14 +300,15 @@ def containers(argv):
     add = parser.add_argument
 
     add("-c", "--clear", action="store_true",
-            help="Clear containers log file")
+            help="Clear the log file that keeps track of stimela containers. This doesn't do anything to the containers.")
 
     args = parser.parse_args(argv)
 
-    conts = stimela_logger.Container(stimela.LOG_CONTAINERS)
-    conts.display()
+    log = logger.StimelaLogger(LOG_FILE)
+    log.display('containers')
     if args.clear:
-        conts.clear()
+        log.clear('containers')
+        log.write()
 
 
 def ps(argv):
@@ -321,14 +320,15 @@ def ps(argv):
     add = parser.add_argument
 
     add("-c", "--clear", action="store_true",
-            help="Clear Log file")
+            help="Clear logfile that keeps track of stimela processes. This doesn't do anything ot the processes themselves.")
 
     args = parser.parse_args(argv)
 
-    procs = stimela_logger.Process(stimela.LOG_PROCESS)
-    procs.display()
+    log = logger.StimelaLogger(LOG_FILE)
+    log.display('processes')
     if args.clear:
-        procs.clear()
+        log.clear('processes')
+        log.write()
 
 
 def kill(argv):
@@ -344,42 +344,96 @@ def kill(argv):
 
     args = parser.parse_args(argv)
 
-    procs = stimela_logger.Process(LOG_PROCESS)
+    log = logger.StimelaLogger(LOG_FILE)
 
-    for pid in map(int, args.pid):
+    for pid in args.pid:
 
-        found = procs.find(pid)
+        found = pid in log.info['processes'].keys()
 
         if not found:
-            print "Could not find process {0}".format(pid)
+            print("Could not find process {0}".format(pid))
             continue
 
-        conts = stimela_logger.Container(LOG_CONTAINERS)
-        lines = []
+        try:
+            os.kill(int(pid), signal.SIGINT)
+        except OSError:
+            raise OSError('Process with PID {} could not be killed'.format(pid))
+
+        log.remove('processes', pid)
+    log.write()
 
-        procs.rm(pid)
-        procs.write()
 
-        for cont in conts.lines:
-            if cont.find(str(pid)) > 0:
-                lines.append(cont)
+def clean(argv):
+    for i, arg in enumerate(argv):
+        if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg
 
-        for line in lines:
-            cont, _id, utime, _pid, status = line.split()
-            if status.find("removed")<0:
-                cont_ = docker.Container(None, cont, None, None)
-                cont_.started = True
-                cont_.stop()
-                cont_.remove()
+    parser = ArgumentParser(description='Convience tools for cleaning up after stimela')
+    add = parser.add_argument
 
-                conts.rm(line)
+    add("-ai", "--all-images", action="store_true",
+        help="Remove all images pulled/built by stimela. This include CAB images")
+    
+    add("-ab", "--all-base", action="store_true",
+        help="Remove all base images")
 
-        conts.write()
+    add("-ac", "--all-cabs", action="store_true", 
+        help="Remove all CAB images")
+
+    add("-aC", "--all-containers", action="store_true",
+        help="Stop and/or Remove all stimela containers")
+
+    args = parser.parse_args(argv)
+
+    log = logger.StimelaLogger(LOG_FILE)
+
+    if args.all_images:
+        images = log.info['images'].keys()
+        for image in images:
+            utils.xrun('docker', ['rmi', image])
+            log.remove('images', image)
+            log.write()
+
+    if args.all_base:
+        images = log.info['images'].keys()
+        for image in images:
+            if log.info['images'][image]['CAB'] is False:
+                utils.xrun('docker', ['rmi', image])
+                log.remove('images', image)
+                log.write()
+
+    if args.all_cabs:
+        images = log.info['images'].keys()
+        for image in images:
+            if log.info['images'][image]['CAB']:
+                utils.xrun('docker', ['rmi', image])
+                log.remove('images', image)
+                log.write()
+
+        
+    if args.all_containers:
+        containers = log.info['containers'].keys()
+        for container in containers:
+            cont = docker.Container(log.info['containers'][container]['IMAGE'], container)
+            try:
+                status = cont.info()['State']['Status']
+            except OSError:
+                print('Could not inspect container {}. It probably doesn\'t exist, will remove it from log'.format(container))
+                status = False
+                log.remove('containers', container)
+                log.write()
+                continue
+
+            if status == 'running':
+            # Kill the container instead of stopping it, so that effect can be felt py parent process
+                utils.xrun('docker', ['kill', container])
+                cont.remove()
+                log.remove('containers', container)
+                log.write()
+            elif status == 'exited':
+                cont.remove()
+                log.remove('containers', container)
+                log.write()
 
-        try:
-            os.kill(pid, signal.SIGKILL)
-        except OSError:
-            pass
 
 def main(argv):
     for i, arg in enumerate(argv):
@@ -404,7 +458,8 @@ def main(argv):
     options = []
     commands = dict(pull=pull, build=build, run=run,
                     images=images, cabs=cabs, ps=ps,
-                    containers=containers, kill=kill)
+                    containers=containers, kill=kill,
+                    clean=clean)
 
     command = "failure"
 
@@ -438,6 +493,7 @@ def main(argv):
 cabs    : List active stimela containers
 ps      : List running stimela scripts
 kill    : Gracefully kill runing stimela process
+clean   : Clean up tools for stimela
 
 """)
 

stimela/cargo/base/aoflagger/Dockerfile

@@ -1,2 +1,2 @@
-FROM stimela/base:0.2.4
+FROM stimela/base:0.2.5
 RUN docker-apt-install aoflagger

stimela/cargo/base/astropy/Dockerfile

@@ -1,4 +1,4 @@
-FROM stimela/base:0.2.4
+FROM stimela/base:0.2.5
 RUN pip install astropy \
         numpy \
         scipy \

stimela/cargo/base/casarest/Dockerfile

@@ -1,3 +1,3 @@
-FROM stimela/base:0.2.4
+FROM stimela/base:0.2.5
 RUN docker-apt-install casarest python-owlcat casacore-data
 RUN pip install msutils