Merge branch 'master' of github.com:SpheMakh/Stimela

Author: SpheMakh

examples/simulation_pipeline.py

@@ -82,4 +82,4 @@
 
 # Run recipe. The 'steps' added above will be executed in the sequence that they were adde. The 'steps' added above will be
 # executed in the sequence that they were addedd
-pipeline.run()
+pipeline.run([1])

setup.py

@@ -1,7 +1,12 @@
 #!/usr/bin/env python
 
 import os
-from distutils.core import setup
+
+try:
+  from setuptools import setup
+except ImportError as e:
+  from distutils.core import setup
+
 import stimela
 
 setup(name = "stimela",
@@ -11,16 +16,16 @@
     author_email = "Sphesihle Makhathini <sphemakh@gmail.com>",
     url = "https://github.com/sphemakh/Stimela",
     packages = ["stimela", "stimela/cargo","stimela/utils"],
-    package_data = { "stimela/cargo" : ["data/skymodels/*.lsm.html", 
+    package_data = { "stimela/cargo" : ["data/skymodels/*.lsm.html",
                                    "configs/*.json",
-                                   "data/observatories/*.txt", 
+                                   "data/observatories/*.txt",
                                    "cab/*/Dockerfile",
                                    "base/*/Dockerfile",
-                                   "cab/*/src/*.py", 
+                                   "cab/*/src/*.py",
                                    "cab/*/src/*.sh",
                                    "cab/*/src/*.json",
                                    "cab/*/src/tdlconf.profiles"]},
     requires = ["docker", "python"],
-    scripts = ["bin/" + i for i in os.listdir("bin")], 
+    scripts = ["bin/" + i for i in os.listdir("bin")],
     classifiers = [],
      )

stimela/cargo/base/katdal/Dockerfile

@@ -0,0 +1,23 @@
+FROM radioastro/base
+MAINTAINER <sphemakh@gmail.com>
+
+RUN apt-add-repository -s ppa:radio-astro/main && \
+    apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git-core \
+        libpython-dev \
+        libcasacore2-dev \
+        libboost-python-dev \
+        libcfitsio3-dev \
+        wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Force upgrade of pip since pip install --upgrade pip doesn't work
+RUN easy_install -U pip setuptools
+
+RUN pip install -I -U git+git://github.com/ska-sa/katdal.git \
+    python-casacore
+
+RUN mkdir -p /var/kat/static && wget https://github.com/ska-sa/katdal/raw/master/blank.ms.tgz -O - | tar -C ./var/kat/static -xzvf -
+

stimela/cargo/cab/specfit/src/run.py

@@ -32,6 +32,8 @@
 
 if spi_image:
     spi_image = substitute(spi_image) or "%s/%s"%(INPUT, spi_image)
+    if spi_err:
+        spi_err = substitute(spi_err) or "%s/%s"%(INPUT, spi_err)
 
     if make_spi:
         spi_image = spi_image + ".alpha.fits"
@@ -41,27 +43,28 @@
 
 if image and make_spi:
     if isinstance(image, (str, unicode)):
-        cube = substitute(image) or "%s/%s"%(INPUT,image) 
+        cube = substitute(image) or "%s/%s"%(INPUT,image)
 
     elif isinstance(image, (list, tuple)):
         for i,im in enumerate(image):
             image[i] = substitute(im) or "%s/%s"%(INPUT, im)
+        cube = image
     else:
         raise TypeError("Image has to be either a string or list of strings")
 
     made_spi = True
     if mask:
             mask = substitute(mask) or "%s/%s"%(INPUT, mask)
     import specfit
-    specfit.spifit(cube, mask=mask, sigma=sigma, 
+    specfit.spifit(cube, mask=mask, sigma=sigma,
                   spi_image=spi_image,
                   spi_err_image=spi_err)
 
 
 if add_spi and lsmname:
     print "Extracting spi from image"
-    if not made_spi:
-        spi_image = substitute(spi_image) or "%s/%s"%(INPUT, spi_image)
+#    if not made_spi:
+#        spi_image = substitute(spi_image) or "%s/%s"%(INPUT, spi_image)
 
     if isinstance(freq0, (str, unicode)):
         freq0 = str(freq0)

stimela/cargo/cab/specfit/src/specfit.py

@@ -48,25 +48,25 @@ def freqInfo(fits):
 
 
 def spifit(cube, prefix=None, mask=None, thresh=None,
-           sigma=20, spi_image=None, 
+           sigma=20, spi_image=None,
            spi_err_image=None):
-    
 
     if isinstance(cube, (list, tuple)):
         ims = []
         nchan = len(cube)
-        freqs = [] 
+        freqs = []
         prefix = prefix or cube[0][-3:]
 
         for im in cube:
             with pyfits.open(im) as hdu:
                 hdr = hdu[0].header
+                hdu_data = hdu[0].data
                 ndim = hdr["NAXIS"]
-                freq_axis = fitsFreqInd(hdr)
-                freqs.append( hdr["CRVAL%d"%freq_axis])
-                ims.append( im[get_imslice(ndim)]  )
+                ind = fitsFreqInd(hdr)
+                freqs.append( hdr["CRVAL%d"%ind])
+                ims.append( hdu_data[get_imslice(ndim)]  )
 
-        data = numpy.vstack(ims)
+        data = numpy.dstack(ims).T
         ndim = data.ndim
         cnt_freq = freqs[nchan/2]
 
@@ -78,8 +78,8 @@ def spifit(cube, prefix=None, mask=None, thresh=None,
             ndim = data.ndim
 
         freqs, cnt_freq, bw, nchan, ind = freqInfo(cube)
-    
-    freq_ind = ndim - ind 
+
+    freq_ind = ndim - ind
     imslice = get_imslice(ndim)
     imslice[freq_ind] = slice(None)
     data = data[imslice]
@@ -90,7 +90,7 @@ def spifit(cube, prefix=None, mask=None, thresh=None,
     aa = []
     bb = []
     I0 = []
-    
+
     if mask:
         with pyfits.open(mask) as hdu:
             mdata = hdu[0].data
@@ -104,8 +104,11 @@ def spifit(cube, prefix=None, mask=None, thresh=None,
             thresh = sigma*noise
         ind = numpy.where(mfs>thresh)
 
+    if len(ind) < 1:
+        raise RunTimeError("No pixels above set threshold, or outside masked region")
+
     for i,j in zip(ind[0], ind[1]):
-        x = numpy.log(freqs/cnt_freq)
+        x = numpy.log(numpy.array(freqs)/cnt_freq)
         val = data[:,i,j]
         if val.any() <= 0:
             continue
@@ -120,7 +123,6 @@ def spifit(cube, prefix=None, mask=None, thresh=None,
     #    aa.append(intercept)
     #    bb.append(std_err)
     #    I0.append(numpy.log(data[nchan/2,i,j]))
-    
 
     nans = numpy.isnan(alpha)
     alpha[nans] = 0.0

stimela/docker.py

@@ -140,27 +140,28 @@ def get_log(self):
     def start(self):
         running = True
         tstart = time.time()
-        utils.xrun("docker start -a", [self.name])
         self.status = "running"
         self.log_container.update(self.info(), uptime="00:00:00")
+        try:
+            utils.xrun("docker start -a", [self.name])
+        except KeyboardInterrupt:
+           self.stop()
+           self.remove()
+           raise KeyboardInterrupt("Terminating process")
+           
+#       while running:
+#           try:
+#               uptime = seconds_hms(time.time() - tstart)
+#               self.log_container.update(self.info(), uptime=uptime)
+#               self.uptime = uptime
+#               self.log_container.write()
+#               status = self.info()["State"]["Status"]
+#               if status != "running":
+#                   running = False
 
-        while running:
-            time.sleep(1)
-            uptime = seconds_hms(time.time() - tstart)
-            self.log_container.update(self.info(), uptime=uptime)
-            self.uptime = uptime
-            self.log_container.write()
-
-            status = self.info()["State"]["Status"]
-            if status != "running":
-                running = False
-
-            #with open(self.logfile, "w") as stdw:
-            #    stdw.write(self.get_log())
-
-        self._print("Container [%s] has executed successfully."%(self.name))
         uptime = seconds_hms(time.time() - tstart)
         self.uptime = uptime
+        self._print("Container [%s] has executed successfully. \n Runtime was %s"%(self.name, uptime))
         self.log_container.update(self.info(), uptime=uptime)
 
         self.container_logger = self.get_log()
@@ -171,19 +172,33 @@ def start(self):
     def stop(self):
         dinfo = self.info()
         status = dinfo["State"]["Status"]
+        killed = False
         if status == "running":
-            utils.xrun("docker stop", [self.name])
+            try:
+                utils.xrun("docker stop", [self.name])
+            except KeyboardInterrupt("Received terminate signal. Will stop and remove container first"):
+                killed = True
 
         self._print("Container [self.name] has been")
         self.log_container.update(self.info(), self.uptime)
         self.log_container.write()
+        if killed:
+            self.remove()
+            raise KeyboardInterrupt
 
 
     def remove(self):
         dinfo = self.info()
         status = dinfo["State"]["Status"]
+        killed = False
         if status != "running":
-            utils.xrun("docker rm", [self.name])
+            try:
+                utils.xrun("docker rm", [self.name])
+            except KeyboardInterrupt:
+                killed = True
+            if killed:
+                raise KeyboardInterrupt
+           
         else:
             raise DockerError("Container [%s] has not been stopped, cannot remove"%(self.name))
 

stimela/recipe.py

@@ -202,7 +202,7 @@ def run(self, steps=None, log=True):
                 containers = [ self.containers[i-1] for i in steps[:len(self.containers)] ]
         else:
             containers = self.containers
-
+        
         for i, container in enumerate(containers):
             self.log.info("Running Container %s"%container.name)
             self.log.info("STEP %d :: %s"%(i, container.label))