Merge branch 'master' of github.com:sphemakh/Penthesilea

Author: SpheMakh

README.md

@@ -35,19 +35,28 @@ Each of these executor images has an execution script (generally a pyxis script)
 
 
 
-
-
-
 ## Requires 
 * [Docker](http://docs.docker.com/)
 * Python
 
 ## Install
+For a system wide install run (requires sudo powers) :
 ```
 pip install penthesilea
 ```
+Alternatively, you can do a local install by enbabling the `--user` flag
+```
+pip install penthesilea --user
+```
+This will install the binaries in `$HOME/.local/bin` and the python packages at `$HOME/.local/lib//python2.7/site-packages`. Finally, add these paths to your *PATH* and *PYTHONPATH* respectively. 
+On my bash shell I add the following to `$HOME/.bashrc` (you do you):
 
-Or
+```
+export PATH=$PATH:$HOME/.local/bin
+export PYTHONPATH=$PYTHONPATH:$HOME/.local/lib//python2.7/site-package
+```
+
+**Or**
 
 Download the repo.
 ```
@@ -57,13 +66,56 @@ Then install
 ```
 cd Penthesilea
 python setup.py install
-penthesilea -pull # This will take some time
-penthesilea -build
 ```
+Similarly, you can add the `--user` to `setup.py` flag to do a local install.
 
 ## Uninstall
 ```
 pip uninstall penthesilea
 ```
 
-See the [wiki](../../wiki/) for tutorials. 
+## For Mac Users
+
+**Prerequisites**
+- install docker for Mac OS X (https://docs.docker.com/v1.8/installation/mac/)
+
+- open a terminal and do the following commands:
+
+```
+docker-machine env default
+eval "$(docker-machine env default)"
+```
+
+This will set up the default docker environment (report to the documentation for user specific environment settings)
+
+
+Using docker on a Mac requires a particular setup and has some limitations
+- Mac OS does not allow folders not owned by the host user to be mounted onto containers. So, on a Mac you have to do the local install. 
+
+- Also, Casacore is known to have I/O issues when trying to read/write a file in a mounted volume on a Mac OS host (see Issue
+[#5](https://github.com/SpheMakh/Penthesilea/issues/5)).
+
+So every I/O operation require additional data management to transfer the files in and out. If you are using a Mac you will have tell penthesilea that. Do this by enabling the `mac_os` flag in the *Pipeline* instance when writing your penthesilea script. 
+
+```
+from otrera import Pipeline
+Pipeline("The one pipeline to rule them all", ..., mac_os=True)
+```
+
+## Building Penthesilea infrastructure
+As mentioned earlier, penthesilea is based on Docker. And before you start scripting penthesilea pipelines you first need to either build or pull the neccessary Docker images (base images). To pull the images from the Docker hub:
+```
+penthesilea pull 
+```
+or build them locally
+```
+penthesilea -build-base
+```
+
+Then finally, build the executor images
+```
+penthesilea -build
+```
+
+
+See the [wiki](../../wiki/) for tutorials.

examples/simulation_pipeline.py

@@ -20,18 +20,26 @@
 
 
 # start oterera instance
-pipeline = Pipeline("Simulation Example", CONFIGS, data=DATA, ms_dir=MSDIR)
+pipeline = Pipeline("Simulation Example", CONFIGS, data=DATA, ms_dir=MSDIR, mac_os=True)
 
 # Make empty MS 
 simms_dict = pipeline.readJson(simms_template)
 simms_dict["msname"] = MS
 simms_dict["telescope"] = "meerkat"
+simms_dict["synthesis"] = 0.5
+simms_dict["direction"] = "J2000,90deg,-45deg"
+simms_dict["dtime"] = 60
+simms_dict["freq0"] = "750MHz"
+simms_dict["dfreq"] = "1MHz"
+simms_dict["nchan"] = 10
 pipeline.add("ares/simms", "simms_example", simms_dict, input=INPUT, output=OUTPUT, 
              label="Creating MS")
 
 # Simulate visibilities into it
 simulator_dict = pipeline.readJson(simulator_template)
 simulator_dict["msname"] = MS
+simulator_dict["addnoise"] = True
+simulator_dict["sefd"] = 831
 simulator_dict["skymodel"] = LSM
 pipeline.add("ares/simulator", "simulator_example", simulator_dict, input=INPUT, output=OUTPUT,
              label="Simulating visibilities")
@@ -42,8 +50,9 @@
 
 imager_dict = pipeline.readJson(imager_template)
 imager_dict["weight"] = "briggs"
+imager_dict["imager"] = "wsclean"
 imager_dict["clean_iterations"] = 1000
-briggs_robust = 2,0,-2
+briggs_robust = 2, 0, -2
 prefix = imager_dict["imageprefix"]
 
 for i, robust in enumerate(briggs_robust):

otrera/__init__.py

@@ -1,5 +1,5 @@
 from pipeliner import Pipeline
 import os
 
-__version__ = "0.0.4"
+__version__ = "0.0.5"
 

otrera/pipeliner.py

@@ -5,11 +5,29 @@
 import sys
 from otrera import penthesilea_docker as docker
 import otrera.utils as utils
+import penthesilea
 import tempfile
+import time
+
+
+ekhaya = penthesilea.__path__[0]
+
+CONFIGS_ = {
+    "ares/simms" : "{:s}/configs/simms_params.json".format(ekhaya),
+    "ares/simulator" : "{:s}/configs/simulator_params.json".format(ekhaya),
+    "ares/imager" : "{:s}/configs/imager_params.json".format(ekhaya),
+    "ares/predict" : "{:s}/configs/simulator_params.json".format(ekhaya),
+    "ares/calibrator" : "{:s}/configs/calibrator_params.json".format(ekhaya),
+    "ares/sourcery" : "{:s}/configs/sourcery_params.json".format(ekhaya),
+    "ares/flagms" : "{:s}/configs/flagms_params.json".format(ekhaya),
+    "ares/autoflagger" : "{:s}/configs/autoflagger_params.json".format(ekhaya),
+    "ares/subtract" : "{:s}/configs/subtract_params.json".format(ekhaya)
+}
+
 
 class Pipeline(object):
 
-    def __init__(self, name, configs, data, ms_dir=None):
+    def __init__(self, name, configs, data, ms_dir=None, mac_os=False):
 
         self.name = name
         self.log = utils.logger(0,
@@ -21,6 +39,7 @@ def __init__(self, name, configs, data, ms_dir=None):
         self.data_path = data
         self.configs_path_container = "/configs"
         self.otrera_path = os.path.dirname(docker.__file__)
+        self.MAC_OS = mac_os
 
         self.ms_dir = ms_dir
         if ms_dir:
@@ -30,14 +49,19 @@ def __init__(self, name, configs, data, ms_dir=None):
 
     def add(self, image, name, config, 
             input=None, output=None, label="", 
-            build_first=False, build_dest=None, saveconf=None):
+            build_first=False, build_dest=None,
+            saveconf=None, add_time_stamp=True):
 
         if build_first and build_dest:
             self.build(image, build_dest)
 
+        if add_time_stamp:
+            name = "%s-%s"%(name, str(time.time()).replace(".",""))
 
         cont = docker.Load(image, name, label=label, logger=self.log)
 
+        cont.add_environ("MAC_OS", str(self.MAC_OS))
+
         # add standard volumes
         cont.add_volume(self.otrera_path, "/utils", perm="ro")
         cont.add_volume(self.data_path, "/data", perm="ro")
@@ -67,7 +91,11 @@ def add(self, image, name, config,
             confname_container = "%s/%s"%(self.configs_path_container, 
                         os.path.basename(saveconf))
 
-            utils.writeJson(saveconf, config)
+
+            template = utils.readJson(CONFIGS_[image])
+            template.update(config)
+            utils.writeJson(saveconf, template)
+
             config = confname_container
             cont.add_volume("configs", self.configs_path_container, perm="ro")
         else:

penthesilea/__init__.py

@@ -1,9 +1,9 @@
 import os
 import otrera
 
-__version__ = otrera.__version__
+#__version__ = otrera.__version__
 
-ekhaya = os.path.dirname(__file__)
+ekhaya = __path__[0]
 STABLE_TAG = "stable.11.15"
 
 # Path to base images
@@ -16,4 +16,5 @@
 PENTHESILEA_DATA = "{:s}/data".format(ekhaya)
 
 # Path to config templates
-PENTHESILEA_CONFIG_TEMPLATES = "{:s}/configs".format(ekhaya)
+PENTHESILEA_CONFIG_TEMPLATES = configs = "{:s}/configs".format(ekhaya)
+

penthesilea/ares/imager/src/pyxis-run.py

@@ -8,9 +8,19 @@
 
 
 INDIR = os.environ["INPUT"]
-v.OUTDIR = os.environ["OUTPUT"]
 CONFIG = os.environ["CONFIG"]
 MSDIR = os.environ["MSDIR"]
+MAC_OS = os.environ["MAC_OS"]
+
+if MAC_OS.lower() in ["yes", "true", "yebo", "1"]:
+    MAC_OS = True
+else:
+    MAC_OS = False
+
+output = "./temp-output-xaba"
+v.OUTDIR = output if MAC_OS else os.environ["OUTPUT"]
+outdir = os.environ["OUTPUT"]
+
 v.DESTDIR = "."
 
 OUTFILE_Template = "${OUTDIR>/}results-${MS:BASE}"
@@ -79,3 +89,6 @@ def azishe():
                   restore_lsm=False, psf=psf,
                   channelize=channelise,
                   **options)
+
+    if MAC_OS:
+        x.sh("mv $output/* $outdir")

penthesilea/ares/simms/Dockerfile

@@ -6,4 +6,4 @@ RUN mkdir -p /output /input
 ADD src /code
 WORKDIR /code
 
-CMD sh run.sh
+CMD python run.py

penthesilea/ares/simms/src/run.py

@@ -0,0 +1,124 @@
+import os
+import sys
+import json
+import pyfits
+import codecs
+
+sys.path.append("/utils")
+import utils
+
+CONFIG = os.environ["CONFIG"]
+INDIR = os.environ["INPUT"]
+MSDIR = os.environ["MSDIR"]
+MAC_OS = os.environ["MAC_OS"]
+
+if MAC_OS.lower() in ["yes", "true", "yebo", "1"]:
+    MAC_OS = True
+else:
+    MAC_OS = False
+
+outfile = "temp.json"
+
+_ANTENNAS = {
+     "meerkat": "meerkat.itrf.txt",
+     "kat-7": "kat-7.itrf.txt",
+     "jvlaa": "vlaa.itrf.txt",
+     "jvlab": "vlab.itrf.txt",
+     "jvlac": "vlac.itrf.txt",
+     "jvlad": "vlad.itrf.txt",
+     "wsrt": "wsrt.itrf.txt",
+     "ska1mid254": "skamid254.itrf.txt",
+     "ska1mid197": "skamid197.itrf.txt",
+}
+
+_OBS = {
+     "meerkat": "meerkat",
+     "kat-7": "kat-7",
+     "jvlaa": "vla",
+     "jvlab": "vla",
+     "jvlac": "vla",
+     "jvlad": "vla",
+     "wsrt": "wsrt",
+     "ska1mid254": "meerkat",
+     "ska1mid197": "meerkat",
+}
+
+
+
+def readJson(conf):
+
+    with open(conf) as _std:
+        jdict = json.load(_std)
+
+    out = {}
+
+    for key,val in jdict.iteritems():
+
+        if isinstance(val, unicode):
+            val = str(val)
+
+        out[str(key)] = val
+
+    return out
+
+
+jdict = readJson(CONFIG)
+telescope = jdict["telescope"]
+
+
+jdict["outdir"] = "." if MAC_OS else MSDIR
+jdict["tel"] = _OBS[telescope]
+msname = jdict["msname"]
+
+if jdict.get("antennas", False):
+    jdict["pos"] = INDIR+"/"+jdict.get("antennas", "meetkat")
+    jdict["coords"] = jdict.get("coord_sys", "enu")
+else:
+    jdict["pos"] = "/data/observatories/"+_ANTENNAS[telescope]
+    if not os.path.isdir(jdict["pos"]):
+        jdict["pos_type"] = "ascii"
+        jdict["coords"] = "itrf"
+
+for item in ["antennas", "coord_sys", "telescope"]:
+    jdict.pop(item, None)
+
+imagename = jdict.pop("skymodel", False)
+
+if jdict.pop("predict", False) and imagename:
+
+    for item in [INDIR, "/data/skymodels"]:
+        _imagename = "{:s}/{:s}".format(item, imagename)
+        if os.path.exists(_imagename):
+            break
+
+    imagename = _imagename
+
+    with pyfits.open(imagename) as hdu:
+        hdr = hdu[0].header
+
+        freq = filter(lambda ind: hdr["ctype%d"%ind].startswith("FREQ"),
+                range(2, 5, 1))
+        if freq:
+            freq = freq[0]
+        else:
+            raise TypeError("Your FITS image has no frequency information")
+
+        nchan = hdr["naxis%d"%freq]
+        freq0 = hdr["crval%d"%freq]
+        dfreq = hdr["cdelt%d"%freq]
+
+        jdict["freq0"] = freq0
+        jdict["dfreq"] = dfreq
+        jdict["nchan"] = nchan
+
+with codecs.open(outfile, "w", "utf8") as std:
+    std.write( json.dumps(jdict, ensure_ascii=False) )
+
+# Run simms
+utils.xrun("simms", ["-jc", outfile])
+
+# move to
+if MAC_OS:
+    if os.path.exists("{:s}/{:s}".format(MSDIR, msname)):
+        utils.xrun("rm", ["-fr", "{:s}/{:s}".format(MSDIR, msname)])
+    utils.xrun("mv", [msname, MSDIR])