Merge pull request #306 from openworm/development

Switch Dockerfile to use Python 3

Author: pgleeson

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
 output/C2*
-/rebuild.sh
+/rebuild2.sh
 /output/siminfo.py
 /output/wor*

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 LABEL maintainer="David Lung ([email protected]); Padraig Gleeson ([email protected])"
 
@@ -38,13 +38,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \
   g++ rpm libtool libncurses5-dev zlib1g-dev bison flex lsb-core \
   sudo xorg openbox x11-xserver-utils \
   libxext-dev libncurses-dev python-dev mercurial \
-  freeglut3-dev libglu1-mesa-dev libglew-dev python-dev python-pip python-lxml python-numpy python-scipy python-tk \
+  freeglut3-dev libglu1-mesa-dev libglew-dev python3-dev python3-pip python3-lxml  python3-scipy python3-tk \
   kmod dkms linux-source linux-headers-generic \
   maven openjdk-8-jdk \
-  python-setuptools python-yaml libnuma1 \
-  openmpi-bin libopenmpi-dev \
+  python3-setuptools python3-yaml libnuma1 \
+  openmpi-bin  libopenmpi-dev \
   libgl1-mesa-glx libgl1-mesa-dri libfreetype6-dev \ 
-  libpng12-dev libxft-dev python-matplotlib xubuntu-desktop ffmpeg xvfb tmux
+  libxft-dev python3-matplotlib unzip ffmpeg xvfb tmux
 
 #RUN  sudo pip install --upgrade pip
 #RUN sudo apt-get install nvidia-opencl-dev
@@ -66,36 +66,35 @@ RUN mkdir neuron && \
   git checkout 76c123b && \
   ./build.sh && \
   ./configure --prefix=`pwd` && \
-  make && \
+  make -j3 && \
   sudo make install && \
   cd ../nrn && \
   git checkout e0950a1 && \
   ./build.sh && \
-  ./configure --prefix=`pwd` --with-iv=$HOME/neuron/iv --with-nrnpython=/usr/bin/python --with-paranrn && \
-  make && \
+  ./configure --prefix=`pwd` --with-iv=$HOME/neuron/iv --with-nrnpython=/usr/bin/python3 --with-paranrn && \
+  make -j3 && \
   sudo make install && \
   cd src/nrnpython && \
-  sudo python setup.py install
+  sudo python3 setup.py install
 
 
 ################################################################################
 ########     Install pyNeuroML for handling NeuroML network model
 
 RUN git clone https://github.com/NeuroML/pyNeuroML.git && \
   cd pyNeuroML && \
-  git checkout ow-0.9  && \
-  sudo python setup.py install
+  git checkout master  && \
+  sudo python3 setup.py install
 
 
 ################################################################################
 ########     Install PyOpenWorm
 
-RUN pip install pyparsing==2.0.3 Jinja2==2.11.1 configparser==4.0.2 GitPython==3.0.7 gitdb2==2.0.6 
 RUN git clone https://github.com/openworm/PyOpenWorm.git && \
   cd PyOpenWorm && \
   git checkout ow-0.9 && \
-  sudo apt-get install -y python-cffi && \
-  sudo python setup.py install && \
+  sudo apt-get install -y python3-cffi && \
+  sudo python3 setup.py install && \
   pow clone https://github.com/openworm/OpenWormData.git
 
 
@@ -104,17 +103,16 @@ RUN git clone https://github.com/openworm/PyOpenWorm.git && \
 
 RUN git clone https://github.com/openworm/c302.git && \  
   cd c302 && \
-  git checkout ow-0.9 && \
-  sudo python setup.py install
+  git checkout ow-0.9.1 && \
+  sudo python3 setup.py install
 
 
 ################################################################################
 ########     Install Sibernetic for the worm body model
 
 RUN git clone https://github.com/openworm/sibernetic.git && \
   cd sibernetic && \
-  # fixed to a specific branch
-  git checkout ow-0.9
+  git checkout ow-0.9.1 # fixed to a specific branch
 
 RUN cp c302/pyopenworm.conf sibernetic/   # Temp step until PyOpenWorm can be run from any dir...
 
@@ -155,15 +153,17 @@ RUN mkdir intel-opencl-tmp && \
   sudo rm -r intel-opencl-tmp
 
 RUN sudo cp -R /opt/intel/opencl/include/CL /usr/include/ && \
-sudo apt install -y ocl-icd-opencl-dev
+sudo apt install -y ocl-icd-opencl-dev vim
 #sudo ln -s /opt/intel/opencl/libOpenCL.so.1 /usr/lib/libOpenCL.so
 
 
 ################################################################################
 ########     Build Sibernetic
 
 RUN cd sibernetic && \
-make clean && make all
+    sed -i -e "s/lpython2.7/lpython3.6m/g" makefile && \
+    sed -i -e "s/n2.7/n3.6/g" makefile && \
+    make clean && make all  # Use python 3 libs
 
 # intel i5, hd 5500, linux 4.15.0-39-generic
 # ./Release/Sibernetic -f worm -no_g device=CPU    190ms
@@ -179,3 +179,12 @@ make clean && make all
 ## sudo apt-get update
 ## sudo apt-get install -y cuda-drivers
 # ./Release/Sibernetic -f worm -no_g device=GPU    37ms
+
+
+
+#### TODO: check that this is the best way to switch to py3...
+RUN  sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+
+RUN echo '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alt"' >> ~/.bashrc
+
+RUN echo "Built the OpenWorm Docker image!"

Dockerfile3 renamed to Dockerfile2

@@ -1,7 +1,10 @@
-FROM ubuntu:18.04
+FROM ubuntu:16.04
 
-MAINTAINER David Lung "[email protected]"
+# For generating/running a Dockerfile image based on Python 2
+# Note: Python 2 is no longer officially supported and this Docker image will
+# probably stop working eventually...
 
+LABEL maintainer="David Lung ([email protected]); Padraig Gleeson ([email protected])"
 
 ARG INTEL_SDK_VERSION=2017_7.0.0.2511_x64
 
@@ -31,38 +34,23 @@ ENV DEBIAN_FRONTEND noninteractive # TODO: change
 #RUN useradd -ms /bin/bash $USER
 
 
+################################################################################
+########     Update/install essential libraries
+
 RUN apt-get update && apt-get install -y --no-install-recommends apt-utils \
-  wget \
-  nano \
-  htop \
-  build-essential \
-  make \
-  git \
-  automake \
-  autoconf \
-  g++ \
-  rpm \
-  libtool \
-  libncurses5-dev \
-  zlib1g-dev \
-  bison \
-  flex \
-  lsb-core \
-  sudo \
-  xorg \
-  openbox \
-  x11-xserver-utils \
-  libxext-dev libncurses-dev python3-dev mercurial \
-  freeglut3-dev libglu1-mesa-dev libglew-dev python3-dev python3-pip python3-lxml python3-numpy python3-scipy python3-tk \
-  kmod dkms \
-  linux-source linux-headers-generic \
+  wget nano htop build-essential make git automake autoconf \
+  g++ rpm libtool libncurses5-dev zlib1g-dev bison flex lsb-core \
+  sudo xorg openbox x11-xserver-utils \
+  libxext-dev libncurses-dev python-dev mercurial \
+  freeglut3-dev libglu1-mesa-dev libglew-dev python-dev python-pip python-lxml python-numpy python-scipy python-tk \
+  kmod dkms linux-source linux-headers-generic \
   maven openjdk-8-jdk \
-  python3-setuptools python3-yaml libnuma1 \
-  openmpi-bin  libopenmpi-dev \
-  libgl1-mesa-glx libgl1-mesa-dri libfreetype6-dev \ 
-  libxft-dev python3-matplotlib xubuntu-desktop ffmpeg xvfb tmux
-
+  python-setuptools python-yaml libnuma1 \
+  openmpi-bin libopenmpi-dev \
+  libgl1-mesa-glx libgl1-mesa-dri libfreetype6-dev \
+  libpng12-dev libxft-dev python-matplotlib unzip ffmpeg xvfb tmux
 
+#RUN  sudo pip install --upgrade pip
 #RUN sudo apt-get install nvidia-opencl-dev
 
 RUN sudo usermod -a -G video $USER
@@ -71,6 +59,9 @@ USER $USER
 ENV HOME /home/$USER
 WORKDIR $HOME
 
+################################################################################
+########     Install NEURON simulator
+
 RUN mkdir neuron && \
   cd neuron && \
   git clone https://github.com/nrnhines/iv.git && \
@@ -84,44 +75,57 @@ RUN mkdir neuron && \
   cd ../nrn && \
   git checkout e0950a1 && \
   ./build.sh && \
-  ./configure --prefix=`pwd` --with-iv=$HOME/neuron/iv --with-nrnpython=/usr/bin/python3 --with-paranrn && \
+  ./configure --prefix=`pwd` --with-iv=$HOME/neuron/iv --with-nrnpython=/usr/bin/python --with-paranrn && \
   make -j3 && \
   sudo make install && \
   cd src/nrnpython && \
-  sudo python3 setup.py install
+  sudo python setup.py install
+
+
+################################################################################
+########     Install pyNeuroML for handling NeuroML network model
 
-  
+RUN pip install cachetools==0.8.0
 RUN git clone https://github.com/NeuroML/pyNeuroML.git && \
   cd pyNeuroML && \
-  git checkout ow-0.9  && \
-  sudo python3 setup.py install
+  git checkout master  && \
+  sudo python setup.py install
 
-RUN pip3 install pyparsing==2.0.3
-RUN pip3 install Jinja2==2.11.1
-RUN pip3 install configparser==4.0.2
+
+################################################################################
+########     Install PyOpenWorm
+
+RUN pip install pyparsing==2.0.3 Jinja2==2.11.1 configparser==4.0.2 GitPython==3.0.7 gitdb2==2.0.6 numpydoc==0.9.2 Sphinx==1.8.3 future==0.18.2 setuptools==41.5.1
 RUN git clone https://github.com/openworm/PyOpenWorm.git && \
   cd PyOpenWorm && \
   git checkout ow-0.9 && \
-  sudo apt-get install -y python3-cffi && \
-  sudo python3 setup.py install && \
+  sudo apt-get install -y python-cffi && \
+  sudo python setup.py install && \
   pow clone https://github.com/openworm/OpenWormData.git
 
-###############################RUN pyconfif
 
-RUN git clone https://github.com/openworm/c302.git && \  
+################################################################################
+########     Install c302 for building neuronal network models
+
+RUN git clone https://github.com/openworm/c302.git && \
   cd c302 && \
-  git checkout ow-0.9 && \
-  sudo python3 setup.py install
+  git checkout ow-0.9.1 && \
+  sudo python setup.py install
+
 
+################################################################################
+########     Install Sibernetic for the worm body model
 
-RUN git clone https://github.com/pgleeson/sibernetic.git && \
+RUN git clone https://github.com/openworm/sibernetic.git && \
   cd sibernetic && \
-  # fixed to a specific commit in development branch:
-  # https://github.com/openworm/sibernetic/commit/3eb9914db040fff852cba76ef8f4f39d0bed3294
-  git checkout test_dev 
+  git checkout ow-0.9.1 # fixed to a specific branch
 
 RUN cp c302/pyopenworm.conf sibernetic/   # Temp step until PyOpenWorm can be run from any dir...
 
+
+################################################################################
+########     Set some paths//environment variables
+
 ENV JNML_HOME=$HOME/jNeuroML
 ENV PATH=$PATH:$JNML_HOME
 ENV IV=$HOME/neuron/iv
@@ -138,6 +142,9 @@ COPY ./master_openworm.py $HOME/master_openworm.py
 RUN sudo chown $USER:$USER $HOME/master_openworm.py
 
 
+################################################################################
+########     Install Intel OpenCL libraries needed for Sibernetic
+
 RUN mkdir intel-opencl-tmp && \
   cd intel-opencl-tmp && \
   mkdir intel-opencl && \
@@ -155,9 +162,12 @@ RUN sudo cp -R /opt/intel/opencl/include/CL /usr/include/ && \
 sudo apt install -y ocl-icd-opencl-dev vim
 #sudo ln -s /opt/intel/opencl/libOpenCL.so.1 /usr/lib/libOpenCL.so
 
+
+################################################################################
+########     Build Sibernetic
+
 RUN cd sibernetic && \
-    sed -i -e "s/n2.7/n3.6/g" makefile  # Use python 3 libs && \
-    make clean && make all
+make clean && make all
 
 # intel i5, hd 5500, linux 4.15.0-39-generic
 # ./Release/Sibernetic -f worm -no_g device=CPU    190ms
@@ -175,13 +185,6 @@ RUN cd sibernetic && \
 # ./Release/Sibernetic -f worm -no_g device=GPU    37ms
 
 
-
-
-
-
-#### TODO: check that this is the best way to switch to py3...
-RUN  sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
-
 RUN echo '\n\nalias cd..="cd .."\nalias h=history\nalias ll="ls -alt"' >> ~/.bashrc
 
-RUN echo "Built the OpenWorm Docker image!"
+RUN echo "Built the OpenWorm Docker image!"

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2011 Individual Authors of code
+Copyright (c) 2012-2021 Individual Authors of code
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

build.cmd

@@ -1 +1 @@
-docker build -t "openworm/openworm:0.9" .
+docker build -t "openworm/openworm:0.9.1" .

build.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-docker build "$@" -t "openworm/openworm:0.9" . 
+docker build "$@" -t "openworm/openworm:0.9.1" . 

build2.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# For generating/running a Dockerfile image based on Python 2
+# Note: Python 2 is no longer officially supported and this Docker image will
+# probably stop working eventually...
+
+docker build "$@" -t "openworm/openworm:0.9.1_py2"  -f Dockerfile2 .

build3.sh

@@ -3,7 +3,7 @@
 import matplotlib
 matplotlib.use('Agg')
 import shutil
-from subprocess import call, Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output
 import os
 import pwd
 import shlex
@@ -12,9 +12,9 @@
 import glob
 import math
 
-print("***************************")
-print("OpenWorm Master Script v0.9")
-print("***************************")
+print("*****************************")
+print("OpenWorm Master Script v0.9.1")
+print("*****************************")
 print("")
 print("This script attempts to run a full pass through the OpenWorm scientific libraries.")
 print("This depends on several other repositories being loaded to work and presumes it is running in a preloaded Docker instance.")
@@ -100,7 +100,7 @@ def execute_with_realtime_output(command, directory, env=None):
 
 
 try:
-    if pwd.getpwuid(os.stat(OW_OUT_DIR).st_uid).pw_name != os.environ['USER']:
+    if os.access(OW_OUT_DIR, os.W_OK) is not True:
         os.system('sudo chown -R %s:%s %s' % (os.environ['USER'], os.environ['USER'], OW_OUT_DIR))
 except:
     print("Unexpected error: %s" % sys.exc_info()[0])
@@ -215,7 +215,7 @@ def execute_with_realtime_output(command, directory, env=None):
 
 # Remove black frames at the beginning of the recorded video
 command = "ffmpeg -i %s/%s -vf blackdetect=d=0:pic_th=0.70:pix_th=0.10 -an -f null - 2>&1 | grep blackdetect" % (new_sim_out, sibernetic_movie_name)
-outstr = check_output(command, shell=True)
+outstr = str(check_output(command, shell=True).decode('utf-8'))
 outstr = outstr.split('\n')
 
 black_start = 0.0

master_openworm.py

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build "$@" -t "openworm/openworm:0.9.1"  --no-cache .