Merge pull request #680 from DiamondLightSource/py37_local

Savu python37 move

Author: nicwade

.gitignore

@@ -6,9 +6,9 @@
 
 # Built documenation
 /doc/build
-/doc/source/_autosummary
-scripts/config_generator/test_processes
-scripts/config_generator/test_processes_2
+/doc/source/_autosummary
+scripts/config_generator/test_processes
+scripts/config_generator/test_processes_2
 
 # Python egg metadata, regenerated from source files by setuptools.
 /*.egg-info
@@ -22,5 +22,11 @@ scripts/config_generator/test_processes_2
 
 # nfs failures
 .nfs*
-# vim swap files
-*.swp
+# vim swap files
+*.swp
+# conda build directory
+build/
+
+# VSCode settings
+.vscode
+.vscode/

.travis.yml

@@ -1,39 +1,24 @@
+dist: bionic
 cache: apt
-sudo: true
 language: python
 python:
-  - "2.7"
-#  - "3.5"
-#addons:
-#  apt:
-#    packages:
-#    - libblas-dev
-#    - liblapack-dev
-#    - gfortran
-#    - libhdf5-serial-dev
-#    - libboost-all-dev
+  - 3.7
 
 before_install:
-  - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
-  - bash miniconda.sh -b -p $HOME/miniconda
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - hash -r
-  - conda config --set always_yes yes --set changeps1 no
-  - conda update -q conda
-  - conda info -a
+  - cd $TRAVIS_BUILD_DIR/install/savu_hpc/savu_installer
+  - ls -lah
+#  - touch ~/.condarc
+#  - "echo channel_priority: strict >> ~/.condarc"
 
 install:
-  - conda env create --file environment.yml
-  - source activate test-environment
+  - PREFIX=$HOME bash savu_installer.sh --local --no_prompts
+  - export PATH=$HOME/miniconda/bin:$PATH
+  - export FACILITY=dls
+  - cd $TRAVIS_BUILD_DIR
   - python setup.py install
 
 script:
-  - py.test scripts/configurator_tests/ -v
-  - py.test savu/test/travis -v --cov savu --cov-report term-missing
-
-after_script:
-  - coveralls
-  
-#allow_failures:
-#    - python: "3.5"
-
+  - pip install coverage pytest-cov
+  - cd $TRAVIS_BUILD_DIR
+  - pytest scripts/configurator_tests/ --cov=scripts
+  - pytest savu/test/travis --cov=savu --cov-report term-missing

GUI/GUI_comparison/PyQt/create_plugin_template_v4.py

@@ -96,7 +96,7 @@ def CreateModule(self):
 
 
         f.close
-        print 'done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname
+        print('done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname)
 
 
 # creates the GUI window, with specifications given above 

GUI/GUI_comparison/qml/plugin_template_v6.py

@@ -48,7 +48,7 @@ def on_create_template(plugin_info):
              f.write("\n\n    def get_max_frames(self):\n       return 'multiple'")
 
         f.close    
-        print 'done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname
+        print('done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname)
 
     QGuiApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
     myApp = QApplication(sys.argv)

GUI/plugin_template_wizard/auto_plugin_wizard.py

@@ -13,7 +13,7 @@
 from PyQt5.QtGui import QGuiApplication
 from PyQt5.QtQml import QQmlApplicationEngine
 
-from template import Template   
+from .template import Template   
 
 current_template = Template()
 

GUI/plugin_template_wizard/template.py

@@ -89,4 +89,4 @@ def write_template(self,plugin_info):
 
 
           f.close        
-          print 'done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname
+          print('done, go to :  '+ os.path.dirname(os.path.realpath(__file__)) + '/' + modname)

Jenkinsfile

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# This script should be used to test Local MPI runs on
+# DEVELOPMENT versions of Savu. It takes the path to the
+# Savu source to be used as its first argument, and the rest
+# are forwarded to Savu itself.
+#
+# The Python environment containing all the dependencies of Savu
+# must already be activated.
+#
+# Example usage with full paths:
+# /scratch/dev/savu/bin/savu_mpi_dev.sh /scratch/dev/savu /scratch/dev/savu/test_data/data/24737.nxs /scratch/dev/savu/test_data/process_lists/ica_test.nxs /scratch/output
+#
+# Example usage while inside the repository root:
+# bin/savu_mpi_local_dev.sh . test_data/data/24737.nxs test_data/process_lists/ica_test.nxs /scratch/output
+
+echo "SAVU_MPI_LOCAL:: Running Job with Savu at $1"
+
+nNodes=1
+nCoresPerNode=`lscpu --all --parse=CORE,SOCKET | grep -E "^[0-9]" | wc -l`
+nGPUs=$(nvidia-smi -L | wc -l)
+
+echo "***********************************************"
+echo -e "\tRunning on $nCoresPerNode CPUs and $nGPUs GPUs"
+echo "***********************************************"
+
+savupath=$1
+datafile=$2
+processfile=$3
+outpath=$4
+shift 4
+options=$@
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+echo "savupath is:" $savupath
+
+nCPUs=$((nNodes*nCoresPerNode))
+
+# launch mpi job
+filename=$savupath/savu/tomo_recon.py
+
+echo "running on host: "$HOSTNAME
+echo "Processes running are : ${nCPUs}"
+
+processes=`bc <<< "$nCPUs"`
+
+for i in $(seq 0 $((nGPUs-1))); do GPUs+="GPU$i " ; done
+for i in $(seq 0 $((nCPUs-1-nGPUs))); do CPUs+="CPU$i " ; done
+CPUs=$(echo $GPUs$CPUs | tr ' ' ,)
+
+echo "running the savu mpi local job with process parameters: $CPUs"
+
+echo "Using python at $(which python)"
+
+OMPI_MCA_opal_cuda_support=true PYTHONPATH=$savupath:$PYTHONPATH mpirun -np $nCPUs --use-hwthread-cpus \
+                                                                -mca btl self,vader -mca orte_forward_job_control 1 \
+                                                                python $filename $datafile $processfile $outpath -n $CPUs -v $options
+
+echo "SAVU_MPI_LOCAL:: Process complete"