ENH: Add publishing to PyPi to continuous integration

JoostJM · JoostJM · commit e6afeda3cbca · 2018-07-04T17:45:35.000+02:00
Adds a deploy step for all 3 ci-servers used by pyradiomics (supporting Linux, Windows and Mac OSX).
The deployment step is only run for tagged releases, i.e. when a build is triggered for a tag.
Furthermore, this tag is checked against a regex expression, to restrict deployment only to releases (e.g. v1.0, v2.1.3) and/or release candidates (e.g. v1.0-rc1, v2.1.3-rc12).

Authentication to the PyPi server is done using (private) environment variables set in the accounts on the various CI-servers. These variables will be only available to builds on branches/tags in the repository that is linked to the CI-account (i.e. not available for forked builds).

Finally, the current deployment step publishes to the test server, rather than the live server. This is done for testing of the workflow. Prior to integration into the master this will be rectified.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,13 +10,16 @@ jobs:
       - image: jupyter/datascience-notebook
         user: root
     steps:
-      - checkout
       - run:
           # CircleCI says we need SSH and the docker doesn't have it installed
           name: Update
           command: |
             apt-get update
             apt-get install -y openssh-client openssh-server
+      - run:
+          name: Avoid hosts unknown for github
+          command: echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > /etc/ssh/ssh_config
+      - checkout
       - run:
           # Jupyter datascience notebook does not support python 2 anymore, install it manually.
           # See also https://github.com/jupyter/docker-stacks/issues/432
@@ -55,3 +58,82 @@ jobs:
             jupyter nbconvert --ExecutePreprocessor.kernel_name=python2 --ExecutePreprocessor.timeout=-1 --to notebook --output-dir /tmp --execute notebooks/helloRadiomics.ipynb notebooks/helloFeatureClass.ipynb notebooks/PyRadiomicsExample.ipynb
 
             jupyter nbconvert --ExecutePreprocessor.kernel_name=python3 --ExecutePreprocessor.timeout=-1 --to notebook --output-dir /tmp --execute  notebooks/helloRadiomics.ipynb notebooks/helloFeatureClass.ipynb notebooks/PyRadiomicsExample.ipynb
+  deploy:
+    working_directory: /pyradiomics
+    docker:
+      - image: jupyter/datascience-notebook
+        user: root
+    steps:
+      - run:
+          # CircleCI says we need SSH and the docker doesn't have it installed
+          name: Update
+          command: |
+            apt-get update
+            apt-get install -y openssh-client openssh-server
+      - run:
+          name: Avoid hosts unknown for github
+          command: echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > /etc/ssh/ssh_config
+      - checkout
+      - run:
+          # Jupyter datascience notebook does not support python 2 anymore, install it manually.
+          # See also https://github.com/jupyter/docker-stacks/issues/432
+          # Next, install python 2 kernel globally, so it can be found from the root
+          name: Install Python 2 Kernel
+          command: |
+            conda create -n python2 python=2 ipykernel
+            pip install kernda --no-cache
+            $CONDA_DIR/envs/python2/bin/python -m ipykernel install
+            kernda -o -y /usr/local/share/jupyter/kernels/python2/kernel.json
+            pip uninstall kernda -y
+      - run:
+          name: Install pyradiomics in Python 2 and 3
+          command: |
+            source activate python2
+            python -m pip install --no-cache-dir -r requirements.txt
+            python -m pip install --no-cache-dir -r requirements-dev.txt
+            python setup.py install
+            source activate root
+            python -m pip install --no-cache-dir -r requirements.txt
+            python -m pip install --no-cache-dir -r requirements-dev.txt
+            python setup.py install
+      - run:
+          name: install twine, auditwheel
+          command: |
+            python -m pip install auditwheel
+            python -m pip install twine
+      - run:
+          name: creat source and wheel distribution
+          command: |
+            source activate python2
+            python setup.py bdist_wheel
+            source activate root
+            python setup.py sdist bdist_wheel
+            # Since there are no external shared libraries to bundle into the wheels
+            # this step will fixup the wheel switching from 'linux' to 'manylinux1' tag
+            for whl in dist/*$(uname -p).whl; do
+              auditwheel repair $whl -w ./dist/
+              rm $whl
+            done
+      - run:
+          name: deploy source and linux wheels
+          command: twine upload --repository-url https://test.pypi.org/legacy/ ./dist/*.whl ./dist/*.tar.gz -u $PYPI_USER -p $PYPI_PASSWORD
+
+workflows:
+  version: 2
+  build_and_deploy:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only:
+                - /^v?[0-9]+(\.[0-9]+)*(-rc[0-9]+)?/
+      - deploy:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore:
+                - /.*/
+            tags:
+              only:
+                - /^v?[0-9]+(\.[0-9]+)*(-rc[0-9]+)?/
diff --git a/.travis.yml b/.travis.yml
@@ -1,9 +1,5 @@
 # Config file for automatic testing at travis-ci.org
 
-branches:
- only:
-  - master
-
 language: python
 
 matrix:
@@ -44,3 +40,13 @@ script:
 
 after_success:
   - ci after_test
+
+before_deploy:
+  - sudo pip install twine  # Twine installation requires sudo to get access to /usr/local/man
+deploy:
+  provider: script
+  skip_cleanup: true
+  script: twine upload --repository-url https://test.pypi.org/legacy/ dist/*.whl -u $PYPI_USER -p $PYPI_PASSWORD
+  on:
+    tags: true
+    condition: $TRAVIS_TAG =~ ^v?[0-9]+(\.[0-9]+)*(-rc[0-9]+)?$
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,4 +1,4 @@
-include CONTRIBUTING.md
+include CONTRIBUTING.rst
 include LICENSE.txt
 include README.md
 include requirements.txt
@@ -8,10 +8,20 @@ include versioneer.py
 
 recursive-include radiomics *
 
-recursive-include data *
-exclude data/PyradiomicsFeatures.csv data/Baseline2PyradiomicsFeaturesDiff.csv
+recursive-include data/baseline *
+recursive-include data *_image.nrrd
+recursive-include data *_label.nrrd
+include data/README.md
+
+recursive-include examples/exampleSettings *.yaml
+recursive-include examples batch*.py
+recursive-include examples hello*.py
+include examples/testCases.csv
 
 recursive-include tests *
 
+recursive-include bin *.py
+
 recursive-exclude * __pycache__
 recursive-exclude * *.py[cod]
+recursive-exclude * nosetests.xml
diff --git a/appveyor.yml b/appveyor.yml
@@ -1,7 +1,3 @@
-branches:
- only:
-  - master
-
 version: "0.0.1.{build}"
 
 environment:
@@ -27,6 +23,7 @@ environment:
 init:
   - python -m pip install scikit-ci==0.13.0 scikit-ci-addons==0.11.0
   - python -m ci_addons --install ../addons
+  - python -m pip install twine
 
   - ps: ../addons/appveyor/rolling-build.ps1
 
@@ -42,8 +39,19 @@ test_script:
 after_test:
   - python -m ci after_test
 
+artifacts:
+  - path: dist/*
+    name: pypiartefacts
+
 on_finish:
   - ps: ../addons/appveyor/enable-worker-remote-access.ps1 -check_for_block
 
+deploy_script:
+  - echo "checking deployment"
+  - ps: if ($env:APPVEYOR_REPO_TAG_NAME -notmatch '^v?\d(\.\d)*(-rc\d+)?$') { appveyor exit }
+  - echo "starting deployment"
+  - twine upload --repository-url https://test.pypi.org/legacy/ dist/*.whl -u %PYPI_USER% -p %PYPI_PASSWORD%
+  - echo "finished deployment"
+
 matrix:
   fast_finish: false
diff --git a/setup.cfg b/setup.cfg
@@ -1,3 +1,6 @@
+[metadata]
+description-file = README.md
+
 [nosetests]
 verbosity=3
 where=tests
diff --git a/setup.py b/setup.py
@@ -25,6 +25,9 @@
 with open('requirements-setup.txt', 'r') as fp:
   setup_requirements = list(filter(bool, (line.strip() for line in fp)))
 
+with open('README.md', 'r') as fp:
+  long_description = fp.read()
+
 
 class NoseTestCommand(TestCommand):
   """Command to run unit tests using nose driver after in-place build"""
