Merge pull request #196 from htcondor/v0.6.0

v0.6.0

Author: JoshKarpel

.coveragerc

@@ -5,17 +5,16 @@ data_file = /tmp/htmap-test-coverage
 
 include =
     htmap/*
+    tests/*
 
 omit =
-    tests/*
     htmap/run/*
-    htmap/checkpointing.py
-    htmap/transfer_output.py
 
 [report]
 exclude_lines =
     pragma: no cover
     pragma: unreachable
+    pragma: execute-only
     def __repr__
     def __str__
     raise NotImplementedError

.gitignore

@@ -110,3 +110,4 @@ venv.bak/
 !htmap-exec/singularity.d/*
 
 prof/
+docs/source/tutorials/*.txt

.readthedocs.yml

@@ -11,4 +11,4 @@ formats: all
 python:
   version: 3.7
   install:
-    - requirements: docs/requirements.txt
+    - requirements: requirements-docs.txt

.travis.yml

@@ -16,8 +16,7 @@ jobs:
   fast_finish: true
 
 install:
-  - pip install codecov
-  - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} .
+  - travis_retry docker build -t htmap-test --file docker/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} .
 
 script:
-  - docker run --mount type=bind,src="$PWD",dst=/home/mapper/htmap,readonly htmap-test bash tests/_inf/travis.sh
+  - docker run --mount type=bind,src="$PWD",dst=/home/mapper/htmap,readonly htmap-test bash tests/travis.sh

CONTRIBUTORS

@@ -0,0 +1,2 @@
+Joshua Karpel
+Scott Sievert

README.md

@@ -4,11 +4,20 @@
 [![Build Status](https://travis-ci.org/htcondor/htmap.svg?branch=master)](https://travis-ci.org/htcondor/htmap)
 [![codecov](https://codecov.io/gh/htcondor/htmap/branch/master/graph/badge.svg)](https://codecov.io/gh/htcondor/htmap)
 
-**Jump right in to the tutorials with Binder!** [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/htcondor/htmap/master?urlpath=lab%2Ftree%2Ffirst-steps.ipynb)
+**Jump right in to the tutorials with Binder!** 
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/htcondor/htmap/master?urlpath=lab%2Ftree%2Ffirst-steps.ipynb)
 
-HTMap is a library that wraps the process of mapping Python function calls out to an HTCondor pool.
-It provides tools for submitting, managing, and processing the output of arbitrary functions.
+HTMap is a library that wraps the process of mapping Python function calls out 
+to an HTCondor pool.
+It provides tools for submitting, managing, and processing the output of 
+arbitrary functions.
 
-Our goal is to provide as transparent an interface as possible to high-throughput computing resources so that you can spend more time thinking about your own code, and less about how to get it running on a cluster.
+Our goal is to provide as transparent an interface as possible to 
+high-throughput computing resources so that you can spend more time thinking 
+about your own code, and less about how to get it running on a cluster.
 
-For tutorials, installation instructions, and more details, see the [documentation](https://htmap.readthedocs.io/en/latest/).
+For tutorials, installation instructions, and more details, see the 
+[documentation](https://htmap.readthedocs.io/en/latest/).
+
+Please post bug reports and feature requests to 
+[our issue tracker](https://github.com/htcondor/htmap/issues).

binder/Dockerfile

@@ -14,19 +14,22 @@
 # limitations under the License.
 
 FROM htcondor/htc-minimal-notebook:latest
-ENV NB_USER="jovyan"
 
-USER ${NB_USER}
+USER root
 
-# copy the tutorial notebooks into the container
-COPY --chown=jovyan:0 docs/source/tutorials ${HOME}/tutorials
+RUN echo '#!/bin/bash\nfind ${HOME}/tutorials -name '\''*.ipynb'\'' -and -not -iname '\''*-checkpoint.ipynb'\''' > /usr/bin/find_notebooks \
+ && chmod +x /usr/bin/find_notebooks
 
-# use the repository version of HTMap, not whatever was in the htc notebook
-# also, strip any run results out of the tutorial notebooks
-COPY --chown=jovyan:0 . /tmp/htmap
-RUN pip install --no-cache-dir nbstripout \
+# Use the repository version of HTMap, not whatever was in the htc-notebook.
+COPY . ${HOME}/htmap
+RUN chown -R ${NB_UID}:${NB_GID} ${HOME}/htmap
+
+USER ${NB_UID}:${NB_GID}
+
+# Install HTMap and strip any run results out of the tutorial notebooks.
+RUN cp -r ${HOME}/htmap/docs/source/tutorials ${HOME}/tutorials \
+ && pip install --no-cache-dir nbstripout \
  && nbstripout ${HOME}/tutorials/* \
- && pip install --no-cache-dir --upgrade /tmp/htmap \
- && rm -rf /tmp/htmap
+ && pip install --no-cache-dir --upgrade -e ${HOME}/htmap
 
 WORKDIR ${HOME}/tutorials

binder/condor_config.local

@@ -4,7 +4,5 @@ set -e
 
 CONTAINER_TAG=htmap-binder-edit
 
-echo "Building HTMap Binder container..."
-
 docker build -t ${CONTAINER_TAG} --file binder/Dockerfile .
 docker run -it --rm -p 8888:8888 --mount type=bind,source="$(pwd)"/docs/source/tutorials,target=/home/jovyan/tutorials ${CONTAINER_TAG}

binder/edit.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+CONTAINER_TAG=htmap-binder-exec
+
+docker build -t ${CONTAINER_TAG} --file binder/Dockerfile .
+docker run --rm --mount type=bind,source="$(pwd)"/docs/source/tutorials,target=/home/jovyan/tutorials ${CONTAINER_TAG} -- bash -l -c 'sleep 5 && condor_who -wait:60 "IsReady && STARTD_State =?= \"Ready\"" && rm -r /home/jovyan/tutorials/*.txt ; for x in $(find_notebooks); do nbstripout $x && jupyter nbconvert --to notebook --inplace --execute --allow-errors --ExecutePreprocessor.timeout=None $x && htmap remove --all ; done && rm -r /home/jovyan/tutorials/*.txt'

binder/exec.sh

@@ -4,7 +4,5 @@ set -e
 
 CONTAINER_TAG=htmap-binder-test
 
-echo "Building HTMap Binder container..."
-
 docker build -t ${CONTAINER_TAG} --file binder/Dockerfile .
 docker run -it --rm -p 8888:8888 ${CONTAINER_TAG} "$@"

binder/test.sh binder/run.sh

@@ -0,0 +1 @@
+DELIVERY_METHOD = "shared"

binder/test.cmd

@@ -19,7 +19,7 @@ ARG PYTHON_VERSION=3.6
 FROM python:${PYTHON_VERSION}
 
 # build config
-ARG HTCONDOR_VERSION=8.8
+ARG HTCONDOR_VERSION=8.9
 
 # switch to root to do root-level config
 USER root
@@ -29,7 +29,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # install utils and dependencies
 RUN apt-get update \
- && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz \
+ && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz pandoc strace \
  && apt-get -y clean \
  && rm -rf /var/lib/apt/lists/* \
  && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
@@ -48,7 +48,7 @@ RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg
  && rm -rf /var/lib/apt/lists/*
 
 # copy entrypoint into place and make executable
-COPY tests/_inf/entrypoint.sh /.entrypoint.sh
+COPY docker/entrypoint.sh /.entrypoint.sh
 RUN chmod +x /.entrypoint.sh
 
 # create a user, set their PATH and PYTHONPATH
@@ -61,17 +61,17 @@ RUN groupadd ${SUBMIT_USER} \
 # switch to the user, don't need root anymore
 USER ${SUBMIT_USER}
 
-# install htmap dependencies and debugging tools early for docker build caching
+# install dependencies and debugging tools early for docker build caching
 COPY requirements* /tmp/
-RUN pip install --user --no-cache-dir -r /tmp/requirements-dev.txt \
- && pip install --user --no-cache-dir ipython \
- && pip install --user --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.*
+RUN python -m pip install --user --no-cache-dir -r /tmp/requirements-dev.txt -r /tmp/requirements-docs.txt ipython \
+ && python -m pip install --user --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.*
 
-# copy HTCondor and HTMap testing configs into place
-COPY tests/_inf/condor_config.local /etc/condor/condor_config.local
+# copy testing configs into place
+COPY docker/condor_config.local /etc/condor/condor_config.local
+COPY --chown=mapper:mapper docker/.htmaprc /home/${SUBMIT_USER}/
 
 # set default entrypoint and command
 # the entrypoint is critical: it starts HTCondor in the container
 WORKDIR /home/${SUBMIT_USER}/htmap
 ENTRYPOINT ["/.entrypoint.sh"]
-CMD ["pytest"]
+CMD ["bash"]

docker/.htmaprc

@@ -4,8 +4,13 @@ use ROLE: CentralManager
 use ROLE: Submit
 use ROLE: Execute
 
+# Logging setup
+#ALL_DEBUG = D_ALL
+#TOOL_DEBUG = D_ALL
+STARTER_LOG_NAME_APPEND=jobid
+
 # Edit paths so all state gets stored in user home directory
-LOCAL_DIR=$ENV(HOME)/.condor/state
+LOCAL_DIR=$ENV(HOME)/.condor/local
 LOCK=$(LOCAL_DIR)/lock
 LOG=$(LOCAL_DIR)/log
 RUN=$(LOCAL_DIR)/run
@@ -33,6 +38,7 @@ SCHED_UNIV_RENICE_INCREMENT=5
 SHADOW_RENICE_INCREMENT=5
 
 # Get the HTMap source into the Python path
+# We do this instead of doing an editable pip install because we don't necessarily want to write to the git dir
 JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SetPyVars
 JOB_TRANSFORM_SetPyVars @=end
 [

tests/_inf/Dockerfile docker/Dockerfile

@@ -8,7 +8,8 @@ mkdir -p "$_condor_local_dir/lock" "$_condor_local_dir/log" "$_condor_local_dir/
 
 # start condor
 condor_master
-echo "Starting HTCondor..."
+
+condor_version
 
 # once the shared port daemon wakes up, use condor_who to wait for condor to stand up
 while [[ ! -s "${_condor_local_dir}/log/SharedPortLog" ]]
@@ -18,13 +19,4 @@ done
 sleep 1  # fudge factor to let shared port *actually* wake up
 condor_who -wait:60 'IsReady && STARTD_State =?= "Ready"' > /dev/null
 
-if [[ -n $@ ]];
-then
-    echo "Executing command \"$@\" in container..."
-else
-    echo "Executing default test command in container..."
-fi
-
-condor_version
-
 exec "$@"

tests/_inf/condor_config.local docker/condor_config.local

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -e
+
+export PYTHONPATH="$PWD:$PYTHONPATH"
+
+echo "NOTE: CONNECT TO http://127.0.0.1:8000 NOT WHAT SPHINX-AUTOBUILD TELLS YOU"
+sleep 3
+
+sphinx-autobuild docs/source docs/_build --host 0.0.0.0 --poll --watch htmap/

tests/_inf/entrypoint.sh docker/entrypoint.sh

@@ -149,15 +149,11 @@ Additionally, default map options can be set globally via ``settings['MAP_OPTION
 .. autoclass:: htmap.MapOptions
    :members:
 
-Input File Transfer
+File Transfer
 -------------------
 
 .. autoclass:: htmap.TransferPath
 
-
-Output File Transfer
---------------------
-
 .. autofunction:: htmap.transfer_output_files
 
 
@@ -194,6 +190,10 @@ These functions are useful for generating machine-readable status information.
 
 .. autofunction:: htmap.status_csv
 
+Delivery Methods
+----------------
+
+.. autofunction:: htmap.register_delivery_method
 
 Transplant Installs
 +++++++++++++++++++

docs/autobuild.sh

@@ -0,0 +1,23 @@
+.. _cli:
+
+CLI Reference
+=============
+
+HTMap provides a command line tool called ``htmap`` that exposes a subset
+of functionality focused around monitoring long-running maps without needing
+to run Python yourself.
+
+View the available sub-commands by running:
+
+.. code:: shell
+
+   htmap --help  # View available commands
+
+Some useful commands are highlighted in the Tips and Tricks section at
+:ref:`cli-tips`.
+
+Here's the full documentation on all of the available commands:
+
+.. click:: htmap.cli:cli
+   :prog: htmap
+   :show-nested:

docs/source/api.rst

@@ -22,7 +22,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'HTMap'
-copyright = '2018, HTCondor Team, Computer Sciences Department, University of Wisconsin-Madison, WI'
+copyright = '2018-2020, HTCondor Team, Computer Sciences Department, University of Wisconsin-Madison, WI'
 author = 'HTCondor Team'
 
 # The short X.Y version
@@ -49,6 +49,8 @@
     'sphinx.ext.viewcode',
     'nbsphinx',
     'IPython.sphinxext.ipython_console_highlighting',
+    'sphinx_click.ext',
+    'sphinx_issues',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -175,6 +177,7 @@
 
 napoleon_use_rtype = False
 
+issues_github_path = "htcondor/htmap"
 
 def setup(app):
     app.add_stylesheet('css/custom.css')