v0.5.0 (#161)

v0.5.0

Author: JoshKarpel

.codecov.yml

@@ -0,0 +1,7 @@
+coverage:
+  range: 50..90
+  round: down
+  precision: 0
+  
+comment:
+  layout: "diff, files"

.coveragerc

@@ -6,16 +6,18 @@ include =
 
 omit =
     tests/*
+    htmap/run/*
+    htmap/checkpointing.py
+    htmap/transfer_output.py
 
 [report]
 exclude_lines =
     pragma: no cover
-
+    pragma: unreachable
     def __repr__
-
     def __str__
-
     raise NotImplementedError
-
-    for _ in \[\]:
-
+    raise AssertionError
+    if 0:
+    if False:
+    if __name__ == .__main__.:

.readthedocs.yml

@@ -1,7 +1,14 @@
+version: 2
+
 build:
   image: latest
+  
+sphinx:
+  configuration: docs/source/conf.py
+  
+formats: all
 
 python:
-  version: 3.6
-
-requirements_file: requirements_docs.txt
+  version: 3.7
+  install:
+    - requirements: docs/requirements.txt

.travis.yml

@@ -15,7 +15,7 @@ matrix:
   fast_finish: true
 
 install:
-  - docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=$TRAVIS_PYTHON_VERSION .
+  - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=$TRAVIS_PYTHON_VERSION .
 
 script:
   - docker run htmap-test tests/_inf/travis.sh

MANIFEST.in

@@ -0,0 +1,2 @@
+include LICENSE README.md
+include requirements.txt

requirements_docs.txt docs/requirements.txt

@@ -1,7 +1,8 @@
--r requirements_dev.txt
+-r ../requirements-dev.txt
 
 sphinx
 sphinx_rtd_theme
 sphinx_autodoc_typehints
 pygments-github-lexers
+ipython
 nbsphinx

docs/source/conf.py

@@ -11,7 +11,7 @@
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#
+
 import os
 import sys
 
@@ -48,6 +48,7 @@
     'sphinx.ext.ifconfig',
     'sphinx.ext.viewcode',
     'nbsphinx',
+    'IPython.sphinxext.ipython_console_highlighting',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -170,7 +171,7 @@
 
 autodoc_member_order = 'bysource'
 autoclass_content = 'both'
-autodoc_default_flags = ['undoc-members']
+autodoc_default_options = {"undoc-members": True}
 
 napoleon_use_rtype = False
 

docs/source/devs.rst

@@ -7,10 +7,14 @@ For Developers
 :doc:`devs/env`
    How to set up an environment for development and testing.
 
+:doc:`devs/release`
+   How to release a new version of HTMap.
+
 
 .. toctree::
    :maxdepth: 2
    :hidden:
 
    devs/innards
    devs/env
+   devs/release

docs/source/devs/release.rst

@@ -0,0 +1,21 @@
+How to Release a New HTMap Version
+==================================
+
+.. py:currentmodule:: htmap
+
+
+To release a new version of HTMap:
+
+1. Merge the version PR into ``master`` via GitHub.
+1. Make a GitHub release from https://github.com/htcondor/htmap/releases .
+Name it exactly ``vX.Y.Z``, and link to the release notes for that version
+(like https://htmap.readthedocs.io/en/latest/versions/vX_Y_Z.html )
+in the description (the page will not actually exist yet).
+1. Delete anything in the ``dist/`` directory in your copy of the repository.
+1. On your machine, make sure ``master`` is up-to-date, then run
+``python3 setup.py sdist bdist_wheel`` to create the source distribution
+and the wheel. (This is where the files in ``dist/`` are created.)
+1. Install Twine: ``pip install twine``.
+1. Upload to PyPI:
+``python3 -m twine upload dist/*``.
+You will be prompted for your PyPI login.

docs/source/recipes.rst

@@ -16,6 +16,9 @@ Recipes
 :doc:`recipes/checkpointing-maps`
    How to write a function that can continue from partial progress after being evicted.
 
+:doc:`recipes/using-htmap-on-osg`
+   How to use HTMap on the `Open Science Grid <https://opensciencegrid.org/>`_.
+
 
 .. toctree::
    :maxdepth: 2
@@ -25,3 +28,4 @@ Recipes
    recipes/output-files
    recipes/wrapping-external-programs
    recipes/checkpointing-maps
+   recipes/using-htmap-on-osg

docs/source/recipes/docker-image-cookbook.rst

@@ -125,7 +125,9 @@ Some notes on the above:
 * The ``-y`` options for the ``conda`` commands are the equivalent of answering "yes" to questions that ``conda`` asks on the command line, since the Docker build is non-interactive.
 * A trailing ``\`` is a line continuation, so that first command is equivalent to running ``conda install -y foobar && conda clean -y --all``, which is just ``bash`` shorthand for "do both of these things".
 
-If you need install many packages, we recommend writing a ``requirements.txt`` file (see `the docs <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_) and using
+If you need install many packages, we recommend writing a ``requirements.txt``
+file (see `the Python packaging docs <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_)
+and using
 
 .. code-block:: docker
 

docs/source/tutorials.rst

@@ -3,10 +3,10 @@ Tutorials
 
 .. attention::
 
-    The most convenient way to go through these tutorials is through Binder, which requires no setup on your part.
+    The most convenient way to go through these tutorials is through Binder, which requires no setup on your part: |binder|
 
-    .. image:: https://mybinder.org/badge_logo.svg
-       :target: https://mybinder.org/v2/gh/htcondor/htmap/master?urlpath=lab/tree/tutorials/first-steps.ipynb
+.. |binder| image:: https://mybinder.org/badge_logo.svg
+   :target: https://mybinder.org/v2/gh/htcondor/htmap/master?urlpath=lab/tree/tutorials/first-steps.ipynb
 
 
 :doc:`tutorials/first-steps`

docs/source/versions/v0_5_0.rst

@@ -0,0 +1,60 @@
+v0.5.0
+======
+
+New Features
+------------
+
+* HTMap CLI commands that operate on tags can now pattern-match for tags using
+  glob syntax. Try adding ``-p "<pattern>"`` to commands like ``htmap remove``
+  or ``htmap release``!
+  Issue: https://github.com/htcondor/htmap/issues/159
+* Component status tracking is now preserved between sessions, so it won't be
+  performed from scratch every time. This will only work if the HTCondor Python
+  bindings version is 8.9.3 or greater. You can upgrade your bindings version
+  roughly-independently of HTMap by running ``pip install --upgrade htcondor``.
+  Issue: https://github.com/htcondor/htmap/issues/166
+* :class:`htmap.Map`, :class:`htmap.MapStdOut`, :class:`htmap.MapStdErr`,
+  and :class:`htmap.MapOutputFiles` now all support in the ``in`` operator to check
+  if a component index is in the map.
+
+
+Deprecated Features
+-------------------
+
+* The various iteration methods on :class:`htmap.Map` no longer have a
+  ``callback`` argument.
+
+
+Bug Fixes
+---------
+
+* It should now be much harder to accidentally get a dangling, inaccessible map
+  due to an interrupted ``remove``.
+  Issue: https://github.com/htcondor/htmap/issues/127
+* When an execution errors occurs, the exception and traceback will be printed
+  to stderr execute-side (in addition to being brought back submit-side). This
+  should make some debugging patterns work as expected.
+  Issue: https://github.com/htcondor/htmap/issues/178
+* The CLI command ``htmap status --live`` now has much better behavior when
+  the table width is nearly the width of the terminal. It should now never wrap
+  unless the table is actually wider than the terminal, instead of a few
+  characters before the actual width.
+* HTMap now handles late materialized jobs much more smoothly: maps with
+  unmaterialized components can be removed, and various CLI commands that output
+  color won't fail when acting on maps with unmaterialized components. However,
+  unmaterialized components do not show as `IDLE`, which mirrors the behavior
+  of `condor_q`. This does make it hard to know how many components are in a
+  late-materialized map at a glance; we are thinking about how to address this.
+  Issue: https://github.com/htcondor/htmap/issues/158
+
+
+Known Issues
+------------
+
+* Execution errors that result in the job being terminated but no output being
+  produced are still not handled entirely gracefully. Right now, the component
+  state will just show as ``ERRORED``, but there won't be an actual error report.
+* Map component state may become corrupted when a map is manually vacated.
+  Force-removal may be needed to clean up maps if HTCondor and HTMap disagree
+  about the state of their components.
+  Issue: https://github.com/htcondor/htmap/issues/129

dr.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+SET CONTAINER_TAG=htmap-tests
+
+ECHO Building HTMap testing container...
+
+docker build --quiet -t %CONTAINER_TAG% --file tests/_inf/Dockerfile .
+docker run -it --rm %CONTAINER_TAG% %*

htmap-exec/Dockerfile

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM continuumio/anaconda3:2019.03
+FROM continuumio/anaconda3:2019.07
+ENV PATH=/opt/conda/bin/:${PATH}
 
 LABEL maintainer="[email protected]"
 

htmap/__init__.py

@@ -25,9 +25,9 @@
 from .settings import settings, USER_SETTINGS, BASE_SETTINGS
 
 # SET UP NULL LOG HANDLER
-logger = _logging.getLogger(__name__)
-logger.setLevel(_logging.DEBUG)
-logger.addHandler(_logging.NullHandler())
+_logger = _logging.getLogger(__name__)
+_logger.setLevel(_logging.DEBUG)
+_logger.addHandler(_logging.NullHandler())
 
 from .mapping import (
     map,
@@ -60,8 +60,8 @@
 )
 from .tags import get_tags
 from .checkpointing import checkpoint
-from .output_files import transfer_output_files
-from .transfer import TransferPath, TransferWindowsPath, TransferPosixPath
+from .transfer_output import transfer_output_files
+from .transfer_input import TransferPath, TransferWindowsPath, TransferPosixPath
 from . import exceptions
 
 from . import _startup

htmap/_startup.py

@@ -7,12 +7,11 @@
 
 logger = logging.getLogger('htmap')
 
-LOGS_DIR_PATH = Path(settings['HTMAP_DIR']) / names.LOGS_DIR
-
 
 def setup_internal_file_logger():
-    LOGS_DIR_PATH.mkdir(parents = True, exist_ok = True)
+    LOGS_DIR_PATH = Path(settings['HTMAP_DIR']) / names.LOGS_DIR
     LOG_FILE = LOGS_DIR_PATH / 'htmap.log'
+    LOGS_DIR_PATH.mkdir(parents = True, exist_ok = True)
     _logfile_handler = handlers.RotatingFileHandler(
         filename = LOG_FILE,
         mode = 'a',
@@ -36,15 +35,16 @@ def ensure_htmap_dir_exists():
             _htmap_dir,
             _htmap_dir / _names.MAPS_DIR,
             _htmap_dir / _names.TAGS_DIR,
-            _htmap_dir / _names.REMOVED_TAGS_DIR
+            _htmap_dir / _names.LOGS_DIR,
+            _htmap_dir / _names.REMOVED_TAGS_DIR,
         )
         for dir in dirs:
             dir.mkdir(parents = True, exist_ok = True)
 
         if did_not_exist:
-            logger.debug(f'created HTMap dir at {_htmap_dir}')
+            logger.debug(f'Created HTMap dir at {_htmap_dir}')
     except PermissionError as e:
-        raise PermissionError(f'the HTMap directory ({_htmap_dir}) needs to be writable') from e
+        raise PermissionError(f'The HTMap directory ({_htmap_dir}) needs to be writable') from e
 
 
 if os.getenv('HTMAP_ON_EXECUTE') != '1':

htmap/checkpointing.py

@@ -20,18 +20,20 @@
 from . import names
 
 
-def checkpoint(*paths: os.PathLike):
+def checkpoint(*paths: os.PathLike) -> None:
     """
     Informs HTMap about the existence of checkpoint files.
     This function should be called every time the checkpoint files are changed, even if they have the same names as before.
 
     .. attention::
 
-        This function is a no-op when executing locally, so you if you're testing your function it won't do anything.
+        This function is a no-op when executing locally (i.e., not execute-side),
+        so you if you're testing your function locally it won't do anything.
 
     .. attention::
 
-        The files will be copied, so try not to make the checkpoint files too large.
+        The files will be **copied** by this function,
+        so try not to make the checkpoint files too large.
 
     Parameters
     ----------