v0.5.1 (#189)

* Provide htcondor module version info as a constant instead of a function

* Test infrastructure update (#188)

* resolve #186

Author: JoshKarpel

.codecov.yml

@@ -1,6 +1,8 @@
 [run]
 branch = True
 
+data_file = /tmp/htmap-test-coverage
+
 include =
     htmap/*
 

.coveragerc

@@ -108,3 +108,5 @@ venv.bak/
 .vscode/
 
 !htmap-exec/singularity.d/*
+
+prof/

.gitignore

@@ -1,21 +1,23 @@
+os: linux
 dist: xenial
-sudo: required
 services:
   - docker
 
 language: python
 python:
   - "3.6"
   - "3.7"
+#  - "3.8"
 env:
   - HTCONDOR_VERSION=8.8
   - HTCONDOR_VERSION=8.9
 
-matrix:
+jobs:
   fast_finish: true
 
 install:
-  - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=$TRAVIS_PYTHON_VERSION .
+  - pip install codecov
+  - travis_retry docker build -t htmap-test --file tests/_inf/Dockerfile --build-arg HTCONDOR_VERSION --build-arg PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} .
 
 script:
-  - docker run htmap-test tests/_inf/travis.sh
+  - docker run --mount type=bind,src="$PWD",dst=/home/mapper/htmap,readonly htmap-test bash tests/_inf/travis.sh

.travis.yml

@@ -0,0 +1,11 @@
+coverage:
+  range: 50..90
+  round: down
+  precision: 0
+  status:
+    project:
+      default:
+        threshold: 1%  # allow coverage to drop by up to 1% in a PR before marking it failed
+  
+comment:
+  layout: "diff, files"

codecov.yml

@@ -6,16 +6,16 @@ How to Release a New HTMap Version
 
 To release a new version of HTMap:
 
-1. Merge the version PR into ``master`` via GitHub.
-1. Make a GitHub release from https://github.com/htcondor/htmap/releases .
-Name it exactly ``vX.Y.Z``, and link to the release notes for that version
-(like https://htmap.readthedocs.io/en/latest/versions/vX_Y_Z.html )
-in the description (the page will not actually exist yet).
-1. Delete anything in the ``dist/`` directory in your copy of the repository.
-1. On your machine, make sure ``master`` is up-to-date, then run
-``python3 setup.py sdist bdist_wheel`` to create the source distribution
-and the wheel. (This is where the files in ``dist/`` are created.)
-1. Install Twine: ``pip install twine``.
-1. Upload to PyPI:
-``python3 -m twine upload dist/*``.
-You will be prompted for your PyPI login.
+#. Merge the version PR into ``master`` via GitHub.
+#. Make a GitHub release from https://github.com/htcondor/htmap/releases .
+   Name it exactly ``vX.Y.Z``, and link to the release notes for that version
+   (like https://htmap.readthedocs.io/en/latest/versions/vX_Y_Z.html )
+   in the description (the page will not actually exist yet).
+#. Delete anything in the ``dist/`` directory in your copy of the repository.
+#. On your machine, make sure ``master`` is up-to-date, then run
+   ``python3 setup.py sdist bdist_wheel`` to create the source distribution
+   and the wheel. (This is where the files in ``dist/`` are created.)
+#. Install Twine: ``pip install twine``.
+#. Upload to PyPI:
+   ``python3 -m twine upload dist/*``.
+   You will be prompted for your PyPI login.

docs/source/devs/release.rst

@@ -0,0 +1,29 @@
+v0.5.1
+======
+
+New Features
+------------
+
+
+Deprecated Features
+-------------------
+
+
+Bug Fixes
+---------
+
+* Maps can now be force-removed even if the schedd cannot be contacted.
+  Graceful removal still requires contact with the schedd.
+  Issue: https://github.com/htcondor/htmap/issues/186
+
+
+Known Issues
+------------
+
+* Execution errors that result in the job being terminated but no output being
+  produced are still not handled entirely gracefully. Right now, the component
+  state will just show as ``ERRORED``, but there won't be an actual error report.
+* Map component state may become corrupted when a map is manually vacated.
+  Force-removal may be needed to clean up maps if HTCondor and HTMap disagree
+  about the state of their components.
+  Issue: https://github.com/htcondor/htmap/issues/129

docs/source/versions/v0_5_1.rst

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-CONTAINER_TAG=htmap-tests
+CONTAINER_TAG='htmap-tests'
 
 set -e
 echo "Building HTMap testing container..."
 docker build --quiet -t ${CONTAINER_TAG} --file tests/_inf/Dockerfile .
-docker run -it --rm ${CONTAINER_TAG} $@
+docker run -it --rm --mount type=bind,src="$PWD",dst=/home/mapper/htmap ${CONTAINER_TAG} $@

dr

@@ -262,7 +262,7 @@ def wait(
 
                 previous_pbar_len = 0
 
-            ok_statuses = set([state.ComponentStatus.COMPLETED])
+            ok_statuses = {state.ComponentStatus.COMPLETED}
             if holds_ok:
                 ok_statuses.add(state.ComponentStatus.HELD)
             if errors_ok:
@@ -707,7 +707,14 @@ def remove(self, force: bool = False) -> None:
         force
             If ``True``, do not wait for HTCondor to confirm that all map components have been removed.
         """
-        self._remove_from_queue()
+        try:
+            self._remove_from_queue()
+        except Exception as e:
+            if not force:
+                raise e
+
+            logger.exception(f"Encountered error while force-removing map {self.tag}; ignoring and moving to cleanup step")
+
         self._cleanup_local_data(force = force)
         MAPS.remove(self)
 
@@ -736,7 +743,7 @@ def _cleanup_local_data(self, force: bool = False) -> None:
                 )
                 for cs in self.component_statuses
             ):
-                time.sleep(.01)
+                time.sleep(settings["WAIT_TIME"])
 
         # move the tagfile to the removed tags dir
         # renamed by uid to prevent duplicates
@@ -757,8 +764,8 @@ def _cleanup_local_data(self, force: bool = False) -> None:
                 logger.debug(f'Removed tag file for map {self.tag}')
                 return  # break out of the loop
             except OSError:
-                logger.exception(f'Failed to remove map directory for map {self.tag}, retrying in .1 seconds')
-                time.sleep(.1)
+                logger.exception(f'Failed to remove map directory for map {self.tag}, retrying in {settings["WAIT_TIME"]} seconds')
+                time.sleep(settings["WAIT_TIME"])
         logger.error(f'Failed to remove map directory for map {self.tag}, run htmap.clean() to try to remove later')
 
     @property

htmap/maps.py

@@ -170,7 +170,7 @@ def _read_events(self):
             if handled_events:
                 self.map._local_data = None  # invalidate cache if any events were received
 
-                if utils.htcondor_version_info() >= (8, 9, 3):
+                if utils.HTCONDOR_VERSION_INFO >= (8, 9, 3):
                     self.save()
 
     def save(self) -> Path:
@@ -188,7 +188,7 @@ def save(self) -> Path:
 
     @staticmethod
     def load(map):
-        if utils.htcondor_version_info() < (8, 9, 3):
+        if utils.HTCONDOR_VERSION_INFO < (8, 9, 3):
             raise exceptions.InsufficientHTCondorVersion("Map state can only be saved with HTCondor 8.9.3 or greater")
 
         with (map._map_dir / names.MAP_STATE).open(mode = 'rb') as f:

htmap/state.py

@@ -253,6 +253,4 @@ def parse_version(v: str) -> Tuple[int, int, int, str, int]:
 
 EXTRACT_HTCONDOR_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+)", flags = re.ASCII)
 
-
-def htcondor_version_info() -> Tuple[int, int, int, str, int]:
-    return parse_version(EXTRACT_HTCONDOR_VERSION_RE.search(htcondor.version()).group(0))
+HTCONDOR_VERSION_INFO = parse_version(EXTRACT_HTCONDOR_VERSION_RE.search(htcondor.version()).group(0))

htmap/utils.py

@@ -17,7 +17,7 @@
 
 from . import utils
 
-__version__ = '0.5.0'
+__version__ = '0.5.1'
 
 
 def version() -> str:

htmap/version.py

@@ -3,3 +3,12 @@
 testspaths = tests
 
 console_output_style = count
+
+log_format = %(levelname)-8s  %(asctime)s  %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+
+cache_dir = /tmp/htmap-pytest-cache
+
+markers =
+    issue: marks a test as corresponding to a particular GitHub issue
+

pytest.ini

@@ -3,6 +3,8 @@ pytest >= 4.1.1
 pytest-mock
 pytest-xdist
 pytest-timeout
+pytest-watch
+pytest-profiling
 coverage
 pytest-cov
 codecov

requirements-dev.txt

@@ -1,7 +1,7 @@
 htcondor >= 8.8.0
 cloudpickle ~= 1.2
-toml ~= 0.10.0
+toml ~= 0.10
 tqdm ~= 4.36
 click ~= 7.0
-click-didyoumean == 0.0.3
-halo == 0.0.28
+click-didyoumean ~= 0.0.3
+halo ~= 0.0.28

requirements.txt

@@ -15,22 +15,21 @@
 
 # this dockerfile builds a test environment for HTMap
 
-FROM ubuntu:bionic
+ARG PYTHON_VERSION=3.6
+FROM python:${PYTHON_VERSION}
+
+# build config
+ARG HTCONDOR_VERSION=8.8
 
 # switch to root to do root-level config
 USER root
 
-# build config
-ARG PYTHON_VERSION=3.7
-ARG HTCONDOR_VERSION=8.9
-ARG MINICONDA_VERSION=latest
-
 # environment setup
 ENV DEBIAN_FRONTEND=noninteractive
 
 # install utils and dependencies
 RUN apt-get update \
- && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales \
+ && apt-get -y install --no-install-recommends vim less git gnupg wget ca-certificates locales graphviz \
  && apt-get -y clean \
  && rm -rf /var/lib/apt/lists/* \
  && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
@@ -41,50 +40,38 @@ ENV LC_ALL=en_US.UTF-8 \
     LANGUAGE=en_US.UTF-8
 
 # install HTCondor version specified in config
-RUN wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | apt-key add - \
- && echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/${HTCONDOR_VERSION}/bionic bionic contrib" >> /etc/apt/sources.list \
+RUN wget -qO - https://research.cs.wisc.edu/htcondor/debian/HTCondor-Release.gpg.key | apt-key add - \
+ && echo "deb http://research.cs.wisc.edu/htcondor/debian/${HTCONDOR_VERSION}/buster buster contrib" >> /etc/apt/sources.list.d/htcondor.list \
  && apt-get -y update \
  && apt-get -y install --no-install-recommends htcondor \
  && apt-get -y clean \
  && rm -rf /var/lib/apt/lists/*
 
-# create a user to be our submitter and set conda install location
-ENV SUBMIT_USER=mapper
-ENV CONDA_DIR=/home/${SUBMIT_USER}/conda
-ENV PATH=${CONDA_DIR}/bin:${PATH}
+# copy entrypoint into place and make executable
+COPY tests/_inf/entrypoint.sh /.entrypoint.sh
+RUN chmod +x /.entrypoint.sh
+
+# create a user, set their PATH and PYTHONPATH
+ENV SUBMIT_USER=mapper \
+    PATH="/home/mapper/.local/bin:${PATH}" \
+    PYTHONPATH="/home/mapper/htmap:${PYTHONPATH}"
 RUN groupadd ${SUBMIT_USER} \
  && useradd -m -g ${SUBMIT_USER} ${SUBMIT_USER}
 
-# switch to submit user, don't need root anymore
+# switch to the user, don't need root anymore
 USER ${SUBMIT_USER}
 
-# install miniconda and python version specified in config
-# (and ipython, which is nice for debugging inside the container)
-RUN cd /tmp \
- && wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh \
- && bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR \
- && rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh \
- && conda install python=${PYTHON_VERSION} ipython \
- && conda clean -y --all
-
-# install htmap dependencies early for docker build caching
-COPY requirements* /home/${SUBMIT_USER}/
-RUN pip install --no-cache-dir -r /home/${SUBMIT_USER}/requirements-dev.txt \
- && pip install --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.* \
- && rm /home/${SUBMIT_USER}/requirements*
-
-# set default entrypoint and command
-# the entrypoint is critical: it starts HTCondor in the container
-ENTRYPOINT ["tests/_inf/entrypoint.sh"]
-CMD ["pytest"]
+# install htmap dependencies and debugging tools early for docker build caching
+COPY requirements* /tmp/
+RUN pip install --user --no-cache-dir -r /tmp/requirements-dev.txt \
+ && pip install --user --no-cache-dir ipython \
+ && pip install --user --no-cache-dir --upgrade htcondor==${HTCONDOR_VERSION}.*
 
 # copy HTCondor and HTMap testing configs into place
 COPY tests/_inf/condor_config.local /etc/condor/condor_config.local
-COPY tests/_inf/.htmaprc /home/${SUBMIT_USER}/.htmaprc
 
-# copy htmap package into container and install it
-# this is the only part that can't be cached against editing the package
-COPY --chown=mapper:mapper . /home/${SUBMIT_USER}/htmap
-RUN chmod +x /home/${SUBMIT_USER}/htmap/tests/_inf/entrypoint.sh /home/${SUBMIT_USER}/htmap/tests/_inf/travis.sh \
- && pip install --no-cache-dir --no-deps -e /home/${SUBMIT_USER}/htmap
+# set default entrypoint and command
+# the entrypoint is critical: it starts HTCondor in the container
 WORKDIR /home/${SUBMIT_USER}/htmap
+ENTRYPOINT ["/.entrypoint.sh"]
+CMD ["pytest"]

tests/_inf/.htmaprc

@@ -22,18 +22,21 @@ SHADOW_QUEUE_UPDATE_INTERVAL=10
 UPDATE_INTERVAL=5
 RUNBENCHMARKS=0
 
-# Don't use all the machine resources
-RESERVED_MEMORY = ( $(DETECTED_MEMORY) / 2 )
+# Put all of the machine resources under a single partitionable slot
+NUM_SLOTS = 1
+NUM_SLOTS_TYPE_1 = 1
+SLOT_TYPE_1 = 100%
+SLOT_TYPE_1_PARTITIONABLE = TRUE
+
 JOB_RENICE_INCREMENT=5
 SCHED_UNIV_RENICE_INCREMENT=5
 SHADOW_RENICE_INCREMENT=5
 
-# If the job does not explicitly set an environment, define
-# some default environment variables that put Conda in the path.
-JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SetCondaVars
-JOB_TRANSFORM_SetCondaVars @=end
+# Get the HTMap source into the Python path
+JOB_TRANSFORM_NAMES = $(JOB_TRANSFORM_NAMES) SetPyVars
+JOB_TRANSFORM_SetPyVars @=end
 [
   Requirements = ((Env?:"") == "") && ((Environment?:"") == "");
-  set_Environment = "PATH=/home/mapper/conda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin CONDA_DIR=/home/mapper/conda";
+  set_Environment = "PATH=/home/mapper/.local/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin PYTHONPATH=/home/mapper/htmap";
 ]
 @end

tests/_inf/Dockerfile

@@ -8,13 +8,15 @@ mkdir -p "$_condor_local_dir/lock" "$_condor_local_dir/log" "$_condor_local_dir/
 
 # start condor
 condor_master
-echo "HTCondor is starting..."
+echo "Starting HTCondor..."
 
-# wait until the scheduler is awake
-while [[ ! -f ${_condor_local_dir}/spool/job_queue.log ]]
+# once the shared port daemon wakes up, use condor_who to wait for condor to stand up
+while [[ ! -s "${_condor_local_dir}/log/SharedPortLog" ]]
 do
   sleep .01
 done
+sleep 1  # fudge factor to let shared port *actually* wake up
+condor_who -wait:60 'IsReady && STARTD_State =?= "Ready"' > /dev/null
 
 if [[ -n $@ ]];
 then