Merge pull request #225 from htcondor/v0.6.1

v0.6.1

Author: JoshKarpel

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.2.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -30,3 +30,15 @@ repos:
     rev: v2.0.0
     hooks:
       - id: hadolint
+  - repo: https://github.com/asottile/setup-cfg-fmt
+    rev: v1.11.0
+    hooks:
+      - id: setup-cfg-fmt
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
+    hooks:
+      - id: seed-isort-config
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.3.2
+    hooks:
+      - id: isort

docker/Dockerfile

@@ -67,6 +67,6 @@ ENTRYPOINT ["/home/mapper/htmap/docker/entrypoint.sh"]
 CMD ["bash"]
 
 COPY --chown=mapper:mapper . /home/${USER}/htmap
-RUN python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.*
+RUN python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.*
 
 WORKDIR /home/${USER}/htmap

docs/source/versions/v0_6_1.rst

@@ -0,0 +1,28 @@
+v0.6.1
+======
+
+.. py:currentmodule:: htmap
+
+
+This version is a drop-in replacement for v0.6.0, except that it relaxes the
+version requirements for several dependencies to accommodate upcoming changes
+to the
+`pip dependency resolver <https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html>`_.
+
+
+Known Issues
+------------
+
+* HTMap does not currently allow "directory content transfers", which is an HTCondor
+  feature where naming a directory in ``transfer_input_files`` with a trailing
+  slash transfers the contents of the directory, not the directory itself.
+  (If you try it, the directory itself will be transferred, as if you had not
+  used a trailing slash).
+  Issue: :issue:`215`
+* Execution errors that result in the job being terminated but no output being
+  produced are still not handled entirely gracefully. Right now, the component
+  state will just show as ``ERRORED``, but there won't be an actual error report.
+* Map component state may become corrupted when a map is manually vacated.
+  Force-removal may be needed to clean up maps if HTCondor and HTMap disagree
+  about the state of their components.
+  Issue: :issue:`129`

htmap/__init__.py

@@ -16,47 +16,34 @@
 
 import logging as _logging
 
+from .settings import BASE_SETTINGS, USER_SETTINGS, settings
 from .version import __version__, version, version_info
 
-from .settings import settings, USER_SETTINGS, BASE_SETTINGS
-
 # SET UP NULL LOG HANDLER
 _logger = _logging.getLogger(__name__)
 _logger.setLevel(_logging.DEBUG)
 _logger.addHandler(_logging.NullHandler())
 
-from .mapping import (
-    map,
-    starmap,
-    build_map,
-    MapBuilder,
-)
-from .mapped import mapped, MappedFunction
-from .maps import (
-    Map,
-    MapStdOut,
-    MapStdErr,
-    MapOutputFiles,
-)
-from .holds import ComponentHold
+from . import _startup, exceptions
+from .checkpointing import checkpoint
 from .errors import ComponentError
-from .state import ComponentStatus
-from .options import MapOptions, register_delivery_method
+from .holds import ComponentHold
 from .management import (
-    status,
-    status_json,
-    status_csv,
+    Transplant,
+    clean,
     load,
     load_maps,
     remove,
-    clean,
-    Transplant,
-    transplants,
+    status,
+    status_csv,
+    status_json,
     transplant_info,
+    transplants,
 )
+from .mapped import MappedFunction, mapped
+from .mapping import MapBuilder, build_map, map, starmap
+from .maps import Map, MapOutputFiles, MapStdErr, MapStdOut
+from .options import MapOptions, register_delivery_method
+from .state import ComponentStatus
 from .tags import get_tags
-from .checkpointing import checkpoint
 from .transfer import TransferPath, transfer_output_files
-from . import exceptions
-
-from . import _startup

htmap/__main__.py

@@ -1,5 +1,4 @@
 from .cli import cli
 
-
 if __name__ == "__main__":
     exit(cli.main(prog_name="htmap"))

htmap/_startup.py

@@ -1,9 +1,9 @@
-import os
 import logging
+import os
 from logging import handlers
 from pathlib import Path
 
-from . import settings, names
+from . import names, settings
 
 logger = logging.getLogger("htmap")
 

htmap/checkpointing.py

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 import os
-from pathlib import Path
 import shutil
+from pathlib import Path
 
 from . import names
 

htmap/cli.py

@@ -13,28 +13,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional, List, Collection, Tuple
-
-import logging
-import sys
-import time
 import collections
-import random
 import functools
+import logging
+import random
 import shutil
+import sys
+import time
 from pathlib import Path
-
-import htcondor
-import htmap
-from htmap import names, __version__
-from htmap.management import _status, read_events
+from typing import Collection, List, Optional, Tuple
 
 import click
+import htcondor
 from click_didyoumean import DYMGroup
-
 from halo import Halo
 from spinners import Spinners
 
+import htmap
+from htmap import __version__, names
+from htmap.management import _status, read_events
+
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
 

htmap/htio.py

@@ -13,12 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, List, Iterator, Callable, Iterable
-import logging
-
 import gzip
 import json
+import logging
 from pathlib import Path
+from typing import Any, Callable, Iterable, Iterator, List
 
 import cloudpickle
 import htcondor

htmap/management.py

@@ -13,21 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Tuple, Iterable, Dict, Union, NamedTuple, Callable, List, Optional
-import logging
-
-from pathlib import Path
-import datetime
 import collections
-import json
 import csv
+import datetime
 import io
-import textwrap
+import json
+import logging
 import shutil
+import textwrap
 import uuid
 from concurrent.futures.thread import ThreadPoolExecutor
+from pathlib import Path
+from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union
 
-from . import maps, tags, mapping, utils, state, names, settings, exceptions
+from . import exceptions, mapping, maps, names, settings, state, tags, utils
 
 logger = logging.getLogger(__name__)
 

htmap/mapped.py

@@ -13,10 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Iterable, Dict, Union, Optional, Callable, Any
 import logging
+from typing import Any, Callable, Dict, Iterable, Optional, Union
 
-from . import mapping, options, maps
+from . import mapping, maps, options
 
 logger = logging.getLogger(__name__)
 

htmap/mapping.py

@@ -13,30 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Tuple, Iterable, Dict, Optional, Callable, Iterator, Any, List, Union
+import itertools
 import logging
-
-import uuid
 import shutil
+import uuid
 from pathlib import Path
-import itertools
 from pprint import pformat
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
 
 import htcondor
 
-from . import (
-    htio,
-    tags,
-    exceptions,
-    maps,
-    transfer,
-    options,
-    condor,
-    settings,
-    names,
-    utils,
-)
-from .types import KWARGS, ARGS_OR_KWARGS, ARGS_AND_KWARGS, ARGS
+from . import condor, exceptions, htio, maps, names, options, settings, tags, transfer, utils
+from .types import ARGS, ARGS_AND_KWARGS, ARGS_OR_KWARGS, KWARGS
 
 logger = logging.getLogger(__name__)
 

htmap/maps.py

@@ -13,48 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import (
-    Tuple,
-    List,
-    Iterable,
-    Any,
-    Optional,
-    Iterator,
-    Dict,
-    Mapping,
-    MutableMapping,
-)
-import logging
-
+import collections
+import collections.abc
 import datetime
-import shutil
-import time
 import functools
 import inspect
-import collections
-import collections.abc
+import logging
+import shutil
+import time
 import weakref
 from copy import copy
 from pathlib import Path
+from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple
 
-from tqdm import tqdm
-
-import htcondor
 import classad
+import htcondor
+from tqdm import tqdm
 
-from . import (
-    htio,
-    state,
-    tags,
-    errors,
-    holds,
-    mapping,
-    condor,
-    settings,
-    utils,
-    names,
-    exceptions,
-)
+from . import condor, errors, exceptions, holds, htio, mapping, names, settings, state, tags, utils
 
 logger = logging.getLogger(__name__)
 

htmap/options.py

@@ -13,19 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Union, Iterable, Optional, Callable, Dict, List, Tuple
-import logging
-
-import sys
-import shutil
 import collections
 import hashlib
+import logging
+import shutil
+import sys
 from pathlib import Path
+from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union
 
 import htcondor
 
-from . import utils, transfer, exceptions, names, settings
-from .types import TRANSFER_PATH, REMAPS
+from . import exceptions, names, settings, transfer, utils
+from .types import REMAPS, TRANSFER_PATH
 
 logger = logging.getLogger(__name__)
 

htmap/run/_htmap_run.py

@@ -15,16 +15,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
+import getpass
+import gzip
 import os
 import shutil
-import sys
 import socket
-import datetime
-import gzip
+import subprocess
+import sys
 import textwrap
 import traceback
-import subprocess
-import getpass
 from pathlib import Path
 
 TRANSFER_DIR = "_htmap_transfer"