Merge branch 'master' into doc-usecases

Author: chaselgrove

.travis.yml

@@ -17,6 +17,7 @@ matrix:
     - REPROMAN_TESTS_SSH=1
     - INSTALL_DATALAD=1
     - INSTALL_CONDOR=1
+    - SETUP_SLURM=1
   - python: 3.5
     env:
     - REPROMAN_TESTS_SSH=1
@@ -38,7 +39,7 @@ matrix:
     env:
     - REPROMAN_LOGLEVEL=2
     - REPROMAN_LOGTARGET=/dev/null
-  - python: 3.4
+  - python: 3.6
     # Note: This this no network run appears to hang or otherwise
     # behave oddly on 3.5.
     env:
@@ -66,6 +67,9 @@ before_install:
       sudo eatmydata tools/ci/prep-travis-forssh-sudo.sh;
       tools/ci/prep-travis-forssh.sh;
     fi
+  - if [ ! -z "${SETUP_SLURM:-}" ]; then
+      tools/ci/setup-slurm-container.sh;
+    fi
   - git config --global user.email "[email protected]"
   - git config --global user.name "Travis Almighty"
 

CONTRIBUTING.md

@@ -113,7 +113,7 @@ we outline the workflow used by the developers:
 Development environment
 -----------------------
 
-We support Python 3 (>= 3.4).
+We support Python 3 (>= 3.5).
 
 See [README.md:Dependencies](README.md#Dependencies) for basic information
 about installation of reproman itself.

README.md

@@ -27,7 +27,7 @@ internals and/or contributing to the project.
 
 # Installation
 
-ReproMan requires Python 3 (>= 3.4).
+ReproMan requires Python 3 (>= 3.5).
 
 ## Debian-based systems
 

reproman/cmd.py

@@ -249,8 +249,12 @@ def run(self, cmd, log_stdout=True, log_stderr=True, log_online=False,
 
             except Exception as e:
                 prot_exc = e
-                lgr.error("Failed to start %r%r: %s" %
-                          (cmd, " under %r" % cwd if cwd else '', exc_str(e)))
+                if isinstance(e, FileNotFoundError) and expect_fail:
+                    logfn = lgr.debug
+                else:
+                    logfn = lgr.error
+                logfn("Failed to start %r%r: %s" %
+                      (cmd, " under %r" % cwd if cwd else '', exc_str(e)))
                 raise
 
             finally:

reproman/distributions/conda.py

@@ -85,9 +85,7 @@ def get_miniconda_url(conda_platform, python_version):
         raise ValueError("Unsupported platform %s for conda installation" %
                          conda_platform)
     platform += "-x86_64" if ("64" in conda_platform) else "-x86"
-    # FIXME: We need to update this our conda tracer to work with conda's newer
-    # than 4.6.14. See gh-443.
-    return "https://repo.continuum.io/miniconda/Miniconda%s-4.6.14-%s.sh" \
+    return "https://repo.anaconda.com/miniconda/Miniconda%s-latest-%s.sh" \
                     % (python_version[0], platform)
 
 
@@ -184,22 +182,22 @@ def install_packages(self, session=None):
                 # TODO: Determine if we can detect miniconda vs anaconad
                 miniconda_url = get_miniconda_url(self.platform,
                                                   self.python_version)
-                session.execute_command("curl %s -o %s/miniconda.sh" %
-                                        (miniconda_url, tmp_dir))
+                session.execute_command(
+                    "curl --fail --silent --show-error --location "
+                    "--output {}/miniconda.sh {}"
+                    .format(tmp_dir, miniconda_url))
                 # NOTE: miniconda.sh makes parent directories automatically
                 session.execute_command("bash -b %s/miniconda.sh -b -p %s" %
                                         (tmp_dir, self.path))
-            ## Update root version of conda
-            session.execute_command(
-               "%s/bin/conda install -y conda=%s python=%s" %
-               (self.path, self.conda_version,
-                self.get_simple_python_version(self.python_version)))
-
-            # Loop through non-root packages, creating the conda-env config
-            for env in self.environments:
+            envs = sorted(
+                self.environments,
+                # Create/update the root environment before handling anything
+                # else.
+                key=lambda x: "_" if x.name == "root" else "_" + x.name)
+            for env in envs:
                 export_contents = self.create_conda_export(env)
                 with make_tempfile(export_contents) as local_config:
-                    remote_config = os.path.join(tmp_dir, env.name)
+                    remote_config = os.path.join(tmp_dir, env.name + ".yaml")
                     session.put(local_config, remote_config)
                     if not session.isdir(env.path):
                         try:
@@ -280,6 +278,7 @@ class CondaTracer(DistributionTracer):
 
     def _init(self):
         self._get_conda_env_path = PathRoot(self._is_conda_env_path)
+        self._get_conda_dist_path = PathRoot(self._is_conda_dist_path)
 
     def _get_packagefields_for_files(self, files):
         raise NotImplementedError("TODO")
@@ -335,20 +334,27 @@ def _get_conda_pip_package_details(self, env_export, conda_path):
 
         # If there are pip dependencies, they'll be listed under a
         # {"pip": [...]} entry.
-        pip_pkgs = []
+        pip_pkgs = set()
         for dep in dependencies:
             if isinstance(dep, dict) and "pip" in dep:
                 # Pip packages are recorded in conda exports as "name (loc)",
                 # "name==version" or "name (loc)==version".
-                pip_pkgs = [p.split("=")[0].split(" ")[0] for p in dep["pip"]]
+                pip_pkgs = {p.split("=")[0].split(" ")[0] for p in dep["pip"]}
                 break
 
+        pip = conda_path + "/bin/pip"
+        if not self._session.exists(pip):
+            return {}, {}
+
+        pkgs_editable = set(piputils.get_pip_packages(
+            self._session, pip, restriction="editable"))
+        pip_pkgs.update(pkgs_editable)
+
         if not pip_pkgs:
             return {}, {}
 
-        pip = conda_path + "/bin/pip"
         packages, file_to_package_map = piputils.get_package_details(
-            self._session, pip, pip_pkgs)
+            self._session, pip, pip_pkgs, editable_packages=pkgs_editable)
         for entry in packages.values():
             entry["installer"] = "pip"
         return packages, file_to_package_map
@@ -391,6 +397,10 @@ def _get_conda_info(self, conda_path):
     def _is_conda_env_path(self, path):
         return self._session.exists('%s/conda-meta' % path)
 
+    def _is_conda_dist_path(self, path):
+        return (self._session.exists(path + "/envs")
+                and self._is_conda_env_path(path))
+
     def identify_distributions(self, paths):
         conda_paths = set()
         root_to_envs = defaultdict(list)
@@ -417,8 +427,7 @@ def identify_distributions(self, paths):
             # Find the root path for the environment
             # TODO: cache/memoize for those paths which have been considered
             # since will be asked again below
-            conda_info = self._get_conda_info(conda_path)
-            root_path = conda_info.get('root_prefix')
+            root_path = self._get_conda_dist_path(conda_path)
             if not root_path:
                 lgr.warning("Could not find root path for conda environment %s"
                             % conda_path)

reproman/distributions/piputils.py

@@ -117,7 +117,8 @@ def get_pip_packages(session, which_pip, restriction=None):
     return (p["name"] for p in json.loads(out))
 
 
-def get_package_details(session, which_pip, packages=None):
+def get_package_details(session, which_pip, packages=None,
+                        editable_packages=None):
     """Get package details from `pip show` and `pip list`.
 
     This is similar to `pip_show`, but it uses `pip list` to get information
@@ -132,6 +133,9 @@ def get_package_details(session, which_pip, packages=None):
     packages : list of str, optional
         Package names.  If not given, all packages returned by `pip list` are
         used.
+    editable_packages : collection of str
+        If a package name is in this collection, mark it as editable. Passing
+        this saves a call to `which_pip`.
 
     Returns
     -------
@@ -140,8 +144,9 @@ def get_package_details(session, which_pip, packages=None):
     """
     if packages is None:
         packages = list(get_pip_packages(session, which_pip))
-    editable_packages = set(
-        get_pip_packages(session, which_pip, restriction="editable"))
+    if editable_packages is None:
+        editable_packages = set(
+            get_pip_packages(session, which_pip, restriction="editable"))
     details, file_to_pkg = pip_show(session, which_pip, packages)
 
     for pkg in details:

reproman/distributions/tests/test_conda.py

@@ -33,11 +33,11 @@ def test_get_conda_platform_from_python():
 
 def test_get_miniconda_url():
     assert get_miniconda_url("linux-64", "2.7") == \
-           "https://repo.continuum.io/miniconda/Miniconda2-4.6.14-Linux-x86_64.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda2-latest-Linux-x86_64.sh"
     assert get_miniconda_url("linux-32", "3.4") == \
-           "https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86.sh"
     assert get_miniconda_url("osx-64", "3.5.1") == \
-           "https://repo.continuum.io/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh"
+           "https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
 
 
 def test_get_simple_python_version():
@@ -83,7 +83,7 @@ def test_create_conda_export():
                 md5=None,
                 size=None,
                 url=None,
-                files=["lib/python2.7/site-packages/rpaths.py"])],
+                files=["lib/python3.7/site-packages/rpaths.py"])],
         channels=[
             CondaChannel(
                 name="conda-forge",
@@ -107,8 +107,8 @@ def test_conda_init_install_and_detect(tmpdir):
     dist = CondaDistribution(
         name="conda",
         path=test_dir,
-        conda_version="4.3.31",
-        python_version="2.7.14.final.0",
+        conda_version="4.8.2",
+        python_version="3.8.2",
         platform=get_conda_platform_from_python(sys.platform) + "-64",
         environments=[
             CondaEnvironment(
@@ -118,7 +118,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="conda",
                         installer=None,
-                        version="4.3.31",
+                        version="4.8.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -128,7 +128,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="9.0.1",
+                        version="20.0.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -159,7 +159,7 @@ def test_conda_init_install_and_detect(tmpdir):
                     CondaPackage(
                         name="pip",
                         installer=None,
-                        version="9.0.1",
+                        version="20.0.2",
                         build=None,
                         channel_name=None,
                         md5=None,
@@ -185,7 +185,7 @@ def test_conda_init_install_and_detect(tmpdir):
                         md5=None,
                         size=None,
                         url=None,
-                        files=["lib/python2.7/site-packages/rpaths.py"])],
+                        files=["lib/python3.8/site-packages/rpaths.py"])],
                 channels=[
                     CondaChannel(
                         name="conda-forge",

reproman/resource/session.py

@@ -17,6 +17,8 @@
 import os.path as op
 import re
 from shlex import quote as shlex_quote
+import subprocess
+from ..support.exceptions import CommandError
 
 from reproman.support.exceptions import SessionRuntimeError
 from reproman.cmd import Runner
@@ -492,10 +494,20 @@ class POSIXSession(Session):
     # -0 is not provided by busybox's env command.  So if we decide to make it
     # even more portable - something to be done
     _GET_ENVIRON_CMD = ['env', '-0']
+    _ALT_GET_ENVIRON_CMD = ['perl', '-e', r'foreach (keys %ENV) {print "$_=$ENV{$_}\0";}']
 
     @borrowdoc(Session)
     def query_envvars(self):
-        out, err = self.execute_command(self._GET_ENVIRON_CMD)
+        try:
+            out, err = self.execute_command(self._GET_ENVIRON_CMD)
+        except CommandError:
+            # if this fails, we might need the altenative command...
+            if self._GET_ENVIRON_CMD == self.__class__._ALT_GET_ENVIRON_CMD:
+                # ...if it's already installed, we fail...
+                raise
+            # ...otherwise we install it and try again
+            self._GET_ENVIRON_CMD = self._ALT_GET_ENVIRON_CMD
+            return self.query_envvars()
         env = self._parse_envvars_output(out)
         # TODO:  should we update with our .env or .env_permament????
         return env
@@ -577,7 +589,7 @@ def source_script(self, command, permanent=False, diff=True, shell=None):
         # just run it and be done
         marker = "== =ReproMan == ="  # unique marker to be able to split away
         # possible output from the sourced script
-        get_env_command = " ".join('"%s"' % s for s in self._GET_ENVIRON_CMD)
+        get_env_command = " ".join("'%s'" % s for s in self._GET_ENVIRON_CMD)
         shell = shell or self.query_envvars().get('SHELL', None)
         if not isinstance(command, list):
             command = [command]

reproman/support/external_versions.py

@@ -44,27 +44,32 @@ def __cmp__(self, other):
 _runner = Runner()
 
 
+def _try_run(cmd):
+    return _runner.run(cmd, expect_fail=True, expect_stderr=True)
+
+
 def _get_annex_version():
     """Return version of available git-annex"""
-    return _runner.run('git annex version --raw'.split())[0]
+    return _try_run(['git', 'annex', 'version', '--raw'])[0]
 
 
 def _get_git_version():
     """Return version of available git"""
-    return _runner.run('git version'.split())[0].split()[-1]
+    out = _try_run(['git', 'version'])[0]
+    return out.split()[-1]
 
 
 def _get_apt_cache_version():
     """Return version of available apt-cache."""
-    return _runner.run('apt-cache -v'.split())[0].split()[1]
+    out = _try_run(['apt-cache', '-v'])[0]
+    return out.split()[1]
 
 
 def _get_system_ssh_version():
     """Return version of ssh available system-wide
     """
     try:
-        out, err = _runner.run('ssh -V'.split(),
-                               expect_fail=True, expect_stderr=True)
+        out, err = _try_run(['ssh', '-V'])
         # apparently spits out to err but I wouldn't trust it blindly
         if err.startswith('OpenSSH'):
             out = err
@@ -80,7 +85,7 @@ def _get_singularity_version():
     # example output:
     #  "singularity version 3.0.3+ds"
     #  "2.6.1-dist"
-    out = _runner.run(["singularity", "--version"])[0]
+    out = _try_run(["singularity", "--version"])[0]
     return out.split(' ')[-1].split("-")[0].split("+")[0]
 
 
@@ -90,15 +95,17 @@ def _get_svn_version():
     #
     # svn, version 1.9.5 (r1770682)
     # [...]
-    return _runner.run(["svn", "--version"])[0].split()[2]
+    out = _try_run(["svn", "--version"])[0]
+    return out.split()[2]
 
 
 def _get_condor_version():
     """Return version of available condor"""
     # Example output:
     #
     # $CondorVersion: 8.6.8 Nov 30 2017 BuildID: [...]
-    return _runner.run(['condor_version'])[0].split()[1]
+    out = _try_run(['condor_version'])[0]
+    return out.split()[1]
 
 
 class ExternalVersions(object):