Wheelable production install

.github/docker/rez-win-py/entrypoint.ps1

@@ -36,7 +36,7 @@ if (-not $?) {exit 1}
 
 # Run Rez Tests
 #
-.\build\Scripts\rez\rez-selftest.exe
+.\build\Scripts\rez\rez-selftest
 
 # Pass on exit code to runner
 exit $LASTEXITCODE

.github/workflows/installation.yaml

@@ -23,7 +23,7 @@ jobs:
           - method: 'python ./install.py'
             exports: 'PATH=${PATH}:/opt/rez/bin/rez'
           - method: 'pip install --target /opt/rez .'
-            exports: 'PATH=${PATH}:/opt/rez/bin PYTHONPATH=${PYTHONPATH}:/opt/rez'
+            exports: 'PATH=${PATH}:/opt/rez/bin/rez REZ_PRODUCTION_PATH=/opt/rez'
 
     steps:
     - uses: actions/checkout@master

install.py

@@ -20,9 +20,7 @@
 # though rez is not yet built.
 #
 from rez.utils._version import _rez_version  # noqa: E402
-from rez.cli._entry_points import get_specifications  # noqa: E402
 from rez.backport.shutilwhich import which  # noqa: E402
-from rez.vendor.distlib.scripts import ScriptMaker  # noqa: E402
 
 # switch to builtin venv in python 3.7+
 #
@@ -74,63 +72,6 @@ def run_command(args, cwd=source_path):
     return subprocess.check_output(args, cwd=source_path)
 
 
-def patch_rez_binaries(dest_dir):
-    virtualenv_bin_path, py_executable = get_virtualenv_py_executable(dest_dir)
-
-    specs = get_specifications()
-
-    # delete rez bin files written into virtualenv
-    for name in specs.keys():
-        filepath = os.path.join(virtualenv_bin_path, name)
-        if os.path.isfile(filepath):
-            os.remove(filepath)
-
-    # write patched bins instead. These go into 'bin/rez' subdirectory, which
-    # gives us a bin dir containing only rez binaries. This is what we want -
-    # we don't want resolved envs accidentally getting the virtualenv's 'python'.
-    dest_bin_path = os.path.join(virtualenv_bin_path, "rez")
-    if os.path.exists(dest_bin_path):
-        shutil.rmtree(dest_bin_path)
-    os.makedirs(dest_bin_path)
-
-    maker = ScriptMaker(
-        # note: no filenames are referenced in any specifications, so
-        # source_dir is unused
-        source_dir=None,
-        target_dir=dest_bin_path
-    )
-
-    maker.executable = py_executable
-
-    maker.make_multiple(
-        specifications=specs.values(),
-        # the -E arg is crucial - it means rez cli tools still work within a
-        # rez-resolved env, even if PYTHONPATH or related env-vars would have
-        # otherwise changed rez's behaviour
-        options=dict(interpreter_args=["-E"])
-    )
-
-
-def copy_completion_scripts(dest_dir):
-    # find completion dir in rez package
-    path = os.path.join(dest_dir, "lib")
-    completion_path = None
-    for root, _, _ in os.walk(path):
-        if root.endswith(os.path.sep + "rez" + os.path.sep + "completion"):
-            completion_path = root
-            break
-
-    # copy completion scripts into root of virtualenv for ease of use
-    if completion_path:
-        dest_path = os.path.join(dest_dir, "completion")
-        if os.path.exists(dest_path):
-            shutil.rmtree(dest_path)
-        shutil.copytree(completion_path, dest_path)
-        return dest_path
-
-    return None
-
-
 def install(dest_dir, print_welcome=False):
     """Install rez into the given directory.
 
@@ -145,23 +86,12 @@ def install(dest_dir, print_welcome=False):
     # install rez from source
     install_rez_from_source(dest_dir)
 
-    # patch the rez binaries
-    patch_rez_binaries(dest_dir)
-
-    # copy completion scripts into virtualenv
-    completion_path = copy_completion_scripts(dest_dir)
-
-    # mark virtualenv as production rez install. Do not remove - rez uses this!
-    virtualenv_bin_dir = get_virtualenv_bin_dir(dest_dir)
-    dest_bin_dir = os.path.join(virtualenv_bin_dir, "rez")
-    validation_file = os.path.join(dest_bin_dir, ".rez_production_install")
-    with open(validation_file, 'w') as f:
-        f.write(_rez_version)
-
     # done
     if print_welcome:
         print()
         print("SUCCESS!")
+        virtualenv_bin_dir = get_virtualenv_bin_dir(dest_dir)
+        dest_bin_dir = os.path.join(virtualenv_bin_dir, "rez")
         rez_exe = os.path.realpath(os.path.join(dest_bin_dir, "rez"))
         print("Rez executable installed to: %s" % rez_exe)
 
@@ -181,7 +111,8 @@ def install(dest_dir, print_welcome=False):
         print("To activate Rez, add the following path to $PATH:")
         print(dest_bin_dir)
 
-        if completion_path:
+        completion_path = os.path.join(dest_dir, "completion")
+        if os.path.isdir(completion_path):
             print('')
             shell = os.getenv('SHELL')
 

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+# Minimum requirements for the build system to execute.
+requires = ["setuptools", "wheel"]  # PEP 508 specifications.

setup.py

@@ -8,6 +8,7 @@
 
 try:
     from setuptools import setup, find_packages
+    from setuptools.command import install_scripts
 except ImportError:
     print("install failed - requires setuptools", file=sys.stderr)
     sys.exit(1)
@@ -27,7 +28,32 @@
 from rez.cli._entry_points import get_specifications
 
 
-def find_files(pattern, path=None, root="rez"):
+class InstallRezScripts(install_scripts.install_scripts):
+
+    def run(self):
+        install_scripts.install_scripts.run(self)
+        self.patch_rez_binaries()
+
+    def patch_rez_binaries(self):
+        from rez.utils.installer import create_rez_production_scripts
+
+        build_path = os.path.join(self.build_dir, "rez")
+        install_path = os.path.join(self.install_dir, "rez")
+
+        specifications = get_specifications().values()
+        create_rez_production_scripts(build_path, specifications)
+
+        validation_file = os.path.join(build_path, ".rez_production_install")
+        with open(validation_file, "w") as vfn:
+            # PEP-427, wheel will rewrite this *shebang* to the python that
+            # used to install rez. And we'll use this to run rez cli tools.
+            vfn.write("#!python\n")
+            vfn.write(_rez_version)
+
+        self.outfiles += self.copy_tree(build_path, install_path)
+
+
+def find_files(pattern, path=None, root="rez", prefix=""):
     paths = []
     basepath = os.path.realpath(os.path.join("src", root))
     path_ = basepath
@@ -39,7 +65,7 @@ def find_files(pattern, path=None, root="rez"):
         files = [os.path.join(root, x) for x in files]
         paths += [x[len(basepath):].lstrip(os.path.sep) for x in files]
 
-    return paths
+    return [prefix + p for p in paths]
 
 
 this_directory = os.path.abspath(os.path.dirname(__file__))
@@ -61,7 +87,7 @@ def find_files(pattern, path=None, root="rez"):
     author_email="[email protected]",
     license="LGPL",
     entry_points={
-        "console_scripts": get_specifications().values()
+        "console_scripts": []
     },
     include_package_data=True,
     zip_safe=False,
@@ -84,6 +110,12 @@ def find_files(pattern, path=None, root="rez"):
             find_files('rezguiconfig', root='rezgui') +
             find_files('*', 'icons', root='rezgui')
     },
+    data_files=[
+        ("completion", find_files('*', 'completion', prefix='src/rez/'))
+    ],
+    cmdclass={
+        "install_scripts": InstallRezScripts,
+    },
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",

src/rez/__init__.py

@@ -12,6 +12,7 @@
 
 
 module_root_path = __path__[0]
+production_bin_path = None
 
 
 # TODO: Revamp logging. For now, this is here for backwards compatibility

src/rez/cli/_entry_points.py

@@ -1,7 +1,7 @@
 """
 Entry points.
 """
-import os.path
+import os
 import sys
 
 
@@ -46,7 +46,17 @@ def check_production_install():
     path = os.path.dirname(sys.argv[0])
     filepath = os.path.join(path, ".rez_production_install")
 
-    if not os.path.exists(filepath):
+    if os.path.exists(filepath):
+        try:
+            # For case like `pip install rez --target <dst>`, which rez tools
+            # may not have the same directory hierarchy as normal install and
+            # makes `rez.system.rez_bin_path` not able to find the validation
+            # file from module path.
+            import rez
+            rez.production_bin_path = path
+        except ImportError:
+            pass
+    else:
         sys.stderr.write(
             "Pip-based rez installation detected. Please be aware that rez command "
             "line tools are not guaranteed to function correctly in this case. See "

src/rez/package_cache.py

@@ -18,6 +18,7 @@
 from rez.exceptions import PackageCacheError
 from rez.vendor.lockfile import LockFile, NotLocked
 from rez.utils import json
+from rez.utils.execution import Popen
 from rez.utils.filesystem import safe_listdir, safe_makedirs, safe_remove, \
     forceful_rmtree
 from rez.utils.colorize import ColorizedStreamHandler
@@ -460,7 +461,7 @@ def add_variants_async(self, variants):
                 else:
                     out_target = devnull
 
-                subprocess.Popen(
+                Popen(
                     [exe, "--daemon", self.path],
                     stdout=out_target,
                     stderr=out_target,

src/rez/rezconfig.py

@@ -912,6 +912,14 @@
         "pip_install": r"\1",
         "rez_install": r"python{s}\1",
     },
+    # Path in record          | pip installed to    | copy to rez destination
+    # ------------------------|---------------------|--------------------------
+    # ../../*                 | *                   | *
+    {
+        "record_path": r"^{p}{s}{p}{s}(.*)",
+        "pip_install": r"\1",
+        "rez_install": r"\1",
+    },
 ]
 
 ###############################################################################

src/rez/system.py

@@ -198,14 +198,17 @@ def rez_bin_path(self):
         """Get path containing rez binaries, or None if no binaries are
         available, or Rez is not a production install.
         """
+        import rez
+
+        if rez.production_bin_path:
+            return rez.production_bin_path
 
         # Rez install layout will be like:
         #
         # /<install>/lib/python2.7/site-packages/rez  <- module path
         # /<install>/(bin or Scripts)/rez/rez  <- rez executable
         #
-        import rez
-        module_path = rez.__path__[0]
+        module_path = rez.module_root_path
 
         parts = module_path.split(os.path.sep)
         parts_lower = module_path.lower().split(os.path.sep)

src/rez/tests/test_shells.py

@@ -8,7 +8,7 @@
 from rez.resolved_context import ResolvedContext
 from rez.rex import literal, expandable
 from rez.utils.execution import create_executable_script, ExecutableScriptMode, \
-    _get_python_script_files
+    _get_python_script_files, Popen
 from rez.tests.util import TestBase, TempdirMixin, per_available_shell, \
     install_dependent
 from rez.util import which
@@ -191,7 +191,7 @@ def test_rez_env_output(self):
 
         # Assumes that the shell has an echo command, build-in or alias
         cmd = [os.path.join(system.rez_bin_path, "rez-env"), "--", "echo", "hey"]
-        process = subprocess.Popen(
+        process = Popen(
             cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE, universal_newlines=True
         )

src/rez/utils/execution.py

@@ -81,6 +81,10 @@ def __init__(self, args, **kwargs):
             if sys.version_info[:2] >= (3, 6) and "encoding" not in kwargs:
                 kwargs["encoding"] = "utf-8"
 
+        # Patch batch script extension for Windows
+        if sys.platform == "win32" and os.path.isfile(args[0] + ".cmd"):
+            args[0] += ".cmd"
+
         super(Popen, self).__init__(args, **kwargs)
 
 
@@ -167,7 +171,7 @@ def create_executable_script(filepath, body, program=None, py_script_mode=None):
                 # following lines of batch script will be stripped
                 # before yaml.load
                 f.write("@echo off\n")
-                f.write("%s.exe %%~dpnx0 %%*\n" % program)
+                f.write("%s %%~dpnx0 %%*\n" % program)
                 f.write("goto :eof\n")  # skip YAML body
                 f.write(":: YAML\n")    # comment for human
             else: