diff --git a/.github/docker/rez-win-py/entrypoint.ps1 b/.github/docker/rez-win-py/entrypoint.ps1 index 0d2cb100e..3f6146aa6 100644 --- a/.github/docker/rez-win-py/entrypoint.ps1 +++ b/.github/docker/rez-win-py/entrypoint.ps1 @@ -36,7 +36,7 @@ if (-not $?) {exit 1} # Run Rez Tests # -.\build\Scripts\rez\rez-selftest.exe +.\build\Scripts\rez\rez-selftest # Pass on exit code to runner exit $LASTEXITCODE diff --git a/.github/workflows/installation.yaml b/.github/workflows/installation.yaml index 54f99a795..13c025b9d 100644 --- a/.github/workflows/installation.yaml +++ b/.github/workflows/installation.yaml @@ -23,7 +23,7 @@ jobs: - method: 'python ./install.py' exports: 'PATH=${PATH}:/opt/rez/bin/rez' - method: 'pip install --target /opt/rez .' - exports: 'PATH=${PATH}:/opt/rez/bin PYTHONPATH=${PYTHONPATH}:/opt/rez' + exports: 'PATH=${PATH}:/opt/rez/bin/rez REZ_PRODUCTION_PATH=/opt/rez' steps: - uses: actions/checkout@master diff --git a/install.py b/install.py index 5a3a63dc5..201fbb7ef 100644 --- a/install.py +++ b/install.py @@ -20,9 +20,7 @@ # though rez is not yet built. # from rez.utils._version import _rez_version # noqa: E402 -from rez.cli._entry_points import get_specifications # noqa: E402 from rez.backport.shutilwhich import which # noqa: E402 -from rez.vendor.distlib.scripts import ScriptMaker # noqa: E402 # switch to builtin venv in python 3.7+ # @@ -74,63 +72,6 @@ def run_command(args, cwd=source_path): return subprocess.check_output(args, cwd=source_path) -def patch_rez_binaries(dest_dir): - virtualenv_bin_path, py_executable = get_virtualenv_py_executable(dest_dir) - - specs = get_specifications() - - # delete rez bin files written into virtualenv - for name in specs.keys(): - filepath = os.path.join(virtualenv_bin_path, name) - if os.path.isfile(filepath): - os.remove(filepath) - - # write patched bins instead. These go into 'bin/rez' subdirectory, which - # gives us a bin dir containing only rez binaries. This is what we want - - # we don't want resolved envs accidentally getting the virtualenv's 'python'. - dest_bin_path = os.path.join(virtualenv_bin_path, "rez") - if os.path.exists(dest_bin_path): - shutil.rmtree(dest_bin_path) - os.makedirs(dest_bin_path) - - maker = ScriptMaker( - # note: no filenames are referenced in any specifications, so - # source_dir is unused - source_dir=None, - target_dir=dest_bin_path - ) - - maker.executable = py_executable - - maker.make_multiple( - specifications=specs.values(), - # the -E arg is crucial - it means rez cli tools still work within a - # rez-resolved env, even if PYTHONPATH or related env-vars would have - # otherwise changed rez's behaviour - options=dict(interpreter_args=["-E"]) - ) - - -def copy_completion_scripts(dest_dir): - # find completion dir in rez package - path = os.path.join(dest_dir, "lib") - completion_path = None - for root, _, _ in os.walk(path): - if root.endswith(os.path.sep + "rez" + os.path.sep + "completion"): - completion_path = root - break - - # copy completion scripts into root of virtualenv for ease of use - if completion_path: - dest_path = os.path.join(dest_dir, "completion") - if os.path.exists(dest_path): - shutil.rmtree(dest_path) - shutil.copytree(completion_path, dest_path) - return dest_path - - return None - - def install(dest_dir, print_welcome=False): """Install rez into the given directory. @@ -145,23 +86,12 @@ def install(dest_dir, print_welcome=False): # install rez from source install_rez_from_source(dest_dir) - # patch the rez binaries - patch_rez_binaries(dest_dir) - - # copy completion scripts into virtualenv - completion_path = copy_completion_scripts(dest_dir) - - # mark virtualenv as production rez install. Do not remove - rez uses this! - virtualenv_bin_dir = get_virtualenv_bin_dir(dest_dir) - dest_bin_dir = os.path.join(virtualenv_bin_dir, "rez") - validation_file = os.path.join(dest_bin_dir, ".rez_production_install") - with open(validation_file, 'w') as f: - f.write(_rez_version) - # done if print_welcome: print() print("SUCCESS!") + virtualenv_bin_dir = get_virtualenv_bin_dir(dest_dir) + dest_bin_dir = os.path.join(virtualenv_bin_dir, "rez") rez_exe = os.path.realpath(os.path.join(dest_bin_dir, "rez")) print("Rez executable installed to: %s" % rez_exe) @@ -181,7 +111,8 @@ def install(dest_dir, print_welcome=False): print("To activate Rez, add the following path to $PATH:") print(dest_bin_dir) - if completion_path: + completion_path = os.path.join(dest_dir, "completion") + if os.path.isdir(completion_path): print('') shell = os.getenv('SHELL') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..019b0d848 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +# Minimum requirements for the build system to execute. +requires = ["setuptools", "wheel"] # PEP 508 specifications. diff --git a/setup.py b/setup.py index 0522bd4ee..5838aaaf5 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ try: from setuptools import setup, find_packages + from setuptools.command import install_scripts except ImportError: print("install failed - requires setuptools", file=sys.stderr) sys.exit(1) @@ -27,7 +28,32 @@ from rez.cli._entry_points import get_specifications -def find_files(pattern, path=None, root="rez"): +class InstallRezScripts(install_scripts.install_scripts): + + def run(self): + install_scripts.install_scripts.run(self) + self.patch_rez_binaries() + + def patch_rez_binaries(self): + from rez.utils.installer import create_rez_production_scripts + + build_path = os.path.join(self.build_dir, "rez") + install_path = os.path.join(self.install_dir, "rez") + + specifications = get_specifications().values() + create_rez_production_scripts(build_path, specifications) + + validation_file = os.path.join(build_path, ".rez_production_install") + with open(validation_file, "w") as vfn: + # PEP-427, wheel will rewrite this *shebang* to the python that + # used to install rez. And we'll use this to run rez cli tools. + vfn.write("#!python\n") + vfn.write(_rez_version) + + self.outfiles += self.copy_tree(build_path, install_path) + + +def find_files(pattern, path=None, root="rez", prefix=""): paths = [] basepath = os.path.realpath(os.path.join("src", root)) path_ = basepath @@ -39,7 +65,7 @@ def find_files(pattern, path=None, root="rez"): files = [os.path.join(root, x) for x in files] paths += [x[len(basepath):].lstrip(os.path.sep) for x in files] - return paths + return [prefix + p for p in paths] this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -61,7 +87,7 @@ def find_files(pattern, path=None, root="rez"): author_email="nerdvegas@gmail.com", license="LGPL", entry_points={ - "console_scripts": get_specifications().values() + "console_scripts": [] }, include_package_data=True, zip_safe=False, @@ -84,6 +110,12 @@ def find_files(pattern, path=None, root="rez"): find_files('rezguiconfig', root='rezgui') + find_files('*', 'icons', root='rezgui') }, + data_files=[ + ("completion", find_files('*', 'completion', prefix='src/rez/')) + ], + cmdclass={ + "install_scripts": InstallRezScripts, + }, classifiers=[ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", diff --git a/src/rez/__init__.py b/src/rez/__init__.py index 5c24c22d9..f798fe8b6 100644 --- a/src/rez/__init__.py +++ b/src/rez/__init__.py @@ -12,6 +12,7 @@ module_root_path = __path__[0] +production_bin_path = None # TODO: Revamp logging. For now, this is here for backwards compatibility diff --git a/src/rez/cli/_entry_points.py b/src/rez/cli/_entry_points.py index 7a94f78ee..d943c6639 100644 --- a/src/rez/cli/_entry_points.py +++ b/src/rez/cli/_entry_points.py @@ -1,7 +1,7 @@ """ Entry points. """ -import os.path +import os import sys @@ -46,7 +46,17 @@ def check_production_install(): path = os.path.dirname(sys.argv[0]) filepath = os.path.join(path, ".rez_production_install") - if not os.path.exists(filepath): + if os.path.exists(filepath): + try: + # For case like `pip install rez --target `, which rez tools + # may not have the same directory hierarchy as normal install and + # makes `rez.system.rez_bin_path` not able to find the validation + # file from module path. + import rez + rez.production_bin_path = path + except ImportError: + pass + else: sys.stderr.write( "Pip-based rez installation detected. Please be aware that rez command " "line tools are not guaranteed to function correctly in this case. See " diff --git a/src/rez/package_cache.py b/src/rez/package_cache.py index 0e750a5cb..dbcb3a0b4 100644 --- a/src/rez/package_cache.py +++ b/src/rez/package_cache.py @@ -18,6 +18,7 @@ from rez.exceptions import PackageCacheError from rez.vendor.lockfile import LockFile, NotLocked from rez.utils import json +from rez.utils.execution import Popen from rez.utils.filesystem import safe_listdir, safe_makedirs, safe_remove, \ forceful_rmtree from rez.utils.colorize import ColorizedStreamHandler @@ -460,7 +461,7 @@ def add_variants_async(self, variants): else: out_target = devnull - subprocess.Popen( + Popen( [exe, "--daemon", self.path], stdout=out_target, stderr=out_target, diff --git a/src/rez/rezconfig.py b/src/rez/rezconfig.py index ea6e64526..863634025 100644 --- a/src/rez/rezconfig.py +++ b/src/rez/rezconfig.py @@ -912,6 +912,14 @@ "pip_install": r"\1", "rez_install": r"python{s}\1", }, + # Path in record | pip installed to | copy to rez destination + # ------------------------|---------------------|-------------------------- + # ../../* | * | * + { + "record_path": r"^{p}{s}{p}{s}(.*)", + "pip_install": r"\1", + "rez_install": r"\1", + }, ] ############################################################################### diff --git a/src/rez/system.py b/src/rez/system.py index 95d78e34b..88895064f 100644 --- a/src/rez/system.py +++ b/src/rez/system.py @@ -198,14 +198,17 @@ def rez_bin_path(self): """Get path containing rez binaries, or None if no binaries are available, or Rez is not a production install. """ + import rez + + if rez.production_bin_path: + return rez.production_bin_path # Rez install layout will be like: # # //lib/python2.7/site-packages/rez <- module path # //(bin or Scripts)/rez/rez <- rez executable # - import rez - module_path = rez.__path__[0] + module_path = rez.module_root_path parts = module_path.split(os.path.sep) parts_lower = module_path.lower().split(os.path.sep) diff --git a/src/rez/tests/test_shells.py b/src/rez/tests/test_shells.py index 9773c4e5b..f876aa6c5 100644 --- a/src/rez/tests/test_shells.py +++ b/src/rez/tests/test_shells.py @@ -8,7 +8,7 @@ from rez.resolved_context import ResolvedContext from rez.rex import literal, expandable from rez.utils.execution import create_executable_script, ExecutableScriptMode, \ - _get_python_script_files + _get_python_script_files, Popen from rez.tests.util import TestBase, TempdirMixin, per_available_shell, \ install_dependent from rez.util import which @@ -191,7 +191,7 @@ def test_rez_env_output(self): # Assumes that the shell has an echo command, build-in or alias cmd = [os.path.join(system.rez_bin_path, "rez-env"), "--", "echo", "hey"] - process = subprocess.Popen( + process = Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True ) diff --git a/src/rez/utils/execution.py b/src/rez/utils/execution.py index 93279377d..cdc88bb17 100644 --- a/src/rez/utils/execution.py +++ b/src/rez/utils/execution.py @@ -81,6 +81,10 @@ def __init__(self, args, **kwargs): if sys.version_info[:2] >= (3, 6) and "encoding" not in kwargs: kwargs["encoding"] = "utf-8" + # Patch batch script extension for Windows + if sys.platform == "win32" and os.path.isfile(args[0] + ".cmd"): + args[0] += ".cmd" + super(Popen, self).__init__(args, **kwargs) @@ -167,7 +171,7 @@ def create_executable_script(filepath, body, program=None, py_script_mode=None): # following lines of batch script will be stripped # before yaml.load f.write("@echo off\n") - f.write("%s.exe %%~dpnx0 %%*\n" % program) + f.write("%s %%~dpnx0 %%*\n" % program) f.write("goto :eof\n") # skip YAML body f.write(":: YAML\n") # comment for human else: diff --git a/src/rez/utils/installer.py b/src/rez/utils/installer.py index a4837b24f..4308fc2f1 100644 --- a/src/rez/utils/installer.py +++ b/src/rez/utils/installer.py @@ -4,6 +4,7 @@ from rez.package_maker import make_package from rez.system import system import os.path +import re import sys import shutil @@ -39,3 +40,117 @@ def make_root(variant, root): print('') print("Success! Rez was installed to %s/rez/%s" % (repo_path, rez.__version__)) + + +def create_rez_production_scripts(target_dir, specifications): + """Create Rez production scripts + + The script will be executed with Python interpreter flag -E, which will + ignore all PYTHON* env vars, e.g. PYTHONPATH and PYTHONHOME. + + But for case like installing rez with `pip install rez --target `, + which may install rez packages into a custom location that cannot be + seen by Python unless setting PYTHONPATH, use REZ_PRODUCTION_PATH to + expose , it will be appended into sys.path before execute. + + """ + import stat + + PYTHON_TEMPLATE = r'''# -*- coding: utf-8 -*- +import re +import os +import sys +if "REZ_PRODUCTION_PATH" in os.environ: + sys.path.insert(0, os.environ["REZ_PRODUCTION_PATH"]) +from %(module)s import %(import_name)s +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe\.cmd)?$', '', sys.argv[0]) + sys.exit(%(func)s()) +''' + + CMD_TEMPLATE = r'''@echo off +set /p _rez_python=< %%~dp0.rez_production_install +%%_rez_python:~2%% -E %%~dp0%(name)s_.py %%* +''' + + BASH_TEMPLATE = r'''#!/bin/bash +export _rez_python=$(head -1 $(dirname $0)/.rez_production_install) +${_rez_python:2} -E $(dirname $0)/%(name)s_.py "$@" +''' + + scripts = [] + + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + + for specification in specifications: + entry = _get_export_entry(specification) + # add a trailing "_" to avoid module name conflict + python_script = os.path.join(target_dir, entry.name) + "_.py" + bash_script = os.path.join(target_dir, entry.name) + cmd_script = os.path.join(target_dir, entry.name) + ".cmd" + + with open(python_script, "w") as s: + s.write(PYTHON_TEMPLATE % dict( + module=entry.prefix, + import_name=entry.suffix.split('.')[0], + func=entry.suffix + )) + + with open(cmd_script, "w") as s: + s.write(CMD_TEMPLATE % dict(name=entry.name)) + + with open(bash_script, "w") as s: + s.write(BASH_TEMPLATE % dict(name=entry.name)) + os.chmod(bash_script, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + scripts += [python_script, bash_script, cmd_script] + + return scripts + + +class _ExportEntry(object): + """vended from distlib.scripts + """ + def __init__(self, name, prefix, suffix, flags): + self.name = name + self.prefix = prefix + self.suffix = suffix + self.flags = flags + + +def _get_export_entry(specification): + """vended from distlib.scripts + """ + ENTRY_RE = re.compile( + r'''(?P(\w|[-.+])+) + \s*=\s*(?P(\w+)([:\.]\w+)*) + \s*(\[\s*(?P[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])? + ''', re.VERBOSE) + + m = ENTRY_RE.search(specification) + if not m: + result = None + if '[' in specification or ']' in specification: + raise Exception("Invalid specification '%s'" % specification) + else: + d = m.groupdict() + name = d['name'] + path = d['callable'] + colons = path.count(':') + if colons == 0: + prefix, suffix = path, None + else: + if colons != 1: + raise Exception("Invalid specification '%s'" % specification) + prefix, suffix = path.split(':') + flags = d['flags'] + if flags is None: + if '[' in specification or ']' in specification: + raise Exception("Invalid specification '%s'" % specification) + flags = [] + else: + flags = [f.strip() for f in flags.split(',')] + result = _ExportEntry(name, prefix, suffix, flags) + return result