Make rez wheels production ready

.github/scripts/create_python_package.py

@@ -0,0 +1,46 @@
+import os
+import tarfile
+import argparse
+import platform
+import urllib.request
+
+import rez.packages
+import rez.package_maker
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("version", help="Python version")
+parser.add_argument("repository", help="Repository path")
+
+args = parser.parse_args()
+
+
+INTERPRETERS = {
+    ("Windows", "AMD64"): "https://github.com/indygreg/python-build-standalone/releases/download/20240415/cpython-{version}+20240415-x86_64-pc-windows-msvc-install_only.tar.gz",
+    ("Linux", "x86_64"): "https://github.com/indygreg/python-build-standalone/releases/download/20240415/cpython-{version}+20240415-x86_64-unknown-linux-gnu-install_only.tar.gz",
+    ("Darwin", "x86_64"): "https://github.com/indygreg/python-build-standalone/releases/download/20240415/cpython-{version}+20240415-x86_64-apple-darwin-install_only.tar.gz",
+}
+
+def make_root(variant: rez.packages.Variant, path: str):
+    url = INTERPRETERS[(platform.system(), platform.machine())]
+
+    print(f"Downloading {url!r} and extracting to {path!r}")
+    with urllib.request.urlopen(url.format(version=variant.version)) as response:
+        tar = tarfile.open(fileobj=response, mode="r:gz")
+        tar.extractall(path=path)
+
+
+with rez.package_maker.make_package(
+    "python", os.path.expanduser(args.repository), make_root=make_root
+) as package:
+    package.version = args.version
+    commands = [
+        "env.PATH.prepend('{root}/python/bin')",
+    ]
+    if platform.system() == "Windows":
+        commands = [
+            "env.PATH.prepend('{root}/python')",
+            "env.PATH.prepend('{root}/python/DLLs')",
+        ]
+
+    package.commands = "\n".join(commands)

.github/workflows/installation.yaml

@@ -41,17 +41,25 @@ jobs:
           REZ_INSTALL_COMMAND: python ./install.py /opt/rez
         - os: ubuntu-latest
           method: pip
-          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:/opt/rez/bin PYTHONPATH=${PYTHONPATH}:/opt/rez'
-          REZ_INSTALL_COMMAND: pip install --target /opt/rez .
+          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:/opt/rez/bin/rez PYTHONPATH=${PYTHONPATH}:/opt/rez'
+          REZ_INSTALL_COMMAND: |
+            set -ex
+            python -m pip install virtualenv
+            python -m virtualenv /opt/rez
+            /opt/rez/bin/python -m pip install .
         # macOS
         - os: macos-latest
           method: install
           REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:~/rez/bin/rez'
           REZ_INSTALL_COMMAND: python ./install.py ~/rez
         - os: macos-latest
           method: pip
-          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin" PYTHONPATH=$PYTHONPATH:$HOME/rez'
-          REZ_INSTALL_COMMAND: pip install --target ~/rez .
+          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin/rez" PYTHONPATH=$PYTHONPATH:$HOME/rez'
+          REZ_INSTALL_COMMAND: |
+            set -ex
+            python -m pip install virtualenv
+            python -m virtualenv ~/rez
+            ~/rez/bin/python -m pip install .
         # macOS
         # Python 3.7 is not supported on Apple Silicon.
         # macos-13 is the last macos runner image to run on Intel CPUs.
@@ -63,17 +71,24 @@ jobs:
         - os: macos-13
           python-version: '3.7'
           method: pip
-          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin" PYTHONPATH=$PYTHONPATH:$HOME/rez'
-          REZ_INSTALL_COMMAND: pip install --target ~/rez .
+          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin/rez" PYTHONPATH=$PYTHONPATH:$HOME/rez'
+          REZ_INSTALL_COMMAND: |
+            set -ex
+            python -m pip install virtualenv
+            python -m virtualenv ~/rez
+            ~/rez/bin/python -m pip install .
         # windows
         - os: windows-latest
           method: install
           REZ_SET_PATH_COMMAND: '$env:PATH="$env:PATH;C:\ProgramData\rez\Scripts\rez"'
           REZ_INSTALL_COMMAND: python ./install.py C:\ProgramData\rez
         - os: windows-latest
           method: pip
-          REZ_SET_PATH_COMMAND: '[System.Environment]::SetEnvironmentVariable("PATH","$env:PATH;C:\ProgramData\rez\bin"); $env:PYTHONPATH="$env:PYTHONPATH;C:\ProgramData\rez"'
-          REZ_INSTALL_COMMAND: pip install --target C:\ProgramData\rez .
+          REZ_SET_PATH_COMMAND: '[System.Environment]::SetEnvironmentVariable("PATH","$env:PATH;C:\ProgramData\rez\Scripts\rez")'
+          REZ_INSTALL_COMMAND: |
+            python -m pip install virtualenv
+            python -m virtualenv C:\ProgramData\rez
+            C:\ProgramData\rez\Scripts\python.exe -m pip install .
 
     steps:
     - uses: actions/checkout@v4

.github/workflows/wheel.yaml

@@ -0,0 +1,155 @@
+name: wheel
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  wheel:
+    name: Build ${{ matrix.title }} wheel
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - os: ubuntu-latest
+          title: pure
+        - os: windows-latest
+          arch: win_amd64
+          title: windows amd64
+          msvc-arch: amd64
+        - os: windows-latest
+          arch: win_arm64
+          title: windows arm64
+          msvc-arch: amd64_arm64
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - name: Build wheel
+      shell: bash
+      run: |
+        set -ex
+        python -m pip install build
+        python -m build -w . --outdir ./dist
+      env:
+        PYTHON_PLAT_NAME: ${{ matrix.arch }}
+
+    - uses: actions/upload-artifact@v4
+      with:
+        path: dist
+        name: wheels-${{ matrix.arch }}
+
+    - uses: ilammy/msvc-dev-cmd@v1
+      if: ${{ matrix.os == 'windows-latest' }}
+      with:
+        arch: ${{ matrix.msvc-arch }}
+
+    - name: Analyze EXEs
+      if: ${{ matrix.os == 'windows-latest' }}
+      run: |
+        cd dist
+        unzip *.whl
+        dumpbin.exe /DEPENDENTS rez-*.data/scripts/rez/rez.exe
+        echo ((Get-Item -Path rez-*.data/scripts/rez/rez.exe).Length / 1kb)
+
+  test:
+    name: Test wheels
+    needs: ["wheel"]
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Run on macos-13 to get conda... We should install miniconda manually.
+        os: [macos-13, ubuntu-latest, windows-latest]
+
+    env:
+      CURRENT_PLATFORM: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: 'wheels-*'
+        merge-multiple: true
+        path: wheels
+
+    - name: Install wheel
+      shell: bash
+      run: |
+        set -ex
+        python -m venv .venv
+        if [[ "${CURRENT_PLATFORM}" == "windows-latest" ]]; then
+            .venv/Scripts/python.exe -m pip install rez --find-links ./wheels --no-index
+            ls -la .venv/Scripts/
+            ls -la .venv/Scripts/rez
+        else
+            .venv/bin/python -m pip install rez --find-links ./wheels --no-index
+            ls -la .venv/bin/
+            ls -la .venv/bin/rez
+        fi
+
+    - name: Test commands
+      shell: bash
+      run: |
+        set -ex
+        if [[ "${CURRENT_PLATFORM}" == "windows-latest" ]]; then
+            export PATH=$(pwd)/.venv/Scripts/rez:$PATH
+        else
+            export PATH=$(pwd)/.venv/bin/rez:$PATH
+        fi
+
+        echo 'Running jctest with REZ_LAUNCHER_DEBUG=1'
+        export REZ_LAUNCHER_DEBUG=1
+
+        _rez-install-test
+
+        rez --help
+        rez --version
+        rez-env --help
+
+    - name: Integration test
+      shell: bash
+      run: |
+        set -ex
+
+        interpreter_path=""
+        export REZ_PACKAGES_PATH="~/rez_packages"
+        if [[ "${CURRENT_PLATFORM}" == "windows-latest" ]]; then
+            interpreter_path='.venv/Scripts/python.exe'
+            export PATH=$(pwd)/.venv/Scripts/rez:$PATH
+        else
+            interpreter_path=.'venv/bin/python'
+            export PATH=$(pwd)/.venv/bin/rez:$PATH
+        fi
+        "${interpreter_path}" .github/scripts/create_python_package.py 3.12.3 $REZ_PACKAGES_PATH
+
+        # First, test that the "python" package is found and is the right version.
+        test "$(rez-env python -- python --version)" = 'Python 3.12.3'
+
+        # Now test that the -E flag is used. This manifest with sys.flags.ignore_environment=1
+        test $(rez-env python -- _rez-install-test | jq '.sysflags.ignore_environment' -r) -eq 1
+
+        # Now test that the executable used for runnin the rez command is the one from the rez
+        # install, not the rez package.
+        expected_path="$(pwd)/${interpreter_path}"
+        if [[ "${CURRENT_PLATFORM}" == "windows-latest" ]]; then
+            # Convert unix path to windows path
+            expected_path=$(cygpath -w "${expected_path}")
+        fi
+        test $(rez-env python -- _rez-install-test | jq '.executable' -r) = "${expected_path}"

install.py

@@ -78,19 +78,23 @@ def get_virtualenv_py_executable(dest_dir):
     return bin_dir, which("python", env=env)
 
 
-def run_command(args, cwd=source_path):
+def run_command(args, cwd=source_path, env=None):
     if opts.verbose:
         print("running in %s: %s" % (cwd, " ".join(args)))
-    return subprocess.check_output(args, cwd=source_path)
+
+    if env is None:
+        env = os.environ.copy()
+
+    return subprocess.check_output(args, cwd=source_path, env=env)
 
 
 def patch_rez_binaries(dest_dir):
     virtualenv_bin_path, py_executable = get_virtualenv_py_executable(dest_dir)
 
-    specs = get_specifications()
+    entrypoints = get_specifications()
 
     # delete rez bin files written into virtualenv
-    for name in specs.keys():
+    for name in entrypoints.keys():
         basepath = os.path.join(virtualenv_bin_path, name)
         filepaths = [
             basepath,
@@ -119,7 +123,7 @@ def patch_rez_binaries(dest_dir):
     maker.executable = py_executable
 
     maker.make_multiple(
-        specifications=specs.values(),
+        specifications=[ep.spec for ep in entrypoints.values()],
         # the -E arg is crucial - it means rez cli tools still work within a
         # rez-resolved env, even if PYTHONPATH or related env-vars would have
         # otherwise changed rez's behaviour
@@ -162,6 +166,9 @@ def install(dest_dir, print_welcome=False, editable=False):
     install_rez_from_source(dest_dir, editable=editable)
 
     # patch the rez binaries
+    # Note that we don't use our custom launcher when using the installer.
+    # This is for performance reasons as the new launcher as a slight startup cost
+    # increase compared to our patched binaries on Unix.
     patch_rez_binaries(dest_dir)
 
     # copy completion scripts into virtualenv
@@ -170,9 +177,6 @@ def install(dest_dir, print_welcome=False, editable=False):
     # mark virtualenv as production rez install. Do not remove - rez uses this!
     virtualenv_bin_dir = get_virtualenv_bin_dir(dest_dir)
     dest_bin_dir = os.path.join(virtualenv_bin_dir, "rez")
-    validation_file = os.path.join(dest_bin_dir, ".rez_production_install")
-    with open(validation_file, 'w') as f:
-        f.write(_rez_version)
 
     # done
     if print_welcome:
@@ -230,7 +234,15 @@ def install_rez_from_source(dest_dir, editable):
     if editable:
         args.append("-e")
     args.append(".")
-    run_command(args)
+
+    env = os.environ.copy()
+
+    # patch the rez binaries
+    # Note that we don't use our custom launcher when using the installer.
+    # This is for performance reasons as the new launcher as a slight startup cost
+    # increase compared to our patched binaries on Unix.
+    env["REZ_USE_STANDARD_LAUNCHER"] = "1"
+    run_command(args, env=env)
 
 
 def install_as_rez_package(repo_path):

launcher/LICENSE

@@ -0,0 +1,22 @@
+Copyright (C) 2011-2022 Vinay Sajip. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

launcher/README.md

@@ -0,0 +1,12 @@
+# Script launchers
+
+This folder contains the script launchers. It is basically https://github.com/pypa/distlib/blob/0.3.7/PC/launcher.c
+with some light modifications:
+
+1. We don't embbed the scripts in the executable.
+2. We removed the support to use environment variables to locate the interpreter.
+3. We added a small patch to print the command executed if the environment variable
+   REZ_LAUNCHER_DEBUG is set.
+
+The launcher is only needed on Windows and is compiled from the setup.py file. MSVC
+isn't required. We use the [Zig](https://ziglang.org) compiler (zig cc) to compile.