Merge pull request #548 from nerdvegas/misc-build

Misc build issues addressed

Author: nerdvegas

src/rez/build_process_.py

@@ -193,6 +193,7 @@ def visit_variants(self, func, variants=None, **kwargs):
                     % (variant.index, self._n_of_m(variant)))
                 continue
 
+            # visit the variant
             result = func(variant, **kwargs)
             results.append(result)
             num_visited += 1

src/rez/build_system.py

@@ -1,7 +1,9 @@
+import os.path
+
 from rez.build_process_ import BuildType
 from rez.exceptions import BuildSystemError
 from rez.packages_ import get_developer_package
-import os.path
+from rez.backport.lru_cache import lru_cache
 
 
 def get_buildsys_types():
@@ -10,8 +12,16 @@ def get_buildsys_types():
     return plugin_manager.get_plugins('build_system')
 
 
+@lru_cache()
 def get_valid_build_systems(working_dir):
-    """Returns the build system classes that could build the source in given dir."""
+    """Returns the build system classes that could build the source in given dir.
+
+    Note: This function is cached because the 'custom' build system type causes
+    a package load (in order to get the 'build_command' package attribute). This
+    in turn causes early-bound attribs to be evaluated, and those could be
+    expensive (eg, a smart installer pkg that hits a website to get its valid
+    versions).
+    """
     from rez.plugin_managers import plugin_manager
 
     clss = []
@@ -79,8 +89,8 @@ def __init__(self, working_dir, opts=None, package=None,
             working_dir: Directory to build source from.
             opts: argparse.Namespace object which may contain constructor
                 params, as set by our bind_cli() classmethod.
-            package (`Package`): Package to build. If None, defaults to the
-                unbuilt ('developer') package in the working directory.
+            package (`DeveloperPackage`): Package to build. If None, defaults to
+                the package in the working directory.
             write_build_scripts: If True, create build scripts rather than
                 perform the full build. The user can then run these scripts to
                 place themselves into a build environment and invoke the build
@@ -169,12 +179,14 @@ def get_standard_vars(cls, context, variant, build_type, install,
         from rez.config import config
 
         package = variant.parent
+        variant_requires = map(str, variant.variant_requires)
 
         vars_ = {
             'REZ_BUILD_ENV': 1,
             'REZ_BUILD_PATH': build_path,
             'REZ_BUILD_THREAD_COUNT': package.config.build_thread_count,
             'REZ_BUILD_VARIANT_INDEX': variant.index or 0,
+            'REZ_BUILD_VARIANT_REQUIRES': ' '.join(variant_requires),
             'REZ_BUILD_PROJECT_VERSION': str(package.version),
             'REZ_BUILD_PROJECT_NAME': package.name,
             'REZ_BUILD_PROJECT_DESCRIPTION': (package.description or '').strip(),

src/rez/cli/cp.py

@@ -27,7 +27,11 @@ def setup_parser(parser, completions=False):
         help="overwrite existing package/variants")
     parser.add_argument(
         "-s", "--shallow", action="store_true",
-        help="perform a shallow copy (symlink directories)")
+        help="perform a shallow copy (symlinks topmost directories)")
+    parser.add_argument(
+        "--follow-symlinks", action="store_true",
+        help="follow symlinks when copying package payload, rather than copying "
+        "the symlinks themselves.")
     parser.add_argument(
         "-k", "--keep-timestamp", action="store_true",
         help="keep timestamp of source package. Note that this is ignored if "
@@ -134,6 +138,7 @@ def command(opts, parser, extra_arg_groups=None):
         variants=variants,
         overwrite=opts.overwrite,
         shallow=opts.shallow,
+        follow_symlinks=opts.follow_symlinks,
         keep_timestamp=opts.keep_timestamp,
         force=opts.force,
         verbose=opts.verbose,

src/rez/cli/depends.py

@@ -15,7 +15,7 @@ def setup_parser(parser, completions=False):
         help="Include build requirements")
     parser.add_argument(
         "-p", "--private-build-requires", action="store_true",
-        help="Include private build requirements")
+        help="Include private build requirements of PKG, if any")
     parser.add_argument(
         "-g", "--graph", action="store_true",
         help="display the dependency tree as an image")

src/rez/developer_package.py

@@ -1,6 +1,6 @@
 from rez.config import config
 from rez.packages_ import Package
-from rez.serialise import load_from_file, FileFormat
+from rez.serialise import load_from_file, FileFormat, set_objects
 from rez.packages_ import create_package
 from rez.exceptions import PackageMetadataError, InvalidPackageError
 from rez.utils.system import add_sys_paths
@@ -118,6 +118,23 @@ def visit(d):
 
         return package
 
+    def get_reevaluated(self, objects):
+        """Get a newly loaded and re-evaluated package.
+
+        Values in `objects` are made available to early-bound package
+        attributes. For example, a re-evaluated package might return a different
+        value for an early-bound 'private_build_requires', depending on the
+        variant currently being built.
+
+        Args:
+            objects (`dict`): Variables to expose to early-bound package attribs.
+
+        Returns:
+            `DeveloperPackage`: New package.
+        """
+        with set_objects(objects):
+            return self.from_path(self.root)
+
     def _validate_includes(self):
         if not self.includes:
             return

src/rez/package_copy.py

@@ -1,3 +1,4 @@
+from functools import partial
 import os.path
 import time
 
@@ -11,8 +12,8 @@
 
 def copy_package(package, dest_repository, variants=None, shallow=False,
                  dest_name=None, dest_version=None, overwrite=False, force=False,
-                 dry_run=False, keep_timestamp=False, skip_payload=False,
-                 overrides=None, verbose=False):
+                 follow_symlinks=False, dry_run=False, keep_timestamp=False,
+                 skip_payload=False, overrides=None, verbose=False):
     """Copy a package from one package repository to another.
 
     This copies the package definition and payload. The package can also be
@@ -58,6 +59,8 @@ def copy_package(package, dest_repository, variants=None, shallow=False,
         force (bool): Copy the package regardless of its relocatable attribute.
             Use at your own risk (there is no guarantee the resulting package
             will be functional).
+        follow_symlinks (bool): Follow symlinks when copying package payload,
+            rather than copying the symlinks themselves.
         keep_timestamp (bool): By default, a newly copied package will get a
             new timestamp (because that's when it was added to the target repo).
             By setting this option to True, the original package's timestamp
@@ -182,6 +185,7 @@ def finalize():
                     src_variant=src_variant,
                     dest_pkg_repo=dest_pkg_repo,
                     shallow=shallow,
+                    follow_symlinks=follow_symlinks,
                     overrides=overrides
                 )
 
@@ -207,7 +211,7 @@ def finalize():
 
 
 def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
-                          overrides=None):
+                          follow_symlinks=False, overrides=None):
         # Get payload path of source variant. For some types (eg from a "memory"
         # type repo) there may not be a root.
         #
@@ -241,7 +245,12 @@ def _copy_variant_payload(src_variant, dest_pkg_repo, shallow=False,
         if shallow:
             maybe_symlink = replacing_symlink
         else:
-            maybe_symlink = replacing_copy
+            maybe_symlink = partial(
+                replacing_copy,
+                copytree_kwargs={
+                    "symlinks": (not follow_symlinks)
+                }
+            )
 
         if src_variant.subpath:
             # symlink/copy the last install dir to the variant root

src/rez/package_search.py

@@ -37,6 +37,9 @@ def get_reverse_dependency_tree(package_name, depth=None, paths=None,
         depth (int): Tree depth limit, unlimited if None.
         paths (list of str): paths to search for packages, defaults to
             `config.packages_path`.
+        build_requires (bool): If True, includes packages' build_requires.
+        private_build_requires (bool): If True, include `package_name`'s
+            private_build_requires.
 
     Returns:
         A 2-tuple:
@@ -71,7 +74,12 @@ def get_reverse_dependency_tree(package_name, depth=None, paths=None,
         requires = []
 
         for variant in pkg.iter_variants():
-            requires += variant.get_requires(build_requires, private_build_requires)
+            pbr = (private_build_requires and pkg.name == package_name)
+
+            requires += variant.get_requires(
+                build_requires=build_requires,
+                private_build_requires=pbr
+            )
 
         for req in requires:
             if not req.conflict:

src/rez/packages_.py

@@ -72,7 +72,9 @@ class PackageBaseResourceWrapper(PackageRepositoryResourceWrapper):
     def __init__(self, resource, context=None):
         super(PackageBaseResourceWrapper, self).__init__(resource)
         self.context = context
-        self._late_bindings = {}
+
+        # cached results of late-bound funcs
+        self._late_binding_returnvalues = {}
 
     def set_context(self, context):
         self.context = context
@@ -134,16 +136,19 @@ def print_info(self, buf=None, format_=FileFormat.yaml,
 
     def _wrap_forwarded(self, key, value):
         if isinstance(value, SourceCode) and value.late_binding:
-            value_ = self._late_bindings.get(key, KeyError)
+            # get cached return value if present
+            value_ = self._late_binding_returnvalues.get(key, KeyError)
 
             if value_ is KeyError:
+                # evaluate the late-bound function
                 value_ = self._eval_late_binding(value)
 
                 schema = self.late_bind_schemas.get(key)
                 if schema is not None:
                     value_ = schema.validate(value_)
 
-                self._late_bindings[key] = value_
+                # cache result of late bound func
+                self._late_binding_returnvalues[key] = value_
 
             return value_
         else:
@@ -162,14 +167,12 @@ def _eval_late_binding(self, sourcecode):
             bindings = self.context._get_pre_resolve_bindings()
             g.update(bindings)
 
-        # note that what 'this' actually points to depends on whether the context
-        # is available or not. If not, then 'this' is a Package instance; if the
-        # context is available, it is a Variant instance. So for example, if
-        # in_context() is True, 'this' will have a 'root' attribute, but will
-        # not if in_context() is False.
+        # Note that 'this' could be a `Package` or `Variant` instance. This is
+        # intentional; it just depends on how the package is accessed.
         #
         g["this"] = self
 
+        # evaluate the late-bound function
         sourcecode.set_package(self)
         return sourcecode.exec_(globals_=g)
 
@@ -183,6 +186,12 @@ class Package(PackageBaseResourceWrapper):
     """
     keys = schema_keys(package_schema)
 
+    # This is to allow for a simple check like 'this.is_package' in late-bound
+    # funcs, where 'this' may be a package or variant.
+    #
+    is_package = True
+    is_variant = False
+
     def __init__(self, resource, context=None):
         _check_class(resource, PackageResource)
         super(Package, self).__init__(resource, context)
@@ -268,6 +277,10 @@ class Variant(PackageBaseResourceWrapper):
     keys = schema_keys(variant_schema)
     keys.update(["index", "root", "subpath"])
 
+    # See comment in `Package`
+    is_package = False
+    is_variant = True
+
     def __init__(self, resource, context=None, parent=None):
         _check_class(resource, VariantResource)
         super(Variant, self).__init__(resource, context)
@@ -316,23 +329,31 @@ def parent(self):
 
         return self._parent
 
+    @property
+    def variant_requires(self):
+        """Get the subset of requirements specific to this variant.
+
+        Returns:
+            List of `Requirement` objects.
+        """
+        if self.index is None:
+            return []
+        else:
+            return self.parent.variants[self.index] or []
+
     @property
     def requires(self):
         """Get variant requirements.
 
         This is a concatenation of the package requirements and those of this
         specific variant.
-        """
-        try:
-            package_requires = self.parent.requires or []
 
-            if self.index is None:
-                return package_requires
-            else:
-                variant_requires = self.parent.variants[self.index] or []
-                return package_requires + variant_requires
-        except AttributeError as e:
-            reraise(e, ValueError)
+        Returns:
+            List of `Requirement` objects.
+        """
+        return (
+            (self.parent.requires or []) + self.variant_requires
+        )
 
     def get_requires(self, build_requires=False, private_build_requires=False):
         """Get the requirements of the variant.
@@ -433,9 +454,9 @@ def _repository_uids(self):
         return uids
 
 
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 # resource acquisition functions
-#------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 
 def iter_package_families(paths=None):
     """Iterate over package families, in no particular order.
@@ -551,6 +572,15 @@ def get_package_from_string(txt, paths=None):
 
 
 def get_developer_package(path, format=None):
+    """Create a developer package.
+
+    Args:
+        path (str): Path to dir containing package definition file.
+        format (str): Package definition file format, detected if None.
+
+    Returns:
+        `DeveloperPackage`.
+    """
     from rez.developer_package import DeveloperPackage
     return DeveloperPackage.from_path(path, format=format)
 

src/rez/rex.py

@@ -1141,8 +1141,8 @@ def compile_code(cls, code, filename=None, exec_namespace=None):
             code (str or SourceCode): The python code to compile.
             filename (str): File to associate with the code, will default to
                 '<string>'.
-            namespace (dict): Namespace to execute the code in. If None, the
-                code is not executed.
+            exec_namespace (dict): Namespace to execute the code in. If None,
+                the code is not executed.
 
         Returns:
             Compiled code object.