- reimplemented expand_requirement() to be more robust, allow any valid version range

- exposed all variant attribs in rex binding object
- expose 'this' in @early bound package functions
- strip functions, leading-__ variables from package.py
- added tests for expand_requirement()
- added Version.as_tuple(), and matching unit test

Author: ajohns

src/rez/developer_package.py

@@ -124,10 +124,9 @@ def _get_preprocessed(self, data):
         from copy import deepcopy
 
         with add_sys_paths(config.package_definition_build_python_paths):
-            preprocess = getattr(self, "preprocess", None)
+            preprocess_func = getattr(self, "preprocess", None)
 
-            if preprocess:
-                preprocess_func = preprocess.func
+            if preprocess_func:
                 print_info("Applying preprocess from package.py")
             else:
                 # load globally configured preprocess function
@@ -173,7 +172,7 @@ def _get_preprocessed(self, data):
                             % (e.__class__.__name__, str(e)))
                 return None
 
-        # if preprocess added functions, these need to be converted to
+        # if preprocess added functions, these may need to be converted to
         # SourceCode instances
         preprocessed_data = process_python_objects(preprocessed_data)
 

src/rez/package_py_utils.py

@@ -1,6 +1,6 @@
 """
-This sourcefile is intended to only be imported in package.py files, in
-functions including:
+This sourcefile is intended to be imported in package.py files, in functions
+including:
 
 - the special 'preprocess' function;
 - early bound functions that use the @early decorator.
@@ -11,46 +11,133 @@
 from rez.exceptions import InvalidPackageError
 
 
-def expand_requirement(request):
-    """Expands a requirement string like 'python-2.*'
+def expand_requirement(request, paths=None):
+    """Expands a requirement string like 'python-2.*', 'foo-2.*+<*', etc.
 
-    Only trailing wildcards are supported; they will be replaced with the
-    latest package version found within the range. If none are found, the
-    wildcards will just be stripped.
+    Wildcards are expanded to the latest version that matches. There is also a
+    special wildcard '**' that will expand to the full version, but it cannot
+    be used in combination with '*'.
 
-    Example:
+    Wildcards MUST placehold a whole version token, not partial - while 'foo-2.*'
+    is valid, 'foo-2.v*' is not.
+
+    Wildcards MUST appear at the end of version numbers - while 'foo-1.*.*' is
+    valid, 'foo-1.*.0' is not.
+
+    It is possible that an expansion will result in an invalid request string
+    (such as 'foo-2+<2'). The appropriate exception will be raised if this
+    happens.
+
+    Examples:
 
         >>> print expand_requirement('python-2.*')
         python-2.7
+        >>> print expand_requirement('python==2.**')
+        python==2.7.12
+        >>> print expand_requirement('python<**')
+        python<3.0.5
 
     Args:
         request (str): Request to expand, eg 'python-2.*'
+        paths (list of str, optional): paths to search for package families,
+            defaults to `config.packages_path`.
 
     Returns:
         str: Expanded request string.
     """
     if '*' not in request:
         return request
 
-    from rez.vendor.version.requirement import VersionedObject, Requirement
+    from rez.vendor.version.version import VersionRange
+    from rez.vendor.version.requirement import Requirement
     from rez.packages_ import get_latest_package
+    from uuid import uuid4
 
-    txt = request.replace('*', '_')
-    obj = VersionedObject(txt)
-    rank = len(obj.version)
-
+    wildcard_map = {}
+    expanded_versions = {}
     request_ = request
-    while request_.endswith('*'):
-        request_ = request_[:-2]  # strip sep + *
-
-    req = Requirement(request_)
-    package = get_latest_package(name=req.name, range_=req.range_)
-
-    if package is None:
-        return request_
 
-    obj.version_ = package.version.trim(rank)
-    return str(obj)
+    # replace wildcards with valid version tokens that can be replaced again
+    # afterwards. This produces a horrendous, but both valid and temporary,
+    # version string.
+    #
+    while "**" in request_:
+        uid = "_%s_" % uuid4().hex
+        request_ = request_.replace("**", uid, 1)
+        wildcard_map[uid] = "**"
+
+    while '*' in request_:
+        uid = "_%s_" % uuid4().hex
+        request_ = request_.replace('*', uid, 1)
+        wildcard_map[uid] = '*'
+
+    # create the requirement, then expand wildcards
+    #
+    req = Requirement(request_, invalid_bound_error=False)
+
+    def expand_version(version):
+        rank = len(version)
+        wildcard_found = False
+
+        while version and str(version[-1]) in wildcard_map:
+            token = wildcard_map[str(version[-1])]
+            version = version.trim(len(version) - 1)
+
+            if token == "**":
+                if wildcard_found:  # catches bad syntax '**.*'
+                    return None
+                else:
+                    wildcard_found = True
+                    rank = 0
+                    break
+
+            wildcard_found = True
+
+        if not wildcard_found:
+            return None
+
+        range_ = VersionRange(str(version))
+        package = get_latest_package(name=req.name, range_=range_, paths=paths)
+
+        if package is None:
+            return version
+
+        if rank:
+            return package.version.trim(rank)
+        else:
+            return package.version
+
+    def visit_version(version):
+        # requirements like 'foo-1' are actually represented internally as
+        # 'foo-1+<1_' - '1_' is the next possible version after '1'. So we have
+        # to detect this case and remap the uid-ified wildcard back here too.
+        #
+        for v, expanded_v in expanded_versions.iteritems():
+            if version == v.next():
+                return expanded_v.next()
+
+        version_ = expand_version(version)
+        if version_ is None:
+            return None
+
+        expanded_versions[version] = version_
+        return version_
+
+    if req.range_ is not None:
+        req.range_.visit_versions(visit_version)
+
+    result = str(req)
+
+    # do some cleanup so that long uids aren't left in invalid wildcarded strings
+    for uid, token in wildcard_map.iteritems():
+        result = result.replace(uid, token)
+
+    # cast back to a Requirement again, then back to a string. This will catch
+    # bad verison ranges, but will also put OR'd version ranges into the correct
+    # order
+    expanded_req = Requirement(result)
+
+    return str(expanded_req)
 
 
 def expand_requires(*requests):

src/rez/rex_bindings.py

@@ -103,14 +103,22 @@ def __iter__(self):
 class VariantBinding(Binding):
     """Binds a packages.Variant object."""
     def __init__(self, variant):
-        doc = dict(
-            name=variant.name,
-            version=VersionBinding(variant.version),
-            base=variant.base,
-            root=variant.root)
+        doc = dict(version=VersionBinding(variant.version))
         super(VariantBinding, self).__init__(doc)
         self.__variant = variant
 
+    # hacky, but we'll be deprecating all these bindings..
+    def __getattr__(self, attr):
+        try:
+            return super(VariantBinding, self).__getattr__(attr)
+        except:
+            missing = object()
+            value = getattr(self.__variant, attr, missing)
+            if value is missing:
+                raise
+
+            return value
+
     def _attr_error(self, attr):
         raise AttributeError("package %s has no attribute '%s'"
                              % (str(self), attr))

src/rez/serialise.py

@@ -170,40 +170,84 @@ def load_py(stream, filepath=None):
     return result
 
 
+class EarlyThis(object):
+    """The 'this' object for @early bound functions."""
+    def __init__(self, data):
+        self._data = data
+
+    def __getattr__(self, attr):
+        missing = object()
+        value = self._data.get(attr, missing)
+        if value is missing:
+            raise AttributeError("No such package attribute '%s'" % attr)
+
+        if isfunction(value) and (hasattr(value, "_early") or hasattr(value, "_late")):
+            raise ValueError(
+                "An early binding function cannot refer to another early or "
+                "late binding function: '%s'" % attr)
+
+        return value
+
+
 def process_python_objects(data, filepath=None):
+    """Replace certain values in the given package data dict.
 
-    _remove = object()
+    Does things like:
+    * evaluates @early decorated functions, and replaces with return value;
+    * converts functions into `SourceCode` instances so they can be serialized
+      out to installed packages, and evaluated later;
+    * strips some values (modules, __-leading variables) that are never to be
+      part of installed packages.
 
+    Returns:
+        dict: Updated dict.
+    """
     def _process(value):
         if isinstance(value, dict):
             for k, v in value.items():
-                new_value = _process(v)
-
-                if new_value is _remove:
-                    del value[k]
-                else:
-                    value[k] = new_value
+                value[k] = _process(v)
 
             return value
         elif isfunction(value):
-            if hasattr(value, "_early"):
+            func = value
+
+            if hasattr(func, "_early"):
                 # run the function now, and replace with return value
-                with add_sys_paths(config.package_definition_build_python_paths):
-                    func = value
+                #
+
+                # make a copy of the func with its own globals, and add 'this'
+                import types
+                fn = types.FunctionType(func.func_code,
+                                        func.func_globals.copy(),
+                                        name=func.func_name,
+                                        argdefs=func.func_defaults,
+                                        closure=func.func_closure)
 
+                this = EarlyThis(data)
+                fn.func_globals.update({"this": this})
+
+                with add_sys_paths(config.package_definition_build_python_paths):
+                    # this 'data' arg support isn't needed anymore, but I'm
+                    # supporting it til I know nobody is using it...
+                    #
                     spec = getargspec(func)
                     args = spec.args or []
                     if len(args) not in (0, 1):
                         raise ResourceError("@early decorated function must "
                                             "take zero or one args only")
                     if args:
-                        value_ = func(data)
+                        value_ = fn(data)
                     else:
-                        value_ = func()
+                        value_ = fn()
 
                 # process again in case this is a function returning a function
                 return _process(value_)
-            else:
+
+            elif hasattr(func, "_late"):
+                return SourceCode(func=func, filepath=filepath,
+                                  eval_as_function=True)
+
+            elif func.__name__ in package_rex_keys:
                 # if a rex function, the code has to be eval'd NOT as a function,
                 # otherwise the globals dict doesn't get updated with any vars
                 # defined in the code, and that means rex code like this:
@@ -214,20 +258,37 @@ def _process(value):
                 # ..won't work. It was never intentional that the above work, but
                 # it does, so now we have to keep it so.
                 #
-                as_function = (value.__name__ not in package_rex_keys)
-
-                return SourceCode(func=value, filepath=filepath,
-                                  eval_as_function=as_function)
-        elif ismodule(value):
-            # modules cannot be installed as package attributes. They are present
-            # in developer packages sometimes though - it's fine for a package
-            # attribute to use an imported module at build time.
-            #
-            return _remove
+                return SourceCode(func=func, filepath=filepath,
+                                  eval_as_function=False)
+
+            else:
+                # a normal function. Leave unchanged, it will be stripped after
+                return func
         else:
             return value
 
-    return _process(data)
+    def _trim(value):
+        if isinstance(value, dict):
+            for k, v in value.items():
+                if isfunction(v):
+                    if v.__name__ == "preprocess":
+                        # preprocess is a special case. It has to stay intact
+                        # until the `DeveloperPackage` has a chance to apply it;
+                        # after which it gets removed from the package attributes.
+                        #
+                        pass
+                    else:
+                        del value[k]
+                elif ismodule(v) or k.startswith("__"):
+                    del value[k]
+                else:
+                    value[k] = _trim(v)
+
+        return value
+
+    data = _process(data)
+    data = _trim(data)
+    return data
 
 
 def load_yaml(stream, **kwargs):