Merge branch 'issue_323'

# Conflicts:
#	src/rez/utils/_version.py
#	src/rezplugins/package_repository/filesystem.py

Author: ajohns

src/rez/build_process_.py

@@ -189,13 +189,23 @@ def create_build_context(self, variant, build_type, build_path):
 
         requests_str = ' '.join(map(str, request))
         self._print("Resolving build environment: %s", requests_str)
+
         if build_type == BuildType.local:
             packages_path = self.package.config.packages_path
         else:
             packages_path = self.package.config.nonlocal_packages_path
 
+        if self.package.config.is_overridden("package_filter"):
+            from rez.package_filter import PackageFilterList
+
+            data = self.package.config.package_filter
+            package_filter = PackageFilterList.from_pod(data)
+        else:
+            package_filter = None
+
         context = ResolvedContext(request,
                                   package_paths=packages_path,
+                                  package_filter=package_filter,
                                   building=True)
         if self.verbose:
             context.print_info()

src/rez/config.py

@@ -388,6 +388,9 @@ def override(self, key, value):
             self.overrides[key] = value
             self._uncache(key)
 
+    def is_overridden(self, key):
+        return (key in self.overrides)
+
     def remove_override(self, key):
         """Remove a setting override, if one exists."""
         keys = key.split('.')

src/rez/package_repository.py

@@ -141,6 +141,14 @@ def iter_variants(self, package_resource):
         """
         raise NotImplementedError
 
+    def pre_variant_install(self, variant_resource):
+        """Called before a variant is installed.
+
+        If any directories are created on disk for the variant to install into,
+        this is called before that happens.
+        """
+        pass
+
     def install_variant(self, variant_resource, dry_run=False, overrides=None):
         """Install a variant into this repository.
 

src/rez/utils/_version.py

@@ -1,7 +1,7 @@
 
 
 # Update this value to version up Rez. Do not place anything else in this file.
-_rez_version = "2.1.0"
+_rez_version = "2.2.0"
 
 
 # Copyright 2013-2016 Allan Johns.

src/rezplugins/build_process/local.py

@@ -1,6 +1,7 @@
 """
 Builds packages on local host
 """
+from rez.package_repository import package_repository_manager
 from rez.build_process_ import BuildProcessHelper, BuildType
 from rez.release_hook import ReleaseHookEvent
 from rez.exceptions import BuildError, ReleaseError
@@ -96,10 +97,16 @@ def _build_variant_base(self, variant, build_type, install_path=None,
         install_path = install_path or self.package.config.local_packages_path
         variant_install_path = self.get_package_install_path(install_path)
         variant_build_path = self.build_path
+
         if variant.subpath:
             variant_build_path = os.path.join(variant_build_path, variant.subpath)
             variant_install_path = os.path.join(variant_install_path, variant.subpath)
 
+        # inform package repo that a variant is about to be built/installed
+        pkg_repo = package_repository_manager.get_repository(install_path)
+        pkg_repo.pre_variant_install(variant.resource)
+
+        # create directories (build, install)
         if clean and os.path.exists(variant_build_path):
             shutil.rmtree(variant_build_path)
         if not os.path.exists(variant_build_path):

src/rezplugins/package_repository/filesystem.py

@@ -74,7 +74,6 @@ def iter_packages(self):
 
         # versioned packages
         for version_str in self._repository._get_version_dirs(self.path):
-
             if _settings.check_package_definition_files:
                 path = os.path.join(self.path, version_str)
                 if not self._repository._get_file(path)[0]:
@@ -85,6 +84,7 @@ def iter_packages(self):
                 location=self.location,
                 name=self.name,
                 version=version_str)
+
             yield package
 
 
@@ -177,8 +177,6 @@ def _load_old_formats(self):
                         pass
         return data
 
-    # should be static or classmethod, since it's passed as an arg to
-    # load_from_file, which is memcached
     @staticmethod
     def _update_changelog(file_format, data):
         # this is to deal with older package releases. They can contain long
@@ -404,6 +402,8 @@ class FileSystemPackageRepository(PackageRepository):
                    "file_lock_dir": Or(None, str),
                    "package_filenames": [basestring]}
 
+    building_prefix = ".building"
+
     @classmethod
     def name(cls):
         return "filesystem"
@@ -489,6 +489,24 @@ def file_lock_dir(self):
 
         return dirname
 
+    def pre_variant_install(self, variant_resource):
+        if not variant_resource.version:
+            return
+
+        # create 'building' tagfile, this makes sure that a resolve doesn't
+        # pick up this package if it doesn't yet have a package.py created.
+        path = self.location
+
+        family_path = os.path.join(path, variant_resource.name)
+        if not os.path.isdir(family_path):
+            os.makedirs(family_path)
+
+        filename = self.building_prefix + str(variant_resource.version)
+        filepath = os.path.join(family_path, filename)
+
+        with open(filepath, 'w'):  # create empty file
+            pass
+
     def install_variant(self, variant_resource, dry_run=False, overrides=None):
         if variant_resource._repository is self:
             return variant_resource
@@ -549,6 +567,7 @@ def _get_family_dirs(self):
         dirs = []
         if not os.path.isdir(self.location):
             return dirs
+
         for name in os.listdir(self.location):
             path = os.path.join(self.location, name)
             if os.path.isdir(path):
@@ -558,6 +577,7 @@ def _get_family_dirs(self):
                 name_, ext_ = os.path.splitext(name)
                 if ext_ in (".py", ".yaml") and is_valid_package_name(name_):
                     dirs.append((name_, ext_[1:]))
+
         return dirs
 
     def _get_version_dirs__key(self, root):
@@ -569,14 +589,59 @@ def _get_version_dirs__key(self, root):
                key=_get_version_dirs__key,
                debug=config.debug_memcache)
     def _get_version_dirs(self, root):
-        dirs = []
+
+        # simpler case if this test is on
+        #
+        if _settings.check_package_definition_files:
+            dirs = []
+
+            for name in os.listdir(root):
+                if name.startswith('.'):
+                    continue
+
+                path = os.path.join(root, name)
+                if os.path.isdir(path):
+                    if not self._is_valid_package_directory(path):
+                        continue
+
+                dirs.append(name)
+            return dirs
+
+        # with test off, we have to check for 'building' dirs, these have to be
+        # tested regardless. Failed releases may cause 'building files' to be
+        # left behind, so we need to clear these out also
+        #
+        dirs = set()
+        building_dirs = set()
+
+        # find dirs and dirs marked as 'building'
         for name in os.listdir(root):
             if name.startswith('.'):
-                continue
+                if not name.startswith(self.building_prefix):
+                    continue
+
+                ver_str = name[len(self.building_prefix):]
+                building_dirs.add(ver_str)
+
             path = os.path.join(root, name)
             if os.path.isdir(path):
-                dirs.append(name)
-        return dirs
+                dirs.add(name)
+
+        # check 'building' dirs for validity
+        for name in building_dirs:
+            if name not in dirs:
+                continue
+
+            path = os.path.join(root, name)
+            if not self._is_valid_package_directory(path):
+                # package probably still being built
+                dirs.remove(name)
+
+        return list(dirs)
+
+    # True if `path` contains package.py or similar
+    def _is_valid_package_directory(self, path):
+        return bool(self._get_file(path, "package")[0])
 
     def _get_families(self):
         families = []
@@ -593,6 +658,7 @@ def _get_families(self):
                     name=name,
                     ext=ext)
             families.append(family)
+
         return families
 
     def _get_family(self, name):
@@ -772,7 +838,7 @@ def _create_variant(self, variant, dry_run=False, overrides=None):
         package_data["config"] = parent_package._data.get("config")
         package_data.pop("base", None)
 
-        # create version dir and write out the new package definition file
+        # create version dir if it doesn't already exist
         family_path = os.path.join(self.location, variant.name)
         if variant.version:
             path = os.path.join(family_path, str(variant.version))
@@ -792,9 +858,27 @@ def _create_variant(self, variant, dry_run=False, overrides=None):
 
         package_file = ".".join([package_filename, package_extension])
         filepath = os.path.join(path, package_file)
+
         with open_file_for_write(filepath) as f:
             dump_package_data(package_data, buf=f, format_=package_format)
 
+        # delete the tmp 'building' file.
+        if variant.version:
+            filename = self.building_prefix + str(variant.version)
+            filepath = os.path.join(family_path, filename)
+            if os.path.exists(filepath):
+                try:
+                    os.remove(filepath)
+                except:
+                    pass
+
+        # delete other stale building files; previous failed releases may have
+        # left some around
+        try:
+            self._delete_stale_build_tagfiles(family_path)
+        except:
+            pass
+
         # touch the family dir, this keeps memcached resolves updated properly
         os.utime(family_path, None)
 
@@ -816,6 +900,32 @@ def _create_variant(self, variant, dry_run=False, overrides=None):
             raise RezSystemError("Internal failure - expected installed variant")
         return new_variant
 
+    def _delete_stale_build_tagfiles(self, family_path):
+        now = time.time()
+
+        for name in os.listdir(family_path):
+            if not name.startswith(self.building_prefix):
+                continue
+
+            tagfilepath = os.path.join(family_path, name)
+            ver_str = name[len(self.building_prefix):]
+            pkg_path = os.path.join(family_path, ver_str)
+
+            if os.path.exists(pkg_path):
+                # build tagfile not needed if package is valid
+                if self._is_valid_package_directory(pkg_path):
+                    os.remove(tagfilepath)
+                    continue
+            else:
+                # remove tagfile if pkg is gone. Delete only tagfiles over a certain
+                # age, otherwise might delete a tagfile another process has created
+                # just before it created the package directory.
+                st = os.stat(tagfilepath)
+                age = now - st.st_mtime
+
+                if age > 3600:
+                    os.remove(tagfilepath)
+
 
 def register_plugin():
     return FileSystemPackageRepository