Merge pull request #30 from annatisch/bug-fixes

Bug fixes

Author: davidkydd

CHANGES.txt

@@ -1,4 +1,11 @@
 
+unreleased v0.17.0
+------------------
+- Restructured dependency installation for wider compatibility:
+    get_pip.py is now packaged with the plugin.
+- Fixed unicode filename errors in thumbnail generation.
+- Fixed detection of Bifrost caches as assets.
+
 2017.10.16 v0.16.0
 ------------------
 - Asset path redirection updates

azure_batch_maya/modules/arnold_renderer.py

@@ -18,9 +18,9 @@
 from default import AzureBatchRenderJob, AzureBatchRenderAssets
 
 try:
-    str = unicode
+    str_type = unicode
 except NameError:
-    pass
+    str_type = str
 
 
 class ArnoldRenderJob(AzureBatchRenderJob):
@@ -40,7 +40,7 @@ def settings(self):
         if self.scene_name == '':
             job_name = "Untitled"
         else:
-            job_name = str(os.path.splitext(os.path.basename(self.scene_name))[0])
+            job_name = str_type(os.path.splitext(os.path.basename(self.scene_name))[0])
         file_prefix = cmds.getAttr("defaultRenderGlobals.imageFilePrefix")
         if file_prefix:
             file_prefix = os.path.split(file_prefix)[1]
@@ -59,7 +59,7 @@ def settings(self):
         self.logging = self.display_menu("Logging:   ", self.log_levels, log_level+1)
 
     def get_title(self):
-        return str(cmds.textField(self.job_name, query=True, text=True))
+        return str_type(cmds.textField(self.job_name, query=True, text=True))
 
     def render_enabled(self):
         return True
@@ -147,9 +147,12 @@ def renderer_assets(self):
 
     def setup_script(self, script_handle, pathmap, searchpaths):
         search_path = ';'.join(searchpaths).encode('utf-8')
-        script_handle.write("setAttr -type \"string\" defaultArnoldRenderOptions.procedural_searchpath \"{}\";\n".format(search_path))
-        script_handle.write("setAttr -type \"string\" defaultArnoldRenderOptions.plugin_searchpath \"{}\";\n".format(search_path))
-        script_handle.write("setAttr -type \"string\" defaultArnoldRenderOptions.texture_searchpath \"{}\";\n".format(search_path))
+        procedural_searchpath = str("setAttr -type \"string\" defaultArnoldRenderOptions.procedural_searchpath \"{}\";\n").format(search_path)
+        plugin_searchpath = str("setAttr -type \"string\" defaultArnoldRenderOptions.plugin_searchpath \"{}\";\n").format(search_path)
+        texture_searchpath = str("setAttr -type \"string\" defaultArnoldRenderOptions.texture_searchpath \"{}\";\n").format(search_path)
+        script_handle.write(procedural_searchpath)
+        script_handle.write(plugin_searchpath)
+        script_handle.write(texture_searchpath)
 
         # This kind of explicit asset re-direct is kinda ugly - so far
         # it only seems to be needed on aiImage nodes, which appear to

azure_batch_maya/plug-in/AzureBatch.py

@@ -30,7 +30,7 @@
 
 INSTALL_DIR = os.path.normpath(
     os.path.join(cmds.internalVar(userScriptDir=True), 'azure-batch-libs'))
-sys.path.append(INSTALL_DIR)
+sys.path.insert(0, INSTALL_DIR)
 
 REQUIREMENTS = {
     "pathlib==1.0.1": "pathlib",
@@ -47,7 +47,7 @@
     "azure-batch-extensions==1.0.1": "azure.batch_extensions"
 }
 
-VERSION = "0.16.0"
+VERSION = "0.17.0"
 EULA_PREF = "AzureBatch_EULA"
 SHELF_FILE = "shelf_AzureBatch.mel"
 cmd_name = "AzureBatch"
@@ -269,6 +269,7 @@ def set_environment(plugin_path):
         sys.path.append(modpath)
         sys.path.append(srcpath)
         sys.path.append(os.path.join(srcpath, "ui"))
+        sys.path.append(tolpath)
 
         script_dirs = os.environ["MAYA_SCRIPT_PATH"] + os.pathsep
         os.environ["AZUREBATCH_ICONS"] = AzureBatchSetup.clean(icnpath)
@@ -349,7 +350,7 @@ def dependency_installed(package, namespace):
         return True
 
 
-def install_pkg(package):
+def install_pkg(pip, package):
     """Install the specified package by shelling out to pip.
     :param str package: A pip-formatted package reference.
 
@@ -358,9 +359,7 @@ def install_pkg(package):
     """
     if not os.path.isdir(INSTALL_DIR):
         os.makedirs(INSTALL_DIR)
-    pip_cmds = ['mayapy', os.path.join(INSTALL_DIR, 'pip'), 
-                'install', package, 
-                '--target', INSTALL_DIR]
+    pip_cmds = ['mayapy', pip, 'install', package, '--target', INSTALL_DIR]
     print(pip_cmds)
     installer = subprocess.Popen(pip_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     installer.wait()
@@ -370,7 +369,7 @@ def install_pkg(package):
         raise RuntimeError("Failed to install package: {}".format(package))
 
 
-def install_namespace_pkg(package, namespace):
+def install_namespace_pkg(pip, package, namespace):
     """Azure packages have issues installing one by one as they don't
     unpackage correctly into the namespace directory. So we have to install
     to a temp directory and move it to the right place.
@@ -381,10 +380,7 @@ def install_namespace_pkg(package, namespace):
     temp_target = os.path.join(INSTALL_DIR, 'temp-target')
     if not os.path.isdir(temp_target):
         os.makedirs(temp_target)
-    pip_cmds = ['mayapy', os.path.join(INSTALL_DIR, 'pip'),
-                'install', package, 
-                '--no-deps',
-                '--target', temp_target]
+    pip_cmds = ['mayapy', pip, 'install', package, '--no-deps', '--target', temp_target]
     print(pip_cmds)
     installer = subprocess.Popen(pip_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     installer.wait()
@@ -447,25 +443,39 @@ def initializePlugin(obj):
 
         print("Attempting to install dependencies via Pip.")
         try:
-            install_script = os.path.normpath(os.path.join( os.environ['AZUREBATCH_TOOLS'], 'install_pip.py'))
-            installer = subprocess.Popen(["mayapy", install_script, '--target', INSTALL_DIR],
-                                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            installer.wait()
-            if installer.returncode != 0:
+            import pip
+            if StrictVersion(pip.__version__) < StrictVersion("9.0.0"):
+                print("Found out-of-date pip ({}) - attempting to upgrade.".format(pip.__version__))
+                raise ImportError
+            print("Found pip installed, version: {}".format(pip.__version__))
+            pip_location = os.path.dirname(pip.__file__)
+        except ImportError:
+            try:
+                import getpip
+                print("Running getpip command")
+                install_script = getpip.__file__
+                args = ["mayapy", install_script, "--target", INSTALL_DIR]
+                print(args)
+                installer = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                installer.wait()
                 print(installer.stdout.read())
                 print(installer.stderr.read())
-                raise RuntimeError("Failed to install pip")
-        except BaseException as exp:
-            print("Failed to install Pip. Please install dependencies manually to continue.")
-            raise
+                if installer.returncode != 0:
+                    raise RuntimeError("Failed to install pip")
+            except ImportError as e:
+                print("Unable to load getpip")
+                raise
+            else:
+                print("Getpip complete")
+                pip_location = os.path.join(INSTALL_DIR, "pip")
         try:
-            print("Installing dependencies")
+            print("Installing dependencies using {}".format(pip_location))
             for package in missing_libs:
                 if package in NAMESPACE_PACKAGES:
                     package_path = NAMESPACE_PACKAGES[package].split('.')
-                    install_namespace_pkg(package, os.path.join(*package_path))
+                    install_namespace_pkg(pip_location, package, os.path.join(*package_path))
                 else:
-                    install_pkg(package)
+                    install_pkg(pip_location, package)
             shutil.copy(os.path.join(INSTALL_DIR, 'azure', '__init__.py'), os.path.join(INSTALL_DIR, 'azure', 'mgmt', '__init__.py'))
         except:
             error = "Failed to install dependencies - please install manually"
@@ -521,6 +531,7 @@ def uninitializePlugin(obj):
 try:
     sys.path.extend(os.environ["AZUREBATCH_SCRIPTS"].split(os.pathsep))
     sys.path.append(os.environ['AZUREBATCH_MODULES'])
+    sys.path.append(os.environ['AZUREBATCH_TOOLS'])
 except KeyError as e:
     print("Couldn't find Azure Batch environment, setting up now...")
     setup_module()

azure_batch_maya/scripts/assets.py

@@ -500,6 +500,36 @@ def _get_references(self):
         self._log.debug("Found {0} references.".format(len(assets)))
         return assets
 
+    def _get_bifrost_caches(self, assets):
+        start = maya.start_frame()
+        end = maya.end_frame()
+        step = maya.frame_step()
+        caches = []
+        cache_paths = []
+        containers = maya.get_list(type="bifrostContainer")
+        for container in containers:
+            for cache_type in ['Foam', 'Guide', 'Liquid', 'LiquidMesh', 'Solid']:
+                try:
+                    enabled = maya.get_attr(container + ".enable{}Cache".format(cache_type))
+                except ValueError:
+                    continue
+                if enabled:
+                    cache_name = maya.get_attr(container + ".{}CacheFileName".format(cache_type.lower()))
+                    cache_path = maya.get_attr(container + ".{}CachePath".format(cache_type.lower()))
+                    cache_paths.append(os.path.join(cache_path, cache_name))
+                    self.pathmaps[cache_path] = utils.get_remote_file_path(cache_path)
+                    self.pathmaps[cache_paths[-1]] = utils.get_remote_file_path(cache_paths[-1])
+        cache_paths = list(set(cache_paths))
+        for cache_path in cache_paths:
+            for frame in range(start, end+step, step):
+                frame_name = "*.{}.bif".format(str(frame).zfill(4))
+                caches.extend(glob.glob(os.path.join(cache_path, "**", frame_name)))
+        caches = list(set(caches))
+        for cache in caches:
+            asset = Asset(cache, assets, self.batch, self._log)
+            if not asset.is_duplicate(assets):
+                assets.append(asset)
+
     def _get_caches(self):
         """Gather data caches as assets. TODO: This needs to be tested.
         We generally only want to gather caches relevant to the current
@@ -518,20 +548,7 @@ def _get_caches(self):
                     asset = Asset(cache_path, assets, self.batch, self._log)
                     assets.append(asset)
 
-        containers = maya.get_list(type="bifrostContainer")
-        for container in containers:
-            if not maya.get_attr(container + ".enableDiskCache"):
-                continue
-            bifrost_path = maya.get_attr(container + ".cacheDir")
-            bifrost_name = maya.get_attr(container + ".cacheName")
-            start = maya.start_frame()
-            end = maya.end_frame()
-            step = maya.frame_step()
-            for frame in range(start, end+step, step):
-                cache_file = os.path.join(bifrost_path, bifrost_name + "_p." + str(frame).zfill(4) + ".bif")
-                if os.path.exists(cache_file):
-                    asset = Asset(cache_file, assets, self.batch, self._log)
-                    assets.append(asset)
+        self._get_bifrost_caches(assets)
         self._log.debug("Found {0} caches.".format(len(assets)))
         return assets
 
@@ -769,4 +786,3 @@ def __call__(self, data, total):
         if int(progress) > self.progress:
             self.progress = int(progress)
             self.queue.put(lambda: self.command(progress))
-

azure_batch_maya/scripts/tools/generate_thumbnails.py

@@ -6,7 +6,12 @@
 import os
 import sys
 import subprocess
+import traceback
+import shutil
+import string
 
+
+printable = set(string.printable)
 SUPPORTED_FORMATS =  { ".png", ".bmp", ".jpg", ".tga", ".exr", ".jpeg" }
 
 if __name__ == '__main__':
@@ -39,27 +44,39 @@
             beauty_pass = [o for o in filtered if 'beauty' in o.lower()]
             if beauty_pass:
                 input_file = beauty_pass[0]
-        print("Using output '{}' for thumbnail generation.".format(input_file))
+        try:
+            filtered_input_file = input_file.encode('utf-8')
+        except UnicodeDecodeError:
+            temp_dir = os.path.join(cwd, 'temp_images')
+            if not os.path.isdir(temp_dir):
+                print("Creating directory for non-unicode outputs.")
+                os.makedirs(temp_dir)
+            filtered_input_file = filter(lambda x: x in printable, input_file)
+            filtered_input_file = os.path.join(temp_dir, os.path.basename(filtered_input_file))
+            shutil.copyfile(input_file, filtered_input_file)
+        print("Using output '{}' for thumbnail generation.".format(filtered_input_file))
 
         task_id = os.environ['AZ_BATCH_TASK_ID']
         output_file = os.path.join(thumb_dir, task_id + '_thumb.png')
-        commands = ['convert', input_file, '-thumbnail', '200x150', output_file]
+        commands = ['convert', filtered_input_file, '-thumbnail', '200x150', output_file]
         if os.name == 'nt':
             commands.insert(0, 'magick')
+
         print("Running imagemagick: {}".format(commands))
         conversion = subprocess.Popen(commands, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         conversion.wait()
         if conversion.returncode != 0:
             print("convert exited with code: {}".format(conversion.returncode))
             stdout, stderr = conversion.communicate()
-            print("Stdout: {}".format(stdout))
-            print("Stderr: {}".format(stderr))
+            print(stdout)
+            print(stderr)
             raise Exception("Thumbnail conversion failed.")
         if not os.path.isfile(output_file):
             raise Exception("No output file generated: {}".format(output_file))
         print("Successfully created thumbnail: {}".format(output_file))
     except Exception as exp:
-        print(exp)
+        exc_type, exc_value, exc_traceback = sys.exc_info()
+        print("".join(traceback.format_exception(exc_type, exc_value, exc_traceback)))
     finally:
         print("Exiting with code: {}".format(render_exit_code))
         sys.exit(render_exit_code)