Compilation: Use try_execute on all major thread launchers

Tweak try_exec algo so it actually retries the wanted amount.

Author: jasugun

nimp/build.py

@@ -35,13 +35,13 @@
 import nimp.sys.process
 
 
-def _try_excecute(command, max_attemtps=3, delay=5, time_out=120):
+def _try_excecute(command, cwd='.', capture_output=True, max_attemtps=5, delay=5, time_out=120):
     ''' retry in case autoSDK or devenv cache fails us '''
-    attempt = 1
+    attempt = 0
     while attempt <= max_attemtps:
         retry = False
         start_time = time.time()
-        result, output, err = nimp.sys.process.call(command, capture_output=True)
+        result, output, err = nimp.sys.process.call(command, cwd=cwd, capture_output=True)
         time_passed = time.time() - start_time
 
         if result != 0:
@@ -50,22 +50,25 @@ def _try_excecute(command, max_attemtps=3, delay=5, time_out=120):
                 logging.error('Visual Studio appears to have failed')
                 return False
 
-            if attempt > max_attemtps:
-                logging.error('Max attempts reached, bailing...')
-                return result
             if "ERROR: Unhandled exception: System." in output and ":\\autoSDK\\HostWin64\\" in output:
                 logging.warn(f'AutoSDK error.')
                 retry = True
-            if "Package 'RoslynPackage' failed to load." in output or "Package 'Visual Studio Build Manager Package' failed to load." in output:
+            if "Package 'RoslynPackage' failed to load" in output:
+                logging.warn(f'Devenv cache error')
+                retry = True
+            if "Package 'Visual Studio Build Manager Package' failed to load" in output:
                 logging.warn(f'Devenv cache error')
                 retry = True
 
             if retry:
+                if attempt >= max_attemtps:
+                    logging.error('Max attempts reached, bailing...')
+                    return result
                 if time_passed > time_out:
                     logging.warn(f'Not retrying error that happened late in the process')
                     return result
-                logging.warn(f'Retrying : attempt {attempt} out of {max_attemtps}...')
                 attempt += 1
+                logging.warn(f'Retrying : attempt {attempt} out of {max_attemtps}...')
                 time.sleep(delay)
             else:
                 return result
@@ -111,11 +114,7 @@ def msbuild(project_file, platform_name, configuration, project=None,
     if additional_flags is not None:
         command += additional_flags
 
-    result, output, _ = nimp.sys.process.call(command, capture_output=needCapture)
-
-    if nimp.sys.platform.is_windows() and 'Cannot run if when setup is in progress.' in output:
-        logging.error('Visual Studio appears to have failed')
-        return False
+    result = _try_excecute(command, capture_output=needCapture)
 
     return result == 0
 

nimp/unreal_engine/build.py

@@ -204,28 +204,7 @@ def _unreal_generate_project(env):
     else:
         command = ['/bin/sh', './GenerateProjectFiles.sh']
 
-
-    # Inline this here - minimal change for now - for testing
-    def _try_excecute(env, command, max_attemtps=3, delay=5, time_out=60):
-        ''' retry in cause autoSDK fails us '''
-        attempt = 1
-        while attempt <= max_attemtps:
-            start_time = time.time()
-            result, output, err = nimp.sys.process.call(command, cwd=env.unreal_dir, capture_output=True)
-            time_passed = time.time() - start_time
-            # We don't want to retry long processes and block build machines, just retry quick autoSDK errors.
-            if time_passed > time_out:
-                return result
-            if result != 0 and "ERROR: Unhandled exception: System." in output and ":\\autoSDK\\HostWin64\\" in output:
-                logging.info('AutoSDK issue, retrying : attempt {attempt} out of {max_attemtps}...'.format(**locals()))
-                if attempt > max_attemtps:
-                    return result
-                attempt += 1
-                time.sleep(delay)
-            else:
-                return result
-
-    return _try_excecute(env, command)
+    return nimp.build._try_excecute(command, cwd=env.unreal_dir)
 
 
 def _unreal_build_tool_ubt(env, tool, vs_version=None):
@@ -250,7 +229,7 @@ def _unreal_run_ubt(env, target, build_platform, build_configuration, vs_version
     if flags is not None:
         command += flags
 
-    return nimp.sys.process.call(command, cwd=env.unreal_dir) == 0
+    return nimp.build._try_excecute(command, cwd=env.unreal_dir) == 0
 
 
 def _unreal_run_uat(env, target, build_platforms, flags=None):
@@ -267,7 +246,7 @@ def _unreal_run_uat(env, target, build_platforms, flags=None):
     if flags is not None:
         command += flags
 
-    return nimp.sys.process.call(command, cwd=env.unreal_dir) == 0
+    return nimp.build._try_excecute(command, cwd=env.unreal_dir) == 0
 
 
 ### Targets