[submitter] Add tools to avoid autoretry on farm

Author: Alxiice

bin/meshroom_compute

@@ -3,6 +3,7 @@ import argparse
 import logging
 import os
 import sys
+from typing import NoReturn
 
 try:
     import meshroom
@@ -63,12 +64,28 @@ else:
 
 meshroom.core.initPlugins()
 meshroom.core.initNodes()
+meshroom.core.initSubmitters()
 
 graph = meshroom.core.graph.loadGraph(args.graphFile)
 if args.cache:
     graph.cacheDir = args.cache
 graph.update()
 
+
+def killRunningJob(node) -> NoReturn:
+    """ Kills current job and try to avoid job restarting """
+    jobInfos = node.nodeStatus.jobInfos
+    submitterName = jobInfos.get("submitterName")
+    if not submitterName:
+        sys.exit(meshroom.MeshroomExitStatus.ERROR_NO_RETRY)
+    from meshroom.core import submitters
+    for subName, sub in submitters.items():
+        if submitterName == subName:
+            sub.killRunningJob()
+            break
+    sys.exit(meshroom.MeshroomExitStatus.ERROR_NO_RETRY)
+
+
 if args.node:
     # Execute the node
     node = graph.findNode(args.node)
@@ -109,8 +126,14 @@ if args.node:
     node.preprocess()
     if args.iteration != -1:
         chunk = node.chunks[args.iteration]
+        if chunk._status.status == Status.STOPPED:
+            print(f"Chunk {chunk} : status is STOPPED")
+            killRunningJob(node)
         chunk.process(args.forceCompute, args.inCurrentEnv)
     else:
+        if node.nodeStatus.status == Status.STOPPED:
+            print(f"Node {node} : status is STOPPED")
+            killRunningJob(node)
         node.process(args.forceCompute, args.inCurrentEnv)
     node.postprocess()
     node.restoreLogger()

bin/meshroom_createChunks

@@ -77,19 +77,20 @@ graph.update()
 node = graph.findNode(args.node)
 submittedStatuses = [Status.RUNNING]
 
-if node._nodeStatus.status in (Status.STOPPED, Status.KILLED):
-    logging.error("Node status is STOPPED or KILLED.")
-    sys.exit(666)
-
-
-# Find job
+# Find submitter
 submitter = None
 # It's required if we want to spool chunks on different machines
 for subName, sub in submitters.items():
     if args.submitter == subName:
         submitter = sub
         break
 
+if node._nodeStatus.status in (Status.STOPPED, Status.KILLED):
+    logging.error("Node status is STOPPED or KILLED.")
+    if submitter:
+        submitter.killRunningJob()
+    sys.exit(meshroom.MeshroomExitStatus.ERROR_NO_RETRY)
+
 if not node._chunksCreated:
     # Create node chunks
     # Once created we don't have to do it again even if we relaunch the job

meshroom/__init__.py

@@ -1,5 +1,5 @@
 from distutils import util
-from enum import Enum
+from enum import Enum, IntEnum
 import logging
 import os
 import sys
@@ -72,6 +72,18 @@ def logToRoot(message, *args, **kwargs):
 logging.getLogger().setLevel(logStringToPython[os.environ.get('MESHROOM_VERBOSE', 'warning')])
 
 
+class MeshroomExitStatus(IntEnum):
+    """ In case we want to catch some special case from the parent process
+    We could use 3-125 for custom exist codes :
+    https://tldp.org/LDP/abs/html/exitcodes.html
+    """
+    SUCCESS = 0
+    ERROR = 1
+    # In some farm tools jobs are automatically re-tried, 
+    # using ERROR_NO_RETRY will try to prevent that
+    ERROR_NO_RETRY = -999  # It's actually -999 % 256 => 25
+
+
 def setupEnvironment(backend=Backend.STANDALONE):
     """
     Setup environment for Meshroom to work in a prebuilt, standalone configuration.

meshroom/core/submitter.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 
+import sys
 import logging
 import operator
 
 from enum import IntFlag, auto
 from typing import Optional
 from itertools import accumulate
 
+import meshroom
 from meshroom.common import BaseObject, Property
 
 
@@ -225,4 +227,15 @@ def submit(self, nodes, edges, filepath, submitLabel="{projectName}") -> BaseSub
             return None
         return job
 
+    @staticmethod
+    def killRunningJob():
+        """ Sometimes farms are automatically re-trying job once in case it was
+        killed by a user that don't want his machine to be used. Unfortunately this
+        means jobs will be launched twice even if they failed for a good reason.
+        This function can be used to make sure the current job will not restart
+        Note : the ERROR_NO_RETRY itself won't do anything. This function must be 
+        implemented on a case-by-case for each possible farm system
+        """
+        sys.exit(meshroom.MeshroomExitStatus.ERROR_NO_RETRY)
+
     name = Property(str, lambda self: self._name, constant=True)