[core] pre/post tasks : fix logs & processing issues

Alxiice · Alxiice · commit 683424aafe21 · 2026-02-19T11:33:05.000+01:00
diff --git a/bin/meshroom_compute b/bin/meshroom_compute
@@ -18,6 +18,7 @@ meshroom.setupEnvironment()
 import meshroom.core
 import meshroom.core.graph
 from meshroom.core.node import Status
+from meshroom.core.node import ChunkIndex
 
 
 parser = argparse.ArgumentParser(description='Execute a Graph of processes.')
@@ -49,11 +50,17 @@ parser.add_argument('-v', '--verbose',
                     default=os.environ.get('MESHROOM_VERBOSE', 'info'),
                     choices=['fatal', 'error', 'warning', 'info', 'debug', 'trace'])
 
-parser.add_argument('-i', '--iteration', type=int,
-                    default=-1, help='')
+parser.add_argument('-i', '--iteration', type=int, default=ChunkIndex.NONE, help='')
+parser.add_argument('--preprocess', help='Execute preprocess chunk', action='store_true')
+parser.add_argument('--postprocess', help='Execute postprocess chunk', action='store_true')
 
 args = parser.parse_args()
 
+if args.preprocess:
+    args.iteration = ChunkIndex.PREPROCESS
+elif args.postprocess:
+    args.iteration = ChunkIndex.POSTPROCESS
+
 # Setup the verbose level
 if args.extern:
     # For extern computation, we want to focus on the node computation log.
@@ -111,11 +118,16 @@ if args.node:
         print(f"InputNode: No computation to do.")
         sys.exit(0)
 
+    if args.iteration == ChunkIndex.NONE:
+        chunks = node.chunks
+    elif args.preprocess:
+        chunks = [node._preprocessChunk]
+    elif args.postprocess:
+        chunks = [node._postprocessChunk]
+    else:
+        chunks = [node.chunks[args.iteration]]
+
     if not args.forceStatus and not args.forceCompute:
-        if args.iteration != -1:
-            chunks = [node.chunks[args.iteration]]
-        else:
-            chunks = node.chunks
         for chunk in chunks:
             if chunk.status.status in submittedStatuses:
                 # Particular case for the local isolated, the node status is set to RUNNING by the submitter directly.
@@ -129,24 +141,28 @@ if args.node:
     if args.extern:
         # Restore the log level
         logging.getLogger().setLevel(meshroom.logStringToPython[args.verbose])
-
-    node.prepareLogger(args.iteration)
-    node.preprocess()
-    if args.iteration != -1:
-        chunk = node.chunks[args.iteration]
-        if chunk._status.status == Status.STOPPED:
-            print(f"Chunk {chunk}: status is STOPPED")
-            killRunningJob(node)
-        chunk.process(args.forceCompute, args.inCurrentEnv)
-    else:
+        
+    if args.iteration == ChunkIndex.NONE:
+        # Process the whole node
         if node.nodeStatus.status == Status.STOPPED:
             print(f"Node {node}: status is STOPPED")
             killRunningJob(node)
+        node.preprocess(args.forceCompute, args.inCurrentEnv)
+        node.prepareLogger(args.iteration)
         node.process(args.forceCompute, args.inCurrentEnv)
-    node.postprocess()
-    node.restoreLogger()
+        node.restoreLogger()
+        node.postprocess(args.forceCompute, args.inCurrentEnv)
+    else:
+        chunk = chunks[0]
+        if chunk._status.status == Status.STOPPED:
+            print(f"Chunk {chunk}: status is STOPPED")
+            killRunningJob(node)
+        node.prepareLogger(args.iteration)
+        chunk.process(args.forceCompute, args.inCurrentEnv)
+        node.restoreLogger()
+
 else:
-    if args.iteration != -1:
+    if args.iteration != ChunkIndex.NONE:
         print('Error: "--iteration" only makes sense when used with "--node".')
         sys.exit(-1)
     toNodes = None
diff --git a/bin/meshroom_createChunks b/bin/meshroom_createChunks
@@ -129,13 +129,13 @@ if not args.forceStatus and not args.forceCompute:
                 f"submitterSessionUid: {node._nodeStatus.submitterSessionUid}")
 
 if chunksToProcess:
-    node.prepareLogger()
     node.preprocess()
+    node.prepareLogger()
     for chunk in chunksToProcess:
         logging.info(f"[MeshroomCreateChunks] process chunk {chunk}")
         chunk.process(args.forceCompute, args.inCurrentEnv)
-    node.postprocess()
     node.restoreLogger()
+    node.postprocess()
 else:
     logging.info(f"[MeshroomCreateChunks] -> create job to process chunks {[c for c in node.chunks]}")
     submitter.createChunkTask(node, graphFile=args.graphFile, cache=args.cache, 
diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py
@@ -1,3 +1,5 @@
+# desc/node.py
+
 import enum
 from inspect import getfile
 from pathlib import Path
@@ -420,8 +422,11 @@ def processChunkInEnvironment(self, chunk):
         meshroomComputeCmd = f"{chunk.node.nodeDesc.pythonExecutable} {_MESHROOM_COMPUTE}" + \
                              f" \"{chunk.node.graph.filepath}\" --node {chunk.node.name}" + \
                               " --extern --inCurrentEnv"
-
-        if len(chunk.node.getChunks()) > 1:
+        if chunk.isPreprocess:
+            meshroomComputeCmd += f" --preprocess"
+        elif chunk.isPostprocess:
+            meshroomComputeCmd += f" --postprocess"
+        elif len(chunk.node.getChunks()) >= 1:
             meshroomComputeCmd += f" --iteration {chunk.range.iteration}"
 
         runtimeEnv = chunk.node.nodeDesc.plugin.runtimeEnv
diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py
@@ -1775,25 +1775,32 @@ def executeGraph(graph, toNodes=None, forceCompute=False, forceStatus=False):
         node.initStatusOnCompute(forceCompute)
 
     for n, node in enumerate(nodes):
+        print(f"(executeGraph) {n} {node}")
         try:
             # If the node is in compatibility mode, it cannot be computed
             if node.isCompatibilityNode:
                 logging.warning(f"{node.name} is in Compatibility Mode and cannot be computed: {node.issueDetails}.")
                 continue
 
-            node.preprocess()
+            print("A")
+            node.preprocess(forceCompute)
             if not node._chunksCreated:
+                print("B create chunks")
                 node.createChunks()
             multiChunks = len(node.chunks) > 1
+            print("C")
             for c, chunk in enumerate(node.chunks):
+                print("D", c, chunk)
                 if multiChunks:
                     print('\n[{node}/{nbNodes}]({chunk}/{nbChunks}) {nodeName}'.format(
                         node=n+1, nbNodes=len(nodes),
                         chunk=c+1, nbChunks=len(node.chunks), nodeName=node.nodeType))
                 else:
                     print(f'\n[{n + 1}/{len(nodes)}] {node.nodeType}')
+                print("E")
                 chunk.process(forceCompute)
-            node.postprocess()
+            print("F")
+            node.postprocess(forceCompute)
         except Exception as exc:
             logging.error(f"Error on node computation: {exc}")
             graph.clearSubmittedNodes()
diff --git a/meshroom/core/node.py b/meshroom/core/node.py
@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+
+# core/node.py
+
 import sys
 import atexit
 import copy
@@ -62,6 +65,7 @@ class ExecMode(Enum):
 
 
 class ChunkIndex(IntEnum):
+    NONE=-3
     PREPROCESS=-2
     POSTPROCESS=-1
     # Standard chunks are indexed from 0
@@ -404,25 +408,22 @@ def makeProgressBar(self, end, message=''):
 
             f.close()
 
-        with open(self.logFile) as f:
+        with open(self.logFile, "r") as f:
             content = f.read()
             self.progressBarPosition = content.rfind('\n')
 
-            f.close()
-
     def updateProgressBar(self, value):
         assert self.progressBar
         assert value <= self.progressEnd
 
         tics = round((value/self.progressEnd)*51)
 
-        with open(self.logFile, 'r+') as f:
+        with open(self.logFile, "r+") as f:
             text = f.read()
             for i in range(tics-self.currentProgressTics):
                 text = text[:self.progressBarPosition]+'*'+text[self.progressBarPosition:]
             f.seek(0)
             f.write(text)
-            f.close()
 
         self.currentProgressTics = tics
 
@@ -657,13 +658,6 @@ def isFinished(self):
         return self._status.status == Status.SUCCESS
 
     def process(self, forceCompute=False, inCurrentEnv=False):
-        if self.isPreprocess:
-            return self.node.nodeDesc.preprocess(self.node)
-        if self.isPostprocess:
-            return self.node.nodeDesc.postprocess(self.node)
-        return self._process(forceCompute, inCurrentEnv)
-
-    def _process(self, forceCompute=False, inCurrentEnv=False):
         if not forceCompute and self._status.status == Status.SUCCESS:
             logging.info(f"Node chunk already computed: {self.name}")
             return
@@ -685,8 +679,12 @@ def _process(self, forceCompute=False, inCurrentEnv=False):
         self.statThread.start()
 
         try:
-            logging.info(f"[Process chunk] Start processing...")
-            self.node.nodeDesc.processChunk(self)
+            if self.isPreprocess:
+                self.node.nodeDesc.preprocess(self.node)
+            elif self.isPostprocess:
+                self.node.nodeDesc.postprocess(self.node)
+            else:
+                self.node.nodeDesc.processChunk(self)
             # NOTE: this assumes saving the output attributes for each chunk
             self.node.saveOutputAttr()
             executionStatus = Status.SUCCESS
@@ -712,7 +710,6 @@ def _process(self, forceCompute=False, inCurrentEnv=False):
             self.statistics = stats.Statistics()
             del runningProcesses[self.name]
 
-
     def _processInIsolatedEnvironment(self):
         """
         Process this node chunk in the isolated environment defined in the environment
@@ -1704,42 +1701,47 @@ def initStatusOnCompute(self, forceCompute=False):
                 chunkPlaceholder._status.status = self._nodeStatus.status
                 self._chunkPlaceholder.setObjectList([chunkPlaceholder])
                 self.chunksChanged.emit()
+    
+    def getChunkName(self, iteration: int):
+        if iteration >= 0:
+            return str(self.chunks[iteration].index)
+        elif iteration == ChunkIndex.PREPROCESS:
+            return "preprocess"
+        elif iteration == ChunkIndex.POSTPROCESS:
+            return "postprocess"
+        return "0"
 
     def processIteration(self, iteration):
         self._chunks[iteration].process()
 
-    def preprocess(self):
+    def preprocess(self, forceCompute=False, inCurrentEnv=False):
         """ Prepare the node processing """
-        print("(preprocess)")
+        self.prepareLogger(ChunkIndex.PREPROCESS)
         if self.nodeDesc._hasPreprocess:
-            print("-> has preprocess")
-            # self.nodeDesc.preprocess(self)
-            self._preprocessChunk.process()
+            self._preprocessChunk.process(forceCompute, inCurrentEnv)
+        self.restoreLogger()
 
     def process(self, forceCompute=False, inCurrentEnv=False):
-        print("(process)")
         for chunk in self._chunks:
-            print("-> chunk", chunk)
             chunk.process(forceCompute, inCurrentEnv)
 
-    def postprocess(self):
+    def postprocess(self, forceCompute=False, inCurrentEnv=False):
         """
         Invoke the post process on Client Node to execute after the processing on the
         node is completed
         """
-        print("(postprocess)")
+        self.prepareLogger(ChunkIndex.POSTPROCESS)
         if self.nodeDesc._hasPostprocess:
-            print("-> has postprocess")
-            # self.nodeDesc.postprocess(self)
-            self._postprocessChunk.process()
+            self._postprocessChunk.process(forceCompute, inCurrentEnv)
+        self.restoreLogger()
 
     def getLogHandlers(self):
         return self._handlers
 
-    def prepareLogger(self, iteration=-1):
+    def prepareLogger(self, iteration=ChunkIndex.NONE):
         # Get file handler path
-        chunkIndex = self.chunks[iteration].index if iteration != -1 else 0
-        logFileName = f"{chunkIndex}.log"
+        chunkName = self.getChunkName(iteration)
+        logFileName = f"{chunkName}.log"
         logFile = os.path.join(self.internalFolder, logFileName)
         # Setup logger
         rootLogger = logging.getLogger()
@@ -2319,10 +2321,14 @@ def _resetChunks(self):
         """ Set chunks on the node.
         # TODO : Maybe don't delete chunks if we will recreate them as before ?
         """
-        if self.isInputNode:
+        if not self.isComputableType:
             self._chunksCreated = True
             return
         # Disconnect signals
+        if self._preprocessChunk:
+            self._preprocessChunk.statusChanged.disconnect(self.globalStatusChanged)
+        if self._postprocessChunk:
+            self._postprocessChunk.statusChanged.disconnect(self.globalStatusChanged)
         for chunk in self._chunks:
             chunk.statusChanged.disconnect(self.globalStatusChanged)
         # Empty list
@@ -2358,7 +2364,18 @@ def _resetChunks(self):
             self._chunksCreated = False
             self.setSize(0)
             self._chunkPlaceholder.setObjectList([NodeChunk(self, desc.computation.Range())])
-
+        # Pre/post process
+        if self.nodeDesc._hasPreprocess:
+            self._preprocessChunk = NodeChunk(self, desc.Range(ChunkIndex.PREPROCESS))
+            self._preprocessChunk.statusChanged.connect(self.globalStatusChanged)
+        else:
+            self._preprocessChunk = None
+        if self.nodeDesc._hasPostprocess:
+            self._postprocessChunk = NodeChunk(self, desc.Range(ChunkIndex.POSTPROCESS))
+            self._postprocessChunk.statusChanged.connect(self.globalStatusChanged)
+        else:
+            self._postprocessChunk = None
+        
         # Create chunks when possible
         self.chunksCreatedChanged.emit()
         self.chunksChanged.emit()
diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py
@@ -95,8 +95,9 @@ def run(self):
                 multiChunks = len(node.chunks) > 1
             except TypeError:
                 continue
+            
 
-            node.preprocess()
+            node.preprocess(self.forceCompute)
             for cId, chunk in enumerate(node.chunks):
                 if chunk.isFinishedOrRunning() or not self.isRunning():
                     continue
@@ -128,7 +129,7 @@ def run(self):
                                 # Node already removed (for instance a global clear of _nodesToProcess)
                                 pass
                             n.clearSubmittedChunks()
-            node.postprocess()
+            node.postprocess(self.forceCompute)
 
             if stopAndRestart:
                 break
diff --git a/meshroom/ui/qml/GraphEditor/ChunksListView.qml b/meshroom/ui/qml/GraphEditor/ChunksListView.qml
@@ -22,7 +22,6 @@ ColumnLayout {
     property int currentIndex: 0
 
     function getCurrentChunkIndex() {
-        console.log("(getCurrentChunkIndex)", currentIndex, chunks.length)
         if ( currentIndex == undefined || !chunks || currentIndex == ChunksListView.IndexItems.NULL ) { return -1 }
         let hasPreprocess  = chunks.some(function(chk) { return chk.chunkIndex == ChunksListView.IndexItems.PREPROCESS })
         let hasPostprocess = chunks.some(function(chk) { return chk.chunkIndex == ChunksListView.IndexItems.POSTPROCESS })
@@ -33,7 +32,7 @@ ColumnLayout {
 
     property int currentItemIndex: getCurrentChunkIndex()
 
-    property variant currentChunk: (currentItemIndex >= 0 && chunks && chunks.length > currentItemIndex) ? chunks[currentItemIndex].chunk : null
+    property variant currentChunk: (currentItemIndex >= 0 && chunks && chunks.length > currentItemIndex) ? chunks[currentItemIndex].chunk : undefined
 
     onChunksChanged: {
         // When the list changes, ensure the current index is in the new range
@@ -95,7 +94,6 @@ ColumnLayout {
             width: ListView.view.width
             leftPadding: 8
             onClicked: {
-                console.log("clicked on chunk", chunkIndex, "(", text, ")")
                 chunksLV.forceActiveFocus()
                 root.currentIndex = chunkIndex
             }
diff --git a/meshroom/ui/qml/GraphEditor/NodeLog.qml b/meshroom/ui/qml/GraphEditor/NodeLog.qml
@@ -17,10 +17,6 @@ FocusScope {
     property int currentChunkIndex
     property variant currentChunk
 
-    onCurrentChunkIndexChanged: {
-        console.log("[log] changed chunk :", currentChunkIndex, root.currentChunk.logFile)
-    }
-
     Layout.fillWidth: true
     Layout.fillHeight: true
 
diff --git a/meshroom/ui/qml/Utils/Colors.qml b/meshroom/ui/qml/Utils/Colors.qml
diff --git a/tests/test_compute.py b/tests/test_compute.py
