From 78fddabe45030bf9528eeddd0740b98416fdaf05 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 17 Mar 2025 11:48:08 +0000 Subject: [PATCH 01/49] [ui] graph: status check simplification --- meshroom/ui/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 5a0d1b2799..1f255a2545 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -186,7 +186,7 @@ def watchedStatusFiles(self): # Only chunks that are run externally should be monitored; when run locally, status changes are already notified if c.isExtern(): # Chunks with an ERROR status may be re-submitted externally and should thus still be monitored - if c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: + if c._status.status in {Status.SUBMITTED, Status.RUNNING, Status.ERROR}: files.append(c.statusFile) chunks.append(c) return files, chunks From 8c6fca2a281e70988f6529445c2347ce7250bb03 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 17 Mar 2025 11:48:49 +0000 Subject: [PATCH 02/49] [ui] graph: loadOutputAttr is enough --- meshroom/ui/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 1f255a2545..6ef58ad809 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -204,7 +204,7 @@ def compareFilesTimes(self, times): # update chunk status if last modification time has changed since previous record if fileModTime != chunk.statusFileLastModTime: chunk.updateStatusFromCache() - chunk.node.updateOutputAttr() + chunk.node.loadOutputAttr() def onFilePollerRefreshUpdated(self): """ From 830372b326bd9f7a569f064f334ebcda8e946db2 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 17 Mar 2025 16:19:31 +0100 Subject: [PATCH 03/49] [core] saveOutputAttr directly after the processChunk --- meshroom/core/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 6e8de5f120..25009f9fc1 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -415,6 +415,8 @@ def process(self, forceCompute=False): self.statThread.start() try: self.node.nodeDesc.processChunk(self) + # NOTE: this assumes saving the output attributes for each chunk + self.node.saveOutputAttr() except Exception: if self._status.status != Status.STOPPED: exceptionStatus = Status.ERROR @@ -1086,7 +1088,6 @@ def process(self, forceCompute=False): def postprocess(self): # Invoke the post process on Client Node to execute after the processing on the node is completed self.nodeDesc.postprocess(self) - self.saveOutputAttr() def updateOutputAttr(self): if not self.nodeDesc: From 8e5f8a55d1c3c7b8658f382c9a41daa530fa981a Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 21 Mar 2025 16:46:26 +0100 Subject: [PATCH 04/49] Add some python typing --- meshroom/core/graph.py | 4 ++-- meshroom/core/node.py | 12 ++++++------ meshroom/core/taskManager.py | 7 ++++--- meshroom/ui/graph.py | 14 +++++++------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index ce39d0f3ef..15c84faf93 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -818,11 +818,11 @@ def findInitNodes(self): nodes = [n for n in self._nodes.values() if isinstance(n.nodeDesc, meshroom.core.desc.InitNode)] return nodes - def findNodeCandidates(self, nodeNameExpr): + def findNodeCandidates(self, nodeNameExpr: str) -> list[Node]: pattern = re.compile(nodeNameExpr) return [v for k, v in self._nodes.objects.items() if pattern.match(k)] - def findNode(self, nodeExpr): + def findNode(self, nodeExpr: str) -> Node: candidates = self.findNodeCandidates('^' + nodeExpr) if not candidates: raise KeyError('No node candidate for "{}"'.format(nodeExpr)) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 25009f9fc1..e2fbe03f83 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -13,7 +13,7 @@ import types import uuid from collections import namedtuple -from enum import Enum +from enum import Enum, auto from typing import Callable, Optional import meshroom @@ -55,9 +55,9 @@ class Status(Enum): class ExecMode(Enum): - NONE = 0 - LOCAL = 1 - EXTERN = 2 + NONE = auto() + LOCAL = auto() + EXTERN = auto() class StatusData(BaseObject): @@ -65,7 +65,7 @@ class StatusData(BaseObject): """ dateTimeFormatting = '%Y-%m-%d %H:%M:%S.%f' - def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', parent=None): + def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', parent: BaseObject = None): super(StatusData, self).__init__(parent) self.status = Status.NONE self.execMode = ExecMode.NONE @@ -1221,7 +1221,7 @@ def _isInputNode(self): def globalExecMode(self): return self._chunks.at(0).execModeName - def getChunks(self): + def getChunks(self) -> list[NodeChunk]: return self._chunks def getSize(self): diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py index b1d225a6c1..c8650b1919 100644 --- a/meshroom/core/taskManager.py +++ b/meshroom/core/taskManager.py @@ -4,7 +4,8 @@ import meshroom from meshroom.common import BaseObject, DictModel, Property, Signal, Slot -from meshroom.core.node import Status +from meshroom.core.node import Status, Node +from meshroom.core.graph import Graph import meshroom.core.graph @@ -96,7 +97,7 @@ class TaskManager(BaseObject): """ Manage graph - local and external - computation tasks. """ - def __init__(self, parent=None): + def __init__(self, parent: BaseObject = None): super(TaskManager, self).__init__(parent) self._graph = None self._nodes = DictModel(keyAttrName='_name', parent=self) @@ -163,7 +164,7 @@ def restart(self): self._thread = TaskThread(self) self._thread.start() - def compute(self, graph=None, toNodes=None, forceCompute=False, forceStatus=False): + def compute(self, graph: Graph = None, toNodes: list[Node] = None, forceCompute: bool = False, forceStatus: bool = False): """ Start graph computation, from root nodes to leaves - or nodes in 'toNodes' if specified. Computation tasks (NodeChunk) happen in a separate thread (see TaskThread). diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 6ef58ad809..13b8bb8edb 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -359,20 +359,20 @@ class UIGraph(QObject): UIGraph exposes undoable methods on its graph and computation in a separate thread. It also provides a monitoring of all its computation units (NodeChunks). """ - def __init__(self, undoStack, taskManager, parent=None): + def __init__(self, undoStack: commands.UndoStack, taskManager: TaskManager, parent: QObject = None): super(UIGraph, self).__init__(parent) self._undoStack = undoStack self._taskManager = taskManager - self._graph = Graph('', self) + self._graph: Graph = Graph('', self) self._modificationCount = 0 - self._chunksMonitor = ChunksMonitor(parent=self) - self._computeThread = Thread() + self._chunksMonitor: ChunksMonitor = ChunksMonitor(parent=self) + self._computeThread: Thread = Thread() self._computingLocally = self._submitted = False - self._sortedDFSChunks = QObjectListModel(parent=self) - self._layout = GraphLayout(self) + self._sortedDFSChunks: QObjectListModel = QObjectListModel(parent=self) + self._layout: GraphLayout = GraphLayout(self) self._selectedNode = None - self._nodeSelection = QItemSelectionModel(self._graph.nodes, parent=self) + self._nodeSelection: QItemSelectionModel = QItemSelectionModel(self._graph.nodes, parent=self) self._hoveredNode = None self.submitLabel = "{projectName}" From c4f64d718d940790ccb3f4ad73f1f7d955ba5aac Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 21 Mar 2025 16:47:15 +0100 Subject: [PATCH 05/49] [bin] Ensure that meshroom_compute could be launched without any environment configuration --- bin/meshroom_compute | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index a2922a88d8..d5612a5b34 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -2,7 +2,14 @@ import argparse import sys -import meshroom +try: + import meshroom +except: + # If meshroom module is not in the PYTHONPATH, add our root using the relative path + import pathlib + meshroomRootFolder = pathlib.Path(__file__).parent.parent.resolve() + sys.path.append(meshroomRootFolder) + import meshroom meshroom.setupEnvironment() import meshroom.core.graph From 0106a3b5887a7176744f2c1db91119a4ca231fa4 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 21 Mar 2025 16:49:47 +0100 Subject: [PATCH 06/49] [core] Detailed error message for plugin load failure --- meshroom/core/__init__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 43127b7cd0..eb59582e58 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -7,8 +7,8 @@ import uuid import logging import pkgutil - import sys +import traceback try: # for cx_freeze @@ -65,6 +65,7 @@ def loadPlugins(folder, packageName, classType): package = importlib.import_module(packageName) packageName = package.packageName if hasattr(package, 'packageName') else package.__name__ packageVersion = getattr(package, "__version__", None) + packagePath = os.path.dirname(package.__file__) for importer, pluginName, ispkg in pkgutil.iter_modules(package.__path__): pluginModuleName = '.' + pluginName @@ -75,7 +76,7 @@ def loadPlugins(folder, packageName, classType): if plugin.__module__ == '{}.{}'.format(package.__name__, pluginName) and issubclass(plugin, classType)] if not plugins: - logging.warning("No class defined in plugin: {}".format(pluginModuleName)) + logging.warning(f"No class defined in plugin: {pluginModuleName}") importPlugin = True for p in plugins: @@ -88,13 +89,23 @@ def loadPlugins(folder, packageName, classType): break p.packageName = packageName p.packageVersion = packageVersion + p.packagePath = packagePath if importPlugin: pluginTypes.extend(plugins) except Exception as e: - errors.append(' * {}: {}'.format(pluginName, str(e))) + tb = traceback.extract_tb(e.__traceback__) + last_call = tb[-1] + errors.append(f' * {pluginName} ({type(e).__name__}): {str(e)}\n' + # filename:lineNumber functionName + f'{last_call.filename}:{last_call.lineno} {last_call.name}\n' + # line of code with the error + f'{last_call.line}' + # Full traceback + f'\n{traceback.format_exc()}\n\n' + ) if errors: - logging.warning('== The following "{package}" plugins could not be loaded ==\n' + logging.warning(' The following "{package}" plugins could not be loaded:\n' '{errorMsg}\n' .format(package=packageName, errorMsg='\n'.join(errors))) return pluginTypes From 988da857a31341b8d2485bf3a29d8f65a7c4b154 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 21 Mar 2025 17:05:03 +0100 Subject: [PATCH 07/49] python typing --- meshroom/core/graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 15c84faf93..01181290af 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -672,7 +672,7 @@ def _createUniqueNodeName(self, inputName: str, existingNames: Optional[set[str] return newName idx += 1 - def node(self, nodeName): + def node(self, nodeName) -> Optional[Node]: return self._nodes.get(nodeName) def upgradeNode(self, nodeName) -> Node: From f665b94ca81565ec290bcb552574993614668ef4 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 21 Mar 2025 17:06:14 +0100 Subject: [PATCH 08/49] [core] NodeChunk: Init the subprocess variable --- meshroom/core/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index e2fbe03f83..b5c49cc5df 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -255,7 +255,7 @@ def __init__(self, node, range, parent=None): self._status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) self.statistics = stats.Statistics() self.statusFileLastModTime = -1 - self._subprocess = None + self.subprocess = None # Notify update in filepaths when node's internal folder changes self.node.internalFolderChanged.connect(self.nodeFolderChanged) From 66fb8134c90acdaba50006f5c15fe324e8680101 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 22:59:11 +0100 Subject: [PATCH 09/49] Remove ripple submitter --- meshroom/submitters/rippleSubmitter.py | 151 ------------------------- 1 file changed, 151 deletions(-) delete mode 100644 meshroom/submitters/rippleSubmitter.py diff --git a/meshroom/submitters/rippleSubmitter.py b/meshroom/submitters/rippleSubmitter.py deleted file mode 100644 index 3a13957571..0000000000 --- a/meshroom/submitters/rippleSubmitter.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import json - -#meshroom modules -from meshroom.core.desc import Level -from meshroom.core.submitter import BaseSubmitter - -#mpc logging import -import mpc.logging - -#Ripple imports -from mpc.ripple.rippleConfig import RippleConfig as _RippleConfig -from mpc.ripple.rippleProcess import RippleProcess -from mpc.ripple.dispatcher import DefaultDispatcher -from mpc.ripple.rippleStorage import RippleStorage -from mpc.ripple.rippleUtilities import RippleGroup -from mpc.ripple.rippleAttribute import RippleAttribute - -#validators for numbers -from mpc.pyCore.validators import IntValidator - -_log = mpc.logging.getLogger() - -currentDir = os.path.dirname(os.path.realpath(__file__)) -binDir = os.path.dirname(os.path.dirname(os.path.dirname(currentDir))) - -# Give access to min/maxProcessors, which is an alias to slots -class RippleProcessWithSlots(RippleProcess): - minProcessors = RippleAttribute('', IntValidator(), 1, True) - maxProcessors = RippleAttribute('', IntValidator(), 1, True) - -class RippleSubmitter(BaseSubmitter): - def __init__(self, parent=None): - super(RippleSubmitter, self).__init__(name='Ripple', parent=parent) - - def createTask(self, meshroomFile, node, parents): - - nbBlocks = 1 - - #Map meshroom GPU modes to MPC services - gpudict = { - "NONE":"", - "NORMAL":",cuda8G", - "INTENSIVE":",cuda16G" - } - - #Specify some constraints - requirements = "!\"rs*\",@.mem>25{gpu}".format(gpu=gpudict[node.nodeDesc.gpu.name]) - - #decide if we need multiple slots - minProcessors = 1 - maxProcessors = 1 - if Level.INTENSIVE in (node.nodeDesc.ram, node.nodeDesc.cpu): - #at least 2 slots - minProcessors = 2 - #if more than 2 are available without waiting, use 3 or 4 - maxProcessors = 4 - requirements = requirements + ",!\"rr*\"" - elif Level.NORMAL in (node.nodeDesc.ram, node.nodeDesc.cpu): - #if 2 are available, otherwise 1 - maxProcessors = 2 - requirements = requirements + ",!\"rr*\"" - - #specify which node to wait before launching the current one - waitsFor = [] - for parent in parents: - waitsFor.append(parent.name) - - #Basic command line for this node - command='meshroom_compute --node {nodeName} "{meshroomFile}" --extern'.format(nodeName=node.name, meshroomFile=meshroomFile) - - if node.isParallelized: - _, _, nbBlocks = node.nodeDesc.parallelization.getSizes(node) - - #Create as many process as iteration (or chunks) - rippleprocs = [] - for iteration in range(0, nbBlocks): - - #Add iteration number - commandext = '{cmd} --iteration {iter}'.format(cmd=command, iter=iteration) - - #Create process task with parameters - rippleproc = RippleProcessWithSlots(name='{name} iteration {iter}'.format(name=node.name, iter=iteration), discipline='ripple', appendKeys=True, keys=requirements, label=node.name, cmdList=[commandext], waitsFor=waitsFor, minProcessors=minProcessors, maxProcessors=maxProcessors) - rippleprocs.append(rippleproc) - - rippleObj = RippleGroup(label="{name} Group".format(name=node.name), tasks=rippleprocs, name=node.name, waitsFor=waitsFor) - else: - rippleObj = RippleProcessWithSlots(name=node.name, discipline='ripple', appendKeys=True, keys=requirements, label=node.name, cmdList=[command], waitsFor=waitsFor, minProcessors=minProcessors, maxProcessors=maxProcessors) - - return rippleObj - - def submit(self, nodes, edges, filepath, submitLabel="{projectName}"): - - projectName = os.path.splitext(os.path.basename(filepath))[0] - label = submitLabel.format(projectName=projectName) - - #Build a tree - tree = {} - for node in nodes: - tree[node] = [] - - for end, start in edges: - tree[end].append(start) - - nodesDone = set() - hasChange = True - tasks = [] - - #As long as a valid node was found in the previous iteration - while hasChange: - - hasChange = False - toRemove = [] - - #Loop over all nodes in the graph - for node in tree.keys(): - - #Ignore a node already processed - found = False - if node.name in nodesDone: - found = True - - if found: - continue - - #Check if all parents are already visited - valid = True - for parent in tree[node]: - found = False - if parent.name in nodesDone: - found = True - if found is False: - valid = False - - if valid is False: - continue - - tasks.append(self.createTask(filepath, node, tree[node])) - - toRemove.append(node.name) - hasChange = True - - for itemRemove in toRemove: - nodesDone.add(itemRemove) - - if (len(tasks) == 0): - return True - - DefaultDispatcher(label=label, tasks=tasks, jobType='release', paused=False)() - - return True From ad1d97f2023296d5ecca2e250fb22cc8ba8f6fad Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 23:02:26 +0100 Subject: [PATCH 10/49] [bin] meshroom_compute: verbosity - add option to control verbosity - in extern mode: disable logging and enable it only for the node computation (to avoid polluting the node's log with general warnings) --- bin/meshroom_compute | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index d5612a5b34..208152feec 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -1,5 +1,7 @@ #!/usr/bin/env python import argparse +import logging +import os import sys try: @@ -12,6 +14,7 @@ except: import meshroom meshroom.setupEnvironment() +import meshroom.core import meshroom.core.graph from meshroom.core.node import Status @@ -32,12 +35,30 @@ parser.add_argument('--extern', help='Use this option when you compute externall parser.add_argument('--cache', metavar='FOLDER', type=str, default=None, help='Override the cache folder') +parser.add_argument('-v', '--verbose', + help='Set the verbosity level for logging:\n' + ' - fatal: Show only critical errors.\n' + ' - error: Show errors only.\n' + ' - warning: Show warnings and errors.\n' + ' - info: Show standard informational messages.\n' + ' - debug: Show detailed debug information.\n' + ' - trace: Show all messages, including trace-level details.', + default=os.environ.get('MESHROOM_VERBOSE', 'info'), + choices=['fatal', 'error', 'warning', 'info', 'debug', 'trace']) parser.add_argument('-i', '--iteration', type=int, default=-1, help='') args = parser.parse_args() +# Setup the verbose level +if args.extern: + # For extern computation, we want to focus on the node computation log. + # So, we avoid polluting the log with general warning about plugins, versions of nodes in file, etc. + logging.getLogger().setLevel(level=logging.ERROR) +else: + logging.getLogger().setLevel(meshroom.logStringToPython[args.verbose]) + meshroom.core.initNodes() graph = meshroom.core.graph.loadGraph(args.graphFile) @@ -63,6 +84,10 @@ if args.node: print('Warning: Node is already submitted with status "{}". See file: "{}"'.format(chunk.status.status.name, chunk.statusFile)) # sys.exit(-1) + if args.extern: + # Restore the log level + logging.getLogger().setLevel(meshroom.logStringToPython[args.verbose]) + node.preprocess() if args.iteration != -1: chunk = node.chunks[args.iteration] From c1a862d0cd6c1d270a66eb41df96155f96964dcb Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 23:03:48 +0100 Subject: [PATCH 11/49] remove duplication for verbose options --- meshroom/ui/app.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 2bf13329be..9d51f0f01a 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -209,15 +209,7 @@ def __init__(self, inputArgs): self.debugger = QQmlDebuggingEnabler(printWarning=True) qtArgs = [f"-qmljsdebugger={debuggerParams}"] - logStringToPython = { - 'fatal': logging.FATAL, - 'error': logging.ERROR, - 'warning': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG, - 'trace': logging.DEBUG, - } - logging.getLogger().setLevel(logStringToPython[args.verbose]) + logging.getLogger().setLevel(meshroom.logStringToPython[args.verbose]) super(MeshroomApp, self).__init__(inputArgs[:1] + qtArgs) From 3f69724788f60a732ca73cdc0b4003dfebf03ecb Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 23:08:35 +0100 Subject: [PATCH 12/49] Add some typing and str format --- meshroom/core/desc/node.py | 2 +- meshroom/core/node.py | 6 +++--- meshroom/ui/reconstruction.py | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index a70053f8f0..b0b37b8033 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -116,7 +116,7 @@ def stopProcess(self, chunk): raise NotImplementedError('No stopProcess implementation on node: {}'.format(chunk.node.name)) def processChunk(self, chunk): - raise NotImplementedError('No processChunk implementation on node: "{}"'.format(chunk.node.name)) + raise NotImplementedError(f'No processChunk implementation on node: "{chunk.node.name}"') class InputNode(Node): diff --git a/meshroom/core/node.py b/meshroom/core/node.py index b5c49cc5df..2d92ea3f87 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -23,11 +23,11 @@ from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError -def getWritingFilepath(filepath): +def getWritingFilepath(filepath: str) -> str: return filepath + '.writing.' + str(uuid.uuid4()) -def renameWritingToFinalPath(writingFilepath, filepath): +def renameWritingToFinalPath(writingFilepath: str, filepath: str) -> str: if platform.system() == 'Windows': # On Windows, attempting to remove a file that is in use causes an exception to be raised. # So we may need multiple trials, if someone is reading it at the same time. @@ -1061,7 +1061,7 @@ def updateStatusFromCache(self): s = self.globalStatus for chunk in self._chunks: chunk.updateStatusFromCache() - # logging.warning("updateStatusFromCache: {}, status: {} => {}".format(self.name, s, self.globalStatus)) + # logging.warning(f"updateStatusFromCache: {self.name}, status: {s} => {self.globalStatus}") self.updateOutputAttr() def submit(self, forceCompute=False): diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index c96edfb183..93b7bb7380 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -12,10 +12,14 @@ import meshroom.core import meshroom.common + from meshroom import multiview from meshroom.common.qt import QObjectListModel from meshroom.core import Version from meshroom.core.node import Node, CompatibilityNode, Status, Position, CompatibilityIssue +from meshroom.core.taskManager import TaskManager + +from meshroom.ui import commands from meshroom.ui.graph import UIGraph from meshroom.ui.utils import makeProperty from meshroom.ui.components.filepath import FilepathHelper @@ -450,7 +454,7 @@ class Reconstruction(UIGraph): "matchProvider": ["FeatureMatching", "StructureFromMotion"] } - def __init__(self, undoStack, taskManager, defaultPipeline="", parent=None): + def __init__(self, undoStack: commands.UndoStack, taskManager: TaskManager, defaultPipeline: str="", parent: QObject=None): super(Reconstruction, self).__init__(undoStack, taskManager, parent) # initialize member variables for key steps of the 3D reconstruction pipeline From 1c9c027b0039e90ec8a2bd98f8c7bf4e622d1647 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 23:56:19 +0100 Subject: [PATCH 13/49] Use EnvVar for loading nodes, pipeline templates and submitters --- meshroom/core/__init__.py | 52 +++++++++++++++++++++++---------------- meshroom/env.py | 17 ++++++++++++- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index eb59582e58..8c2dd9770f 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,8 +19,10 @@ pass from meshroom.core.submitter import BaseSubmitter +from meshroom.env import EnvVar, meshroomFolder from . import desc + # Setup logging logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) @@ -55,7 +57,6 @@ def add_to_path(p): def loadPlugins(folder, packageName, classType): """ """ - pluginTypes = [] errors = [] @@ -296,7 +297,7 @@ def registerNodeType(nodeType): """ global nodesDesc if nodeType.__name__ in nodesDesc: - logging.error("Node Desc {} is already registered.".format(nodeType.__name__)) + logging.error(f"Node Desc {nodeType.__name__} is already registered.") nodesDesc[nodeType.__name__] = nodeType @@ -308,7 +309,11 @@ def unregisterNodeType(nodeType): def loadNodes(folder, packageName): - return loadPlugins(folder, packageName, desc.Node) + if not os.path.isdir(folder): + logging.error("Node folder '{folder}' does not exist.") + return + + return loadPlugins(folder, packageName, desc.BaseNode) def loadAllNodes(folder): @@ -318,51 +323,56 @@ def loadAllNodes(folder): nodeTypes = loadNodes(folder, package) for nodeType in nodeTypes: registerNodeType(nodeType) - logging.debug('Nodes loaded [{}]: {}'.format(package, ', '.join([nodeType.__name__ for nodeType in nodeTypes]))) + nodesStr = ', '.join([nodeType.__name__ for nodeType in nodeTypes]) + logging.debug(f'Nodes loaded [{package}]: {nodesStr}') def registerSubmitter(s): global submitters if s.name in submitters: - logging.error("Submitter {} is already registered.".format(s.name)) + logging.error(f"Submitter {s.name} is already registered.") submitters[s.name] = s def loadSubmitters(folder, packageName): + if not os.path.isdir(folder): + logging.error(f"Submitters folder '{folder}' does not exist.") + return + return loadPlugins(folder, packageName, BaseSubmitter) def loadPipelineTemplates(folder): global pipelineTemplates + if not os.path.isdir(folder): + logging.error(f"Pipeline templates folder '{folder}' does not exist.") + return for file in os.listdir(folder): if file.endswith(".mg") and file not in pipelineTemplates: pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file) def initNodes(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) - additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep) - # filter empty strings - additionalNodesPath = [i for i in additionalNodesPath if i] + additionalNodesPath = EnvVar.getList(EnvVar.MESHROOM_NODES_PATH) nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath for f in nodesFolders: loadAllNodes(folder=f) def initSubmitters(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) - subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters') - for sub in subs: - registerSubmitter(sub()) + additionalPaths = EnvVar.getList(EnvVar.MESHROOM_SUBMITTERS_PATH) + allSubmittersFolders = [meshroomFolder] + additionalPaths + for folder in allSubmittersFolders: + subs = loadSubmitters(folder, 'submitters') + for sub in subs: + registerSubmitter(sub()) def initPipelines(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) - # Load pipeline templates: check in any folder the user might have added to the environment variable - pipelinesPath = os.environ.get("MESHROOM_PIPELINE_TEMPLATES_PATH", "").split(os.pathsep) - pipelineTemplatesFolders = [i for i in pipelinesPath if i] + # Load pipeline templates: check in the default folder and any folder the user might have + # added to the environment variable + additionalPipelinesPath = EnvVar.getList(EnvVar.MESHROOM_PIPELINE_TEMPLATES_PATH) + pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath for f in pipelineTemplatesFolders: - if os.path.isdir(f): - loadPipelineTemplates(f) - else: - logging.warning("Pipeline templates folder '{}' does not exist.".format(f)) + loadPipelineTemplates(f) + diff --git a/meshroom/env.py b/meshroom/env.py index 3036db206f..6e94e5f873 100644 --- a/meshroom/env.py +++ b/meshroom/env.py @@ -15,6 +15,8 @@ from typing import Any, Type +meshroomFolder = os.path.dirname(__file__) + @dataclass class VarDefinition: """Environment variable definition.""" @@ -39,18 +41,31 @@ class EnvVar(Enum): str, "port:3768", "QML debugging params as expected by -qmljsdebugger" ) + # Core + MESHROOM_PLUGINS_PATH = VarDefinition(str, "", "Paths to plugins folders") + MESHROOM_NODES_PATH = VarDefinition(str, "", "Paths to set of nodes folders") + MESHROOM_SUBMITTERS_PATH = VarDefinition(str, "", "Paths to set of submitters folders") + MESHROOM_PIPELINE_TEMPLATES_PATH = VarDefinition(str, "", "Paths to pipeline templates folders") + @staticmethod def get(envVar: "EnvVar") -> Any: """Get the value of `envVar`, cast to the variable type.""" value = os.environ.get(envVar.name, envVar.value.default) return EnvVar._cast(value, envVar.value.valueType) + @staticmethod + def getList(envVar: "EnvVar") -> list[Any]: + """Get the value of `envVar` as a list of non-empty strings.""" + paths = EnvVar.get(envVar).split(os.pathsep) + # filter empty values + return [p for p in paths if p] + @staticmethod def _cast(value: str, valueType: Type) -> Any: if valueType is str: return value elif valueType is bool: - return value.lower() in {"true", "1", "on"} + return value.lower() in {"true", "1", "on", "yes", "y"} return valueType(value) @classmethod From 0c961a5b684773a08c907fd5e7ad0fb0cfbb4c1e Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 23 Mar 2025 23:57:54 +0100 Subject: [PATCH 14/49] [ui] call loadOutputAttr only once (and not per chunk) --- meshroom/ui/graph.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 13b8bb8edb..aa29400cdc 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -200,11 +200,14 @@ def compareFilesTimes(self, times): times: the last modification times for currently monitored files. """ newRecords = dict(zip(self.monitoredChunks, times)) + hasChanges = False for chunk, fileModTime in newRecords.items(): # update chunk status if last modification time has changed since previous record if fileModTime != chunk.statusFileLastModTime: chunk.updateStatusFromCache() - chunk.node.loadOutputAttr() + hasChanges = True + if hasChanges: + chunk.node.loadOutputAttr() def onFilePollerRefreshUpdated(self): """ From 1ad526d627a1df1c6c2aa50d54b0e78ceb6eb7dd Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 24 Mar 2025 00:00:11 +0100 Subject: [PATCH 15/49] [core] Use exist_ok on makedirs --- meshroom/core/node.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 2d92ea3f87..62ad1777da 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -335,10 +335,7 @@ def saveStatusFile(self): data = self._status.toDict() statusFilepath = self.statusFile folder = os.path.dirname(statusFilepath) - try: - os.makedirs(folder) - except Exception: - pass + os.makedirs(folder, exist_ok=True) statusFilepathWriting = getWritingFilepath(statusFilepath) with open(statusFilepathWriting, 'w') as jsonFile: @@ -376,8 +373,7 @@ def saveStatistics(self): data = self.statistics.toDict() statisticsFilepath = self.statisticsFile folder = os.path.dirname(statisticsFilepath) - if not os.path.exists(folder): - os.makedirs(folder) + os.makedirs(folder, exist_ok=True) statisticsFilepathWriting = getWritingFilepath(statisticsFilepath) with open(statisticsFilepathWriting, 'w') as jsonFile: json.dump(data, jsonFile, indent=4) From faece7efca73036ba5d0475d92553780342b5b1a Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 24 Mar 2025 00:00:56 +0100 Subject: [PATCH 16/49] minor wording --- meshroom/core/desc/attribute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/desc/attribute.py b/meshroom/core/desc/attribute.py index 4970ff873c..434d39e731 100644 --- a/meshroom/core/desc/attribute.py +++ b/meshroom/core/desc/attribute.py @@ -80,7 +80,7 @@ def matchDescription(self, value, strict=True): # This property only makes sense for output attributes. isExpression = Property(bool, lambda self: self._isExpression, constant=True) # isDynamicValue - # The default value of the attribute's descriptor is None, so it's not an input value, + # The default value of the attribute's descriptor is None, so it is not an input value, # but an output value that is computed during the Node's process execution. isDynamicValue = Property(bool, lambda self: self._isDynamicValue, constant=True) group = Property(str, lambda self: self._group, constant=True) From 727a4d129b653e1f7fc53236eb97224738061211 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 24 Mar 2025 00:03:45 +0100 Subject: [PATCH 17/49] New notion of local isolated computation for python nodes using meshroom_compute Reoganization - BaseNode: is the base class for all nodes - Node: is now dedicated to python nodes, with the implentation directly in the process function - CommandLineNode: dedicated to generate and run external command lines --- bin/meshroom_compute | 14 ++- meshroom/core/desc/__init__.py | 29 +---- meshroom/core/desc/node.py | 198 ++++++++++++++++++++++++--------- meshroom/core/node.py | 183 ++++++++++++++++++++---------- meshroom/core/nodeFactory.py | 3 +- meshroom/ui/graph.py | 13 ++- 6 files changed, 288 insertions(+), 152 deletions(-) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index 208152feec..04b03d3e9f 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -16,7 +16,7 @@ meshroom.setupEnvironment() import meshroom.core import meshroom.core.graph -from meshroom.core.node import Status +from meshroom.core.node import Status, ExecMode parser = argparse.ArgumentParser(description='Execute a Graph of processes.') @@ -26,6 +26,8 @@ parser.add_argument('--node', metavar='NODE_NAME', type=str, help='Process the node. It will generate an error if the dependencies are not already computed.') parser.add_argument('--toNode', metavar='NODE_NAME', type=str, help='Process the node with its dependencies.') +parser.add_argument('--inCurrentEnv', help='Execute process in current env without creating a dedicated runtime environment.', + action='store_true') parser.add_argument('--forceStatus', help='Force computation if status is RUNNING or SUBMITTED.', action='store_true') parser.add_argument('--forceCompute', help='Compute in all cases even if already computed.', @@ -81,7 +83,11 @@ if args.node: chunks = node.chunks for chunk in chunks: if chunk.status.status in submittedStatuses: - print('Warning: Node is already submitted with status "{}". See file: "{}"'.format(chunk.status.status.name, chunk.statusFile)) + # Particular case for the LOCAL_ISOLATED, the node status is set to RUNNING by the submitter directly. + # We ensure that no other instance has started to compute, by checking that the sessionUid is empty. + if chunk.status.execMode == ExecMode.LOCAL_ISOLATED and not chunk.status.sessionUid and chunk.status.submitterSessionUid: + continue + print(f'Warning: Node is already submitted with status "{chunk.status.status.name}". See file: "{chunk.statusFile}". ExecMode: {chunk.status.execMode.name}, SessionUid: {chunk.status.sessionUid}, submitterSessionUid: {chunk.status.submitterSessionUid}') # sys.exit(-1) if args.extern: @@ -91,9 +97,9 @@ if args.node: node.preprocess() if args.iteration != -1: chunk = node.chunks[args.iteration] - chunk.process(args.forceCompute) + chunk.process(args.forceCompute, args.inCurrentEnv) else: - node.process(args.forceCompute) + node.process(args.forceCompute, args.inCurrentEnv) node.postprocess() else: if args.iteration != -1: diff --git a/meshroom/core/desc/__init__.py b/meshroom/core/desc/__init__.py index b60a9d5164..16bab71158 100644 --- a/meshroom/core/desc/__init__.py +++ b/meshroom/core/desc/__init__.py @@ -21,36 +21,9 @@ ) from .node import ( AVCommandLineNode, + BaseNode, CommandLineNode, InitNode, InputNode, Node, ) - -__all__ = [ - # attribute - "Attribute", - "BoolParam", - "ChoiceParam", - "ColorParam", - "File", - "FloatParam", - "GroupAttribute", - "IntParam", - "ListAttribute", - "PushButtonParam", - "StringParam", - # computation - "DynamicNodeSize", - "Level", - "MultiDynamicNodeSize", - "Parallelization", - "Range", - "StaticNodeSize", - # node - "AVCommandLineNode", - "CommandLineNode", - "InitNode", - "InputNode", - "Node", -] diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index b0b37b8033..07dc5377c4 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -1,16 +1,37 @@ from inspect import getfile from pathlib import Path +import logging import os import psutil import shlex +import shutil +import sys from .computation import Level, StaticNodeSize from .attribute import StringParam, ColorParam +import meshroom from meshroom.core import cgroup -class Node(object): +_MESHROOM_ROOT = Path(meshroom.__file__).parent.parent +_MESHROOM_COMPUTE = _MESHROOM_ROOT / "bin" / "meshroom_compute" + + +def isNodeSaved(node): + """Returns whether a node is identical to its serialized counterpart in the current graph file.""" + filepath = node.graph.filepath + if not filepath: + return False + + from meshroom.core.graph import loadGraph + graphSaved = loadGraph(filepath) + nodeSaved = graphSaved.node(node.name) + if nodeSaved is None: + return False + return nodeSaved._uid == node._uid + +class BaseNode(object): """ """ cpu = Level.NORMAL @@ -62,7 +83,7 @@ class Node(object): category = 'Other' def __init__(self): - super(Node, self).__init__() + super(BaseNode, self).__init__() self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs) self.sourceCodeFolder = Path(getfile(self.__class__)).parent.resolve().as_posix() @@ -113,13 +134,102 @@ def postprocess(self, node): pass def stopProcess(self, chunk): - raise NotImplementedError('No stopProcess implementation on node: {}'.format(chunk.node.name)) + logging.warning(f'No stopProcess implementation on node: {chunk.node.name}') def processChunk(self, chunk): raise NotImplementedError(f'No processChunk implementation on node: "{chunk.node.name}"') + def executeChunkCommandLine(self, chunk, cmd, env=None): + try: + with open(chunk.logFile, 'w') as logF: + chunk.status.commandLine = cmd + chunk.saveStatusFile() + cmdList = shlex.split(cmd) + # Resolve executable to full path + prog = shutil.which(cmdList[0], path=env.get('PATH') if env else None) + + print(f"Starting Process for '{chunk.node.name}'") + print(f' - commandLine: {cmd}') + print(f' - logFile: {chunk.logFile}') + if prog: + cmdList[0] = prog + print(f' - command full path: {prog}') + + # Change the process group to avoid Meshroom main process being killed if the subprocess + # gets terminated by the user or an Out Of Memory (OOM kill). + if sys.platform == "win32": + platformArgs = {"creationflags": psutil.CREATE_NEW_PROCESS_GROUP} + # Note: DETACHED_PROCESS means fully detached process. + # We don't want a fully detached process to ensure that if Meshroom is killed, + # the subprocesses are killed too. + else: + platformArgs = {"start_new_session": True} + # Note: "preexec_fn"=os.setsid is the old way before python-3.2 + + chunk.subprocess = psutil.Popen( + cmdList, + stdout=logF, + stderr=logF, + cwd=chunk.node.internalFolder, + env=env, + **platformArgs, + ) + + if hasattr(chunk, "statThread"): + # We only have a statThread if the node is running in the current process + # and not in a dedicated environment/process. + chunk.statThread.proc = chunk.subprocess + + stdout, stderr = chunk.subprocess.communicate() + + chunk.status.returnCode = chunk.subprocess.returncode + + if chunk.subprocess.returncode and chunk.subprocess.returncode < 0: + signal_num = -chunk.subprocess.returncode + logF.write(f"Process was killed by signal: {signal_num}") + try: + status = chunk.subprocess.status() + logF.write(f"Process status: {status}") + except: + pass + + if chunk.subprocess.returncode != 0: + with open(chunk.logFile, 'r') as logF: + logContent = ''.join(logF.readlines()) + raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent)) + finally: + chunk.subprocess = None + + def stopProcess(self, chunk): + # The same node could exists several times in the graph and + # only one would have the running subprocess; ignore all others + if not chunk.subprocess: + print(f"[{chunk.node.name}] stopProcess: no subprocess") + return + + # Retrieve process tree + processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess] + logging.debug(f"[{chunk.node.name}] Processes to stop: {len(processes)}") + for process in processes: + try: + # With terminate, the process has a chance to handle cleanup + process.terminate() + except psutil.NoSuchProcess: + pass + + # If it is still running, force kill it + for process in processes: + try: + # Use is_running() instead of poll() as we use a psutil.Process object + if process.is_running(): # Check if process is still alive + process.kill() # Forcefully kill it + except psutil.NoSuchProcess: + logging.info(f"[{chunk.node.name}] Process already terminated.") + except psutil.AccessDenied: + logging.info(f"[{chunk.node.name}] Permission denied to kill the process.") + -class InputNode(Node): +class InputNode(BaseNode): """ Node that does not need to be processed, it is just a placeholder for inputs. """ @@ -130,7 +240,24 @@ def processChunk(self, chunk): pass -class CommandLineNode(Node): +class Node(BaseNode): + + def __init__(self): + super(Node, self).__init__() + + def processChunkInEnvironment(self, chunk): + if not isNodeSaved(chunk.node): + raise RuntimeError("File must be saved before computing in isolated environment.") + + meshroomComputeCmd = f"python {_MESHROOM_COMPUTE} {chunk.node.graph.filepath} --node {chunk.node.name} --extern --inCurrentEnv" + if len(chunk.node.getChunks()) > 1: + meshroomComputeCmd += f" --iteration {chunk.range.iteration}" + + runtimeEnv = None + self.executeChunkCommandLine(chunk, meshroomComputeCmd, env=runtimeEnv) + + +class CommandLineNode(BaseNode): """ """ commandLine = '' # need to be defined on the node @@ -143,14 +270,14 @@ def __init__(self): def buildCommandLine(self, chunk): cmdPrefix = '' - # If rez available in env, we use it - if "REZ_ENV" in os.environ and chunk.node.packageVersion: - # If the node package is already in the environment, we don't need a new dedicated rez environment - alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()), - "").startswith(chunk.node.packageVersion) - if not alreadyInEnv: - cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"), - packageFullName=chunk.node.packageFullName) + # # If rez available in env, we use it + # if "REZ_ENV" in os.environ and chunk.node.packageVersion: + # # If the node package is already in the environment, we don't need a new dedicated rez environment + # alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()), + # "").startswith(chunk.node.packageVersion) + # if not alreadyInEnv: + # cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"), + # packageFullName=chunk.node.packageFullName) cmdSuffix = '' if chunk.node.isParallelized and chunk.node.size > 1: @@ -158,48 +285,10 @@ def buildCommandLine(self, chunk): return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix - def stopProcess(self, chunk): - # The same node could exists several times in the graph and - # only one would have the running subprocess; ignore all others - if not hasattr(chunk, "subprocess"): - return - if chunk.subprocess: - # Kill process tree - processes = chunk.subprocess.children(recursive=True) + [chunk.subprocess] - try: - for process in processes: - process.terminate() - except psutil.NoSuchProcess: - pass - def processChunk(self, chunk): - try: - with open(chunk.logFile, 'w') as logF: - cmd = self.buildCommandLine(chunk) - chunk.status.commandLine = cmd - chunk.saveStatusFile() - print(' - commandLine: {}'.format(cmd)) - print(' - logFile: {}'.format(chunk.logFile)) - chunk.subprocess = psutil.Popen(shlex.split(cmd), stdout=logF, stderr=logF, cwd=chunk.node.internalFolder) - - # Store process static info into the status file - # chunk.status.env = node.proc.environ() - # chunk.status.createTime = node.proc.create_time() - - chunk.statThread.proc = chunk.subprocess - stdout, stderr = chunk.subprocess.communicate() - chunk.subprocess.wait() - - chunk.status.returnCode = chunk.subprocess.returncode - - if chunk.subprocess.returncode != 0: - with open(chunk.logFile, 'r') as logF: - logContent = ''.join(logF.readlines()) - raise RuntimeError('Error on node "{}":\nLog:\n{}'.format(chunk.name, logContent)) - except Exception: - raise - finally: - chunk.subprocess = None + cmd = self.buildCommandLine(chunk) + # TODO: Setup runtime env + self.executeChunkCommandLine(chunk, cmd) # Specific command line node for AliceVision apps @@ -282,3 +371,4 @@ def setAttributes(self, node, attributesDict): for attr in attributesDict: if node.hasAttribute(attr): node.attribute(attr).value = attributesDict[attr] + diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 62ad1777da..bfc7a0c13f 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -57,6 +57,7 @@ class Status(Enum): class ExecMode(Enum): NONE = auto() LOCAL = auto() + LOCAL_ISOLATED = auto() EXTERN = auto() @@ -67,20 +68,13 @@ class StatusData(BaseObject): def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', parent: BaseObject = None): super(StatusData, self).__init__(parent) - self.status = Status.NONE - self.execMode = ExecMode.NONE - self.nodeName = nodeName - self.nodeType = nodeType - self.packageName = packageName - self.packageVersion = packageVersion - self.graph = '' - self.commandLine = None - self.env = None - self.startDateTime = "" - self.endDateTime = "" - self.elapsedTime = 0 - self.hostname = "" - self.sessionUid = meshroom.core.sessionUid + self.reset() + self.nodeName: str = nodeName + self.nodeType: str = nodeType + self.packageName: str = packageName + self.packageVersion: str = packageVersion + self.sessionUid: Optional[str] = meshroom.core.sessionUid + self.submitterSessionUid: Optional[str] = None def merge(self, other): self.startDateTime = min(self.startDateTime, other.startDateTime) @@ -88,27 +82,44 @@ def merge(self, other): self.elapsedTime += other.elapsedTime def reset(self): - self.status = Status.NONE - self.execMode = ExecMode.NONE - self.graph = '' - self.commandLine = None - self.env = None - self.startDateTime = "" - self.endDateTime = "" - self.elapsedTime = 0 - self.hostname = "" - self.sessionUid = meshroom.core.sessionUid + self.nodeName: str = "" + self.nodeType: str = "" + self.packageName: str = "" + self.packageVersion: str = "" + self.resetDynamicValues() + + def resetDynamicValues(self): + self.status: Status = Status.NONE + self.execMode: ExecMode = ExecMode.NONE + self.graph = "" + self.commandLine: str = "" + self.env: str = "" + self._startTime: Optional[datetime.datetime] = None + self.startDateTime: str = "" + self.endDateTime: str = "" + self.elapsedTime: float = 0.0 + self.hostname: str = "" def initStartCompute(self): import platform self.sessionUid = meshroom.core.sessionUid self.hostname = platform.node() + self._startTime = time.time() self.startDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting) # to get datetime obj: datetime.datetime.strptime(obj, self.dateTimeFormatting) + def initSubmit(self): + ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. + ''' + self.resetDynamicValues() + self.sessionUid = None + self.submitterSessionUid = meshroom.core.sessionUid + def initEndCompute(self): self.sessionUid = meshroom.core.sessionUid self.endDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting) + if self._startTime != None: + self.elapsedTime = time.time() - self._startTime @property def elapsedTimeStr(self): @@ -118,9 +129,12 @@ def toDict(self): d = self.__dict__.copy() d["elapsedTimeStr"] = self.elapsedTimeStr - # Skip non data attributes from BaseObject + # Skip some attributes (some are from BaseObject) d.pop("destroyed", None) d.pop("objectNameChanged", None) + d.pop("_parent", None) + d.pop("_startTime", None) + return d def fromDict(self, d): @@ -142,8 +156,9 @@ def fromDict(self, d): self.elapsedTime = d.get('elapsedTime', 0) self.hostname = d.get('hostname', '') self.sessionUid = d.get('sessionUid', '') + self.submitterSessionUid = d.get('submitterSessionUid', '') - + class LogManager: dateTimeFormatting = '%H:%M:%S' @@ -251,9 +266,9 @@ def __init__(self, node, range, parent=None): super(NodeChunk, self).__init__(parent) self.node = node self.range = range - self.logManager = LogManager(self) - self._status = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) - self.statistics = stats.Statistics() + self.logManager: LogManager = LogManager(self) + self._status: StatusData = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) + self.statistics: stats.Statistics = stats.Statistics() self.statusFileLastModTime = -1 self.subprocess = None # Notify update in filepaths when node's internal folder changes @@ -298,6 +313,7 @@ def updateStatusFromCache(self): try: with open(statusFile, 'r') as jsonFile: statusData = json.load(jsonFile) + # logging.debug(f"updateStatusFromCache({self.node.name}): From status {self.status.status} to {statusData['status']}") self.status.fromDict(statusData) self.statusFileLastModTime = os.path.getmtime(statusFile) except Exception: @@ -343,12 +359,9 @@ def saveStatusFile(self): renameWritingToFinalPath(statusFilepathWriting, statusFilepath) def upgradeStatusTo(self, newStatus, execMode=None): - if newStatus.value <= self._status.status.value: - logging.warning("Downgrade status on node '{}' from {} to {}". - format(self.name, self._status.status, newStatus)) + if newStatus.value < self._status.status.value: + logging.warning(f"Downgrade status on node '{self.name}' from {self._status.status} to {newStatus}") - if newStatus == Status.SUBMITTED: - self._status = StatusData(self.node.name, self.node.nodeType, self.node.packageName, self.node.packageVersion) if execMode is not None: self._status.execMode = execMode self.execModeNameChanged.emit() @@ -397,15 +410,22 @@ def isStopped(self): def isFinished(self): return self._status.status == Status.SUCCESS - def process(self, forceCompute=False): + def process(self, forceCompute=False, inCurrentEnv=False): if not forceCompute and self._status.status == Status.SUCCESS: logging.info("Node chunk already computed: {}".format(self.name)) return + + # Start the process environment for nodes running in isolation. + # This only happens once, when the node has the SUBMITTED status. + # The sub-process will go through this method again, but the node status will have been set to RUNNING. + if not inCurrentEnv and isinstance(self.node.nodeDesc, desc.Node): + self._processInIsolatedEnvironment() + return + global runningProcesses runningProcesses[self.name] = self self._status.initStartCompute() - exceptionStatus = None - startTime = time.time() + executionStatus = None self.upgradeStatusTo(Status.RUNNING) self.statThread = stats.StatisticsThread(self) self.statThread.start() @@ -413,18 +433,18 @@ def process(self, forceCompute=False): self.node.nodeDesc.processChunk(self) # NOTE: this assumes saving the output attributes for each chunk self.node.saveOutputAttr() + executionStatus = Status.SUCCESS except Exception: if self._status.status != Status.STOPPED: - exceptionStatus = Status.ERROR + executionStatus = Status.ERROR raise except (KeyboardInterrupt, SystemError, GeneratorExit): - exceptionStatus = Status.STOPPED + executionStatus = Status.STOPPED raise finally: self._status.initEndCompute() - self._status.elapsedTime = time.time() - startTime - if exceptionStatus is not None: - self.upgradeStatusTo(exceptionStatus) + if executionStatus: + self.upgradeStatusTo(executionStatus) logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) # Ask and wait for the stats thread to stop self.statThread.stopRequest() @@ -432,19 +452,43 @@ def process(self, forceCompute=False): self.statistics = stats.Statistics() del runningProcesses[self.name] - self.upgradeStatusTo(Status.SUCCESS) + + def _processInIsolatedEnvironment(self): + """Process this node chunk in the isolated environment defined in the environment configuration.""" + try: + self._status.initSubmit() + self.upgradeStatusTo(Status.RUNNING, execMode=ExecMode.LOCAL_ISOLATED) + self.node.nodeDesc.processChunkInEnvironment(self) + except: + # status should be already updated by meshroom_compute + self.updateStatusFromCache() + if self._status.status != Status.ERROR: + # If meshroom_compute has crashed or been killed, the status may have not been set to ERROR. + # In this particular case, we enforce it from here. + self.upgradeStatusTo(Status.ERROR) + raise + # Update the chunk status. + self.updateStatusFromCache() + # Update the output attributes, as any chunk may have modified them. + self.node.updateOutputAttr() def stopProcess(self): - if not self.isExtern(): - if self._status.status == Status.RUNNING: - self.upgradeStatusTo(Status.STOPPED) - elif self._status.status == Status.SUBMITTED: - self.upgradeStatusTo(Status.NONE) + if self.isExtern(): + return + if self._status.status != Status.RUNNING: + return + + self.upgradeStatusTo(Status.STOPPED) self.node.nodeDesc.stopProcess(self) def isExtern(self): - return self._status.execMode == ExecMode.EXTERN or ( - self._status.execMode == ExecMode.LOCAL and self._status.sessionUid != meshroom.core.sessionUid) + """ The computation is managed externally by another instance of Meshroom, or by meshroom_compute on renderfarm). + In the ambiguous case of an isolated environment, it is considered as local as we can stop it. + """ + if self._status.execMode == ExecMode.LOCAL_ISOLATED: + # It is a local isolated node, check if it is submitted by our current session. + return self._status.submitterSessionUid != meshroom.core.sessionUid + return self._status.sessionUid != meshroom.core.sessionUid statusChanged = Signal() status = Property(Variant, lambda self: self._status, notify=statusChanged) @@ -845,7 +889,13 @@ def clearData(self): Status will be reset to Status.NONE """ if self.internalFolder and os.path.exists(self.internalFolder): - shutil.rmtree(self.internalFolder) + try: + shutil.rmtree(self.internalFolder) + except Exception as e: + # We could get some "Device or resource busy" on .nfs file while removing the folder on linux network. + # On windows, some output files may be open for visualization and the removal will fail. + # On both cases, we can ignore it. + logging.warning(f"Failed to remove internal folder: '{self.internalFolder}'. Error: {e}.") self.updateStatusFromCache() @Slot(result=str) @@ -1063,6 +1113,7 @@ def updateStatusFromCache(self): def submit(self, forceCompute=False): for chunk in self._chunks: if forceCompute or chunk.status.status != Status.SUCCESS: + chunk._status.initSubmit() chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.EXTERN) def beginSequence(self, forceCompute=False): @@ -1077,9 +1128,9 @@ def preprocess(self): # Invoke the Node Description's pre-process for the Client Node to prepare its processing self.nodeDesc.preprocess(self) - def process(self, forceCompute=False): + def process(self, forceCompute=False, inCurrentEnv=False): for chunk in self._chunks: - chunk.process(forceCompute) + chunk.process(forceCompute, inCurrentEnv) def postprocess(self): # Invoke the post process on Client Node to execute after the processing on the node is completed @@ -1090,8 +1141,8 @@ def updateOutputAttr(self): return if not self.nodeDesc.hasDynamicOutputAttribute: return - # logging.warning("updateOutputAttr: {}, status: {}".format(self.name, self.globalStatus)) - if self.getGlobalStatus() == Status.SUCCESS: + # logging.warning(f"updateOutputAttr: {self.name}, status: {self.globalStatus}") + if Status.SUCCESS in [c._status.status for c in self.getChunks()]: self.loadOutputAttr() else: self.resetOutputAttr() @@ -1339,19 +1390,33 @@ def statusInThisSession(self): return False return True + def submitterStatusInThisSession(self): + if not self._chunks: + return False + for chunk in self._chunks: + if chunk.status.submitterSessionUid != meshroom.core.sessionUid: + return False + return True + @Slot(result=bool) def canBeStopped(self): # Only locked nodes running in local with the same # sessionUid as the Meshroom instance can be stopped - return (self.locked and self.getGlobalStatus() == Status.RUNNING and - self.globalExecMode == "LOCAL" and self.statusInThisSession()) + # logging.warning(f"[{self.name}] canBeStopped: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") + return (self.getGlobalStatus() == Status.RUNNING and + ((self.globalExecMode == ExecMode.LOCAL.name and self.statusInThisSession()) or + (self.globalExecMode == ExecMode.LOCAL_ISOLATED.name and self.submitterStatusInThisSession()) + )) @Slot(result=bool) def canBeCanceled(self): # Only locked nodes submitted in local with the same # sessionUid as the Meshroom instance can be canceled - return (self.locked and self.getGlobalStatus() == Status.SUBMITTED and - self.globalExecMode == "LOCAL" and self.statusInThisSession()) + # logging.warning(f"[{self.name}] canBeCanceled: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") + return (self.getGlobalStatus() == Status.SUBMITTED and + ((self.globalExecMode == ExecMode.LOCAL.name and self.statusInThisSession()) or + (self.globalExecMode == ExecMode.LOCAL_ISOLATED.name and self.submitterStatusInThisSession()) + )) def hasImageOutputAttribute(self): """ diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index 9322e54a10..a33cd3377c 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -168,6 +168,7 @@ def _checkAttributesCompatibility( def _createNode(self) -> Node: logging.info(f"Creating node '{self.name}'") + # TODO: user inputs/outputs may conflicts with internal names (like position, uid) return Node( self.nodeType, position=self.position, @@ -187,7 +188,7 @@ def _tryUpgradeCompatibilityNode(self, node: CompatibilityNode) -> Union[Node, C """Handle possible upgrades of CompatibilityNodes, when no computed data is associated to the Node.""" if node.issue == CompatibilityIssue.UnknownNodeType: return node - + # Nodes in templates are not meant to hold computation data. if self.inTemplate: logging.warning(f"Compatibility issue in template: performing automatic upgrade on '{self.name}'") diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index aa29400cdc..77e036274d 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -183,10 +183,11 @@ def watchedStatusFiles(self): return self.statusFiles, self.monitorableChunks elif self.filePollerRefresh is PollerRefreshStatus.MINIMAL_ENABLED.value: for c in self.monitorableChunks: - # Only chunks that are run externally should be monitored; when run locally, status changes are already notified - if c.isExtern(): - # Chunks with an ERROR status may be re-submitted externally and should thus still be monitored - if c._status.status in {Status.SUBMITTED, Status.RUNNING, Status.ERROR}: + # Only chunks that are run externally or local_isolated should be monitored, + # when run locally, status changes are already notified. + # Chunks with an ERROR status may be re-submitted externally and should thus still be monitored + if (c.isExtern() and c._status.status in (Status.SUBMITTED, Status.RUNNING, Status.ERROR)) or ( + (c._status.execMode is ExecMode.LOCAL_ISOLATED) and (c._status.status in (Status.SUBMITTED, Status.RUNNING))): files.append(c.statusFile) chunks.append(c) return files, chunks @@ -582,8 +583,8 @@ def submit(self, nodes: Optional[Union[list[Node], Node]] = None): def updateGraphComputingStatus(self): # update graph computing status computingLocally = any([ - (ch.status.execMode == ExecMode.LOCAL and - ch.status.sessionUid == sessionUid and + (((ch.status.execMode == ExecMode.LOCAL and ch.status.sessionUid == sessionUid) or + ch.status.execMode == ExecMode.LOCAL_ISOLATED) and ch.status.status in (Status.RUNNING, Status.SUBMITTED)) for ch in self._sortedDFSChunks]) submitted = any([ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks]) From 426855baa6ef421506e384c5a25f7df68e744c84 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 24 Mar 2025 18:12:27 +0100 Subject: [PATCH 18/49] [ui] ensure all node types used in the UI are declared Avoid qml errors when accessing non-existing nodes. The entry exist in the dict and we retrieve a null node. --- meshroom/ui/reconstruction.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 93b7bb7380..2843fd689b 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -453,6 +453,16 @@ class Reconstruction(UIGraph): # Nodes that can be used to provide matches folders to the UI "matchProvider": ["FeatureMatching", "StructureFromMotion"] } + # Nodes accessed from the UI + uiNodes = [ + "LdrToHdrMerge", + "LdrToHdrCalibration", + "ImageProcessing", + "PhotometricStereo", + "PanoramaInit", + "ColorCheckerDetection", + "SphereDetection", + ] def __init__(self, undoStack: commands.UndoStack, taskManager: TaskManager, defaultPipeline: str="", parent: QObject=None): super(Reconstruction, self).__init__(undoStack, taskManager, parent) @@ -516,7 +526,11 @@ def initActiveNodes(self): # Create all possible entries for category, _ in self.activeNodeCategories.items(): self._activeNodes.add(ActiveNode(category, parent=self)) - for nodeType, _ in meshroom.core.nodesDesc.items(): + # For all nodes declared to be accessed by the UI + usedNodeTypes = {j for i in self.activeNodeCategories.values() for j in i} + allUiNodes = set(self.uiNodes) | usedNodeTypes + allLoadedNodeTypes = set(meshroom.core.nodesDesc.keys()) + for nodeType in allUiNodes: self._activeNodes.add(ActiveNode(nodeType, parent=self)) def clearActiveNodes(self): From eb9df4c90068526a8ddbd09c6fcd2ecb2bb49f60 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 25 Mar 2025 12:28:22 +0100 Subject: [PATCH 19/49] Explicit meshroom node type in status file Avoid an ambiguous LOCAL_ISOLATED, as a process can be extern and using an isolated execution environement. --- bin/meshroom_compute | 4 +-- meshroom/core/__init__.py | 11 ++++--- meshroom/core/desc/__init__.py | 1 + meshroom/core/desc/node.py | 21 +++++++++++- meshroom/core/node.py | 58 +++++++++++++++++++++++----------- meshroom/ui/graph.py | 14 ++++---- 6 files changed, 76 insertions(+), 33 deletions(-) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index 04b03d3e9f..777552047b 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -83,9 +83,9 @@ if args.node: chunks = node.chunks for chunk in chunks: if chunk.status.status in submittedStatuses: - # Particular case for the LOCAL_ISOLATED, the node status is set to RUNNING by the submitter directly. + # Particular case for the local isolated, the node status is set to RUNNING by the submitter directly. # We ensure that no other instance has started to compute, by checking that the sessionUid is empty. - if chunk.status.execMode == ExecMode.LOCAL_ISOLATED and not chunk.status.sessionUid and chunk.status.submitterSessionUid: + if chunk.status.mrNodeType == meshroom.core.MrNodeType.NODE and not chunk.status.sessionUid and chunk.status.submitterSessionUid: continue print(f'Warning: Node is already submitted with status "{chunk.status.status.name}". See file: "{chunk.statusFile}". ExecMode: {chunk.status.execMode.name}, SessionUid: {chunk.status.sessionUid}, submitterSessionUid: {chunk.status.submitterSessionUid}') # sys.exit(-1) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 8c2dd9770f..85b98becbe 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -1,14 +1,14 @@ -import hashlib from contextlib import contextmanager +import hashlib import importlib import inspect -import os -import tempfile -import uuid import logging +import os import pkgutil import sys +import tempfile import traceback +import uuid try: # for cx_freeze @@ -21,7 +21,7 @@ from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc - +from .desc import MrNodeType # Setup logging logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) @@ -36,6 +36,7 @@ pipelineTemplates = {} + def hashValue(value): """ Hash 'value' using sha1. """ hashObject = hashlib.sha1(str(value).encode('utf-8')) diff --git a/meshroom/core/desc/__init__.py b/meshroom/core/desc/__init__.py index 16bab71158..8623b28051 100644 --- a/meshroom/core/desc/__init__.py +++ b/meshroom/core/desc/__init__.py @@ -20,6 +20,7 @@ StaticNodeSize, ) from .node import ( + MrNodeType, AVCommandLineNode, BaseNode, CommandLineNode, diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index 07dc5377c4..378e01fdf4 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -1,3 +1,4 @@ +import enum from inspect import getfile from pathlib import Path import logging @@ -13,11 +14,17 @@ import meshroom from meshroom.core import cgroup - _MESHROOM_ROOT = Path(meshroom.__file__).parent.parent _MESHROOM_COMPUTE = _MESHROOM_ROOT / "bin" / "meshroom_compute" +class MrNodeType(enum.Enum): + NONE = enum.auto() + BASENODE = enum.auto() + NODE = enum.auto() + COMMANDLINE = enum.auto() + INPUT = enum.auto() + def isNodeSaved(node): """Returns whether a node is identical to its serialized counterpart in the current graph file.""" filepath = node.graph.filepath @@ -87,6 +94,9 @@ def __init__(self): self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs) self.sourceCodeFolder = Path(getfile(self.__class__)).parent.resolve().as_posix() + def getMrNodeType(self): + return MrNodeType.BASENODE + def upgradeAttributeValues(self, attrValues, fromVersion): return attrValues @@ -236,6 +246,9 @@ class InputNode(BaseNode): def __init__(self): super(InputNode, self).__init__() + def getMrNodeType(self): + return MrNodeType.INPUT + def processChunk(self, chunk): pass @@ -245,6 +258,9 @@ class Node(BaseNode): def __init__(self): super(Node, self).__init__() + def getMrNodeType(self): + return MrNodeType.NODE + def processChunkInEnvironment(self, chunk): if not isNodeSaved(chunk.node): raise RuntimeError("File must be saved before computing in isolated environment.") @@ -267,6 +283,9 @@ class CommandLineNode(BaseNode): def __init__(self): super(CommandLineNode, self).__init__() + def getMrNodeType(self): + return MrNodeType.COMMANDLINE + def buildCommandLine(self, chunk): cmdPrefix = '' diff --git a/meshroom/core/node.py b/meshroom/core/node.py index bfc7a0c13f..1f997e3c31 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -57,7 +57,6 @@ class Status(Enum): class ExecMode(Enum): NONE = auto() LOCAL = auto() - LOCAL_ISOLATED = auto() EXTERN = auto() @@ -86,6 +85,7 @@ def reset(self): self.nodeType: str = "" self.packageName: str = "" self.packageVersion: str = "" + self.mrNodeType: desc.MrNodeType = desc.MrNodeType.NONE self.resetDynamicValues() def resetDynamicValues(self): @@ -108,6 +108,14 @@ def initStartCompute(self): self.startDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting) # to get datetime obj: datetime.datetime.strptime(obj, self.dateTimeFormatting) + def initIsolatedCompute(self): + ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. + ''' + self.resetDynamicValues() + self.mrNodeType = desc.MrNodeType.NODE + self.sessionUid = None + self.submitterSessionUid = meshroom.core.sessionUid + def initSubmit(self): ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. ''' @@ -144,6 +152,10 @@ def fromDict(self, d): self.execMode = d.get('execMode', ExecMode.NONE) if not isinstance(self.execMode, ExecMode): self.execMode = ExecMode[self.execMode] + self.mrNodeType = d.get('mrNodeType', desc.MrNodeType.NONE) + if not isinstance(self.mrNodeType, desc.MrNodeType): + self.mrNodeType = desc.MrNodeType[self.mrNodeType] + self.nodeName = d.get('nodeName', '') self.nodeType = d.get('nodeType', '') self.packageName = d.get('packageName', '') @@ -456,8 +468,8 @@ def process(self, forceCompute=False, inCurrentEnv=False): def _processInIsolatedEnvironment(self): """Process this node chunk in the isolated environment defined in the environment configuration.""" try: - self._status.initSubmit() - self.upgradeStatusTo(Status.RUNNING, execMode=ExecMode.LOCAL_ISOLATED) + self._status.initIsolatedCompute() + self.upgradeStatusTo(Status.RUNNING, execMode=ExecMode.LOCAL) self.node.nodeDesc.processChunkInEnvironment(self) except: # status should be already updated by meshroom_compute @@ -473,22 +485,25 @@ def _processInIsolatedEnvironment(self): self.node.updateOutputAttr() def stopProcess(self): - if self.isExtern(): - return - if self._status.status != Status.RUNNING: - return + # if self.isExtern() or self._status.status != Status.RUNNING: + # return self.upgradeStatusTo(Status.STOPPED) self.node.nodeDesc.stopProcess(self) def isExtern(self): - """ The computation is managed externally by another instance of Meshroom, or by meshroom_compute on renderfarm). + """ The computation is managed externally by another instance of Meshroom. In the ambiguous case of an isolated environment, it is considered as local as we can stop it. """ - if self._status.execMode == ExecMode.LOCAL_ISOLATED: - # It is a local isolated node, check if it is submitted by our current session. - return self._status.submitterSessionUid != meshroom.core.sessionUid - return self._status.sessionUid != meshroom.core.sessionUid + uid = self._status.submitterSessionUid if self._status.mrNodeType == desc.MrNodeType.NODE else meshroom.core.sessionUid + return uid != meshroom.core.sessionUid + + # def isIndependantProcess(self): + # if self._status.execMode == ExecMode.EXTERN: + # # Compute is managed by another instance of Meshroom + # return True + # # Compute is using a meshroom_compute subprocess + # return self._status.mrNodeType == desc.MrNodeType.NODE statusChanged = Signal() status = Property(Variant, lambda self: self._status, notify=statusChanged) @@ -1397,6 +1412,15 @@ def submitterStatusInThisSession(self): if chunk.status.submitterSessionUid != meshroom.core.sessionUid: return False return True + + def initFromThisSession(self): + if not self._chunks: + return False + for chunk in self._chunks: + uid = chunk.status.submitterSessionUid if chunk.status.mrNodeType == desc.MrNodeType.NODE else chunk.status.sessionUid + if uid != meshroom.core.sessionUid: + return False + return True @Slot(result=bool) def canBeStopped(self): @@ -1404,9 +1428,8 @@ def canBeStopped(self): # sessionUid as the Meshroom instance can be stopped # logging.warning(f"[{self.name}] canBeStopped: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") return (self.getGlobalStatus() == Status.RUNNING and - ((self.globalExecMode == ExecMode.LOCAL.name and self.statusInThisSession()) or - (self.globalExecMode == ExecMode.LOCAL_ISOLATED.name and self.submitterStatusInThisSession()) - )) + self.globalExecMode == ExecMode.LOCAL.name and + self.initFromThisSession()) @Slot(result=bool) def canBeCanceled(self): @@ -1414,9 +1437,8 @@ def canBeCanceled(self): # sessionUid as the Meshroom instance can be canceled # logging.warning(f"[{self.name}] canBeCanceled: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") return (self.getGlobalStatus() == Status.SUBMITTED and - ((self.globalExecMode == ExecMode.LOCAL.name and self.statusInThisSession()) or - (self.globalExecMode == ExecMode.LOCAL_ISOLATED.name and self.submitterStatusInThisSession()) - )) + self.globalExecMode == ExecMode.LOCAL.name and + self.initFromThisSession()) def hasImageOutputAttribute(self): """ diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 77e036274d..85bb3c2a93 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -30,7 +30,7 @@ from meshroom.core.taskManager import TaskManager from meshroom.core.node import NodeChunk, Node, Status, ExecMode, CompatibilityNode, Position -from meshroom.core import submitters +from meshroom.core import submitters, MrNodeType from meshroom.ui import commands from meshroom.ui.utils import makeProperty @@ -187,7 +187,7 @@ def watchedStatusFiles(self): # when run locally, status changes are already notified. # Chunks with an ERROR status may be re-submitted externally and should thus still be monitored if (c.isExtern() and c._status.status in (Status.SUBMITTED, Status.RUNNING, Status.ERROR)) or ( - (c._status.execMode is ExecMode.LOCAL_ISOLATED) and (c._status.status in (Status.SUBMITTED, Status.RUNNING))): + (c._status.mrNodeType == MrNodeType.NODE) and (c._status.status in (Status.SUBMITTED, Status.RUNNING))): files.append(c.statusFile) chunks.append(c) return files, chunks @@ -201,13 +201,14 @@ def compareFilesTimes(self, times): times: the last modification times for currently monitored files. """ newRecords = dict(zip(self.monitoredChunks, times)) - hasChanges = False + hasChangesAndSuccess = False for chunk, fileModTime in newRecords.items(): # update chunk status if last modification time has changed since previous record if fileModTime != chunk.statusFileLastModTime: chunk.updateStatusFromCache() - hasChanges = True - if hasChanges: + if chunk.status.status == Status.SUCCESS: + hasChangesAndSuccess = True + if hasChangesAndSuccess: chunk.node.loadOutputAttr() def onFilePollerRefreshUpdated(self): @@ -583,8 +584,7 @@ def submit(self, nodes: Optional[Union[list[Node], Node]] = None): def updateGraphComputingStatus(self): # update graph computing status computingLocally = any([ - (((ch.status.execMode == ExecMode.LOCAL and ch.status.sessionUid == sessionUid) or - ch.status.execMode == ExecMode.LOCAL_ISOLATED) and + ((ch.status.submitterSessionUid if ch.status.mrNodeType == MrNodeType.NODE else ch.status.sessionUid) == sessionUid) and ( ch.status.status in (Status.RUNNING, Status.SUBMITTED)) for ch in self._sortedDFSChunks]) submitted = any([ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks]) From 4c7ff6eb1a2d2a485391b6ad01f72fdcc31c4da5 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 26 Mar 2025 11:29:09 +0000 Subject: [PATCH 20/49] [core] remove duplicated function from BaseNode --- meshroom/core/desc/node.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index 378e01fdf4..2cbec1110d 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -143,9 +143,6 @@ def postprocess(self, node): """ pass - def stopProcess(self, chunk): - logging.warning(f'No stopProcess implementation on node: {chunk.node.name}') - def processChunk(self, chunk): raise NotImplementedError(f'No processChunk implementation on node: "{chunk.node.name}"') @@ -214,7 +211,7 @@ def stopProcess(self, chunk): # The same node could exists several times in the graph and # only one would have the running subprocess; ignore all others if not chunk.subprocess: - print(f"[{chunk.node.name}] stopProcess: no subprocess") + logging.warning(f"[{chunk.node.name}] stopProcess: no subprocess") return # Retrieve process tree From b3c4f675a87e47b7e13e33ae23326bf29829c336 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 26 Mar 2025 11:32:35 +0000 Subject: [PATCH 21/49] rename "typing" to avoid conflicts --- meshroom/core/graph.py | 4 ++-- meshroom/core/{typing.py => mtyping.py} | 0 meshroom/core/taskManager.py | 2 +- meshroom/ui/commands.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename meshroom/core/{typing.py => mtyping.py} (100%) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 01181290af..11cb1965eb 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -20,7 +20,7 @@ from meshroom.core.graphIO import GraphIO, GraphSerializer, TemplateGraphSerializer, PartialGraphSerializer from meshroom.core.node import BaseNode, Status, Node, CompatibilityNode from meshroom.core.nodeFactory import nodeFactory -from meshroom.core.typing import PathLike +from meshroom.core.mtyping import PathLike # Replace default encoder to support Enums @@ -1613,7 +1613,7 @@ def executeGraph(graph, toNodes=None, forceCompute=False, forceStatus=False): chunk.process(forceCompute) node.postprocess() except Exception as e: - logging.error("Error on node computation: {}".format(e)) + logging.error(f"Error on node computation: {e}") graph.clearSubmittedNodes() raise diff --git a/meshroom/core/typing.py b/meshroom/core/mtyping.py similarity index 100% rename from meshroom/core/typing.py rename to meshroom/core/mtyping.py diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py index c8650b1919..f7de0ad3d5 100644 --- a/meshroom/core/taskManager.py +++ b/meshroom/core/taskManager.py @@ -70,7 +70,7 @@ def run(self): stopAndRestart = True break else: - logging.error("Error on node computation: {}".format(e)) + logging.error(f"Error on node computation: {e}.") nodesToRemove, _ = self._manager._graph.dfsOnDiscover(startNodes=[node], reverse=True) # remove following nodes from the task queue for n in nodesToRemove[1:]: # exclude current node diff --git a/meshroom/ui/commands.py b/meshroom/ui/commands.py index d1e8d8bf3e..424fad1e09 100755 --- a/meshroom/ui/commands.py +++ b/meshroom/ui/commands.py @@ -9,7 +9,7 @@ from meshroom.core.graph import Graph, GraphModification from meshroom.core.node import Position, CompatibilityIssue from meshroom.core.nodeFactory import nodeFactory -from meshroom.core.typing import PathLike +from meshroom.core.mtyping import PathLike class UndoCommand(QUndoCommand): From 1ca83fc6a9a59c6cee7a04e3ed4c0ba6e5155a0b Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 26 Mar 2025 11:33:13 +0000 Subject: [PATCH 22/49] [ui] Check if node types are available before using them --- meshroom/ui/reconstruction.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 2843fd689b..ae3053fb9b 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -215,8 +215,10 @@ def __init__(self, viewpointAttribute, reconstruction): # trigger internal members updates when reconstruction members changes self._reconstruction.cameraInitChanged.connect(self._updateInitialParams) self._reconstruction.sfmReportChanged.connect(self._updateSfMParams) - self._activeNode_PrepareDenseScene.nodeChanged.connect(self._updateUndistortedImageParams) - self._activeNode_ExportAnimatedCamera.nodeChanged.connect(self._updateUndistortedImageParams) + if self._activeNode_PrepareDenseScene: + self._activeNode_PrepareDenseScene.nodeChanged.connect(self._updateUndistortedImageParams) + if self._activeNode_ExportAnimatedCamera: + self._activeNode_ExportAnimatedCamera.nodeChanged.connect(self._updateUndistortedImageParams) def _updateInitialParams(self): """ Update internal members depending on CameraInit. """ @@ -366,7 +368,7 @@ def fieldOfView(self): if not self.solvedIntrinsics: return None focalLength = self.solvedIntrinsics["focalLength"] - + #We assume that if the width is less than the weight #It's because the image has been rotated and not #because the sensor has some unusual shape @@ -379,13 +381,13 @@ def fieldOfView(self): return 2.0 * math.atan(float(sensorWidth) / (2.0 * float(focalLength))) * 180.0 / math.pi else: return 2.0 * math.atan(float(sensorHeight) / (2.0 * float(focalLength))) * 180.0 / math.pi - + @Property(type=float, notify=sfmParamsChanged) def pixelAspectRatio(self): """ Get camera pixel aspect ratio. """ if not self.solvedIntrinsics: return 1.0 - + return float(self.solvedIntrinsics["pixelRatio"]) @Property(type=QUrl, notify=undistortedImageParamsChanged) @@ -887,7 +889,7 @@ def handleFilesUrl(self, filesByType, cameraInit=None, position=None): "Unknown file extensions: " + ', '.join(extensions) ) ) - + # As the boolean is introduced to check if the project is loaded or not, the return value is added to the function. # The default value is False, which means the project is not loaded. return False @@ -1095,7 +1097,9 @@ def setActiveNode(self, node, categories=True, inputs=True): # Set the new active node (if it is not an unknown type) unknownType = isinstance(node, CompatibilityNode) and node.issue == CompatibilityIssue.UnknownNodeType if not unknownType: - self.activeNodes.get(node.nodeType).node = node + activeNode = self.activeNodes.get(node.nodeType) + if activeNode: + activeNode.node = node @Slot(QObject) def setActiveNodes(self, nodes): @@ -1313,4 +1317,4 @@ def setCurrentViewPath(self, path): # Signals to propagate high-level messages error = Signal(Message) warning = Signal(Message) - info = Signal(Message) \ No newline at end of file + info = Signal(Message) From 2f08448310c871448f2694dfbb8c6113888c86e1 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Wed, 26 Mar 2025 11:36:24 +0000 Subject: [PATCH 23/49] Rely on the nodeDesc MrNodeType --- bin/meshroom_compute | 2 +- meshroom/core/node.py | 82 +++++++++++++++++++++++++++---------------- meshroom/ui/graph.py | 4 +-- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index 777552047b..59d8b647e2 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -85,7 +85,7 @@ if args.node: if chunk.status.status in submittedStatuses: # Particular case for the local isolated, the node status is set to RUNNING by the submitter directly. # We ensure that no other instance has started to compute, by checking that the sessionUid is empty. - if chunk.status.mrNodeType == meshroom.core.MrNodeType.NODE and not chunk.status.sessionUid and chunk.status.submitterSessionUid: + if chunk.node.getMrNodeType() == meshroom.core.MrNodeType.NODE and not chunk.status.sessionUid and chunk.status.submitterSessionUid: continue print(f'Warning: Node is already submitted with status "{chunk.status.status.name}". See file: "{chunk.statusFile}". ExecMode: {chunk.status.execMode.name}, SessionUid: {chunk.status.sessionUid}, submitterSessionUid: {chunk.status.submitterSessionUid}') # sys.exit(-1) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 1f997e3c31..a65ae3f4f9 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -16,9 +16,10 @@ from enum import Enum, auto from typing import Callable, Optional + import meshroom from meshroom.common import Signal, Variant, Property, BaseObject, Slot, ListModel, DictModel -from meshroom.core import desc, stats, hashValue, nodeVersion, Version +from meshroom.core import desc, stats, hashValue, nodeVersion, Version, MrNodeType from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError @@ -65,13 +66,14 @@ class StatusData(BaseObject): """ dateTimeFormatting = '%Y-%m-%d %H:%M:%S.%f' - def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', parent: BaseObject = None): + def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', mrNodeType: MrNodeType = MrNodeType.NONE, parent: BaseObject = None): super(StatusData, self).__init__(parent) self.reset() self.nodeName: str = nodeName self.nodeType: str = nodeType self.packageName: str = packageName self.packageVersion: str = packageVersion + self.mrNodeType = mrNodeType self.sessionUid: Optional[str] = meshroom.core.sessionUid self.submitterSessionUid: Optional[str] = None @@ -85,7 +87,7 @@ def reset(self): self.nodeType: str = "" self.packageName: str = "" self.packageVersion: str = "" - self.mrNodeType: desc.MrNodeType = desc.MrNodeType.NONE + self.mrNodeType: MrNodeType = MrNodeType.NONE self.resetDynamicValues() def resetDynamicValues(self): @@ -112,16 +114,27 @@ def initIsolatedCompute(self): ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. ''' self.resetDynamicValues() - self.mrNodeType = desc.MrNodeType.NODE + # assert(self.mrNodeType == MrNodeType.NODE) + self.sessionUid = None + self.submitterSessionUid = meshroom.core.sessionUid + + def initExternSubmit(self): + ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. + ''' + self.resetDynamicValues() self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid + self.status = Status.SUBMITTED + self.execMode = ExecMode.EXTERN - def initSubmit(self): + def initLocalSubmit(self): ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. ''' self.resetDynamicValues() self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid + self.status = Status.SUBMITTED + self.execMode = ExecMode.LOCAL def initEndCompute(self): self.sessionUid = meshroom.core.sessionUid @@ -152,9 +165,9 @@ def fromDict(self, d): self.execMode = d.get('execMode', ExecMode.NONE) if not isinstance(self.execMode, ExecMode): self.execMode = ExecMode[self.execMode] - self.mrNodeType = d.get('mrNodeType', desc.MrNodeType.NONE) - if not isinstance(self.mrNodeType, desc.MrNodeType): - self.mrNodeType = desc.MrNodeType[self.mrNodeType] + self.mrNodeType = d.get('mrNodeType', MrNodeType.NONE) + if not isinstance(self.mrNodeType, MrNodeType): + self.mrNodeType = MrNodeType[self.mrNodeType] self.nodeName = d.get('nodeName', '') self.nodeType = d.get('nodeType', '') @@ -279,7 +292,7 @@ def __init__(self, node, range, parent=None): self.node = node self.range = range self.logManager: LogManager = LogManager(self) - self._status: StatusData = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion) + self._status: StatusData = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion, node.getMrNodeType()) self.statistics: stats.Statistics = stats.Statistics() self.statusFileLastModTime = -1 self.subprocess = None @@ -430,7 +443,7 @@ def process(self, forceCompute=False, inCurrentEnv=False): # Start the process environment for nodes running in isolation. # This only happens once, when the node has the SUBMITTED status. # The sub-process will go through this method again, but the node status will have been set to RUNNING. - if not inCurrentEnv and isinstance(self.node.nodeDesc, desc.Node): + if not inCurrentEnv and self.node.getMrNodeType() == MrNodeType.NODE: self._processInIsolatedEnvironment() return @@ -447,6 +460,7 @@ def process(self, forceCompute=False, inCurrentEnv=False): self.node.saveOutputAttr() executionStatus = Status.SUCCESS except Exception: + self.updateStatusFromCache() # check if the status has been updated by another process if self._status.status != Status.STOPPED: executionStatus = Status.ERROR raise @@ -474,7 +488,7 @@ def _processInIsolatedEnvironment(self): except: # status should be already updated by meshroom_compute self.updateStatusFromCache() - if self._status.status != Status.ERROR: + if self._status.status not in (Status.ERROR, Status.STOPPED, Status.KILLED): # If meshroom_compute has crashed or been killed, the status may have not been set to ERROR. # In this particular case, we enforce it from here. self.upgradeStatusTo(Status.ERROR) @@ -485,26 +499,24 @@ def _processInIsolatedEnvironment(self): self.node.updateOutputAttr() def stopProcess(self): - # if self.isExtern() or self._status.status != Status.RUNNING: - # return + if self.isExtern(): + raise ValueError("Cannot stop process: node is computed externally (another instance of Meshroom)") + if self._status.status != Status.RUNNING: + raise ValueError(f"Cannot stop process: node is not running (status is: {self._status.status}).") - self.upgradeStatusTo(Status.STOPPED) self.node.nodeDesc.stopProcess(self) + # Update the status to get latest information before changing it + self.updateStatusFromCache() + self.upgradeStatusTo(Status.STOPPED) + def isExtern(self): """ The computation is managed externally by another instance of Meshroom. In the ambiguous case of an isolated environment, it is considered as local as we can stop it. """ - uid = self._status.submitterSessionUid if self._status.mrNodeType == desc.MrNodeType.NODE else meshroom.core.sessionUid + uid = self._status.submitterSessionUid if self.node.getMrNodeType() == MrNodeType.NODE else meshroom.core.sessionUid return uid != meshroom.core.sessionUid - # def isIndependantProcess(self): - # if self._status.execMode == ExecMode.EXTERN: - # # Compute is managed by another instance of Meshroom - # return True - # # Compute is using a meshroom_compute subprocess - # return self._status.mrNodeType == desc.MrNodeType.NODE - statusChanged = Signal() status = Property(Variant, lambda self: self._status, notify=statusChanged) statusName = Property(str, statusName.fget, notify=statusChanged) @@ -588,6 +600,11 @@ def __getattr__(self, k): except KeyError: raise e + def getMrNodeType(self): + if self.isCompatibilityNode: + return MrNodeType.NONE + return self.nodeDesc.getMrNodeType() + def getName(self): return self._name @@ -1128,12 +1145,13 @@ def updateStatusFromCache(self): def submit(self, forceCompute=False): for chunk in self._chunks: if forceCompute or chunk.status.status != Status.SUCCESS: - chunk._status.initSubmit() + chunk._status.initExternSubmit() chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.EXTERN) def beginSequence(self, forceCompute=False): for chunk in self._chunks: if forceCompute or (chunk.status.status not in (Status.RUNNING, Status.SUCCESS)): + chunk._status.initLocalSubmit() chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.LOCAL) def processIteration(self, iteration): @@ -1239,6 +1257,8 @@ def getGlobalStatus(self): return Status.INPUT if not self._chunks: return Status.NONE + if len( self._chunks) == 1: + return self._chunks[0].status.status chunksStatus = [chunk.status.status for chunk in self._chunks] @@ -1257,11 +1277,14 @@ def getGlobalStatus(self): @Slot(result=StatusData) def getFusedStatus(self): + if not self._chunks: + return StatusData() + if len(self._chunks) == 1: + return self._chunks[0].status fusedStatus = StatusData() - if self._chunks: - fusedStatus.fromDict(self._chunks[0].status.toDict()) - for chunk in self._chunks[1:]: - fusedStatus.merge(chunk.status) + fusedStatus.fromDict(self._chunks[0].status.toDict()) + for chunk in self._chunks[1:]: + fusedStatus.merge(chunk.status) fusedStatus.status = self.getGlobalStatus() return fusedStatus @@ -1417,7 +1440,8 @@ def initFromThisSession(self): if not self._chunks: return False for chunk in self._chunks: - uid = chunk.status.submitterSessionUid if chunk.status.mrNodeType == desc.MrNodeType.NODE else chunk.status.sessionUid + mrNodeType = chunk.node.getMrNodeType() + uid = chunk.status.submitterSessionUid if mrNodeType == MrNodeType.NODE else chunk.status.sessionUid if uid != meshroom.core.sessionUid: return False return True @@ -1426,7 +1450,6 @@ def initFromThisSession(self): def canBeStopped(self): # Only locked nodes running in local with the same # sessionUid as the Meshroom instance can be stopped - # logging.warning(f"[{self.name}] canBeStopped: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") return (self.getGlobalStatus() == Status.RUNNING and self.globalExecMode == ExecMode.LOCAL.name and self.initFromThisSession()) @@ -1435,7 +1458,6 @@ def canBeStopped(self): def canBeCanceled(self): # Only locked nodes submitted in local with the same # sessionUid as the Meshroom instance can be canceled - # logging.warning(f"[{self.name}] canBeCanceled: globalExecMode={self.globalExecMode} globalStatus={self.getGlobalStatus()} statusInThisSession={self.statusInThisSession()}, submitterStatusInThisSession={self.submitterStatusInThisSession()}") return (self.getGlobalStatus() == Status.SUBMITTED and self.globalExecMode == ExecMode.LOCAL.name and self.initFromThisSession()) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 85bb3c2a93..a41430b2d4 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -187,7 +187,7 @@ def watchedStatusFiles(self): # when run locally, status changes are already notified. # Chunks with an ERROR status may be re-submitted externally and should thus still be monitored if (c.isExtern() and c._status.status in (Status.SUBMITTED, Status.RUNNING, Status.ERROR)) or ( - (c._status.mrNodeType == MrNodeType.NODE) and (c._status.status in (Status.SUBMITTED, Status.RUNNING))): + (c.node.getMrNodeType() == MrNodeType.NODE) and (c._status.status in (Status.SUBMITTED, Status.RUNNING))): files.append(c.statusFile) chunks.append(c) return files, chunks @@ -584,7 +584,7 @@ def submit(self, nodes: Optional[Union[list[Node], Node]] = None): def updateGraphComputingStatus(self): # update graph computing status computingLocally = any([ - ((ch.status.submitterSessionUid if ch.status.mrNodeType == MrNodeType.NODE else ch.status.sessionUid) == sessionUid) and ( + ((ch.status.submitterSessionUid if ch.node.getMrNodeType() == MrNodeType.NODE else ch.status.sessionUid) == sessionUid) and ( ch.status.status in (Status.RUNNING, Status.SUBMITTED)) for ch in self._sortedDFSChunks]) submitted = any([ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks]) From 0c320602647e14ff5d6ac279bc3b52e463191d04 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 12:15:39 +0100 Subject: [PATCH 24/49] [core] node: add some typing --- meshroom/core/node.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index a65ae3f4f9..b86c9a216e 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -68,15 +68,18 @@ class StatusData(BaseObject): def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', mrNodeType: MrNodeType = MrNodeType.NONE, parent: BaseObject = None): super(StatusData, self).__init__(parent) - self.reset() + self.nodeName: str = nodeName self.nodeType: str = nodeType self.packageName: str = packageName self.packageVersion: str = packageVersion self.mrNodeType = mrNodeType + self.sessionUid: Optional[str] = meshroom.core.sessionUid self.submitterSessionUid: Optional[str] = None + self.resetDynamicValues() + def merge(self, other): self.startDateTime = min(self.startDateTime, other.startDateTime) self.endDateTime = max(self.endDateTime, other.endDateTime) @@ -1420,7 +1423,7 @@ def updateDuplicates(self, nodesPerUid): self._hasDuplicates = bool(len(newList)) self.hasDuplicatesChanged.emit() - def statusInThisSession(self): + def statusInThisSession(self) -> bool: if not self._chunks: return False for chunk in self._chunks: @@ -1428,7 +1431,7 @@ def statusInThisSession(self): return False return True - def submitterStatusInThisSession(self): + def submitterStatusInThisSession(self) -> bool: if not self._chunks: return False for chunk in self._chunks: @@ -1436,8 +1439,8 @@ def submitterStatusInThisSession(self): return False return True - def initFromThisSession(self): - if not self._chunks: + def initFromThisSession(self) -> bool: + if len(self._chunks) == 0: return False for chunk in self._chunks: mrNodeType = chunk.node.getMrNodeType() @@ -1447,7 +1450,7 @@ def initFromThisSession(self): return True @Slot(result=bool) - def canBeStopped(self): + def canBeStopped(self) -> bool: # Only locked nodes running in local with the same # sessionUid as the Meshroom instance can be stopped return (self.getGlobalStatus() == Status.RUNNING and @@ -1455,14 +1458,14 @@ def canBeStopped(self): self.initFromThisSession()) @Slot(result=bool) - def canBeCanceled(self): + def canBeCanceled(self) -> bool: # Only locked nodes submitted in local with the same # sessionUid as the Meshroom instance can be canceled return (self.getGlobalStatus() == Status.SUBMITTED and self.globalExecMode == ExecMode.LOCAL.name and self.initFromThisSession()) - def hasImageOutputAttribute(self): + def hasImageOutputAttribute(self) -> bool: """ Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), False otherwise. @@ -1472,7 +1475,7 @@ def hasImageOutputAttribute(self): return True return False - def hasSequenceOutputAttribute(self): + def hasSequenceOutputAttribute(self) -> bool: """ Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), False otherwise. From 75ab823c1893f0864d142ef0f6e19f5611b1c0a6 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 12:16:40 +0100 Subject: [PATCH 25/49] [core] node: udpate isExtern check --- meshroom/core/node.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index b86c9a216e..40c402b440 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -515,9 +515,11 @@ def stopProcess(self): def isExtern(self): """ The computation is managed externally by another instance of Meshroom. - In the ambiguous case of an isolated environment, it is considered as local as we can stop it. + In the ambiguous case of an isolated environment, it is considered as local as we can stop it (if it is run from the current Meshroom instance). """ - uid = self._status.submitterSessionUid if self.node.getMrNodeType() == MrNodeType.NODE else meshroom.core.sessionUid + if self._status.execMode == ExecMode.EXTERN: + return True + uid = self._status.submitterSessionUid if self.node.getMrNodeType() == MrNodeType.NODE else self._status.sessionUid return uid != meshroom.core.sessionUid statusChanged = Signal() From e54127fbf0bfd190dfc47aca5b00ca3ee21d0147 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 12:17:21 +0100 Subject: [PATCH 26/49] [core] statusNodeName change over time due to duplicates --- meshroom/core/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 40c402b440..20776616a0 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -535,7 +535,7 @@ def isExtern(self): statisticsFile = Property(str, statisticsFile.fget, notify=nodeFolderChanged) nodeName = Property(str, lambda self: self.node.name, constant=True) - statusNodeName = Property(str, lambda self: self._status.nodeName, constant=True) + statusNodeName = Property(str, lambda self: self._status.nodeName, notify=statusChanged) elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) From 4e7de577c2ee1224377d16b370d02b008a50f21a Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 12:18:00 +0100 Subject: [PATCH 27/49] [core] Init node name with non-empty unique ID --- meshroom/core/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 20776616a0..74cee1b638 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -577,7 +577,8 @@ def __init__(self, nodeType, position=None, parent=None, uid=None, **kwargs): self._internalFolder = "" self._sourceCodeFolder = "" - self._name = None + # temporary unique name for this node + self._name = f"_{nodeType}_{uuid.uuid1()}" self.graph = None self.dirty = True # whether this node's outputs must be re-evaluated on next Graph update self._chunks = ListModel(parent=self) From 8be90ce362c487aa76d3514ba4df27a71a58793b Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 12:21:16 +0100 Subject: [PATCH 28/49] [core] node: simplify with a new method isMainNode() And add the check about duplicates for canBeStopped/canBeCanceled. --- meshroom/core/node.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 74cee1b638..cf1835bd96 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1339,7 +1339,7 @@ def setLocked(self, lock): @Slot() def updateDuplicatesStatusAndLocked(self): """ Update status of duplicate nodes without any latency and update locked. """ - if self.name == self._chunks.at(0).statusNodeName: + if self.isMainNode(): for node in self._duplicates: node.updateStatusFromCache() @@ -1378,7 +1378,7 @@ def updateLocked(self): # Check if at least one dependentNode is submitted or currently running for node in outputNodes: - if node.getGlobalStatus() in lockedStatus and node._chunks.at(0).statusNodeName == node.name: + if node.getGlobalStatus() in lockedStatus and node.isMainNode(): stayLocked = True break if not stayLocked: @@ -1387,7 +1387,7 @@ def updateLocked(self): for node in inputNodes: node.setLocked(False) return - elif currentStatus in lockedStatus and self._chunks.at(0).statusNodeName == self.name: + elif currentStatus in lockedStatus and self.isMainNode(): self.setLocked(True) inputNodes = self.getInputNodes(recursive=True, dependenciesOnly=True) for node in inputNodes: @@ -1451,6 +1451,16 @@ def initFromThisSession(self) -> bool: if uid != meshroom.core.sessionUid: return False return True + + def isMainNode(self) -> bool: + """ In case of a node with duplicates, we check that the node is the one driving the computation. """ + if len(self._chunks) == 0: + return True + firstChunk = self._chunks.at(0) + if not firstChunk.statusNodeName: + # If nothing is declared, anyone could become the main (if there are duplicates). + return True + return firstChunk.statusNodeName == self.name @Slot(result=bool) def canBeStopped(self) -> bool: @@ -1458,6 +1468,7 @@ def canBeStopped(self) -> bool: # sessionUid as the Meshroom instance can be stopped return (self.getGlobalStatus() == Status.RUNNING and self.globalExecMode == ExecMode.LOCAL.name and + self.isMainNode() and self.initFromThisSession()) @Slot(result=bool) @@ -1466,6 +1477,7 @@ def canBeCanceled(self) -> bool: # sessionUid as the Meshroom instance can be canceled return (self.getGlobalStatus() == Status.SUBMITTED and self.globalExecMode == ExecMode.LOCAL.name and + self.isMainNode() and self.initFromThisSession()) def hasImageOutputAttribute(self) -> bool: From ea3f87b04124d9e184718d69037fc00bbcb927a7 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Fri, 28 Mar 2025 20:23:01 +0100 Subject: [PATCH 29/49] [core] improve the update of the node status --- meshroom/core/node.py | 53 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index cf1835bd96..f435589d12 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -80,6 +80,20 @@ def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', self.resetDynamicValues() + def setNode(self, node): + """ Set the node information from one node instance.""" + self.nodeName = node.name + self.setNodeType(node) + + def setNodeType(self, node): + """ Set the node type and package information from the given node. + We do not set the name in this method as it may vary if there are duplicates. + """ + self.nodeType = node.nodeType + self.packageName = node.packageName + self.packageVersion = node.packageVersion + self.mrNodeType = node.getMrNodeType() + def merge(self, other): self.startDateTime = min(self.startDateTime, other.startDateTime) self.endDateTime = max(self.endDateTime, other.endDateTime) @@ -112,12 +126,15 @@ def initStartCompute(self): self._startTime = time.time() self.startDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting) # to get datetime obj: datetime.datetime.strptime(obj, self.dateTimeFormatting) + self.status = Status.RUNNING + self.execMode = ExecMode.LOCAL def initIsolatedCompute(self): ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. ''' self.resetDynamicValues() - # assert(self.mrNodeType == MrNodeType.NODE) + self.initStartCompute() + assert(self.mrNodeType == MrNodeType.NODE) self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid @@ -337,16 +354,19 @@ def updateStatusFromCache(self): if not os.path.exists(statusFile): self.statusFileLastModTime = -1 self._status.reset() + self._status.setNodeType(self.node) else: try: with open(statusFile, 'r') as jsonFile: statusData = json.load(jsonFile) # logging.debug(f"updateStatusFromCache({self.node.name}): From status {self.status.status} to {statusData['status']}") - self.status.fromDict(statusData) + self._status.fromDict(statusData) self.statusFileLastModTime = os.path.getmtime(statusFile) - except Exception: + except Exception as e: + logging.debug(f"updateStatusFromCache({self.node.name}): Error while loading status file {statusFile}: {e}") self.statusFileLastModTime = -1 - self.status.reset() + self._status.reset() + self._status.setNodeType(self.node) if oldStatus != self.status.status: self.statusChanged.emit() @@ -386,6 +406,12 @@ def saveStatusFile(self): json.dump(data, jsonFile, indent=4) renameWritingToFinalPath(statusFilepathWriting, statusFilepath) + def upgradeStatusFile(self): + """ Upgrade node status file based on the current status. + """ + self.saveStatusFile() + self.statusChanged.emit() + def upgradeStatusTo(self, newStatus, execMode=None): if newStatus.value < self._status.status.value: logging.warning(f"Downgrade status on node '{self.name}' from {self._status.status} to {newStatus}") @@ -394,8 +420,7 @@ def upgradeStatusTo(self, newStatus, execMode=None): self._status.execMode = execMode self.execModeNameChanged.emit() self._status.status = newStatus - self.saveStatusFile() - self.statusChanged.emit() + self.upgradeStatusFile() def updateStatisticsFromCache(self): """ @@ -452,9 +477,10 @@ def process(self, forceCompute=False, inCurrentEnv=False): global runningProcesses runningProcesses[self.name] = self + self._status.setNode(self.node) self._status.initStartCompute() + self.upgradeStatusFile() executionStatus = None - self.upgradeStatusTo(Status.RUNNING) self.statThread = stats.StatisticsThread(self) self.statThread.start() try: @@ -471,7 +497,10 @@ def process(self, forceCompute=False, inCurrentEnv=False): executionStatus = Status.STOPPED raise finally: + self._status.setNode(self.node) self._status.initEndCompute() + self.upgradeStatusFile() + if executionStatus: self.upgradeStatusTo(executionStatus) logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) @@ -485,8 +514,10 @@ def process(self, forceCompute=False, inCurrentEnv=False): def _processInIsolatedEnvironment(self): """Process this node chunk in the isolated environment defined in the environment configuration.""" try: + self._status.setNode(self.node) self._status.initIsolatedCompute() - self.upgradeStatusTo(Status.RUNNING, execMode=ExecMode.LOCAL) + self.upgradeStatusFile() + self.node.nodeDesc.processChunkInEnvironment(self) except: # status should be already updated by meshroom_compute @@ -1151,14 +1182,16 @@ def updateStatusFromCache(self): def submit(self, forceCompute=False): for chunk in self._chunks: if forceCompute or chunk.status.status != Status.SUCCESS: + chunk._status.setNode(self) chunk._status.initExternSubmit() - chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.EXTERN) + chunk.upgradeStatusFile() def beginSequence(self, forceCompute=False): for chunk in self._chunks: if forceCompute or (chunk.status.status not in (Status.RUNNING, Status.SUCCESS)): + chunk._status.setNode(self) chunk._status.initLocalSubmit() - chunk.upgradeStatusTo(Status.SUBMITTED, ExecMode.LOCAL) + chunk.upgradeStatusFile() def processIteration(self, iteration): self._chunks[iteration].process() From 92555f6ab38165290946f7013a40a35c77b1c7ef Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 12 Apr 2025 19:36:05 +0200 Subject: [PATCH 30/49] [core] more explicit error messages when loading plugins --- meshroom/core/__init__.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 85b98becbe..06cbeacd94 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -55,19 +55,34 @@ def add_to_path(p): sys.path = old_path -def loadPlugins(folder, packageName, classType): +def loadClasses(folder, packageName, classType): """ """ - pluginTypes = [] + classes = [] errors = [] + resolvedFolder = str(Path(folder).resolve()) # temporarily add folder to python path - with add_to_path(folder): + with add_to_path(resolvedFolder): # import node package - package = importlib.import_module(packageName) - packageName = package.packageName if hasattr(package, 'packageName') else package.__name__ - packageVersion = getattr(package, "__version__", None) - packagePath = os.path.dirname(package.__file__) + + try: + package = importlib.import_module(packageName) + packageName = package.packageName if hasattr(package, 'packageName') else package.__name__ + packageVersion = getattr(package, "__version__", None) + packagePath = os.path.dirname(package.__file__) + except Exception as e: + tb = traceback.extract_tb(e.__traceback__) + last_call = tb[-1] + logging.warning(f' * Failed to load package "{packageName}" from folder "{resolvedFolder}" ({type(e).__name__}): {str(e)}\n' + # filename:lineNumber functionName + f'{last_call.filename}:{last_call.lineno} {last_call.name}\n' + # line of code with the error + f'{last_call.line}' + # Full traceback + f'\n{traceback.format_exc()}\n\n' + ) + return [] for importer, pluginName, ispkg in pkgutil.iter_modules(package.__path__): pluginModuleName = '.' + pluginName @@ -93,7 +108,7 @@ def loadPlugins(folder, packageName, classType): p.packageVersion = packageVersion p.packagePath = packagePath if importPlugin: - pluginTypes.extend(plugins) + classes.extend(plugins) except Exception as e: tb = traceback.extract_tb(e.__traceback__) last_call = tb[-1] @@ -110,7 +125,7 @@ def loadPlugins(folder, packageName, classType): logging.warning(' The following "{package}" plugins could not be loaded:\n' '{errorMsg}\n' .format(package=packageName, errorMsg='\n'.join(errors))) - return pluginTypes + return classes def validateNodeDesc(nodeDesc): @@ -314,7 +329,7 @@ def loadNodes(folder, packageName): logging.error("Node folder '{folder}' does not exist.") return - return loadPlugins(folder, packageName, desc.BaseNode) + return loadClasses(folder, packageName, desc.BaseNode) def loadAllNodes(folder): @@ -340,7 +355,7 @@ def loadSubmitters(folder, packageName): logging.error(f"Submitters folder '{folder}' does not exist.") return - return loadPlugins(folder, packageName, BaseSubmitter) + return loadClasses(folder, packageName, BaseSubmitter) def loadPipelineTemplates(folder): From db8fd02aebef27d12a80f8c2c177b05d286affbe Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 12 Apr 2025 19:38:58 +0200 Subject: [PATCH 31/49] New plugins load MESHROOM_PLUGINS_PATH can be used to automatically load nodes and pipelines from a folder structure. --- bin/meshroom_batch | 1 + bin/meshroom_compute | 1 + bin/meshroom_submit | 1 + meshroom/core/__init__.py | 36 ++++++++++++++++++++++++++++++++++++ meshroom/env.py | 4 ++-- meshroom/ui/app.py | 1 + 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/bin/meshroom_batch b/bin/meshroom_batch index 6bee4c1f6d..d08d40eae7 100755 --- a/bin/meshroom_batch +++ b/bin/meshroom_batch @@ -146,6 +146,7 @@ if not args.input and not args.inputRecursive: print('Nothing to compute. You need to set --input or --inputRecursive.') sys.exit(1) +meshroom.core.initPlugins() meshroom.core.initNodes() graph = meshroom.core.graph.Graph(name=args.pipeline) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index 59d8b647e2..8d15e7e585 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -61,6 +61,7 @@ if args.extern: else: logging.getLogger().setLevel(meshroom.logStringToPython[args.verbose]) +meshroom.core.initPlugins() meshroom.core.initNodes() graph = meshroom.core.graph.loadGraph(args.graphFile) diff --git a/bin/meshroom_submit b/bin/meshroom_submit index 404a87f95a..34812e3db5 100755 --- a/bin/meshroom_submit +++ b/bin/meshroom_submit @@ -22,6 +22,7 @@ parser.add_argument("--submitLabel", args = parser.parse_args() +meshroom.core.initPlugins() meshroom.core.initNodes() meshroom.core.initSubmitters() diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 06cbeacd94..fe9ec170ed 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -4,6 +4,7 @@ import inspect import logging import os +from pathlib import Path import pkgutil import sys import tempfile @@ -343,6 +344,35 @@ def loadAllNodes(folder): logging.debug(f'Nodes loaded [{package}]: {nodesStr}') +def loadPluginFolder(folder): + if not os.path.isdir(folder): + logging.info(f"Plugin folder '{folder}' does not exist.") + return + + mrFolder = Path(folder, 'meshroom') + if not mrFolder.exists(): + logging.info(f"Plugin folder '{folder}' does not contain a 'meshroom' folder.") + return + + binFolders = [Path(folder, 'bin')] + libFolders = [Path(folder, 'lib'), Path(folder, 'lib64')] + pythonPathFolders = [Path(folder)] + binFolders + + loadAllNodes(folder=mrFolder) + loadPipelineTemplates(folder=mrFolder) + + +def loadPluginsFolder(folder): + if not os.path.isdir(folder): + logging.debug(f"PluginSet folder '{folder}' does not exist.") + return + + for file in os.listdir(folder): + if os.path.isdir(file): + subFolder = os.path.join(folder, file) + loadPluginFolder(subFolder) + + def registerSubmitter(s): global submitters if s.name in submitters: @@ -392,3 +422,9 @@ def initPipelines(): for f in pipelineTemplatesFolders: loadPipelineTemplates(f) + +def initPlugins(): + additionalpluginsPath = EnvVar.getList(EnvVar.MESHROOM_PLUGINS_PATH) + nodesFolders = [os.path.join(meshroomFolder, 'plugins')] + additionalpluginsPath + for f in nodesFolders: + loadPluginFolder(folder=f) diff --git a/meshroom/env.py b/meshroom/env.py index 6e94e5f873..46ce30e9fa 100644 --- a/meshroom/env.py +++ b/meshroom/env.py @@ -42,10 +42,10 @@ class EnvVar(Enum): ) # Core - MESHROOM_PLUGINS_PATH = VarDefinition(str, "", "Paths to plugins folders") + MESHROOM_PLUGINS_PATH = VarDefinition(str, "", "Paths to plugins folders containing nodes, submitters and pipeline templates") MESHROOM_NODES_PATH = VarDefinition(str, "", "Paths to set of nodes folders") MESHROOM_SUBMITTERS_PATH = VarDefinition(str, "", "Paths to set of submitters folders") - MESHROOM_PIPELINE_TEMPLATES_PATH = VarDefinition(str, "", "Paths to pipeline templates folders") + MESHROOM_PIPELINE_TEMPLATES_PATH = VarDefinition(str, "", "Paths to et of pipeline templates folders") @staticmethod def get(envVar: "EnvVar") -> Any: diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 9d51f0f01a..11f921b42d 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -232,6 +232,7 @@ def __init__(self, inputArgs): # - clean cache directory and make sure it exists on disk ThumbnailCache.initialize() + meshroom.core.initPlugins() meshroom.core.initNodes() meshroom.core.initSubmitters() From 008d6c75ee51327b9f14ed7dc5408657cdcbc40f Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 12 Apr 2025 19:49:11 +0200 Subject: [PATCH 32/49] Automatically save the project when computing or submitting to renderfarm If the project is not saved at all, it will suggest to save it manually or to define a project in a temporary folder using date/time for the project name. --- meshroom/core/__init__.py | 1 - meshroom/core/desc/node.py | 15 -------- meshroom/core/graph.py | 5 ++- meshroom/env.py | 4 ++- meshroom/ui/graph.py | 9 +++++ meshroom/ui/qml/Application.qml | 63 ++++++++++++++++++++------------- 6 files changed, 53 insertions(+), 44 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index fe9ec170ed..8263520279 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -31,7 +31,6 @@ sessionUid = str(uuid.uuid1()) cacheFolderName = 'MeshroomCache' -defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName)) nodesDesc = {} submitters = {} pipelineTemplates = {} diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index 2cbec1110d..e524037edf 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -25,18 +25,6 @@ class MrNodeType(enum.Enum): COMMANDLINE = enum.auto() INPUT = enum.auto() -def isNodeSaved(node): - """Returns whether a node is identical to its serialized counterpart in the current graph file.""" - filepath = node.graph.filepath - if not filepath: - return False - - from meshroom.core.graph import loadGraph - graphSaved = loadGraph(filepath) - nodeSaved = graphSaved.node(node.name) - if nodeSaved is None: - return False - return nodeSaved._uid == node._uid class BaseNode(object): """ @@ -259,9 +247,6 @@ def getMrNodeType(self): return MrNodeType.NODE def processChunkInEnvironment(self, chunk): - if not isNodeSaved(chunk.node): - raise RuntimeError("File must be saved before computing in isolated environment.") - meshroomComputeCmd = f"python {_MESHROOM_COMPUTE} {chunk.node.graph.filepath} --node {chunk.node.name} --extern --inCurrentEnv" if len(chunk.node.getChunks()) > 1: meshroomComputeCmd += f" --iteration {chunk.range.iteration}" diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 11cb1965eb..ab28bcc3b1 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -182,7 +182,6 @@ class Graph(BaseObject): edges = {B.input: A.output, C.input: B.output,} """ - _cacheDir = "" def __init__(self, name, parent=None): super(Graph, self).__init__(parent) @@ -199,7 +198,7 @@ def __init__(self, name, parent=None): # Edges: use dst attribute as unique key since it can only have one input connection self._edges = DictModel(keyAttrName='dst', parent=self) self._compatibilityNodes = DictModel(keyAttrName='name', parent=self) - self.cacheDir = meshroom.core.defaultCacheFolder + self._cacheDir = '' self._filepath = '' self._fileDateVersion = 0 self.header = {} @@ -1354,7 +1353,7 @@ def _setFilepath(self, filepath): def _unsetFilepath(self): self._filepath = "" self.name = "" - self.cacheDir = meshroom.core.defaultCacheFolder + self.cacheDir = "" self.filepathChanged.emit() def updateInternals(self, startNodes=None, force=False): diff --git a/meshroom/env.py b/meshroom/env.py index 46ce30e9fa..ddd4857747 100644 --- a/meshroom/env.py +++ b/meshroom/env.py @@ -12,9 +12,9 @@ from dataclasses import dataclass from enum import Enum import sys +import tempfile from typing import Any, Type - meshroomFolder = os.path.dirname(__file__) @dataclass @@ -46,6 +46,8 @@ class EnvVar(Enum): MESHROOM_NODES_PATH = VarDefinition(str, "", "Paths to set of nodes folders") MESHROOM_SUBMITTERS_PATH = VarDefinition(str, "", "Paths to set of submitters folders") MESHROOM_PIPELINE_TEMPLATES_PATH = VarDefinition(str, "", "Paths to et of pipeline templates folders") + MESHROOM_TEMP_PATH = VarDefinition(str, tempfile.gettempdir(), "Path to the temporary folder") + @staticmethod def get(envVar: "EnvVar") -> Any: diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index a41430b2d4..d5062bf86b 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -514,6 +514,14 @@ def _saveAs(self, url, setupProjectFile=True, template=False): # => force re-evaluation of monitored status files paths self.updateChunkMonitor(self._sortedDFSChunks) + @Slot() + def saveAsTemp(self): + from meshroom.env import EnvVar + from datetime import datetime + tempFolder = EnvVar.get(EnvVar.MESHROOM_TEMP_PATH) + timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M") + self._saveAs(os.path.join(tempFolder, f"meshroom_{timestamp}.mg")) + @Slot() def save(self): self._graph.save() @@ -531,6 +539,7 @@ def updateLockedUndoStack(self): @Slot(list) def execute(self, nodes: Optional[Union[list[Node], Node]] = None): nodes = [nodes] if not isinstance(nodes, Iterable) and nodes else nodes + self.save() # always save the graph before computing self._taskManager.compute(self._graph, nodes) self.updateLockedUndoStack() # explicitly call the update while it is already computing diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index ef91bcca4d..59e04d1288 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -134,6 +134,8 @@ Page { id: saveFileDialog options: Platform.FileDialog.DontUseNativeDialog + property var _callback: undefined + signal closed(var result) title: "Save File" @@ -150,8 +152,28 @@ Page { _reconstruction.saveAs(currentFile) MeshroomApp.addRecentProjectFile(currentFile.toString()) closed(Platform.Dialog.Accepted) + fireCallback(Platform.Dialog.Accepted) + } + onRejected: { + closed(Platform.Dialog.Rejected) + fireCallback(Platform.Dialog.Rejected) + } + + function fireCallback(rc) + { + // Call the callback and reset it + if (_callback) + _callback(rc) + _callback = undefined + } + + // Open the unsaved dialog warning with an optional + // callback to fire when the dialog is accepted/discarded + function prompt(callback) + { + _callback = callback + open() } - onRejected: closed(Platform.Dialog.Rejected) } Platform.FileDialog { @@ -218,8 +240,6 @@ Page { Item { id: computeManager - property bool warnIfUnsaved: true - // Evaluate if graph computation can be submitted externally property bool canSubmit: _reconstruction ? _reconstruction.canSubmit // current setup allows to compute externally @@ -227,7 +247,7 @@ Page { false function compute(nodes, force) { - if (!force && warnIfUnsaved && !_reconstruction.graph.filepath) { + if (!force && !_reconstruction.graph.filepath) { unsavedComputeDialog.selectedNodes = nodes; unsavedComputeDialog.open(); } @@ -339,29 +359,28 @@ Page { parent: Overlay.overlay preset: "Warning" title: "Unsaved Project" - text: "Data will be computed in the default cache folder if project remains unsaved." - detailedText: "Default cache folder: " + (_reconstruction ? _reconstruction.graph.cacheDir : "unknown") - helperText: "Save project first?" + text: "Saving the project is required." + helperText: "Choose a location to save the project, or use the default temporary path." standardButtons: Dialog.Discard | Dialog.Cancel | Dialog.Save - CheckBox { - Layout.alignment: Qt.AlignRight - text: "Don't ask again for this session" - padding: 0 - onToggled: computeManager.warnIfUnsaved = !checked - } - Component.onCompleted: { // Set up discard button text - standardButton(Dialog.Discard).text = "Continue without Saving" + standardButton(Dialog.Discard).text = "Continue in Temp Folder" + standardButton(Dialog.Save).text = "Save As" } onDiscarded: { + _reconstruction.saveAsTemp() close() computeManager.compute(selectedNodes, true) } - onAccepted: saveAsAction.trigger() + onAccepted: { + initFileDialogFolder(saveFileDialog) + saveFileDialog.prompt(function(rc) { + computeManager.compute(selectedNodes, true) + }) + } } MessageDialog { @@ -444,14 +463,10 @@ Page { } // Open "Save As" dialog else { - saveFileDialog.open() - function _callbackWrapper(rc) { + saveFileDialog.prompt(function(rc) { if (rc === Platform.Dialog.Accepted) fireCallback() - - saveFileDialog.closed.disconnect(_callbackWrapper) - } - saveFileDialog.closed.connect(_callbackWrapper) + }) } } @@ -463,8 +478,8 @@ Page { _callback = undefined } - /// Open the unsaved dialog warning with an optional - /// callback to fire when the dialog is accepted/discarded + // Open the unsaved dialog warning with an optional + // callback to fire when the dialog is accepted/discarded function prompt(callback) { _callback = callback From 44ec6f0be7c209addab2a88a84c8d122e1c155bc Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 10:08:28 +0200 Subject: [PATCH 33/49] [core] NodeChunk: Do not raise an error when we stop a chunk that is not running When we stop the process of a node with multiple chunks, the Node function will call the stop function of each chunk. So, the chunck status could be SUBMITTED, RUNNING or ERROR. --- meshroom/core/node.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index f435589d12..d55d63e1d6 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -535,8 +535,22 @@ def _processInIsolatedEnvironment(self): def stopProcess(self): if self.isExtern(): raise ValueError("Cannot stop process: node is computed externally (another instance of Meshroom)") + + # Ensure that we are up-to-date + self.updateStatusFromCache() + if self._status.status != Status.RUNNING: - raise ValueError(f"Cannot stop process: node is not running (status is: {self._status.status}).") + # When we stop the process of a node with multiple chunks, the Node function will call the stop function of each chunk. + # So, the chunck status could be SUBMITTED, RUNNING or ERROR. + + if self._status.status is Status.SUBMITTED: + self.upgradeStatusTo(Status.NONE) + elif self._status.status in (Status.ERROR, Status.STOPPED, Status.KILLED, Status.SUCCESS, Status.NONE): + # Nothing to do, the computation is already stopped. + pass + else: + logging.debug(f"Cannot stop process: node is not running (status is: {self._status.status}).") + return self.node.nodeDesc.stopProcess(self) From 299d8f29dfc8161fc65b2a86778424f506313239 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:44:04 +0200 Subject: [PATCH 34/49] [core] NodeChunk: global notification for all status changes and no more separate notification for execMode --- meshroom/core/node.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index d55d63e1d6..41492bdab8 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -319,7 +319,6 @@ def __init__(self, node, range, parent=None): # Notify update in filepaths when node's internal folder changes self.node.internalFolderChanged.connect(self.nodeFolderChanged) - self.execModeNameChanged.connect(self.node.globalExecModeChanged) @property def index(self): @@ -418,7 +417,6 @@ def upgradeStatusTo(self, newStatus, execMode=None): if execMode is not None: self._status.execMode = execMode - self.execModeNameChanged.emit() self._status.status = newStatus self.upgradeStatusFile() @@ -570,8 +568,7 @@ def isExtern(self): statusChanged = Signal() status = Property(Variant, lambda self: self._status, notify=statusChanged) statusName = Property(str, statusName.fget, notify=statusChanged) - execModeNameChanged = Signal() - execModeName = Property(str, execModeName.fget, notify=execModeNameChanged) + execModeName = Property(str, execModeName.fget, notify=statusChanged) statisticsChanged = Signal() nodeFolderChanged = Signal() @@ -1596,9 +1593,8 @@ def has3DOutputAttribute(self): isCompatibilityNode = Property(bool, lambda self: self._isCompatibilityNode(), constant=True) isInputNode = Property(bool, lambda self: self._isInputNode(), constant=True) - globalExecModeChanged = Signal() - globalExecMode = Property(str, globalExecMode.fget, notify=globalExecModeChanged) - isExternal = Property(bool, isExtern, notify=globalExecModeChanged) + globalExecMode = Property(str, globalExecMode.fget, notify=globalStatusChanged) + isExternal = Property(bool, isExtern, notify=globalStatusChanged) isComputed = Property(bool, _isComputed, notify=globalStatusChanged) isComputable = Property(bool, _isComputable, notify=globalStatusChanged) aliveChanged = Signal() From ca75e758be82eb25f76123f8f1dc37595a1cda5b Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:48:01 +0200 Subject: [PATCH 35/49] [core] improve checks for sessionUid and execMode --- meshroom/core/node.py | 24 ++++++++++++++++-------- meshroom/ui/graph.py | 7 +++++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 41492bdab8..1ee3fce3ed 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -75,7 +75,7 @@ def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', self.packageVersion: str = packageVersion self.mrNodeType = mrNodeType - self.sessionUid: Optional[str] = meshroom.core.sessionUid + self.sessionUid: Optional[str] = None self.submitterSessionUid: Optional[str] = None self.resetDynamicValues() @@ -127,7 +127,9 @@ def initStartCompute(self): self.startDateTime = datetime.datetime.now().strftime(self.dateTimeFormatting) # to get datetime obj: datetime.datetime.strptime(obj, self.dateTimeFormatting) self.status = Status.RUNNING - self.execMode = ExecMode.LOCAL + # Note: We do not modify the "execMode" here, as it is set in the init*Submit methods. + # When we compute (from renderfarm or isolated environment), + # we don't want to modify the execMode set from the submit. def initIsolatedCompute(self): ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. @@ -319,7 +321,6 @@ def __init__(self, node, range, parent=None): # Notify update in filepaths when node's internal folder changes self.node.internalFolderChanged.connect(self.nodeFolderChanged) - @property def index(self): return self.range.iteration @@ -562,8 +563,11 @@ def isExtern(self): """ if self._status.execMode == ExecMode.EXTERN: return True - uid = self._status.submitterSessionUid if self.node.getMrNodeType() == MrNodeType.NODE else self._status.sessionUid - return uid != meshroom.core.sessionUid + elif self._status.execMode == ExecMode.LOCAL: + if self._status.status in (Status.SUBMITTED, Status.RUNNING): + return meshroom.core.sessionUid not in (self._status.submitterSessionUid, self._status.sessionUid) + return False + return False statusChanged = Signal() status = Property(Variant, lambda self: self._status, notify=statusChanged) @@ -1033,6 +1037,8 @@ def isExtern(self): chunk has completed locally before the computations were interrupted, its execution mode will always be local, even if computations resume externally. """ + if len(self._chunks) == 0: + return False return any(chunk.isExtern() for chunk in self._chunks) @Slot() @@ -1490,9 +1496,7 @@ def initFromThisSession(self) -> bool: if len(self._chunks) == 0: return False for chunk in self._chunks: - mrNodeType = chunk.node.getMrNodeType() - uid = chunk.status.submitterSessionUid if mrNodeType == MrNodeType.NODE else chunk.status.sessionUid - if uid != meshroom.core.sessionUid: + if meshroom.core.sessionUid not in (chunk.status.sessionUid, chunk.status.submitterSessionUid): return False return True @@ -1508,6 +1512,8 @@ def isMainNode(self) -> bool: @Slot(result=bool) def canBeStopped(self) -> bool: + if not self.isComputable: + return False # Only locked nodes running in local with the same # sessionUid as the Meshroom instance can be stopped return (self.getGlobalStatus() == Status.RUNNING and @@ -1517,6 +1523,8 @@ def canBeStopped(self) -> bool: @Slot(result=bool) def canBeCanceled(self) -> bool: + if not self.isComputable: + return False # Only locked nodes submitted in local with the same # sessionUid as the Meshroom instance can be canceled return (self.getGlobalStatus() == Status.SUBMITTED and diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index d5062bf86b..f2826506c3 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -593,10 +593,13 @@ def submit(self, nodes: Optional[Union[list[Node], Node]] = None): def updateGraphComputingStatus(self): # update graph computing status computingLocally = any([ - ((ch.status.submitterSessionUid if ch.node.getMrNodeType() == MrNodeType.NODE else ch.status.sessionUid) == sessionUid) and ( + ch.status.execMode == ExecMode.LOCAL and + (sessionUid in (ch.status.submitterSessionUid, ch.status.sessionUid)) and ( ch.status.status in (Status.RUNNING, Status.SUBMITTED)) for ch in self._sortedDFSChunks]) - submitted = any([ch.status.status == Status.SUBMITTED for ch in self._sortedDFSChunks]) + # Note: We do not check sessionUid for the submitted status, + # as the source instance of the submit has no importance. + submitted = any([ch.status.execMode == ExecMode.EXTERN and ch.status.status in (Status.RUNNING, Status.SUBMITTED) for ch in self._sortedDFSChunks]) if self._computingLocally != computingLocally or self._submitted != submitted: self._computingLocally = computingLocally self._submitted = submitted From e65dc09710526ffbbdbea8da4a2efe8c0faadd0e Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:49:59 +0200 Subject: [PATCH 36/49] [core] Simplify checks for displayable outputs --- meshroom/core/node.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 1ee3fce3ed..25ce190474 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1538,7 +1538,9 @@ def hasImageOutputAttribute(self) -> bool: False otherwise. """ for attr in self._attributes: - if attr.enabled and attr.isOutput and attr.desc.semantic == "image": + if not attr.enabled or not attr.isOutput: + continue + if attr.desc.semantic == "image": return True return False @@ -1548,8 +1550,9 @@ def hasSequenceOutputAttribute(self) -> bool: False otherwise. """ for attr in self._attributes: - if attr.enabled and attr.isOutput and (attr.desc.semantic == "sequence" or - attr.desc.semantic == "imageList"): + if not attr.enabled or not attr.isOutput: + continue + if attr.desc.semantic in ("sequence", "imageList"): return True return False @@ -1560,9 +1563,11 @@ def has3DOutputAttribute(self): # List of supported extensions, taken from Viewer3DSettings supportedExts = ['.obj', '.stl', '.fbx', '.gltf', '.abc', '.ply'] for attr in self._attributes: + if not attr.enabled or not attr.isOutput: + continue # If the attribute is a File attribute, it is an instance of str and can be iterated over hasSupportedExt = isinstance(attr.value, str) and any(ext in attr.value for ext in supportedExts) - if attr.enabled and attr.isOutput and hasSupportedExt: + if hasSupportedExt: return True return False From 38e82b926f72280ac782fe070f52fb5325a30c30 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:50:42 +0200 Subject: [PATCH 37/49] [core] Add support for "3d" semantic --- meshroom/core/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 25ce190474..f5d2279514 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1565,6 +1565,8 @@ def has3DOutputAttribute(self): for attr in self._attributes: if not attr.enabled or not attr.isOutput: continue + if attr.desc.semantic == "3d": + return True # If the attribute is a File attribute, it is an instance of str and can be iterated over hasSupportedExt = isinstance(attr.value, str) and any(ext in attr.value for ext in supportedExts) if hasSupportedExt: From c01aefc4f3ba4f00bdaa50f42c344d23ed6c8fe8 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:55:55 +0200 Subject: [PATCH 38/49] [ui] GraphEditor: Improved node status computable/submitable checks --- meshroom/core/taskManager.py | 2 +- meshroom/ui/qml/GraphEditor/GraphEditor.qml | 29 ++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py index f7de0ad3d5..dc7c41dcff 100644 --- a/meshroom/core/taskManager.py +++ b/meshroom/core/taskManager.py @@ -330,7 +330,7 @@ def checkDuplicates(self, nodesToProcess, context): def checkNodesDependencies(self, graph, toNodes, context): """ Check dependencies of nodes to process. - Update toNodes with computable/submittable nodes only. + Update toNodes with computable/submitable nodes only. Returns: bool: True if all the nodes can be processed. False otherwise. diff --git a/meshroom/ui/qml/GraphEditor/GraphEditor.qml b/meshroom/ui/qml/GraphEditor/GraphEditor.qml index 79d5c350d2..e9a827cbf6 100755 --- a/meshroom/ui/qml/GraphEditor/GraphEditor.qml +++ b/meshroom/ui/qml/GraphEditor/GraphEditor.qml @@ -574,35 +574,34 @@ Item { }); } - readonly property bool isSelectionOnlyComputableNodes: { - return uigraph.nodeSelection.selectedIndexes.every(function(idx) { + readonly property bool selectionContainsComputableNodes: { + return uigraph.nodeSelection.selectedIndexes.some(function(idx) { const node = uigraph.graph.nodes.at(idx.row); - return ( - node.isComputable - && uigraph.graph.canComputeTopologically(node) - ); + return node.isComputable; }); } readonly property bool canSelectionBeComputed: { - if(!isSelectionOnlyComputableNodes) + if(!selectionContainsComputableNodes) return false; if(isSelectionFullyComputed) return true; - return uigraph.nodeSelection.selectedIndexes.every(function(idx) { + var b = uigraph.nodeSelection.selectedIndexes.every(function(idx) { const node = uigraph.graph.nodes.at(idx.row); return ( - node.isComputed + node.isComputed || + (uigraph.graph.canComputeTopologically(node) && // canCompute if canSubmitOrCompute == 1(can compute) or 3(can compute & submit) - || nodeSubmitOrComputeStatus[node] % 2 == 1 + nodeSubmitOrComputeStatus[node] % 2 == 1) ); }); + return b } - readonly property bool isSelectionSubmittable: uigraph.canSubmit && isSelectionOnlyComputableNodes + readonly property bool isSelectionSubmitable: uigraph.canSubmit && selectionContainsComputableNodes readonly property bool canSelectionBeSubmitted: { - if(!isSelectionOnlyComputableNodes) + if(!selectionContainsComputableNodes) return false; if(isSelectionFullyComputed) return true; @@ -624,7 +623,7 @@ Item { MenuItem { id: computeMenuItem text: nodeMenu.isSelectionFullyComputed ? "Recompute" : "Compute" - visible: nodeMenu.isSelectionOnlyComputableNodes + visible: nodeMenu.selectionContainsComputableNodes height: visible ? implicitHeight : 0 enabled: nodeMenu.canSelectionBeComputed @@ -645,7 +644,7 @@ Item { id: submitMenuItem text: nodeMenu.isSelectionFullyComputed ? "Re-Submit" : "Submit" - visible: nodeMenu.isSelectionSubmittable + visible: nodeMenu.isSelectionSubmitable height: visible ? implicitHeight : 0 enabled: nodeMenu.canSelectionBeSubmitted @@ -678,7 +677,7 @@ Item { } MenuItem { text: "Open Folder" - visible: nodeMenu.currentNode.isComputable + visible: nodeMenu.currentNode.isComputable height: visible ? implicitHeight : 0 onTriggered: Qt.openUrlExternally(Filepath.stringToUrl(nodeMenu.currentNode.internalFolder)) } From 4cbd2e7766b2962a0c5bc3bf1e2d1ebc15b264d2 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 12:57:08 +0200 Subject: [PATCH 39/49] [ui] simplify visible/displayable status --- meshroom/ui/qml/GraphEditor/Node.qml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 65b95cd827..9e0d054d72 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -274,7 +274,7 @@ Item { // Submitted externally indicator MaterialLabel { - visible: ["SUBMITTED", "RUNNING"].includes(node.globalStatus) && node.chunks.count > 0 && node.isExternal + visible: node.isExternal text: MaterialIcons.cloud padding: 2 font.pointSize: 7 @@ -327,7 +327,7 @@ Item { text: MaterialIcons.visibility padding: 2 font.pointSize: 7 - property bool displayable: ((["SUCCESS"].includes(node.globalStatus) && node.chunks.count > 0) || !node.isComputable) + property bool displayable: !node.isComputable || (node.chunks.count > 0 && (["SUCCESS"].includes(node.globalStatus))) color: displayable ? palette.text : Qt.darker(palette.text, 1.8) ToolTip { From cd219fd70ef416d30318d706ec83230ad850562b Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 18:07:47 +0200 Subject: [PATCH 40/49] [core] more typing --- meshroom/core/__init__.py | 9 ++++----- meshroom/core/graph.py | 20 ++++++++++---------- meshroom/core/node.py | 37 +++++++++++++++++++------------------ 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 8263520279..9ce6b8484d 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -31,13 +31,12 @@ sessionUid = str(uuid.uuid1()) cacheFolderName = 'MeshroomCache' -nodesDesc = {} -submitters = {} -pipelineTemplates = {} +nodesDesc: dict[str, desc.BaseNode] = {} +submitters: dict[str, BaseSubmitter] = {} +pipelineTemplates: dict[str, str] = {} - -def hashValue(value): +def hashValue(value) -> str: """ Hash 'value' using sha1. """ hashObject = hashlib.sha1(str(value).encode('utf-8')) return hashObject.hexdigest() diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index ab28bcc3b1..2aa8403f3e 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -183,23 +183,23 @@ class Graph(BaseObject): """ - def __init__(self, name, parent=None): + def __init__(self, name: str = "", parent: BaseObject = None): super(Graph, self).__init__(parent) - self.name = name - self._loading = False - self._saving = False - self._updateEnabled = True - self._updateRequested = False - self.dirtyTopology = False + self.name: str = name + self._loading: bool = False + self._saving: bool = False + self._updateEnabled: bool = True + self._updateRequested: bool = False + self.dirtyTopology: bool = False self._nodesMinMaxDepths = {} self._computationBlocked = {} - self._canComputeLeaves = True + self._canComputeLeaves: bool = True self._nodes = DictModel(keyAttrName='name', parent=self) # Edges: use dst attribute as unique key since it can only have one input connection self._edges = DictModel(keyAttrName='dst', parent=self) self._compatibilityNodes = DictModel(keyAttrName='name', parent=self) - self._cacheDir = '' - self._filepath = '' + self._cacheDir: str = '' + self._filepath: str = '' self._fileDateVersion = 0 self.header = {} diff --git a/meshroom/core/node.py b/meshroom/core/node.py index f5d2279514..89b1328f75 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -601,44 +601,45 @@ class BaseNode(BaseObject): # i.e: a.b, a[0], a[0].b.c[1] attributeRE = re.compile(r'\.?(?P\w+)(?:\[(?P\d+)\])?') - def __init__(self, nodeType, position=None, parent=None, uid=None, **kwargs): + def __init__(self, nodeType: str, position: Position = None, parent: BaseObject = None, uid: str = None, **kwargs): """ Create a new Node instance based on the given node description. Any other keyword argument will be used to initialize this node's attributes. Args: - nodeDesc (desc.Node): the node description for this node - parent (BaseObject): this Node's parent + nodeType: name of the node type + parent: this Node's parent **kwargs: attributes values """ super(BaseNode, self).__init__(parent) - self._nodeType = nodeType - self.nodeDesc = None + self._nodeType: str = nodeType + self.nodeDesc: desc.BaseNode = None # instantiate node description if nodeType is valid if nodeType in meshroom.core.nodesDesc: self.nodeDesc = meshroom.core.nodesDesc[nodeType]() - self.packageName = self.packageVersion = "" - self._internalFolder = "" - self._sourceCodeFolder = "" + self.packageName: str = "" + self.packageVersion: str = "" + self._internalFolder: str = "" + self._sourceCodeFolder: str = "" # temporary unique name for this node - self._name = f"_{nodeType}_{uuid.uuid1()}" + self._name: str = f"_{nodeType}_{uuid.uuid1()}" self.graph = None - self.dirty = True # whether this node's outputs must be re-evaluated on next Graph update + self.dirty: bool = True # whether this node's outputs must be re-evaluated on next Graph update self._chunks = ListModel(parent=self) - self._uid = uid - self._cmdVars = {} - self._size = 0 - self._position = position or Position() + self._uid: str = uid + self._cmdVars: dict = {} + self._size: int = 0 + self._position: Position = position or Position() self._attributes = DictModel(keyAttrName='name', parent=self) self._internalAttributes = DictModel(keyAttrName='name', parent=self) - self.invalidatingAttributes = set() - self._alive = True # for QML side to know if the node can be used or is going to be deleted - self._locked = False + self.invalidatingAttributes: set = set() + self._alive: bool = True # for QML side to know if the node can be used or is going to be deleted + self._locked: bool = False self._duplicates = ListModel(parent=self) # list of nodes with the same uid - self._hasDuplicates = False + self._hasDuplicates: bool = False self.globalStatusChanged.connect(self.updateDuplicatesStatusAndLocked) From 346d78df302fb2ba8a9676a4874a985e9e632ce9 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sun, 13 Apr 2025 18:50:46 +0200 Subject: [PATCH 41/49] Adapt unittests to deal with graph saving --- meshroom/core/graph.py | 32 +++++++++++++++++++++- meshroom/ui/graph.py | 9 ++---- tests/conftest.py | 17 ++---------- tests/test_nodeAttributeChangedCallback.py | 18 ++++++------ 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 2aa8403f3e..05c511cfaf 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -167,6 +167,19 @@ def inner(self, *args, **kwargs): return inner +def generateTempProjectFilepath(tmpFolder=None): + """ + Generate a temporary project filepath. + This method is used to generate a temporary project file for the current graph. + """ + from datetime import datetime + if tmpFolder is None: + from meshroom.env import EnvVar + tmpFolder = EnvVar.get(EnvVar.MESHROOM_TEMP_PATH) + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M") + return os.path.join(tmpFolder, f"meshroom_{timestamp}.mg") + + class Graph(BaseObject): """ _________________ _________________ _________________ @@ -1316,7 +1329,7 @@ def save(self, filepath=None, setupProjectFile=True, template=False): def _save(self, filepath=None, setupProjectFile=True, template=False): path = filepath or self._filepath if not path: - raise ValueError("filepath must be specified for unsaved files.") + path = generateTempProjectFilepath() data = self.serialize(template) @@ -1329,6 +1342,21 @@ def _save(self, filepath=None, setupProjectFile=True, template=False): # update the file date version self._fileDateVersion = os.path.getmtime(path) + def saveAsTemp(self, tmpFolder=None): + """ + Save the current Meshroom graph as a temporary project file. + """ + # Update the saving flag indicating that the current graph is being saved + self._saving = True + try: + self._saveAsTemp(tmpFolder) + finally: + self._saving = False + + def _saveAsTemp(self, tmpFolder=None): + projectPath = generateTempProjectFilepath(tmpFolder) + self._save(projectPath) + def _setFilepath(self, filepath): """ Set the internal filepath of this Graph. @@ -1594,6 +1622,8 @@ def executeGraph(graph, toNodes=None, forceCompute=False, forceStatus=False): print('Nodes to execute: ', str([n.name for n in nodes])) + graph.save() + for node in nodes: node.beginSequence(forceCompute) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index f2826506c3..754faf3a03 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -24,7 +24,7 @@ from meshroom.core import sessionUid from meshroom.common.qt import QObjectListModel from meshroom.core.attribute import Attribute, ListAttribute -from meshroom.core.graph import Graph, Edge +from meshroom.core.graph import Graph, Edge, generateTempProjectFilepath from meshroom.core.graphIO import GraphIO from meshroom.core.taskManager import TaskManager @@ -516,11 +516,8 @@ def _saveAs(self, url, setupProjectFile=True, template=False): @Slot() def saveAsTemp(self): - from meshroom.env import EnvVar - from datetime import datetime - tempFolder = EnvVar.get(EnvVar.MESHROOM_TEMP_PATH) - timestamp = datetime.now().strftime("%Y-%m-%d_%H:%M") - self._saveAs(os.path.join(tempFolder, f"meshroom_{timestamp}.mg")) + projectPath = generateTempProjectFilepath() + self._saveAs(projectPath) @Slot() def save(self): diff --git a/tests/conftest.py b/tests/conftest.py index 08e4918557..dd8db56249 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,19 +6,6 @@ from meshroom.core.graph import Graph -@pytest.fixture -def graphWithIsolatedCache(): - """ - Yield a Graph instance using a unique temporary cache directory. - - Can be used for testing graph computation in isolation, without having to save the graph to disk. - """ - with tempfile.TemporaryDirectory() as cacheDir: - graph = Graph("") - graph.cacheDir = cacheDir - yield graph - - @pytest.fixture def graphSavedOnDisk(): """ @@ -27,6 +14,6 @@ def graphSavedOnDisk(): Can be used for testing graph IO and computation in isolation. """ with tempfile.TemporaryDirectory() as cacheDir: - graph = Graph("") - graph.save(Path(cacheDir) / "test_graph.mg") + graph = Graph() + graph.saveAsTemp(cacheDir) yield graph diff --git a/tests/test_nodeAttributeChangedCallback.py b/tests/test_nodeAttributeChangedCallback.py index faee0e00ba..1ab088ead7 100644 --- a/tests/test_nodeAttributeChangedCallback.py +++ b/tests/test_nodeAttributeChangedCallback.py @@ -5,7 +5,7 @@ from meshroom.core.node import Node -class NodeWithAttributeChangedCallback(desc.Node): +class NodeWithAttributeChangedCallback(desc.BaseNode): """ A Node containing an input Attribute with an 'on{Attribute}Changed' method, called whenever the value of this attribute is changed explicitly. @@ -182,7 +182,7 @@ def test_loadingGraphDoesNotTriggerCallbackForConnectedAttributes( assert loadedNodeB.affectedInput.value == 2 -class NodeWithCompoundAttributes(desc.Node): +class NodeWithCompoundAttributes(desc.BaseNode): """ A Node containing a variation of compound attributes (List/Groups), called whenever the value of this attribute is changed explicitly. @@ -311,7 +311,7 @@ def test_connectionToListElementInGroup(self): assert nodeB.affectedInput.value == 20 -class NodeWithDynamicOutputValue(desc.Node): +class NodeWithDynamicOutputValue(desc.BaseNode): """ A Node containing an output attribute which value is computed dynamically during graph execution. """ @@ -363,9 +363,9 @@ def test_connectingUncomputedDynamicOutputDoesNotTriggerDownstreamAttributeChang assert nodeB.affectedInput.value == 0 def test_connectingComputedDynamicOutputTriggersDownstreamAttributeChangedCallback( - self, graphWithIsolatedCache + self, graphSavedOnDisk ): - graph: Graph = graphWithIsolatedCache + graph: Graph = graphSavedOnDisk nodeA = graph.addNewNode(NodeWithDynamicOutputValue.__name__) nodeB = graph.addNewNode(NodeWithAttributeChangedCallback.__name__) @@ -377,9 +377,9 @@ def test_connectingComputedDynamicOutputTriggersDownstreamAttributeChangedCallba assert nodeB.affectedInput.value == 40 def test_dynamicOutputValueComputeDoesNotTriggerDownstreamAttributeChangedCallback( - self, graphWithIsolatedCache + self, graphSavedOnDisk ): - graph: Graph = graphWithIsolatedCache + graph: Graph = graphSavedOnDisk nodeA = graph.addNewNode(NodeWithDynamicOutputValue.__name__) nodeB = graph.addNewNode(NodeWithAttributeChangedCallback.__name__) @@ -392,9 +392,9 @@ def test_dynamicOutputValueComputeDoesNotTriggerDownstreamAttributeChangedCallba def test_clearingDynamicOutputValueDoesNotTriggerDownstreamAttributeChangedCallback( - self, graphWithIsolatedCache + self, graphSavedOnDisk ): - graph: Graph = graphWithIsolatedCache + graph: Graph = graphSavedOnDisk nodeA = graph.addNewNode(NodeWithDynamicOutputValue.__name__) nodeB = graph.addNewNode(NodeWithAttributeChangedCallback.__name__) From b9a5c002793b815cc597a699e36f3bd6897bb1cb Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 14 Apr 2025 10:26:36 +0200 Subject: [PATCH 42/49] [core] fix logging Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- meshroom/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 9ce6b8484d..1623ee0e55 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -325,7 +325,7 @@ def unregisterNodeType(nodeType): def loadNodes(folder, packageName): if not os.path.isdir(folder): - logging.error("Node folder '{folder}' does not exist.") + logging.error(f"Node folder '{folder}' does not exist.") return return loadClasses(folder, packageName, desc.BaseNode) From 2ad55352eec05ed91fe3cfd74b410d1f2c5d91c7 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 14 Apr 2025 10:50:57 +0200 Subject: [PATCH 43/49] [core] declaring "global" var access is useless --- meshroom/core/__init__.py | 5 ----- meshroom/core/node.py | 4 +--- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 1623ee0e55..148b6119f4 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -310,7 +310,6 @@ def registerNodeType(nodeType): After registration, nodes of this type can be instantiated in a Graph. """ - global nodesDesc if nodeType.__name__ in nodesDesc: logging.error(f"Node Desc {nodeType.__name__} is already registered.") nodesDesc[nodeType.__name__] = nodeType @@ -318,7 +317,6 @@ def registerNodeType(nodeType): def unregisterNodeType(nodeType): """ Remove 'nodeType' from the list of register node types. """ - global nodesDesc assert nodeType.__name__ in nodesDesc del nodesDesc[nodeType.__name__] @@ -332,7 +330,6 @@ def loadNodes(folder, packageName): def loadAllNodes(folder): - global nodesDesc for importer, package, ispkg in pkgutil.walk_packages([folder]): if ispkg: nodeTypes = loadNodes(folder, package) @@ -372,7 +369,6 @@ def loadPluginsFolder(folder): def registerSubmitter(s): - global submitters if s.name in submitters: logging.error(f"Submitter {s.name} is already registered.") submitters[s.name] = s @@ -387,7 +383,6 @@ def loadSubmitters(folder, packageName): def loadPipelineTemplates(folder): - global pipelineTemplates if not os.path.isdir(folder): logging.error(f"Pipeline templates folder '{folder}' does not exist.") return diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 89b1328f75..27f00e3848 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -298,12 +298,11 @@ def textToLevel(self, text): return logging.NOTSET -runningProcesses = {} +runningProcesses: dict[str, "NodeChunk"] = {} @atexit.register def clearProcessesStatus(): - global runningProcesses for k, v in runningProcesses.items(): v.upgradeStatusTo(Status.KILLED) @@ -474,7 +473,6 @@ def process(self, forceCompute=False, inCurrentEnv=False): self._processInIsolatedEnvironment() return - global runningProcesses runningProcesses[self.name] = self self._status.setNode(self.node) self._status.initStartCompute() From 8be8ea570370805279416e235bb51c5834e84e95 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 14 Apr 2025 19:03:12 +0200 Subject: [PATCH 44/49] Fix display of Compatibility nodes "isComputable" is renamed as "isComputableType": this function is only about the Meshroom Node type and not about the computability in the current context. Even if we are in compatibility mode, we may has access to the nodeDesc and its information about the node type. --- meshroom/core/node.py | 23 +++++++++++----- meshroom/core/taskManager.py | 2 +- meshroom/ui/qml/GraphEditor/GraphEditor.qml | 19 ++++++------- meshroom/ui/qml/GraphEditor/Node.qml | 30 ++++++++++----------- meshroom/ui/qml/GraphEditor/NodeEditor.qml | 14 +++++----- meshroom/ui/qml/Viewer/Viewer2D.qml | 4 +-- 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 27f00e3848..d8722de5a7 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -652,7 +652,8 @@ def __getattr__(self, k): raise e def getMrNodeType(self): - if self.isCompatibilityNode: + # In compatibility mode, we may or may not have access to the nodeDesc and its information about the node type. + if self.nodeDesc is None: return MrNodeType.NONE return self.nodeDesc.getMrNodeType() @@ -960,12 +961,16 @@ def hasStatus(self, status): return True def _isComputed(self): - if not self.isComputable: + if not self.isComputableType: return True return self.hasStatus(Status.SUCCESS) - def _isComputable(self): - return self.getGlobalStatus() != Status.INPUT + def _isComputableType(self): + """ Return True if this node type is computable, False otherwise. + A computable node type can be in a context that does not allow computation. + """ + # Ambiguous case for NONE, which could be used for compatibility nodes if we don't have any information about the node descriptor. + return self.getMrNodeType() != MrNodeType.INPUT def clearData(self): """ Delete this Node internal folder. @@ -1511,7 +1516,9 @@ def isMainNode(self) -> bool: @Slot(result=bool) def canBeStopped(self) -> bool: - if not self.isComputable: + if not self.isComputableType: + return False + if self.isCompatibilityNode: return False # Only locked nodes running in local with the same # sessionUid as the Meshroom instance can be stopped @@ -1522,7 +1529,9 @@ def canBeStopped(self) -> bool: @Slot(result=bool) def canBeCanceled(self) -> bool: - if not self.isComputable: + if not self.isComputableType: + return False + if self.isCompatibilityNode: return False # Only locked nodes submitted in local with the same # sessionUid as the Meshroom instance can be canceled @@ -1610,7 +1619,7 @@ def has3DOutputAttribute(self): globalExecMode = Property(str, globalExecMode.fget, notify=globalStatusChanged) isExternal = Property(bool, isExtern, notify=globalStatusChanged) isComputed = Property(bool, _isComputed, notify=globalStatusChanged) - isComputable = Property(bool, _isComputable, notify=globalStatusChanged) + isComputableType = Property(bool, _isComputableType, notify=globalStatusChanged) aliveChanged = Signal() alive = Property(bool, alive.fget, alive.fset, notify=aliveChanged) lockedChanged = Signal() diff --git a/meshroom/core/taskManager.py b/meshroom/core/taskManager.py index dc7c41dcff..7d9c52b1fa 100644 --- a/meshroom/core/taskManager.py +++ b/meshroom/core/taskManager.py @@ -339,7 +339,7 @@ def checkNodesDependencies(self, graph, toNodes, context): computed = [] inputNodes = [] for node in toNodes: - if not node.isComputable: + if not node.isComputableType: inputNodes.append(node) elif context == "COMPUTATION": if graph.canComputeTopologically(node) and graph.canSubmitOrCompute(node) % 2 == 1: diff --git a/meshroom/ui/qml/GraphEditor/GraphEditor.qml b/meshroom/ui/qml/GraphEditor/GraphEditor.qml index e9a827cbf6..b3b402ddbd 100755 --- a/meshroom/ui/qml/GraphEditor/GraphEditor.qml +++ b/meshroom/ui/qml/GraphEditor/GraphEditor.qml @@ -577,7 +577,7 @@ Item { readonly property bool selectionContainsComputableNodes: { return uigraph.nodeSelection.selectedIndexes.some(function(idx) { const node = uigraph.graph.nodes.at(idx.row); - return node.isComputable; + return node.isComputableType; }); } @@ -608,9 +608,10 @@ Item { return uigraph.nodeSelection.selectedIndexes.every(function(idx) { const node = uigraph.graph.nodes.at(idx.row); return ( - node.isComputed - // canSubmit if canSubmitOrCompute == 2(can submit) or 3(can compute & submit) - || nodeSubmitOrComputeStatus[node] > 1 + node.isComputed || + (uigraph.graph.canComputeTopologically(node) && + // canSubmit if canSubmitOrCompute == 2(can submit) or 3(can compute & submit) + nodeSubmitOrComputeStatus[node] > 1) ) }); } @@ -622,7 +623,7 @@ Item { MenuItem { id: computeMenuItem - text: nodeMenu.isSelectionFullyComputed ? "Recompute" : "Compute" + text: nodeMenu.isSelectionFullyComputed ? "Re-Compute" : "Compute" visible: nodeMenu.selectionContainsComputableNodes height: visible ? implicitHeight : 0 enabled: nodeMenu.canSelectionBeComputed @@ -677,12 +678,12 @@ Item { } MenuItem { text: "Open Folder" - visible: nodeMenu.currentNode.isComputable + visible: nodeMenu.currentNode.isComputableType height: visible ? implicitHeight : 0 onTriggered: Qt.openUrlExternally(Filepath.stringToUrl(nodeMenu.currentNode.internalFolder)) } MenuSeparator { - visible: nodeMenu.currentNode.isComputable + visible: nodeMenu.currentNode.isComputableType } MenuItem { text: "Cut Node(s)" @@ -748,12 +749,12 @@ Item { } } MenuSeparator { - visible: nodeMenu.currentNode.isComputable + visible: nodeMenu.currentNode.isComputableType } MenuItem { id: deleteDataMenuItem text: "Delete Data" + (deleteFollowingButton.hovered ? " From Here" : "" ) + "..." - visible: nodeMenu.currentNode.isComputable + visible: nodeMenu.currentNode.isComputableType height: visible ? implicitHeight : 0 enabled: { if (!nodeMenu.currentNode) diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 9e0d054d72..f2d755247a 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -28,7 +28,7 @@ Item { property point position: Qt.point(x, y) /// Styling property color shadowColor: "#cc000000" - readonly property color defaultColor: isCompatibilityNode ? "#444" : !node.isComputable ? "#BA3D69" : activePalette.base + readonly property color defaultColor: isCompatibilityNode ? "#444" : !node.isComputableType ? "#BA3D69" : activePalette.base property color baseColor: defaultColor property point mousePosition: Qt.point(mouseArea.mouseX, mouseArea.mouseY) @@ -327,7 +327,7 @@ Item { text: MaterialIcons.visibility padding: 2 font.pointSize: 7 - property bool displayable: !node.isComputable || (node.chunks.count > 0 && (["SUCCESS"].includes(node.globalStatus))) + property bool displayable: !node.isComputableType || (node.chunks.count > 0 && (["SUCCESS"].includes(node.globalStatus))) color: displayable ? palette.text : Qt.darker(palette.text, 1.8) ToolTip { @@ -363,19 +363,19 @@ Item { } // Node Chunks - NodeChunks { - visible: node.isComputable - defaultColor: Colors.sysPalette.mid - implicitHeight: 3 - width: parent.width - model: node ? node.chunks : undefined - - Rectangle { - anchors.fill: parent - color: Colors.sysPalette.mid - z: -1 - } - } + NodeChunks { + visible: node.isComputableType + defaultColor: Colors.sysPalette.mid + implicitHeight: 3 + width: parent.width + model: node ? node.chunks : undefined + + Rectangle { + anchors.fill: parent + color: Colors.sysPalette.mid + z: -1 + } + } // Vertical Spacer Item { width: parent.width; height: 2 } diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index a208d22ee1..c6ddffbafa 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -53,7 +53,7 @@ Panel { headerBar: RowLayout { Label { id: computationInfo - color: node && node.isComputable ? Colors.statusColors[node.globalStatus] : palette.text + color: node && node.isComputableType ? Colors.statusColors[node.globalStatus] : palette.text Timer { id: timer interval: 2500 @@ -72,7 +72,7 @@ Panel { font.italic: true visible: { if (node !== null) { - if (node.isComputable && (node.isFinishedOrRunning() || node.isSubmittedOrRunning() || node.globalStatus=="ERROR")) { + if (node.isComputableType && (node.isFinishedOrRunning() || node.isSubmittedOrRunning() || node.globalStatus=="ERROR")) { return true } } @@ -378,7 +378,7 @@ Panel { id: tabBar visible: root.node !== null - property bool isComputable: root.node !== null && root.node.isComputable + property bool isComputableType: root.node !== null && root.node.isComputableType // The indices of the tab bar which can be shown for incomputable nodes readonly property var nonComputableTabIndices: [0, 4, 5]; @@ -394,21 +394,21 @@ Panel { rightPadding: leftPadding } TabButton { - visible: tabBar.isComputable + visible: tabBar.isComputableType width: !visible ? 0 : tabBar.width / tabBar.count text: "Log" leftPadding: 8 rightPadding: leftPadding } TabButton { - visible: tabBar.isComputable + visible: tabBar.isComputableType width: !visible ? 0 : tabBar.width / tabBar.count text: "Statistics" leftPadding: 8 rightPadding: leftPadding } TabButton { - visible: tabBar.isComputable + visible: tabBar.isComputableType width: !visible ? 0 : tabBar.width / tabBar.count text: "Status" leftPadding: 8 @@ -429,7 +429,7 @@ Panel { onVisibleChanged: { // If we have a node selected and the node is not Computable // Reset the currentIndex to 0, if the current index is not allowed for an incomputable node - if ((root.node && !root.node.isComputable) && (nonComputableTabIndices.indexOf(tabBar.currentIndex) === -1)) { + if ((root.node && !root.node.isComputableType) && (nonComputableTabIndices.indexOf(tabBar.currentIndex) === -1)) { tabBar.currentIndex = 0; } } diff --git a/meshroom/ui/qml/Viewer/Viewer2D.qml b/meshroom/ui/qml/Viewer/Viewer2D.qml index 4876109070..48a672e76c 100644 --- a/meshroom/ui/qml/Viewer/Viewer2D.qml +++ b/meshroom/ui/qml/Viewer/Viewer2D.qml @@ -198,7 +198,7 @@ FocusScope { } // Node must be computed or at least running - if (node.isComputable && !node.isPartiallyFinished()) { + if (node.isComputableType && !node.isPartiallyFinished()) { return false } @@ -361,7 +361,7 @@ FocusScope { } } - if (!displayedNode || displayedNode.isComputable) + if (!displayedNode || displayedNode.isComputableType) names.push("gallery") outputAttribute.names = names From be43c5c2a7f3ca75da92cabbbcd9f24df39925a0 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 14 Apr 2025 22:28:08 +0200 Subject: [PATCH 45/49] [ui] Compatibility nodes cannot be computed or recomputed When the compatibility node was fully computed, the recompute/resubmit button was available in the UI menu. The reason is that we need to propagate that the connected nodes can be computed if there is a compatibility node in the dependency iff it is already fully computed. Now there is an additional explicit check. --- meshroom/ui/qml/GraphEditor/GraphEditor.qml | 26 ++++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/meshroom/ui/qml/GraphEditor/GraphEditor.qml b/meshroom/ui/qml/GraphEditor/GraphEditor.qml index b3b402ddbd..4d90dbf464 100755 --- a/meshroom/ui/qml/GraphEditor/GraphEditor.qml +++ b/meshroom/ui/qml/GraphEditor/GraphEditor.qml @@ -570,11 +570,21 @@ Item { readonly property bool isSelectionFullyComputed: { return uigraph.nodeSelection.selectedIndexes.every(function(idx) { - return uigraph.graph.nodes.at(idx.row).isComputed; + const node = uigraph.graph.nodes.at(idx.row); + return node.isComputed; + }); + } + + // Selection contains only compatibility nodes + readonly property bool isSelectionFullyCompatibility: { + return uigraph.nodeSelection.selectedIndexes.every(function(idx) { + const node = uigraph.graph.nodes.at(idx.row); + return node.isCompatibilityNode; }); } - readonly property bool selectionContainsComputableNodes: { + // Selection contains at least one computable node type + readonly property bool selectionContainsComputableNodeType: { return uigraph.nodeSelection.selectedIndexes.some(function(idx) { const node = uigraph.graph.nodes.at(idx.row); return node.isComputableType; @@ -582,7 +592,9 @@ Item { } readonly property bool canSelectionBeComputed: { - if(!selectionContainsComputableNodes) + if(!selectionContainsComputableNodeType) + return false; + if(isSelectionFullyCompatibility) return false; if(isSelectionFullyComputed) return true; @@ -598,10 +610,12 @@ Item { return b } - readonly property bool isSelectionSubmitable: uigraph.canSubmit && selectionContainsComputableNodes + readonly property bool isSelectionSubmitable: uigraph.canSubmit && selectionContainsComputableNodeType readonly property bool canSelectionBeSubmitted: { - if(!selectionContainsComputableNodes) + if(!selectionContainsComputableNodeType) + return false; + if(isSelectionFullyCompatibility) return false; if(isSelectionFullyComputed) return true; @@ -624,7 +638,7 @@ Item { MenuItem { id: computeMenuItem text: nodeMenu.isSelectionFullyComputed ? "Re-Compute" : "Compute" - visible: nodeMenu.selectionContainsComputableNodes + visible: nodeMenu.selectionContainsComputableNodeType height: visible ? implicitHeight : 0 enabled: nodeMenu.canSelectionBeComputed From ff9a0370ac95e7bf8bceacdee46ee337cd1cccbc Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Mon, 14 Apr 2025 22:45:47 +0200 Subject: [PATCH 46/49] Fix node states after loading The states of nodes were not valid the first time they were loaded, but became valid after any topological update (such as creating or removing a node or edge). The reason was that the loading of the graph was done before setting the project path (and thus the cacheDir). So the status files of the nodes were not available during the graph creation and were not updated at the end of the load. Now we set the filepath/cache first, so everything is right from the start. --- meshroom/core/graph.py | 6 ++++-- meshroom/core/node.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 05c511cfaf..8197602d17 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -252,8 +252,8 @@ def load(self, filepath: PathLike): Args: filepath: The path to the Meshroom Graph file to load. """ - self._deserialize(Graph._loadGraphData(filepath)) self._setFilepath(filepath) + self._deserialize(Graph._loadGraphData(filepath)) self._fileDateVersion = os.path.getmtime(filepath) def initFromTemplate(self, filepath: PathLike, publishOutputs: bool = False): @@ -293,7 +293,9 @@ def _deserialize(self, graphData: dict): Args: graphData: The serialized Graph. """ - self.clear() + self._clearGraphContent() + self.header.clear() + self.header = graphData.get(GraphIO.Keys.Header, {}) fileVersion = Version(self.header.get(GraphIO.Keys.FileVersion, "0.0")) graphContent = self._normalizeGraphContent(graphData, fileVersion) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index d8722de5a7..dc8fcd225d 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -952,7 +952,7 @@ def isParallelized(self): def nbParallelizationBlocks(self): return len(self._chunks) - def hasStatus(self, status): + def hasStatus(self, status: Status): if not self._chunks: return (status == Status.INPUT) for chunk in self._chunks: From 081d38f78d42fd9422a3f0227e5e6d6e03afd990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 28 Apr 2025 17:28:10 +0200 Subject: [PATCH 47/49] Stop using bare `except` statements --- bin/meshroom_compute | 2 +- meshroom/core/desc/node.py | 3 +-- meshroom/core/node.py | 10 +++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/meshroom_compute b/bin/meshroom_compute index 8d15e7e585..5450ad0b22 100755 --- a/bin/meshroom_compute +++ b/bin/meshroom_compute @@ -6,7 +6,7 @@ import sys try: import meshroom -except: +except Exception: # If meshroom module is not in the PYTHONPATH, add our root using the relative path import pathlib meshroomRootFolder = pathlib.Path(__file__).parent.parent.resolve() diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index e524037edf..910b8df6f6 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -185,7 +185,7 @@ def executeChunkCommandLine(self, chunk, cmd, env=None): try: status = chunk.subprocess.status() logF.write(f"Process status: {status}") - except: + except Exception: pass if chunk.subprocess.returncode != 0: @@ -372,4 +372,3 @@ def setAttributes(self, node, attributesDict): for attr in attributesDict: if node.hasAttribute(attr): node.attribute(attr).value = attributesDict[attr] - diff --git a/meshroom/core/node.py b/meshroom/core/node.py index dc8fcd225d..8c40fdf44a 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -509,18 +509,22 @@ def process(self, forceCompute=False, inCurrentEnv=False): def _processInIsolatedEnvironment(self): - """Process this node chunk in the isolated environment defined in the environment configuration.""" + """ + Process this node chunk in the isolated environment defined in the environment + configuration. + """ try: self._status.setNode(self.node) self._status.initIsolatedCompute() self.upgradeStatusFile() self.node.nodeDesc.processChunkInEnvironment(self) - except: + except Exception: # status should be already updated by meshroom_compute self.updateStatusFromCache() if self._status.status not in (Status.ERROR, Status.STOPPED, Status.KILLED): - # If meshroom_compute has crashed or been killed, the status may have not been set to ERROR. + # If meshroom_compute has crashed or been killed, the status may have not been + # set to ERROR. # In this particular case, we enforce it from here. self.upgradeStatusTo(Status.ERROR) raise From 8300626ef55e93c7808f5fc47963d86025b18f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 28 Apr 2025 18:24:08 +0200 Subject: [PATCH 48/49] [core] Node: Clean-up code --- meshroom/core/node.py | 237 +++++++++++++++++++++++++----------------- 1 file changed, 143 insertions(+), 94 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 8c40fdf44a..4deee8dd70 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -56,6 +56,8 @@ class Status(Enum): class ExecMode(Enum): + """ + """ NONE = auto() LOCAL = auto() EXTERN = auto() @@ -66,7 +68,8 @@ class StatusData(BaseObject): """ dateTimeFormatting = '%Y-%m-%d %H:%M:%S.%f' - def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', mrNodeType: MrNodeType = MrNodeType.NONE, parent: BaseObject = None): + def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', + mrNodeType: MrNodeType = MrNodeType.NONE, parent: BaseObject = None): super(StatusData, self).__init__(parent) self.nodeName: str = nodeName @@ -81,12 +84,13 @@ def __init__(self, nodeName='', nodeType='', packageName='', packageVersion='', self.resetDynamicValues() def setNode(self, node): - """ Set the node information from one node instance.""" + """ Set the node information from one node instance. """ self.nodeName = node.name self.setNodeType(node) def setNodeType(self, node): - """ Set the node type and package information from the given node. + """ + Set the node type and package information from the given node. We do not set the name in this method as it may vary if there are duplicates. """ self.nodeType = node.nodeType @@ -132,17 +136,21 @@ def initStartCompute(self): # we don't want to modify the execMode set from the submit. def initIsolatedCompute(self): - ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. - ''' + """ + When submitting a node, we reset the status information to ensure that we do not keep + outdated information. + """ self.resetDynamicValues() self.initStartCompute() - assert(self.mrNodeType == MrNodeType.NODE) + assert self.mrNodeType == MrNodeType.NODE self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid def initExternSubmit(self): - ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. - ''' + """ + When submitting a node, we reset the status information to ensure that we do not keep + outdated information. + """ self.resetDynamicValues() self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid @@ -150,8 +158,10 @@ def initExternSubmit(self): self.execMode = ExecMode.EXTERN def initLocalSubmit(self): - ''' When submitting a node, we reset the status information to ensure that we do not keep outdated information. - ''' + """ + When submitting a node, we reset the status information to ensure that we do not keep + outdated information. + """ self.resetDynamicValues() self.sessionUid = None self.submitterSessionUid = meshroom.core.sessionUid @@ -181,31 +191,31 @@ def toDict(self): return d def fromDict(self, d): - self.status = d.get('status', Status.NONE) + self.status = d.get("status", Status.NONE) if not isinstance(self.status, Status): self.status = Status[self.status] - self.execMode = d.get('execMode', ExecMode.NONE) + self.execMode = d.get("execMode", ExecMode.NONE) if not isinstance(self.execMode, ExecMode): self.execMode = ExecMode[self.execMode] - self.mrNodeType = d.get('mrNodeType', MrNodeType.NONE) + self.mrNodeType = d.get("mrNodeType", MrNodeType.NONE) if not isinstance(self.mrNodeType, MrNodeType): self.mrNodeType = MrNodeType[self.mrNodeType] - self.nodeName = d.get('nodeName', '') - self.nodeType = d.get('nodeType', '') - self.packageName = d.get('packageName', '') - self.packageVersion = d.get('packageVersion', '') - self.graph = d.get('graph', '') - self.commandLine = d.get('commandLine', '') - self.env = d.get('env', '') - self.startDateTime = d.get('startDateTime', '') - self.endDateTime = d.get('endDateTime', '') - self.elapsedTime = d.get('elapsedTime', 0) - self.hostname = d.get('hostname', '') - self.sessionUid = d.get('sessionUid', '') - self.submitterSessionUid = d.get('submitterSessionUid', '') - - + self.nodeName = d.get("nodeName", "") + self.nodeType = d.get("nodeType", "") + self.packageName = d.get("packageName", "") + self.packageVersion = d.get("packageVersion", "") + self.graph = d.get("graph", "") + self.commandLine = d.get("commandLine", "") + self.env = d.get("env", "") + self.startDateTime = d.get("startDateTime", "") + self.endDateTime = d.get("endDateTime", "") + self.elapsedTime = d.get("elapsedTime", 0) + self.hostname = d.get("hostname", "") + self.sessionUid = d.get("sessionUid", "") + self.submitterSessionUid = d.get("submitterSessionUid", "") + + class LogManager: dateTimeFormatting = '%H:%M:%S' @@ -223,7 +233,8 @@ def configureLogger(self): for handler in self.logger.handlers[:]: self.logger.removeHandler(handler) handler = logging.FileHandler(self.chunk.logFile) - formatter = self.Formatter('[%(asctime)s.%(msecs)03d][%(levelname)s] %(message)s', self.dateTimeFormatting) + formatter = self.Formatter('[%(asctime)s.%(msecs)03d][%(levelname)s] %(message)s', + self.dateTimeFormatting) handler.setFormatter(formatter) self.logger.addHandler(handler) @@ -284,15 +295,15 @@ def completeProgressBar(self): self.progressBar = False def textToLevel(self, text): - if text == 'critical': + if text == "critical": return logging.CRITICAL - elif text == 'error': + elif text == "error": return logging.ERROR - elif text == 'warning': + elif text == "warning": return logging.WARNING - elif text == 'info': + elif text == "info": return logging.INFO - elif text == 'debug': + elif text == "debug": return logging.DEBUG else: return logging.NOTSET @@ -313,7 +324,8 @@ def __init__(self, node, range, parent=None): self.node = node self.range = range self.logManager: LogManager = LogManager(self) - self._status: StatusData = StatusData(node.name, node.nodeType, node.packageName, node.packageVersion, node.getMrNodeType()) + self._status: StatusData = StatusData(node.name, node.nodeType, node.packageName, + node.packageVersion, node.getMrNodeType()) self.statistics: stats.Statistics = stats.Statistics() self.statusFileLastModTime = -1 self.subprocess = None @@ -375,21 +387,24 @@ def statusFile(self): if self.range.blockSize == 0: return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "status") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".status") + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, + str(self.index) + ".status") @property def statisticsFile(self): if self.range.blockSize == 0: return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "statistics") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".statistics") + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, + str(self.index) + ".statistics") @property def logFile(self): if self.range.blockSize == 0: return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, "log") else: - return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, str(self.index) + ".log") + return os.path.join(self.node.graph.cacheDir, self.node.internalFolder, + str(self.index) + ".log") def saveStatusFile(self): """ @@ -406,7 +421,8 @@ def saveStatusFile(self): renameWritingToFinalPath(statusFilepathWriting, statusFilepath) def upgradeStatusFile(self): - """ Upgrade node status file based on the current status. + """ + Upgrade node status file based on the current status. """ self.saveStatusFile() self.statusChanged.emit() @@ -465,10 +481,11 @@ def process(self, forceCompute=False, inCurrentEnv=False): if not forceCompute and self._status.status == Status.SUCCESS: logging.info("Node chunk already computed: {}".format(self.name)) return - + # Start the process environment for nodes running in isolation. # This only happens once, when the node has the SUBMITTED status. - # The sub-process will go through this method again, but the node status will have been set to RUNNING. + # The sub-process will go through this method again, but the node status will + # have been set to RUNNING. if not inCurrentEnv and self.node.getMrNodeType() == MrNodeType.NODE: self._processInIsolatedEnvironment() return @@ -541,12 +558,14 @@ def stopProcess(self): self.updateStatusFromCache() if self._status.status != Status.RUNNING: - # When we stop the process of a node with multiple chunks, the Node function will call the stop function of each chunk. - # So, the chunck status could be SUBMITTED, RUNNING or ERROR. + # When we stop the process of a node with multiple chunks, the Node function will call + # the stop function of each chunk. + # So, the chunk status could be SUBMITTED, RUNNING or ERROR. if self._status.status is Status.SUBMITTED: self.upgradeStatusTo(Status.NONE) - elif self._status.status in (Status.ERROR, Status.STOPPED, Status.KILLED, Status.SUCCESS, Status.NONE): + elif self._status.status in (Status.ERROR, Status.STOPPED, Status.KILLED, + Status.SUCCESS, Status.NONE): # Nothing to do, the computation is already stopped. pass else: @@ -560,8 +579,10 @@ def stopProcess(self): self.upgradeStatusTo(Status.STOPPED) def isExtern(self): - """ The computation is managed externally by another instance of Meshroom. - In the ambiguous case of an isolated environment, it is considered as local as we can stop it (if it is run from the current Meshroom instance). + """ + The computation is managed externally by another instance of Meshroom. + In the ambiguous case of an isolated environment, it is considered as local as we can stop + it (if it is run from the current Meshroom instance). """ if self._status.execMode == ExecMode.EXTERN: return True @@ -603,7 +624,8 @@ class BaseNode(BaseObject): # i.e: a.b, a[0], a[0].b.c[1] attributeRE = re.compile(r'\.?(?P\w+)(?:\[(?P\d+)\])?') - def __init__(self, nodeType: str, position: Position = None, parent: BaseObject = None, uid: str = None, **kwargs): + def __init__(self, nodeType: str, position: Position = None, parent: BaseObject = None, + uid: str = None, **kwargs): """ Create a new Node instance based on the given node description. Any other keyword argument will be used to initialize this node's attributes. @@ -656,7 +678,8 @@ def __getattr__(self, k): raise e def getMrNodeType(self): - # In compatibility mode, we may or may not have access to the nodeDesc and its information about the node type. + # In compatibility mode, we may or may not have access to the nodeDesc and its information + # about the node type. if self.nodeDesc is None: return MrNodeType.NONE return self.nodeDesc.getMrNodeType() @@ -826,22 +849,25 @@ def valuesFile(self): return os.path.join(self.graph.cacheDir, self.internalFolder, 'values') def getInputNodes(self, recursive, dependenciesOnly): - return self.graph.getInputNodes(self, recursive=recursive, dependenciesOnly=dependenciesOnly) + return self.graph.getInputNodes(self, recursive=recursive, + dependenciesOnly=dependenciesOnly) def getOutputNodes(self, recursive, dependenciesOnly): - return self.graph.getOutputNodes(self, recursive=recursive, dependenciesOnly=dependenciesOnly) + return self.graph.getOutputNodes(self, recursive=recursive, + dependenciesOnly=dependenciesOnly) def toDict(self): pass def _computeUid(self): """ Compute node UID by combining associated attributes' UIDs. """ - # If there is no invalidating attribute, then the computation of the UID should not go through as - # it will only include the node type + # If there is no invalidating attribute, then the computation of the UID should not + # go through as it will only include the node type if not self.invalidatingAttributes: return - # UID is computed by hashing the sorted list of tuple (name, value) of all attributes impacting this UID + # UID is computed by hashing the sorted list of tuple (name, value) of all attributes + # impacting this UID uidAttributes = [] for attr in self.invalidatingAttributes: if not attr.enabled: @@ -862,6 +888,10 @@ def _computeUid(self): self._uid = hashValue(uidAttributes) def _buildCmdVars(self): + """ + Generate command variables using input attributes and resolved output attributes + names and values. + """ def _buildAttributeCmdVars(cmdVars, name, attr): if attr.enabled: group = attr.attributeDesc.group(attr.node) \ @@ -886,7 +916,6 @@ def _buildAttributeCmdVars(cmdVars, name, attr): for v in attr._value: _buildAttributeCmdVars(cmdVars, v.name, v) - """ Generate command variables using input attributes and resolved output attributes names and values. """ self._cmdVars["uid"] = self._uid self._cmdVars["nodeCacheFolder"] = self.internalFolder self._cmdVars["nodeSourceCodeFolder"] = self.sourceCodeFolder @@ -901,8 +930,9 @@ def _buildAttributeCmdVars(cmdVars, name, attr): cmdVarsNoCache = self._cmdVars.copy() cmdVarsNoCache["cache"] = "" - # Use "self._internalFolder" instead of "self.internalFolder" because we do not want it to be - # resolved with the {cache} information ("self.internalFolder" resolves "self._internalFolder") + # Use "self._internalFolder" instead of "self.internalFolder" because we do not want it to + # be resolved with the {cache} information ("self.internalFolder" resolves + # "self._internalFolder") cmdVarsNoCache["nodeCacheFolder"] = self._internalFolder.format(**cmdVarsNoCache) # Evaluate output params @@ -919,8 +949,8 @@ def _buildAttributeCmdVars(cmdVars, name, attr): try: defaultValue = attr.defaultValue() except AttributeError: - # If we load an old scene, the lambda associated to the 'value' could try to access other - # params that could not exist yet + # If we load an old scene, the lambda associated to the 'value' could try to + # access other params that could not exist yet logging.warning('Invalid lambda evaluation for "{nodeName}.{attrName}"'. format(nodeName=self.name, attrName=attr.name)) if defaultValue is not None: @@ -945,8 +975,8 @@ def _buildAttributeCmdVars(cmdVars, name, attr): self._cmdVars[name + 'Value'] = attr.getValueStr(withQuotes=False) if v: - self._cmdVars[attr.attributeDesc.group] = self._cmdVars.get(attr.attributeDesc.group, '') + \ - ' ' + self._cmdVars[name] + self._cmdVars[attr.attributeDesc.group] = \ + self._cmdVars.get(attr.attributeDesc.group, '') + ' ' + self._cmdVars[name] @property def isParallelized(self): @@ -958,7 +988,7 @@ def nbParallelizationBlocks(self): def hasStatus(self, status: Status): if not self._chunks: - return (status == Status.INPUT) + return status == Status.INPUT for chunk in self._chunks: if chunk.status.status != status: return False @@ -973,7 +1003,8 @@ def _isComputableType(self): """ Return True if this node type is computable, False otherwise. A computable node type can be in a context that does not allow computation. """ - # Ambiguous case for NONE, which could be used for compatibility nodes if we don't have any information about the node descriptor. + # Ambiguous case for NONE, which could be used for compatibility nodes if we don't have + # any information about the node descriptor. return self.getMrNodeType() != MrNodeType.INPUT def clearData(self): @@ -984,9 +1015,11 @@ def clearData(self): try: shutil.rmtree(self.internalFolder) except Exception as e: - # We could get some "Device or resource busy" on .nfs file while removing the folder on linux network. - # On windows, some output files may be open for visualization and the removal will fail. - # On both cases, we can ignore it. + # We could get some "Device or resource busy" on .nfs file while removing the folder + # on Linux network. + # On Windows, some output files may be open for visualization and the removal will + # fail. + # In both cases, we can ignore it. logging.warning(f"Failed to remove internal folder: '{self.internalFolder}'. Error: {e}.") self.updateStatusFromCache() @@ -1011,7 +1044,10 @@ def isAlreadySubmittedOrFinished(self): @Slot(result=bool) def isSubmittedOrRunning(self): - """ Return True if all chunks are at least submitted and there is one running chunk, False otherwise. """ + """ + Return True if all chunks are at least submitted and there is one running chunk, + False otherwise. + """ if not self.isAlreadySubmittedOrFinished(): return False for chunk in self._chunks: @@ -1026,7 +1062,10 @@ def isRunning(self): @Slot(result=bool) def isFinishedOrRunning(self): - """ Return True if all chunks of this Node is either finished or running, False otherwise. """ + """ + Return True if all chunks of this Node is either finished or running, False + otherwise. + """ return all(chunk.isFinishedOrRunning() for chunk in self._chunks) @Slot(result=bool) @@ -1038,12 +1077,15 @@ def alreadySubmittedChunks(self): return [ch for ch in self._chunks if ch.isAlreadySubmitted()] def isExtern(self): - """ Return True if at least one chunk of this Node has an external execution mode, False otherwise. + """ + Return True if at least one chunk of this Node has an external execution mode, + False otherwise. - It is not enough to check whether the first chunk's execution mode is external, because computations - may have been started locally, interrupted, and restarted externally. In that case, if the first - chunk has completed locally before the computations were interrupted, its execution mode will always - be local, even if computations resume externally. + It is not enough to check whether the first chunk's execution mode is external, + because computations may have been started locally, interrupted, and restarted externally. + In that case, if the first chunk has completed locally before the computations were + interrupted, its execution mode will always be local, even if computations resume + externally. """ if len(self._chunks) == 0: return False @@ -1051,8 +1093,9 @@ def isExtern(self): @Slot() def clearSubmittedChunks(self): - """ Reset all submitted chunks to Status.NONE. This method should be used to clear inconsistent status - if a computation failed without informing the graph. + """ + Reset all submitted chunks to Status.NONE. This method should be used to clear + inconsistent status if a computation failed without informing the graph. Warnings: This must be used with caution. This could lead to inconsistent node status @@ -1069,9 +1112,7 @@ def clearLocallySubmittedChunks(self): chunk.upgradeStatusTo(Status.NONE, ExecMode.NONE) def upgradeStatusTo(self, newStatus): - """ - Upgrade node to the given status and save it on disk. - """ + """ Upgrade node to the given status and save it on disk. """ for chunk in self._chunks: chunk.upgradeStatusTo(newStatus) @@ -1083,7 +1124,7 @@ def _updateChunks(self): pass def _getAttributeChangedCallback(self, attr: Attribute) -> Optional[Callable]: - """Get the node descriptor-defined value changed callback associated to `attr` if any.""" + """ Get the node descriptor-defined value changed callback associated to `attr` if any. """ # Callbacks cannot be defined on nested attributes. if attr.root is not None: @@ -1097,7 +1138,8 @@ def _getAttributeChangedCallback(self, attr: Attribute) -> Optional[Callable]: def _onAttributeChanged(self, attr: Attribute): """ - When an attribute value has changed, a specific function can be defined in the descriptor and be called. + When an attribute value has changed, a specific function can be defined in the descriptor + and be called. Args: attr: The Attribute that has changed. @@ -1116,7 +1158,7 @@ def _onAttributeChanged(self, attr: Attribute): if attr.value is None: # Discard dynamic values depending on the graph processing. return - + if self.graph and self.graph.isLoading: # Do not trigger attribute callbacks during the graph loading. return @@ -1132,7 +1174,9 @@ def _onAttributeChanged(self, attr: Attribute): edge.dst.valueChanged.emit() def onAttributeClicked(self, attr): - """ When an attribute is clicked, a specific function can be defined in the descriptor and be called. + """ + When an attribute is clicked, a specific function can be defined in the descriptor + and be called. Args: attr (Attribute): attribute that has been clicked @@ -1230,7 +1274,8 @@ def process(self, forceCompute=False, inCurrentEnv=False): chunk.process(forceCompute, inCurrentEnv) def postprocess(self): - # Invoke the post process on Client Node to execute after the processing on the node is completed + # Invoke the post process on Client Node to execute after the processing on the + # node is completed self.nodeDesc.postprocess(self) def updateOutputAttr(self): @@ -1499,7 +1544,7 @@ def submitterStatusInThisSession(self) -> bool: if chunk.status.submitterSessionUid != meshroom.core.sessionUid: return False return True - + def initFromThisSession(self) -> bool: if len(self._chunks) == 0: return False @@ -1507,7 +1552,7 @@ def initFromThisSession(self) -> bool: if meshroom.core.sessionUid not in (chunk.status.sessionUid, chunk.status.submitterSessionUid): return False return True - + def isMainNode(self) -> bool: """ In case of a node with duplicates, we check that the node is the one driving the computation. """ if len(self._chunks) == 0: @@ -1546,8 +1591,8 @@ def canBeCanceled(self) -> bool: def hasImageOutputAttribute(self) -> bool: """ - Return True if at least one attribute has the 'image' semantic (and can thus be loaded in the 2D Viewer), - False otherwise. + Return True if at least one attribute has the 'image' semantic (and can thus be loaded in + the 2D Viewer), False otherwise. """ for attr in self._attributes: if not attr.enabled or not attr.isOutput: @@ -1558,8 +1603,8 @@ def hasImageOutputAttribute(self) -> bool: def hasSequenceOutputAttribute(self) -> bool: """ - Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in the 2D Viewer), - False otherwise. + Return True if at least one attribute has the 'sequence' semantic (and can thus be loaded in + the 2D Viewer), False otherwise. """ for attr in self._attributes: if not attr.enabled or not attr.isOutput: @@ -1570,7 +1615,8 @@ def hasSequenceOutputAttribute(self) -> bool: def has3DOutputAttribute(self): """ - Return True if at least one attribute is a File that can be loaded in the 3D Viewer, False otherwise. + Return True if at least one attribute is a File that can be loaded in the 3D Viewer, + False otherwise. """ # List of supported extensions, taken from Viewer3DSettings supportedExts = ['.obj', '.stl', '.fbx', '.gltf', '.abc', '.ply'] @@ -1654,14 +1700,16 @@ def __init__(self, nodeType, position=None, parent=None, uid=None, **kwargs): self._sourceCodeFolder = self.nodeDesc.sourceCodeFolder for attrDesc in self.nodeDesc.inputs: - self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, node=self)) + self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), + isOutput=False, node=self)) for attrDesc in self.nodeDesc.outputs: - self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=True, node=self)) + self._attributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), + isOutput=True, node=self)) for attrDesc in self.nodeDesc.internalInputs: - self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), isOutput=False, - node=self)) + self._internalAttributes.add(attributeFactory(attrDesc, kwargs.get(attrDesc.name, None), + isOutput=False, node=self)) # Declare events for specific output attributes for attr in self._attributes: @@ -1891,7 +1939,8 @@ def attributeDescFromValue(attrName, value, isOutput): @staticmethod def attributeDescFromName(refAttributes, name, value, strict=True): """ - Try to find a matching attribute description in refAttributes for given attribute 'name' and 'value'. + Try to find a matching attribute description in refAttributes for given attribute + 'name' and 'value'. Args: refAttributes ([desc.Attribute]): reference Attributes to look for a description From f8567fb98f17210f1d2de7906309b62930de7075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 28 Apr 2025 18:24:58 +0200 Subject: [PATCH 49/49] [desc] Node: Clean-up code --- meshroom/core/desc/node.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index 910b8df6f6..da887b629f 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -90,7 +90,9 @@ def upgradeAttributeValues(self, attrValues, fromVersion): @classmethod def onNodeCreated(cls, node): - """Called after a node instance had been created from this node descriptor and added to a Graph.""" + """ + Called after a node instance created from this node descriptor has been added to a Graph. + """ pass @classmethod @@ -150,8 +152,8 @@ def executeChunkCommandLine(self, chunk, cmd, env=None): cmdList[0] = prog print(f' - command full path: {prog}') - # Change the process group to avoid Meshroom main process being killed if the subprocess - # gets terminated by the user or an Out Of Memory (OOM kill). + # Change the process group to avoid Meshroom main process being killed if the + # subprocess gets terminated by the user or an Out Of Memory (OOM kill). if sys.platform == "win32": platformArgs = {"creationflags": psutil.CREATE_NEW_PROCESS_GROUP} # Note: DETACHED_PROCESS means fully detached process. @@ -269,17 +271,7 @@ def getMrNodeType(self): return MrNodeType.COMMANDLINE def buildCommandLine(self, chunk): - cmdPrefix = '' - # # If rez available in env, we use it - # if "REZ_ENV" in os.environ and chunk.node.packageVersion: - # # If the node package is already in the environment, we don't need a new dedicated rez environment - # alreadyInEnv = os.environ.get("REZ_{}_VERSION".format(chunk.node.packageName.upper()), - # "").startswith(chunk.node.packageVersion) - # if not alreadyInEnv: - # cmdPrefix = '{rez} {packageFullName} -- '.format(rez=os.environ.get("REZ_ENV"), - # packageFullName=chunk.node.packageFullName) - cmdSuffix = '' if chunk.node.isParallelized and chunk.node.size > 1: cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict()) @@ -333,7 +325,8 @@ def initialize(self, node, inputs, recursiveInputs): Args: node (Node): the node whose attributes must be initialized inputs (list): the user-provided list of input files/directories - recursiveInputs (list): the user-provided list of input directories to search recursively for images + recursiveInputs (list): the user-provided list of input directories to search + recursively for images """ pass @@ -355,7 +348,8 @@ def extendAttributes(self, node, attributesDict): Args: node (Node): the node whose attributes are to be extended - attributesDict (dict): the dictionary containing the attributes' names (as keys) and the values to extend with + attributesDict (dict): the dictionary containing the attributes' names (as keys) and the + values to extend with """ for attr in attributesDict.keys(): if node.hasAttribute(attr): @@ -367,7 +361,8 @@ def setAttributes(self, node, attributesDict): Args: node (Node): the node whose attributes are to be extended - attributesDict (dict): the dictionary containing the attributes' names (as keys) and the values to set + attributesDict (dict): the dictionary containing the attributes' names (as keys) and the + values to set """ for attr in attributesDict: if node.hasAttribute(attr):