From b56420318d59ece37f476270fbadc4c0c9f9c213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 5 May 2025 16:06:52 +0200 Subject: [PATCH 01/39] [core] Add `ProcessEnv` class in new `plugins` module Add a new module named `plugins` with a `ProcessEnv` class which contains the paths describing the environment needed for the plugin's (node's) process. --- meshroom/core/__init__.py | 7 +++---- meshroom/core/plugins.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 meshroom/core/plugins.py diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 0300647799..5257e5f8ad 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,6 +19,7 @@ except Exception: pass +from meshroom.core.plugins import ProcessEnv from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc @@ -349,9 +350,7 @@ def loadPluginFolder(folder): logging.info(f"Plugin folder '{folder}' does not contain a 'meshroom' folder.") return - binFolders = [Path(folder, 'bin')] - libFolders = [Path(folder, 'lib'), Path(folder, 'lib64')] - pythonPathFolders = [Path(folder)] + binFolders + processEnv = ProcessEnv(folder) loadAllNodes(folder=mrFolder) loadPipelineTemplates(folder=mrFolder) @@ -361,7 +360,7 @@ def loadPluginsFolder(folder): if not os.path.isdir(folder): logging.debug(f"PluginSet folder '{folder}' does not exist.") return - + for file in os.listdir(folder): if os.path.isdir(file): subFolder = os.path.join(folder, file) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py new file mode 100644 index 0000000000..e1b4006173 --- /dev/null +++ b/meshroom/core/plugins.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from meshroom.common import BaseObject + +class ProcessEnv(BaseObject): + """ + Describes the environment required by a node's process. + """ + + def __init__(self, folder: str): + super().__init__() + self.binPaths: list = [Path(folder, "bin")] + self.libPaths: list = [Path(folder, "lib"), Path(folder, "lib64")] + self.pythonPathFolders: list = [Path(folder)] + self.binPaths From 6f69588f0b5752b8bab51ae92a14322e5a8b8afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Tue, 6 May 2025 09:55:32 +0100 Subject: [PATCH 02/39] [core] Add typing on methods --- meshroom/core/__init__.py | 16 +-- meshroom/core/attribute.py | 193 ++++++++++++++++++++----------------- 2 files changed, 115 insertions(+), 94 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 5257e5f8ad..799298d222 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -128,7 +128,7 @@ def loadClasses(folder, packageName, classType): return classes -def validateNodeDesc(nodeDesc): +def validateNodeDesc(nodeDesc: desc.Node) -> list: """ Check that the node has a valid description before being loaded. For the description to be valid, the default value of every parameter needs to correspond to the type @@ -140,10 +140,10 @@ def validateNodeDesc(nodeDesc): "group", is invalid, then it will be added to the list as "group:x". Args: - nodeDesc (desc.Node): description of the node + nodeDesc: description of the node Returns: - errors (list): the list of invalid parameters if there are any, empty list otherwise + errors: the list of invalid parameters if there are any, empty list otherwise """ errors = [] @@ -280,7 +280,7 @@ def micro(self): return self.components[2] -def moduleVersion(moduleName, default=None): +def moduleVersion(moduleName: str, default=None): """ Return the version of a module indicated with '__version__' keyword. Args: @@ -293,7 +293,7 @@ def moduleVersion(moduleName, default=None): return getattr(sys.modules[moduleName], "__version__", default) -def nodeVersion(nodeDesc, default=None): +def nodeVersion(nodeDesc: desc.Node, default=None): """ Return node type version for the given node description class. Args: @@ -306,7 +306,7 @@ def nodeVersion(nodeDesc, default=None): return moduleVersion(nodeDesc.__module__, default) -def registerNodeType(nodeType): +def registerNodeType(nodeType: desc.Node): """ Register a Node Type based on a Node Description class. After registration, nodes of this type can be instantiated in a Graph. @@ -316,7 +316,7 @@ def registerNodeType(nodeType): nodesDesc[nodeType.__name__] = nodeType -def unregisterNodeType(nodeType): +def unregisterNodeType(nodeType: desc.Node): """ Remove 'nodeType' from the list of register node types. """ assert nodeType.__name__ in nodesDesc del nodesDesc[nodeType.__name__] @@ -367,7 +367,7 @@ def loadPluginsFolder(folder): loadPluginFolder(subFolder) -def registerSubmitter(s): +def registerSubmitter(s: BaseSubmitter): if s.name in submitters: logging.error(f"Submitter {s.name} is already registered.") submitters[s.name] = s diff --git a/meshroom/core/attribute.py b/meshroom/core/attribute.py index a06e912607..4141cce519 100644 --- a/meshroom/core/attribute.py +++ b/meshroom/core/attribute.py @@ -17,15 +17,16 @@ from meshroom.core.graph import Edge -def attributeFactory(description, value, isOutput, node, root=None, parent=None): +def attributeFactory(description: str, value, isOutput: bool, node, root=None, parent=None): """ Create an Attribute based on description type. Args: description: the Attribute description - value: value of the Attribute. Will be set if not None. - isOutput: whether is Attribute is an output attribute. - node (Node): node owning the Attribute. Note that the created Attribute is not added to Node's attributes + value: value of the Attribute. Will be set if not None. + isOutput: whether the Attribute is an output attribute. + node (Node): node owning the Attribute. Note that the created Attribute is not added to \ + Node's attributes root: (optional) parent Attribute (must be ListAttribute or GroupAttribute) parent (BaseObject): (optional) the parent BaseObject if any """ @@ -53,27 +54,27 @@ class Attribute(BaseObject): VALID_IMAGE_SEMANTICS = ["image", "imageList", "sequence"] VALID_3D_EXTENSIONS = [".obj", ".stl", ".fbx", ".gltf", ".abc", ".ply"] - def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): + def __init__(self, node, attributeDesc: desc.Attribute, isOutput: bool, root=None, parent=None): """ Attribute constructor Args: node (Node): the Node hosting this Attribute - attributeDesc (desc.Attribute): the description of this Attribute - isOutput (bool): whether this Attribute is an output of the Node + attributeDesc: the description of this Attribute + isOutput: whether this Attribute is an output of the Node root (Attribute): (optional) the root Attribute (List or Group) containing this one parent (BaseObject): (optional) the parent BaseObject """ super().__init__(parent) - self._name = attributeDesc.name + self._name: str = attributeDesc.name self._root = None if root is None else weakref.ref(root) self._node = weakref.ref(node) - self.attributeDesc = attributeDesc - self._isOutput = isOutput - self._label = attributeDesc.label - self._enabled = True - self._validValue = True - self._description = attributeDesc.description + self.attributeDesc: desc.Attribute = attributeDesc + self._isOutput: bool = isOutput + self._label: str = attributeDesc.label + self._enabled: bool = True + self._validValue: bool = True + self._description: str = attributeDesc.description self._invalidate = False if self._isOutput else attributeDesc.invalidate # invalidation value for output attributes @@ -91,11 +92,11 @@ def node(self): def root(self): return self._root() if self._root else None - def getName(self): + def getName(self) -> str: """ Attribute name """ return self._name - def getFullName(self): + def getFullName(self) -> str: """ Name inside the Graph: groupName.name """ if isinstance(self.root, ListAttribute): return f'{self.root.getFullName()}[{self.root.index(self)}]' @@ -103,53 +104,55 @@ def getFullName(self): return f'{self.root.getFullName()}.{self.getName()}' return self.getName() - def getFullNameToNode(self): + def getFullNameToNode(self) -> str: """ Name inside the Graph: nodeName.groupName.name """ return f'{self.node.name}.{self.getFullName()}' - def getFullNameToGraph(self): + def getFullNameToGraph(self) -> str: """ Name inside the Graph: graphName.nodeName.groupName.name """ graphName = self.node.graph.name if self.node.graph else "UNDEFINED" return f'{graphName}.{self.getFullNameToNode()}' - def asLinkExpr(self): + def asLinkExpr(self) -> str: """ Return link expression for this Attribute """ return "{" + self.getFullNameToNode() + "}" - def getType(self): + def getType(self) -> str: return self.attributeDesc.type - def _isReadOnly(self): + def _isReadOnly(self) -> bool: return not self._isOutput and self.node.isCompatibilityNode - def getBaseType(self): + def getBaseType(self) -> str: return self.getType() - def getLabel(self): + def getLabel(self) -> str: return self._label @Slot(str, result=bool) - def matchText(self, text): + def matchText(self, text: str) -> bool: return self.fullLabel.lower().find(text.lower()) > -1 - def getFullLabel(self): - """ Full Label includes the name of all parent groups, e.g. 'groupLabel subGroupLabel Label' """ + def getFullLabel(self) -> str: + """ + Full Label includes the name of all parent groups, e.g. 'groupLabel subGroupLabel Label'. + """ if isinstance(self.root, ListAttribute): return self.root.getFullLabel() elif isinstance(self.root, GroupAttribute): return f'{self.root.getFullLabel()} {self.getLabel()}' return self.getLabel() - def getFullLabelToNode(self): + def getFullLabelToNode(self) -> str: """ Label inside the Graph: nodeLabel groupLabel Label """ return f'{self.node.label} {self.getFullLabel()}' - def getFullLabelToGraph(self): + def getFullLabelToGraph(self) -> str: """ Label inside the Graph: graphName nodeLabel groupLabel Label """ graphName = self.node.graph.name if self.node.graph else "UNDEFINED" return f'{graphName} {self.getFullLabelToNode()}' - def getEnabled(self): + def getEnabled(self) -> bool: if isinstance(self.desc.enabled, types.FunctionType): try: return self.desc.enabled(self.node) @@ -205,8 +208,8 @@ def _set_value(self, value): # evaluate the function self._value = value(self) else: - # if we set a new value, we use the attribute descriptor validator to check the validity of the value - # and apply some conversion if needed + # if we set a new value, we use the attribute descriptor validator to check the + # validity of the value and apply some conversion if needed convertedValue = self.validateValue(value) self._value = convertedValue @@ -266,26 +269,27 @@ def requestNodeUpdate(self): self.node.updateInternalAttributes() @property - def isOutput(self): + def isOutput(self) -> bool: return self._isOutput @property - def isInput(self): + def isInput(self) -> bool: return not self._isOutput - def uid(self): + def uid(self) -> str: """ Compute the UID for the attribute. """ if self.isOutput: if self.desc.isDynamicValue: # If the attribute is a dynamic output, the UID is derived from the node UID. - # To guarantee that each output attribute receives a unique ID, we add the attribute name to it. + # To guarantee that each output attribute receives a unique ID, we add the attribute + # name to it. return hashValue((self.name, self.node._uid)) else: # Only dependent on the hash of its value without the cache folder. - # "/" at the end of the link is stripped to prevent having different UIDs depending on - # whether the invalidation value finishes with it or not + # "/" at the end of the link is stripped to prevent having different UIDs depending + # on whether the invalidation value finishes with it or not strippedInvalidationValue = self._invalidationValue.rstrip("/") return hashValue(strippedInvalidationValue) if self.isLink: @@ -298,13 +302,15 @@ def uid(self): return hashValue(self._value) @property - def isLink(self): + def isLink(self) -> bool: """ Whether the input attribute is a link to another attribute. """ - # note: directly use self.node.graph._edges to avoid using the property that may become invalid at some point - return self.node.graph and self.isInput and self.node.graph._edges and self in self.node.graph._edges.keys() + # note: directly use self.node.graph._edges to avoid using the property that may become + # invalid at some point + return self.node.graph and self.isInput and self.node.graph._edges and \ + self in self.node.graph._edges.keys() @staticmethod - def isLinkExpression(value): + def isLinkExpression(value) -> bool: """ Return whether the given argument is a link expression. A link expression is a string matching the {nodeName.attrName} pattern. @@ -322,12 +328,13 @@ def getLinkParam(self, recursive=False): return linkParam @property - def hasOutputConnections(self): - """ Whether the attribute has output connections, i.e is the source of at least one edge. """ + def hasOutputConnections(self) -> bool: + """ + Whether the attribute has output connections, i.e is the source of at least one edge. + """ # safety check to avoid evaluation errors if not self.node.graph or not self.node.graph.edges: return False - return next((edge for edge in self.node.graph.edges.values() if edge.src == self), None) is not None def getInputConnections(self) -> list["Edge"]: @@ -391,28 +398,29 @@ def getExportValue(self): return self.value def getEvalValue(self): - ''' + """ Return the value. If it is a string, expressions will be evaluated. - ''' + """ if isinstance(self.value, str): substituted = Template(self.value).safe_substitute(os.environ) try: varResolved = substituted.format(**self.node._cmdVars) return varResolved except (KeyError, IndexError): - # Catch KeyErrors and IndexErros to be able to open files created prior to the support of - # relative variables (when self.node._cmdVars was not used to evaluate expressions in the attribute) + # Catch KeyErrors and IndexErros to be able to open files created prior to the + # support of relative variables (when self.node._cmdVars was not used to evaluate + # expressions in the attribute) return substituted return self.value - def getValueStr(self, withQuotes=True): - ''' + def getValueStr(self, withQuotes=True) -> str: + """ Return the value formatted as a string with quotes to deal with spaces. If it is a string, expressions will be evaluated. If it is an empty string, it will returns 2 quotes. If it is an empty list, it will returns a really empty string. If it is a list with one empty string element, it will returns 2 quotes. - ''' + """ # ChoiceParam with multiple values should be combined if isinstance(self.attributeDesc, desc.ChoiceParam) and not self.attributeDesc.exclusive: # Ensure value is a list as expected @@ -421,8 +429,10 @@ def getValueStr(self, withQuotes=True): if withQuotes and v: return f'"{v}"' return v - # String, File, single value Choice are based on strings and should includes quotes to deal with spaces - if withQuotes and isinstance(self.attributeDesc, (desc.StringParam, desc.File, desc.ChoiceParam)): + # String, File, single value Choice are based on strings and should includes quotes + # to deal with spaces + if withQuotes and \ + isinstance(self.attributeDesc, (desc.StringParam, desc.File, desc.ChoiceParam)): return f'"{self.getEvalValue()}"' return str(self.getEvalValue()) @@ -436,10 +446,11 @@ def defaultValue(self): logging.warning("Failed to evaluate default value (node lambda) for attribute '{}': {}". format(self.name, e)) return None - # Need to force a copy, for the case where the value is a list (avoid reference to the desc value) + # Need to force a copy, for the case where the value is a list + # (avoid reference to the desc value) return copy.copy(self.desc.value) - def _isDefault(self): + def _isDefault(self) -> bool: return self.value == self.defaultValue() def getPrimitiveValue(self, exportDefault=True): @@ -510,7 +521,8 @@ def _is2D(self) -> bool: isDefault = Property(bool, _isDefault, notify=valueChanged) linkParam = Property(BaseObject, getLinkParam, notify=isLinkChanged) - rootLinkParam = Property(BaseObject, lambda self: self.getLinkParam(recursive=True), notify=isLinkChanged) + rootLinkParam = Property(BaseObject, lambda self: self.getLinkParam(recursive=True), + notify=isLinkChanged) node = Property(BaseObject, node.fget, constant=True) enabledChanged = Signal() enabled = Property(bool, getEnabled, setEnabled, notify=enabledChanged) @@ -522,7 +534,7 @@ def _is2D(self) -> bool: def raiseIfLink(func): - """ If Attribute instance is a link, raise a RuntimeError.""" + """ If Attribute instance is a link, raise a RuntimeError. """ def wrapper(attr, *args, **kwargs): if attr.isLink: raise RuntimeError("Can't modify connected Attribute") @@ -531,7 +543,8 @@ def wrapper(attr, *args, **kwargs): class PushButtonParam(Attribute): - def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): + def __init__(self, node, attributeDesc: desc.PushButtonParam, isOutput: bool, + root=None, parent=None): super().__init__(node, attributeDesc, isOutput, root, parent) @Slot() @@ -541,7 +554,8 @@ def clicked(self): class ChoiceParam(Attribute): - def __init__(self, node, attributeDesc: desc.ChoiceParam, isOutput, root=None, parent=None): + def __init__(self, node, attributeDesc: desc.ChoiceParam, isOutput: bool, + root=None, parent=None): super().__init__(node, attributeDesc, isOutput, root, parent) self._values = None @@ -568,7 +582,7 @@ def validateValue(self, value): raise ValueError("Non exclusive ChoiceParam value should be iterable (param:{}, value:{}, type:{})". format(self.name, value, type(value))) return [self.conformValue(v) for v in value] - + def _set_value(self, value): # Handle alternative serialization for ChoiceParam with overriden values. serializedValueWithValuesOverrides = isinstance(value, dict) @@ -585,7 +599,8 @@ def setValues(self, values): self.valuesChanged.emit() def getExportValue(self): - useStandardSerialization = self.isLink or not self.desc._saveValuesOverride or self._values is None + useStandardSerialization = self.isLink or not self.desc._saveValuesOverride or \ + self._values is None if useStandardSerialization: return super().getExportValue() @@ -602,7 +617,8 @@ def getExportValue(self): class ListAttribute(Attribute): - def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): + def __init__(self, node, attributeDesc: desc.ListAttribute, isOutput: bool, + root=None, parent=None): super().__init__(node, attributeDesc, isOutput, root, parent) def __len__(self): @@ -617,7 +633,7 @@ def getBaseType(self): return self.attributeDesc.elementDesc.__class__.__name__ def at(self, idx): - """ Returns child attribute at index 'idx' """ + """ Returns child attribute at index 'idx'. """ # Implement 'at' rather than '__getitem__' # since the later is called spuriously when object is used in QML return self._value.at(idx) @@ -649,7 +665,8 @@ def _set_value(self, value): def upgradeValue(self, exportedValues): if not isinstance(exportedValues, list): - if isinstance(exportedValues, ListAttribute) or Attribute.isLinkExpression(exportedValues): + if isinstance(exportedValues, ListAttribute) or \ + Attribute.isLinkExpression(exportedValues): self._set_value(exportedValues) return raise RuntimeError("ListAttribute.upgradeValue: the given value is of type " + @@ -657,7 +674,8 @@ def upgradeValue(self, exportedValues): attrs = [] for v in exportedValues: - a = attributeFactory(self.attributeDesc.elementDesc, None, self.isOutput, self.node, self) + a = attributeFactory(self.attributeDesc.elementDesc, None, self.isOutput, + self.node, self) a.upgradeValue(v) attrs.append(a) index = len(self._value) @@ -675,7 +693,8 @@ def insert(self, index, value): if self._value is None: self._value = ListModel(parent=self) values = value if isinstance(value, list) else [value] - attrs = [attributeFactory(self.attributeDesc.elementDesc, v, self.isOutput, self.node, self) for v in values] + attrs = [attributeFactory(self.attributeDesc.elementDesc, v, self.isOutput, self.node, self) + for v in values] self._value.insert(index, attrs) self.valueChanged.emit() self._applyExpr() @@ -725,27 +744,28 @@ def getExportValue(self): return self.getLinkParam().asLinkExpr() return [attr.getExportValue() for attr in self._value] - def defaultValue(self): + def defaultValue(self) -> list: return [] - def _isDefault(self): + def _isDefault(self) -> bool: return len(self._value) == 0 def getPrimitiveValue(self, exportDefault=True): if exportDefault: return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value] - else: - return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value if not attr.isDefault] + return [attr.getPrimitiveValue(exportDefault=exportDefault) for attr in self._value + if not attr.isDefault] - def getValueStr(self, withQuotes=True): + def getValueStr(self, withQuotes=True) -> str: assert isinstance(self.value, ListModel) if self.attributeDesc.joinChar == ' ': - return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) for v in self.value]) - else: - v = self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=False) for v in self.value]) - if withQuotes and v: - return f'"{v}"' - return v + return self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=withQuotes) + for v in self.value]) + v = self.attributeDesc.joinChar.join([v.getValueStr(withQuotes=False) + for v in self.value]) + if withQuotes and v: + return f'"{v}"' + return v def updateInternals(self): super().updateInternals() @@ -753,9 +773,10 @@ def updateInternals(self): attr.updateInternals() @property - def isLinkNested(self): + def isLinkNested(self) -> bool: """ Whether the attribute or any of its elements is a link to another attribute. """ - # note: directly use self.node.graph._edges to avoid using the property that may become invalid at some point + # note: directly use self.node.graph._edges to avoid using the property that may become + # invalid at some point return self.isLink \ or self.node.graph and self.isInput and self.node.graph._edges \ and any(v in self.node.graph._edges.keys() for v in self._value) @@ -799,7 +820,8 @@ def getOutputConnections(self) -> list["Edge"]: class GroupAttribute(Attribute): - def __init__(self, node, attributeDesc, isOutput, root=None, parent=None): + def __init__(self, node, attributeDesc: desc.GroupAttribute, isOutput: bool, + root=None, parent=None): super().__init__(node, attributeDesc, isOutput, root, parent) def __getattr__(self, key): @@ -854,12 +876,12 @@ def resetToDefaultValue(self): self._value.get(attrDesc.name).resetToDefaultValue() @Slot(str, result=Attribute) - def childAttribute(self, key): + def childAttribute(self, key: str) -> Attribute: """ Get child attribute by name or None if none was found. Args: - key (str): the name of the child attribute + key: the name of the child attribute Returns: Attribute: the child attribute or None @@ -892,9 +914,8 @@ def defaultValue(self): def getPrimitiveValue(self, exportDefault=True): if exportDefault: return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items()} - else: - return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() - if not attr.isDefault} + return {name: attr.getPrimitiveValue(exportDefault=exportDefault) for name, attr in self._value.items() + if not attr.isDefault} def getValueStr(self, withQuotes=True): # add brackets if requested @@ -925,7 +946,7 @@ def updateInternals(self): attr.updateInternals() @Slot(str, result=bool) - def matchText(self, text): + def matchText(self, text: str) -> bool: return super().matchText(text) or any(c.matchText(text) for c in self._value) # Override value property From f6694022a3476d9ab1cd0d2b42c583585aa57900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 7 May 2025 14:27:59 +0200 Subject: [PATCH 03/39] [core] plugins: Add a `Plugin` class that contains a set of nodes A `Plugin` object contains a collection of nodes, represented as `NodePlugin` objects. This commit adds the implementation of the `Plugin` class with the methods to add and remove `NodePlugins`, as well as an empty class for the `NodePlugin` objects themselves. --- meshroom/core/plugins.py | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index e1b4006173..08f96bef34 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -1,6 +1,11 @@ +from __future__ import annotations + +import logging + from pathlib import Path from meshroom.common import BaseObject +from meshroom.core import desc class ProcessEnv(BaseObject): """ @@ -12,3 +17,73 @@ def __init__(self, folder: str): self.binPaths: list = [Path(folder, "bin")] self.libPaths: list = [Path(folder, "lib"), Path(folder, "lib64")] self.pythonPathFolders: list = [Path(folder)] + self.binPaths + + +class Plugin(BaseObject): + """ + A collection of node plugins. + + Members: + name: the name of the plugin (e.g. name of the Python module containing the node plugins) + path: the absolute path of the plugin + _nodePlugins: dictionary mapping the name of a node plugin to its corresponding + NodePlugin object + processEnv: the environment required for the nodes' processes to be correctly executed + """ + + def __init__(self, name: str, path: str): + super().__init__() + + self._name: str = name + self._path: str = path + + self._nodePlugins: dict[str: NodePlugin] = {} + self._processEnv: ProcessEnv = ProcessEnv(path) + + @property + def name(self): + """ Return the name of the plugin. """ + return self._name + + @property + def path(self): + """ Return the absolute path of the plugin. """ + return self._path + + @property + def processEnv(self): + """ Return the environment required to successfully execute processes. """ + return self._processEnv + + def addNodePlugin(self, nodePlugin: NodePlugin): + """ + Add a node plugin to the current plugin object and assign it as its containing plugin. + The node plugin is added to the dictionary of node plugins with the name of the node + descriptor as its key. + + Args: + nodePlugin: the NodePlugin object to add to the Plugin. + """ + self._nodePlugins[nodePlugin.nodeDescriptor.__name__] = nodePlugin + nodePlugin.plugin = self + + def removeNodePlugin(self, name: str): + """ + Remove a node plugin from the current plugin object and delete any container relationship. + + Args: + name: the name of the NodePlugin to remove. + """ + if name in self._nodePlugins: + self._nodePlugins[name].plugin = None + del self._nodePlugins[name] + else: + logging.warning(f"Node plugin {name} is not part of the plugin {self.name}.") + + + +class NodePlugin(BaseObject): + """ + """ + def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): + super().__init__() From 1531e11b831f44991296640ad10850fb4e93871b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 7 May 2025 14:29:32 +0200 Subject: [PATCH 04/39] [core] plugins: Add `NodePluginStatus` enum for `NodePlugin` objects --- meshroom/core/plugins.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 08f96bef34..23baf62881 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -2,6 +2,7 @@ import logging +from enum import Enum from pathlib import Path from meshroom.common import BaseObject @@ -19,6 +20,16 @@ def __init__(self, folder: str): self.pythonPathFolders: list = [Path(folder)] + self.binPaths +class NodePluginStatus(Enum): + """ + Loading status for NodePlugin objects. + """ + NOT_LOADED = 0 # The node plugin exists but is not loaded and cannot be used (not registered) + LOADED = 1 # The node plugin is currently loaded and functional (it has been registered) + DESC_ERROR = 2 # The node plugin exists but has an invalid description + ERROR = 3 # The node plugin exists and is valid but could not be successfully loaded + + class Plugin(BaseObject): """ A collection of node plugins. From fa3cb8c4d6658500182be9977eff458eee919cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 7 May 2025 15:25:40 +0100 Subject: [PATCH 05/39] [core] plugins: Move `validateNodeDesc` from __init__.py to plugins.py The validation of the node descriptions will be handled directly within the `NodePlugin` objects and there is thus no need for this method to exist outside of plugins.py. --- meshroom/core/__init__.py | 36 +----------------------------------- meshroom/core/plugins.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 799298d222..5b3bfc2e07 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ except Exception: pass -from meshroom.core.plugins import ProcessEnv +from meshroom.core.plugins import validateNodeDesc, ProcessEnv from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc @@ -128,40 +128,6 @@ def loadClasses(folder, packageName, classType): return classes -def validateNodeDesc(nodeDesc: desc.Node) -> list: - """ - Check that the node has a valid description before being loaded. For the description - to be valid, the default value of every parameter needs to correspond to the type - of the parameter. - An empty returned list means that every parameter is valid, and so is the node's description. - If it is not valid, the returned list contains the names of the invalid parameters. In case - of nested parameters (parameters in groups or lists, for example), the name of the parameter - follows the name of the parent attributes. For example, if the attribute "x", contained in group - "group", is invalid, then it will be added to the list as "group:x". - - Args: - nodeDesc: description of the node - - Returns: - errors: the list of invalid parameters if there are any, empty list otherwise - """ - errors = [] - - for param in nodeDesc.inputs: - err = param.checkValueTypes() - if err: - errors.append(err) - - for param in nodeDesc.outputs: - if param.value is None: - continue - err = param.checkValueTypes() - if err: - errors.append(err) - - return errors - - class Version: """ Version provides convenient properties and methods to manipulate and compare versions. diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 23baf62881..894fc71d11 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -8,6 +8,40 @@ from meshroom.common import BaseObject from meshroom.core import desc +def validateNodeDesc(nodeDesc: desc.Node) -> list: + """ + Check that the node has a valid description before being loaded. For the description + to be valid, the default value of every parameter needs to correspond to the type + of the parameter. + An empty returned list means that every parameter is valid, and so is the node's description. + If it is not valid, the returned list contains the names of the invalid parameters. In case + of nested parameters (parameters in groups or lists, for example), the name of the parameter + follows the name of the parent attributes. For example, if the attribute "x", contained in group + "group", is invalid, then it will be added to the list as "group:x". + + Args: + nodeDesc: description of the node. + + Returns: + errors: the list of invalid parameters if there are any, empty list otherwise + """ + errors = [] + + for param in nodeDesc.inputs: + err = param.checkValueTypes() + if err: + errors.append(err) + + for param in nodeDesc.outputs: + if param.value is None: + continue + err = param.checkValueTypes() + if err: + errors.append(err) + + return errors + + class ProcessEnv(BaseObject): """ Describes the environment required by a node's process. From 63e01d43bf351ea15c48b69fa6fc3504afa6046b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 7 May 2025 15:56:03 +0100 Subject: [PATCH 06/39] [core] plugins: Add working `NodePlugin` class to represent nodes --- meshroom/core/plugins.py | 51 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 894fc71d11..33e5f3af5f 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -3,6 +3,7 @@ import logging from enum import Enum +from inspect import getfile from pathlib import Path from meshroom.common import BaseObject @@ -126,9 +127,57 @@ def removeNodePlugin(self, name: str): logging.warning(f"Node plugin {name} is not part of the plugin {self.name}.") - class NodePlugin(BaseObject): """ + Based on a node description, a NodePlugin represents a loadable node. + + Members: + plugin: the Plugin object that contains this node plugin + path: absolute path to the file containing the node's description + nodeDescriptor: the description of the node + status: the loading status on the node plugin + errors: the list of errors (if there are any) when validating the description + of the node or attempting to load it + processEnv: the environment required for the node plugin's process. It can either + be specific to this node plugin, or be common for all the node plugins within + the plugin """ + def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): super().__init__() + self.plugin: Plugin = plugin + self.path: str = Path(getfile(nodeDesc)).resolve().as_posix() + self.nodeDescriptor: desc.Node = nodeDesc + + self.status: NodePluginStatus = NodePluginStatus.NOT_LOADED + self.errors: list[str] = validateNodeDesc(nodeDesc) + + if self.errors: + self.status = NodePluginStatus.DESC_ERROR + + self._processEnv = None + + @property + def plugin(self): + """ + Return the Plugin object that contains this node plugin. + If the node plugin has not been assigned to a plugin yet, this value will + be set to None. + """ + return self._plugin + + @plugin.setter + def plugin(self, plugin: Plugin): + self._plugin = plugin + + @property + def processEnv(self): + """" + Return the process environment that is specific to the node plugin if it has any. + Otherwise, the Plugin's is returned. + """ + if self._processEnv: + return self._processEnv + if self.plugin: + return self.plugin.processEnv + return None From 836a351cc9e2e1c017131ee8e0ddae0027cd8955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 7 May 2025 17:57:24 +0100 Subject: [PATCH 07/39] [core] plugins: Add a `NodePluginManager` class The `NodePluginManager` class manages is used to load, manage, and register plugins and nodes. --- meshroom/core/plugins.py | 126 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 33e5f3af5f..7913b36698 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -181,3 +181,129 @@ def processEnv(self): if self.plugin: return self.plugin.processEnv return None + + +class NodePluginManager(BaseObject): + """ + Manager for all the loaded Plugin objects as well as the registered NodePlugin objects. + + Members: + _plugins: dictionary containing all the loaded Plugins, with their name as the key + _nodePlugins: dictionary containing all the NodePlugins that have been registered + (a NodePlugin may exist without having been registered) with their name as + the key + """ + + def __init__(self): + super().__init__() + + self._plugins: dict[str: Plugin] = {} # loaded plugins + self._nodePlugins: dict[str: NodePlugin] = {} # registered node plugins + + def isRegistered(self, name: str) -> bool: + """ + Return whether the node plugin has been registered already. + + Args: + name: the name of the node plugin whose registration needs to be checked. + """ + return name in self._nodePlugins + + def getPlugins(self) -> dict[str: Plugin]: + """ + Return a dictionary containing all the loaded Plugins, with {key, value} = + {name, Plugin}. + """ + return self._plugins + + def getPlugin(self, name: str) -> Plugin: + """ + Return the loaded Plugin object named "name". + + Args: + name: the name of the Plugin, used upon its loading. + + Returns: + Plugin | None: the loaded Plugin object if it exists, None otherwise. + """ + if name in self._plugins: + return self._plugins[name] + return None + + def addPlugin(self, plugin: Plugin): + """ + Load a Plugin object. + + Args: + plugin: the Plugin to load and add to the list of loaded plugins. + """ + if not self.getPlugin(plugin.name): + self._plugins[plugin.name] = plugin + + def getNodePlugins(self) -> dict[str: NodePlugin]: + """ + Return a dictionary containing all the registered NodePlugins, with + {key, value} = {name, NodePlugin}. + """ + return self._nodePlugins + + def getNodePlugin(self, name: str) -> NodePlugin: + """ + Return the NodePlugin object that has been registered under the name "name" if it exists. + + Args: + name: the name of the NodePlugin used for its registration. + + Returns: + NodePlugin | None: the loaded NodePlugin object if it exists, None otherwise. + """ + if self.isRegistered(name): + return self._nodePlugins[name] + return None + + def registerPlugin(self, name: str): + """ + Register all the NodePlugins contained in the Plugin loaded as "name". + + Args: + name: the name of the Plugin whose NodePlugins will be registered. + """ + plugin = self.getPlugin(name) + if plugin: + for node in plugin._nodePlugins: + self.registerNode(plugin._nodePlugins[node]) + else: + logging.error(f"No loaded Plugin named {name}.") + + def registerNode(self, nodePlugin: NodePlugin): + """ + Register a node plugin. A registered node plugin will become instantiable. + If it is already registered, or if there is an issue with the node description, + the node plugin will not be registered and its status will be updated. + + Args: + nodePlugin: the node plugin to register. + """ + name = nodePlugin.nodeDescriptor.__name__ + if not self.isRegistered(name) and nodePlugin.status != NodePluginStatus.DESC_ERROR: + try: + self._nodePlugins[name] = nodePlugin + nodePlugin.status = NodePluginStatus.LOADED + except Exception as e: + logging.error(f"NodePlugin {name} could not be loaded: {e}") + nodePlugin.status = NodePluginStatus.ERROR + + def unregisterNode(self, nodePlugin: NodePlugin): + """ + Unregister a node plugin. When unregistered, a node plugin cannot be instantiated anymore. + If it is not registered already, nothing happens. + + Args: + nodePlugin: the node plugin to unregister. + """ + name = nodePlugin.nodeDescriptor.__name__ + if self.isRegistered(name): + if nodePlugin.status != NodePluginStatus.LOADED: + logging.warning(f"NodePlugin {name} is registered but is not correctly loaded.") + del self._nodePlugins[name] + nodePlugin.status = NodePluginStatus.NOT_LOADED From 56eebe4cafa14c6ea66e1e8d934a3e0b6d9cd95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 9 May 2025 16:46:52 +0200 Subject: [PATCH 08/39] [core] plugins: Add templates handling in the `Plugin` class Templates that are available for a plugin are detected and gathered upon the plugin's creation. --- meshroom/core/plugins.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 7913b36698..027fe3fbda 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import os from enum import Enum from inspect import getfile @@ -72,8 +73,10 @@ class Plugin(BaseObject): Members: name: the name of the plugin (e.g. name of the Python module containing the node plugins) path: the absolute path of the plugin - _nodePlugins: dictionary mapping the name of a node plugin to its corresponding - NodePlugin object + _nodePlugins: dictionary mapping the name of a node plugin contained in the plugin + to its corresponding NodePlugin object + _templates: dictionary mapping the name of templates (.mg files) associated to the plugin + with their absolute paths processEnv: the environment required for the nodes' processes to be correctly executed """ @@ -84,8 +87,11 @@ def __init__(self, name: str, path: str): self._path: str = path self._nodePlugins: dict[str: NodePlugin] = {} + self._templates: dict[str: str] = {} self._processEnv: ProcessEnv = ProcessEnv(path) + self.loadTemplates() + @property def name(self): """ Return the name of the plugin. """ @@ -96,6 +102,11 @@ def path(self): """ Return the absolute path of the plugin. """ return self._path + @property + def templates(self): + """ Return the list of templates associated to the plugin. """ + return self._templates + @property def processEnv(self): """ Return the environment required to successfully execute processes. """ @@ -126,6 +137,17 @@ def removeNodePlugin(self, name: str): else: logging.warning(f"Node plugin {name} is not part of the plugin {self.name}.") + def loadTemplates(self): + """ + Load all the pipeline templates that are available within the plugin folder. + Whenever this method is called, the list of templates for the plugin is cleared, + before being filled again. + """ + self._templates.clear() + for file in os.listdir(self.path): + if file.endswith(".mg"): + self._templates[os.path.splitext(file)[0]] = os.path.join(self.path, file) + class NodePlugin(BaseObject): """ From 15c38b6a8b69f9fa537ffe6c1eace3b718781ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 9 May 2025 17:23:55 +0200 Subject: [PATCH 09/39] [core] Instantiate a `NodePluginManager` in the core module --- meshroom/core/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 5b3bfc2e07..df2c9bbabc 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ except Exception: pass -from meshroom.core.plugins import validateNodeDesc, ProcessEnv +from meshroom.core.plugins import validateNodeDesc, ProcessEnv, NodePluginManager from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc @@ -33,6 +33,7 @@ cacheFolderName = 'MeshroomCache' nodesDesc: dict[str, desc.BaseNode] = {} +pluginManager: NodePluginManager = NodePluginManager() submitters: dict[str, BaseSubmitter] = {} pipelineTemplates: dict[str, str] = {} From 4a238b9637a59d5209493d12962805db2b1882a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 9 May 2025 17:51:29 +0200 Subject: [PATCH 10/39] [core] Load and register nodes using the `NodePluginManager` --- meshroom/core/__init__.py | 127 +++++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 36 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index df2c9bbabc..cfc1fc0b84 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ except Exception: pass -from meshroom.core.plugins import validateNodeDesc, ProcessEnv, NodePluginManager +from meshroom.core.plugins import NodePlugin, NodePluginManager, Plugin, ProcessEnv from meshroom.core.submitter import BaseSubmitter from meshroom.env import EnvVar, meshroomFolder from . import desc @@ -43,7 +43,6 @@ def hashValue(value) -> str: hashObject = hashlib.sha1(str(value).encode('utf-8')) return hashObject.hexdigest() - @contextmanager def add_to_path(p): import sys @@ -55,9 +54,15 @@ def add_to_path(p): finally: sys.path = old_path - -def loadClasses(folder, packageName, classType): +def loadClasses(folder: str, packageName: str, classType: type) -> list[type]: """ + Go over the Python module named "packageName" located in "folder" to find files + that contain classes of type "classType" and return these classes in a list. + + Args: + folder: the folder to load the module from. + packageName: the name of the module to look for nodes in. + classType: the class to look for in the files that are inspected. """ classes = [] errors = [] @@ -69,7 +74,8 @@ def loadClasses(folder, packageName, classType): try: package = importlib.import_module(packageName) - packageName = package.packageName if hasattr(package, 'packageName') else package.__name__ + packageName = package.packageName if hasattr(package, "packageName") \ + else package.__name__ packageVersion = getattr(package, "__version__", None) packagePath = os.path.dirname(package.__file__) except Exception as e: @@ -85,31 +91,30 @@ def loadClasses(folder, packageName, classType): ) return [] - for importer, pluginName, ispkg in pkgutil.iter_modules(package.__path__): - pluginModuleName = '.' + pluginName + for _, pluginName, _ in pkgutil.iter_modules(package.__path__): + pluginModuleName = "." + pluginName try: pluginMod = importlib.import_module(pluginModuleName, package=package.__name__) - plugins = [plugin for name, plugin in inspect.getmembers(pluginMod, inspect.isclass) - if plugin.__module__ == f'{package.__name__}.{pluginName}' + plugins = [plugin for _, plugin in inspect.getmembers(pluginMod, inspect.isclass) + if plugin.__module__ == f"{package.__name__}.{pluginName}" and issubclass(plugin, classType)] + if not plugins: logging.warning(f"No class defined in plugin: {pluginModuleName}") - importPlugin = True for p in plugins: - if classType == desc.Node: - nodeErrors = validateNodeDesc(p) - if nodeErrors: - errors.append(" * {}: The following parameters do not have valid default values/ranges: {}" - .format(pluginName, ", ".join(nodeErrors))) - importPlugin = False - break p.packageName = packageName p.packageVersion = packageVersion p.packagePath = packagePath - if importPlugin: - classes.extend(plugins) + if classType == desc.BaseNode: + nodePlugin = NodePlugin(p) + if nodePlugin.errors: + errors.append(" * {}: The following parameters do not have valid " \ + "default values/ranges: {}".format(pluginName, ", ".join(nodePlugin.errors))) + classes.append(nodePlugin) + else: + classes.append(p) except Exception as e: tb = traceback.extract_tb(e.__traceback__) last_call = tb[-1] @@ -126,8 +131,43 @@ def loadClasses(folder, packageName, classType): logging.warning(' The following "{package}" plugins could not be loaded:\n' '{errorMsg}\n' .format(package=packageName, errorMsg='\n'.join(errors))) + return classes +def loadClassesNodes(folder: str, packageName: str) -> list[NodePlugin]: + """ + Return the list of all the NodePlugins that were created following the search of the + Python module named "packageName" located in the folder "folder". + A NodePlugin is created when a file within "packageName" that contains a class inheriting + desc.BaseNode is found. + + Args: + folder: the folder to load the module from. + packageName: the name of the module to look for nodes in. + + Returns: + list[NodePlugin]: a list of all the NodePlugins that were created based on the + module's search. If none has been created, an empty list is returned. + """ + return loadClasses(folder, packageName, desc.BaseNode) + +def loadClassesSubmitters(folder: str, packageName: str) -> list[BaseSubmitter]: + """ + Return the list of all the submitters that were found during the search of the + Python module named "packageName" that located in the folder "folder". + A submitter is found if a file within "packageName" contains a class inheriting + from BaseSubmitter. + + Args: + folder: the folder to load the module from. + packageName: the name of the module to look for nodes in. + + Returns: + list[BaseSubmitter]: a list of all the submitters that were found during the + module's search + """ + return loadClasses(folder, packageName, BaseSubmitter) + class Version: """ @@ -289,22 +329,28 @@ def unregisterNodeType(nodeType: desc.Node): del nodesDesc[nodeType.__name__] -def loadNodes(folder, packageName): +def loadNodes(folder, packageName) -> list[NodePlugin]: if not os.path.isdir(folder): logging.error(f"Node folder '{folder}' does not exist.") - return + return [] - return loadClasses(folder, packageName, desc.BaseNode) + nodes = loadClassesNodes(folder, packageName) + return nodes -def loadAllNodes(folder): - for importer, package, ispkg in pkgutil.walk_packages([folder]): +def loadAllNodes(folder) -> list[Plugin]: + plugins = [] + for _, package, ispkg in pkgutil.walk_packages([folder]): if ispkg: - nodeTypes = loadNodes(folder, package) - for nodeType in nodeTypes: - registerNodeType(nodeType) - nodesStr = ', '.join([nodeType.__name__ for nodeType in nodeTypes]) - logging.debug(f'Nodes loaded [{package}]: {nodesStr}') + plugin = Plugin(package, folder) + nodePlugins = loadNodes(folder, package) + if nodePlugins: + for node in nodePlugins: + plugin.addNodePlugin(node) + nodesStr = ', '.join([node.nodeDescriptor.__name__ for node in nodePlugins]) + logging.debug(f'Nodes loaded [{package}]: {nodesStr}') + plugins.append(plugin) + return plugins def loadPluginFolder(folder): @@ -319,8 +365,13 @@ def loadPluginFolder(folder): processEnv = ProcessEnv(folder) - loadAllNodes(folder=mrFolder) - loadPipelineTemplates(folder=mrFolder) + plugins = loadAllNodes(folder=mrFolder) + if plugins: + for plugin in plugins: + pluginManager.addPlugin(plugin) + pluginManager.registerPlugin(plugin.name) + pipelineTemplates.update(plugin.templates) + return plugins def loadPluginsFolder(folder): @@ -345,10 +396,9 @@ def loadSubmitters(folder, packageName): logging.error(f"Submitters folder '{folder}' does not exist.") return - return loadClasses(folder, packageName, BaseSubmitter) - + return loadClassesSubmitters(folder, packageName) -def loadPipelineTemplates(folder): +def loadPipelineTemplates(folder: str): if not os.path.isdir(folder): logging.error(f"Pipeline templates folder '{folder}' does not exist.") return @@ -356,12 +406,15 @@ def loadPipelineTemplates(folder): if file.endswith(".mg") and file not in pipelineTemplates: pipelineTemplates[os.path.splitext(file)[0]] = os.path.join(folder, file) - def initNodes(): additionalNodesPath = EnvVar.getList(EnvVar.MESHROOM_NODES_PATH) nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath for f in nodesFolders: - loadAllNodes(folder=f) + plugins = loadAllNodes(folder=f) + if plugins: + for plugin in plugins: + pluginManager.addPlugin(plugin) + pluginManager.registerPlugin(plugin.name) def initSubmitters(): @@ -380,6 +433,8 @@ def initPipelines(): pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath for f in pipelineTemplatesFolders: loadPipelineTemplates(f) + for plugin in pluginManager.getPlugins().values(): + pipelineTemplates.update(plugin.templates) def initPlugins(): From 88bee35443cdb171abef26dfa8bd2e3b13d68608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 12 May 2025 08:23:15 +0100 Subject: [PATCH 11/39] [core] Replace `nodesDesc` with the `NodePluginManager` instance --- meshroom/core/__init__.py | 3 +-- meshroom/core/node.py | 10 ++++++---- meshroom/core/nodeFactory.py | 4 +++- meshroom/core/test.py | 10 +++++----- meshroom/ui/app.py | 4 ++-- meshroom/ui/reconstruction.py | 4 ++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index cfc1fc0b84..1eb8bceca5 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -32,7 +32,6 @@ sessionUid = str(uuid.uuid1()) cacheFolderName = 'MeshroomCache' -nodesDesc: dict[str, desc.BaseNode] = {} pluginManager: NodePluginManager = NodePluginManager() submitters: dict[str, BaseSubmitter] = {} pipelineTemplates: dict[str, str] = {} @@ -408,7 +407,7 @@ def loadPipelineTemplates(folder: str): def initNodes(): additionalNodesPath = EnvVar.getList(EnvVar.MESHROOM_NODES_PATH) - nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + nodesFolders = [os.path.join(meshroomFolder, "nodes")] + additionalNodesPath for f in nodesFolders: plugins = loadAllNodes(folder=f) if plugins: diff --git a/meshroom/core/node.py b/meshroom/core/node.py index bafe728854..6d608ab4b6 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -18,7 +18,7 @@ import meshroom from meshroom.common import Signal, Variant, Property, BaseObject, Slot, ListModel, DictModel -from meshroom.core import desc, stats, hashValue, nodeVersion, Version, MrNodeType +from meshroom.core import desc, plugins, stats, hashValue, nodeVersion, Version, MrNodeType from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError @@ -201,7 +201,7 @@ def fromDict(self, d): self.mrNodeType = d.get("mrNodeType", MrNodeType.NONE) if not isinstance(self.mrNodeType, MrNodeType): self.mrNodeType = MrNodeType[self.mrNodeType] - + self.nodeName = d.get("nodeName", "") self.nodeType = d.get("nodeType", "") self.packageName = d.get("packageName", "") @@ -639,10 +639,12 @@ def __init__(self, nodeType: str, position: Position = None, parent: BaseObject super().__init__(parent) self._nodeType: str = nodeType self.nodeDesc: desc.BaseNode = None + self.nodePlugin: plugins.Plugin = None # instantiate node description if nodeType is valid - if nodeType in meshroom.core.nodesDesc: - self.nodeDesc = meshroom.core.nodesDesc[nodeType]() + if meshroom.core.pluginManager.getNodePlugin(nodeType): + self.nodeDesc = meshroom.core.pluginManager.getNodePlugin(nodeType).nodeDescriptor() + self.nodePlugin = meshroom.core.pluginManager.getNodePlugin(nodeType) self.packageName: str = "" self.packageVersion: str = "" diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index 7ca79fe53b..c3b74b3798 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -54,7 +54,9 @@ def __init__( self.internalFolder = self.nodeData.get("internalFolder") self.position = Position(*self.nodeData.get("position", [])) self.uid = self.nodeData.get("uid", None) - self.nodeDesc = meshroom.core.nodesDesc.get(self.nodeType, None) + self.nodeDesc = None + if meshroom.core.pluginManager.isRegistered(self.nodeType): + self.nodeDesc = meshroom.core.pluginManager.getNodePlugin(self.nodeType).nodeDescriptor def create(self) -> Union[Node, CompatibilityNode]: compatibilityIssue = self._checkCompatibilityIssues() diff --git a/meshroom/core/test.py b/meshroom/core/test.py index 728122fb9f..5bd92435fb 100644 --- a/meshroom/core/test.py +++ b/meshroom/core/test.py @@ -28,10 +28,10 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: for _, nodeData in graphData.items(): nodeType = nodeData["nodeType"] - if not nodeType in meshroom.core.nodesDesc: + if not meshroom.core.pluginManager.isRegistered(nodeType): return False - nodeDesc = meshroom.core.nodesDesc[nodeType] + nodeDesc = meshroom.core.pluginManager.getNodePlugin(nodeType) currentNodeVersion = meshroom.core.nodeVersion(nodeDesc) inputs = nodeData.get("inputs", {}) @@ -60,9 +60,9 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: finally: if not nodesAlreadyLoaded: - nodeTypes = [nodeType for _, nodeType in meshroom.core.nodesDesc.items()] - for nodeType in nodeTypes: - unregisterNodeType(nodeType) + nodePlugins = meshroom.core.pluginManager.getNodePlugins() + for node in nodePlugins: + meshroom.core.pluginManager.unregisterNode(node) def checkAllTemplatesVersions() -> bool: diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index e83c90e52b..275ceae0d6 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -13,7 +13,7 @@ from PySide6.QtWidgets import QApplication import meshroom -from meshroom.core import nodesDesc +from meshroom.core import pluginManager from meshroom.core.taskManager import TaskManager from meshroom.common import Property, Variant, Signal, Slot @@ -261,7 +261,7 @@ def __init__(self, inputArgs): self.engine.addImportPath(qmlDir) # expose available node types that can be instantiated - self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": nodesDesc[n].category} for n in sorted(nodesDesc.keys())}) + self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": pluginManager.getNodePlugins()[n].nodeDescriptor.category} for n in sorted(pluginManager.getNodePlugins().keys())}) # instantiate Reconstruction object self._undoStack = commands.UndoStack(self) diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index b6fadba78d..d35a38f03f 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -537,7 +537,7 @@ def initActiveNodes(self): # For all nodes declared to be accessed by the UI usedNodeTypes = {j for i in self.activeNodeCategories.values() for j in i} allUiNodes = set(self.uiNodes) | usedNodeTypes - allLoadedNodeTypes = set(meshroom.core.nodesDesc.keys()) + allLoadedNodeTypes = set(meshroom.core.pluginManager.getNodePlugins().keys()) for nodeType in allUiNodes: self._activeNodes.add(ActiveNode(nodeType, parent=self)) @@ -684,7 +684,7 @@ def setupTempCameraInit(self, node, attrName): if not sfmFile or not os.path.isfile(sfmFile): self.tempCameraInit = None return - nodeDesc = meshroom.core.nodesDesc["CameraInit"]() + nodeDesc = meshroom.core.pluginManager.getNodePlugin("CameraInit") views, intrinsics = nodeDesc.readSfMData(sfmFile) tmpCameraInit = Node("CameraInit", viewpoints=views, intrinsics=intrinsics) tmpCameraInit.locked = True From 441ba37c24bb5fba4313e467f17259c85c7972bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 12 May 2025 08:35:04 +0100 Subject: [PATCH 12/39] [core] Remove `un/registerNodeType` methods Nodes are now registered and unregistered through the `NodePluginManager`. --- meshroom/core/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 1eb8bceca5..e40d5d4e6c 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -312,22 +312,6 @@ def nodeVersion(nodeDesc: desc.Node, default=None): return moduleVersion(nodeDesc.__module__, default) -def registerNodeType(nodeType: desc.Node): - """ Register a Node Type based on a Node Description class. - - After registration, nodes of this type can be instantiated in a Graph. - """ - if nodeType.__name__ in nodesDesc: - logging.error(f"Node Desc {nodeType.__name__} is already registered.") - nodesDesc[nodeType.__name__] = nodeType - - -def unregisterNodeType(nodeType: desc.Node): - """ Remove 'nodeType' from the list of register node types. """ - assert nodeType.__name__ in nodesDesc - del nodesDesc[nodeType.__name__] - - def loadNodes(folder, packageName) -> list[NodePlugin]: if not os.path.isdir(folder): logging.error(f"Node folder '{folder}' does not exist.") From 28042dd2ad9cd7896813fc4adf0bc7c4b232f10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 12 May 2025 17:31:59 +0200 Subject: [PATCH 13/39] [tests] Harmonize and clean-up syntax across test files Also comply more with PEP8 linting rules. --- tests/nodes/test/appendFiles.py | 1 - tests/nodes/test/ls.py | 18 +- tests/test_attributeChoiceParam.py | 4 +- tests/test_compatibility.py | 190 +++++++++++---------- tests/test_graph.py | 126 +++++++------- tests/test_graphIO.py | 3 +- tests/test_invalidation.py | 20 ++- tests/test_nodeAttributeChangedCallback.py | 7 +- tests/test_nodeCommandLineFormatting.py | 34 ++-- tests/test_pipeline.py | 2 +- tests/utils.py | 2 +- 11 files changed, 218 insertions(+), 189 deletions(-) diff --git a/tests/nodes/test/appendFiles.py b/tests/nodes/test/appendFiles.py index 9f9c0b236e..abaa8575b1 100644 --- a/tests/nodes/test/appendFiles.py +++ b/tests/nodes/test/appendFiles.py @@ -39,4 +39,3 @@ class AppendFiles(desc.CommandLineNode): value='{nodeCacheFolder}/appendText.txt', ) ] - diff --git a/tests/nodes/test/ls.py b/tests/nodes/test/ls.py index d359936868..9d683e174c 100644 --- a/tests/nodes/test/ls.py +++ b/tests/nodes/test/ls.py @@ -2,21 +2,21 @@ class Ls(desc.CommandLineNode): - commandLine = 'ls {inputValue} > {outputValue}' + commandLine = "ls {inputValue} > {outputValue}" inputs = [ desc.File( - name='input', - label='Input', - description='''''', - value='', + name="input", + label="Input", + description="", + value="", ) ] outputs = [ desc.File( - name='output', - label='Output', - description='''''', - value='{nodeCacheFolder}/ls.txt', + name="output", + label="Output", + description="", + value="{nodeCacheFolder}/ls.txt", ) ] diff --git a/tests/test_attributeChoiceParam.py b/tests/test_attributeChoiceParam.py index 6527f99d1d..45a364558a 100644 --- a/tests/test_attributeChoiceParam.py +++ b/tests/test_attributeChoiceParam.py @@ -84,7 +84,7 @@ def test_overridenValuesAreNotSerialized(self, graphSavedOnDisk): graph: Graph = graphSavedOnDisk node = graph.addNewNode(NodeWithChoiceParams.__name__) node.choice.values = ["D", "E", "F"] - + graph.save() loadedGraph = loadGraph(graph.filepath) @@ -143,7 +143,7 @@ def test_overridenValuesAreSerialized(self, graphSavedOnDisk): node = graph.addNewNode(NodeWithChoiceParamsSavingValuesOverride.__name__) node.choice.values = ["D", "E", "F"] node.choiceMulti.values = ["D", "E", "F"] - + graph.save() loadedGraph = loadGraph(graph.filepath) diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 174380bc6a..70bb1d2b34 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -20,7 +20,8 @@ desc.IntParam(name="a", label="a", description="", value=0, range=None), desc.ListAttribute( name="b", - elementDesc=desc.FloatParam(name="p", label="", description="", value=0.0, range=None), + elementDesc=desc.FloatParam(name="p", label="", + description="", value=0.0, range=None), label="b", description="", ) @@ -30,7 +31,8 @@ desc.IntParam(name="a", label="a", description="", value=0, range=None), desc.ListAttribute( name="b", - elementDesc=desc.GroupAttribute(name="p", label="", description="", groupDesc=SampleGroupV1), + elementDesc=desc.GroupAttribute(name="p", label="", + description="", groupDesc=SampleGroupV1), label="b", description="", ) @@ -39,10 +41,12 @@ # SampleGroupV3 is SampleGroupV2 with one more int parameter SampleGroupV3 = [ desc.IntParam(name="a", label="a", description="", value=0, range=None), - desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", description="", value=0, range=None), + desc.IntParam(name="notInSampleGroupV2", label="notInSampleGroupV2", + description="", value=0, range=None), desc.ListAttribute( name="b", - elementDesc=desc.GroupAttribute(name="p", label="", description="", groupDesc=SampleGroupV1), + elementDesc=desc.GroupAttribute(name="p", label="", + description="", groupDesc=SampleGroupV1), label="b", description="", ) @@ -52,11 +56,12 @@ class SampleNodeV1(desc.Node): """ Version 1 Sample Node """ inputs = [ - desc.File(name='input', label='Input', description='', value='',), - desc.StringParam(name='paramA', label='ParamA', description='', value='', invalidate=False) # No impact on UID + desc.File(name="input", label="Input", description="", value=""), + desc.StringParam(name="paramA", label="ParamA", description="", + value="", invalidate=False) # No impact on UID ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -65,11 +70,12 @@ class SampleNodeV2(desc.Node): * 'input' has been renamed to 'in' """ inputs = [ - desc.File(name='in', label='Input', description='', value='',), - desc.StringParam(name='paramA', label='ParamA', description='', value='', invalidate=False), # No impact on UID + desc.File(name="in", label="Input", description="", value=""), + desc.StringParam(name="paramA", label="ParamA", description="", + value="", invalidate=False), # No impact on UID ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -79,10 +85,10 @@ class SampleNodeV3(desc.Node): * 'paramA' has been removed' """ inputs = [ - desc.File(name='in', label='Input', description='', value='',), + desc.File(name="in", label="Input", description="", value=""), ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -92,14 +98,14 @@ class SampleNodeV4(desc.Node): * 'paramA' has been added """ inputs = [ - desc.File(name='in', label='Input', description='', value='',), - desc.ListAttribute(name='paramA', label='ParamA', + desc.File(name="in", label="Input", description="", value=""), + desc.ListAttribute(name="paramA", label="ParamA", elementDesc=desc.GroupAttribute( - groupDesc=SampleGroupV1, name='gA', label='gA', description=''), - description='') + groupDesc=SampleGroupV1, name="gA", label="gA", description=""), + description="") ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -109,14 +115,14 @@ class SampleNodeV5(desc.Node): * 'paramA' elementDesc has changed from SampleGroupV1 to SampleGroupV2 """ inputs = [ - desc.File(name='in', label='Input', description='', value=''), - desc.ListAttribute(name='paramA', label='ParamA', + desc.File(name="in", label="Input", description="", value=""), + desc.ListAttribute(name="paramA", label="ParamA", elementDesc=desc.GroupAttribute( - groupDesc=SampleGroupV2, name='gA', label='gA', description=''), - description='') + groupDesc=SampleGroupV2, name="gA", label="gA", description=""), + description="") ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -126,24 +132,25 @@ class SampleNodeV6(desc.Node): * 'paramA' elementDesc has changed from SampleGroupV2 to SampleGroupV3 """ inputs = [ - desc.File(name='in', label='Input', description='', value=''), - desc.ListAttribute(name='paramA', label='ParamA', + desc.File(name="in", label="Input", description="", value=""), + desc.ListAttribute(name="paramA", label="ParamA", elementDesc=desc.GroupAttribute( - groupDesc=SampleGroupV3, name='gA', label='gA', description=''), - description='') + groupDesc=SampleGroupV3, name="gA", label="gA", description=""), + description="") ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] class SampleInputNodeV1(desc.InputNode): """ Version 1 Sample Input Node """ inputs = [ - desc.StringParam(name='path', label='path', description='', value='', invalidate=False) # No impact on UID + desc.StringParam(name="path", label="Path", description="", + value="", invalidate=False) # No impact on UID ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -152,10 +159,11 @@ class SampleInputNodeV2(desc.InputNode): * 'path' has been renamed to 'in' """ inputs = [ - desc.StringParam(name='in', label='path', description='', value='', invalidate=False) # No impact on UID + desc.StringParam(name="in", label="path", description="", + value="", invalidate=False) # No impact on UID ] outputs = [ - desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") + desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}") ] @@ -170,7 +178,7 @@ def test_unknown_node_type(): Test compatibility behavior for unknown node type. """ registerNodeType(SampleNodeV1) - g = Graph('') + g = Graph("") n = g.addNewNode("SampleNodeV1", input="/dev/null", paramA="foo") graphFile = os.path.join(tempfile.mkdtemp(), "test_unknown_node_type.mg") g.save(graphFile) @@ -178,24 +186,25 @@ def test_unknown_node_type(): nodeName = n.name unregisterNodeType(SampleNodeV1) - # reload file + + # Reload file g = loadGraph(graphFile) os.remove(graphFile) assert len(g.nodes) == 1 n = g.node(nodeName) # SampleNodeV1 is now an unknown type - # check node instance type and compatibility issue type + # Check node instance type and compatibility issue type assert isinstance(n, CompatibilityNode) assert n.issue == CompatibilityIssue.UnknownNodeType - # check if attributes are properly restored + # Check if attributes are properly restored assert len(n.attributes) == 3 assert n.input.isInput assert n.output.isOutput - # check if internal folder + # Check if internal folder assert n.internalFolder == internalFolder - # upgrade can't be perform on unknown node types + # Upgrade can't be perform on unknown node types assert not n.canUpgrade with pytest.raises(NodeUpgradeError): g.upgradeNode(nodeName) @@ -205,20 +214,20 @@ def test_description_conflict(): """ Test compatibility behavior for conflicting node descriptions. """ - # copy registered node types to be able to restore them - originalNodeTypes = copy.copy(meshroom.core.nodesDesc) + # Copy registered node types to be able to restore them + originalNodeTypes = copy.deepcopy(pluginManager.getNodePlugins()) nodeTypes = [SampleNodeV1, SampleNodeV2, SampleNodeV3, SampleNodeV4, SampleNodeV5] nodes = [] - g = Graph('') + g = Graph("") - # register and instantiate instances of all node types except last one + # Register and instantiate instances of all node types except last one for nt in nodeTypes[:-1]: - registerNodeType(nt) + pluginManager.registerNode(NodePlugin(nt)) n = g.addNewNode(nt.__name__) if nt == SampleNodeV4: - # initialize list attribute with values to create a conflict with V5 + # Initialize list attribute with values to create a conflict with V5 n.paramA.value = [{'a': 0, 'b': [1.0, 2.0]}] nodes.append(n) @@ -226,15 +235,15 @@ def test_description_conflict(): graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg") g.save(graphFile) - # reload file as-is, ensure no compatibility issue is detected (no CompatibilityNode instances) + # Reload file as-is, ensure no compatibility issue is detected (no CompatibilityNode instances) loadGraph(graphFile, strictCompatibility=True) - # offset node types register to create description conflicts - # each node type name now reference the next one's implementation + # Offset node types register to create description conflicts + # Each node type name now reference the next one's implementation for i, nt in enumerate(nodeTypes[:-1]): - meshroom.core.nodesDesc[nt.__name__] = nodeTypes[i+1] + pluginManager.getNodePlugins()[nt.__name__] = NodePlugin(nodeTypes[i + 1]) - # reload file + # Reload file g = loadGraph(graphFile) os.remove(graphFile) @@ -246,7 +255,7 @@ def test_description_conflict(): assert isinstance(compatNode, CompatibilityNode) assert srcNode.internalFolder == compatNode.internalFolder - # case by case description conflict verification + # Case by case description conflict verification if isinstance(srcNode.nodeDesc, SampleNodeV1): # V1 => V2: 'input' has been renamed to 'in' assert len(compatNode.attributes) == 3 @@ -254,27 +263,29 @@ def test_description_conflict(): assert hasattr(compatNode, "input") assert not hasattr(compatNode, "in") - # perform upgrade + # Perform upgrade upgradedNode = g.upgradeNode(nodeName) - assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV2) + assert isinstance(upgradedNode, Node) and \ + isinstance(upgradedNode.nodeDesc, SampleNodeV2) assert list(upgradedNode.attributes.keys()) == ["in", "paramA", "output"] assert not hasattr(upgradedNode, "input") assert hasattr(upgradedNode, "in") - # check uid has changed (not the same set of attributes) + # Check UID has changed (not the same set of attributes) assert upgradedNode.internalFolder != srcNode.internalFolder elif isinstance(srcNode.nodeDesc, SampleNodeV2): - # V2 => V3: 'paramA' has been removed' + # V2 => V3: 'paramA' has been removed assert len(compatNode.attributes) == 3 assert hasattr(compatNode, "paramA") - # perform upgrade + # Perform upgrade upgradedNode = g.upgradeNode(nodeName) - assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV3) + assert isinstance(upgradedNode, Node) and \ + isinstance(upgradedNode.nodeDesc, SampleNodeV3) assert not hasattr(upgradedNode, "paramA") - # check uid is identical (paramA not part of uid) + # Check UID is identical (paramA not part of UID) assert upgradedNode.internalFolder == srcNode.internalFolder elif isinstance(srcNode.nodeDesc, SampleNodeV3): @@ -282,9 +293,10 @@ def test_description_conflict(): assert len(compatNode.attributes) == 2 assert not hasattr(compatNode, "paramA") - # perform upgrade + # Perform upgrade upgradedNode = g.upgradeNode(nodeName) - assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV4) + assert isinstance(upgradedNode, Node) and \ + isinstance(upgradedNode.nodeDesc, SampleNodeV4) assert hasattr(upgradedNode, "paramA") assert isinstance(upgradedNode.paramA.attributeDesc, desc.ListAttribute) @@ -298,22 +310,24 @@ def test_description_conflict(): groupAttribute = compatNode.paramA.attributeDesc.elementDesc assert isinstance(groupAttribute, desc.GroupAttribute) - # check that Compatibility node respect SampleGroupV1 description + # Check that Compatibility node respect SampleGroupV1 description for elt in groupAttribute.groupDesc: - assert isinstance(elt, next(a for a in SampleGroupV1 if a.name == elt.name).__class__) + assert isinstance(elt, + next(a for a in SampleGroupV1 if a.name == elt.name).__class__) - # perform upgrade + # Perform upgrade upgradedNode = g.upgradeNode(nodeName) - assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV5) + assert isinstance(upgradedNode, Node) and \ + isinstance(upgradedNode.nodeDesc, SampleNodeV5) assert hasattr(upgradedNode, "paramA") - # parameter was incompatible, value could not be restored + # Parameter was incompatible, value could not be restored assert upgradedNode.paramA.isDefault assert upgradedNode.internalFolder != srcNode.internalFolder else: raise ValueError("Unexpected node type: " + srcNode.nodeType) - # restore original node types + # Restore original node types meshroom.core.nodesDesc = originalNodeTypes @@ -323,7 +337,7 @@ def test_upgradeAllNodes(): registerNodeType(SampleInputNodeV1) registerNodeType(SampleInputNodeV2) - g = Graph('') + g = Graph("") n1 = g.addNewNode("SampleNodeV1") n2 = g.addNewNode("SampleNodeV2") n3 = g.addNewNode("SampleInputNodeV1") @@ -335,28 +349,28 @@ def test_upgradeAllNodes(): graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg") g.save(graphFile) - # make SampleNodeV2 and SampleInputNodeV2 an unknown type + # Make SampleNodeV2 and SampleInputNodeV2 an unknown type unregisterNodeType(SampleNodeV2) unregisterNodeType(SampleInputNodeV2) - # replace SampleNodeV1 by SampleNodeV2 and SampleInputNodeV1 by SampleInputNodeV2 meshroom.core.nodesDesc[SampleNodeV1.__name__] = SampleNodeV2 meshroom.core.nodesDesc[SampleInputNodeV1.__name__] = SampleInputNodeV2 + # Replace SampleNodeV1 by SampleNodeV2 and SampleInputNodeV1 by SampleInputNodeV2 - # reload file + # Reload file g = loadGraph(graphFile) os.remove(graphFile) - # both nodes are CompatibilityNodes + # Both nodes are CompatibilityNodes assert len(g.compatibilityNodes) == 4 assert g.node(n1Name).canUpgrade # description conflict assert not g.node(n2Name).canUpgrade # unknown type assert g.node(n3Name).canUpgrade # description conflict assert not g.node(n4Name).canUpgrade # unknown type - # upgrade all upgradable nodes + # Upgrade all upgradable nodes g.upgradeAllNodes() - # only the nodes with an unknown type have not been upgraded + # Only the nodes with an unknown type have not been upgraded assert len(g.compatibilityNodes) == 2 assert n2Name in g.compatibilityNodes.keys() assert n4Name in g.compatibilityNodes.keys() @@ -369,36 +383,36 @@ def test_conformUpgrade(): registerNodeType(SampleNodeV5) registerNodeType(SampleNodeV6) - g = Graph('') + g = Graph("") n1 = g.addNewNode("SampleNodeV5") - n1.paramA.value = [{'a': 0, 'b': [{'a': 0, 'b': [1.0, 2.0]}, {'a': 1, 'b': [1.0, 2.0]}]}] + n1.paramA.value = [{"a": 0, "b": [{"a": 0, "b": [1.0, 2.0]}, {"a": 1, "b": [1.0, 2.0]}]}] n1Name = n1.name graphFile = os.path.join(tempfile.mkdtemp(), "test_conform_upgrade.mg") g.save(graphFile) - # replace SampleNodeV5 by SampleNodeV6 + # Replace SampleNodeV5 by SampleNodeV6 meshroom.core.nodesDesc[SampleNodeV5.__name__] = SampleNodeV6 - # reload file + # Reload file g = loadGraph(graphFile) os.remove(graphFile) - # node is a CompatibilityNode + # Node is a CompatibilityNode assert len(g.compatibilityNodes) == 1 assert g.node(n1Name).canUpgrade - # upgrade all upgradable nodes + # Upgrade all upgradable nodes g.upgradeAllNodes() - # only the node with an unknown type has not been upgraded + # Only the node with an unknown type has not been upgraded assert len(g.compatibilityNodes) == 0 upgradedNode = g.node(n1Name) - # check upgrade + # Check upgrade assert isinstance(upgradedNode, Node) and isinstance(upgradedNode.nodeDesc, SampleNodeV6) - # check conformation + # Check conformation assert len(upgradedNode.paramA.value) == 1 unregisterNodeType(SampleNodeV5) @@ -482,7 +496,7 @@ def test_loadingConflictingNodeVersionCreatesCompatibilityNodes(self, graphSaved with overrideNodeTypeVersion(SampleNodeV1, "1.0"): node = graph.addNewNode(SampleNodeV1.__name__) graph.save() - + with overrideNodeTypeVersion(SampleNodeV1, "2.0"): otherGraph = Graph("") otherGraph.load(graph.filepath) @@ -496,7 +510,7 @@ def test_loadingUnspecifiedNodeVersionAssumesCurrentVersion(self, graphSavedOnDi with registeredNodeTypes([SampleNodeV1]): graph.addNewNode(SampleNodeV1.__name__) graph.save() - + with overrideNodeTypeVersion(SampleNodeV1, "2.0"): otherGraph = Graph("") otherGraph.load(graph.filepath) @@ -508,7 +522,8 @@ class UidTestingNodeV1(desc.Node): inputs = [ desc.File(name="input", label="Input", description="", value="", invalidate=True), ] - outputs = [desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}")] + outputs = [desc.File(name="output", label="Output", + description="", value="{nodeCacheFolder}")] class UidTestingNodeV2(desc.Node): @@ -554,7 +569,8 @@ class UidTestingNodeV3(desc.Node): description="", ), ] - outputs = [desc.File(name="output", label="Output", description="", value="{nodeCacheFolder}")] + outputs = [desc.File(name="output", label="Output", + description="", value="{nodeCacheFolder}")] class TestUidConflict: @@ -626,7 +642,8 @@ def checkNodeAConnectionsToNodeB(): assert len(loadedGraph.compatibilityNodes) == 0 - def test_uidConflictDoesNotPropagateToValidDownstreamNodeThroughConnection(self, graphSavedOnDisk): + def test_uidConflictDoesNotPropagateToValidDownstreamNodeThroughConnection( + self, graphSavedOnDisk): with registeredNodeTypes([UidTestingNodeV1, UidTestingNodeV2]): graph: Graph = graphSavedOnDisk nodeA = graph.addNewNode(UidTestingNodeV2.__name__) @@ -640,7 +657,8 @@ def test_uidConflictDoesNotPropagateToValidDownstreamNodeThroughConnection(self, loadedGraph = loadGraph(graph.filepath) assert len(loadedGraph.compatibilityNodes) == 1 - def test_uidConflictDoesNotPropagateToValidDownstreamNodeThroughListConnection(self, graphSavedOnDisk): + def test_uidConflictDoesNotPropagateToValidDownstreamNodeThroughListConnection( + self,graphSavedOnDisk): with registeredNodeTypes([UidTestingNodeV2, UidTestingNodeV3]): graph: Graph = graphSavedOnDisk nodeA = graph.addNewNode(UidTestingNodeV2.__name__) diff --git a/tests/test_graph.py b/tests/test_graph.py index 003bc6ed2b..280cd46f7d 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -2,11 +2,11 @@ def test_depth(): - graph = Graph('Tests tasks depth') + graph = Graph("Tests tasks depth") - tA = graph.addNewNode('Ls', input='/tmp') - tB = graph.addNewNode('AppendText', inputText='echo B') - tC = graph.addNewNode('AppendText', inputText='echo C') + tA = graph.addNewNode("Ls", input="/tmp") + tB = graph.addNewNode("AppendText", inputText="echo B") + tC = graph.addNewNode("AppendText", inputText="echo C") graph.addEdges( (tA.output, tB.input), @@ -19,19 +19,19 @@ def test_depth(): def test_depth_diamond_graph(): - graph = Graph('Tests tasks depth') + graph = Graph("Tests tasks depth") - tA = graph.addNewNode('Ls', input='/tmp') - tB = graph.addNewNode('AppendText', inputText='echo B') - tC = graph.addNewNode('AppendText', inputText='echo C') - tD = graph.addNewNode('AppendFiles') + tA = graph.addNewNode("Ls", input="/tmp") + tB = graph.addNewNode("AppendText", inputText="echo B") + tC = graph.addNewNode("AppendText", inputText="echo C") + tD = graph.addNewNode("AppendFiles") graph.addEdges( (tA.output, tB.input), (tA.output, tC.input), (tB.output, tD.input), (tC.output, tD.input2), - ) + ) assert tA.depth == 0 assert tB.depth == 1 @@ -58,13 +58,13 @@ def test_depth_diamond_graph(): def test_depth_diamond_graph2(): - graph = Graph('Tests tasks depth') + graph = Graph("Tests tasks depth") - tA = graph.addNewNode('Ls', input='/tmp') - tB = graph.addNewNode('AppendText', inputText='echo B') - tC = graph.addNewNode('AppendText', inputText='echo C') - tD = graph.addNewNode('AppendText', inputText='echo D') - tE = graph.addNewNode('AppendFiles') + tA = graph.addNewNode("Ls", input="/tmp") + tB = graph.addNewNode("AppendText", inputText="echo B") + tC = graph.addNewNode("AppendText", inputText="echo C") + tD = graph.addNewNode("AppendText", inputText="echo D") + tE = graph.addNewNode("AppendFiles") # C # / \ # /---/---->\ @@ -81,7 +81,7 @@ def test_depth_diamond_graph2(): (tB.output, tE.input2), (tC.output, tE.input3), (tD.output, tE.input4), - ) + ) assert tA.depth == 0 assert tB.depth == 1 @@ -116,14 +116,13 @@ def test_depth_diamond_graph2(): def test_transitive_reduction(): + graph = Graph("Tests tasks depth") - graph = Graph('Tests tasks depth') - - tA = graph.addNewNode('Ls', input='/tmp') - tB = graph.addNewNode('AppendText', inputText='echo B') - tC = graph.addNewNode('AppendText', inputText='echo C') - tD = graph.addNewNode('AppendText', inputText='echo D') - tE = graph.addNewNode('AppendFiles') + tA = graph.addNewNode("Ls", input="/tmp") + tB = graph.addNewNode("AppendText", inputText="echo B") + tC = graph.addNewNode("AppendText", inputText="echo C") + tD = graph.addNewNode("AppendText", inputText="echo D") + tE = graph.addNewNode("AppendFiles") # C # / \ # /---/---->\ @@ -141,7 +140,7 @@ def test_transitive_reduction(): (tB.output, tE.input4), (tC.output, tE.input3), (tD.output, tE.input2), - ) + ) flowEdges = graph.flowEdges() flowEdgesRes = [(tB, tA), @@ -153,24 +152,24 @@ def test_transitive_reduction(): assert set(flowEdgesRes) == set(flowEdges) assert len(graph._nodesMinMaxDepths) == len(graph.nodes) - for node, (minDepth, maxDepth) in graph._nodesMinMaxDepths.items(): + for node, (_, maxDepth) in graph._nodesMinMaxDepths.items(): assert node.depth == maxDepth def test_graph_reverse_dfsOnDiscover(): - graph = Graph('Test dfsOnDiscover(reverse=True)') + graph = Graph("Test dfsOnDiscover(reverse=True)") # ------------\ # / ~ C - E - F # A - B # ~ D - A = graph.addNewNode('Ls', input='/tmp') - B = graph.addNewNode('AppendText', inputText=A.output) - C = graph.addNewNode('AppendText', inputText=B.output) - D = graph.addNewNode('AppendText', inputText=B.output) - E = graph.addNewNode('Ls', input=C.output) - F = graph.addNewNode('AppendText', input=A.output, inputText=E.output) + A = graph.addNewNode("Ls", input="/tmp") + B = graph.addNewNode("AppendText", inputText=A.output) + C = graph.addNewNode("AppendText", inputText=B.output) + D = graph.addNewNode("AppendText", inputText=B.output) + E = graph.addNewNode("Ls", input=C.output) + F = graph.addNewNode("AppendText", input=A.output, inputText=E.output) # Get all nodes from A (use set, order not guaranteed) nodes = graph.dfsOnDiscover(startNodes=[A], reverse=True)[0] @@ -179,7 +178,7 @@ def test_graph_reverse_dfsOnDiscover(): nodes = graph.dfsOnDiscover(startNodes=[B], reverse=True)[0] assert set(nodes) == {B, D, C, E, F} # Get all nodes of type AppendText from B - nodes = graph.dfsOnDiscover(startNodes=[B], filterTypes=['AppendText'], reverse=True)[0] + nodes = graph.dfsOnDiscover(startNodes=[B], filterTypes=["AppendText"], reverse=True)[0] assert set(nodes) == {B, D, C, F} # Get all nodes from C (order guaranteed) nodes = graph.dfsOnDiscover(startNodes=[C], reverse=True)[0] @@ -190,7 +189,7 @@ def test_graph_reverse_dfsOnDiscover(): def test_graph_dfsOnDiscover(): - graph = Graph('Test dfsOnDiscover(reverse=False)') + graph = Graph("Test dfsOnDiscover(reverse=False)") # ------------\ # / ~ C - E - F @@ -198,13 +197,13 @@ def test_graph_dfsOnDiscover(): # ~ D # G - G = graph.addNewNode('Ls', input='/tmp') - A = graph.addNewNode('Ls', input='/tmp') - B = graph.addNewNode('AppendText', inputText=A.output) - C = graph.addNewNode('AppendText', inputText=B.output) - D = graph.addNewNode('AppendText', input=G.output, inputText=B.output) - E = graph.addNewNode('Ls', input=C.output) - F = graph.addNewNode('AppendText', input=A.output, inputText=E.output) + G = graph.addNewNode("Ls", input="/tmp") + A = graph.addNewNode("Ls", input="/tmp") + B = graph.addNewNode("AppendText", inputText=A.output) + C = graph.addNewNode("AppendText", inputText=B.output) + D = graph.addNewNode("AppendText", input=G.output, inputText=B.output) + E = graph.addNewNode("Ls", input=C.output) + F = graph.addNewNode("AppendText", input=A.output, inputText=E.output) # Get all nodes from A (use set, order not guaranteed) nodes = graph.dfsOnDiscover(startNodes=[A], reverse=False)[0] @@ -219,7 +218,7 @@ def test_graph_dfsOnDiscover(): nodes = graph.dfsOnDiscover(startNodes=[F], reverse=False)[0] assert set(nodes) == {A, B, C, E, F} # Get all nodes of type AppendText from C - nodes = graph.dfsOnDiscover(startNodes=[C], filterTypes=['AppendText'], reverse=False)[0] + nodes = graph.dfsOnDiscover(startNodes=[C], filterTypes=["AppendText"], reverse=False)[0] assert set(nodes) == {B, C} # Get all nodes from D (order guaranteed) nodes = graph.dfsOnDiscover(startNodes=[D], longestPathFirst=True, reverse=False)[0] @@ -230,21 +229,21 @@ def test_graph_dfsOnDiscover(): def test_graph_nodes_sorting(): - graph = Graph('') + graph = Graph("") - ls0 = graph.addNewNode('Ls') - ls1 = graph.addNewNode('Ls') - ls2 = graph.addNewNode('Ls') + ls0 = graph.addNewNode("Ls") + ls1 = graph.addNewNode("Ls") + ls2 = graph.addNewNode("Ls") - assert graph.nodesOfType('Ls', sortedByIndex=True) == [ls0, ls1, ls2] + assert graph.nodesOfType("Ls", sortedByIndex=True) == [ls0, ls1, ls2] - graph = Graph('') + graph = Graph("") # 'Random' creation order (what happens when loading a file) - ls2 = graph.addNewNode('Ls', name='Ls_2') - ls0 = graph.addNewNode('Ls', name='Ls_0') - ls1 = graph.addNewNode('Ls', name='Ls_1') + ls2 = graph.addNewNode("Ls", name="Ls_2") + ls0 = graph.addNewNode("Ls", name="Ls_0") + ls1 = graph.addNewNode("Ls", name="Ls_1") - assert graph.nodesOfType('Ls', sortedByIndex=True) == [ls0, ls1, ls2] + assert graph.nodesOfType("Ls", sortedByIndex=True) == [ls0, ls1, ls2] def test_duplicate_nodes(): @@ -256,24 +255,25 @@ def test_duplicate_nodes(): # \ \ # ---------- n3 - g = Graph('') - n0 = g.addNewNode('Ls', input='/tmp') - n1 = g.addNewNode('Ls', input=n0.output) - n2 = g.addNewNode('Ls', input=n1.output) - n3 = g.addNewNode('AppendFiles', input=n1.output, input2=n2.output) + g = Graph("") + n0 = g.addNewNode("Ls", input="/tmp") + n1 = g.addNewNode("Ls", input=n0.output) + n2 = g.addNewNode("Ls", input=n1.output) + n3 = g.addNewNode("AppendFiles", input=n1.output, input2=n2.output) - # duplicate from n1 + # Duplicate from n1 nodes_to_duplicate, _ = g.dfsOnDiscover(startNodes=[n1], reverse=True, dependenciesOnly=True) nMap = g.duplicateNodes(srcNodes=nodes_to_duplicate) for s, duplicated in nMap.items(): for d in duplicated: assert s.nodeType == d.nodeType - # check number of duplicated nodes and that every parent node has been duplicated once - assert len(nMap) == 3 and all([len(nMap[i]) == 1 for i in nMap.keys()]) + # Check number of duplicated nodes and that every parent node has been duplicated once + assert len(nMap) == 3 and \ + all([len(nMap[i]) == 1 for i in nMap.keys()]) - # check connections - # access directly index 0 because we know there is a single duplicate for each parent node + # Check connections + # Access directly index 0 because we know there is a single duplicate for each parent node assert nMap[n1][0].input.getLinkParam() == n0.output assert nMap[n2][0].input.getLinkParam() == nMap[n1][0].output assert nMap[n3][0].input.getLinkParam() == nMap[n1][0].output diff --git a/tests/test_graphIO.py b/tests/test_graphIO.py index b742069192..e9de490829 100644 --- a/tests/test_graphIO.py +++ b/tests/test_graphIO.py @@ -255,7 +255,8 @@ def test_listAttributeToListAttributeConnectionIsSerialized(self): otherGraph = Graph("") otherGraph._deserialize(graph.serializePartial([nodeA, nodeB])) - assert otherGraph.node(nodeB.name).listInput.linkParam == otherGraph.node(nodeA.name).listInput + assert otherGraph.node(nodeB.name).listInput.linkParam == \ + otherGraph.node(nodeA.name).listInput def test_singleNodeWithInputConnectionFromNonSerializedNodeRemovesEdge(self): graph = Graph("") diff --git a/tests/test_invalidation.py b/tests/test_invalidation.py index 9dfe0879e7..71b77f7e19 100644 --- a/tests/test_invalidation.py +++ b/tests/test_invalidation.py @@ -7,8 +7,10 @@ class SampleNode(desc.Node): """ Sample Node for unit testing """ inputs = [ - desc.File(name='input', label='Input', description='', value='',), - desc.StringParam(name='paramA', label='ParamA', description='', value='', invalidate=False) # No impact on UID + desc.File(name="input", label="Input", description="", value="",), + desc.StringParam(name="paramA", label="ParamA", + description="", value="", + invalidate=False) # No impact on UID ] outputs = [ desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") @@ -19,10 +21,10 @@ class SampleNode(desc.Node): def test_output_invalidation(): - graph = Graph('') - n1 = graph.addNewNode('SampleNode', input='/tmp') - n2 = graph.addNewNode('SampleNode') - n3 = graph.addNewNode('SampleNode') + graph = Graph("") + n1 = graph.addNewNode("SampleNode", input="/tmp") + n2 = graph.addNewNode("SampleNode") + n3 = graph.addNewNode("SampleNode") graph.addEdges( (n1.output, n2.input), @@ -52,9 +54,9 @@ def test_inputLinkInvalidation(): """ Input links should not change the invalidation. """ - graph = Graph('') - n1 = graph.addNewNode('SampleNode') - n2 = graph.addNewNode('SampleNode') + graph = Graph("") + n1 = graph.addNewNode("SampleNode") + n2 = graph.addNewNode("SampleNode") graph.addEdges((n1.input, n2.input)) assert n1.input.uid() == n2.input.uid() diff --git a/tests/test_nodeAttributeChangedCallback.py b/tests/test_nodeAttributeChangedCallback.py index 1ab088ead7..7f3be63c8f 100644 --- a/tests/test_nodeAttributeChangedCallback.py +++ b/tests/test_nodeAttributeChangedCallback.py @@ -313,7 +313,8 @@ def test_connectionToListElementInGroup(self): class NodeWithDynamicOutputValue(desc.BaseNode): """ - A Node containing an output attribute which value is computed dynamically during graph execution. + A Node containing an output attribute which value is computed dynamically + during graph execution. """ inputs = [ @@ -390,7 +391,6 @@ def test_dynamicOutputValueComputeDoesNotTriggerDownstreamAttributeChangedCallba assert nodeB.input.value == 20 assert nodeB.affectedInput.value == 0 - def test_clearingDynamicOutputValueDoesNotTriggerDownstreamAttributeChangedCallback( self, graphSavedOnDisk ): @@ -450,9 +450,8 @@ def test_importingGraphDoesNotTriggerAttributeChangedCallbacks(self): nodeA.input.value = 5 nodeB.affectedInput.value = 2 - + otherGraph = Graph("") otherGraph.importGraphContent(graph) assert otherGraph.node(nodeB.name).affectedInput.value == 2 - diff --git a/tests/test_nodeCommandLineFormatting.py b/tests/test_nodeCommandLineFormatting.py index e7f2adf7a0..07e46960ce 100644 --- a/tests/test_nodeCommandLineFormatting.py +++ b/tests/test_nodeCommandLineFormatting.py @@ -8,7 +8,8 @@ class NodeWithAttributesNeedingFormatting(desc.Node): """ - A node containing list, file, choice and group attributes in order to test the formatting of the command line. + A node containing list, file, choice and group attributes in order to test the + formatting of the command line. """ inputs = [ desc.ListAttribute( @@ -128,23 +129,28 @@ def test_formatting_listOfFiles(self): # Assert that extending values when the list is not empty is working node.images.extend(inputImages) - assert node.images.getValueStr() == '"single value with space" "{}" "{}"'.format(inputImages[0], - inputImages[1]) + assert node.images.getValueStr() == \ + '"single value with space" "{}" "{}"'.format(inputImages[0], + inputImages[1]) - # Values are not retrieved as strings in the command line, so quotes around them are not expected - assert node._cmdVars["imagesValue"] == 'single value with space {} {}'.format(inputImages[0], - inputImages[1]) + # Values are not retrieved as strings in the command line, so quotes around them are + # not expected + assert node._cmdVars["imagesValue"] == \ + 'single value with space {} {}'.format(inputImages[0], + inputImages[1]) def test_formatting_strings(self): graph = Graph("") node = graph.addNewNode("NodeWithAttributesNeedingFormatting") node._buildCmdVars() - # Assert an empty File attribute generates empty quotes when requesting its value as a string + # Assert an empty File attribute generates empty quotes when requesting its value as + # a string assert node.input.getValueStr() == '""' assert node._cmdVars["inputValue"] == "" - # Assert a Choice attribute with a non-empty default value is surrounded with quotes when requested as a string + # Assert a Choice attribute with a non-empty default value is surrounded with quotes + # when requested as a string assert node.method.getValueStr() == '"MethodC"' assert node._cmdVars["methodValue"] == "MethodC" @@ -154,14 +160,18 @@ def test_formatting_strings(self): # Assert that the list with one empty value generates empty quotes node.images.extend("") - assert node.images.getValueStr() == '""', "A list with one empty string should generate empty quotes" - assert node._cmdVars["imagesValue"] == "", "The value is always only the value, so empty here" + assert node.images.getValueStr() == '""', \ + "A list with one empty string should generate empty quotes" + assert node._cmdVars["imagesValue"] == "", \ + "The value is always only the value, so empty here" # Assert that a list with 2 empty strings generates quotes node.images.extend("") - assert node.images.getValueStr() == '"" ""', "A list with 2 empty strings should generate quotes" + assert node.images.getValueStr() == '"" ""', \ + "A list with 2 empty strings should generate quotes" assert node._cmdVars["imagesValue"] == ' ', \ - "The value is always only the value, so 2 empty strings with the space separator in the middle" + "The value is always only the value, so 2 empty strings with the " \ + "space separator in the middle" def test_formatting_groups(self): graph = Graph("") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a536a04341..ad5e44a5da 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -44,7 +44,7 @@ def test_pipeline(): if attr.isOutput and attr.enabled: otherAttr = otherNode.attribute(key) assert attr.uid() != otherAttr.uid() - + # Test serialization/deserialization on both graphs for graph in [graph1, graph2]: filename = tempfile.mktemp() diff --git a/tests/utils.py b/tests/utils.py index c279a0ad9c..1d0bdbc77d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,7 +18,7 @@ def registeredNodeTypes(nodeTypes: list[Type[desc.Node]]): @contextmanager def overrideNodeTypeVersion(nodeType: Type[desc.Node], version: str): - """Helper context manager to override the version of a given node type.""" + """ Helper context manager to override the version of a given node type. """ unpatchedFunc = meshroom.core.nodeVersion with patch.object( meshroom.core, From 777ed4207e91ee0151a42921d5a4c2934288c251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 12 May 2025 17:33:02 +0200 Subject: [PATCH 14/39] [tests] Use the `NodePluginManager` instance in the unit tests The plugin manager is now effectively used for all the operations that involve registering or unregistering nodes. --- tests/__init__.py | 9 +++- tests/test_attributeChoiceParam.py | 17 ++++--- tests/test_compatibility.py | 53 +++++++++++++--------- tests/test_invalidation.py | 8 ++-- tests/test_nodeAttributeChangedCallback.py | 45 +++++++++++------- tests/test_nodeCallbacks.py | 9 ++-- tests/test_nodeCommandLineFormatting.py | 9 ++-- tests/utils.py | 16 ++++--- 8 files changed, 103 insertions(+), 63 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 8a6bd273ca..bdfc06e15e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,13 @@ import os -from meshroom.core import loadAllNodes, initPipelines +from meshroom.core import loadAllNodes +from meshroom.core import pluginManager + +plugins = loadAllNodes(os.path.join(os.path.dirname(__file__), "nodes")) +for plugin in plugins: + pluginManager.addPlugin(plugin) + pluginManager.registerPlugin(plugin.name) -loadAllNodes(os.path.join(os.path.dirname(__file__), "nodes")) if os.getenv("MESHROOM_PIPELINE_TEMPLATES_PATH", False): os.environ["MESHROOM_PIPELINE_TEMPLATES_PATH"] += os.pathsep + os.path.dirname(os.path.realpath(__file__)) else: diff --git a/tests/test_attributeChoiceParam.py b/tests/test_attributeChoiceParam.py index 45a364558a..125048646a 100644 --- a/tests/test_attributeChoiceParam.py +++ b/tests/test_attributeChoiceParam.py @@ -1,4 +1,5 @@ -from meshroom.core import desc, registerNodeType, unregisterNodeType +from meshroom.core import desc, pluginManager +from meshroom.core.plugins import NodePlugin from meshroom.core.graph import Graph, loadGraph @@ -52,13 +53,15 @@ class NodeWithChoiceParamsSavingValuesOverride(desc.Node): class TestChoiceParam: + nodePlugin = NodePlugin(NodeWithChoiceParams) + @classmethod def setup_class(cls): - registerNodeType(NodeWithChoiceParams) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithChoiceParams) + pluginManager.unregisterNode(cls.nodePlugin) def test_customValueIsSerialized(self, graphSavedOnDisk): graph: Graph = graphSavedOnDisk @@ -117,13 +120,15 @@ def test_connectionsAreSerialized(self, graphSavedOnDisk): class TestChoiceParamSavingCustomValues: + nodePlugin = NodePlugin(NodeWithChoiceParamsSavingValuesOverride) + @classmethod def setup_class(cls): - registerNodeType(NodeWithChoiceParamsSavingValuesOverride) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithChoiceParamsSavingValuesOverride) + pluginManager.unregisterNode(cls.nodePlugin) def test_customValueIsSerialized(self, graphSavedOnDisk): graph: Graph = graphSavedOnDisk @@ -166,4 +171,4 @@ def test_connectionsAreSerialized(self, graphSavedOnDisk): loadedNodeA = loadedGraph.node(nodeA.name) loadedNodeB = loadedGraph.node(nodeB.name) assert loadedNodeB.choice.linkParam == loadedNodeA.choice - assert loadedNodeB.choiceMulti.linkParam == loadedNodeA.choiceMulti \ No newline at end of file + assert loadedNodeB.choiceMulti.linkParam == loadedNodeA.choiceMulti diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 70bb1d2b34..0e80cb2ec2 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -7,8 +7,8 @@ from typing import Type import pytest -import meshroom.core -from meshroom.core import desc, registerNodeType, unregisterNodeType +from meshroom.core import desc, pluginManager +from meshroom.core.plugins import NodePlugin from meshroom.core.exception import GraphCompatibilityError, NodeUpgradeError from meshroom.core.graph import Graph, loadGraph from meshroom.core.node import CompatibilityNode, CompatibilityIssue, Node @@ -170,21 +170,22 @@ class SampleInputNodeV2(desc.InputNode): def replaceNodeTypeDesc(nodeType: str, nodeDesc: Type[desc.Node]): """Change the `nodeDesc` associated to `nodeType`.""" - meshroom.core.nodesDesc[nodeType] = nodeDesc + pluginManager.getNodePlugins()[nodeType] = NodePlugin(nodeDesc) def test_unknown_node_type(): """ Test compatibility behavior for unknown node type. """ - registerNodeType(SampleNodeV1) + nodePlugin = NodePlugin(SampleNodeV1) + pluginManager.registerNode(nodePlugin) g = Graph("") n = g.addNewNode("SampleNodeV1", input="/dev/null", paramA="foo") graphFile = os.path.join(tempfile.mkdtemp(), "test_unknown_node_type.mg") g.save(graphFile) internalFolder = n.internalFolder nodeName = n.name - unregisterNodeType(SampleNodeV1) + pluginManager.unregisterNode(nodePlugin) # Reload file @@ -328,14 +329,18 @@ def test_description_conflict(): raise ValueError("Unexpected node type: " + srcNode.nodeType) # Restore original node types - meshroom.core.nodesDesc = originalNodeTypes - + pluginManager._nodePlugins = originalNodeTypes def test_upgradeAllNodes(): - registerNodeType(SampleNodeV1) - registerNodeType(SampleNodeV2) - registerNodeType(SampleInputNodeV1) - registerNodeType(SampleInputNodeV2) + nodePluginSampleV1 = NodePlugin(SampleNodeV1) + nodePluginSampleV2 = NodePlugin(SampleNodeV2) + nodePluginSampleInputV1 = NodePlugin(SampleInputNodeV1) + nodePluginSampleInputV2 = NodePlugin(SampleInputNodeV2) + + pluginManager.registerNode(nodePluginSampleV1) + pluginManager.registerNode(nodePluginSampleV2) + pluginManager.registerNode(nodePluginSampleInputV1) + pluginManager.registerNode(nodePluginSampleInputV2) g = Graph("") n1 = g.addNewNode("SampleNodeV1") @@ -350,11 +355,13 @@ def test_upgradeAllNodes(): g.save(graphFile) # Make SampleNodeV2 and SampleInputNodeV2 an unknown type - unregisterNodeType(SampleNodeV2) - unregisterNodeType(SampleInputNodeV2) - meshroom.core.nodesDesc[SampleNodeV1.__name__] = SampleNodeV2 - meshroom.core.nodesDesc[SampleInputNodeV1.__name__] = SampleInputNodeV2 + pluginManager.unregisterNode(nodePluginSampleV2) + pluginManager.unregisterNode(nodePluginSampleInputV2) + # Replace SampleNodeV1 by SampleNodeV2 and SampleInputNodeV1 by SampleInputNodeV2 + pluginManager.getNodePlugins()[nodePluginSampleV1.nodeDescriptor.__name__] = nodePluginSampleV2 + pluginManager.getNodePlugins()[nodePluginSampleInputV1.nodeDescriptor.__name__] = \ + nodePluginSampleInputV2 # Reload file g = loadGraph(graphFile) @@ -375,13 +382,15 @@ def test_upgradeAllNodes(): assert n2Name in g.compatibilityNodes.keys() assert n4Name in g.compatibilityNodes.keys() - unregisterNodeType(SampleNodeV1) - unregisterNodeType(SampleInputNodeV1) + pluginManager.unregisterNode(nodePluginSampleV1) + pluginManager.unregisterNode(nodePluginSampleInputV1) def test_conformUpgrade(): - registerNodeType(SampleNodeV5) - registerNodeType(SampleNodeV6) + nodePluginSampleV5 = NodePlugin(SampleNodeV5) + nodePluginSampleV6 = NodePlugin(SampleNodeV6) + pluginManager.registerNode(nodePluginSampleV5) + pluginManager.registerNode(nodePluginSampleV6) g = Graph("") n1 = g.addNewNode("SampleNodeV5") @@ -391,7 +400,7 @@ def test_conformUpgrade(): g.save(graphFile) # Replace SampleNodeV5 by SampleNodeV6 - meshroom.core.nodesDesc[SampleNodeV5.__name__] = SampleNodeV6 + pluginManager.getNodePlugins()[nodePluginSampleV5.nodeDescriptor.__name__] = nodePluginSampleV6 # Reload file g = loadGraph(graphFile) @@ -415,8 +424,8 @@ def test_conformUpgrade(): # Check conformation assert len(upgradedNode.paramA.value) == 1 - unregisterNodeType(SampleNodeV5) - unregisterNodeType(SampleNodeV6) + pluginManager.unregisterNode(nodePluginSampleV5) + pluginManager.unregisterNode(nodePluginSampleV6) class TestGraphLoadingWithStrictCompatibility: diff --git a/tests/test_invalidation.py b/tests/test_invalidation.py index 71b77f7e19..ea4cee45d1 100644 --- a/tests/test_invalidation.py +++ b/tests/test_invalidation.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # coding:utf-8 from meshroom.core.graph import Graph -from meshroom.core import desc, registerNodeType +from meshroom.core import desc, pluginManager +from meshroom.core.plugins import NodePlugin class SampleNode(desc.Node): @@ -16,9 +17,8 @@ class SampleNode(desc.Node): desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") ] - -registerNodeType(SampleNode) - +nodePlugin = NodePlugin(SampleNode) +pluginManager.registerNode(nodePlugin) # register standalone NodePlugin def test_output_invalidation(): graph = Graph("") diff --git a/tests/test_nodeAttributeChangedCallback.py b/tests/test_nodeAttributeChangedCallback.py index 7f3be63c8f..6e3ce2fbca 100644 --- a/tests/test_nodeAttributeChangedCallback.py +++ b/tests/test_nodeAttributeChangedCallback.py @@ -1,8 +1,9 @@ # coding:utf-8 from meshroom.core.graph import Graph, loadGraph, executeGraph -from meshroom.core import desc, registerNodeType, unregisterNodeType +from meshroom.core import desc, pluginManager from meshroom.core.node import Node +from meshroom.core.plugins import NodePlugin class NodeWithAttributeChangedCallback(desc.BaseNode): @@ -37,13 +38,15 @@ def processChunk(self, chunk): class TestNodeWithAttributeChangedCallback: + nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributeChangedCallback) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributeChangedCallback) + pluginManager.unregisterNode(cls.nodePlugin) def test_assignValueTriggersCallback(self): node = Node(NodeWithAttributeChangedCallback.__name__) @@ -68,13 +71,15 @@ def test_assignNonDefaultValueTriggersCallback(self): class TestAttributeCallbackTriggerInGraph: + nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributeChangedCallback) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributeChangedCallback) + pluginManager.unregisterNode(cls.nodePlugin) def test_connectionTriggersCallback(self): graph = Graph("") @@ -219,7 +224,7 @@ class NodeWithCompoundAttributes(desc.BaseNode): desc.IntParam( name="int", label="Int", description="", value=0, range=None ) - ], + ], ) ), desc.GroupAttribute( @@ -241,15 +246,18 @@ class NodeWithCompoundAttributes(desc.BaseNode): class TestAttributeCallbackBehaviorWithUpstreamCompoundAttributes: + nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) + nodePluginCompoundAttributes = NodePlugin(NodeWithCompoundAttributes) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributeChangedCallback) - registerNodeType(NodeWithCompoundAttributes) + pluginManager.registerNode(cls.nodePluginAttributeChangedCallback) + pluginManager.registerNode(cls.nodePluginCompoundAttributes) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributeChangedCallback) - unregisterNodeType(NodeWithCompoundAttributes) + pluginManager.unregisterNode(cls.nodePluginAttributeChangedCallback) + pluginManager.unregisterNode(cls.nodePluginCompoundAttributes) def test_connectionToListElement(self): graph = Graph("") @@ -341,15 +349,18 @@ def processChunk(self, chunk): class TestAttributeCallbackBehaviorWithUpstreamDynamicOutputs: + nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) + nodePluginDynamicOutputValue = NodePlugin(NodeWithDynamicOutputValue) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributeChangedCallback) - registerNodeType(NodeWithDynamicOutputValue) + pluginManager.registerNode(cls.nodePluginAttributeChangedCallback) + pluginManager.registerNode(cls.nodePluginDynamicOutputValue) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributeChangedCallback) - unregisterNodeType(NodeWithDynamicOutputValue) + pluginManager.unregisterNode(cls.nodePluginAttributeChangedCallback) + pluginManager.unregisterNode(cls.nodePluginDynamicOutputValue) def test_connectingUncomputedDynamicOutputDoesNotTriggerDownstreamAttributeChangedCallback( self, @@ -432,13 +443,15 @@ def test_loadingGraphWithComputedDynamicOutputValueDoesNotTriggerDownstreamAttri class TestAttributeCallbackBehaviorOnGraphImport: + nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributeChangedCallback) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributeChangedCallback) + pluginManager.unregisterNode(cls.nodePlugin) def test_importingGraphDoesNotTriggerAttributeChangedCallbacks(self): graph = Graph("") diff --git a/tests/test_nodeCallbacks.py b/tests/test_nodeCallbacks.py index 149151e803..8ea3e58cbc 100644 --- a/tests/test_nodeCallbacks.py +++ b/tests/test_nodeCallbacks.py @@ -1,6 +1,7 @@ -from meshroom.core import desc, registerNodeType, unregisterNodeType +from meshroom.core import desc, pluginManager from meshroom.core.node import Node from meshroom.core.graph import Graph, loadGraph +from meshroom.core.plugins import NodePlugin class NodeWithCreationCallback(desc.InputNode): @@ -22,13 +23,15 @@ def onNodeCreated(cls, node: Node): class TestNodeCreationCallback: + nodePlugin = NodePlugin(NodeWithCreationCallback) + @classmethod def setup_class(cls): - registerNodeType(NodeWithCreationCallback) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithCreationCallback) + pluginManager.unregisterNode(cls.nodePlugin) def test_notTriggeredOnNodeInstantiation(self): node = Node(NodeWithCreationCallback.__name__) diff --git a/tests/test_nodeCommandLineFormatting.py b/tests/test_nodeCommandLineFormatting.py index 07e46960ce..6a59fbd097 100644 --- a/tests/test_nodeCommandLineFormatting.py +++ b/tests/test_nodeCommandLineFormatting.py @@ -2,8 +2,9 @@ # coding:utf-8 from meshroom.core.graph import Graph, loadGraph, executeGraph -from meshroom.core import desc, registerNodeType, unregisterNodeType +from meshroom.core import desc, pluginManager from meshroom.core.node import Node +from meshroom.core.plugins import NodePlugin class NodeWithAttributesNeedingFormatting(desc.Node): @@ -100,13 +101,15 @@ class NodeWithAttributesNeedingFormatting(desc.Node): ] class TestCommandLineFormatting: + nodePlugin = NodePlugin(NodeWithAttributesNeedingFormatting) + @classmethod def setup_class(cls): - registerNodeType(NodeWithAttributesNeedingFormatting) + pluginManager.registerNode(cls.nodePlugin) @classmethod def teardown_class(cls): - unregisterNodeType(NodeWithAttributesNeedingFormatting) + pluginManager.unregisterNode(cls.nodePlugin) def test_formatting_listOfFiles(self): inputImages = ["/non/existing/fileA", "/non/existing/with space/fileB"] diff --git a/tests/utils.py b/tests/utils.py index 1d0bdbc77d..bf93ad5570 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,23 +1,25 @@ from contextlib import contextmanager from unittest.mock import patch -from typing import Type import meshroom -from meshroom.core import registerNodeType, unregisterNodeType -from meshroom.core import desc +from meshroom.core import desc, pluginManager +from meshroom.core.plugins import NodePlugin @contextmanager -def registeredNodeTypes(nodeTypes: list[Type[desc.Node]]): +def registeredNodeTypes(nodeTypes: list[desc.Node]): + nodePluginsList = {} for nodeType in nodeTypes: - registerNodeType(nodeType) + nodePlugin = NodePlugin(nodeType) + pluginManager.registerNode(nodePlugin) + nodePluginsList[nodeType] = nodePlugin yield for nodeType in nodeTypes: - unregisterNodeType(nodeType) + pluginManager.unregisterNode(nodePluginsList[nodeType]) @contextmanager -def overrideNodeTypeVersion(nodeType: Type[desc.Node], version: str): +def overrideNodeTypeVersion(nodeType: desc.Node, version: str): """ Helper context manager to override the version of a given node type. """ unpatchedFunc = meshroom.core.nodeVersion with patch.object( From c16b56c7e3a414188c7f662d5c930cb78efeb5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 22 May 2025 12:02:16 +0200 Subject: [PATCH 15/39] Clean-up: PEP8 linting and quotes harmonization --- meshroom/core/__init__.py | 10 ++++++---- meshroom/core/desc/node.py | 12 ++++++++---- meshroom/core/nodeFactory.py | 7 ++++--- meshroom/ui/reconstruction.py | 29 +++++++++++++++++++---------- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index e40d5d4e6c..08b13cc706 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -257,7 +257,8 @@ def toComponents(versionName): status = '' # If there is a status, it is placed after a "-" splitComponents = versionName.split("-", maxsplit=1) - if (len(splitComponents) > 1): # If there is no status, splitComponents is equal to [versionName] + # If there is no status, splitComponents is equal to [versionName] + if len(splitComponents) > 1: status = splitComponents[-1] return tuple([int(v) for v in splitComponents[0].split(".")]), status @@ -354,6 +355,7 @@ def loadPluginFolder(folder): pluginManager.addPlugin(plugin) pluginManager.registerPlugin(plugin.name) pipelineTemplates.update(plugin.templates) + return plugins @@ -404,7 +406,7 @@ def initSubmitters(): additionalPaths = EnvVar.getList(EnvVar.MESHROOM_SUBMITTERS_PATH) allSubmittersFolders = [meshroomFolder] + additionalPaths for folder in allSubmittersFolders: - subs = loadSubmitters(folder, 'submitters') + subs = loadSubmitters(folder, "submitters") for sub in subs: registerSubmitter(sub()) @@ -413,7 +415,7 @@ def initPipelines(): # Load pipeline templates: check in the default folder and any folder the user might have # added to the environment variable additionalPipelinesPath = EnvVar.getList(EnvVar.MESHROOM_PIPELINE_TEMPLATES_PATH) - pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + pipelineTemplatesFolders = [os.path.join(meshroomFolder, "pipelines")] + additionalPipelinesPath for f in pipelineTemplatesFolders: loadPipelineTemplates(f) for plugin in pluginManager.getPlugins().values(): @@ -422,6 +424,6 @@ def initPipelines(): def initPlugins(): additionalpluginsPath = EnvVar.getList(EnvVar.MESHROOM_PLUGINS_PATH) - nodesFolders = [os.path.join(meshroomFolder, 'plugins')] + additionalpluginsPath + nodesFolders = [os.path.join(meshroomFolder, "plugins")] + additionalpluginsPath for f in nodesFolders: loadPluginFolder(folder=f) diff --git a/meshroom/core/desc/node.py b/meshroom/core/desc/node.py index 5c663e019c..4d940c7592 100644 --- a/meshroom/core/desc/node.py +++ b/meshroom/core/desc/node.py @@ -39,8 +39,10 @@ class BaseNode(object): name="invalidation", label="Invalidation Message", description="A message that will invalidate the node's output folder.\n" - "This is useful for development, we can invalidate the output of the node when we modify the code.\n" - "It is displayed in bold font in the invalidation/comment messages tooltip.", + "This is useful for development, we can invalidate the output of the node " + "when we modify the code.\n" + "It is displayed in bold font in the invalidation/comment messages " + "tooltip.", value="", semantic="multiline", advanced=True, @@ -50,7 +52,8 @@ class BaseNode(object): name="comment", label="Comments", description="User comments describing this specific node instance.\n" - "It is displayed in regular font in the invalidation/comment messages tooltip.", + "It is displayed in regular font in the invalidation/comment messages " + "tooltip.", value="", semantic="multiline", invalidate=False, @@ -58,7 +61,8 @@ class BaseNode(object): StringParam( name="label", label="Node's Label", - description="Customize the default label (to replace the technical name of the node instance).", + description="Customize the default label (to replace the technical name of the node " + "instance).", value="", invalidate=False, ), diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index c3b74b3798..1b3e678f35 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -123,13 +123,14 @@ def _checkAttributesNamesMatchDescription(self) -> bool: def _checkAttributesAreCompatibleWithDescription(self) -> bool: return ( self._checkAttributesCompatibility(self.nodeDesc.inputs, self.inputs) - and self._checkAttributesCompatibility(self.nodeDesc.internalInputs, self.internalInputs) + and self._checkAttributesCompatibility(self.nodeDesc.internalInputs, + self.internalInputs) and self._checkAttributesCompatibility(self.nodeDesc.outputs, self.outputs) ) def _checkInputAttributesNames(self) -> bool: def serializedInput(attr: desc.Attribute) -> bool: - """Filter that excludes not-serialized desc input attributes.""" + """ Filter that excludes not-serialized desc input attributes. """ if isinstance(attr, desc.PushButtonParam): # PushButtonParam are not serialized has they do not hold a value. return False @@ -140,7 +141,7 @@ def serializedInput(attr: desc.Attribute) -> bool: def _checkOutputAttributesNames(self) -> bool: def serializedOutput(attr: desc.Attribute) -> bool: - """Filter that excludes not-serialized desc output attributes.""" + """ Filter that excludes not-serialized desc output attributes. """ if attr.isDynamicValue: # Dynamic outputs values are not serialized with the node, # as their value is written in the computed output data. diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index d35a38f03f..7e61a07ebb 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -736,14 +736,16 @@ def lastNodeOfType(self, nodeTypes, startNode, preferredStatus=None): """ if not startNode: return None - nodes = self._graph.dfsOnDiscover(startNodes=[startNode], filterTypes=nodeTypes, reverse=True)[0] + nodes = self._graph.dfsOnDiscover(startNodes=[startNode], + filterTypes=nodeTypes, reverse=True)[0] if not nodes: return None # order the nodes according to their depth in the graph, then according to their name nodes.sort(key=lambda n: (n.depth, n.name)) node = nodes[-1] if preferredStatus: - node = next((n for n in reversed(nodes) if n.getGlobalStatus() == preferredStatus), node) + node = next((n for n in reversed(nodes) + if n.getGlobalStatus() == preferredStatus), node) return node def addSfmAugmentation(self, withMVS=False): @@ -800,7 +802,8 @@ def handleFilesUrl(self, filesByType, cameraInit=None, position=None): This method allows to reduce process time by doing it on Python side. Args: - {images, videos, panoramaInfo, meshroomScenes, otherFiles}: Map containing the lists of paths for recognized images, videos, Meshroom scenes and other files. + {images, videos, panoramaInfo, meshroomScenes, otherFiles}: Map containing the + lists of paths for recognized images, videos, Meshroom scenes and other files. Node: cameraInit node used to add new images to it QPoint: position to locate the node (usually the mouse position) """ @@ -821,7 +824,8 @@ def handleFilesUrl(self, filesByType, cameraInit=None, position=None): else: p = position cameraInit = self.addNewNode("CameraInit", position=p) - self._workerThreads.apply_async(func=self.importImagesSync, args=(filesByType["images"], cameraInit,)) + self._workerThreads.apply_async(func=self.importImagesSync, + args=(filesByType["images"], cameraInit,)) if filesByType["videos"]: if self.nodes: boundingBox = self.layout.boundingBox() @@ -840,7 +844,8 @@ def handleFilesUrl(self, filesByType, cameraInit=None, position=None): newVideoNodeMessage, "Warning: You need to manually compute the KeyframeSelection node \n" "and then reimport the created images into Meshroom for the reconstruction.\n\n" - "If you know the Camera Make/Model, it is highly recommended to declare them in the Node." + "If you know the Camera Make/Model, it is highly recommended to declare " + "them in the Node." )) if filesByType["panoramaInfo"]: @@ -848,15 +853,15 @@ def handleFilesUrl(self, filesByType, cameraInit=None, position=None): self.error.emit( Message( "Multiple XML files in input", - "Ignore the xml Panorama files:\n\n'{}'.".format(',\n'.join(filesByType["panoramaInfo"])), + "Ignore the XML Panorama files:\n\n'{}'.".format(',\n'.join(filesByType["panoramaInfo"])), "", )) else: - panoramaInitNodes = self.graph.nodesOfType('PanoramaInit') + panoramaInitNodes = self.graph.nodesOfType("PanoramaInit") for panoramaInfoFile in filesByType["panoramaInfo"]: for panoramaInitNode in panoramaInitNodes: - panoramaInitNode.attribute('initializeCameras').value = 'File' - panoramaInitNode.attribute('config').value = panoramaInfoFile + panoramaInitNode.attribute("initializeCameras").value = "File" + panoramaInitNode.attribute("config").value = panoramaInfoFile if panoramaInitNodes: self.info.emit( Message( @@ -918,7 +923,11 @@ def getFilesByTypeFromDrop(self, urls): filesByType.extend(multiview.findFilesByTypeInFolder(localFile)) else: filesByType.addFile(localFile) - return {"images": filesByType.images, "videos": filesByType.videos, "panoramaInfo": filesByType.panoramaInfo, "meshroomScenes": filesByType.meshroomScenes, "other": filesByType.other} + return {"images": filesByType.images, + "videos": filesByType.videos, + "panoramaInfo": filesByType.panoramaInfo, + "meshroomScenes": filesByType.meshroomScenes, + "other": filesByType.other} def importImagesFromFolder(self, path, recursive=False): """ From 98d90dae81276b72af0872a12af61c02467fcce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 22 May 2025 16:17:20 +0200 Subject: [PATCH 16/39] [core] plugins: Rename `getNodePlugin...` to `getRegisteredNodePlugin...` This prevents ambiguities between `NodePlugin` objects that have been registered (and are thus instantiable) and those that belong to `Plugin` objects but have not been registered. --- meshroom/core/node.py | 6 +++--- meshroom/core/nodeFactory.py | 2 +- meshroom/core/plugins.py | 4 ++-- meshroom/core/test.py | 4 ++-- meshroom/ui/app.py | 2 +- meshroom/ui/reconstruction.py | 4 ++-- tests/test_compatibility.py | 12 ++++++------ 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 6d608ab4b6..ddf8931df6 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -642,9 +642,9 @@ def __init__(self, nodeType: str, position: Position = None, parent: BaseObject self.nodePlugin: plugins.Plugin = None # instantiate node description if nodeType is valid - if meshroom.core.pluginManager.getNodePlugin(nodeType): - self.nodeDesc = meshroom.core.pluginManager.getNodePlugin(nodeType).nodeDescriptor() - self.nodePlugin = meshroom.core.pluginManager.getNodePlugin(nodeType) + if meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType): + self.nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType).nodeDescriptor() + self.nodePlugin = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType) self.packageName: str = "" self.packageVersion: str = "" diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index 1b3e678f35..ed4c8977bd 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -56,7 +56,7 @@ def __init__( self.uid = self.nodeData.get("uid", None) self.nodeDesc = None if meshroom.core.pluginManager.isRegistered(self.nodeType): - self.nodeDesc = meshroom.core.pluginManager.getNodePlugin(self.nodeType).nodeDescriptor + self.nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(self.nodeType).nodeDescriptor def create(self) -> Union[Node, CompatibilityNode]: compatibilityIssue = self._checkCompatibilityIssues() diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 027fe3fbda..ac8c92a938 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -262,14 +262,14 @@ def addPlugin(self, plugin: Plugin): if not self.getPlugin(plugin.name): self._plugins[plugin.name] = plugin - def getNodePlugins(self) -> dict[str: NodePlugin]: + def getRegisteredNodePlugins(self) -> dict[str: NodePlugin]: """ Return a dictionary containing all the registered NodePlugins, with {key, value} = {name, NodePlugin}. """ return self._nodePlugins - def getNodePlugin(self, name: str) -> NodePlugin: + def getRegisteredNodePlugin(self, name: str) -> NodePlugin: """ Return the NodePlugin object that has been registered under the name "name" if it exists. diff --git a/meshroom/core/test.py b/meshroom/core/test.py index 5bd92435fb..2ea2a1c07d 100644 --- a/meshroom/core/test.py +++ b/meshroom/core/test.py @@ -31,7 +31,7 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: if not meshroom.core.pluginManager.isRegistered(nodeType): return False - nodeDesc = meshroom.core.pluginManager.getNodePlugin(nodeType) + nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin(nodeType) currentNodeVersion = meshroom.core.nodeVersion(nodeDesc) inputs = nodeData.get("inputs", {}) @@ -60,7 +60,7 @@ def checkTemplateVersions(path: str, nodesAlreadyLoaded: bool = False) -> bool: finally: if not nodesAlreadyLoaded: - nodePlugins = meshroom.core.pluginManager.getNodePlugins() + nodePlugins = meshroom.core.pluginManager.getRegisteredNodePlugins() for node in nodePlugins: meshroom.core.pluginManager.unregisterNode(node) diff --git a/meshroom/ui/app.py b/meshroom/ui/app.py index 275ceae0d6..37a44c974e 100644 --- a/meshroom/ui/app.py +++ b/meshroom/ui/app.py @@ -261,7 +261,7 @@ def __init__(self, inputArgs): self.engine.addImportPath(qmlDir) # expose available node types that can be instantiated - self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": pluginManager.getNodePlugins()[n].nodeDescriptor.category} for n in sorted(pluginManager.getNodePlugins().keys())}) + self.engine.rootContext().setContextProperty("_nodeTypes", {n: {"category": pluginManager.getRegisteredNodePlugins()[n].nodeDescriptor.category} for n in sorted(pluginManager.getRegisteredNodePlugins().keys())}) # instantiate Reconstruction object self._undoStack = commands.UndoStack(self) diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 7e61a07ebb..1d05ec0851 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -537,7 +537,7 @@ def initActiveNodes(self): # For all nodes declared to be accessed by the UI usedNodeTypes = {j for i in self.activeNodeCategories.values() for j in i} allUiNodes = set(self.uiNodes) | usedNodeTypes - allLoadedNodeTypes = set(meshroom.core.pluginManager.getNodePlugins().keys()) + allLoadedNodeTypes = set(meshroom.core.pluginManager.getRegisteredNodePlugins().keys()) for nodeType in allUiNodes: self._activeNodes.add(ActiveNode(nodeType, parent=self)) @@ -684,7 +684,7 @@ def setupTempCameraInit(self, node, attrName): if not sfmFile or not os.path.isfile(sfmFile): self.tempCameraInit = None return - nodeDesc = meshroom.core.pluginManager.getNodePlugin("CameraInit") + nodeDesc = meshroom.core.pluginManager.getRegisteredNodePlugin("CameraInit") views, intrinsics = nodeDesc.readSfMData(sfmFile) tmpCameraInit = Node("CameraInit", viewpoints=views, intrinsics=intrinsics) tmpCameraInit.locked = True diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index 0e80cb2ec2..ae0338e0ed 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -170,7 +170,7 @@ class SampleInputNodeV2(desc.InputNode): def replaceNodeTypeDesc(nodeType: str, nodeDesc: Type[desc.Node]): """Change the `nodeDesc` associated to `nodeType`.""" - pluginManager.getNodePlugins()[nodeType] = NodePlugin(nodeDesc) + pluginManager.getRegisteredNodePlugins()[nodeType] = NodePlugin(nodeDesc) def test_unknown_node_type(): @@ -216,7 +216,7 @@ def test_description_conflict(): Test compatibility behavior for conflicting node descriptions. """ # Copy registered node types to be able to restore them - originalNodeTypes = copy.deepcopy(pluginManager.getNodePlugins()) + originalNodeTypes = copy.deepcopy(pluginManager.getRegisteredNodePlugins()) nodeTypes = [SampleNodeV1, SampleNodeV2, SampleNodeV3, SampleNodeV4, SampleNodeV5] nodes = [] @@ -242,7 +242,7 @@ def test_description_conflict(): # Offset node types register to create description conflicts # Each node type name now reference the next one's implementation for i, nt in enumerate(nodeTypes[:-1]): - pluginManager.getNodePlugins()[nt.__name__] = NodePlugin(nodeTypes[i + 1]) + pluginManager.getRegisteredNodePlugins()[nt.__name__] = NodePlugin(nodeTypes[i + 1]) # Reload file g = loadGraph(graphFile) @@ -359,8 +359,8 @@ def test_upgradeAllNodes(): pluginManager.unregisterNode(nodePluginSampleInputV2) # Replace SampleNodeV1 by SampleNodeV2 and SampleInputNodeV1 by SampleInputNodeV2 - pluginManager.getNodePlugins()[nodePluginSampleV1.nodeDescriptor.__name__] = nodePluginSampleV2 - pluginManager.getNodePlugins()[nodePluginSampleInputV1.nodeDescriptor.__name__] = \ + pluginManager.getRegisteredNodePlugins()[nodePluginSampleV1.nodeDescriptor.__name__] = nodePluginSampleV2 + pluginManager.getRegisteredNodePlugins()[nodePluginSampleInputV1.nodeDescriptor.__name__] = \ nodePluginSampleInputV2 # Reload file @@ -400,7 +400,7 @@ def test_conformUpgrade(): g.save(graphFile) # Replace SampleNodeV5 by SampleNodeV6 - pluginManager.getNodePlugins()[nodePluginSampleV5.nodeDescriptor.__name__] = nodePluginSampleV6 + pluginManager.getRegisteredNodePlugins()[nodePluginSampleV5.nodeDescriptor.__name__] = nodePluginSampleV6 # Reload file g = loadGraph(graphFile) From a8a54f67fb43a6eb6adbcee3061a0c9ff4f7d24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 22 May 2025 16:22:32 +0200 Subject: [PATCH 17/39] [core] plugins: Support finding unregistered nodes in plugins A `NodePlugin` may be part of a `Plugin` even if it has not been registered. `containsNode` allows to check whether a `NodePlugin` is contained within a `Plugin`, and `belongsToPlugin` --- meshroom/core/plugins.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index ac8c92a938..e75af08c17 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -148,6 +148,16 @@ def loadTemplates(self): if file.endswith(".mg"): self._templates[os.path.splitext(file)[0]] = os.path.join(self.path, file) + def containsNodePlugin(self, name: str) -> bool: + """ + Return whether the node plugin "name" is part of the plugin, independently from its + status. + + Args: + name: the name of the node plugin to be checked. + """ + return name in self._nodePlugins + class NodePlugin(BaseObject): """ @@ -231,6 +241,22 @@ def isRegistered(self, name: str) -> bool: """ return name in self._nodePlugins + def belongsToPlugin(self, name: str) -> Plugin: + """ + Check whether the node plugin belongs to a loaded plugin, independently from + whether it has been registered or not. + + Args: + name: the name of the node plugin that needs to be searched for across plugins. + + Returns: + Plugin | None: the Plugin the node belongs to if it exists, None otherwise. + """ + for plugin in self._plugins.values(): + if plugin.containsNodePlugin(name): + return plugin + return None + def getPlugins(self) -> dict[str: Plugin]: """ Return a dictionary containing all the loaded Plugins, with {key, value} = From 0adb3754f39ca8b24f67425fb1850d1e8262adab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 22 May 2025 16:28:41 +0200 Subject: [PATCH 18/39] [core] Add `PluginIssue` compatibility issue --- meshroom/core/node.py | 1 + meshroom/core/nodeFactory.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index ddf8931df6..50ee7ba617 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1816,6 +1816,7 @@ class CompatibilityIssue(Enum): VersionConflict = 2 # mismatch between node's description version and serialized node data DescriptionConflict = 3 # mismatch between node's description attributes and serialized node data UidConflict = 4 # mismatch between computed UIDs and UIDs stored in serialized node data + PluginIssue = 5 # issue when loading the associated plugin class CompatibilityNode(BaseNode): diff --git a/meshroom/core/nodeFactory.py b/meshroom/core/nodeFactory.py index ed4c8977bd..23997f7c83 100644 --- a/meshroom/core/nodeFactory.py +++ b/meshroom/core/nodeFactory.py @@ -76,6 +76,8 @@ def _normalizeNodeData(self): def _checkCompatibilityIssues(self) -> Optional[CompatibilityIssue]: if self.nodeDesc is None: + if meshroom.core.pluginManager.belongsToPlugin(self.nodeType) is not None: + return CompatibilityIssue.PluginIssue return CompatibilityIssue.UnknownNodeType if not self._checkUidCompatibility(): From b3ee2ad329cee3c4ed6a7adac58813fb2276638e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 26 May 2025 15:27:38 +0100 Subject: [PATCH 19/39] [core] plugins: Add new methods to the manager to manipulate plugins --- meshroom/core/plugins.py | 53 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index e75af08c17..bfa7bc2549 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -102,6 +102,14 @@ def path(self): """ Return the absolute path of the plugin. """ return self._path + @property + def nodes(self): + """ + Return the dictionary containing the NodePlugin objects associated to + the plugin. + """ + return self._nodePlugins + @property def templates(self): """ Return the list of templates associated to the plugin. """ @@ -278,15 +286,36 @@ def getPlugin(self, name: str) -> Plugin: return self._plugins[name] return None - def addPlugin(self, plugin: Plugin): + def addPlugin(self, plugin: Plugin, registerNodePlugins: bool = True): """ Load a Plugin object. Args: plugin: the Plugin to load and add to the list of loaded plugins. + registerNodePlugins: True if all the NodePlugins from the plugin should be registered + at the same time the plugin is being loaded. Otherwise, the + NodePlugins will have to be registered at a later occasion. """ if not self.getPlugin(plugin.name): self._plugins[plugin.name] = plugin + if registerNodePlugins: + self.registerPlugin(plugin.name) + + def removePlugin(self, plugin: Plugin, unregisterNodePlugins: bool = True): + """ + Remove a loaded Plugin object. + + Args: + plugin: the Plugin to remove from the list of loaded plugins. + unregisterNodePlugins: True if all the nodes from the plugin should be unregistered (if they + are registered) at the same time as the plugin is unloaded. Otherwise, + the registered NodePlugins will remain while the Plugin itself will + be unloaded. + """ + if self.getPlugin(plugin.name): + if unregisterNodePlugins: + self.unregisterPlugin(plugin.name) + del self._plugins[plugin.name] def getRegisteredNodePlugins(self) -> dict[str: NodePlugin]: """ @@ -314,15 +343,28 @@ def registerPlugin(self, name: str): Register all the NodePlugins contained in the Plugin loaded as "name". Args: - name: the name of the Plugin whose NodePlugins will be registered. + name: the name of the Plugin whose NodePlugins will be registered. """ plugin = self.getPlugin(name) if plugin: - for node in plugin._nodePlugins: - self.registerNode(plugin._nodePlugins[node]) + for node in plugin.nodes: + self.registerNode(plugin.nodes[node]) else: logging.error(f"No loaded Plugin named {name}.") + def unregisterPlugin(self, name: str): + """ + Unregister all the NodePlugins contained in the Plugin loaded as "name" + that are currently registered. + + Args: + name: the name of the Plugin whose NodePlugins will be unregistered. + """ + plugin = self.getPlugin(name) + if plugin: + for node in plugin.nodes.values(): + self.unregisterNode(node) + def registerNode(self, nodePlugin: NodePlugin): """ Register a node plugin. A registered node plugin will become instantiable. @@ -353,5 +395,6 @@ def unregisterNode(self, nodePlugin: NodePlugin): if self.isRegistered(name): if nodePlugin.status != NodePluginStatus.LOADED: logging.warning(f"NodePlugin {name} is registered but is not correctly loaded.") + else: + nodePlugin.status = NodePluginStatus.NOT_LOADED del self._nodePlugins[name] - nodePlugin.status = NodePluginStatus.NOT_LOADED From 3af5acfa912337e4c2bbe5f69188cd8ee762d34e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 26 May 2025 15:28:31 +0100 Subject: [PATCH 20/39] Plugins: Use simplified load/register of plugins and nodes when possible --- meshroom/core/__init__.py | 2 -- tests/__init__.py | 1 - 2 files changed, 3 deletions(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 08b13cc706..84cd06a430 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -353,7 +353,6 @@ def loadPluginFolder(folder): if plugins: for plugin in plugins: pluginManager.addPlugin(plugin) - pluginManager.registerPlugin(plugin.name) pipelineTemplates.update(plugin.templates) return plugins @@ -399,7 +398,6 @@ def initNodes(): if plugins: for plugin in plugins: pluginManager.addPlugin(plugin) - pluginManager.registerPlugin(plugin.name) def initSubmitters(): diff --git a/tests/__init__.py b/tests/__init__.py index bdfc06e15e..6ebeba20f3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,6 @@ plugins = loadAllNodes(os.path.join(os.path.dirname(__file__), "nodes")) for plugin in plugins: pluginManager.addPlugin(plugin) - pluginManager.registerPlugin(plugin.name) if os.getenv("MESHROOM_PIPELINE_TEMPLATES_PATH", False): os.environ["MESHROOM_PIPELINE_TEMPLATES_PATH"] += os.pathsep + os.path.dirname(os.path.realpath(__file__)) From 5bc09c88474f2e759e9893baf13bfc9dadbe8386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Mon, 26 May 2025 17:41:37 +0200 Subject: [PATCH 21/39] [core] plugins: Add a method to reload a `NodePlugin` Additionnally, add a new error status to distinguish an error during the registration of the plugin (`LOADING_ERROR`) and an error when the module itself is reloaded (through the `reload` function from importlib). --- meshroom/core/plugins.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index bfa7bc2549..9aff48b086 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -1,7 +1,9 @@ from __future__ import annotations +import importlib import logging import os +import sys from enum import Enum from inspect import getfile @@ -63,7 +65,8 @@ class NodePluginStatus(Enum): NOT_LOADED = 0 # The node plugin exists but is not loaded and cannot be used (not registered) LOADED = 1 # The node plugin is currently loaded and functional (it has been registered) DESC_ERROR = 2 # The node plugin exists but has an invalid description - ERROR = 3 # The node plugin exists and is valid but could not be successfully loaded + LOADING_ERROR = 3 # The node plugin exists and is valid but could not be successfully registered + ERROR = 4 # Error when importing the node plugin from its module class Plugin(BaseObject): @@ -197,6 +200,23 @@ def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): self._processEnv = None + def reload(self): + """ Reload the node plugin and update its status accordingly. """ + updated = importlib.reload(sys.modules.get(self.nodeDescriptor.__module__)) + descriptor = getattr(updated, self.nodeDescriptor.__name__) + + if not descriptor: + self.status = NodePluginStatus.ERROR + return + + self.nodeDescriptor = descriptor + self.errors = validateNodeDesc(descriptor) + + if self.errors: + self.status = NodePluginStatus.DESC_ERROR + else: + self.status = NodePluginStatus.NOT_LOADED + @property def plugin(self): """ @@ -375,13 +395,14 @@ def registerNode(self, nodePlugin: NodePlugin): nodePlugin: the node plugin to register. """ name = nodePlugin.nodeDescriptor.__name__ - if not self.isRegistered(name) and nodePlugin.status != NodePluginStatus.DESC_ERROR: + if not self.isRegistered(name) and nodePlugin.status not in (NodePluginStatus.DESC_ERROR, + NodePluginStatus.ERROR): try: self._nodePlugins[name] = nodePlugin nodePlugin.status = NodePluginStatus.LOADED except Exception as e: logging.error(f"NodePlugin {name} could not be loaded: {e}") - nodePlugin.status = NodePluginStatus.ERROR + nodePlugin.status = NodePluginStatus.LOADING_ERROR def unregisterNode(self, nodePlugin: NodePlugin): """ From aa4d9ad92bd8044b0cac99a2026e12311d13685a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Tue, 27 May 2025 11:48:14 +0200 Subject: [PATCH 22/39] [tests] Add initial set of unit tests for plugins Tests cover loading/unloading plugins, registering/unregistering node plugins, and reloading them. --- .../plugins/meshroom/pluginA/PluginANodeA.py | 22 ++ .../plugins/meshroom/pluginA/PluginANodeB.py | 28 +++ tests/plugins/meshroom/pluginA/__init__.py | 0 .../plugins/meshroom/pluginB/PluginBNodeA.py | 22 ++ .../plugins/meshroom/pluginB/PluginBNodeB.py | 28 +++ tests/plugins/meshroom/pluginB/__init__.py | 0 tests/plugins/meshroom/sharedTemplate.mg | 10 + tests/test_plugins.py | 200 ++++++++++++++++++ 8 files changed, 310 insertions(+) create mode 100644 tests/plugins/meshroom/pluginA/PluginANodeA.py create mode 100644 tests/plugins/meshroom/pluginA/PluginANodeB.py create mode 100644 tests/plugins/meshroom/pluginA/__init__.py create mode 100644 tests/plugins/meshroom/pluginB/PluginBNodeA.py create mode 100644 tests/plugins/meshroom/pluginB/PluginBNodeB.py create mode 100644 tests/plugins/meshroom/pluginB/__init__.py create mode 100644 tests/plugins/meshroom/sharedTemplate.mg create mode 100644 tests/test_plugins.py diff --git a/tests/plugins/meshroom/pluginA/PluginANodeA.py b/tests/plugins/meshroom/pluginA/PluginANodeA.py new file mode 100644 index 0000000000..95182ec993 --- /dev/null +++ b/tests/plugins/meshroom/pluginA/PluginANodeA.py @@ -0,0 +1,22 @@ +__version__ = "1.0" + +from meshroom.core import desc + +class PluginANodeA(desc.Node): + inputs = [ + desc.File( + name="input", + label="Input", + description="", + value="", + ), + ] + + outputs = [ + desc.File( + name="output", + label="Output", + description="", + value="", + ), + ] \ No newline at end of file diff --git a/tests/plugins/meshroom/pluginA/PluginANodeB.py b/tests/plugins/meshroom/pluginA/PluginANodeB.py new file mode 100644 index 0000000000..ff048e943c --- /dev/null +++ b/tests/plugins/meshroom/pluginA/PluginANodeB.py @@ -0,0 +1,28 @@ +__version__ = "1.0" + +from meshroom.core import desc + +class PluginANodeB(desc.Node): + inputs = [ + desc.File( + name="input", + label="Input", + description="", + value="", + ), + desc.IntParam( + name="int", + label="Integer", + description="", + value=1, + ), + ] + + outputs = [ + desc.File( + name="output", + label="Output", + description="", + value="", + ), + ] \ No newline at end of file diff --git a/tests/plugins/meshroom/pluginA/__init__.py b/tests/plugins/meshroom/pluginA/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plugins/meshroom/pluginB/PluginBNodeA.py b/tests/plugins/meshroom/pluginB/PluginBNodeA.py new file mode 100644 index 0000000000..f25ce6b0b8 --- /dev/null +++ b/tests/plugins/meshroom/pluginB/PluginBNodeA.py @@ -0,0 +1,22 @@ +__version__ = "1.0" + +from meshroom.core import desc + +class PluginBNodeA(desc.Node): + inputs = [ + desc.File( + name="input", + label="Input", + description="", + value="", + ), + ] + + outputs = [ + desc.File( + name="output", + label="Output", + description="", + value="", + ), + ] \ No newline at end of file diff --git a/tests/plugins/meshroom/pluginB/PluginBNodeB.py b/tests/plugins/meshroom/pluginB/PluginBNodeB.py new file mode 100644 index 0000000000..613d39362e --- /dev/null +++ b/tests/plugins/meshroom/pluginB/PluginBNodeB.py @@ -0,0 +1,28 @@ +__version__ = "1.0" + +from meshroom.core import desc + +class PluginBNodeB(desc.Node): + inputs = [ + desc.File( + name="input", + label="Input", + description="", + value="", + ), + desc.IntParam( + name="int", + label="Integer", + description="", + value="not an integer", + ), + ] + + outputs = [ + desc.File( + name="output", + label="Output", + description="", + value="", + ), + ] \ No newline at end of file diff --git a/tests/plugins/meshroom/pluginB/__init__.py b/tests/plugins/meshroom/pluginB/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/plugins/meshroom/sharedTemplate.mg b/tests/plugins/meshroom/sharedTemplate.mg new file mode 100644 index 0000000000..4e265f8c9f --- /dev/null +++ b/tests/plugins/meshroom/sharedTemplate.mg @@ -0,0 +1,10 @@ +{ + "header": { + "releaseVersion": "2025.1.0-develop", + "fileVersion": "2.0", + "nodesVersions": {}, + "template": true + }, + "graph": { + } +} \ No newline at end of file diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000000..a32ec12935 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,200 @@ +# coding:utf-8 + +from meshroom.core import desc, pluginManager, loadClassesNodes +from meshroom.core.plugins import NodePluginStatus, Plugin + +from itertools import islice +import os + +class TestPluginWithValidNodesOnly: + plugin = None + + @classmethod + def setup_class(cls): + folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + package = "pluginA" + cls.plugin = Plugin(package, folder) + nodes = loadClassesNodes(folder, package) + for node in nodes: + cls.plugin.addNodePlugin(node) + pluginManager.addPlugin(cls.plugin) + + @classmethod + def teardown_class(cls): + pluginManager.unregisterPlugin(cls.plugin) + cls.plugin = None + + def test_loadedPlugin(self): + # Assert that there are loaded plugins, and that "pluginA" is one of them + assert len(pluginManager.getPlugins()) >= 1 + plugin = pluginManager.getPlugin("pluginA") + assert plugin == self.plugin + assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + + # Assert that the nodes of pluginA have been successfully registered + assert len(pluginManager.getRegisteredNodePlugins()) >= 2 + for nodeName, nodePlugin in plugin.nodes.items(): + assert nodePlugin.status == NodePluginStatus.LOADED + assert pluginManager.isRegistered(nodeName) + + # Assert the template has been loaded + assert len(plugin.templates) == 1 + name = list(plugin.templates.keys())[0] + assert name == "sharedTemplate" + assert plugin.templates[name] == os.path.join(str(plugin.path), "sharedTemplate.mg") + + def test_unloadPlugin(self): + plugin = pluginManager.getPlugin("pluginA") + assert plugin == self.plugin + + # Unload the plugin without unregistering the nodes + pluginManager.removePlugin(plugin, unregisterNodePlugins=False) + + # Assert the plugin is not loaded anymore + assert pluginManager.getPlugin(plugin.name) is None + + # Assert the nodes are still registered and belong to an unloaded plugin + for nodeName, nodePlugin in plugin.nodes.items(): + assert nodePlugin.status == NodePluginStatus.LOADED + assert pluginManager.isRegistered(nodeName) + assert pluginManager.belongsToPlugin(nodeName) is None + + # Re-add the plugin + pluginManager.addPlugin(plugin, registerNodePlugins=False) + assert pluginManager.getPlugin(plugin.name) + + # Unload the plugin with a full unregistration of the nodes + pluginManager.removePlugin(plugin) + + # Assert the plugin is not loaded anymore + assert pluginManager.getPlugin(plugin.name) is None + + # Assert the nodes have been successfully unregistered + for nodeName, nodePlugin in plugin.nodes.items(): + assert nodePlugin.status == NodePluginStatus.NOT_LOADED + assert not pluginManager.isRegistered(nodeName) + + # Re-add the plugin and re-register the nodes + pluginManager.addPlugin(plugin) + assert pluginManager.getPlugin(plugin.name) + for nodeName, nodePlugin in plugin.nodes.items(): + assert nodePlugin.status == NodePluginStatus.LOADED + assert pluginManager.isRegistered(nodeName) + + def test_updateRegisteredNodes(self): + nbRegisteredNodes = len(pluginManager.getRegisteredNodePlugins()) + plugin = pluginManager.getPlugin("pluginA") + assert plugin == self.plugin + nodeA = pluginManager.getRegisteredNodePlugin("PluginANodeA") + nodeAName = nodeA.nodeDescriptor.__name__ + + # Unregister a node + assert nodeA + pluginManager.unregisterNode(nodeA) + + # Check that the node has been fully unregistered: + # - its status is "NOT_LOADED" + # - it is still part of pluginA + # - it is not in the list of registered plugins anymore (and returns None when requested) + assert nodeA.status == NodePluginStatus.NOT_LOADED + assert plugin.containsNodePlugin(nodeAName) + assert nodeA.plugin == plugin + + assert pluginManager.getRegisteredNodePlugin(nodeAName) is None + assert nodeAName not in pluginManager.getRegisteredNodePlugins() + assert len(pluginManager.getRegisteredNodePlugins()) == nbRegisteredNodes - 1 + + # Re-register the node + pluginManager.registerNode(nodeA) + + assert nodeA.status == NodePluginStatus.LOADED + assert pluginManager.getRegisteredNodePlugin(nodeAName) + assert len(pluginManager.getRegisteredNodePlugins()) == nbRegisteredNodes + + +class TestPluginWithInvalidNodes: + plugin = None + + @classmethod + def setup_class(cls): + folder = os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + package = "pluginB" + cls.plugin = Plugin(package, folder) + nodes = loadClassesNodes(folder, package) + for node in nodes: + cls.plugin.addNodePlugin(node) + pluginManager.addPlugin(cls.plugin) + + @classmethod + def teardown_class(cls): + pluginManager.unregisterPlugin(cls.plugin) + cls.plugin = None + + def test_loadedPlugin(self): + # Assert that there are loaded plugins, and that "pluginB" is one of them + assert len(pluginManager.getPlugins()) >= 1 + plugin = pluginManager.getPlugin("pluginB") + assert plugin == self.plugin + assert str(plugin.path) == os.path.join(os.path.dirname(__file__), "plugins", "meshroom") + + # Assert that PluginBNodeA is successfully registered + assert pluginManager.isRegistered("PluginBNodeA") + assert plugin.nodes["PluginBNodeA"].status == NodePluginStatus.LOADED + assert plugin.nodes["PluginBNodeA"].plugin == plugin + + # Assert that PluginBNodeB has not been registered (description error) + assert not pluginManager.isRegistered("PluginBNodeB") + assert plugin.nodes["PluginBNodeB"].status == NodePluginStatus.DESC_ERROR + assert plugin.nodes["PluginBNodeB"].plugin == plugin + + # Assert the template has been loaded + assert len(plugin.templates) == 1 + name = list(plugin.templates.keys())[0] + assert name == "sharedTemplate" + assert plugin.templates[name] == os.path.join(str(plugin.path), "sharedTemplate.mg") + + def test_reloadNodePlugin(self): + plugin = pluginManager.getPlugin("pluginB") + assert plugin == self.plugin + node = plugin.nodes["PluginBNodeB"] + nodeName = node.nodeDescriptor.__name__ + + # Check that the node has not been registered + assert node.status == NodePluginStatus.DESC_ERROR + assert not pluginManager.isRegistered(nodeName) + + # Check that the node cannot be registered + pluginManager.registerNode(node) + assert not pluginManager.isRegistered(nodeName) + + # Replace directly in the node file the line that fails the validation + # on the description with a line that will pass + originalFileContent = None + with open(node.path, "r") as f: + originalFileContent = f.read() + + replaceFileContent = originalFileContent.replace('"not an integer"', '1') + with open(node.path, "w") as f: + f.write(replaceFileContent) + + # Reload the node and assert it is valid + node.reload() + assert node.status == NodePluginStatus.NOT_LOADED + + # Attempt to register node plugin + pluginManager.registerNode(node) + assert pluginManager.isRegistered(nodeName) + + # Restore the node file to its original state (with a description error) + with open(node.path, "w") as f: + f.write(originalFileContent) + + # Reload the node and assert it is invalid while still registered + node.reload() + assert node.status == NodePluginStatus.DESC_ERROR + assert pluginManager.isRegistered(nodeName) + + # Unregister it + pluginManager.unregisterNode(node) + assert node.status == NodePluginStatus.DESC_ERROR # Not NOT_LOADED + assert not pluginManager.isRegistered(nodeName) From 424abbff82e137440e38cce9d800319481bbbabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 4 Jun 2025 20:29:25 +0200 Subject: [PATCH 23/39] [core] plugins: Check `NodePlugin`'s timestamp before reloading it When calling the `reload()` method for `NodePlugins`, we now check that the timestamp of the node's description file doesn't match with the timestamp of that same file when the `NodePlugin` was created. If it does match, then nothing happens during the `reload()`. --- meshroom/core/plugins.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 9aff48b086..c354070973 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -184,6 +184,8 @@ class NodePlugin(BaseObject): processEnv: the environment required for the node plugin's process. It can either be specific to this node plugin, or be common for all the node plugins within the plugin + timestamp: the timestamp corresponding to the last time the node description's file has been + modified """ def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): @@ -199,14 +201,23 @@ def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): self.status = NodePluginStatus.DESC_ERROR self._processEnv = None + self._timestamp = os.path.getmtime(self.path) def reload(self): """ Reload the node plugin and update its status accordingly. """ + if self._timestamp == os.path.getmtime(self.path): + logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Not reloading. The node description " + f"at {self.path} has not been modified since the last load.") + return + updated = importlib.reload(sys.modules.get(self.nodeDescriptor.__module__)) descriptor = getattr(updated, self.nodeDescriptor.__name__) + self._timestamp = os.path.getmtime(self.path) if not descriptor: self.status = NodePluginStatus.ERROR + logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " + "was not found.") return self.nodeDescriptor = descriptor @@ -214,8 +225,11 @@ def reload(self): if self.errors: self.status = NodePluginStatus.DESC_ERROR + logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " + "has description errors.") else: self.status = NodePluginStatus.NOT_LOADED + logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Successful reloading.") @property def plugin(self): From 3c57afb4d01ef0cb1ec7041ec958193f46f99b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 4 Jun 2025 21:18:16 +0200 Subject: [PATCH 24/39] [tests] Simplify registration/unregistration of nodes in tests Add two methods that are local to tests, `registerNodeDesc` and `unregisterNodeDesc`, which handle the registration and unregistration of `NodePlugins` from a node description. This reduces the amount of code in tests whenever `NodePlugin` need to be instantiated prior to their registration and so on. --- tests/test_attributeChoiceParam.py | 12 ++--- tests/test_compatibility.py | 56 ++++++++++------------ tests/test_invalidation.py | 6 +-- tests/test_nodeAttributeChangedCallback.py | 41 +++++++--------- tests/test_nodeCallbacks.py | 8 ++-- tests/test_nodeCommandLineFormatting.py | 8 ++-- tests/test_plugins.py | 7 +-- tests/utils.py | 15 +++++- 8 files changed, 76 insertions(+), 77 deletions(-) diff --git a/tests/test_attributeChoiceParam.py b/tests/test_attributeChoiceParam.py index 125048646a..094bb3a12b 100644 --- a/tests/test_attributeChoiceParam.py +++ b/tests/test_attributeChoiceParam.py @@ -1,7 +1,7 @@ from meshroom.core import desc, pluginManager -from meshroom.core.plugins import NodePlugin from meshroom.core.graph import Graph, loadGraph +from .utils import registerNodeDesc, unregisterNodeDesc class NodeWithChoiceParams(desc.Node): inputs = [ @@ -53,15 +53,14 @@ class NodeWithChoiceParamsSavingValuesOverride(desc.Node): class TestChoiceParam: - nodePlugin = NodePlugin(NodeWithChoiceParams) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithChoiceParams) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithChoiceParams) def test_customValueIsSerialized(self, graphSavedOnDisk): graph: Graph = graphSavedOnDisk @@ -120,15 +119,14 @@ def test_connectionsAreSerialized(self, graphSavedOnDisk): class TestChoiceParamSavingCustomValues: - nodePlugin = NodePlugin(NodeWithChoiceParamsSavingValuesOverride) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithChoiceParamsSavingValuesOverride) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithChoiceParamsSavingValuesOverride) def test_customValueIsSerialized(self, graphSavedOnDisk): graph: Graph = graphSavedOnDisk diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py index ae0338e0ed..3d3ea4d811 100644 --- a/tests/test_compatibility.py +++ b/tests/test_compatibility.py @@ -13,7 +13,7 @@ from meshroom.core.graph import Graph, loadGraph from meshroom.core.node import CompatibilityNode, CompatibilityIssue, Node -from .utils import registeredNodeTypes, overrideNodeTypeVersion +from .utils import registeredNodeTypes, overrideNodeTypeVersion, registerNodeDesc, unregisterNodeDesc SampleGroupV1 = [ @@ -177,15 +177,14 @@ def test_unknown_node_type(): """ Test compatibility behavior for unknown node type. """ - nodePlugin = NodePlugin(SampleNodeV1) - pluginManager.registerNode(nodePlugin) + registerNodeDesc(SampleNodeV1) g = Graph("") n = g.addNewNode("SampleNodeV1", input="/dev/null", paramA="foo") graphFile = os.path.join(tempfile.mkdtemp(), "test_unknown_node_type.mg") g.save(graphFile) internalFolder = n.internalFolder nodeName = n.name - pluginManager.unregisterNode(nodePlugin) + unregisterNodeDesc(SampleNodeV1) # Reload file @@ -224,7 +223,7 @@ def test_description_conflict(): # Register and instantiate instances of all node types except last one for nt in nodeTypes[:-1]: - pluginManager.registerNode(NodePlugin(nt)) + registerNodeDesc(nt) n = g.addNewNode(nt.__name__) if nt == SampleNodeV4: @@ -332,15 +331,10 @@ def test_description_conflict(): pluginManager._nodePlugins = originalNodeTypes def test_upgradeAllNodes(): - nodePluginSampleV1 = NodePlugin(SampleNodeV1) - nodePluginSampleV2 = NodePlugin(SampleNodeV2) - nodePluginSampleInputV1 = NodePlugin(SampleInputNodeV1) - nodePluginSampleInputV2 = NodePlugin(SampleInputNodeV2) - - pluginManager.registerNode(nodePluginSampleV1) - pluginManager.registerNode(nodePluginSampleV2) - pluginManager.registerNode(nodePluginSampleInputV1) - pluginManager.registerNode(nodePluginSampleInputV2) + registerNodeDesc(SampleNodeV1) + registerNodeDesc(SampleNodeV2) + registerNodeDesc(SampleInputNodeV1) + registerNodeDesc(SampleInputNodeV2) g = Graph("") n1 = g.addNewNode("SampleNodeV1") @@ -354,14 +348,15 @@ def test_upgradeAllNodes(): graphFile = os.path.join(tempfile.mkdtemp(), "test_description_conflict.mg") g.save(graphFile) - # Make SampleNodeV2 and SampleInputNodeV2 an unknown type - pluginManager.unregisterNode(nodePluginSampleV2) - pluginManager.unregisterNode(nodePluginSampleInputV2) - # Replace SampleNodeV1 by SampleNodeV2 and SampleInputNodeV1 by SampleInputNodeV2 - pluginManager.getRegisteredNodePlugins()[nodePluginSampleV1.nodeDescriptor.__name__] = nodePluginSampleV2 - pluginManager.getRegisteredNodePlugins()[nodePluginSampleInputV1.nodeDescriptor.__name__] = \ - nodePluginSampleInputV2 + pluginManager.getRegisteredNodePlugins()[SampleNodeV1.__name__] = \ + pluginManager.getRegisteredNodePlugin(SampleNodeV2.__name__) + pluginManager.getRegisteredNodePlugins()[SampleInputNodeV1.__name__] = \ + pluginManager.getRegisteredNodePlugin(SampleInputNodeV2.__name__) + + # Make SampleNodeV2 and SampleInputNodeV2 an unknown type + unregisterNodeDesc(SampleNodeV2) + unregisterNodeDesc(SampleInputNodeV2) # Reload file g = loadGraph(graphFile) @@ -382,15 +377,13 @@ def test_upgradeAllNodes(): assert n2Name in g.compatibilityNodes.keys() assert n4Name in g.compatibilityNodes.keys() - pluginManager.unregisterNode(nodePluginSampleV1) - pluginManager.unregisterNode(nodePluginSampleInputV1) + unregisterNodeDesc(SampleNodeV1) + unregisterNodeDesc(SampleInputNodeV1) def test_conformUpgrade(): - nodePluginSampleV5 = NodePlugin(SampleNodeV5) - nodePluginSampleV6 = NodePlugin(SampleNodeV6) - pluginManager.registerNode(nodePluginSampleV5) - pluginManager.registerNode(nodePluginSampleV6) + registerNodeDesc(SampleNodeV5) + registerNodeDesc(SampleNodeV6) g = Graph("") n1 = g.addNewNode("SampleNodeV5") @@ -400,7 +393,8 @@ def test_conformUpgrade(): g.save(graphFile) # Replace SampleNodeV5 by SampleNodeV6 - pluginManager.getRegisteredNodePlugins()[nodePluginSampleV5.nodeDescriptor.__name__] = nodePluginSampleV6 + pluginManager.getRegisteredNodePlugins()[SampleNodeV5.__name__] = \ + pluginManager.getRegisteredNodePlugin(SampleNodeV6.__name__) # Reload file g = loadGraph(graphFile) @@ -424,8 +418,8 @@ def test_conformUpgrade(): # Check conformation assert len(upgradedNode.paramA.value) == 1 - pluginManager.unregisterNode(nodePluginSampleV5) - pluginManager.unregisterNode(nodePluginSampleV6) + unregisterNodeDesc(SampleNodeV5) + unregisterNodeDesc(SampleNodeV6) class TestGraphLoadingWithStrictCompatibility: @@ -441,7 +435,6 @@ def test_failsOnUnknownNodeType(self, graphSavedOnDisk): def test_failsOnNodeDescriptionCompatibilityIssue(self, graphSavedOnDisk): - with registeredNodeTypes([SampleNodeV1, SampleNodeV2]): graph: Graph = graphSavedOnDisk graph.addNewNode(SampleNodeV1.__name__) @@ -456,7 +449,6 @@ def test_failsOnNodeDescriptionCompatibilityIssue(self, graphSavedOnDisk): class TestGraphTemplateLoading: def test_failsOnUnknownNodeTypeError(self, graphSavedOnDisk): - with registeredNodeTypes([SampleNodeV1, SampleNodeV2]): graph: Graph = graphSavedOnDisk graph.addNewNode(SampleNodeV1.__name__) diff --git a/tests/test_invalidation.py b/tests/test_invalidation.py index ea4cee45d1..f9ef972c86 100644 --- a/tests/test_invalidation.py +++ b/tests/test_invalidation.py @@ -2,7 +2,8 @@ # coding:utf-8 from meshroom.core.graph import Graph from meshroom.core import desc, pluginManager -from meshroom.core.plugins import NodePlugin + +from .utils import registerNodeDesc class SampleNode(desc.Node): @@ -17,8 +18,7 @@ class SampleNode(desc.Node): desc.File(name='output', label='Output', description='', value="{nodeCacheFolder}") ] -nodePlugin = NodePlugin(SampleNode) -pluginManager.registerNode(nodePlugin) # register standalone NodePlugin +registerNodeDesc(SampleNode) # register standalone NodePlugin def test_output_invalidation(): graph = Graph("") diff --git a/tests/test_nodeAttributeChangedCallback.py b/tests/test_nodeAttributeChangedCallback.py index 6e3ce2fbca..1628d61248 100644 --- a/tests/test_nodeAttributeChangedCallback.py +++ b/tests/test_nodeAttributeChangedCallback.py @@ -3,7 +3,8 @@ from meshroom.core.graph import Graph, loadGraph, executeGraph from meshroom.core import desc, pluginManager from meshroom.core.node import Node -from meshroom.core.plugins import NodePlugin + +from .utils import registerNodeDesc, unregisterNodeDesc class NodeWithAttributeChangedCallback(desc.BaseNode): @@ -38,15 +39,14 @@ def processChunk(self, chunk): class TestNodeWithAttributeChangedCallback: - nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithAttributeChangedCallback) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithAttributeChangedCallback) def test_assignValueTriggersCallback(self): node = Node(NodeWithAttributeChangedCallback.__name__) @@ -71,15 +71,14 @@ def test_assignNonDefaultValueTriggersCallback(self): class TestAttributeCallbackTriggerInGraph: - nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithAttributeChangedCallback) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithAttributeChangedCallback) def test_connectionTriggersCallback(self): graph = Graph("") @@ -246,18 +245,16 @@ class NodeWithCompoundAttributes(desc.BaseNode): class TestAttributeCallbackBehaviorWithUpstreamCompoundAttributes: - nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) - nodePluginCompoundAttributes = NodePlugin(NodeWithCompoundAttributes) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePluginAttributeChangedCallback) - pluginManager.registerNode(cls.nodePluginCompoundAttributes) + registerNodeDesc(NodeWithAttributeChangedCallback) + registerNodeDesc(NodeWithCompoundAttributes) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePluginAttributeChangedCallback) - pluginManager.unregisterNode(cls.nodePluginCompoundAttributes) + unregisterNodeDesc(NodeWithAttributeChangedCallback) + unregisterNodeDesc(NodeWithCompoundAttributes) def test_connectionToListElement(self): graph = Graph("") @@ -349,18 +346,18 @@ def processChunk(self, chunk): class TestAttributeCallbackBehaviorWithUpstreamDynamicOutputs: - nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) - nodePluginDynamicOutputValue = NodePlugin(NodeWithDynamicOutputValue) + # nodePluginAttributeChangedCallback = NodePlugin(NodeWithAttributeChangedCallback) + # nodePluginDynamicOutputValue = NodePlugin(NodeWithDynamicOutputValue) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePluginAttributeChangedCallback) - pluginManager.registerNode(cls.nodePluginDynamicOutputValue) + registerNodeDesc(NodeWithAttributeChangedCallback) + registerNodeDesc(NodeWithDynamicOutputValue) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePluginAttributeChangedCallback) - pluginManager.unregisterNode(cls.nodePluginDynamicOutputValue) + unregisterNodeDesc(NodeWithAttributeChangedCallback) + unregisterNodeDesc(NodeWithDynamicOutputValue) def test_connectingUncomputedDynamicOutputDoesNotTriggerDownstreamAttributeChangedCallback( self, @@ -443,15 +440,13 @@ def test_loadingGraphWithComputedDynamicOutputValueDoesNotTriggerDownstreamAttri class TestAttributeCallbackBehaviorOnGraphImport: - nodePlugin = NodePlugin(NodeWithAttributeChangedCallback) - @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithAttributeChangedCallback) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithAttributeChangedCallback) def test_importingGraphDoesNotTriggerAttributeChangedCallbacks(self): graph = Graph("") diff --git a/tests/test_nodeCallbacks.py b/tests/test_nodeCallbacks.py index 8ea3e58cbc..9fbc676ca7 100644 --- a/tests/test_nodeCallbacks.py +++ b/tests/test_nodeCallbacks.py @@ -1,7 +1,8 @@ from meshroom.core import desc, pluginManager from meshroom.core.node import Node from meshroom.core.graph import Graph, loadGraph -from meshroom.core.plugins import NodePlugin + +from .utils import registerNodeDesc, unregisterNodeDesc class NodeWithCreationCallback(desc.InputNode): @@ -23,15 +24,14 @@ def onNodeCreated(cls, node: Node): class TestNodeCreationCallback: - nodePlugin = NodePlugin(NodeWithCreationCallback) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithCreationCallback) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithCreationCallback) def test_notTriggeredOnNodeInstantiation(self): node = Node(NodeWithCreationCallback.__name__) diff --git a/tests/test_nodeCommandLineFormatting.py b/tests/test_nodeCommandLineFormatting.py index 6a59fbd097..9c5ea69b05 100644 --- a/tests/test_nodeCommandLineFormatting.py +++ b/tests/test_nodeCommandLineFormatting.py @@ -4,7 +4,8 @@ from meshroom.core.graph import Graph, loadGraph, executeGraph from meshroom.core import desc, pluginManager from meshroom.core.node import Node -from meshroom.core.plugins import NodePlugin + +from .utils import registerNodeDesc, unregisterNodeDesc class NodeWithAttributesNeedingFormatting(desc.Node): @@ -101,15 +102,14 @@ class NodeWithAttributesNeedingFormatting(desc.Node): ] class TestCommandLineFormatting: - nodePlugin = NodePlugin(NodeWithAttributesNeedingFormatting) @classmethod def setup_class(cls): - pluginManager.registerNode(cls.nodePlugin) + registerNodeDesc(NodeWithAttributesNeedingFormatting) @classmethod def teardown_class(cls): - pluginManager.unregisterNode(cls.nodePlugin) + unregisterNodeDesc(NodeWithAttributesNeedingFormatting) def test_formatting_listOfFiles(self): inputImages = ["/non/existing/fileA", "/non/existing/with space/fileB"] diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a32ec12935..40d13d699e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -3,7 +3,6 @@ from meshroom.core import desc, pluginManager, loadClassesNodes from meshroom.core.plugins import NodePluginStatus, Plugin -from itertools import islice import os class TestPluginWithValidNodesOnly: @@ -21,7 +20,8 @@ def setup_class(cls): @classmethod def teardown_class(cls): - pluginManager.unregisterPlugin(cls.plugin) + for node in cls.plugin.nodes.values(): + pluginManager.unregisterNode(node) cls.plugin = None def test_loadedPlugin(self): @@ -127,7 +127,8 @@ def setup_class(cls): @classmethod def teardown_class(cls): - pluginManager.unregisterPlugin(cls.plugin) + for node in cls.plugin.nodes.values(): + pluginManager.unregisterNode(node) cls.plugin = None def test_loadedPlugin(self): diff --git a/tests/utils.py b/tests/utils.py index bf93ad5570..93b4bf6d90 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,7 +3,7 @@ import meshroom from meshroom.core import desc, pluginManager -from meshroom.core.plugins import NodePlugin +from meshroom.core.plugins import NodePlugin, NodePluginStatus @contextmanager def registeredNodeTypes(nodeTypes: list[desc.Node]): @@ -28,3 +28,16 @@ def overrideNodeTypeVersion(nodeType: desc.Node, version: str): side_effect=lambda type: version if type is nodeType else unpatchedFunc(type), ): yield + +def registerNodeDesc(nodeDesc: desc.Node): + name = nodeDesc.__name__ + if not pluginManager.isRegistered(name): + pluginManager._nodePlugins[name] = NodePlugin(nodeDesc) + pluginManager._nodePlugins[name].status = NodePluginStatus.LOADED + +def unregisterNodeDesc(nodeDesc: desc.Node): + name = nodeDesc.__name__ + if pluginManager.isRegistered(name): + plugin = pluginManager.getRegisteredNodePlugin(name) + plugin.status = NodePluginStatus.NOT_LOADED + del pluginManager._nodePlugins[name] From a19c306cf34d3f498f76e24765954db6760a2654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Wed, 4 Jun 2025 21:21:42 +0200 Subject: [PATCH 25/39] [core] plugins: Remove `register/unregisterPlugin` methods Remove the methods and perform the registration/unregistration directly within the functions that called them. This simplifies the code and prevents ambiguity between the different functions. --- meshroom/core/plugins.py | 33 ++++----------------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index c354070973..46a63397d7 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -333,7 +333,8 @@ def addPlugin(self, plugin: Plugin, registerNodePlugins: bool = True): if not self.getPlugin(plugin.name): self._plugins[plugin.name] = plugin if registerNodePlugins: - self.registerPlugin(plugin.name) + for node in plugin.nodes: + self.registerNode(plugin.nodes[node]) def removePlugin(self, plugin: Plugin, unregisterNodePlugins: bool = True): """ @@ -348,7 +349,8 @@ def removePlugin(self, plugin: Plugin, unregisterNodePlugins: bool = True): """ if self.getPlugin(plugin.name): if unregisterNodePlugins: - self.unregisterPlugin(plugin.name) + for node in plugin.nodes.values(): + self.unregisterNode(node) del self._plugins[plugin.name] def getRegisteredNodePlugins(self) -> dict[str: NodePlugin]: @@ -372,33 +374,6 @@ def getRegisteredNodePlugin(self, name: str) -> NodePlugin: return self._nodePlugins[name] return None - def registerPlugin(self, name: str): - """ - Register all the NodePlugins contained in the Plugin loaded as "name". - - Args: - name: the name of the Plugin whose NodePlugins will be registered. - """ - plugin = self.getPlugin(name) - if plugin: - for node in plugin.nodes: - self.registerNode(plugin.nodes[node]) - else: - logging.error(f"No loaded Plugin named {name}.") - - def unregisterPlugin(self, name: str): - """ - Unregister all the NodePlugins contained in the Plugin loaded as "name" - that are currently registered. - - Args: - name: the name of the Plugin whose NodePlugins will be unregistered. - """ - plugin = self.getPlugin(name) - if plugin: - for node in plugin.nodes.values(): - self.unregisterNode(node) - def registerNode(self, nodePlugin: NodePlugin): """ Register a node plugin. A registered node plugin will become instantiable. From 91e753c1146df35b44c5816a013a9459fe8f4089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 5 Jun 2025 16:04:55 +0200 Subject: [PATCH 26/39] [core] plugins: Handle corner cases when reloading nodes --- meshroom/core/plugins.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 46a63397d7..9f42e4bd2e 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -205,23 +205,32 @@ def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): def reload(self): """ Reload the node plugin and update its status accordingly. """ - if self._timestamp == os.path.getmtime(self.path): + timestamp = 0.0 + try: + timestamp = os.path.getmtime(self.path) + except FileNotFoundError: + self.status = NodePluginStatus.ERROR + logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The path at {self.path} was not " + "not found.") + return + + if self._timestamp == timestamp: logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Not reloading. The node description " f"at {self.path} has not been modified since the last load.") return updated = importlib.reload(sys.modules.get(self.nodeDescriptor.__module__)) descriptor = getattr(updated, self.nodeDescriptor.__name__) - self._timestamp = os.path.getmtime(self.path) if not descriptor: self.status = NodePluginStatus.ERROR logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " - "was not found.") + "was not found.") return self.nodeDescriptor = descriptor self.errors = validateNodeDesc(descriptor) + self._timestamp = timestamp if self.errors: self.status = NodePluginStatus.DESC_ERROR From 3d36ea5f2a51b52617777c56c25a5a63557e7b12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 5 Jun 2025 16:06:43 +0200 Subject: [PATCH 27/39] [core] graph: Add a `reloadAllNodes` method to reload the current graph This will replace all the nodes in the current graph with a new instance of the same type. If any change has occured in the description of these nodes since their last instantiation, the new nodes will reflect them. --- meshroom/core/graph.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 4a0f9aae8a..5a6269fa33 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -759,6 +759,16 @@ def upgradeAllNodes(self): for nodeName in nodeNames: self.upgradeNode(nodeName) + def reloadAllNodes(self): + """ + Replace all the node instances in the current graph with new node instances of the same + type. If the description of the nodes has changed, the reloaded nodes will reflect theses + changes. + """ + for node in self._nodes.values(): + newNode = nodeFactory(node.toDict(), node.nodeType, expectedUid=node._uid) + self.replaceNode(node.name, newNode) + @Slot(str, result=Attribute) def attribute(self, fullName): # type: (str) -> Attribute From 0c5e76997a959172f06abd3d721b8b584e36ca9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 5 Jun 2025 16:09:56 +0200 Subject: [PATCH 28/39] [ui] Add a "Reload All Nodes" menu to reload all the plugins' nodes "Reload All Nodes" will reload all the descriptions for `NodePlugins` that are part of loaded plugins, whether they are registered or not. Once the reload of the descriptions has been performed, the node instances from the current graph will be reloaded as well to reflect any change in the description. --- meshroom/ui/qml/Application.qml | 15 +++++++++++++++ meshroom/ui/reconstruction.py | 13 +++++++++++++ 2 files changed, 28 insertions(+) diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index de7201449c..ecf5d49aac 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -552,6 +552,15 @@ Page { } } + Action { + id: reloadAllNodesAction + property string tooltip: "Reload all the node descriptions for all the registered plugins" + text: "Reload All Nodes" + onTriggered: { + _reconstruction.reloadAllNodes() + } + } + Action { id: undoAction @@ -830,6 +839,12 @@ Page { ToolTip.visible: hovered ToolTip.text: removeImagesFromAllGroupsAction.tooltip } + + MenuItem { + action: reloadAllNodesAction + ToolTip.visible: hovered + ToolTip.text: reloadAllNodesAction.tooltip + } } MenuSeparator { } Action { diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 1d05ec0851..e0f8d98b81 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -552,6 +552,19 @@ def onCameraInitChanged(self): nodes = self._graph.dfsOnDiscover(startNodes=[self._cameraInit], reverse=True)[0] self.setActiveNodes(nodes) + @Slot() + def reloadAllNodes(self): + """ + Reload all the NodePlugins from all the registered plugins. + The nodes in the graph will be updated to match the changes in the description, if + there was any. + """ + for plugin in meshroom.core.pluginManager.getPlugins().values(): + for node in plugin.nodes.values(): + node.reload() + + self._graph.reloadAllNodes() + @Slot() @Slot(str) def new(self, pipeline=None): From 98fbfae0133d4f0bb7ffeaf1ba9af3f20ca9db68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 5 Jun 2025 16:54:03 +0200 Subject: [PATCH 29/39] [tests] Plugins: Add a `sleep` between file rewrites On some systems such as GitHub's Windows CI, the `write` operation is too fast and does not cause a change in the timestamp of the file we're reloading, hence causing the test to fail for external reasons. Adding a sleep does not change anything to the test functionally, but on the contrary ensures that we are actually testing the feature. https://stackoverflow.com/questions/19059877/python-os-path-getmtime-time-not-changing --- tests/test_plugins.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 40d13d699e..4ee2e0a4b9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -4,6 +4,7 @@ from meshroom.core.plugins import NodePluginStatus, Plugin import os +import time class TestPluginWithValidNodesOnly: plugin = None @@ -185,10 +186,23 @@ def test_reloadNodePlugin(self): # Attempt to register node plugin pluginManager.registerNode(node) assert pluginManager.isRegistered(nodeName) + + # Reload the node again without any change + node.reload() + assert pluginManager.isRegistered(nodeName) + # Hack to ensure that the timestamp of the file will be different after being rewritten + # Without it, on some systems, the operation is too fast and the timestamp does not change, + # cause the test to fail + time.sleep(0.1) + # Restore the node file to its original state (with a description error) with open(node.path, "w") as f: f.write(originalFileContent) + + timestampOr2 = os.path.getmtime(node.path) + print(f"New timestamp: {timestampOr2}") + print(os.stat(node.path)) # Reload the node and assert it is invalid while still registered node.reload() From a6d80b3de784f0c41df98b8f0f9568a0e3a87d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Thu, 5 Jun 2025 16:55:59 +0200 Subject: [PATCH 30/39] [ui] Application: Add shortcut for the "Reload All Nodes" menu --- meshroom/ui/qml/Application.qml | 1 + 1 file changed, 1 insertion(+) diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index ecf5d49aac..c19ce6af9e 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -556,6 +556,7 @@ Page { id: reloadAllNodesAction property string tooltip: "Reload all the node descriptions for all the registered plugins" text: "Reload All Nodes" + shortcut: "Ctrl+Shift+R" onTriggered: { _reconstruction.reloadAllNodes() } From 24e14572bac998b00e5bca5eed6eefe80808398c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 6 Jun 2025 16:06:57 +0200 Subject: [PATCH 31/39] [core] graph: Ensure all nodes are looped over in `reloadAllNodes` --- meshroom/core/graph.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 5a6269fa33..cd9a6e91da 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -765,9 +765,15 @@ def reloadAllNodes(self): type. If the description of the nodes has changed, the reloaded nodes will reflect theses changes. """ + newNodes: dict[str, BaseNode] = {} for node in self._nodes.values(): newNode = nodeFactory(node.toDict(), node.nodeType, expectedUid=node._uid) - self.replaceNode(node.name, newNode) + newNodes[node.name] = newNode + + # Replace in a different loop to ensure all the nodes have been looped over: when looping + # over self._nodes and replacing nodes at the same time, some nodes might not be reached + for name, node in newNodes.items(): + self.replaceNode(name, node) @Slot(str, result=Attribute) def attribute(self, fullName): From c58cf999206ebc8a6d4000866881b6b8c392d391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 6 Jun 2025 17:53:38 +0200 Subject: [PATCH 32/39] [core] plugins: `reload`: Return a bool depending on the reloading status If the plugin has successfully been reloaded, return `True`. If it has not been reloaded for any reason (either an error or because no modification has been made to it since it has been loaded last), then return `False`. --- meshroom/core/plugins.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/meshroom/core/plugins.py b/meshroom/core/plugins.py index 9f42e4bd2e..a25ff5563e 100644 --- a/meshroom/core/plugins.py +++ b/meshroom/core/plugins.py @@ -203,8 +203,15 @@ def __init__(self, nodeDesc: desc.Node, plugin: Plugin = None): self._processEnv = None self._timestamp = os.path.getmtime(self.path) - def reload(self): - """ Reload the node plugin and update its status accordingly. """ + def reload(self) -> bool: + """ + Reload the node plugin and update its status accordingly. If the timestamp of the node plugin's + path has not changed since the last time the plugin has been loaded, then nothing will happen. + + Returns: + bool: True if the node plugin has successfully been reloaded (i.e. there was no error, and + some changes were made since its last loading), False otherwise. + """ timestamp = 0.0 try: timestamp = os.path.getmtime(self.path) @@ -212,12 +219,12 @@ def reload(self): self.status = NodePluginStatus.ERROR logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The path at {self.path} was not " "not found.") - return + return False if self._timestamp == timestamp: logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Not reloading. The node description " f"at {self.path} has not been modified since the last load.") - return + return False updated = importlib.reload(sys.modules.get(self.nodeDescriptor.__module__)) descriptor = getattr(updated, self.nodeDescriptor.__name__) @@ -226,19 +233,20 @@ def reload(self): self.status = NodePluginStatus.ERROR logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " "was not found.") - return + return False - self.nodeDescriptor = descriptor self.errors = validateNodeDesc(descriptor) - self._timestamp = timestamp - if self.errors: self.status = NodePluginStatus.DESC_ERROR logging.error(f"[Reload] {self.nodeDescriptor.__name__}: The node description at {self.path} " "has description errors.") - else: - self.status = NodePluginStatus.NOT_LOADED - logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Successful reloading.") + return False + + self.nodeDescriptor = descriptor + self._timestamp = timestamp + self.status = NodePluginStatus.NOT_LOADED + logging.info(f"[Reload] {self.nodeDescriptor.__name__}: Successful reloading.") + return True @property def plugin(self): From 8349814e8cb2a8cfaf82e02ffa7a867a280fa837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 6 Jun 2025 17:57:33 +0200 Subject: [PATCH 33/39] `reloadAllNodes`: Save reloaded types and only replace targeted nodes When requesting a `NodePlugin` to reload, the status of the reloading is now saved. If it has been successful, all the nodes of this type in the graph will effectively be removed. If it has not (either because of an error, or because no change has been done to the description), then nothing will happen. Replacing only the nodes whose description has been updated prevents from having to update heavy graphs for very few nodes to change (if any). --- meshroom/core/graph.py | 20 ++++++++++++++------ meshroom/ui/reconstruction.py | 6 ++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index cd9a6e91da..9834d9c59c 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -759,16 +759,24 @@ def upgradeAllNodes(self): for nodeName in nodeNames: self.upgradeNode(nodeName) - def reloadAllNodes(self): + def reloadAllNodes(self, nodeTypes: list[str]): """ - Replace all the node instances in the current graph with new node instances of the same - type. If the description of the nodes has changed, the reloaded nodes will reflect theses - changes. + Replace all the node instances of "nodeTypes" in the current graph with new node instances of the + same type. If the description of the nodes has changed, the reloaded nodes will reflect theses + changes. If "nodeTypes" is empty, then the function returns immediately. + + Args: + nodeTypes: the list of node types that will be reloaded. """ + if not nodeTypes: + # No updated node to replace in the graph, nothing to do + return + newNodes: dict[str, BaseNode] = {} for node in self._nodes.values(): - newNode = nodeFactory(node.toDict(), node.nodeType, expectedUid=node._uid) - newNodes[node.name] = newNode + if node.nodeType in nodeTypes: + newNode = nodeFactory(node.toDict(), node.nodeType, expectedUid=node._uid) + newNodes[node.name] = newNode # Replace in a different loop to ensure all the nodes have been looped over: when looping # over self._nodes and replacing nodes at the same time, some nodes might not be reached diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index e0f8d98b81..60851b6640 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -559,11 +559,13 @@ def reloadAllNodes(self): The nodes in the graph will be updated to match the changes in the description, if there was any. """ + nodeTypes: list[str] = [] for plugin in meshroom.core.pluginManager.getPlugins().values(): for node in plugin.nodes.values(): - node.reload() + if node.reload(): + nodeTypes.append(node.nodeDescriptor.__name__) - self._graph.reloadAllNodes() + self._graph.reloadAllNodes(nodeTypes) @Slot() @Slot(str) From fc62ef772201e1267b862b8e5844bd0cb2a24fcf Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 18:38:21 +0200 Subject: [PATCH 34/39] [core/common] raise an error instead of an assert if the key does not exist --- meshroom/common/qt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meshroom/common/qt.py b/meshroom/common/qt.py index f479ddfd2a..0fc4a1ef62 100644 --- a/meshroom/common/qt.py +++ b/meshroom/common/qt.py @@ -305,7 +305,8 @@ def _dereferenceItem(self, item): key = getattr(item, self._keyAttrName, None) if key is None: return - assert key in self._objectByKey + if key not in self._objectByKey: + raise RuntimeError(f"{key} is not in the Model: {self._objectByKey.keys()}") del self._objectByKey[key] def onRequestDeletion(self, item): From 2ad431af746ca76ef7e27cc28691c2f7de77e110 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 18:40:32 +0200 Subject: [PATCH 35/39] [core] Node upgrade: check if the attributes exist --- meshroom/core/graph.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index 9834d9c59c..eba556cc3a 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -748,7 +748,12 @@ def _recreateTargetListAttributeChildren(listAttrName: str, index: int, value: A if dstName in outListAttributes: _recreateTargetListAttributeChildren(*outListAttributes[dstName]) try: - self.addEdge(self.attribute(srcName), self.attribute(dstName)) + srcAttr = self.attribute(srcName) + dstAttr = self.attribute(dstName) + if srcAttr is None or dstAttr is None: + logging.warning(f"Failed to restore edge {srcName}{' (missing)' if srcAttr is None else ''} -> {dstName}{' (missing)' if dstAttr is None else ''}") + continue + self.addEdge(srcAttr, dstAttr) except (KeyError, ValueError) as e: logging.warning(f"Failed to restore edge {srcName} -> {dstName}: {e}") From e0ba3731501a62b133b54a24f31e692595997e27 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 19:37:26 +0200 Subject: [PATCH 36/39] Minor renaming --- meshroom/core/graph.py | 2 +- meshroom/ui/qml/Application.qml | 12 ++++++------ meshroom/ui/reconstruction.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meshroom/core/graph.py b/meshroom/core/graph.py index eba556cc3a..a5bff2a250 100644 --- a/meshroom/core/graph.py +++ b/meshroom/core/graph.py @@ -764,7 +764,7 @@ def upgradeAllNodes(self): for nodeName in nodeNames: self.upgradeNode(nodeName) - def reloadAllNodes(self, nodeTypes: list[str]): + def reloadNodePlugins(self, nodeTypes: list[str]): """ Replace all the node instances of "nodeTypes" in the current graph with new node instances of the same type. If the description of the nodes has changed, the reloaded nodes will reflect theses diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index c19ce6af9e..648992f183 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -553,12 +553,12 @@ Page { } Action { - id: reloadAllNodesAction - property string tooltip: "Reload all the node descriptions for all the registered plugins" - text: "Reload All Nodes" + id: reloadPluginsAction + property string tooltip: "Reload the source code for all nodes from all registered plugins" + text: "Reload Plugins Source Code" shortcut: "Ctrl+Shift+R" onTriggered: { - _reconstruction.reloadAllNodes() + _reconstruction.reloadPlugins() } } @@ -842,9 +842,9 @@ Page { } MenuItem { - action: reloadAllNodesAction + action: reloadPluginsAction ToolTip.visible: hovered - ToolTip.text: reloadAllNodesAction.tooltip + ToolTip.text: reloadPluginsAction.tooltip } } MenuSeparator { } diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 60851b6640..62ff4ccee3 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -553,7 +553,7 @@ def onCameraInitChanged(self): self.setActiveNodes(nodes) @Slot() - def reloadAllNodes(self): + def reloadPlugins(self): """ Reload all the NodePlugins from all the registered plugins. The nodes in the graph will be updated to match the changes in the description, if @@ -565,7 +565,7 @@ def reloadAllNodes(self): if node.reload(): nodeTypes.append(node.nodeDescriptor.__name__) - self._graph.reloadAllNodes(nodeTypes) + self._graph.reloadNodePlugins(nodeTypes) @Slot() @Slot(str) From 9cffabbe1455f9153b5e1303d475f5d0c79b6b7c Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 23:12:07 +0200 Subject: [PATCH 37/39] [core] load all modules but not recursively --- meshroom/core/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 84cd06a430..0bf4a7a404 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -324,7 +324,7 @@ def loadNodes(folder, packageName) -> list[NodePlugin]: def loadAllNodes(folder) -> list[Plugin]: plugins = [] - for _, package, ispkg in pkgutil.walk_packages([folder]): + for _, package, ispkg in pkgutil.iter_modules([folder]): if ispkg: plugin = Plugin(package, folder) nodePlugins = loadNodes(folder, package) From 9dc0aca8edba4b2500a66613f32ce273d9b1ba60 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 23:41:13 +0200 Subject: [PATCH 38/39] [core] Adjust message when there is no class in the module Warning only on a module/file without any element in it and provide full-path to ease debugging. For package/folder, it does not generate any message. --- meshroom/core/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 0bf4a7a404..e3eab31630 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -100,7 +100,11 @@ def loadClasses(folder: str, packageName: str, classType: type) -> list[type]: and issubclass(plugin, classType)] if not plugins: - logging.warning(f"No class defined in plugin: {pluginModuleName}") + # Only packages/folders have __path__, single module/file do not have it. + isPackage = hasattr(pluginMod, "__path__") + # Sub-folders/Packages should not raise a warning + if not isPackage: + logging.warning(f"No class defined in plugin: {package.__name__}.{pluginName} ('{pluginMod.__file__}')") for p in plugins: p.packageName = packageName From d3738fbe8dae7884bda09eab0eec870b645ceef5 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Sat, 7 Jun 2025 23:51:59 +0200 Subject: [PATCH 39/39] Disable too verbose debug message --- meshroom/ui/components/thumbnail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/ui/components/thumbnail.py b/meshroom/ui/components/thumbnail.py index acfd1591f7..06203dbc54 100644 --- a/meshroom/ui/components/thumbnail.py +++ b/meshroom/ui/components/thumbnail.py @@ -129,7 +129,7 @@ def clean(): # Compute storage duration since last usage of thumbnail lastUsage = f_stat.st_mtime storageTime = now - lastUsage - logging.debug(f'[ThumbnailCache] Thumbnail {f_name} has been stored for {storageTime}s') + # logging.debug(f'[ThumbnailCache] Thumbnail {f_name} has been stored for {storageTime}s') if storageTime > ThumbnailCache.storageTimeLimit * 3600 * 24: # Mark as removable if storage time exceeds limit