From c2aac17c88bf8508f275ffc1e7044afe54c0cf94 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Mon, 15 Jul 2024 14:55:56 +0200 Subject: [PATCH 01/10] created plugin and environment node systems --- .bandit | 43 +++ README.md | 8 + meshroom/core/__init__.py | 18 +- meshroom/core/node.py | 94 +++--- meshroom/core/plugin.py | 368 ++++++++++++++++++++++++ meshroom/plugins/README.md | 40 +++ meshroom/plugins/catalog.json | 9 + meshroom/ui/qml/GraphEditor/common.js | 4 +- meshroom/ui/qml/Utils/Colors.qml | 6 +- meshroom/ui/qml/main.qml | 67 +++++ meshroom/ui/reconstruction.py | 34 ++- tests/nodes/plugins/Dockerfile | 5 + tests/nodes/plugins/__init__.py | 0 tests/nodes/plugins/dummyNodes.py | 155 ++++++++++ tests/nodes/plugins/env.yaml | 8 + tests/nodes/plugins/meshroomPlugin.json | 6 + tests/nodes/plugins/requirements.txt | 2 + tests/test_plugin_nodes.py | 17 ++ 18 files changed, 824 insertions(+), 60 deletions(-) create mode 100644 .bandit create mode 100644 meshroom/core/plugin.py create mode 100644 meshroom/plugins/README.md create mode 100644 meshroom/plugins/catalog.json create mode 100644 tests/nodes/plugins/Dockerfile create mode 100644 tests/nodes/plugins/__init__.py create mode 100644 tests/nodes/plugins/dummyNodes.py create mode 100644 tests/nodes/plugins/env.yaml create mode 100644 tests/nodes/plugins/meshroomPlugin.json create mode 100644 tests/nodes/plugins/requirements.txt create mode 100644 tests/test_plugin_nodes.py diff --git a/.bandit b/.bandit new file mode 100644 index 0000000000..a52326bcea --- /dev/null +++ b/.bandit @@ -0,0 +1,43 @@ +[tool.bandit] +skips = [B101, B102, B105, B106, B107, B113, B202, B401, B402, B403, B404, B405, B406, B407, B408, B409, B410, B413, B307, B311, B507, B602, B603, B605, B607, B610, B611, B703] + +[tool.bandit.any_other_function_with_shell_equals_true] +no_shell = [ + "os.execl", + "os.execle", + "os.execlp", + "os.execlpe", + "os.execv", + "os.execve", + "os.execvp", + "os.execvpe", + "os.spawnl", + "os.spawnle", + "os.spawnlp", + "os.spawnlpe", + "os.spawnv", + "os.spawnve", + "os.spawnvp", + "os.spawnvpe", + "os.startfile" +] +shell = [ + "os.system", + "os.popen", + "os.popen2", + "os.popen3", + "os.popen4", + "popen2.popen2", + "popen2.popen3", + "popen2.popen4", + "popen2.Popen3", + "popen2.Popen4", + "commands.getoutput", + "commands.getstatusoutput" +] +subprocess = [ + "subprocess.Popen", + "subprocess.call", + "subprocess.check_call", + "subprocess.check_output" +] diff --git a/README.md b/README.md index 870dbda783..578b08d240 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,14 @@ You can create custom nodes in python and make them available in Meshroom using In a standard precompiled version of Meshroom, you can also directly add custom nodes in `lib/meshroom/nodes`. To be recognized by Meshroom, a custom folder with nodes should be a Python module (an `__init__.py` file is needed). +### Plugins + +Meshroom supports installing containerised plugins via Docker (with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)) or [Anaconda](https://docs.anaconda.com/free/miniconda/index.html). + +To do so, make sure docker or anaconda is installed properly and available from the command line. +Then click on `File > Advanced > Install Plugin From URL` or `File > Advanced > Install Plugin From Local Folder` to begin the installation. + +To learn more about using or creating plugins, check the explanations [here](meshroom/plugins/README.md). ## License diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 3bd30d5bbd..176dcdd0bf 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -27,12 +27,19 @@ # make a UUID based on the host ID and current time sessionUid = str(uuid.uuid1()) -cacheFolderName = 'MeshroomCache' -defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName)) nodesDesc = {} submitters = {} pipelineTemplates = {} +#meshroom paths +meshroomFolder = os.path.dirname(os.path.dirname(__file__)) +cacheFolderName = 'MeshroomCache' +defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName)) + +#plugin paths +pluginsNodesFolder = os.path.join(meshroomFolder, "plugins") +pluginsPipelinesFolder = os.path.join(meshroomFolder, "pipelines") +pluginCatalogFile = os.path.join(meshroomFolder, "plugins", "catalog.json") def hashValue(value): """ Hash 'value' using sha1. """ @@ -329,29 +336,26 @@ def loadPipelineTemplates(folder): def initNodes(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep) # filter empty strings additionalNodesPath = [i for i in additionalNodesPath if i] - nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + [pluginsNodesFolder] for f in nodesFolders: loadAllNodes(folder=f) def initSubmitters(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters') for sub in subs: registerSubmitter(sub()) def initPipelines(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) # Load pipeline templates: check in the default folder and any folder the user might have # added to the environment variable additionalPipelinesPath = os.environ.get("MESHROOM_PIPELINE_TEMPLATES_PATH", "").split(os.pathsep) additionalPipelinesPath = [i for i in additionalPipelinesPath if i] - pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + [pluginsPipelinesFolder] for f in pipelineTemplatesFolders: if os.path.isdir(f): loadPipelineTemplates(f) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 2c9466b879..a905487958 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -51,6 +51,8 @@ class Status(Enum): KILLED = 5 SUCCESS = 6 INPUT = 7 # Special status for input nodes + BUILD = 8 + FIRST_RUN = 9 class ExecMode(Enum): @@ -380,16 +382,16 @@ def saveStatistics(self): renameWritingToFinalPath(statisticsFilepathWriting, statisticsFilepath) def isAlreadySubmitted(self): - return self._status.status in (Status.SUBMITTED, Status.RUNNING) + return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isAlreadySubmittedOrFinished(self): - return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS) + return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS, Status.BUILD, Status.FIRST_RUN) def isFinishedOrRunning(self): - return self._status.status in (Status.SUCCESS, Status.RUNNING) + return self._status.status in (Status.SUCCESS, Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isRunning(self): - return self._status.status == Status.RUNNING + return self._status.status in (Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isStopped(self): return self._status.status == Status.STOPPED @@ -401,36 +403,56 @@ def process(self, forceCompute=False): if not forceCompute and self._status.status == Status.SUCCESS: logging.info("Node chunk already computed: {}".format(self.name)) return - global runningProcesses - runningProcesses[self.name] = self - self._status.initStartCompute() - exceptionStatus = None - startTime = time.time() - self.upgradeStatusTo(Status.RUNNING) - self.statThread = stats.StatisticsThread(self) - self.statThread.start() - try: - self.node.nodeDesc.processChunk(self) - except Exception: - if self._status.status != Status.STOPPED: - exceptionStatus = Status.ERROR - raise - except (KeyboardInterrupt, SystemError, GeneratorExit): - exceptionStatus = Status.STOPPED - raise - finally: - self._status.initEndCompute() - self._status.elapsedTime = time.time() - startTime - if exceptionStatus is not None: - self.upgradeStatusTo(exceptionStatus) - logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) - # Ask and wait for the stats thread to stop - self.statThread.stopRequest() - self.statThread.join() - self.statistics = stats.Statistics() - del runningProcesses[self.name] - - self.upgradeStatusTo(Status.SUCCESS) + + #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk + # of the node will be ran into the env + if hasattr(self.node.nodeDesc, 'envFile') and self._status.status!=Status.FIRST_RUN: + try: + if not self.node.nodeDesc.isBuild(): + self.upgradeStatusTo(Status.BUILD) + self.node.nodeDesc.build() + self.upgradeStatusTo(Status.FIRST_RUN) + command = self.node.nodeDesc.getCommandLine(self) + #NOTE: docker returns 0 even if mount fail (it fails on the deamon side) + logging.info("Running plugin node with "+command) + status = os.system(command) + if status != 0: + raise RuntimeError("Error in node execution") + self.updateStatusFromCache() + except Exception as ex: + self.logger.exception(ex) + self.upgradeStatusTo(Status.ERROR) + else: + global runningProcesses + runningProcesses[self.name] = self + self._status.initStartCompute() + exceptionStatus = None + startTime = time.time() + self.upgradeStatusTo(Status.RUNNING) + self.statThread = stats.StatisticsThread(self) + self.statThread.start() + try: + self.node.nodeDesc.processChunk(self) + except Exception: + if self._status.status != Status.STOPPED: + exceptionStatus = Status.ERROR + raise + except (KeyboardInterrupt, SystemError, GeneratorExit): + exceptionStatus = Status.STOPPED + raise + finally: + self._status.initEndCompute() + self._status.elapsedTime = time.time() - startTime + if exceptionStatus is not None: + self.upgradeStatusTo(exceptionStatus) + logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) + # Ask and wait for the stats thread to stop + self.statThread.stopRequest() + self.statThread.join() + self.statistics = stats.Statistics() + del runningProcesses[self.name] + + self.upgradeStatusTo(Status.SUCCESS) def stopProcess(self): if not self.isExtern(): @@ -1130,8 +1152,8 @@ def getGlobalStatus(self): return Status.INPUT chunksStatus = [chunk.status.status for chunk in self._chunks] - anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, - Status.RUNNING, Status.SUBMITTED) + anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN, + Status.SUBMITTED,) allOf = (Status.SUCCESS,) for status in anyOf: diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py new file mode 100644 index 0000000000..c7db6c95f3 --- /dev/null +++ b/meshroom/core/plugin.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python +# coding:utf-8 + +""" +This file defines the nodes and logic needed for the plugin system in meshroom. +A plugin is a collection of node(s) of any type with their rutime environnement setup file attached. +We use the term 'environment' to abstract a docker container or a conda/virtual environment. +""" + +from enum import Enum +import json +import os, sys +import logging +import urllib +from distutils.dir_util import copy_tree, remove_tree +import subprocess +import venv +import inspect + +from meshroom.core import desc, hashValue +from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, defaultCacheFolder, pluginCatalogFile +from meshroom.core import meshroomFolder +from meshroom.core.graph import loadGraph + +#where the executables are (eg meshroom compute) +meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) + +class EnvType(Enum): + """ + enum for the type of env used (by degree of encapsulation) + """ + NONE = 0 + PIP = 1 + VENV = 10 + CONDA = 20 + DOCKER = 30 + +#NOTE: could add the concept of dependencies between plugins +class PluginParams(): + """" + Class that holds parameters to install one plugin from a folder and optionally from a json structure + """ + def __init__(self, pluginUrl, jsonData=None): + #get the plugin name from folder + self.pluginName = os.path.basename(pluginUrl) + #default node and pipeline locations + self.nodesFolder = os.path.join(pluginUrl, "meshroomNodes") + self.pipelineFolder = os.path.join(pluginUrl, "meshroomPipelines") + #overwrite is json is passed + if jsonData is not None: + self.pluginName = jsonData["pluginName"] + #default node and pipeline locations + self.nodesFolder = os.path.join(pluginUrl, jsonData["nodesFolder"]) + if "pipelineFolder" in jsonData.keys(): + self.pipelineFolder = os.path.join(pluginUrl, jsonData["pipelineFolder"]) + +def _dockerImageExists(image_name, tag='latest'): + """ + Check if the desired image:tag exists + """ + try: + result = subprocess.run( ['docker', 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) + if result.returncode != 0: + return False + images = result.stdout.splitlines() + image_tag = f"{image_name}:{tag}" + return image_tag in images + except Exception as e: + return False + +def _cleanEnvVarsRez(): + """ + Used to unset all rez defined env that mess up with conda. + """ + cmd = "unset python; unset PYTHONPATH; " + return cmd + +def _condaEnvExist(envName): + """ + Checks if a specified env exists + """ + cmd = "conda list --name "+envName + return os.system(cmd) == 0 + +def _formatPluginName(pluginName): + """ + Replaces spaces for env naming + """ + return pluginName.replace(" ", "_") + +def getVenvExe(venvPath): + """ + Returns the path for the python in a virtual env + """ + if not os.path.isdir(venvPath): + raise ValueError("The specified path "+venvPath+" is not a directory") + if sys.platform == "win32": + executable = os.path.join(venvPath, 'Scripts', 'python.exe') + else: + executable = os.path.join(venvPath, 'bin', 'python') + if not os.path.isfile(executable): + raise FileNotFoundError(f"Python executable not found in the specified virtual environment: "+executable) + return executable + +def _venvExists(envName): + """ + Check if the following virtual env exists + """ + return os.path.isdir(os.path.join(defaultCacheFolder, envName)) + +def installPlugin(pluginUrl): + """ + Install plugin from an url or local path. + Regardless of the method, the content will be copied in the plugin folder of meshroom (which is added to the list of directory to load nodes from). + There are two options : + - having the following structure : + - [plugin folder] (will be the plugin name) + - meshroomNodes + - [code for your nodes] that contains relative path to a DockerFile|env.yaml|requirements.txt + - [...] + - meshroomPipelines + - [your meshroom templates] + - having a meshroomPlugin.json file at the root of the plugin folder + With this solution, you may have several separate node folders. + """ + logging.info("Installing plugin from "+pluginUrl) + try: + isLocal = True + #if git repo, clone the repo in cache + if urllib.parse.urlparse(pluginUrl).scheme in ('http', 'https','git'): + os.chdir(defaultCacheFolder) + os.system("git clone "+pluginUrl) + pluginName = pluginUrl.split('.git')[0].split('/')[-1] + pluginUrl = os.path.join(defaultCacheFolder, pluginName) + isLocal = False + #sanity check + if not os.path.isdir(pluginUrl): + ValueError("Invalid plugin path :"+pluginUrl) + #by default only one plugin, and with default file hierachy + pluginParamList=[PluginParams(pluginUrl)] + #location of the json file if any + paramFile=os.path.join(pluginUrl, "meshroomPlugin.json") + #load json for custom install if any + if os.path.isfile(paramFile): + jsonData=json.load(open(paramFile,"r")) + pluginParamList = [PluginParams(pluginUrl, jsonDataplugin) for jsonDataplugin in jsonData] + #for each plugin, run the 'install' + for pluginParam in pluginParamList: + intallFolder = os.path.join(pluginsNodesFolder, _formatPluginName(pluginParam.pluginName)) + logging.info("Installing "+pluginParam.pluginName+" from "+pluginUrl+" in "+intallFolder) + #check if folder valid + if not os.path.isdir(pluginParam.nodesFolder): + raise RuntimeError("Invalid node folder: "+pluginParam.nodesFolder) + #check if already installed + if os.path.isdir(intallFolder): + logging.warn("Plugin already installed, will overwrite") + if os.path.islink(intallFolder): + os.unlink(intallFolder) + else: + remove_tree(intallFolder) + #install via symlink if local, otherwise copy (usefull to develop) + if isLocal: + os.symlink(pluginParam.nodesFolder, intallFolder) + if os.path.isdir(pluginParam.pipelineFolder): + os.symlink(pluginParam.pipelineFolder, os.path.join(pluginsPipelinesFolder, pluginParam.pluginName)) + else: + copy_tree(pluginParam.nodesFolder, intallFolder) + if os.path.isdir(pluginParam.pipelineFolder): + copy_tree(pluginParam.pipelineFolder, os.path.join(pluginsPipelinesFolder, pluginParam.pluginName)) + #remove repo if was cloned + if not isLocal: + os.removedirs(pluginUrl) + #NOTE: could try to auto load the plugins to avoid restart and test files + except Exception as ex: + logging.error(ex) + return False + + return True + +def getCatalog(): + """ + Returns the plugin catalog + """ + jsonData=json.load(open(pluginCatalogFile,"r")) + return jsonData + +def getInstalledPlugin(): + """ + Returns the list of installed plugins + """ + installedPlugins = [os.path.join(pluginsNodesFolder, f) for f in os.listdir(pluginsNodesFolder)] + return installedPlugins + +def uninstallPlugin(pluginUrl): + """ + Uninstall a plugin + """ + #NOTE: could also remove the env files + if not os.path.exists(pluginUrl): + raise RuntimeError("Plugin "+pluginUrl+" is not installed") + if os.path.islink(pluginUrl): + os.unlink(pluginUrl) + else: + os.removedirs(pluginUrl) + +class PluginNode(desc.Node): + """ + Class to be used to make a plugin node, you need to overwrite envType and envFile + """ + + @property + def envType(cls): + """ + Dynamic env type + """ + raise NotImplementedError("You must specify one or several envtype in the node description") + + @property + def envFile(cls): + """ + Env file used to build the environement, you may overwrite this to custom the behaviour + """ + raise NotImplementedError("You must specify an env file") + + @property + def _envName(cls): + """ + Get the env name by hashing the env files, overwrite this to use a custom pre-build env + """ + with open(cls.envFile, 'r') as file: + envContent = file.read() + return "meshroom_plugin_"+hashValue(envContent) + + def isBuild(cls): + """ + Check if the env needs to be build. + """ + if cls.envType == EnvType.NONE: + return True + elif cls.envType == EnvType.PIP: + #NOTE: could find way to check for installed packages instead of rebuilding all the time + return False + elif cls.envType == EnvType.VENV: + return _venvExists(cls._envName) + elif cls.envType == EnvType.CONDA: + return _condaEnvExist(cls._envName) + elif cls.envType == EnvType.DOCKER: + return _dockerImageExists(cls._envName) + + def build(cls): + """ + Perform the needed steps to prepare the environement in which to run the node. + """ + if cls.envType == EnvType.NONE: + pass + elif cls.envType == EnvType.PIP: + #install packages in the same python as meshroom + logging.info("Installing packages from "+ cls.envFile) + buildCommand = sys.executable+" -m pip install "+ cls.envFile + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + elif cls.envType == EnvType.VENV: + #create venv in default cache folder + logging.info("Creating virtual env "+os.path.join(defaultCacheFolder, cls._envName)+" from "+cls.envFile) + envPath = os.path.join(defaultCacheFolder, cls._envName) + venv.create(envPath, with_pip=True) + logging.info("Installing dependencies") + envExe = getVenvExe(envPath) + returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ cls.envFile) + venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') + venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) + for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) + logging.info("Done") + elif cls.envType == EnvType.CONDA: + #build a conda env from a yaml file + logging.info("Creating conda env "+cls._envName+" from "+cls.envFile) + makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " + +" conda env create -v -v --name "+cls._envName + +" --file "+cls.envFile+" ") + logging.info("Making conda env") + logging.info(makeEnvCommand) + returnValue = os.system(makeEnvCommand) + #find path to env's folder and add symlink to meshroom + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+cls._envName + +" python -c \"import sys; print(sys.executable)\"", + shell=True).strip().decode('UTF-8') + condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') + condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) + for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) + logging.info("Done making conda env") + elif cls.envType == EnvType.DOCKER: + #build docker image + logging.info("Creating image "+cls._envName+" from "+ cls.envFile) + buildCommand = "docker build -f "+cls.envFile+" -t "+cls._envName+" "+os.path.dirname(cls.envFile) + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + if returnValue != 0: + raise RuntimeError("Something went wrong during build") + + def getCommandLine(cls, chunk): + """ + Return the command line needed to enter the environment + meshroom_compute + Will make meshroom available in the environment. + """ + if chunk.node.isParallelized: + raise RuntimeError("Parallelisation not supported for plugin nodes") + if chunk.node.graph.filepath == "": + raise RuntimeError("The project needs to be saved to use plugin nodes") + saved_graph = loadGraph(chunk.node.graph.filepath) + if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] + or chunk.node._uids[0] != saved_graph.findNode(str(chunk.node))._uids[0] ): + raise RuntimeError("The changes needs to be saved to use plugin nodes") + + cmdPrefix = "" + # vars common to venv and conda, that will be passed when runing conda run or venv + meshroomCompute= meshroomBinDir+"/meshroom_compute" + meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" + pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" + + if cls.envType == EnvType.VENV: + envPath = os.path.join(defaultCacheFolder, cls._envName) + envExe = getVenvExe(envPath) + #make sure meshroom in in pythonpath and that we call the right python + cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif cls.envType == EnvType.CONDA: + #NOTE: system env vars are not passed to conda run, we installed it 'manually' before + cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ + +" --no-capture-output -n "+cls._envName+" "+" python "+meshroomCompute + elif cls.envType == EnvType.DOCKER: + #path to the selected plugin + classFile=inspect.getfile(chunk.node.nodeDesc.__class__) + pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) + #path to the project/cache + projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) + mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy + +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) + +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' + +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') + #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path + envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" + #adds the gpu arg if needed + runtimeArg="" + if cls.gpu != desc.Level.NONE: + runtimeArg="--runtime=nvidia --gpus all" + #compose cl + cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+cls._envName +" \"python /meshroomBinDir/meshroom_compute " + meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + + command=cmdPrefix+" "+meshroomComputeArgs + + return command + +#class that call command line nodes in an env +class PluginCommandLineNode(PluginNode, desc.CommandLineNode): + def buildCommandLine(self, chunk): + cmd = super().buildCommandLine(chunk) + #the process in Popen does not seem to use the right python, even if meshroom_compute is call within the env + #so in the case of command line using python, we have to make sur it is using the correct python + if self.envType == EnvType.VENV: + envPath = os.path.join(defaultCacheFolder, self._envName) + envExe = getVenvExe(envPath) + cmd=cmd.replace("python", envExe) + return cmd diff --git a/meshroom/plugins/README.md b/meshroom/plugins/README.md new file mode 100644 index 0000000000..f7bdddc01a --- /dev/null +++ b/meshroom/plugins/README.md @@ -0,0 +1,40 @@ +# Intro + +A plugin is a set of one or several nodes that are not part of the Meshroom/Alicevision main project. +They are meant to facilitate the creation of custom pipeline, make distribution and installation of extra nodes easy, and allow the use of different level of isolation at the node level. +Each node within a plugin may use the same or differents environnement. + +# Making Meshroom Plugins + +To make a new plugin, make your node inheriting from `meshroom.core.plugins.PluginNode` +In your new node class, overwrite the variable `envFile` to point to the environment file (e.g. the `yaml` or `dockerfile`) that sets up your installation, end `envType` to specify the type of plugin. The path to this file should be relative to the path of the node, and within the same folder (or subsequent child folder) as the node definition. + +The code in `processChunk` in your node definition will be automatically executed within the envirenoment, using `meshroom_compute`. +A new status `FIRST_RUN` denotes the stage in between the environement startup and the execution of the node. + +Make sur your imports are lazy, in `processChunk`. +Several nodes share the same environment as long as they point to the same environment file. +Changing this file will trigger a rebuild on the environment. + +You may install plugin from a git repository or from a local folder. In the later case, you may edit the code directly from your source folder. + +By default, Meshroom will look for node definition in `[plugin folder]/meshroomNodes` and new pipelines in `[plugin folder]/meshroomPipelines` and assumes only one environement is needed. + +To modify this behavior, you may put a json file named `meshroomPlugin.json` at the root of your folder/repository. +The file must have the following structure: +``` +[ + { + "pluginName":"[YOUR_PLUGIN_NAME]", + "nodesFolder":"[YOUR_FOLDER_RELATIVE_TO_THE_ROOT_REPO_OR_FOLDER], + "pipelineFolder":"[YOUR_CUSTOM_PIEPILINE_FOLDER" + }, + { + "pluginName":"Dummy Plugin", + "nodesFolder":"dummy" + } +] +``` + +The environment of the nodes are going to be build the first time it is needed (status will be `BUILD`, in purple). + diff --git a/meshroom/plugins/catalog.json b/meshroom/plugins/catalog.json new file mode 100644 index 0000000000..556ac56429 --- /dev/null +++ b/meshroom/plugins/catalog.json @@ -0,0 +1,9 @@ +[ + { + "pluginName":"Meshroom Research", + "pluginUrl":"https://github.com/alicevision/MeshroomResearch/", + "description":"Meshroom-Research comprises a collection of plugins for Meshroom, mostly develloped in-house at MikrosImage", + "isCollection":true, + "nodeTypes":["Python", "Docker", "Conda"] + } +] \ No newline at end of file diff --git a/meshroom/ui/qml/GraphEditor/common.js b/meshroom/ui/qml/GraphEditor/common.js index 35c754ce29..3633b049b6 100644 --- a/meshroom/ui/qml/GraphEditor/common.js +++ b/meshroom/ui/qml/GraphEditor/common.js @@ -5,7 +5,9 @@ var statusColors = { "RUNNING": "#FF9800", "ERROR": "#F44336", "SUCCESS": "#4CAF50", - "STOPPED": "#E91E63" + "STOPPED": "#E91E63", + "BUILD": "#66207f", + "FIRST_RUN": "#A52A2A" } var statusColorsExternOverrides = { diff --git a/meshroom/ui/qml/Utils/Colors.qml b/meshroom/ui/qml/Utils/Colors.qml index 51af70e25c..6215bcf3ad 100644 --- a/meshroom/ui/qml/Utils/Colors.qml +++ b/meshroom/ui/qml/Utils/Colors.qml @@ -19,6 +19,8 @@ QtObject { readonly property color lime: "#CDDC39" readonly property color grey: "#555555" readonly property color lightgrey: "#999999" + readonly property color deeppurple: "#66207F" + readonly property color brown: "#A52A2A" readonly property var statusColors: { "NONE": "transparent", @@ -26,7 +28,9 @@ QtObject { "RUNNING": orange, "ERROR": red, "SUCCESS": green, - "STOPPED": pink + "STOPPED": pink, + "BUILD": deeppurple, + "FIRST_RUN": brown } readonly property var ghostColors: { diff --git a/meshroom/ui/qml/main.qml b/meshroom/ui/qml/main.qml index f011326450..5832af07d8 100644 --- a/meshroom/ui/qml/main.qml +++ b/meshroom/ui/qml/main.qml @@ -132,6 +132,73 @@ ApplicationWindow { } } + //File browser for plugin + Dialog { + id: pluginURLDialog + title: "Plugin URL" + height: 150 + width: 300 + standardButtons: StandardButton.Ok | StandardButton.Cancel + //focus: true + Column { + anchors.fill: parent + Text { + text: "Plugin URL" + height: 40 + } + TextField { + id: urlInput + width: parent.width * 0.75 + focus: true + } + } + onButtonClicked: { + if (clickedButton==StandardButton.Ok) { + console.log("Accepted " + clickedButton) + if (_reconstruction.installPlugin(urlInput.text)) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + } + + // dialogs for plugins + MessageDialog { + id: pluginInstalledDialog + title: "Plugin installed" + modal: true + canCopy: false + Label { + text: "Plugin installed, please restart meshroom for the changes to take effect" + } + } + + MessageDialog { + id: pluginNotInstalledDialog + title: "Plugin not installed" + modal: true + canCopy: false + Label { + text: "Something went wrong, plugin not installed" + } + } + + // plugin installation from path or url + Platform.FolderDialog { + id: intallPluginDialog + options: Platform.FolderDialog.DontUseNativeDialog + title: "Install Plugin" + onAccepted: { + if (_reconstruction.installPlugin(currentFolder.toString())) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + // Check if document has been saved function ensureSaved(callback) { diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index e1aaaa5f77..225ded9bae 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -18,6 +18,7 @@ from meshroom.ui.graph import UIGraph from meshroom.ui.utils import makeProperty from meshroom.ui.components.filepath import FilepathHelper +from meshroom.core.plugin import installPlugin class Message(QObject): @@ -413,6 +414,16 @@ def __init__(self, nodeType, parent=None): nodeChanged = Signal() node = makeProperty(QObject, "_node", nodeChanged, resetOnDestroy=True) +def prepareUrlLocalFile(url): + if isinstance(url, (QUrl)): + # depending how the QUrl has been initialized, + # toLocalFile() may return the local path or an empty string + localFile = url.toLocalFile() + if not localFile: + localFile = url.toString() + else: + localFile = url + return localFile class Reconstruction(UIGraph): """ @@ -567,16 +578,14 @@ def load(self, filepath, setupProjectFile=True, publishOutputs=False): @Slot(QUrl, result=bool) @Slot(QUrl, bool, bool, result=bool) def loadUrl(self, url, setupProjectFile=True, publishOutputs=False): - if isinstance(url, (QUrl)): - # depending how the QUrl has been initialized, - # toLocalFile() may return the local path or an empty string - localFile = url.toLocalFile() - if not localFile: - localFile = url.toString() - else: - localFile = url + localFile = prepareUrlLocalFile(url) return self.load(localFile, setupProjectFile, publishOutputs) + @Slot(QUrl, result=bool) + def installPlugin(self, url): + localFile = prepareUrlLocalFile(url) + return installPlugin(localFile) + def onGraphChanged(self): """ React to the change of the internal graph. """ self._liveSfmManager.reset() @@ -904,12 +913,7 @@ def importImagesFromFolder(self, path, recursive=False): def importImagesUrls(self, imagePaths, recursive=False): paths = [] for imagePath in imagePaths: - if isinstance(imagePath, (QUrl)): - p = imagePath.toLocalFile() - if not p: - p = imagePath.toString() - else: - p = imagePath + p = prepareUrlLocalFile(imagePath) paths.append(p) self.importImagesFromFolder(paths) @@ -1285,4 +1289,4 @@ def setCurrentViewPath(self, path): # Signals to propagate high-level messages error = Signal(Message) warning = Signal(Message) - info = Signal(Message) \ No newline at end of file + info = Signal(Message) diff --git a/tests/nodes/plugins/Dockerfile b/tests/nodes/plugins/Dockerfile new file mode 100644 index 0000000000..463ebb3896 --- /dev/null +++ b/tests/nodes/plugins/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3 +RUN python -m pip install --no-cache-dir numpy +RUN python -m pip install --no-cache-dir psutil +#overwrides entry point otherwise will directly execute python +ENTRYPOINT [ "/bin/bash", "-l", "-c" ] \ No newline at end of file diff --git a/tests/nodes/plugins/__init__.py b/tests/nodes/plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/nodes/plugins/dummyNodes.py b/tests/nodes/plugins/dummyNodes.py new file mode 100644 index 0000000000..b0446c90f3 --- /dev/null +++ b/tests/nodes/plugins/dummyNodes.py @@ -0,0 +1,155 @@ + + +import os +from meshroom.core.plugin import PluginNode, PluginCommandLineNode, EnvType + +#Python nodes + +class DummyConda(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.CONDA + envFile = os.path.join(os.path.dirname(__file__), "env.yaml") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyDocker(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.DOCKER + envFile = os.path.join(os.path.dirname(__file__), "Dockerfile") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + + +class DummyVenv(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.VENV + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyPip(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.PIP + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyNone(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.NONE + envFile = None + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +#Command line node + +class DummyCondaCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.CONDA + envFile = os.path.join(os.path.dirname(__file__), "env.yaml") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyDockerCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.DOCKER + envFile = os.path.join(os.path.dirname(__file__), "Dockerfile") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + + +class DummyVenvCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.VENV + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyPipCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.PIP + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyNoneCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.NONE + envFile = None + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" \ No newline at end of file diff --git a/tests/nodes/plugins/env.yaml b/tests/nodes/plugins/env.yaml new file mode 100644 index 0000000000..f09be04127 --- /dev/null +++ b/tests/nodes/plugins/env.yaml @@ -0,0 +1,8 @@ +name: dummy +channels: + - defaults + - conda-forge +dependencies: + - python + - numpy + - psutil \ No newline at end of file diff --git a/tests/nodes/plugins/meshroomPlugin.json b/tests/nodes/plugins/meshroomPlugin.json new file mode 100644 index 0000000000..b1d03bd6d4 --- /dev/null +++ b/tests/nodes/plugins/meshroomPlugin.json @@ -0,0 +1,6 @@ +[ + { + "pluginName":"Dummy", + "nodesFolder":"." + } +] \ No newline at end of file diff --git a/tests/nodes/plugins/requirements.txt b/tests/nodes/plugins/requirements.txt new file mode 100644 index 0000000000..386e3f591a --- /dev/null +++ b/tests/nodes/plugins/requirements.txt @@ -0,0 +1,2 @@ +numpy +psutil \ No newline at end of file diff --git a/tests/test_plugin_nodes.py b/tests/test_plugin_nodes.py new file mode 100644 index 0000000000..1df862553d --- /dev/null +++ b/tests/test_plugin_nodes.py @@ -0,0 +1,17 @@ +import logging +import os + +from meshroom.core.graph import Graph + +logging = logging.getLogger(__name__) + +def test_pluginNodes(): + if "CI" in os.environ: + return + graph = Graph('') + graph.addNewNode('DummyCondaNode') + graph.addNewNode('DummyDockerNode') + graph.addNewNode('DummyPipNode') + graph.addNewNode('DummyVenvNode') + + \ No newline at end of file From b477031f8a66225c33dd6e1a022fddd2b0e668fe Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Mon, 2 Sep 2024 16:49:58 +0200 Subject: [PATCH 02/10] moved plugin logic to nodeDesc instead of class added build badge --- meshroom/core/__init__.py | 2 +- meshroom/core/desc.py | 56 +++- meshroom/core/node.py | 13 +- meshroom/core/plugin.py | 296 +++++++++---------- meshroom/plugins/catalog.json | 4 +- meshroom/ui/qml/Application.qml | 96 ++++++ meshroom/ui/qml/GraphEditor/Node.qml | 15 +- meshroom/ui/qml/GraphEditor/NodeEditor.qml | 14 + meshroom/ui/qml/GraphEditor/ToBuildBadge.qml | 66 +++++ meshroom/ui/qml/GraphEditor/qmldir | 3 +- meshroom/ui/qml/main.qml | 67 ----- meshroom/ui/reconstruction.py | 8 + tests/test_plugin_nodes.py | 3 +- 13 files changed, 407 insertions(+), 236 deletions(-) create mode 100644 meshroom/ui/qml/GraphEditor/ToBuildBadge.qml diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 176dcdd0bf..55a192cb77 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ pass from meshroom.core.submitter import BaseSubmitter -from . import desc +from meshroom.core import desc # Setup logging logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index 1a7f4893c6..a501a926b1 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -717,9 +717,53 @@ class Node(object): documentation = '' category = 'Other' + _isPlugin = True + def __init__(self): super(Node, self).__init__() self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs) + try: + self.envFile + self.envType + except: + self._isPlugin=False + + @property + def envType(cls): + from core.plugin import EnvType #lazy import for plugin to avoid circular dependency + return EnvType.NONE + + @property + def envFile(cls): + """ + Env file used to build the environement, you may overwrite this to custom the behaviour + """ + raise NotImplementedError("You must specify an env file") + + @property + def _envName(cls): + """ + Get the env name by hashing the env files, overwrite this to use a custom pre-build env + """ + with open(cls.envFile, 'r') as file: + envContent = file.read() + from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + return getEnvName(envContent) + + @property + def isPlugin(self): + """ + Tests if the node is a valid plugin node + """ + return self._isPlugin + + @property + def isBuilt(self): + """ + Tests if the environnement is built + """ + from meshroom.core.plugin import isBuilt + return self._isPlugin and isBuilt(self) def upgradeAttributeValues(self, attrValues, fromVersion): return attrValues @@ -806,7 +850,17 @@ def buildCommandLine(self, chunk): if chunk.node.isParallelized and chunk.node.size > 1: cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict()) - return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + cmd=cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + + #the process in Popen does not seem to use the right python, even if meshroom_compute is called within the env + #so in the case of command line using python, we have to make sure it is using the correct python + from meshroom.core.plugin import EnvType, getVenvPath, getVenvExe #lazy import to prevent circular dep + if self.isPlugin and self.envType == EnvType.VENV: + envPath = getVenvPath(self._envName) + envExe = getVenvExe(envPath) + cmd=cmd.replace("python", envExe) + + return cmd def stopProcess(self, chunk): # The same node could exists several times in the graph and diff --git a/meshroom/core/node.py b/meshroom/core/node.py index a905487958..aeb3903727 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -21,7 +21,6 @@ from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError - def getWritingFilepath(filepath): return filepath + '.writing.' + str(uuid.uuid4()) @@ -406,13 +405,14 @@ def process(self, forceCompute=False): #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk # of the node will be ran into the env - if hasattr(self.node.nodeDesc, 'envFile') and self._status.status!=Status.FIRST_RUN: + if self.node.nodeDesc.isPlugin and self._status.status!=Status.FIRST_RUN: try: - if not self.node.nodeDesc.isBuild(): + from meshroom.core.plugin import isBuilt, build, getCommandLine #lazy import to avoid circular dep + if not isBuilt(self.node.nodeDesc): self.upgradeStatusTo(Status.BUILD) - self.node.nodeDesc.build() + build(self.node.nodeDesc) self.upgradeStatusTo(Status.FIRST_RUN) - command = self.node.nodeDesc.getCommandLine(self) + command = getCommandLine(self) #NOTE: docker returns 0 even if mount fail (it fails on the deamon side) logging.info("Running plugin node with "+command) status = os.system(command) @@ -482,6 +482,7 @@ def isExtern(self): statusNodeName = Property(str, lambda self: self._status.nodeName, constant=True) elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) + # Simple structure for storing node position @@ -1416,6 +1417,8 @@ def has3DOutputAttribute(self): hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) + isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin, constant=True) + isBuilt = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) class Node(BaseNode): """ diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index c7db6c95f3..d54e97f0ed 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -17,10 +17,11 @@ import venv import inspect -from meshroom.core import desc, hashValue -from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, defaultCacheFolder, pluginCatalogFile +from meshroom.core import desc +from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, pluginCatalogFile, defaultCacheFolder from meshroom.core import meshroomFolder from meshroom.core.graph import loadGraph +from meshroom.core import hashValue #where the executables are (eg meshroom compute) meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) @@ -54,6 +55,9 @@ def __init__(self, pluginUrl, jsonData=None): if "pipelineFolder" in jsonData.keys(): self.pipelineFolder = os.path.join(pluginUrl, jsonData["pipelineFolder"]) +def getEnvName(envContent): + return "meshroom_plugin_"+hashValue(envContent) + def _dockerImageExists(image_name, tag='latest'): """ Check if the desired image:tag exists @@ -103,11 +107,14 @@ def getVenvExe(venvPath): raise FileNotFoundError(f"Python executable not found in the specified virtual environment: "+executable) return executable +def getVenvPath(envName): + return os.path.join(defaultCacheFolder, envName) + def _venvExists(envName): """ Check if the following virtual env exists """ - return os.path.isdir(os.path.join(defaultCacheFolder, envName)) + return os.path.isdir(getVenvPath(envName)) def installPlugin(pluginUrl): """ @@ -203,166 +210,141 @@ def uninstallPlugin(pluginUrl): os.unlink(pluginUrl) else: os.removedirs(pluginUrl) - -class PluginNode(desc.Node): + +def isBuilt(nodeDesc): """ - Class to be used to make a plugin node, you need to overwrite envType and envFile + Check if the env needs to be build for a specific nodesc. """ + if nodeDesc.envType == EnvType.NONE: + return True + elif nodeDesc.envType == EnvType.PIP: + #NOTE: could find way to check for installed packages instead of rebuilding all the time + return False + elif nodeDesc.envType == EnvType.VENV: + return _venvExists(nodeDesc._envName) + elif nodeDesc.envType == EnvType.CONDA: + return _condaEnvExist(nodeDesc._envName) + elif nodeDesc.envType == EnvType.DOCKER: + return _dockerImageExists(nodeDesc._envName) - @property - def envType(cls): - """ - Dynamic env type - """ - raise NotImplementedError("You must specify one or several envtype in the node description") - - @property - def envFile(cls): - """ - Env file used to build the environement, you may overwrite this to custom the behaviour - """ - raise NotImplementedError("You must specify an env file") +def build(nodeDesc): + """ + Perform the needed steps to prepare the environement in which to run the node. + """ + if not hasattr(nodeDesc, 'envFile'): + raise RuntimeError("The nodedesc has no env file") + returnValue = 0 + if nodeDesc.envType == EnvType.NONE: + pass + elif nodeDesc.envType == EnvType.PIP: + #install packages in the same python as meshroom + logging.info("Installing packages from "+ nodeDesc.envFile) + buildCommand = sys.executable+" -m pip install "+ nodeDesc.envFile + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + elif nodeDesc.envType == EnvType.VENV: + #create venv in default cache folder + envPath = getVenvPath(nodeDesc._envName) + logging.info("Creating virtual env "+envPath+" from "+nodeDesc.envFile) + venv.create(envPath, with_pip=True) + logging.info("Installing dependencies") + envExe = getVenvExe(envPath) + returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ nodeDesc.envFile) + venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') + venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) + for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) + logging.info("Done") + elif nodeDesc.envType == EnvType.CONDA: + #build a conda env from a yaml file + logging.info("Creating conda env "+nodeDesc._envName+" from "+nodeDesc.envFile) + makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " + +" conda env create -v -v --name "+nodeDesc._envName + +" --file "+nodeDesc.envFile+" ") + logging.info("Making conda env") + logging.info(makeEnvCommand) + returnValue = os.system(makeEnvCommand) + #find path to env's folder and add symlink to meshroom + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+nodeDesc._envName + +" python -c \"import sys; print(sys.executable)\"", + shell=True).strip().decode('UTF-8') + condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') + condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) + for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) + logging.info("Done making conda env") + elif nodeDesc.envType == EnvType.DOCKER: + #build docker image + logging.info("Creating image "+nodeDesc._envName+" from "+ nodeDesc.envFile) + buildCommand = "docker build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + else: + raise RuntimeError("Invalid env type") + if returnValue != 0: + raise RuntimeError("Something went wrong during build") - @property - def _envName(cls): - """ - Get the env name by hashing the env files, overwrite this to use a custom pre-build env - """ - with open(cls.envFile, 'r') as file: - envContent = file.read() - return "meshroom_plugin_"+hashValue(envContent) +def getCommandLine(chunk): + """ + Return the command line needed to enter the environment + meshroom_compute + Will make meshroom available in the environment. + """ + nodeDesc=chunk.node.nodeDesc + if chunk.node.isParallelized: + raise RuntimeError("Parallelisation not supported for plugin nodes") + if chunk.node.graph.filepath == "": + raise RuntimeError("The project needs to be saved to use plugin nodes") + saved_graph = loadGraph(chunk.node.graph.filepath) + if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] + or chunk.node._uids[0] != saved_graph.findNode(str(chunk.node))._uids[0] ): + raise RuntimeError("The changes needs to be saved to use plugin nodes") - def isBuild(cls): - """ - Check if the env needs to be build. - """ - if cls.envType == EnvType.NONE: - return True - elif cls.envType == EnvType.PIP: - #NOTE: could find way to check for installed packages instead of rebuilding all the time - return False - elif cls.envType == EnvType.VENV: - return _venvExists(cls._envName) - elif cls.envType == EnvType.CONDA: - return _condaEnvExist(cls._envName) - elif cls.envType == EnvType.DOCKER: - return _dockerImageExists(cls._envName) + cmdPrefix = "" + # vars common to venv and conda, that will be passed when runing conda run or venv + meshroomCompute= meshroomBinDir+"/meshroom_compute" + meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" + pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" - def build(cls): - """ - Perform the needed steps to prepare the environement in which to run the node. - """ - if cls.envType == EnvType.NONE: - pass - elif cls.envType == EnvType.PIP: - #install packages in the same python as meshroom - logging.info("Installing packages from "+ cls.envFile) - buildCommand = sys.executable+" -m pip install "+ cls.envFile - logging.info("Building with "+buildCommand+" ...") - returnValue = os.system(buildCommand) - logging.info("Done") - elif cls.envType == EnvType.VENV: - #create venv in default cache folder - logging.info("Creating virtual env "+os.path.join(defaultCacheFolder, cls._envName)+" from "+cls.envFile) - envPath = os.path.join(defaultCacheFolder, cls._envName) - venv.create(envPath, with_pip=True) - logging.info("Installing dependencies") - envExe = getVenvExe(envPath) - returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ cls.envFile) - venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') - venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) - for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] - os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) - logging.info("Done") - elif cls.envType == EnvType.CONDA: - #build a conda env from a yaml file - logging.info("Creating conda env "+cls._envName+" from "+cls.envFile) - makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " - +" conda env create -v -v --name "+cls._envName - +" --file "+cls.envFile+" ") - logging.info("Making conda env") - logging.info(makeEnvCommand) - returnValue = os.system(makeEnvCommand) - #find path to env's folder and add symlink to meshroom - condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+cls._envName - +" python -c \"import sys; print(sys.executable)\"", - shell=True).strip().decode('UTF-8') - condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') - condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) - for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] - os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) - logging.info("Done making conda env") - elif cls.envType == EnvType.DOCKER: - #build docker image - logging.info("Creating image "+cls._envName+" from "+ cls.envFile) - buildCommand = "docker build -f "+cls.envFile+" -t "+cls._envName+" "+os.path.dirname(cls.envFile) - logging.info("Building with "+buildCommand+" ...") - returnValue = os.system(buildCommand) - logging.info("Done") - if returnValue != 0: - raise RuntimeError("Something went wrong during build") - - def getCommandLine(cls, chunk): - """ - Return the command line needed to enter the environment + meshroom_compute - Will make meshroom available in the environment. - """ - if chunk.node.isParallelized: - raise RuntimeError("Parallelisation not supported for plugin nodes") - if chunk.node.graph.filepath == "": - raise RuntimeError("The project needs to be saved to use plugin nodes") - saved_graph = loadGraph(chunk.node.graph.filepath) - if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] - or chunk.node._uids[0] != saved_graph.findNode(str(chunk.node))._uids[0] ): - raise RuntimeError("The changes needs to be saved to use plugin nodes") - - cmdPrefix = "" - # vars common to venv and conda, that will be passed when runing conda run or venv - meshroomCompute= meshroomBinDir+"/meshroom_compute" - meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" - pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" + if nodeDesc.envType == EnvType.VENV: + envPath = getVenvPath(nodeDesc._envName) + envExe = getVenvExe(envPath) + #make sure meshroom in in pythonpath and that we call the right python + cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.CONDA: + #NOTE: system env vars are not passed to conda run, we installed it 'manually' before + cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ + +" --no-capture-output -n "+nodeDesc._envName+" "+" python "+meshroomCompute + elif nodeDesc.envType == EnvType.DOCKER: + #path to the selected plugin + classFile=inspect.getfile(chunk.node.nodeDesc.__class__) + pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) + #path to the project/cache + projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) + mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy + +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) + +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' + +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') + #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path + envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" + #adds the gpu arg if needed + runtimeArg="" + if chunk.node.nodeDesc.gpu != desc.Level.NONE: + runtimeArg="--runtime=nvidia --gpus all" + #compose cl + cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " + meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + else: + raise RuntimeError("NodeType not recognised") - if cls.envType == EnvType.VENV: - envPath = os.path.join(defaultCacheFolder, cls._envName) - envExe = getVenvExe(envPath) - #make sure meshroom in in pythonpath and that we call the right python - cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " - elif cls.envType == EnvType.CONDA: - #NOTE: system env vars are not passed to conda run, we installed it 'manually' before - cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ - +" --no-capture-output -n "+cls._envName+" "+" python "+meshroomCompute - elif cls.envType == EnvType.DOCKER: - #path to the selected plugin - classFile=inspect.getfile(chunk.node.nodeDesc.__class__) - pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) - #path to the project/cache - projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) - mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy - +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) - +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' - +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') - #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path - envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" - #adds the gpu arg if needed - runtimeArg="" - if cls.gpu != desc.Level.NONE: - runtimeArg="--runtime=nvidia --gpus all" - #compose cl - cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+cls._envName +" \"python /meshroomBinDir/meshroom_compute " - meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + command=cmdPrefix+" "+meshroomComputeArgs + + return command - command=cmdPrefix+" "+meshroomComputeArgs - - return command +# you may use these to esplicitly define Pluginnodes +class PluginNode(desc.Node): + pass -#class that call command line nodes in an env -class PluginCommandLineNode(PluginNode, desc.CommandLineNode): - def buildCommandLine(self, chunk): - cmd = super().buildCommandLine(chunk) - #the process in Popen does not seem to use the right python, even if meshroom_compute is call within the env - #so in the case of command line using python, we have to make sur it is using the correct python - if self.envType == EnvType.VENV: - envPath = os.path.join(defaultCacheFolder, self._envName) - envExe = getVenvExe(envPath) - cmd=cmd.replace("python", envExe) - return cmd +class PluginCommandLineNode(desc.CommandLineNode): + pass \ No newline at end of file diff --git a/meshroom/plugins/catalog.json b/meshroom/plugins/catalog.json index 556ac56429..f4609c0c9d 100644 --- a/meshroom/plugins/catalog.json +++ b/meshroom/plugins/catalog.json @@ -2,8 +2,8 @@ { "pluginName":"Meshroom Research", "pluginUrl":"https://github.com/alicevision/MeshroomResearch/", - "description":"Meshroom-Research comprises a collection of plugins for Meshroom, mostly develloped in-house at MikrosImage", + "description":"Meshroom-Research comprises a collection of experimental plugins for Meshroom", "isCollection":true, "nodeTypes":["Python", "Docker", "Conda"] } -] \ No newline at end of file +] diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index ab06b9b347..345eb62b9e 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -131,6 +131,73 @@ Page { } } + //File browser for plugin + Dialog { + id: pluginURLDialog + title: "Plugin URL" + height: 150 + width: 300 + standardButtons: StandardButton.Ok | StandardButton.Cancel + //focus: true + Column { + anchors.fill: parent + Text { + text: "Plugin URL" + height: 40 + } + TextField { + id: urlInput + width: parent.width * 0.75 + focus: true + } + } + onButtonClicked: { + if (clickedButton==StandardButton.Ok) { + console.log("Accepted " + clickedButton) + if (_reconstruction.installPlugin(urlInput.text)) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + } + + // dialogs for plugins + MessageDialog { + id: pluginInstalledDialog + title: "Plugin installed" + modal: true + canCopy: false + Label { + text: "Plugin installed, please restart meshroom for the changes to take effect" + } + } + + MessageDialog { + id: pluginNotInstalledDialog + title: "Plugin not installed" + modal: true + canCopy: false + Label { + text: "Something went wrong, plugin not installed" + } + } + + // plugin installation from path or url + Platform.FolderDialog { + id: intallPluginDialog + options: Platform.FolderDialog.DontUseNativeDialog + title: "Install Plugin" + onAccepted: { + if (_reconstruction.installPlugin(currentFolder.toString())) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + Item { id: computeManager @@ -525,6 +592,23 @@ Page { } } + Action { + id: installPluginFromFolderAction + text: "Install Plugin From Local Folder" + onTriggered: { + initFileDialogFolder(intallPluginDialog) + intallPluginDialog.open() + } + } + + Action { + id: installPluginFromURLAction + text: "Install Plugin From URL" + onTriggered: { + pluginURLDialog.open() + } + } + header: RowLayout { spacing: 0 MaterialToolButton { @@ -1215,6 +1299,18 @@ Page { _reconstruction.selectedNode = n } } + + onDoBuild: { + try { + _reconstruction.buildNode(node.name) + node.isNotBuilt=false + } catch (error) { + //NOTE: could do an error popup + console.log("Build error:") + console.log(error) + } + + } } } } diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 9a8bb05a17..4de416c420 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -19,6 +19,9 @@ Item { property bool readOnly: node.locked /// Whether the node is in compatibility mode readonly property bool isCompatibilityNode: node ? node.hasOwnProperty("compatibilityIssue") : false + /// Whether the node is a plugin that needs to be build + readonly property bool isPlugin: node ? node.isPlugin : false + property bool isNotBuilt: node ? (!node.isBuilt) : false /// Mouse related states property bool mainSelected: false property bool selected: false @@ -28,7 +31,8 @@ Item { property point position: Qt.point(x, y) /// Styling property color shadowColor: "#cc000000" - readonly property color defaultColor: isCompatibilityNode ? "#444" : !node.isComputable ? "#BA3D69" : activePalette.base + //readonly property color defaultColor: isCompatibilityNode ? "#444" : ((isPlugin && isNotBuilt) ? "#444": (!node.isComputable ? "#BA3D69" : activePalette.base)) + readonly property color defaultColor: isCompatibilityNode ? "#444" : (!node.isComputable ? "#BA3D69" : activePalette.base) property color baseColor: defaultColor property point mousePosition: Qt.point(mouseArea.mouseX, mouseArea.mouseY) @@ -232,6 +236,15 @@ Item { issueDetails: root.node.issueDetails } } + + // ToBuild icon for PluginNodes + Loader { + active: root.isPlugin && root.isNotBuilt + sourceComponent: ToBuildBadge { + sourceComponent: iconDelegate + } + } + // Data sharing indicator // Note: for an unknown reason, there are some performance issues with the UI refresh. diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index f5998f87ef..84daec1ddf 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -18,9 +18,12 @@ Panel { property bool readOnly: false property bool isCompatibilityNode: node && node.compatibilityIssue !== undefined property string nodeStartDateTime: "" + readonly property bool isPlugin: node ? node.isPlugin : false + readonly property bool isNotBuilt: node ? (!node.isBuilt) : false signal attributeDoubleClicked(var mouse, var attribute) signal upgradeRequest() + signal doBuild() title: "Node" + (node !== null ? " - " + node.label + "" + (node.label !== node.defaultLabel ? " (" + node.defaultLabel + ")" : "") : "") icon: MaterialLabel { text: MaterialIcons.tune } @@ -225,6 +228,17 @@ Panel { } } + Loader { + active: root.isPlugin && root.isNotBuilt + Layout.fillWidth: true + visible: active // for layout update + + sourceComponent: ToBuildBadge { + onDoBuild: root.doBuild() + sourceComponent: bannerDelegate + } + } + Loader { Layout.fillHeight: true Layout.fillWidth: true diff --git a/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml new file mode 100644 index 0000000000..f7333700f0 --- /dev/null +++ b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml @@ -0,0 +1,66 @@ +import QtQuick 2.15 +import QtQuick.Controls 2.15 +import QtQuick.Layouts 1.11 +import MaterialIcons 2.2 + +Loader { + id: root + + sourceComponent: iconDelegate + + signal doBuild() + + property Component iconDelegate: Component { + + Label { + text: MaterialIcons.warning + font.family: MaterialIcons.fontFamily + font.pointSize: 12 + color: "#66207F" + + MouseArea { + anchors.fill: parent + hoverEnabled: true + onPressed: mouse.accepted = false + ToolTip.text: "Node env needs to be built" + ToolTip.visible: containsMouse + } + } + } + + property Component bannerDelegate: Component { + + Pane { + padding: 6 + clip: true + background: Rectangle { color: "#66207F" } + + RowLayout { + width: parent.width + Column { + Layout.fillWidth: true + Label { + width: parent.width + elide: Label.ElideMiddle + font.bold: true + text: "Env needs to be built" + color: "white" + } + Label { + width: parent.width + elide: Label.ElideMiddle + color: "white" + } + } + Button { + visible: (parent.width > width) ? 1 : 0 + palette.window: root.color + palette.button: Qt.darker(root.color, 1.2) + palette.buttonText: "white" + text: "Build" + onClicked: doBuild() + } + } + } + } +} diff --git a/meshroom/ui/qml/GraphEditor/qmldir b/meshroom/ui/qml/GraphEditor/qmldir index 4a4d4ca460..dccf2d5ba9 100644 --- a/meshroom/ui/qml/GraphEditor/qmldir +++ b/meshroom/ui/qml/GraphEditor/qmldir @@ -9,7 +9,8 @@ AttributePin 1.0 AttributePin.qml AttributeEditor 1.0 AttributeEditor.qml AttributeItemDelegate 1.0 AttributeItemDelegate.qml CompatibilityBadge 1.0 CompatibilityBadge.qml +ToBuildBadge 1.0 ToBuildBadge.qml CompatibilityManager 1.0 CompatibilityManager.qml singleton GraphEditorSettings 1.0 GraphEditorSettings.qml TaskManager 1.0 TaskManager.qml -ScriptEditor 1.0 ScriptEditor.qml \ No newline at end of file +ScriptEditor 1.0 ScriptEditor.qml diff --git a/meshroom/ui/qml/main.qml b/meshroom/ui/qml/main.qml index 5832af07d8..f011326450 100644 --- a/meshroom/ui/qml/main.qml +++ b/meshroom/ui/qml/main.qml @@ -132,73 +132,6 @@ ApplicationWindow { } } - //File browser for plugin - Dialog { - id: pluginURLDialog - title: "Plugin URL" - height: 150 - width: 300 - standardButtons: StandardButton.Ok | StandardButton.Cancel - //focus: true - Column { - anchors.fill: parent - Text { - text: "Plugin URL" - height: 40 - } - TextField { - id: urlInput - width: parent.width * 0.75 - focus: true - } - } - onButtonClicked: { - if (clickedButton==StandardButton.Ok) { - console.log("Accepted " + clickedButton) - if (_reconstruction.installPlugin(urlInput.text)) { - pluginInstalledDialog.open() - } else { - pluginNotInstalledDialog.open() - } - } - } - } - - // dialogs for plugins - MessageDialog { - id: pluginInstalledDialog - title: "Plugin installed" - modal: true - canCopy: false - Label { - text: "Plugin installed, please restart meshroom for the changes to take effect" - } - } - - MessageDialog { - id: pluginNotInstalledDialog - title: "Plugin not installed" - modal: true - canCopy: false - Label { - text: "Something went wrong, plugin not installed" - } - } - - // plugin installation from path or url - Platform.FolderDialog { - id: intallPluginDialog - options: Platform.FolderDialog.DontUseNativeDialog - title: "Install Plugin" - onAccepted: { - if (_reconstruction.installPlugin(currentFolder.toString())) { - pluginInstalledDialog.open() - } else { - pluginNotInstalledDialog.open() - } - } - } - // Check if document has been saved function ensureSaved(callback) { diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 225ded9bae..8918d22ac6 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -586,6 +586,14 @@ def installPlugin(self, url): localFile = prepareUrlLocalFile(url) return installPlugin(localFile) + @Slot(str, result=bool) + def buildNode(self, nodeName): + print("***Building "+nodeName) + node = self._graph.node(nodeName) + from meshroom.core.plugin import isBuilt, build #lazy import to avoid circular dep + if not isBuilt(node.nodeDesc): + build(node.nodeDesc) + def onGraphChanged(self): """ React to the change of the internal graph. """ self._liveSfmManager.reset() diff --git a/tests/test_plugin_nodes.py b/tests/test_plugin_nodes.py index 1df862553d..cc44461a0c 100644 --- a/tests/test_plugin_nodes.py +++ b/tests/test_plugin_nodes.py @@ -6,6 +6,7 @@ logging = logging.getLogger(__name__) def test_pluginNodes(): + #Dont run the tests in the CI as we are unable to install plugins beforehand if "CI" in os.environ: return graph = Graph('') @@ -14,4 +15,4 @@ def test_pluginNodes(): graph.addNewNode('DummyPipNode') graph.addNewNode('DummyVenvNode') - \ No newline at end of file + From b3aef333adbbd4b29f5eaa4b73c39cafcfd98ea4 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Tue, 24 Sep 2024 17:27:58 +0200 Subject: [PATCH 03/10] changes for uid renaming added back menu after update fix for existing symlink --- meshroom/core/plugin.py | 10 ++++++--- meshroom/ui/qml/Application.qml | 31 +++++++++++++++++++--------- meshroom/ui/qml/GraphEditor/Node.qml | 1 - 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index d54e97f0ed..2ef50f252d 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -170,7 +170,11 @@ def installPlugin(pluginUrl): if isLocal: os.symlink(pluginParam.nodesFolder, intallFolder) if os.path.isdir(pluginParam.pipelineFolder): - os.symlink(pluginParam.pipelineFolder, os.path.join(pluginsPipelinesFolder, pluginParam.pluginName)) + pipelineFolderLink = os.path.join(pluginsPipelinesFolder, pluginParam.pluginName) + if os.path.exists(pipelineFolderLink): + logging.warn("Plugin already installed, will overwrite") + os.unlink(pipelineFolderLink) + os.symlink(pluginParam.pipelineFolder, pipelineFolderLink) else: copy_tree(pluginParam.nodesFolder, intallFolder) if os.path.isdir(pluginParam.pipelineFolder): @@ -298,7 +302,7 @@ def getCommandLine(chunk): raise RuntimeError("The project needs to be saved to use plugin nodes") saved_graph = loadGraph(chunk.node.graph.filepath) if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] - or chunk.node._uids[0] != saved_graph.findNode(str(chunk.node))._uids[0] ): + or chunk.node._uid != saved_graph.findNode(str(chunk.node))._uid ): raise RuntimeError("The changes needs to be saved to use plugin nodes") cmdPrefix = "" @@ -342,7 +346,7 @@ def getCommandLine(chunk): return command -# you may use these to esplicitly define Pluginnodes +# you may use these to explicitly define Pluginnodes class PluginNode(desc.Node): pass diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index 345eb62b9e..00dc59b6a3 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -825,6 +825,18 @@ Page { ToolTip.visible: hovered ToolTip.text: removeImagesFromAllGroupsAction.tooltip } + + MenuItem { + action: installPluginFromFolderAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a folder" + } + + MenuItem { + action: installPluginFromURLAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a local or online url" + } } MenuSeparator { } Action { @@ -1298,18 +1310,17 @@ Page { var n = _reconstruction.upgradeNode(node) _reconstruction.selectedNode = n } - } - onDoBuild: { - try { - _reconstruction.buildNode(node.name) - node.isNotBuilt=false - } catch (error) { - //NOTE: could do an error popup - console.log("Build error:") - console.log(error) + onDoBuild: { + try { + _reconstruction.buildNode(node.name) + node.isNotBuilt=false + } catch (error) { + //NOTE: could do an error popup + console.log("Build error:") + console.log(error) + } } - } } } diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 4de416c420..6c1eb385c9 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -31,7 +31,6 @@ Item { property point position: Qt.point(x, y) /// Styling property color shadowColor: "#cc000000" - //readonly property color defaultColor: isCompatibilityNode ? "#444" : ((isPlugin && isNotBuilt) ? "#444": (!node.isComputable ? "#BA3D69" : activePalette.base)) readonly property color defaultColor: isCompatibilityNode ? "#444" : (!node.isComputable ? "#BA3D69" : activePalette.base) property color baseColor: defaultColor From f982c1a2a77cf4d0e32d39d3a4238fcc87aa9873 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Thu, 3 Oct 2024 11:34:20 +0200 Subject: [PATCH 04/10] added support for rez envs --- meshroom/core/desc.py | 6 +++- meshroom/core/plugin.py | 46 +++++++++++++++++++++++-------- tests/nodes/plugins/dummyNodes.py | 16 +++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index a501a926b1..b5ab4e188e 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -745,9 +745,13 @@ def _envName(cls): """ Get the env name by hashing the env files, overwrite this to use a custom pre-build env """ + from core.plugin import EnvType + from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + if cls.envType.value == EnvType.REZ.value: + return cls.envFile with open(cls.envFile, 'r') as file: envContent = file.read() - from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + return getEnvName(envContent) @property diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index 2ef50f252d..a2c88494e2 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -23,8 +23,10 @@ from meshroom.core.graph import loadGraph from meshroom.core import hashValue -#where the executables are (eg meshroom compute) +#executables def meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) +condaBin = "conda" +dockerBin = "docker" class EnvType(Enum): """ @@ -32,6 +34,7 @@ class EnvType(Enum): """ NONE = 0 PIP = 1 + REZ = 2 VENV = 10 CONDA = 20 DOCKER = 30 @@ -63,7 +66,7 @@ def _dockerImageExists(image_name, tag='latest'): Check if the desired image:tag exists """ try: - result = subprocess.run( ['docker', 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], + result = subprocess.run( [dockerBin, 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) if result.returncode != 0: return False @@ -84,8 +87,8 @@ def _condaEnvExist(envName): """ Checks if a specified env exists """ - cmd = "conda list --name "+envName - return os.system(cmd) == 0 + cmd = condaBin+" list --name "+envName + return os.system(cmd) == 0 def _formatPluginName(pluginName): """ @@ -116,6 +119,23 @@ def _venvExists(envName): """ return os.path.isdir(getVenvPath(envName)) +def getActiveRezPackages(): + """ + Returns a list containing the active explicit packages + """ + packages = [] + if 'REZ_REQUEST' in os.environ: + nondefpackages = [n.split("-")[0] for n in os.environ.get('REZ_REQUEST', '').split()] + resolvedPackages = os.environ.get('REZ_RESOLVE', '').split() + resolvedVersions = {} + for r in resolvedPackages: + if r.startswith('~'): + continue + v = r.split('-') + resolvedVersions[v[0]] = v[1] + packages = [p+"-"+resolvedVersions[p] for p in resolvedVersions.keys() if p in nondefpackages] + return packages + def installPlugin(pluginUrl): """ Install plugin from an url or local path. @@ -219,7 +239,7 @@ def isBuilt(nodeDesc): """ Check if the env needs to be build for a specific nodesc. """ - if nodeDesc.envType == EnvType.NONE: + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: return True elif nodeDesc.envType == EnvType.PIP: #NOTE: could find way to check for installed packages instead of rebuilding all the time @@ -238,7 +258,7 @@ def build(nodeDesc): if not hasattr(nodeDesc, 'envFile'): raise RuntimeError("The nodedesc has no env file") returnValue = 0 - if nodeDesc.envType == EnvType.NONE: + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: pass elif nodeDesc.envType == EnvType.PIP: #install packages in the same python as meshroom @@ -263,14 +283,14 @@ def build(nodeDesc): elif nodeDesc.envType == EnvType.CONDA: #build a conda env from a yaml file logging.info("Creating conda env "+nodeDesc._envName+" from "+nodeDesc.envFile) - makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " - +" conda env create -v -v --name "+nodeDesc._envName + makeEnvCommand = ( _cleanEnvVarsRez()+condaBin+" config --set channel_priority strict ; " + +condaBin+" env create -v -v --name "+nodeDesc._envName +" --file "+nodeDesc.envFile+" ") logging.info("Making conda env") logging.info(makeEnvCommand) returnValue = os.system(makeEnvCommand) #find path to env's folder and add symlink to meshroom - condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+nodeDesc._envName + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+condaBin+" run -n "+nodeDesc._envName +" python -c \"import sys; print(sys.executable)\"", shell=True).strip().decode('UTF-8') condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') @@ -281,7 +301,7 @@ def build(nodeDesc): elif nodeDesc.envType == EnvType.DOCKER: #build docker image logging.info("Creating image "+nodeDesc._envName+" from "+ nodeDesc.envFile) - buildCommand = "docker build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) + buildCommand = dockerBin+" build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) logging.info("Building with "+buildCommand+" ...") returnValue = os.system(buildCommand) logging.info("Done") @@ -316,9 +336,11 @@ def getCommandLine(chunk): envExe = getVenvExe(envPath) #make sure meshroom in in pythonpath and that we call the right python cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.REZ: + cmdPrefix = "rez env "+" ".join(getActiveRezPackages())+" "+nodeDesc._envName+" -- "+ meshroomCompute +" " elif nodeDesc.envType == EnvType.CONDA: #NOTE: system env vars are not passed to conda run, we installed it 'manually' before - cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ + cmdPrefix = _cleanEnvVarsRez()+condaBin+" run --cwd "+os.path.join(meshroomFolder, "..")\ +" --no-capture-output -n "+nodeDesc._envName+" "+" python "+meshroomCompute elif nodeDesc.envType == EnvType.DOCKER: #path to the selected plugin @@ -337,7 +359,7 @@ def getCommandLine(chunk): if chunk.node.nodeDesc.gpu != desc.Level.NONE: runtimeArg="--runtime=nvidia --gpus all" #compose cl - cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " + cmdPrefix = dockerBin+" run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" else: raise RuntimeError("NodeType not recognised") diff --git a/tests/nodes/plugins/dummyNodes.py b/tests/nodes/plugins/dummyNodes.py index b0446c90f3..e187b7fdb1 100644 --- a/tests/nodes/plugins/dummyNodes.py +++ b/tests/nodes/plugins/dummyNodes.py @@ -86,6 +86,22 @@ def processChunk(self, chunk): chunk.logManager.start("info") chunk.logger.info(np.abs(-1)) +class DummyRez(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.REZ + envFile = "numpy" + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + #Command line node class DummyCondaCL(PluginCommandLineNode): From 24c4585878bf6682a022f8db5a3baee77d22bdf5 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Mon, 7 Oct 2024 17:04:24 +0200 Subject: [PATCH 05/10] auto update --- meshroom/core/node.py | 6 ++++- meshroom/ui/graph.py | 28 +++++++++++++++++++--- meshroom/ui/qml/GraphEditor/Node.qml | 2 +- meshroom/ui/qml/GraphEditor/NodeEditor.qml | 2 +- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index aeb3903727..2e3c9cfa82 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1418,7 +1418,11 @@ def has3DOutputAttribute(self): has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin, constant=True) - isBuilt = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) + + isEnvBuild = (not isPlugin) #init build status false its not a plugin + buildStatusChanged = Signal() #event to notify change in status + isBuiltStatus = Property(bool, lambda self: self.isEnvBuild, notify = buildStatusChanged) + # isBuiltStatus = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) class Node(BaseNode): """ diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 448f25484a..0c3fa816fa 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -44,6 +44,7 @@ def __init__(self, parent=None): self._stopFlag = Event() self._refreshInterval = 5 # refresh interval in seconds self._files = [] + self._nodes = [] if submitters: self._filePollerRefresh = PollerRefreshStatus.MINIMAL_ENABLED else: @@ -53,7 +54,7 @@ def __del__(self): self._threadPool.terminate() self._threadPool.join() - def start(self, files=None): + def start(self, files=None, nodes=None): """ Start polling thread. Args: @@ -66,6 +67,7 @@ def start(self, files=None): return self._stopFlag.clear() self._files = files or [] + self._nodes = nodes or [] self._thread = Thread(target=self.run) self._thread.start() @@ -78,6 +80,15 @@ def setFiles(self, files): with self._mutex: self._files = files + def setNodes(self, nodes): + """ Set the list of nodes to monitor + + Args: + nodes: the list of nodes to monitor + """ + with self._mutex: + self._nodes = nodes + def stop(self): """ Request polling thread to stop. """ if not self._thread: @@ -94,6 +105,13 @@ def getFileLastModTime(f): except OSError: return -1 + @staticmethod + def updatePluginEnvStatus(n): + """ Will update the status of the plugin env """ + print("Refreshing "+str(n)) + n.isEnvBuild=n.nodeDesc.isBuilt + n.buildStatusChanged.emit() + def run(self): """ Poll watched files for last modification time. """ while not self._stopFlag.wait(self._refreshInterval): @@ -103,6 +121,8 @@ def run(self): with self._mutex: if files == self._files: self.timesAvailable.emit(times) + #update plugin nodes + _ = self._threadPool.map(self.updatePluginEnvStatus, self._nodes) def onFilePollerRefreshChanged(self, value): """ Stop or start the file poller depending on the new refresh status. """ @@ -116,7 +136,6 @@ def onFilePollerRefreshChanged(self, value): filePollerRefresh = Property(int, lambda self: self._filePollerRefresh.value, constant=True) filePollerRefreshReady = Signal() # The refresh status has been updated and is ready to be used - class ChunksMonitor(QObject): """ ChunksMonitor regularly check NodeChunks' status files for modification and trigger their update on change. @@ -147,6 +166,8 @@ def setChunks(self, chunks): self.monitorableChunks = chunks files, monitoredChunks = self.watchedStatusFiles self._filesTimePoller.setFiles(files) + pluginNodes = [c.node for c in chunks if c.node.isPlugin] + self._filesTimePoller.setNodes(pluginNodes) self.monitoredChunks = monitoredChunks def stop(self): @@ -172,7 +193,8 @@ def watchedStatusFiles(self): elif self.filePollerRefresh is PollerRefreshStatus.MINIMAL_ENABLED.value: for c in self.monitorableChunks: # When a chunk's status is ERROR, it may be externally re-submitted and it should thus still be monitored - if c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: + #Plugin nodes are always moniotored + if c.node.isPlugin or c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: files.append(c.statusFile) chunks.append(c) return files, chunks diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 6c1eb385c9..34707b9ed7 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -21,7 +21,7 @@ Item { readonly property bool isCompatibilityNode: node ? node.hasOwnProperty("compatibilityIssue") : false /// Whether the node is a plugin that needs to be build readonly property bool isPlugin: node ? node.isPlugin : false - property bool isNotBuilt: node ? (!node.isBuilt) : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false /// Mouse related states property bool mainSelected: false property bool selected: false diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index 84daec1ddf..a16f92e9fa 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -19,7 +19,7 @@ Panel { property bool isCompatibilityNode: node && node.compatibilityIssue !== undefined property string nodeStartDateTime: "" readonly property bool isPlugin: node ? node.isPlugin : false - readonly property bool isNotBuilt: node ? (!node.isBuilt) : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false signal attributeDoubleClicked(var mouse, var attribute) signal upgradeRequest() From 370d1346b4d13c3085a49faf0242e609ff1a837a Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Mon, 2 Sep 2024 16:49:58 +0200 Subject: [PATCH 06/10] moved plugin logic to nodeDesc instead of class added build badge changes for uid renaming added back menu after update fix for existing symlink --- meshroom/core/__init__.py | 2 +- meshroom/core/desc.py | 56 +++- meshroom/core/node.py | 13 +- meshroom/core/plugin.py | 302 +++++++++---------- meshroom/plugins/catalog.json | 4 +- meshroom/ui/qml/Application.qml | 107 +++++++ meshroom/ui/qml/GraphEditor/Node.qml | 14 +- meshroom/ui/qml/GraphEditor/NodeEditor.qml | 14 + meshroom/ui/qml/GraphEditor/ToBuildBadge.qml | 66 ++++ meshroom/ui/qml/GraphEditor/qmldir | 3 +- meshroom/ui/qml/main.qml | 67 ---- meshroom/ui/reconstruction.py | 8 + tests/test_plugin_nodes.py | 3 +- 13 files changed, 422 insertions(+), 237 deletions(-) create mode 100644 meshroom/ui/qml/GraphEditor/ToBuildBadge.qml diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 176dcdd0bf..55a192cb77 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ pass from meshroom.core.submitter import BaseSubmitter -from . import desc +from meshroom.core import desc # Setup logging logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index 1a7f4893c6..a501a926b1 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -717,9 +717,53 @@ class Node(object): documentation = '' category = 'Other' + _isPlugin = True + def __init__(self): super(Node, self).__init__() self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs) + try: + self.envFile + self.envType + except: + self._isPlugin=False + + @property + def envType(cls): + from core.plugin import EnvType #lazy import for plugin to avoid circular dependency + return EnvType.NONE + + @property + def envFile(cls): + """ + Env file used to build the environement, you may overwrite this to custom the behaviour + """ + raise NotImplementedError("You must specify an env file") + + @property + def _envName(cls): + """ + Get the env name by hashing the env files, overwrite this to use a custom pre-build env + """ + with open(cls.envFile, 'r') as file: + envContent = file.read() + from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + return getEnvName(envContent) + + @property + def isPlugin(self): + """ + Tests if the node is a valid plugin node + """ + return self._isPlugin + + @property + def isBuilt(self): + """ + Tests if the environnement is built + """ + from meshroom.core.plugin import isBuilt + return self._isPlugin and isBuilt(self) def upgradeAttributeValues(self, attrValues, fromVersion): return attrValues @@ -806,7 +850,17 @@ def buildCommandLine(self, chunk): if chunk.node.isParallelized and chunk.node.size > 1: cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict()) - return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + cmd=cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + + #the process in Popen does not seem to use the right python, even if meshroom_compute is called within the env + #so in the case of command line using python, we have to make sure it is using the correct python + from meshroom.core.plugin import EnvType, getVenvPath, getVenvExe #lazy import to prevent circular dep + if self.isPlugin and self.envType == EnvType.VENV: + envPath = getVenvPath(self._envName) + envExe = getVenvExe(envPath) + cmd=cmd.replace("python", envExe) + + return cmd def stopProcess(self, chunk): # The same node could exists several times in the graph and diff --git a/meshroom/core/node.py b/meshroom/core/node.py index a905487958..aeb3903727 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -21,7 +21,6 @@ from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError - def getWritingFilepath(filepath): return filepath + '.writing.' + str(uuid.uuid4()) @@ -406,13 +405,14 @@ def process(self, forceCompute=False): #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk # of the node will be ran into the env - if hasattr(self.node.nodeDesc, 'envFile') and self._status.status!=Status.FIRST_RUN: + if self.node.nodeDesc.isPlugin and self._status.status!=Status.FIRST_RUN: try: - if not self.node.nodeDesc.isBuild(): + from meshroom.core.plugin import isBuilt, build, getCommandLine #lazy import to avoid circular dep + if not isBuilt(self.node.nodeDesc): self.upgradeStatusTo(Status.BUILD) - self.node.nodeDesc.build() + build(self.node.nodeDesc) self.upgradeStatusTo(Status.FIRST_RUN) - command = self.node.nodeDesc.getCommandLine(self) + command = getCommandLine(self) #NOTE: docker returns 0 even if mount fail (it fails on the deamon side) logging.info("Running plugin node with "+command) status = os.system(command) @@ -482,6 +482,7 @@ def isExtern(self): statusNodeName = Property(str, lambda self: self._status.nodeName, constant=True) elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) + # Simple structure for storing node position @@ -1416,6 +1417,8 @@ def has3DOutputAttribute(self): hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) + isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin, constant=True) + isBuilt = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) class Node(BaseNode): """ diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index c7db6c95f3..2ef50f252d 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -17,10 +17,11 @@ import venv import inspect -from meshroom.core import desc, hashValue -from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, defaultCacheFolder, pluginCatalogFile +from meshroom.core import desc +from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, pluginCatalogFile, defaultCacheFolder from meshroom.core import meshroomFolder from meshroom.core.graph import loadGraph +from meshroom.core import hashValue #where the executables are (eg meshroom compute) meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) @@ -54,6 +55,9 @@ def __init__(self, pluginUrl, jsonData=None): if "pipelineFolder" in jsonData.keys(): self.pipelineFolder = os.path.join(pluginUrl, jsonData["pipelineFolder"]) +def getEnvName(envContent): + return "meshroom_plugin_"+hashValue(envContent) + def _dockerImageExists(image_name, tag='latest'): """ Check if the desired image:tag exists @@ -103,11 +107,14 @@ def getVenvExe(venvPath): raise FileNotFoundError(f"Python executable not found in the specified virtual environment: "+executable) return executable +def getVenvPath(envName): + return os.path.join(defaultCacheFolder, envName) + def _venvExists(envName): """ Check if the following virtual env exists """ - return os.path.isdir(os.path.join(defaultCacheFolder, envName)) + return os.path.isdir(getVenvPath(envName)) def installPlugin(pluginUrl): """ @@ -163,7 +170,11 @@ def installPlugin(pluginUrl): if isLocal: os.symlink(pluginParam.nodesFolder, intallFolder) if os.path.isdir(pluginParam.pipelineFolder): - os.symlink(pluginParam.pipelineFolder, os.path.join(pluginsPipelinesFolder, pluginParam.pluginName)) + pipelineFolderLink = os.path.join(pluginsPipelinesFolder, pluginParam.pluginName) + if os.path.exists(pipelineFolderLink): + logging.warn("Plugin already installed, will overwrite") + os.unlink(pipelineFolderLink) + os.symlink(pluginParam.pipelineFolder, pipelineFolderLink) else: copy_tree(pluginParam.nodesFolder, intallFolder) if os.path.isdir(pluginParam.pipelineFolder): @@ -203,166 +214,141 @@ def uninstallPlugin(pluginUrl): os.unlink(pluginUrl) else: os.removedirs(pluginUrl) - -class PluginNode(desc.Node): + +def isBuilt(nodeDesc): """ - Class to be used to make a plugin node, you need to overwrite envType and envFile + Check if the env needs to be build for a specific nodesc. """ + if nodeDesc.envType == EnvType.NONE: + return True + elif nodeDesc.envType == EnvType.PIP: + #NOTE: could find way to check for installed packages instead of rebuilding all the time + return False + elif nodeDesc.envType == EnvType.VENV: + return _venvExists(nodeDesc._envName) + elif nodeDesc.envType == EnvType.CONDA: + return _condaEnvExist(nodeDesc._envName) + elif nodeDesc.envType == EnvType.DOCKER: + return _dockerImageExists(nodeDesc._envName) - @property - def envType(cls): - """ - Dynamic env type - """ - raise NotImplementedError("You must specify one or several envtype in the node description") - - @property - def envFile(cls): - """ - Env file used to build the environement, you may overwrite this to custom the behaviour - """ - raise NotImplementedError("You must specify an env file") +def build(nodeDesc): + """ + Perform the needed steps to prepare the environement in which to run the node. + """ + if not hasattr(nodeDesc, 'envFile'): + raise RuntimeError("The nodedesc has no env file") + returnValue = 0 + if nodeDesc.envType == EnvType.NONE: + pass + elif nodeDesc.envType == EnvType.PIP: + #install packages in the same python as meshroom + logging.info("Installing packages from "+ nodeDesc.envFile) + buildCommand = sys.executable+" -m pip install "+ nodeDesc.envFile + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + elif nodeDesc.envType == EnvType.VENV: + #create venv in default cache folder + envPath = getVenvPath(nodeDesc._envName) + logging.info("Creating virtual env "+envPath+" from "+nodeDesc.envFile) + venv.create(envPath, with_pip=True) + logging.info("Installing dependencies") + envExe = getVenvExe(envPath) + returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ nodeDesc.envFile) + venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') + venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) + for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) + logging.info("Done") + elif nodeDesc.envType == EnvType.CONDA: + #build a conda env from a yaml file + logging.info("Creating conda env "+nodeDesc._envName+" from "+nodeDesc.envFile) + makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " + +" conda env create -v -v --name "+nodeDesc._envName + +" --file "+nodeDesc.envFile+" ") + logging.info("Making conda env") + logging.info(makeEnvCommand) + returnValue = os.system(makeEnvCommand) + #find path to env's folder and add symlink to meshroom + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+nodeDesc._envName + +" python -c \"import sys; print(sys.executable)\"", + shell=True).strip().decode('UTF-8') + condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') + condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) + for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) + logging.info("Done making conda env") + elif nodeDesc.envType == EnvType.DOCKER: + #build docker image + logging.info("Creating image "+nodeDesc._envName+" from "+ nodeDesc.envFile) + buildCommand = "docker build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + else: + raise RuntimeError("Invalid env type") + if returnValue != 0: + raise RuntimeError("Something went wrong during build") - @property - def _envName(cls): - """ - Get the env name by hashing the env files, overwrite this to use a custom pre-build env - """ - with open(cls.envFile, 'r') as file: - envContent = file.read() - return "meshroom_plugin_"+hashValue(envContent) +def getCommandLine(chunk): + """ + Return the command line needed to enter the environment + meshroom_compute + Will make meshroom available in the environment. + """ + nodeDesc=chunk.node.nodeDesc + if chunk.node.isParallelized: + raise RuntimeError("Parallelisation not supported for plugin nodes") + if chunk.node.graph.filepath == "": + raise RuntimeError("The project needs to be saved to use plugin nodes") + saved_graph = loadGraph(chunk.node.graph.filepath) + if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] + or chunk.node._uid != saved_graph.findNode(str(chunk.node))._uid ): + raise RuntimeError("The changes needs to be saved to use plugin nodes") - def isBuild(cls): - """ - Check if the env needs to be build. - """ - if cls.envType == EnvType.NONE: - return True - elif cls.envType == EnvType.PIP: - #NOTE: could find way to check for installed packages instead of rebuilding all the time - return False - elif cls.envType == EnvType.VENV: - return _venvExists(cls._envName) - elif cls.envType == EnvType.CONDA: - return _condaEnvExist(cls._envName) - elif cls.envType == EnvType.DOCKER: - return _dockerImageExists(cls._envName) + cmdPrefix = "" + # vars common to venv and conda, that will be passed when runing conda run or venv + meshroomCompute= meshroomBinDir+"/meshroom_compute" + meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" + pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" - def build(cls): - """ - Perform the needed steps to prepare the environement in which to run the node. - """ - if cls.envType == EnvType.NONE: - pass - elif cls.envType == EnvType.PIP: - #install packages in the same python as meshroom - logging.info("Installing packages from "+ cls.envFile) - buildCommand = sys.executable+" -m pip install "+ cls.envFile - logging.info("Building with "+buildCommand+" ...") - returnValue = os.system(buildCommand) - logging.info("Done") - elif cls.envType == EnvType.VENV: - #create venv in default cache folder - logging.info("Creating virtual env "+os.path.join(defaultCacheFolder, cls._envName)+" from "+cls.envFile) - envPath = os.path.join(defaultCacheFolder, cls._envName) - venv.create(envPath, with_pip=True) - logging.info("Installing dependencies") - envExe = getVenvExe(envPath) - returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ cls.envFile) - venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') - venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) - for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] - os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) - logging.info("Done") - elif cls.envType == EnvType.CONDA: - #build a conda env from a yaml file - logging.info("Creating conda env "+cls._envName+" from "+cls.envFile) - makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " - +" conda env create -v -v --name "+cls._envName - +" --file "+cls.envFile+" ") - logging.info("Making conda env") - logging.info(makeEnvCommand) - returnValue = os.system(makeEnvCommand) - #find path to env's folder and add symlink to meshroom - condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+cls._envName - +" python -c \"import sys; print(sys.executable)\"", - shell=True).strip().decode('UTF-8') - condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') - condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) - for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] - os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) - logging.info("Done making conda env") - elif cls.envType == EnvType.DOCKER: - #build docker image - logging.info("Creating image "+cls._envName+" from "+ cls.envFile) - buildCommand = "docker build -f "+cls.envFile+" -t "+cls._envName+" "+os.path.dirname(cls.envFile) - logging.info("Building with "+buildCommand+" ...") - returnValue = os.system(buildCommand) - logging.info("Done") - if returnValue != 0: - raise RuntimeError("Something went wrong during build") - - def getCommandLine(cls, chunk): - """ - Return the command line needed to enter the environment + meshroom_compute - Will make meshroom available in the environment. - """ - if chunk.node.isParallelized: - raise RuntimeError("Parallelisation not supported for plugin nodes") - if chunk.node.graph.filepath == "": - raise RuntimeError("The project needs to be saved to use plugin nodes") - saved_graph = loadGraph(chunk.node.graph.filepath) - if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] - or chunk.node._uids[0] != saved_graph.findNode(str(chunk.node))._uids[0] ): - raise RuntimeError("The changes needs to be saved to use plugin nodes") - - cmdPrefix = "" - # vars common to venv and conda, that will be passed when runing conda run or venv - meshroomCompute= meshroomBinDir+"/meshroom_compute" - meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" - pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" + if nodeDesc.envType == EnvType.VENV: + envPath = getVenvPath(nodeDesc._envName) + envExe = getVenvExe(envPath) + #make sure meshroom in in pythonpath and that we call the right python + cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.CONDA: + #NOTE: system env vars are not passed to conda run, we installed it 'manually' before + cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ + +" --no-capture-output -n "+nodeDesc._envName+" "+" python "+meshroomCompute + elif nodeDesc.envType == EnvType.DOCKER: + #path to the selected plugin + classFile=inspect.getfile(chunk.node.nodeDesc.__class__) + pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) + #path to the project/cache + projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) + mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy + +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) + +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' + +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') + #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path + envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" + #adds the gpu arg if needed + runtimeArg="" + if chunk.node.nodeDesc.gpu != desc.Level.NONE: + runtimeArg="--runtime=nvidia --gpus all" + #compose cl + cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " + meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + else: + raise RuntimeError("NodeType not recognised") - if cls.envType == EnvType.VENV: - envPath = os.path.join(defaultCacheFolder, cls._envName) - envExe = getVenvExe(envPath) - #make sure meshroom in in pythonpath and that we call the right python - cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " - elif cls.envType == EnvType.CONDA: - #NOTE: system env vars are not passed to conda run, we installed it 'manually' before - cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ - +" --no-capture-output -n "+cls._envName+" "+" python "+meshroomCompute - elif cls.envType == EnvType.DOCKER: - #path to the selected plugin - classFile=inspect.getfile(chunk.node.nodeDesc.__class__) - pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) - #path to the project/cache - projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) - mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy - +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) - +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' - +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') - #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path - envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" - #adds the gpu arg if needed - runtimeArg="" - if cls.gpu != desc.Level.NONE: - runtimeArg="--runtime=nvidia --gpus all" - #compose cl - cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+cls._envName +" \"python /meshroomBinDir/meshroom_compute " - meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + command=cmdPrefix+" "+meshroomComputeArgs + + return command - command=cmdPrefix+" "+meshroomComputeArgs - - return command +# you may use these to explicitly define Pluginnodes +class PluginNode(desc.Node): + pass -#class that call command line nodes in an env -class PluginCommandLineNode(PluginNode, desc.CommandLineNode): - def buildCommandLine(self, chunk): - cmd = super().buildCommandLine(chunk) - #the process in Popen does not seem to use the right python, even if meshroom_compute is call within the env - #so in the case of command line using python, we have to make sur it is using the correct python - if self.envType == EnvType.VENV: - envPath = os.path.join(defaultCacheFolder, self._envName) - envExe = getVenvExe(envPath) - cmd=cmd.replace("python", envExe) - return cmd +class PluginCommandLineNode(desc.CommandLineNode): + pass \ No newline at end of file diff --git a/meshroom/plugins/catalog.json b/meshroom/plugins/catalog.json index 556ac56429..f4609c0c9d 100644 --- a/meshroom/plugins/catalog.json +++ b/meshroom/plugins/catalog.json @@ -2,8 +2,8 @@ { "pluginName":"Meshroom Research", "pluginUrl":"https://github.com/alicevision/MeshroomResearch/", - "description":"Meshroom-Research comprises a collection of plugins for Meshroom, mostly develloped in-house at MikrosImage", + "description":"Meshroom-Research comprises a collection of experimental plugins for Meshroom", "isCollection":true, "nodeTypes":["Python", "Docker", "Conda"] } -] \ No newline at end of file +] diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index ab06b9b347..00dc59b6a3 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -131,6 +131,73 @@ Page { } } + //File browser for plugin + Dialog { + id: pluginURLDialog + title: "Plugin URL" + height: 150 + width: 300 + standardButtons: StandardButton.Ok | StandardButton.Cancel + //focus: true + Column { + anchors.fill: parent + Text { + text: "Plugin URL" + height: 40 + } + TextField { + id: urlInput + width: parent.width * 0.75 + focus: true + } + } + onButtonClicked: { + if (clickedButton==StandardButton.Ok) { + console.log("Accepted " + clickedButton) + if (_reconstruction.installPlugin(urlInput.text)) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + } + + // dialogs for plugins + MessageDialog { + id: pluginInstalledDialog + title: "Plugin installed" + modal: true + canCopy: false + Label { + text: "Plugin installed, please restart meshroom for the changes to take effect" + } + } + + MessageDialog { + id: pluginNotInstalledDialog + title: "Plugin not installed" + modal: true + canCopy: false + Label { + text: "Something went wrong, plugin not installed" + } + } + + // plugin installation from path or url + Platform.FolderDialog { + id: intallPluginDialog + options: Platform.FolderDialog.DontUseNativeDialog + title: "Install Plugin" + onAccepted: { + if (_reconstruction.installPlugin(currentFolder.toString())) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + Item { id: computeManager @@ -525,6 +592,23 @@ Page { } } + Action { + id: installPluginFromFolderAction + text: "Install Plugin From Local Folder" + onTriggered: { + initFileDialogFolder(intallPluginDialog) + intallPluginDialog.open() + } + } + + Action { + id: installPluginFromURLAction + text: "Install Plugin From URL" + onTriggered: { + pluginURLDialog.open() + } + } + header: RowLayout { spacing: 0 MaterialToolButton { @@ -741,6 +825,18 @@ Page { ToolTip.visible: hovered ToolTip.text: removeImagesFromAllGroupsAction.tooltip } + + MenuItem { + action: installPluginFromFolderAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a folder" + } + + MenuItem { + action: installPluginFromURLAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a local or online url" + } } MenuSeparator { } Action { @@ -1214,6 +1310,17 @@ Page { var n = _reconstruction.upgradeNode(node) _reconstruction.selectedNode = n } + + onDoBuild: { + try { + _reconstruction.buildNode(node.name) + node.isNotBuilt=false + } catch (error) { + //NOTE: could do an error popup + console.log("Build error:") + console.log(error) + } + } } } } diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 9a8bb05a17..6c1eb385c9 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -19,6 +19,9 @@ Item { property bool readOnly: node.locked /// Whether the node is in compatibility mode readonly property bool isCompatibilityNode: node ? node.hasOwnProperty("compatibilityIssue") : false + /// Whether the node is a plugin that needs to be build + readonly property bool isPlugin: node ? node.isPlugin : false + property bool isNotBuilt: node ? (!node.isBuilt) : false /// Mouse related states property bool mainSelected: false property bool selected: false @@ -28,7 +31,7 @@ Item { property point position: Qt.point(x, y) /// Styling property color shadowColor: "#cc000000" - readonly property color defaultColor: isCompatibilityNode ? "#444" : !node.isComputable ? "#BA3D69" : activePalette.base + readonly property color defaultColor: isCompatibilityNode ? "#444" : (!node.isComputable ? "#BA3D69" : activePalette.base) property color baseColor: defaultColor property point mousePosition: Qt.point(mouseArea.mouseX, mouseArea.mouseY) @@ -232,6 +235,15 @@ Item { issueDetails: root.node.issueDetails } } + + // ToBuild icon for PluginNodes + Loader { + active: root.isPlugin && root.isNotBuilt + sourceComponent: ToBuildBadge { + sourceComponent: iconDelegate + } + } + // Data sharing indicator // Note: for an unknown reason, there are some performance issues with the UI refresh. diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index f5998f87ef..84daec1ddf 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -18,9 +18,12 @@ Panel { property bool readOnly: false property bool isCompatibilityNode: node && node.compatibilityIssue !== undefined property string nodeStartDateTime: "" + readonly property bool isPlugin: node ? node.isPlugin : false + readonly property bool isNotBuilt: node ? (!node.isBuilt) : false signal attributeDoubleClicked(var mouse, var attribute) signal upgradeRequest() + signal doBuild() title: "Node" + (node !== null ? " - " + node.label + "" + (node.label !== node.defaultLabel ? " (" + node.defaultLabel + ")" : "") : "") icon: MaterialLabel { text: MaterialIcons.tune } @@ -225,6 +228,17 @@ Panel { } } + Loader { + active: root.isPlugin && root.isNotBuilt + Layout.fillWidth: true + visible: active // for layout update + + sourceComponent: ToBuildBadge { + onDoBuild: root.doBuild() + sourceComponent: bannerDelegate + } + } + Loader { Layout.fillHeight: true Layout.fillWidth: true diff --git a/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml new file mode 100644 index 0000000000..f7333700f0 --- /dev/null +++ b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml @@ -0,0 +1,66 @@ +import QtQuick 2.15 +import QtQuick.Controls 2.15 +import QtQuick.Layouts 1.11 +import MaterialIcons 2.2 + +Loader { + id: root + + sourceComponent: iconDelegate + + signal doBuild() + + property Component iconDelegate: Component { + + Label { + text: MaterialIcons.warning + font.family: MaterialIcons.fontFamily + font.pointSize: 12 + color: "#66207F" + + MouseArea { + anchors.fill: parent + hoverEnabled: true + onPressed: mouse.accepted = false + ToolTip.text: "Node env needs to be built" + ToolTip.visible: containsMouse + } + } + } + + property Component bannerDelegate: Component { + + Pane { + padding: 6 + clip: true + background: Rectangle { color: "#66207F" } + + RowLayout { + width: parent.width + Column { + Layout.fillWidth: true + Label { + width: parent.width + elide: Label.ElideMiddle + font.bold: true + text: "Env needs to be built" + color: "white" + } + Label { + width: parent.width + elide: Label.ElideMiddle + color: "white" + } + } + Button { + visible: (parent.width > width) ? 1 : 0 + palette.window: root.color + palette.button: Qt.darker(root.color, 1.2) + palette.buttonText: "white" + text: "Build" + onClicked: doBuild() + } + } + } + } +} diff --git a/meshroom/ui/qml/GraphEditor/qmldir b/meshroom/ui/qml/GraphEditor/qmldir index 4a4d4ca460..dccf2d5ba9 100644 --- a/meshroom/ui/qml/GraphEditor/qmldir +++ b/meshroom/ui/qml/GraphEditor/qmldir @@ -9,7 +9,8 @@ AttributePin 1.0 AttributePin.qml AttributeEditor 1.0 AttributeEditor.qml AttributeItemDelegate 1.0 AttributeItemDelegate.qml CompatibilityBadge 1.0 CompatibilityBadge.qml +ToBuildBadge 1.0 ToBuildBadge.qml CompatibilityManager 1.0 CompatibilityManager.qml singleton GraphEditorSettings 1.0 GraphEditorSettings.qml TaskManager 1.0 TaskManager.qml -ScriptEditor 1.0 ScriptEditor.qml \ No newline at end of file +ScriptEditor 1.0 ScriptEditor.qml diff --git a/meshroom/ui/qml/main.qml b/meshroom/ui/qml/main.qml index 5832af07d8..f011326450 100644 --- a/meshroom/ui/qml/main.qml +++ b/meshroom/ui/qml/main.qml @@ -132,73 +132,6 @@ ApplicationWindow { } } - //File browser for plugin - Dialog { - id: pluginURLDialog - title: "Plugin URL" - height: 150 - width: 300 - standardButtons: StandardButton.Ok | StandardButton.Cancel - //focus: true - Column { - anchors.fill: parent - Text { - text: "Plugin URL" - height: 40 - } - TextField { - id: urlInput - width: parent.width * 0.75 - focus: true - } - } - onButtonClicked: { - if (clickedButton==StandardButton.Ok) { - console.log("Accepted " + clickedButton) - if (_reconstruction.installPlugin(urlInput.text)) { - pluginInstalledDialog.open() - } else { - pluginNotInstalledDialog.open() - } - } - } - } - - // dialogs for plugins - MessageDialog { - id: pluginInstalledDialog - title: "Plugin installed" - modal: true - canCopy: false - Label { - text: "Plugin installed, please restart meshroom for the changes to take effect" - } - } - - MessageDialog { - id: pluginNotInstalledDialog - title: "Plugin not installed" - modal: true - canCopy: false - Label { - text: "Something went wrong, plugin not installed" - } - } - - // plugin installation from path or url - Platform.FolderDialog { - id: intallPluginDialog - options: Platform.FolderDialog.DontUseNativeDialog - title: "Install Plugin" - onAccepted: { - if (_reconstruction.installPlugin(currentFolder.toString())) { - pluginInstalledDialog.open() - } else { - pluginNotInstalledDialog.open() - } - } - } - // Check if document has been saved function ensureSaved(callback) { diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index 225ded9bae..8918d22ac6 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -586,6 +586,14 @@ def installPlugin(self, url): localFile = prepareUrlLocalFile(url) return installPlugin(localFile) + @Slot(str, result=bool) + def buildNode(self, nodeName): + print("***Building "+nodeName) + node = self._graph.node(nodeName) + from meshroom.core.plugin import isBuilt, build #lazy import to avoid circular dep + if not isBuilt(node.nodeDesc): + build(node.nodeDesc) + def onGraphChanged(self): """ React to the change of the internal graph. """ self._liveSfmManager.reset() diff --git a/tests/test_plugin_nodes.py b/tests/test_plugin_nodes.py index 1df862553d..cc44461a0c 100644 --- a/tests/test_plugin_nodes.py +++ b/tests/test_plugin_nodes.py @@ -6,6 +6,7 @@ logging = logging.getLogger(__name__) def test_pluginNodes(): + #Dont run the tests in the CI as we are unable to install plugins beforehand if "CI" in os.environ: return graph = Graph('') @@ -14,4 +15,4 @@ def test_pluginNodes(): graph.addNewNode('DummyPipNode') graph.addNewNode('DummyVenvNode') - \ No newline at end of file + From f2423b956c1af3ac1216010c8125b0d28398e976 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Thu, 3 Oct 2024 11:34:20 +0200 Subject: [PATCH 07/10] added support for rez envs --- meshroom/core/desc.py | 6 +++- meshroom/core/plugin.py | 46 +++++++++++++++++++++++-------- tests/nodes/plugins/dummyNodes.py | 16 +++++++++++ 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index a501a926b1..b5ab4e188e 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -745,9 +745,13 @@ def _envName(cls): """ Get the env name by hashing the env files, overwrite this to use a custom pre-build env """ + from core.plugin import EnvType + from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + if cls.envType.value == EnvType.REZ.value: + return cls.envFile with open(cls.envFile, 'r') as file: envContent = file.read() - from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + return getEnvName(envContent) @property diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index 2ef50f252d..a2c88494e2 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -23,8 +23,10 @@ from meshroom.core.graph import loadGraph from meshroom.core import hashValue -#where the executables are (eg meshroom compute) +#executables def meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) +condaBin = "conda" +dockerBin = "docker" class EnvType(Enum): """ @@ -32,6 +34,7 @@ class EnvType(Enum): """ NONE = 0 PIP = 1 + REZ = 2 VENV = 10 CONDA = 20 DOCKER = 30 @@ -63,7 +66,7 @@ def _dockerImageExists(image_name, tag='latest'): Check if the desired image:tag exists """ try: - result = subprocess.run( ['docker', 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], + result = subprocess.run( [dockerBin, 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) if result.returncode != 0: return False @@ -84,8 +87,8 @@ def _condaEnvExist(envName): """ Checks if a specified env exists """ - cmd = "conda list --name "+envName - return os.system(cmd) == 0 + cmd = condaBin+" list --name "+envName + return os.system(cmd) == 0 def _formatPluginName(pluginName): """ @@ -116,6 +119,23 @@ def _venvExists(envName): """ return os.path.isdir(getVenvPath(envName)) +def getActiveRezPackages(): + """ + Returns a list containing the active explicit packages + """ + packages = [] + if 'REZ_REQUEST' in os.environ: + nondefpackages = [n.split("-")[0] for n in os.environ.get('REZ_REQUEST', '').split()] + resolvedPackages = os.environ.get('REZ_RESOLVE', '').split() + resolvedVersions = {} + for r in resolvedPackages: + if r.startswith('~'): + continue + v = r.split('-') + resolvedVersions[v[0]] = v[1] + packages = [p+"-"+resolvedVersions[p] for p in resolvedVersions.keys() if p in nondefpackages] + return packages + def installPlugin(pluginUrl): """ Install plugin from an url or local path. @@ -219,7 +239,7 @@ def isBuilt(nodeDesc): """ Check if the env needs to be build for a specific nodesc. """ - if nodeDesc.envType == EnvType.NONE: + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: return True elif nodeDesc.envType == EnvType.PIP: #NOTE: could find way to check for installed packages instead of rebuilding all the time @@ -238,7 +258,7 @@ def build(nodeDesc): if not hasattr(nodeDesc, 'envFile'): raise RuntimeError("The nodedesc has no env file") returnValue = 0 - if nodeDesc.envType == EnvType.NONE: + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: pass elif nodeDesc.envType == EnvType.PIP: #install packages in the same python as meshroom @@ -263,14 +283,14 @@ def build(nodeDesc): elif nodeDesc.envType == EnvType.CONDA: #build a conda env from a yaml file logging.info("Creating conda env "+nodeDesc._envName+" from "+nodeDesc.envFile) - makeEnvCommand = ( _cleanEnvVarsRez()+" conda config --set channel_priority strict ; " - +" conda env create -v -v --name "+nodeDesc._envName + makeEnvCommand = ( _cleanEnvVarsRez()+condaBin+" config --set channel_priority strict ; " + +condaBin+" env create -v -v --name "+nodeDesc._envName +" --file "+nodeDesc.envFile+" ") logging.info("Making conda env") logging.info(makeEnvCommand) returnValue = os.system(makeEnvCommand) #find path to env's folder and add symlink to meshroom - condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+"conda run -n "+nodeDesc._envName + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+condaBin+" run -n "+nodeDesc._envName +" python -c \"import sys; print(sys.executable)\"", shell=True).strip().decode('UTF-8') condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') @@ -281,7 +301,7 @@ def build(nodeDesc): elif nodeDesc.envType == EnvType.DOCKER: #build docker image logging.info("Creating image "+nodeDesc._envName+" from "+ nodeDesc.envFile) - buildCommand = "docker build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) + buildCommand = dockerBin+" build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) logging.info("Building with "+buildCommand+" ...") returnValue = os.system(buildCommand) logging.info("Done") @@ -316,9 +336,11 @@ def getCommandLine(chunk): envExe = getVenvExe(envPath) #make sure meshroom in in pythonpath and that we call the right python cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.REZ: + cmdPrefix = "rez env "+" ".join(getActiveRezPackages())+" "+nodeDesc._envName+" -- "+ meshroomCompute +" " elif nodeDesc.envType == EnvType.CONDA: #NOTE: system env vars are not passed to conda run, we installed it 'manually' before - cmdPrefix = _cleanEnvVarsRez()+" conda run --cwd "+os.path.join(meshroomFolder, "..")\ + cmdPrefix = _cleanEnvVarsRez()+condaBin+" run --cwd "+os.path.join(meshroomFolder, "..")\ +" --no-capture-output -n "+nodeDesc._envName+" "+" python "+meshroomCompute elif nodeDesc.envType == EnvType.DOCKER: #path to the selected plugin @@ -337,7 +359,7 @@ def getCommandLine(chunk): if chunk.node.nodeDesc.gpu != desc.Level.NONE: runtimeArg="--runtime=nvidia --gpus all" #compose cl - cmdPrefix = "docker run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " + cmdPrefix = dockerBin+" run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" else: raise RuntimeError("NodeType not recognised") diff --git a/tests/nodes/plugins/dummyNodes.py b/tests/nodes/plugins/dummyNodes.py index b0446c90f3..e187b7fdb1 100644 --- a/tests/nodes/plugins/dummyNodes.py +++ b/tests/nodes/plugins/dummyNodes.py @@ -86,6 +86,22 @@ def processChunk(self, chunk): chunk.logManager.start("info") chunk.logger.info(np.abs(-1)) +class DummyRez(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.REZ + envFile = "numpy" + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + #Command line node class DummyCondaCL(PluginCommandLineNode): From ee7d432bb020fd9e54e075cddc5d076780b7e735 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Mon, 7 Oct 2024 17:04:24 +0200 Subject: [PATCH 08/10] auto update --- meshroom/core/node.py | 6 ++++- meshroom/ui/graph.py | 28 +++++++++++++++++++--- meshroom/ui/qml/GraphEditor/Node.qml | 2 +- meshroom/ui/qml/GraphEditor/NodeEditor.qml | 2 +- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/meshroom/core/node.py b/meshroom/core/node.py index aeb3903727..2e3c9cfa82 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1418,7 +1418,11 @@ def has3DOutputAttribute(self): has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin, constant=True) - isBuilt = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) + + isEnvBuild = (not isPlugin) #init build status false its not a plugin + buildStatusChanged = Signal() #event to notify change in status + isBuiltStatus = Property(bool, lambda self: self.isEnvBuild, notify = buildStatusChanged) + # isBuiltStatus = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) class Node(BaseNode): """ diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 448f25484a..0c3fa816fa 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -44,6 +44,7 @@ def __init__(self, parent=None): self._stopFlag = Event() self._refreshInterval = 5 # refresh interval in seconds self._files = [] + self._nodes = [] if submitters: self._filePollerRefresh = PollerRefreshStatus.MINIMAL_ENABLED else: @@ -53,7 +54,7 @@ def __del__(self): self._threadPool.terminate() self._threadPool.join() - def start(self, files=None): + def start(self, files=None, nodes=None): """ Start polling thread. Args: @@ -66,6 +67,7 @@ def start(self, files=None): return self._stopFlag.clear() self._files = files or [] + self._nodes = nodes or [] self._thread = Thread(target=self.run) self._thread.start() @@ -78,6 +80,15 @@ def setFiles(self, files): with self._mutex: self._files = files + def setNodes(self, nodes): + """ Set the list of nodes to monitor + + Args: + nodes: the list of nodes to monitor + """ + with self._mutex: + self._nodes = nodes + def stop(self): """ Request polling thread to stop. """ if not self._thread: @@ -94,6 +105,13 @@ def getFileLastModTime(f): except OSError: return -1 + @staticmethod + def updatePluginEnvStatus(n): + """ Will update the status of the plugin env """ + print("Refreshing "+str(n)) + n.isEnvBuild=n.nodeDesc.isBuilt + n.buildStatusChanged.emit() + def run(self): """ Poll watched files for last modification time. """ while not self._stopFlag.wait(self._refreshInterval): @@ -103,6 +121,8 @@ def run(self): with self._mutex: if files == self._files: self.timesAvailable.emit(times) + #update plugin nodes + _ = self._threadPool.map(self.updatePluginEnvStatus, self._nodes) def onFilePollerRefreshChanged(self, value): """ Stop or start the file poller depending on the new refresh status. """ @@ -116,7 +136,6 @@ def onFilePollerRefreshChanged(self, value): filePollerRefresh = Property(int, lambda self: self._filePollerRefresh.value, constant=True) filePollerRefreshReady = Signal() # The refresh status has been updated and is ready to be used - class ChunksMonitor(QObject): """ ChunksMonitor regularly check NodeChunks' status files for modification and trigger their update on change. @@ -147,6 +166,8 @@ def setChunks(self, chunks): self.monitorableChunks = chunks files, monitoredChunks = self.watchedStatusFiles self._filesTimePoller.setFiles(files) + pluginNodes = [c.node for c in chunks if c.node.isPlugin] + self._filesTimePoller.setNodes(pluginNodes) self.monitoredChunks = monitoredChunks def stop(self): @@ -172,7 +193,8 @@ def watchedStatusFiles(self): elif self.filePollerRefresh is PollerRefreshStatus.MINIMAL_ENABLED.value: for c in self.monitorableChunks: # When a chunk's status is ERROR, it may be externally re-submitted and it should thus still be monitored - if c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: + #Plugin nodes are always moniotored + if c.node.isPlugin or c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: files.append(c.statusFile) chunks.append(c) return files, chunks diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 6c1eb385c9..34707b9ed7 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -21,7 +21,7 @@ Item { readonly property bool isCompatibilityNode: node ? node.hasOwnProperty("compatibilityIssue") : false /// Whether the node is a plugin that needs to be build readonly property bool isPlugin: node ? node.isPlugin : false - property bool isNotBuilt: node ? (!node.isBuilt) : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false /// Mouse related states property bool mainSelected: false property bool selected: false diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index 84daec1ddf..a16f92e9fa 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -19,7 +19,7 @@ Panel { property bool isCompatibilityNode: node && node.compatibilityIssue !== undefined property string nodeStartDateTime: "" readonly property bool isPlugin: node ? node.isPlugin : false - readonly property bool isNotBuilt: node ? (!node.isBuilt) : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false signal attributeDoubleClicked(var mouse, var attribute) signal upgradeRequest() From 74fcfb96f6389495b391bac0607dea26def6c1ac Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Fri, 29 Nov 2024 15:19:00 +0100 Subject: [PATCH 09/10] minor modifs --- meshroom/core/desc.py | 3 +-- meshroom/core/node.py | 2 +- meshroom/core/plugin.py | 2 +- meshroom/ui/graph.py | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index b5ab4e188e..ea5e783163 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -745,8 +745,7 @@ def _envName(cls): """ Get the env name by hashing the env files, overwrite this to use a custom pre-build env """ - from core.plugin import EnvType - from meshroom.core.plugin import getEnvName #lazy import as to avoid circular dep + from meshroom.core.plugin import getEnvName, EnvType #lazy import as to avoid circular dep if cls.envType.value == EnvType.REZ.value: return cls.envFile with open(cls.envFile, 'r') as file: diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 2e3c9cfa82..a07fe0a8ec 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -1417,7 +1417,7 @@ def has3DOutputAttribute(self): hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) - isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin, constant=True) + isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin if self.nodeDesc is not None else False, constant=True) isEnvBuild = (not isPlugin) #init build status false its not a plugin buildStatusChanged = Signal() #event to notify change in status diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py index a2c88494e2..d5de93a3ee 100644 --- a/meshroom/core/plugin.py +++ b/meshroom/core/plugin.py @@ -87,7 +87,7 @@ def _condaEnvExist(envName): """ Checks if a specified env exists """ - cmd = condaBin+" list --name "+envName + cmd = condaBin+" list --name "+envName+" > /dev/null" return os.system(cmd) == 0 def _formatPluginName(pluginName): diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 0c3fa816fa..f4ee391afa 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -108,9 +108,9 @@ def getFileLastModTime(f): @staticmethod def updatePluginEnvStatus(n): """ Will update the status of the plugin env """ - print("Refreshing "+str(n)) - n.isEnvBuild=n.nodeDesc.isBuilt - n.buildStatusChanged.emit() + if n.nodeDesc is not None: + n.isEnvBuild=n.nodeDesc.isBuilt + n.buildStatusChanged.emit() def run(self): """ Poll watched files for last modification time. """ From cdf74fc8c58d75e35e427cb5cddcc5ce2356a439 Mon Sep 17 00:00:00 2001 From: Matthieu Hog Date: Tue, 10 Dec 2024 11:05:26 +0100 Subject: [PATCH 10/10] handling node deletion in status update --- meshroom/ui/graph.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index f4ee391afa..97dcb24124 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -109,8 +109,11 @@ def getFileLastModTime(f): def updatePluginEnvStatus(n): """ Will update the status of the plugin env """ if n.nodeDesc is not None: - n.isEnvBuild=n.nodeDesc.isBuilt - n.buildStatusChanged.emit() + try: + n.isEnvBuild=n.nodeDesc.isBuilt + n.buildStatusChanged.emit() + except Exception as E: + logging.warn("Plugin status update failed, node may be already deleted") def run(self): """ Poll watched files for last modification time. """