diff --git a/.bandit b/.bandit new file mode 100644 index 0000000000..a52326bcea --- /dev/null +++ b/.bandit @@ -0,0 +1,43 @@ +[tool.bandit] +skips = [B101, B102, B105, B106, B107, B113, B202, B401, B402, B403, B404, B405, B406, B407, B408, B409, B410, B413, B307, B311, B507, B602, B603, B605, B607, B610, B611, B703] + +[tool.bandit.any_other_function_with_shell_equals_true] +no_shell = [ + "os.execl", + "os.execle", + "os.execlp", + "os.execlpe", + "os.execv", + "os.execve", + "os.execvp", + "os.execvpe", + "os.spawnl", + "os.spawnle", + "os.spawnlp", + "os.spawnlpe", + "os.spawnv", + "os.spawnve", + "os.spawnvp", + "os.spawnvpe", + "os.startfile" +] +shell = [ + "os.system", + "os.popen", + "os.popen2", + "os.popen3", + "os.popen4", + "popen2.popen2", + "popen2.popen3", + "popen2.popen4", + "popen2.Popen3", + "popen2.Popen4", + "commands.getoutput", + "commands.getstatusoutput" +] +subprocess = [ + "subprocess.Popen", + "subprocess.call", + "subprocess.check_call", + "subprocess.check_output" +] diff --git a/README.md b/README.md index 870dbda783..578b08d240 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,14 @@ You can create custom nodes in python and make them available in Meshroom using In a standard precompiled version of Meshroom, you can also directly add custom nodes in `lib/meshroom/nodes`. To be recognized by Meshroom, a custom folder with nodes should be a Python module (an `__init__.py` file is needed). +### Plugins + +Meshroom supports installing containerised plugins via Docker (with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)) or [Anaconda](https://docs.anaconda.com/free/miniconda/index.html). + +To do so, make sure docker or anaconda is installed properly and available from the command line. +Then click on `File > Advanced > Install Plugin From URL` or `File > Advanced > Install Plugin From Local Folder` to begin the installation. + +To learn more about using or creating plugins, check the explanations [here](meshroom/plugins/README.md). ## License diff --git a/meshroom/core/__init__.py b/meshroom/core/__init__.py index 3bd30d5bbd..55a192cb77 100644 --- a/meshroom/core/__init__.py +++ b/meshroom/core/__init__.py @@ -19,7 +19,7 @@ pass from meshroom.core.submitter import BaseSubmitter -from . import desc +from meshroom.core import desc # Setup logging logging.basicConfig(format='[%(asctime)s][%(levelname)s] %(message)s', level=logging.INFO) @@ -27,12 +27,19 @@ # make a UUID based on the host ID and current time sessionUid = str(uuid.uuid1()) -cacheFolderName = 'MeshroomCache' -defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName)) nodesDesc = {} submitters = {} pipelineTemplates = {} +#meshroom paths +meshroomFolder = os.path.dirname(os.path.dirname(__file__)) +cacheFolderName = 'MeshroomCache' +defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName)) + +#plugin paths +pluginsNodesFolder = os.path.join(meshroomFolder, "plugins") +pluginsPipelinesFolder = os.path.join(meshroomFolder, "pipelines") +pluginCatalogFile = os.path.join(meshroomFolder, "plugins", "catalog.json") def hashValue(value): """ Hash 'value' using sha1. """ @@ -329,29 +336,26 @@ def loadPipelineTemplates(folder): def initNodes(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep) # filter empty strings additionalNodesPath = [i for i in additionalNodesPath if i] - nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + [pluginsNodesFolder] for f in nodesFolders: loadAllNodes(folder=f) def initSubmitters(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters') for sub in subs: registerSubmitter(sub()) def initPipelines(): - meshroomFolder = os.path.dirname(os.path.dirname(__file__)) # Load pipeline templates: check in the default folder and any folder the user might have # added to the environment variable additionalPipelinesPath = os.environ.get("MESHROOM_PIPELINE_TEMPLATES_PATH", "").split(os.pathsep) additionalPipelinesPath = [i for i in additionalPipelinesPath if i] - pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + [pluginsPipelinesFolder] for f in pipelineTemplatesFolders: if os.path.isdir(f): loadPipelineTemplates(f) diff --git a/meshroom/core/desc.py b/meshroom/core/desc.py index 1a7f4893c6..ea5e783163 100644 --- a/meshroom/core/desc.py +++ b/meshroom/core/desc.py @@ -717,9 +717,56 @@ class Node(object): documentation = '' category = 'Other' + _isPlugin = True + def __init__(self): super(Node, self).__init__() self.hasDynamicOutputAttribute = any(output.isDynamicValue for output in self.outputs) + try: + self.envFile + self.envType + except: + self._isPlugin=False + + @property + def envType(cls): + from core.plugin import EnvType #lazy import for plugin to avoid circular dependency + return EnvType.NONE + + @property + def envFile(cls): + """ + Env file used to build the environement, you may overwrite this to custom the behaviour + """ + raise NotImplementedError("You must specify an env file") + + @property + def _envName(cls): + """ + Get the env name by hashing the env files, overwrite this to use a custom pre-build env + """ + from meshroom.core.plugin import getEnvName, EnvType #lazy import as to avoid circular dep + if cls.envType.value == EnvType.REZ.value: + return cls.envFile + with open(cls.envFile, 'r') as file: + envContent = file.read() + + return getEnvName(envContent) + + @property + def isPlugin(self): + """ + Tests if the node is a valid plugin node + """ + return self._isPlugin + + @property + def isBuilt(self): + """ + Tests if the environnement is built + """ + from meshroom.core.plugin import isBuilt + return self._isPlugin and isBuilt(self) def upgradeAttributeValues(self, attrValues, fromVersion): return attrValues @@ -806,7 +853,17 @@ def buildCommandLine(self, chunk): if chunk.node.isParallelized and chunk.node.size > 1: cmdSuffix = ' ' + self.commandLineRange.format(**chunk.range.toDict()) - return cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + cmd=cmdPrefix + chunk.node.nodeDesc.commandLine.format(**chunk.node._cmdVars) + cmdSuffix + + #the process in Popen does not seem to use the right python, even if meshroom_compute is called within the env + #so in the case of command line using python, we have to make sure it is using the correct python + from meshroom.core.plugin import EnvType, getVenvPath, getVenvExe #lazy import to prevent circular dep + if self.isPlugin and self.envType == EnvType.VENV: + envPath = getVenvPath(self._envName) + envExe = getVenvExe(envPath) + cmd=cmd.replace("python", envExe) + + return cmd def stopProcess(self, chunk): # The same node could exists several times in the graph and diff --git a/meshroom/core/node.py b/meshroom/core/node.py index 2c9466b879..a07fe0a8ec 100644 --- a/meshroom/core/node.py +++ b/meshroom/core/node.py @@ -21,7 +21,6 @@ from meshroom.core.attribute import attributeFactory, ListAttribute, GroupAttribute, Attribute from meshroom.core.exception import NodeUpgradeError, UnknownNodeTypeError - def getWritingFilepath(filepath): return filepath + '.writing.' + str(uuid.uuid4()) @@ -51,6 +50,8 @@ class Status(Enum): KILLED = 5 SUCCESS = 6 INPUT = 7 # Special status for input nodes + BUILD = 8 + FIRST_RUN = 9 class ExecMode(Enum): @@ -380,16 +381,16 @@ def saveStatistics(self): renameWritingToFinalPath(statisticsFilepathWriting, statisticsFilepath) def isAlreadySubmitted(self): - return self._status.status in (Status.SUBMITTED, Status.RUNNING) + return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isAlreadySubmittedOrFinished(self): - return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS) + return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS, Status.BUILD, Status.FIRST_RUN) def isFinishedOrRunning(self): - return self._status.status in (Status.SUCCESS, Status.RUNNING) + return self._status.status in (Status.SUCCESS, Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isRunning(self): - return self._status.status == Status.RUNNING + return self._status.status in (Status.RUNNING, Status.BUILD, Status.FIRST_RUN) def isStopped(self): return self._status.status == Status.STOPPED @@ -401,36 +402,57 @@ def process(self, forceCompute=False): if not forceCompute and self._status.status == Status.SUCCESS: logging.info("Node chunk already computed: {}".format(self.name)) return - global runningProcesses - runningProcesses[self.name] = self - self._status.initStartCompute() - exceptionStatus = None - startTime = time.time() - self.upgradeStatusTo(Status.RUNNING) - self.statThread = stats.StatisticsThread(self) - self.statThread.start() - try: - self.node.nodeDesc.processChunk(self) - except Exception: - if self._status.status != Status.STOPPED: - exceptionStatus = Status.ERROR - raise - except (KeyboardInterrupt, SystemError, GeneratorExit): - exceptionStatus = Status.STOPPED - raise - finally: - self._status.initEndCompute() - self._status.elapsedTime = time.time() - startTime - if exceptionStatus is not None: - self.upgradeStatusTo(exceptionStatus) - logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) - # Ask and wait for the stats thread to stop - self.statThread.stopRequest() - self.statThread.join() - self.statistics = stats.Statistics() - del runningProcesses[self.name] - - self.upgradeStatusTo(Status.SUCCESS) + + #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk + # of the node will be ran into the env + if self.node.nodeDesc.isPlugin and self._status.status!=Status.FIRST_RUN: + try: + from meshroom.core.plugin import isBuilt, build, getCommandLine #lazy import to avoid circular dep + if not isBuilt(self.node.nodeDesc): + self.upgradeStatusTo(Status.BUILD) + build(self.node.nodeDesc) + self.upgradeStatusTo(Status.FIRST_RUN) + command = getCommandLine(self) + #NOTE: docker returns 0 even if mount fail (it fails on the deamon side) + logging.info("Running plugin node with "+command) + status = os.system(command) + if status != 0: + raise RuntimeError("Error in node execution") + self.updateStatusFromCache() + except Exception as ex: + self.logger.exception(ex) + self.upgradeStatusTo(Status.ERROR) + else: + global runningProcesses + runningProcesses[self.name] = self + self._status.initStartCompute() + exceptionStatus = None + startTime = time.time() + self.upgradeStatusTo(Status.RUNNING) + self.statThread = stats.StatisticsThread(self) + self.statThread.start() + try: + self.node.nodeDesc.processChunk(self) + except Exception: + if self._status.status != Status.STOPPED: + exceptionStatus = Status.ERROR + raise + except (KeyboardInterrupt, SystemError, GeneratorExit): + exceptionStatus = Status.STOPPED + raise + finally: + self._status.initEndCompute() + self._status.elapsedTime = time.time() - startTime + if exceptionStatus is not None: + self.upgradeStatusTo(exceptionStatus) + logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr)) + # Ask and wait for the stats thread to stop + self.statThread.stopRequest() + self.statThread.join() + self.statistics = stats.Statistics() + del runningProcesses[self.name] + + self.upgradeStatusTo(Status.SUCCESS) def stopProcess(self): if not self.isExtern(): @@ -460,6 +482,7 @@ def isExtern(self): statusNodeName = Property(str, lambda self: self._status.nodeName, constant=True) elapsedTime = Property(float, lambda self: self._status.elapsedTime, notify=statusChanged) + # Simple structure for storing node position @@ -1130,8 +1153,8 @@ def getGlobalStatus(self): return Status.INPUT chunksStatus = [chunk.status.status for chunk in self._chunks] - anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, - Status.RUNNING, Status.SUBMITTED) + anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN, + Status.SUBMITTED,) allOf = (Status.SUCCESS,) for status in anyOf: @@ -1394,6 +1417,12 @@ def has3DOutputAttribute(self): hasSequenceOutput = Property(bool, hasSequenceOutputAttribute, notify=outputAttrEnabledChanged) has3DOutput = Property(bool, has3DOutputAttribute, notify=outputAttrEnabledChanged) + isPlugin = Property(bool, lambda self: self.nodeDesc.isPlugin if self.nodeDesc is not None else False, constant=True) + + isEnvBuild = (not isPlugin) #init build status false its not a plugin + buildStatusChanged = Signal() #event to notify change in status + isBuiltStatus = Property(bool, lambda self: self.isEnvBuild, notify = buildStatusChanged) + # isBuiltStatus = Property(bool, lambda self: self.nodeDesc.isBuilt, constant=True) class Node(BaseNode): """ diff --git a/meshroom/core/plugin.py b/meshroom/core/plugin.py new file mode 100644 index 0000000000..d5de93a3ee --- /dev/null +++ b/meshroom/core/plugin.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python +# coding:utf-8 + +""" +This file defines the nodes and logic needed for the plugin system in meshroom. +A plugin is a collection of node(s) of any type with their rutime environnement setup file attached. +We use the term 'environment' to abstract a docker container or a conda/virtual environment. +""" + +from enum import Enum +import json +import os, sys +import logging +import urllib +from distutils.dir_util import copy_tree, remove_tree +import subprocess +import venv +import inspect + +from meshroom.core import desc +from meshroom.core import pluginsNodesFolder, pluginsPipelinesFolder, pluginCatalogFile, defaultCacheFolder +from meshroom.core import meshroomFolder +from meshroom.core.graph import loadGraph +from meshroom.core import hashValue + +#executables def +meshroomBinDir = os.path.abspath(os.path.join(meshroomFolder, "..", "bin")) +condaBin = "conda" +dockerBin = "docker" + +class EnvType(Enum): + """ + enum for the type of env used (by degree of encapsulation) + """ + NONE = 0 + PIP = 1 + REZ = 2 + VENV = 10 + CONDA = 20 + DOCKER = 30 + +#NOTE: could add the concept of dependencies between plugins +class PluginParams(): + """" + Class that holds parameters to install one plugin from a folder and optionally from a json structure + """ + def __init__(self, pluginUrl, jsonData=None): + #get the plugin name from folder + self.pluginName = os.path.basename(pluginUrl) + #default node and pipeline locations + self.nodesFolder = os.path.join(pluginUrl, "meshroomNodes") + self.pipelineFolder = os.path.join(pluginUrl, "meshroomPipelines") + #overwrite is json is passed + if jsonData is not None: + self.pluginName = jsonData["pluginName"] + #default node and pipeline locations + self.nodesFolder = os.path.join(pluginUrl, jsonData["nodesFolder"]) + if "pipelineFolder" in jsonData.keys(): + self.pipelineFolder = os.path.join(pluginUrl, jsonData["pipelineFolder"]) + +def getEnvName(envContent): + return "meshroom_plugin_"+hashValue(envContent) + +def _dockerImageExists(image_name, tag='latest'): + """ + Check if the desired image:tag exists + """ + try: + result = subprocess.run( [dockerBin, 'images', image_name, '--format', '{{.Repository}}:{{.Tag}}'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) + if result.returncode != 0: + return False + images = result.stdout.splitlines() + image_tag = f"{image_name}:{tag}" + return image_tag in images + except Exception as e: + return False + +def _cleanEnvVarsRez(): + """ + Used to unset all rez defined env that mess up with conda. + """ + cmd = "unset python; unset PYTHONPATH; " + return cmd + +def _condaEnvExist(envName): + """ + Checks if a specified env exists + """ + cmd = condaBin+" list --name "+envName+" > /dev/null" + return os.system(cmd) == 0 + +def _formatPluginName(pluginName): + """ + Replaces spaces for env naming + """ + return pluginName.replace(" ", "_") + +def getVenvExe(venvPath): + """ + Returns the path for the python in a virtual env + """ + if not os.path.isdir(venvPath): + raise ValueError("The specified path "+venvPath+" is not a directory") + if sys.platform == "win32": + executable = os.path.join(venvPath, 'Scripts', 'python.exe') + else: + executable = os.path.join(venvPath, 'bin', 'python') + if not os.path.isfile(executable): + raise FileNotFoundError(f"Python executable not found in the specified virtual environment: "+executable) + return executable + +def getVenvPath(envName): + return os.path.join(defaultCacheFolder, envName) + +def _venvExists(envName): + """ + Check if the following virtual env exists + """ + return os.path.isdir(getVenvPath(envName)) + +def getActiveRezPackages(): + """ + Returns a list containing the active explicit packages + """ + packages = [] + if 'REZ_REQUEST' in os.environ: + nondefpackages = [n.split("-")[0] for n in os.environ.get('REZ_REQUEST', '').split()] + resolvedPackages = os.environ.get('REZ_RESOLVE', '').split() + resolvedVersions = {} + for r in resolvedPackages: + if r.startswith('~'): + continue + v = r.split('-') + resolvedVersions[v[0]] = v[1] + packages = [p+"-"+resolvedVersions[p] for p in resolvedVersions.keys() if p in nondefpackages] + return packages + +def installPlugin(pluginUrl): + """ + Install plugin from an url or local path. + Regardless of the method, the content will be copied in the plugin folder of meshroom (which is added to the list of directory to load nodes from). + There are two options : + - having the following structure : + - [plugin folder] (will be the plugin name) + - meshroomNodes + - [code for your nodes] that contains relative path to a DockerFile|env.yaml|requirements.txt + - [...] + - meshroomPipelines + - [your meshroom templates] + - having a meshroomPlugin.json file at the root of the plugin folder + With this solution, you may have several separate node folders. + """ + logging.info("Installing plugin from "+pluginUrl) + try: + isLocal = True + #if git repo, clone the repo in cache + if urllib.parse.urlparse(pluginUrl).scheme in ('http', 'https','git'): + os.chdir(defaultCacheFolder) + os.system("git clone "+pluginUrl) + pluginName = pluginUrl.split('.git')[0].split('/')[-1] + pluginUrl = os.path.join(defaultCacheFolder, pluginName) + isLocal = False + #sanity check + if not os.path.isdir(pluginUrl): + ValueError("Invalid plugin path :"+pluginUrl) + #by default only one plugin, and with default file hierachy + pluginParamList=[PluginParams(pluginUrl)] + #location of the json file if any + paramFile=os.path.join(pluginUrl, "meshroomPlugin.json") + #load json for custom install if any + if os.path.isfile(paramFile): + jsonData=json.load(open(paramFile,"r")) + pluginParamList = [PluginParams(pluginUrl, jsonDataplugin) for jsonDataplugin in jsonData] + #for each plugin, run the 'install' + for pluginParam in pluginParamList: + intallFolder = os.path.join(pluginsNodesFolder, _formatPluginName(pluginParam.pluginName)) + logging.info("Installing "+pluginParam.pluginName+" from "+pluginUrl+" in "+intallFolder) + #check if folder valid + if not os.path.isdir(pluginParam.nodesFolder): + raise RuntimeError("Invalid node folder: "+pluginParam.nodesFolder) + #check if already installed + if os.path.isdir(intallFolder): + logging.warn("Plugin already installed, will overwrite") + if os.path.islink(intallFolder): + os.unlink(intallFolder) + else: + remove_tree(intallFolder) + #install via symlink if local, otherwise copy (usefull to develop) + if isLocal: + os.symlink(pluginParam.nodesFolder, intallFolder) + if os.path.isdir(pluginParam.pipelineFolder): + pipelineFolderLink = os.path.join(pluginsPipelinesFolder, pluginParam.pluginName) + if os.path.exists(pipelineFolderLink): + logging.warn("Plugin already installed, will overwrite") + os.unlink(pipelineFolderLink) + os.symlink(pluginParam.pipelineFolder, pipelineFolderLink) + else: + copy_tree(pluginParam.nodesFolder, intallFolder) + if os.path.isdir(pluginParam.pipelineFolder): + copy_tree(pluginParam.pipelineFolder, os.path.join(pluginsPipelinesFolder, pluginParam.pluginName)) + #remove repo if was cloned + if not isLocal: + os.removedirs(pluginUrl) + #NOTE: could try to auto load the plugins to avoid restart and test files + except Exception as ex: + logging.error(ex) + return False + + return True + +def getCatalog(): + """ + Returns the plugin catalog + """ + jsonData=json.load(open(pluginCatalogFile,"r")) + return jsonData + +def getInstalledPlugin(): + """ + Returns the list of installed plugins + """ + installedPlugins = [os.path.join(pluginsNodesFolder, f) for f in os.listdir(pluginsNodesFolder)] + return installedPlugins + +def uninstallPlugin(pluginUrl): + """ + Uninstall a plugin + """ + #NOTE: could also remove the env files + if not os.path.exists(pluginUrl): + raise RuntimeError("Plugin "+pluginUrl+" is not installed") + if os.path.islink(pluginUrl): + os.unlink(pluginUrl) + else: + os.removedirs(pluginUrl) + +def isBuilt(nodeDesc): + """ + Check if the env needs to be build for a specific nodesc. + """ + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: + return True + elif nodeDesc.envType == EnvType.PIP: + #NOTE: could find way to check for installed packages instead of rebuilding all the time + return False + elif nodeDesc.envType == EnvType.VENV: + return _venvExists(nodeDesc._envName) + elif nodeDesc.envType == EnvType.CONDA: + return _condaEnvExist(nodeDesc._envName) + elif nodeDesc.envType == EnvType.DOCKER: + return _dockerImageExists(nodeDesc._envName) + +def build(nodeDesc): + """ + Perform the needed steps to prepare the environement in which to run the node. + """ + if not hasattr(nodeDesc, 'envFile'): + raise RuntimeError("The nodedesc has no env file") + returnValue = 0 + if nodeDesc.envType in [EnvType.NONE, EnvType.REZ]: + pass + elif nodeDesc.envType == EnvType.PIP: + #install packages in the same python as meshroom + logging.info("Installing packages from "+ nodeDesc.envFile) + buildCommand = sys.executable+" -m pip install "+ nodeDesc.envFile + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + elif nodeDesc.envType == EnvType.VENV: + #create venv in default cache folder + envPath = getVenvPath(nodeDesc._envName) + logging.info("Creating virtual env "+envPath+" from "+nodeDesc.envFile) + venv.create(envPath, with_pip=True) + logging.info("Installing dependencies") + envExe = getVenvExe(envPath) + returnValue = os.system(_cleanEnvVarsRez()+envExe+" -m pip install -r "+ nodeDesc.envFile) + venvPythonLibFolder = os.path.join(os.path.dirname(envExe), '..', 'lib') + venvPythonLibFolder = [os.path.join(venvPythonLibFolder, p) + for p in os.listdir(venvPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(venvPythonLibFolder, 'site-packages', 'meshroom')) + logging.info("Done") + elif nodeDesc.envType == EnvType.CONDA: + #build a conda env from a yaml file + logging.info("Creating conda env "+nodeDesc._envName+" from "+nodeDesc.envFile) + makeEnvCommand = ( _cleanEnvVarsRez()+condaBin+" config --set channel_priority strict ; " + +condaBin+" env create -v -v --name "+nodeDesc._envName + +" --file "+nodeDesc.envFile+" ") + logging.info("Making conda env") + logging.info(makeEnvCommand) + returnValue = os.system(makeEnvCommand) + #find path to env's folder and add symlink to meshroom + condaPythonExecudable=subprocess.check_output(_cleanEnvVarsRez()+condaBin+" run -n "+nodeDesc._envName + +" python -c \"import sys; print(sys.executable)\"", + shell=True).strip().decode('UTF-8') + condaPythonLibFolder=os.path.join(os.path.dirname(condaPythonExecudable), '..', 'lib') + condaPythonLibFolder=[os.path.join(condaPythonLibFolder, p) + for p in os.listdir(condaPythonLibFolder) if p.startswith("python")][0] + os.symlink(meshroomFolder,os.path.join(condaPythonLibFolder, 'meshroom')) + logging.info("Done making conda env") + elif nodeDesc.envType == EnvType.DOCKER: + #build docker image + logging.info("Creating image "+nodeDesc._envName+" from "+ nodeDesc.envFile) + buildCommand = dockerBin+" build -f "+nodeDesc.envFile+" -t "+nodeDesc._envName+" "+os.path.dirname(nodeDesc.envFile) + logging.info("Building with "+buildCommand+" ...") + returnValue = os.system(buildCommand) + logging.info("Done") + else: + raise RuntimeError("Invalid env type") + if returnValue != 0: + raise RuntimeError("Something went wrong during build") + +def getCommandLine(chunk): + """ + Return the command line needed to enter the environment + meshroom_compute + Will make meshroom available in the environment. + """ + nodeDesc=chunk.node.nodeDesc + if chunk.node.isParallelized: + raise RuntimeError("Parallelisation not supported for plugin nodes") + if chunk.node.graph.filepath == "": + raise RuntimeError("The project needs to be saved to use plugin nodes") + saved_graph = loadGraph(chunk.node.graph.filepath) + if (str(chunk.node) not in [str(f) for f in saved_graph._nodes._objects] + or chunk.node._uid != saved_graph.findNode(str(chunk.node))._uid ): + raise RuntimeError("The changes needs to be saved to use plugin nodes") + + cmdPrefix = "" + # vars common to venv and conda, that will be passed when runing conda run or venv + meshroomCompute= meshroomBinDir+"/meshroom_compute" + meshroomComputeArgs = "--node "+chunk.node.name+" \""+chunk.node.graph.filepath+"\"" + pythonsetMeshroomPath = "export PYTHONPATH="+meshroomFolder+":$PYTHONPATH;" + + if nodeDesc.envType == EnvType.VENV: + envPath = getVenvPath(nodeDesc._envName) + envExe = getVenvExe(envPath) + #make sure meshroom in in pythonpath and that we call the right python + cmdPrefix = _cleanEnvVarsRez()+pythonsetMeshroomPath+" "+envExe + " "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.REZ: + cmdPrefix = "rez env "+" ".join(getActiveRezPackages())+" "+nodeDesc._envName+" -- "+ meshroomCompute +" " + elif nodeDesc.envType == EnvType.CONDA: + #NOTE: system env vars are not passed to conda run, we installed it 'manually' before + cmdPrefix = _cleanEnvVarsRez()+condaBin+" run --cwd "+os.path.join(meshroomFolder, "..")\ + +" --no-capture-output -n "+nodeDesc._envName+" "+" python "+meshroomCompute + elif nodeDesc.envType == EnvType.DOCKER: + #path to the selected plugin + classFile=inspect.getfile(chunk.node.nodeDesc.__class__) + pluginDir = os.path.abspath(os.path.realpath(os.path.join(os.path.dirname(classFile),".."))) + #path to the project/cache + projectDir=os.path.abspath(os.path.realpath(os.path.dirname(chunk.node.graph.filepath))) + mountCommand = (' --mount type=bind,source="'+projectDir+'",target="'+projectDir+'"' #mount with same file hierarchy + +' --mount type=bind,source="'+pluginDir+'",target=/meshroomPlugin,readonly' #mount the plugin folder (because of symbolic link, not necesseraly physically in meshroom's folder) + +' --mount type=bind,source="'+meshroomFolder+'",target=/meshroomFolder/meshroom,readonly' + +' --mount type=bind,source="'+meshroomBinDir+'",target=/meshroomBinDir,readonly') + #adds meshroom's code(& the plugin actual path that can be different (because of the dymbolic link)) to the python path + envCommand = " --env PYTHONPATH=/meshroomFolder --env MESHROOM_NODES_PATH=/meshroomPlugin" + #adds the gpu arg if needed + runtimeArg="" + if chunk.node.nodeDesc.gpu != desc.Level.NONE: + runtimeArg="--runtime=nvidia --gpus all" + #compose cl + cmdPrefix = dockerBin+" run -it --rm "+runtimeArg+" "+mountCommand+" "+envCommand+" "+nodeDesc._envName +" \"python /meshroomBinDir/meshroom_compute " + meshroomComputeArgs="--node "+chunk.node.name+" "+chunk.node.graph.filepath+"\"" + else: + raise RuntimeError("NodeType not recognised") + + command=cmdPrefix+" "+meshroomComputeArgs + + return command + +# you may use these to explicitly define Pluginnodes +class PluginNode(desc.Node): + pass + +class PluginCommandLineNode(desc.CommandLineNode): + pass \ No newline at end of file diff --git a/meshroom/plugins/README.md b/meshroom/plugins/README.md new file mode 100644 index 0000000000..f7bdddc01a --- /dev/null +++ b/meshroom/plugins/README.md @@ -0,0 +1,40 @@ +# Intro + +A plugin is a set of one or several nodes that are not part of the Meshroom/Alicevision main project. +They are meant to facilitate the creation of custom pipeline, make distribution and installation of extra nodes easy, and allow the use of different level of isolation at the node level. +Each node within a plugin may use the same or differents environnement. + +# Making Meshroom Plugins + +To make a new plugin, make your node inheriting from `meshroom.core.plugins.PluginNode` +In your new node class, overwrite the variable `envFile` to point to the environment file (e.g. the `yaml` or `dockerfile`) that sets up your installation, end `envType` to specify the type of plugin. The path to this file should be relative to the path of the node, and within the same folder (or subsequent child folder) as the node definition. + +The code in `processChunk` in your node definition will be automatically executed within the envirenoment, using `meshroom_compute`. +A new status `FIRST_RUN` denotes the stage in between the environement startup and the execution of the node. + +Make sur your imports are lazy, in `processChunk`. +Several nodes share the same environment as long as they point to the same environment file. +Changing this file will trigger a rebuild on the environment. + +You may install plugin from a git repository or from a local folder. In the later case, you may edit the code directly from your source folder. + +By default, Meshroom will look for node definition in `[plugin folder]/meshroomNodes` and new pipelines in `[plugin folder]/meshroomPipelines` and assumes only one environement is needed. + +To modify this behavior, you may put a json file named `meshroomPlugin.json` at the root of your folder/repository. +The file must have the following structure: +``` +[ + { + "pluginName":"[YOUR_PLUGIN_NAME]", + "nodesFolder":"[YOUR_FOLDER_RELATIVE_TO_THE_ROOT_REPO_OR_FOLDER], + "pipelineFolder":"[YOUR_CUSTOM_PIEPILINE_FOLDER" + }, + { + "pluginName":"Dummy Plugin", + "nodesFolder":"dummy" + } +] +``` + +The environment of the nodes are going to be build the first time it is needed (status will be `BUILD`, in purple). + diff --git a/meshroom/plugins/catalog.json b/meshroom/plugins/catalog.json new file mode 100644 index 0000000000..f4609c0c9d --- /dev/null +++ b/meshroom/plugins/catalog.json @@ -0,0 +1,9 @@ +[ + { + "pluginName":"Meshroom Research", + "pluginUrl":"https://github.com/alicevision/MeshroomResearch/", + "description":"Meshroom-Research comprises a collection of experimental plugins for Meshroom", + "isCollection":true, + "nodeTypes":["Python", "Docker", "Conda"] + } +] diff --git a/meshroom/ui/graph.py b/meshroom/ui/graph.py index 448f25484a..97dcb24124 100644 --- a/meshroom/ui/graph.py +++ b/meshroom/ui/graph.py @@ -44,6 +44,7 @@ def __init__(self, parent=None): self._stopFlag = Event() self._refreshInterval = 5 # refresh interval in seconds self._files = [] + self._nodes = [] if submitters: self._filePollerRefresh = PollerRefreshStatus.MINIMAL_ENABLED else: @@ -53,7 +54,7 @@ def __del__(self): self._threadPool.terminate() self._threadPool.join() - def start(self, files=None): + def start(self, files=None, nodes=None): """ Start polling thread. Args: @@ -66,6 +67,7 @@ def start(self, files=None): return self._stopFlag.clear() self._files = files or [] + self._nodes = nodes or [] self._thread = Thread(target=self.run) self._thread.start() @@ -78,6 +80,15 @@ def setFiles(self, files): with self._mutex: self._files = files + def setNodes(self, nodes): + """ Set the list of nodes to monitor + + Args: + nodes: the list of nodes to monitor + """ + with self._mutex: + self._nodes = nodes + def stop(self): """ Request polling thread to stop. """ if not self._thread: @@ -94,6 +105,16 @@ def getFileLastModTime(f): except OSError: return -1 + @staticmethod + def updatePluginEnvStatus(n): + """ Will update the status of the plugin env """ + if n.nodeDesc is not None: + try: + n.isEnvBuild=n.nodeDesc.isBuilt + n.buildStatusChanged.emit() + except Exception as E: + logging.warn("Plugin status update failed, node may be already deleted") + def run(self): """ Poll watched files for last modification time. """ while not self._stopFlag.wait(self._refreshInterval): @@ -103,6 +124,8 @@ def run(self): with self._mutex: if files == self._files: self.timesAvailable.emit(times) + #update plugin nodes + _ = self._threadPool.map(self.updatePluginEnvStatus, self._nodes) def onFilePollerRefreshChanged(self, value): """ Stop or start the file poller depending on the new refresh status. """ @@ -116,7 +139,6 @@ def onFilePollerRefreshChanged(self, value): filePollerRefresh = Property(int, lambda self: self._filePollerRefresh.value, constant=True) filePollerRefreshReady = Signal() # The refresh status has been updated and is ready to be used - class ChunksMonitor(QObject): """ ChunksMonitor regularly check NodeChunks' status files for modification and trigger their update on change. @@ -147,6 +169,8 @@ def setChunks(self, chunks): self.monitorableChunks = chunks files, monitoredChunks = self.watchedStatusFiles self._filesTimePoller.setFiles(files) + pluginNodes = [c.node for c in chunks if c.node.isPlugin] + self._filesTimePoller.setNodes(pluginNodes) self.monitoredChunks = monitoredChunks def stop(self): @@ -172,7 +196,8 @@ def watchedStatusFiles(self): elif self.filePollerRefresh is PollerRefreshStatus.MINIMAL_ENABLED.value: for c in self.monitorableChunks: # When a chunk's status is ERROR, it may be externally re-submitted and it should thus still be monitored - if c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: + #Plugin nodes are always moniotored + if c.node.isPlugin or c._status.status is Status.SUBMITTED or c._status.status is Status.RUNNING or c._status.status is Status.ERROR: files.append(c.statusFile) chunks.append(c) return files, chunks diff --git a/meshroom/ui/qml/Application.qml b/meshroom/ui/qml/Application.qml index ab06b9b347..00dc59b6a3 100644 --- a/meshroom/ui/qml/Application.qml +++ b/meshroom/ui/qml/Application.qml @@ -131,6 +131,73 @@ Page { } } + //File browser for plugin + Dialog { + id: pluginURLDialog + title: "Plugin URL" + height: 150 + width: 300 + standardButtons: StandardButton.Ok | StandardButton.Cancel + //focus: true + Column { + anchors.fill: parent + Text { + text: "Plugin URL" + height: 40 + } + TextField { + id: urlInput + width: parent.width * 0.75 + focus: true + } + } + onButtonClicked: { + if (clickedButton==StandardButton.Ok) { + console.log("Accepted " + clickedButton) + if (_reconstruction.installPlugin(urlInput.text)) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + } + + // dialogs for plugins + MessageDialog { + id: pluginInstalledDialog + title: "Plugin installed" + modal: true + canCopy: false + Label { + text: "Plugin installed, please restart meshroom for the changes to take effect" + } + } + + MessageDialog { + id: pluginNotInstalledDialog + title: "Plugin not installed" + modal: true + canCopy: false + Label { + text: "Something went wrong, plugin not installed" + } + } + + // plugin installation from path or url + Platform.FolderDialog { + id: intallPluginDialog + options: Platform.FolderDialog.DontUseNativeDialog + title: "Install Plugin" + onAccepted: { + if (_reconstruction.installPlugin(currentFolder.toString())) { + pluginInstalledDialog.open() + } else { + pluginNotInstalledDialog.open() + } + } + } + Item { id: computeManager @@ -525,6 +592,23 @@ Page { } } + Action { + id: installPluginFromFolderAction + text: "Install Plugin From Local Folder" + onTriggered: { + initFileDialogFolder(intallPluginDialog) + intallPluginDialog.open() + } + } + + Action { + id: installPluginFromURLAction + text: "Install Plugin From URL" + onTriggered: { + pluginURLDialog.open() + } + } + header: RowLayout { spacing: 0 MaterialToolButton { @@ -741,6 +825,18 @@ Page { ToolTip.visible: hovered ToolTip.text: removeImagesFromAllGroupsAction.tooltip } + + MenuItem { + action: installPluginFromFolderAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a folder" + } + + MenuItem { + action: installPluginFromURLAction + ToolTip.visible: hovered + ToolTip.text: "Install plugin from a local or online url" + } } MenuSeparator { } Action { @@ -1214,6 +1310,17 @@ Page { var n = _reconstruction.upgradeNode(node) _reconstruction.selectedNode = n } + + onDoBuild: { + try { + _reconstruction.buildNode(node.name) + node.isNotBuilt=false + } catch (error) { + //NOTE: could do an error popup + console.log("Build error:") + console.log(error) + } + } } } } diff --git a/meshroom/ui/qml/GraphEditor/Node.qml b/meshroom/ui/qml/GraphEditor/Node.qml index 9a8bb05a17..34707b9ed7 100755 --- a/meshroom/ui/qml/GraphEditor/Node.qml +++ b/meshroom/ui/qml/GraphEditor/Node.qml @@ -19,6 +19,9 @@ Item { property bool readOnly: node.locked /// Whether the node is in compatibility mode readonly property bool isCompatibilityNode: node ? node.hasOwnProperty("compatibilityIssue") : false + /// Whether the node is a plugin that needs to be build + readonly property bool isPlugin: node ? node.isPlugin : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false /// Mouse related states property bool mainSelected: false property bool selected: false @@ -28,7 +31,7 @@ Item { property point position: Qt.point(x, y) /// Styling property color shadowColor: "#cc000000" - readonly property color defaultColor: isCompatibilityNode ? "#444" : !node.isComputable ? "#BA3D69" : activePalette.base + readonly property color defaultColor: isCompatibilityNode ? "#444" : (!node.isComputable ? "#BA3D69" : activePalette.base) property color baseColor: defaultColor property point mousePosition: Qt.point(mouseArea.mouseX, mouseArea.mouseY) @@ -232,6 +235,15 @@ Item { issueDetails: root.node.issueDetails } } + + // ToBuild icon for PluginNodes + Loader { + active: root.isPlugin && root.isNotBuilt + sourceComponent: ToBuildBadge { + sourceComponent: iconDelegate + } + } + // Data sharing indicator // Note: for an unknown reason, there are some performance issues with the UI refresh. diff --git a/meshroom/ui/qml/GraphEditor/NodeEditor.qml b/meshroom/ui/qml/GraphEditor/NodeEditor.qml index f5998f87ef..a16f92e9fa 100644 --- a/meshroom/ui/qml/GraphEditor/NodeEditor.qml +++ b/meshroom/ui/qml/GraphEditor/NodeEditor.qml @@ -18,9 +18,12 @@ Panel { property bool readOnly: false property bool isCompatibilityNode: node && node.compatibilityIssue !== undefined property string nodeStartDateTime: "" + readonly property bool isPlugin: node ? node.isPlugin : false + property bool isNotBuilt: node ? (!node.isBuiltStatus) : false signal attributeDoubleClicked(var mouse, var attribute) signal upgradeRequest() + signal doBuild() title: "Node" + (node !== null ? " - " + node.label + "" + (node.label !== node.defaultLabel ? " (" + node.defaultLabel + ")" : "") : "") icon: MaterialLabel { text: MaterialIcons.tune } @@ -225,6 +228,17 @@ Panel { } } + Loader { + active: root.isPlugin && root.isNotBuilt + Layout.fillWidth: true + visible: active // for layout update + + sourceComponent: ToBuildBadge { + onDoBuild: root.doBuild() + sourceComponent: bannerDelegate + } + } + Loader { Layout.fillHeight: true Layout.fillWidth: true diff --git a/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml new file mode 100644 index 0000000000..f7333700f0 --- /dev/null +++ b/meshroom/ui/qml/GraphEditor/ToBuildBadge.qml @@ -0,0 +1,66 @@ +import QtQuick 2.15 +import QtQuick.Controls 2.15 +import QtQuick.Layouts 1.11 +import MaterialIcons 2.2 + +Loader { + id: root + + sourceComponent: iconDelegate + + signal doBuild() + + property Component iconDelegate: Component { + + Label { + text: MaterialIcons.warning + font.family: MaterialIcons.fontFamily + font.pointSize: 12 + color: "#66207F" + + MouseArea { + anchors.fill: parent + hoverEnabled: true + onPressed: mouse.accepted = false + ToolTip.text: "Node env needs to be built" + ToolTip.visible: containsMouse + } + } + } + + property Component bannerDelegate: Component { + + Pane { + padding: 6 + clip: true + background: Rectangle { color: "#66207F" } + + RowLayout { + width: parent.width + Column { + Layout.fillWidth: true + Label { + width: parent.width + elide: Label.ElideMiddle + font.bold: true + text: "Env needs to be built" + color: "white" + } + Label { + width: parent.width + elide: Label.ElideMiddle + color: "white" + } + } + Button { + visible: (parent.width > width) ? 1 : 0 + palette.window: root.color + palette.button: Qt.darker(root.color, 1.2) + palette.buttonText: "white" + text: "Build" + onClicked: doBuild() + } + } + } + } +} diff --git a/meshroom/ui/qml/GraphEditor/common.js b/meshroom/ui/qml/GraphEditor/common.js index 35c754ce29..3633b049b6 100644 --- a/meshroom/ui/qml/GraphEditor/common.js +++ b/meshroom/ui/qml/GraphEditor/common.js @@ -5,7 +5,9 @@ var statusColors = { "RUNNING": "#FF9800", "ERROR": "#F44336", "SUCCESS": "#4CAF50", - "STOPPED": "#E91E63" + "STOPPED": "#E91E63", + "BUILD": "#66207f", + "FIRST_RUN": "#A52A2A" } var statusColorsExternOverrides = { diff --git a/meshroom/ui/qml/GraphEditor/qmldir b/meshroom/ui/qml/GraphEditor/qmldir index 4a4d4ca460..dccf2d5ba9 100644 --- a/meshroom/ui/qml/GraphEditor/qmldir +++ b/meshroom/ui/qml/GraphEditor/qmldir @@ -9,7 +9,8 @@ AttributePin 1.0 AttributePin.qml AttributeEditor 1.0 AttributeEditor.qml AttributeItemDelegate 1.0 AttributeItemDelegate.qml CompatibilityBadge 1.0 CompatibilityBadge.qml +ToBuildBadge 1.0 ToBuildBadge.qml CompatibilityManager 1.0 CompatibilityManager.qml singleton GraphEditorSettings 1.0 GraphEditorSettings.qml TaskManager 1.0 TaskManager.qml -ScriptEditor 1.0 ScriptEditor.qml \ No newline at end of file +ScriptEditor 1.0 ScriptEditor.qml diff --git a/meshroom/ui/qml/Utils/Colors.qml b/meshroom/ui/qml/Utils/Colors.qml index 51af70e25c..6215bcf3ad 100644 --- a/meshroom/ui/qml/Utils/Colors.qml +++ b/meshroom/ui/qml/Utils/Colors.qml @@ -19,6 +19,8 @@ QtObject { readonly property color lime: "#CDDC39" readonly property color grey: "#555555" readonly property color lightgrey: "#999999" + readonly property color deeppurple: "#66207F" + readonly property color brown: "#A52A2A" readonly property var statusColors: { "NONE": "transparent", @@ -26,7 +28,9 @@ QtObject { "RUNNING": orange, "ERROR": red, "SUCCESS": green, - "STOPPED": pink + "STOPPED": pink, + "BUILD": deeppurple, + "FIRST_RUN": brown } readonly property var ghostColors: { diff --git a/meshroom/ui/reconstruction.py b/meshroom/ui/reconstruction.py index e1aaaa5f77..8918d22ac6 100755 --- a/meshroom/ui/reconstruction.py +++ b/meshroom/ui/reconstruction.py @@ -18,6 +18,7 @@ from meshroom.ui.graph import UIGraph from meshroom.ui.utils import makeProperty from meshroom.ui.components.filepath import FilepathHelper +from meshroom.core.plugin import installPlugin class Message(QObject): @@ -413,6 +414,16 @@ def __init__(self, nodeType, parent=None): nodeChanged = Signal() node = makeProperty(QObject, "_node", nodeChanged, resetOnDestroy=True) +def prepareUrlLocalFile(url): + if isinstance(url, (QUrl)): + # depending how the QUrl has been initialized, + # toLocalFile() may return the local path or an empty string + localFile = url.toLocalFile() + if not localFile: + localFile = url.toString() + else: + localFile = url + return localFile class Reconstruction(UIGraph): """ @@ -567,16 +578,22 @@ def load(self, filepath, setupProjectFile=True, publishOutputs=False): @Slot(QUrl, result=bool) @Slot(QUrl, bool, bool, result=bool) def loadUrl(self, url, setupProjectFile=True, publishOutputs=False): - if isinstance(url, (QUrl)): - # depending how the QUrl has been initialized, - # toLocalFile() may return the local path or an empty string - localFile = url.toLocalFile() - if not localFile: - localFile = url.toString() - else: - localFile = url + localFile = prepareUrlLocalFile(url) return self.load(localFile, setupProjectFile, publishOutputs) + @Slot(QUrl, result=bool) + def installPlugin(self, url): + localFile = prepareUrlLocalFile(url) + return installPlugin(localFile) + + @Slot(str, result=bool) + def buildNode(self, nodeName): + print("***Building "+nodeName) + node = self._graph.node(nodeName) + from meshroom.core.plugin import isBuilt, build #lazy import to avoid circular dep + if not isBuilt(node.nodeDesc): + build(node.nodeDesc) + def onGraphChanged(self): """ React to the change of the internal graph. """ self._liveSfmManager.reset() @@ -904,12 +921,7 @@ def importImagesFromFolder(self, path, recursive=False): def importImagesUrls(self, imagePaths, recursive=False): paths = [] for imagePath in imagePaths: - if isinstance(imagePath, (QUrl)): - p = imagePath.toLocalFile() - if not p: - p = imagePath.toString() - else: - p = imagePath + p = prepareUrlLocalFile(imagePath) paths.append(p) self.importImagesFromFolder(paths) @@ -1285,4 +1297,4 @@ def setCurrentViewPath(self, path): # Signals to propagate high-level messages error = Signal(Message) warning = Signal(Message) - info = Signal(Message) \ No newline at end of file + info = Signal(Message) diff --git a/tests/nodes/plugins/Dockerfile b/tests/nodes/plugins/Dockerfile new file mode 100644 index 0000000000..463ebb3896 --- /dev/null +++ b/tests/nodes/plugins/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3 +RUN python -m pip install --no-cache-dir numpy +RUN python -m pip install --no-cache-dir psutil +#overwrides entry point otherwise will directly execute python +ENTRYPOINT [ "/bin/bash", "-l", "-c" ] \ No newline at end of file diff --git a/tests/nodes/plugins/__init__.py b/tests/nodes/plugins/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/nodes/plugins/dummyNodes.py b/tests/nodes/plugins/dummyNodes.py new file mode 100644 index 0000000000..e187b7fdb1 --- /dev/null +++ b/tests/nodes/plugins/dummyNodes.py @@ -0,0 +1,171 @@ + + +import os +from meshroom.core.plugin import PluginNode, PluginCommandLineNode, EnvType + +#Python nodes + +class DummyConda(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.CONDA + envFile = os.path.join(os.path.dirname(__file__), "env.yaml") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyDocker(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.DOCKER + envFile = os.path.join(os.path.dirname(__file__), "Dockerfile") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + + +class DummyVenv(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.VENV + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyPip(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.PIP + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyNone(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.NONE + envFile = None + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +class DummyRez(PluginNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.REZ + envFile = "numpy" + + inputs = [] + outputs = [] + + def processChunk(self, chunk): + import numpy as np + chunk.logManager.start("info") + chunk.logger.info(np.abs(-1)) + +#Command line node + +class DummyCondaCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.CONDA + envFile = os.path.join(os.path.dirname(__file__), "env.yaml") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyDockerCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.DOCKER + envFile = os.path.join(os.path.dirname(__file__), "Dockerfile") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + + +class DummyVenvCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.VENV + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyPipCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.PIP + envFile = os.path.join(os.path.dirname(__file__), "requirements.txt") + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" + +class DummyNoneCL(PluginCommandLineNode): + + category = 'Dummy' + documentation = ''' ''' + + envType = EnvType.NONE + envFile = None + + inputs = [] + outputs = [] + + commandLine = "python -c \"import numpy as np; print(np.abs(-1))\"" \ No newline at end of file diff --git a/tests/nodes/plugins/env.yaml b/tests/nodes/plugins/env.yaml new file mode 100644 index 0000000000..f09be04127 --- /dev/null +++ b/tests/nodes/plugins/env.yaml @@ -0,0 +1,8 @@ +name: dummy +channels: + - defaults + - conda-forge +dependencies: + - python + - numpy + - psutil \ No newline at end of file diff --git a/tests/nodes/plugins/meshroomPlugin.json b/tests/nodes/plugins/meshroomPlugin.json new file mode 100644 index 0000000000..b1d03bd6d4 --- /dev/null +++ b/tests/nodes/plugins/meshroomPlugin.json @@ -0,0 +1,6 @@ +[ + { + "pluginName":"Dummy", + "nodesFolder":"." + } +] \ No newline at end of file diff --git a/tests/nodes/plugins/requirements.txt b/tests/nodes/plugins/requirements.txt new file mode 100644 index 0000000000..386e3f591a --- /dev/null +++ b/tests/nodes/plugins/requirements.txt @@ -0,0 +1,2 @@ +numpy +psutil \ No newline at end of file diff --git a/tests/test_plugin_nodes.py b/tests/test_plugin_nodes.py new file mode 100644 index 0000000000..cc44461a0c --- /dev/null +++ b/tests/test_plugin_nodes.py @@ -0,0 +1,18 @@ +import logging +import os + +from meshroom.core.graph import Graph + +logging = logging.getLogger(__name__) + +def test_pluginNodes(): + #Dont run the tests in the CI as we are unable to install plugins beforehand + if "CI" in os.environ: + return + graph = Graph('') + graph.addNewNode('DummyCondaNode') + graph.addNewNode('DummyDockerNode') + graph.addNewNode('DummyPipNode') + graph.addNewNode('DummyVenvNode') + +