Merge pull request #2867 from alicevision/dev/batchPublish

[bin] `meshroom_batch`: Add support for setting multiple `Publish` nodes' output folders

Author: cbentejac

.git-blame-ignore-revs

@@ -1,3 +1,5 @@
+# [bin] `meshroom_batch`: Minor clean-up in the file
+15d9ecd888faa7216cfc5d97d473f5717a3118a3
 # [core] Linting following CI's flake8 report
 9b4bd68d5aa9e5c3af5e4bfc4fe6aae06437ca88
 # [tests] Linting following CI's flake8 report

bin/meshroom_batch

@@ -1,18 +1,17 @@
 #!/usr/bin/env python
 import argparse
-import os
-import sys
 import distutils.util
 import json
+import logging
+import os
 import re
+import sys
 
+import meshroom.core.graph
 from meshroom import setupEnvironment
 
 setupEnvironment()
 
-import meshroom.core.graph
-import logging
-
 meshroom.core.initPipelines()
 
 parser = argparse.ArgumentParser(
@@ -56,19 +55,25 @@ general_group.add_argument(
     metavar='FILE.mg / PIPELINE',
     type=str,
     default=os.environ.get('MESHROOM_DEFAULT_PIPELINE', 'photogrammetry'),
-    help='Template pipeline among those listed or a Meshroom file containing a custom pipeline to run on input images:\n'
-    + '\n'.join(['    - ' + p for p in meshroom.core.pipelineTemplates])
-    + '\nRequirements: the graph must contain one CameraInit node, and one Publish node if --output is set.',
+    help='Template pipeline among those listed or a Meshroom file containing a custom pipeline '
+         'to run on input images:\n' +
+         '\n'.join(['    - ' + p for p in meshroom.core.pipelineTemplates]) + 
+         '\nRequirements: the graph must contain at least one CameraInit node, and at least '
+         'one Publish node if --output is set.',
     )
 
 general_group.add_argument(
-    '-o', '--output', metavar='FOLDER', type=str, required=False,
+    '-o', '--output', metavar='FOLDER PUBLISH_INSTANCE=FOLDER',
+    type=str, required=False, nargs='*',
     help='Output folder where results should be copied to. '
+         'If the output folder is provided without specifiying the instance of a Publish node, '
+         'all the Publish nodes in the scene will be set with the same ouput folder value. '
          'If not set, results will have to be retrieved directly from the cache folder.')
 
 general_group.add_argument(
     '-s', '--save', metavar='FILE', type=str, required=False,
-    help='Save the configured Meshroom graph to a project file. It will setup the cache folder accordingly if not explicitly changed by --cache.')
+    help='Save the configured Meshroom graph to a project file. It will setup the cache folder '
+         'accordingly if not explicitly changed by --cache.')
 
 general_group.add_argument(
     '--submit', help='Submit on renderfarm instead of local computation.',
@@ -130,7 +135,6 @@ advanced_group.add_argument(
 
 args = parser.parse_args()
 
-
 logStringToPython = {
     'fatal': logging.FATAL,
     'error': logging.ERROR,
@@ -154,11 +158,12 @@ with meshroom.core.graph.GraphModification(graph):
     # initialize template pipeline
     loweredPipelineTemplates = {k.lower(): v for k, v in meshroom.core.pipelineTemplates.items()}
     if args.pipeline.lower() in loweredPipelineTemplates:
-        graph.initFromTemplate(loweredPipelineTemplates[args.pipeline.lower()], publishOutputs=True if args.output else False)
+        graph.initFromTemplate(loweredPipelineTemplates[args.pipeline.lower()],
+                               publishOutputs=True if args.output else False)
     else:
         # custom pipeline
         graph.initFromTemplate(args.pipeline, publishOutputs=True if args.output else False)
-    
+
     def parseInputs(inputs, uniqueInitNode):
         """Utility method for parsing the input and inputRecursive arguments."""
         mapInputs = {}
@@ -193,14 +198,14 @@ with meshroom.core.graph.GraphModification(graph):
             nodeInputs = inputGroup[-1].split(';')
             mapInputs[nodeName] = [os.path.abspath(path) for path in nodeInputs]
         return mapInputs
-    
+
     # get init nodes
     initNodes = graph.findInitNodes()
     uniqueInitNode = initNodes[0] if (len(initNodes) == 1) else None
 
     # parse inputs for each init node
     mapInput = parseInputs(args.input, uniqueInitNode)
-    
+
     # parse recursive inputs for each init node
     mapInputRecursive = parseInputs(args.inputRecursive, uniqueInitNode)
 
@@ -221,15 +226,48 @@ with meshroom.core.graph.GraphModification(graph):
         graph.setVerbose(args.verbose)
 
     if args.output:
-        # if there is more than 1 Publish node, they will all be set to the same output;
-        # depending on what they are connected to, some input files might be overwritten in the output folder
-        # (e.g. if two Publish nodes are connected to two Texturing nodes)
+        # The output folders for Publish nodes can be set as follows:
+        # - for each node, the output folder is specified following the
+        #   "Publish_name=/output/folder/path" convention.
+        # - a path is provided without specifying which Publish node should be set with it:
+        #   all the Publish nodes will be set with it.
+        # - some Publish nodes have their path specified, and another path is provided
+        #   without specifying a node: all Publish nodes with dedicated will have their own
+        #   output folders set, and those which have not been specified will be set with the
+        #   other path.
+        # - some Publish nodes have their output folder specified while others do not: all
+        #   the nodes with specified folders will use the provided values, and those without
+        #   any will be set with the output folder of the first specified Publish node.
+        # - several output folders are provided without specifying any node: the last one will
+        #   be used to set all the Publish nodes' output folders.
+
+        # Check that there is at least one Publish node
         publishNodes = graph.nodesOfType('Publish')
-        if len(publishNodes) > 0:
-            for node in publishNodes:
-                node.output.value = os.path.abspath(args.output)
-        else:
-            raise RuntimeError("meshroom_batch requires a pipeline graph with at least one Publish node, none found.")
+        if len(publishNodes) == 0:
+            raise RuntimeError('meshroom_batch requires a pipeline graph with at least ' +
+                               'one Publish node, none found.')
+
+        reExtract = re.compile(r'(\w+)=(.*)')  # NodeName=value
+        globalPublishPath = ""
+        for p in args.output:
+            result = reExtract.match(p)
+            if not result:  # If the argument is only a path, set it for the global path
+                globalPublishPath = p
+                continue
+
+            node, value = result.groups()
+            for i, n in enumerate(publishNodes):  # Find the correct Publish node in the list
+                if n.name == node:  # If found, set the value, and remove it from the list
+                    n.output.value = value
+                    publishNodes.pop(i)
+                    if globalPublishPath == "":  # Fallback in case some nodes would have no path
+                        globalPublishPath = value
+                    break
+
+        for n in publishNodes:  # Set the remaining Publish nodes with the global path
+            n.output.value = globalPublishPath
+    else:
+        print(f'No output set, results will be available in the cache folder: "{graph.cacheDir}"')
 
     if args.overrides:
         with open(args.overrides, encoding='utf-8', errors='ignore') as f:
@@ -267,9 +305,6 @@ if args.save:
     graph.save(args.save, setupProjectFile=not bool(args.cache))
     print(f'File successfully saved: "{args.save}"')
 
-if not args.output:
-    print(f'No output set, results will be available in the cache folder: "{graph.cacheDir}"')
-
 # find end nodes (None will compute all graph)
 toNodes = graph.findNodes(args.toNode) if args.toNode else None
 
@@ -278,10 +313,11 @@ if args.submit:
     if not args.save:
         raise ValueError('Need to save the project to file to submit on renderfarm.')
     # submit on renderfarm
-    meshroom.core.graph.submit(args.save, args.submitter, toNode=args.toNode, submitLabel=args.submitLabel)
+    meshroom.core.graph.submit(args.save, args.submitter, toNode=args.toNode,
+                               submitLabel=args.submitLabel)
 elif args.compute:
     # find end nodes (None will compute all graph)
     toNodes = graph.findNodes(args.toNode) if args.toNode else None
     # start computation
-    meshroom.core.graph.executeGraph(graph, toNodes=toNodes, forceCompute=args.forceCompute, forceStatus=args.forceStatus)
-
+    meshroom.core.graph.executeGraph(graph, toNodes=toNodes, forceCompute=args.forceCompute,
+                                     forceStatus=args.forceStatus)

meshroom/nodes/general/Publish.py

@@ -75,7 +75,7 @@ def processChunk(self, chunk):
                 raise RuntimeError(error)
 
             if not os.path.exists(chunk.node.output.value):
-                os.mkdir(chunk.node.output.value)
+                os.makedirs(chunk.node.output.value)
 
             for iFile, oFile in outFiles.items():
                 if os.path.isdir(iFile):  # If the input is a directory, copy the directory's content