created plugin and environment node systems

Author: Matthieu Hog

.bandit

@@ -0,0 +1,43 @@
+[tool.bandit]
+skips = [B101, B102, B105, B106, B107, B113, B202, B401, B402, B403, B404, B405, B406, B407, B408, B409, B410, B413, B307, B311, B507, B602, B603, B605, B607, B610, B611, B703]
+
+[tool.bandit.any_other_function_with_shell_equals_true]
+no_shell = [
+  "os.execl",
+  "os.execle",
+  "os.execlp",
+  "os.execlpe",
+  "os.execv",
+  "os.execve",
+  "os.execvp",
+  "os.execvpe",
+  "os.spawnl",
+  "os.spawnle",
+  "os.spawnlp",
+  "os.spawnlpe",
+  "os.spawnv",
+  "os.spawnve",
+  "os.spawnvp",
+  "os.spawnvpe",
+  "os.startfile"
+]
+shell = [
+  "os.system",
+  "os.popen",
+  "os.popen2",
+  "os.popen3",
+  "os.popen4",
+  "popen2.popen2",
+  "popen2.popen3",
+  "popen2.popen4",
+  "popen2.Popen3",
+  "popen2.Popen4",
+  "commands.getoutput",
+  "commands.getstatusoutput"
+]
+subprocess = [
+  "subprocess.Popen",
+  "subprocess.call",
+  "subprocess.check_call",
+  "subprocess.check_output"
+]

README.md

@@ -66,6 +66,14 @@ You can create custom nodes in python and make them available in Meshroom using
 In a standard precompiled version of Meshroom, you can also directly add custom nodes in `lib/meshroom/nodes`.
 To be recognized by Meshroom, a custom folder with nodes should be a Python module (an `__init__.py` file is needed).
 
+### Plugins
+
+Meshroom supports installing containerised plugins via Docker (with the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)) or [Anaconda](https://docs.anaconda.com/free/miniconda/index.html).
+
+To do so, make sure docker or anaconda is installed properly and available from the command line.
+Then click on `File > Advanced > Install Plugin From URL` or `File > Advanced > Install Plugin From Local Folder` to begin the installation.
+
+To learn more about using or creating plugins, check the explanations [here](meshroom/plugins/README.md).
 
 ## License
 

meshroom/core/__init__.py

@@ -27,12 +27,19 @@
 # make a UUID based on the host ID and current time
 sessionUid = str(uuid.uuid1())
 
-cacheFolderName = 'MeshroomCache'
-defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName))
 nodesDesc = {}
 submitters = {}
 pipelineTemplates = {}
 
+#meshroom paths
+meshroomFolder = os.path.dirname(os.path.dirname(__file__))
+cacheFolderName = 'MeshroomCache'
+defaultCacheFolder = os.environ.get('MESHROOM_CACHE', os.path.join(tempfile.gettempdir(), cacheFolderName))
+
+#plugin paths
+pluginsNodesFolder = os.path.join(meshroomFolder, "plugins")
+pluginsPipelinesFolder = os.path.join(meshroomFolder, "pipelines")
+pluginCatalogFile = os.path.join(meshroomFolder, "plugins", "catalog.json")
 
 def hashValue(value):
     """ Hash 'value' using sha1. """
@@ -329,28 +336,25 @@ def loadPipelineTemplates(folder):
 
 
 def initNodes():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     additionalNodesPath = os.environ.get("MESHROOM_NODES_PATH", "").split(os.pathsep)
     # filter empty strings
     additionalNodesPath = [i for i in additionalNodesPath if i]
-    nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath
+    nodesFolders = [os.path.join(meshroomFolder, 'nodes')] + additionalNodesPath + [pluginsNodesFolder]
     for f in nodesFolders:
         loadAllNodes(folder=f)
 
 
 def initSubmitters():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     subs = loadSubmitters(os.environ.get("MESHROOM_SUBMITTERS_PATH", meshroomFolder), 'submitters')
     for sub in subs:
         registerSubmitter(sub())
 
 
 def initPipelines():
-    meshroomFolder = os.path.dirname(os.path.dirname(__file__))
     # Load pipeline templates: check in the default folder and any folder the user might have
     # added to the environment variable
     additionalPipelinesPath = os.environ.get("MESHROOM_PIPELINE_TEMPLATES_PATH", "").split(os.pathsep)
     additionalPipelinesPath = [i for i in additionalPipelinesPath if i]
-    pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath
+    pipelineTemplatesFolders = [os.path.join(meshroomFolder, 'pipelines')] + additionalPipelinesPath + [pluginsPipelinesFolder]
     for f in pipelineTemplatesFolders:
         loadPipelineTemplates(f)

meshroom/core/node.py

@@ -51,6 +51,8 @@ class Status(Enum):
     KILLED = 5
     SUCCESS = 6
     INPUT = 7  # Special status for input nodes
+    BUILD = 8
+    FIRST_RUN = 9
 
 
 class ExecMode(Enum):
@@ -380,16 +382,16 @@ def saveStatistics(self):
         renameWritingToFinalPath(statisticsFilepathWriting, statisticsFilepath)
 
     def isAlreadySubmitted(self):
-        return self._status.status in (Status.SUBMITTED, Status.RUNNING)
+        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isAlreadySubmittedOrFinished(self):
-        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS)
+        return self._status.status in (Status.SUBMITTED, Status.RUNNING, Status.SUCCESS, Status.BUILD, Status.FIRST_RUN)
 
     def isFinishedOrRunning(self):
-        return self._status.status in (Status.SUCCESS, Status.RUNNING)
+        return self._status.status in (Status.SUCCESS, Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isRunning(self):
-        return self._status.status == Status.RUNNING
+        return self._status.status in (Status.RUNNING, Status.BUILD, Status.FIRST_RUN)
 
     def isStopped(self):
         return self._status.status == Status.STOPPED
@@ -401,36 +403,56 @@ def process(self, forceCompute=False):
         if not forceCompute and self._status.status == Status.SUCCESS:
             logging.info("Node chunk already computed: {}".format(self.name))
             return
-        global runningProcesses
-        runningProcesses[self.name] = self
-        self._status.initStartCompute()
-        exceptionStatus = None
-        startTime = time.time()
-        self.upgradeStatusTo(Status.RUNNING)
-        self.statThread = stats.StatisticsThread(self)
-        self.statThread.start()
-        try:
-            self.node.nodeDesc.processChunk(self)
-        except Exception:
-            if self._status.status != Status.STOPPED:
-                exceptionStatus = Status.ERROR
-            raise
-        except (KeyboardInterrupt, SystemError, GeneratorExit):
-            exceptionStatus = Status.STOPPED
-            raise
-        finally:
-            self._status.initEndCompute()
-            self._status.elapsedTime = time.time() - startTime
-            if exceptionStatus is not None:
-                self.upgradeStatusTo(exceptionStatus)
-            logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
-            # Ask and wait for the stats thread to stop
-            self.statThread.stopRequest()
-            self.statThread.join()
-            self.statistics = stats.Statistics()
-            del runningProcesses[self.name]
-
-        self.upgradeStatusTo(Status.SUCCESS)
+
+	    #if plugin node and if first call call meshroom_compute inside the env on 'host' so that the processchunk 
+        # of the node will be ran into the env
+        if hasattr(self.node.nodeDesc, 'envFile') and self._status.status!=Status.FIRST_RUN:
+            try:
+                if not self.node.nodeDesc.isBuild():
+                    self.upgradeStatusTo(Status.BUILD)
+                    self.node.nodeDesc.build()
+                self.upgradeStatusTo(Status.FIRST_RUN)
+                command = self.node.nodeDesc.getCommandLine(self)
+                #NOTE: docker returns 0 even if mount fail (it fails on the deamon side)
+                logging.info("Running plugin node with "+command)
+                status = os.system(command) 
+                if status != 0:
+                    raise RuntimeError("Error in node execution")
+                self.updateStatusFromCache()
+            except Exception as ex:
+                self.logger.exception(ex)
+                self.upgradeStatusTo(Status.ERROR)
+        else:
+            global runningProcesses
+            runningProcesses[self.name] = self
+            self._status.initStartCompute()
+            exceptionStatus = None
+            startTime = time.time()
+            self.upgradeStatusTo(Status.RUNNING)
+            self.statThread = stats.StatisticsThread(self)
+            self.statThread.start()
+            try:
+                self.node.nodeDesc.processChunk(self)
+            except Exception:
+                if self._status.status != Status.STOPPED:
+                    exceptionStatus = Status.ERROR
+                raise
+            except (KeyboardInterrupt, SystemError, GeneratorExit):
+                exceptionStatus = Status.STOPPED
+                raise
+            finally:
+                self._status.initEndCompute()
+                self._status.elapsedTime = time.time() - startTime
+                if exceptionStatus is not None:
+                    self.upgradeStatusTo(exceptionStatus)
+                logging.info(" - elapsed time: {}".format(self._status.elapsedTimeStr))
+                # Ask and wait for the stats thread to stop
+                self.statThread.stopRequest()
+                self.statThread.join()
+                self.statistics = stats.Statistics()
+                del runningProcesses[self.name]
+
+            self.upgradeStatusTo(Status.SUCCESS)
 
     def stopProcess(self):
         if not self.isExtern():
@@ -1127,8 +1149,8 @@ def getGlobalStatus(self):
             return Status.INPUT
         chunksStatus = [chunk.status.status for chunk in self._chunks]
 
-        anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED,
-                 Status.RUNNING, Status.SUBMITTED)
+        anyOf = (Status.ERROR, Status.STOPPED, Status.KILLED, Status.RUNNING, Status.BUILD, Status.FIRST_RUN, 
+                 Status.SUBMITTED,)
         allOf = (Status.SUCCESS,)
 
         for status in anyOf: