Merge pull request #4737 from zenoss/backport/ZEN-35390.7x

ZEN-35390: Adding and downloading MIBs takes around 20 minutes

Author: Deer-WarLord

src/Products/ZenModel/MibOrganizer.py

@@ -253,6 +253,7 @@ def handleUploadedFile(self, REQUEST):
             savedMIBPath,
             "--path=%s" % mypath,
             "--mibdepsdir=%s" % zenPath(_pathToMIB),
+            "--inMainProcess",
         ]
         return self.dmd.JobManager.addJob(
             SubprocessJob,

src/Products/ZenModel/tests/test_zenmib.py

@@ -82,7 +82,7 @@ def test_MIBFileProcessor(self, mockSMIDumpTool):
         processor = MIBFileProcessor(log, self.dmd, options, [source])
         processor.run()
 
-        mockTool.run.assert_called_with(expectedMIBFile)
+        mockTool.run.assert_called_with(False, expectedMIBFile)
         self._checkResults()
 
     def test_DumpFileProcessor(self):

src/Products/ZenModel/zenmib.py

@@ -114,7 +114,11 @@ def __init__(self, log, dmd, options, mibfiles):
         self._mibsdir = options.mibsdir
         self._mibfiles = mibfiles
 
-    def run(self):
+    def run(self, in_main_process=False):
+        """
+        By default, we execute command in the sub process.
+        In case of zenjob usage we explicitly pass --inMainProcess cli arg for zenmib
+        """
         mibfiles = self._getMIBFiles()
         paths = [zenPath("share", "mibs"), self._mibdepsdir]
 
@@ -130,7 +134,7 @@ def run(self):
             tool = SMIDumpTool(config=cfg)
             # returns string containing all the MIB definitions found
             # in the provided set of MIBFile objects.
-            dump = tool.run(*mibfiles)
+            dump = tool.run(in_main_process, *mibfiles)
             # for defn in dump.definitions:
             #     print defn
             if dump:
@@ -207,7 +211,7 @@ def run(self):
                     self.log, self.dmd, self.options, self.args
                 )
 
-            processor.run()
+            processor.run(in_main_process=self.options.in_main_process)
 
             if not self.options.nocommit:
                 transaction.commit()
@@ -316,6 +320,13 @@ def buildOptions(self):
             default=False,
             help="Remove zeros found in the middle of the OID",
         )
+        self.parser.add_option(
+            "--inMainProcess",
+            dest="in_main_process",
+            action="store_true",
+            default=False,
+            help="Run the nested linux command in a current process (default: False)",
+        )
 
 
 def main():

src/Products/ZenUtils/mib/smidump.py

@@ -1,3 +1,5 @@
+import uuid
+import os
 import logging
 import re
 
@@ -216,7 +218,7 @@ def __init__(self, config=None):
         if config:
             self._cmd.extend(["--config", config.filename])
 
-    def run(self, *mibfiles):
+    def run(self, in_main_process=False, *mibfiles):
         """Takes the given MIBFile objects and returns a string containing
         the output of the 'smidump -f python' command.  If multiple MIBFiles
         are given, the returned string will contain the MIB definitions from
@@ -227,14 +229,35 @@ def run(self, *mibfiles):
         if log.getEffectiveLevel() <= logging.DEBUG:
             log.debug("Executing command: %s", ' '.join(cmd))
 
-        process = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True)
-        output, error = process.communicate()  # blocks until done
-        rc = process.poll()
+        error = ""
+        if not in_main_process:
+            process = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=False)
+            output, error = process.communicate()  # blocks until done
+            rc = process.poll()
+        else:
+            unique_id = uuid.uuid4().hex
+            output_file = "/tmp/mib_command_output_%s.txt" % unique_id
+            error_file = "/tmp/mib_command_error_%s.txt" % unique_id
+            cmd.extend([">", output_file, "2>", error_file])
+
+            try:
+                with open(error_file, 'r') as ef:
+                    error = ef.read()
+            except (OSError, IOError):
+                pass # No errors while running the command
+            else:
+                os.system("rm -f " + error_file)
+
+            rc = os.system(" ".join(cmd))
+
+            with open(output_file, 'r') as f:
+                output = f.read()
+            os.system("rm -f " + output_file)
 
         if rc != 0:
             raise RuntimeError("smidump failed:\n" + error.strip())
 
-        if log.getEffectiveLevel() <= logging.DEBUG:
+        if log.getEffectiveLevel() <= logging.DEBUG and error:
             for line in (ln.strip() for ln in error.split("\n")):
                 if line:
                     log.warn("[smidump] %s", line)

src/Products/ZenUtils/mib/tests/test_smidump.py

@@ -190,10 +190,10 @@ def test_cmdline_no_config(self):
         ]
 
         tool = SMIDumpTool()
-        dump = tool.run(mibfile)
+        dump = tool.run(False, mibfile)
         self.assertIsInstance(dump, SMIDump)
         self.mock_popen.assert_called_once_with(
-            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=True
+            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=False
         )
 
     @mock.patch("Products.ZenUtils.mib.tests.test_smidump.SMIConfigFile")
@@ -217,11 +217,11 @@ def test_cmdline_with_config(self, mockConfig):
 
         with mockConfig as cfg:
             tool = SMIDumpTool(config=cfg)
-            dump = tool.run(mibfile)
+            dump = tool.run(False, mibfile)
 
         self.assertIsInstance(dump, SMIDump)
         self.mock_popen.assert_called_once_with(
-            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=True
+            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=False
         )
 
     def test_cmdline_n_files(self):
@@ -238,10 +238,10 @@ def test_cmdline_n_files(self):
         ]
 
         tool = SMIDumpTool()
-        dump = tool.run(mibfile1, mibfile2)
+        dump = tool.run(False, mibfile1, mibfile2)
         self.assertIsInstance(dump, SMIDump)
         self.mock_popen.assert_called_once_with(
-            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=True
+            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=False
         )
 
     def test_cmdline_one_file_n_modules(self):
@@ -257,10 +257,10 @@ def test_cmdline_one_file_n_modules(self):
         expected_cmd.extend(mibfile.module_names[:-1])
 
         tool = SMIDumpTool()
-        dump = tool.run(mibfile)
+        dump = tool.run(False,mibfile)
         self.assertIsInstance(dump, SMIDump)
         self.mock_popen.assert_called_once_with(
-            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=True
+            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=False
         )
 
     def test_cmdline_n_files_n_modules(self):
@@ -285,10 +285,10 @@ def test_cmdline_n_files_n_modules(self):
         ]
 
         tool = SMIDumpTool()
-        dump = tool.run(mockMibFile1, mockMibFile2)
+        dump = tool.run(False, mockMibFile1, mockMibFile2)
         self.assertIsInstance(dump, SMIDump)
         self.mock_popen.assert_called_once_with(
-            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=True
+            expected_cmd, stdout=PIPE, stderr=PIPE, close_fds=False
         )
 
     def test_cmd_error(self):
@@ -301,5 +301,5 @@ def test_cmd_error(self):
 
         tool = SMIDumpTool()
         with self.assertRaises(RuntimeError) as ex:
-            tool.run(mockMibFile)
+            tool.run(False, mockMibFile)
         self.assertMultiLineEqual(str(ex.exception), "smidump failed:\nboom")

src/Products/Zuul/facades/mibfacade.py

@@ -169,7 +169,7 @@ def getMibBaseNodeTree(self, id, relation='nodes'):
     @info
     def addMibPackage(self, package, organizer):
         args = [binPath('zenmib'), 'run', package,
-                '--path=%s' % organizer]
+                '--path=%s' % organizer, "--inMainProcess"]
         jobStatus = self._dmd.JobManager.addJob(
             SubprocessJob,
             description="Add MIB package %s" % package,