Add unit test for interpretermanager instantiation and module registry (#116)

Summary:
The InterpreterManager instantiation is throwing a segfault for cpython 3.10; debugging points to individual interpreter instantiation : https://github.com/pytorch/multipy/blob/a0b1f945bf79ab335bbc141f72b2e92910fe5577/multipy/runtime/interpreter/interpreter_impl.cpp#L415, which in turn has a fixed number of frozen python modules.

Creating a basic unit test to ensure InterpreterManager can be built, and the explicit module registration (https://github.com/pytorch/multipy/blob/a0b1f945bf79ab335bbc141f72b2e92910fe5577/multipy/runtime/deploy.cpp#L125) works.

Pull Request resolved: #116

Reviewed By: PaliC

Differential Revision: D38429675

Pulled By: anirbanr-fb-r2p

fbshipit-source-id: f3d1123f0b92d8411db00089a4e04659b8bda1d5

Author: anirbanr-fb-r2p

multipy/runtime/deploy.h

@@ -158,6 +158,11 @@ struct TORCH_API InterpreterManager {
     registeredModuleSource_[std::move(name)] = std::move(src);
   }
 
+  // Util function for debugging.
+  size_t countRegisteredModuleSources() {
+    return registeredModuleSource_.size();
+  }
+
   InterpreterManager(const InterpreterManager&) = delete;
   InterpreterManager& operator=(const InterpreterManager&) = delete;
   InterpreterManager& operator=(InterpreterManager&&) = delete;

multipy/runtime/interpreter/CMakeLists.txt

@@ -49,19 +49,27 @@ include(GoogleTest)
 # We find the built python modules, this is confusing because python build already outputs
 # the modules in a strange nested path, and then that path is relative to the
 # Cmake ExternalProject root in the cmake build dir.
-## From test devsrvr
 ExternalProject_Get_property(cpython SOURCE_DIR)
 SET(PYTHON_MODULE_DIR "${SOURCE_DIR}/build/temp.linux-x86_64-${CPYTHON_VERSION}/${SOURCE_DIR}/Modules")
 SET(PYTHON_STDLIB_DIR "${SOURCE_DIR}/Lib")
 SET(PYTHON_STDLIB "${PYTHON_INSTALL_DIR}/lib/libpython_stdlib${CPYTHON_VERSION}.a")
+
+if(${CPYTHON_VERSION} MATCHES "3\.(7|8)")
+  SET(PYTHON_MEM_MODULE "${PYTHON_MODULE_DIR}/_decimal/libmpdec/memory.o")
+  SET(LEGACY_PARSER_MODULE "${PYTHON_MODULE_DIR}/parsermodule.o")
+elseif(${CPYTHON_VERSION} MATCHES "3\.(9|1[0-9]*)")
+  SET(PYTHON_MEM_MODULE "${PYTHON_MODULE_DIR}/_decimal/libmpdec/mpalloc.o")
+  SET(LEGACY_PARSER_MODULE "")
+endif()
+
 # Then we use a hardcoded list of expected module names and include them in our lib
 include("CMakePythonModules.txt")
 ExternalProject_Add_Step(
   cpython
   archive_stdlib
   DEPENDEES install
   BYPRODUCTS ${PYTHON_STDLIB}
-  COMMAND ar -rc ${PYTHON_STDLIB} ${PYTHON_MODULES}
+  COMMAND ar -rc ${PYTHON_STDLIB} ${PYTHON_MODULES} ${PYTHON_MEM_MODULE} ${LEGACY_PARSER_MODULE}
   VERBATIM
 )
 

multipy/runtime/interpreter/CMakePythonModules.txt

@@ -39,7 +39,6 @@ SET(PYTHON_MODULES
   ${PYTHON_MODULE_DIR}/nismodule.o
   ${PYTHON_MODULE_DIR}/_opcode.o
   ${PYTHON_MODULE_DIR}/ossaudiodev.o
-  ${PYTHON_MODULE_DIR}/parsermodule.o
   ${PYTHON_MODULE_DIR}/_pickle.o
   ${PYTHON_MODULE_DIR}/_posixsubprocess.o
   ${PYTHON_MODULE_DIR}/pyexpat.o ${PYTHON_MODULE_DIR}/expat/xmlparse.o ${PYTHON_MODULE_DIR}/expat/xmlrole.o ${PYTHON_MODULE_DIR}/expat/xmltok.o

multipy/runtime/test_deploy.cpp

@@ -48,6 +48,11 @@ const char* path(const char* envname, const char* path) {
   return e ? e : path;
 }
 
+TEST(TorchpyTest, InitManagerBasic) {
+  torch::deploy::InterpreterManager m(1);
+  ASSERT_EQ(m.countRegisteredModuleSources(), 1);
+}
+
 TEST(TorchpyTest, LoadLibrary) {
   torch::deploy::InterpreterManager m(1);
   torch::deploy::Package p = m.loadPackage(