gh-128911: Add tests on the PyImport C API (#128915)

* Add Modules/_testlimitedcapi/import.c
* Add Lib/test/test_capi/test_import.py
* Remove _testcapi.check_pyimport_addmodule(): tests already covered
  by newly added tests.

Co-authored-by: Serhiy Storchaka <[email protected]>

Author: vstinner

Lib/test/test_capi/test_import.py

@@ -0,0 +1,327 @@
+import importlib.util
+import os.path
+import sys
+import types
+import unittest
+from test.support import os_helper
+from test.support import import_helper
+from test.support.warnings_helper import check_warnings
+
+_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
+NULL = None
+
+
+class ImportTests(unittest.TestCase):
+    def test_getmagicnumber(self):
+        # Test PyImport_GetMagicNumber()
+        magic = _testlimitedcapi.PyImport_GetMagicNumber()
+        self.assertEqual(magic,
+                         int.from_bytes(importlib.util.MAGIC_NUMBER, 'little'))
+
+    def test_getmagictag(self):
+        # Test PyImport_GetMagicTag()
+        tag = _testlimitedcapi.PyImport_GetMagicTag()
+        self.assertEqual(tag, sys.implementation.cache_tag)
+
+    def test_getmoduledict(self):
+        # Test PyImport_GetModuleDict()
+        modules = _testlimitedcapi.PyImport_GetModuleDict()
+        self.assertIs(modules, sys.modules)
+
+    def check_import_loaded_module(self, import_module):
+        for name in ('os', 'sys', 'test', 'unittest'):
+            with self.subTest(name=name):
+                self.assertIn(name, sys.modules)
+                old_module = sys.modules[name]
+                module = import_module(name)
+                self.assertIsInstance(module, types.ModuleType)
+                self.assertIs(module, old_module)
+
+    def check_import_fresh_module(self, import_module):
+        old_modules = dict(sys.modules)
+        try:
+            for name in ('colorsys', 'math'):
+                with self.subTest(name=name):
+                    sys.modules.pop(name, None)
+                    module = import_module(name)
+                    self.assertIsInstance(module, types.ModuleType)
+                    self.assertIs(module, sys.modules[name])
+                    self.assertEqual(module.__name__, name)
+        finally:
+            sys.modules.clear()
+            sys.modules.update(old_modules)
+
+    def test_getmodule(self):
+        # Test PyImport_GetModule()
+        getmodule = _testlimitedcapi.PyImport_GetModule
+        self.check_import_loaded_module(getmodule)
+
+        nonexistent = 'nonexistent'
+        self.assertNotIn(nonexistent, sys.modules)
+        self.assertIs(getmodule(nonexistent), KeyError)
+        self.assertIs(getmodule(''), KeyError)
+        self.assertIs(getmodule(object()), KeyError)
+
+        self.assertRaises(TypeError, getmodule, [])  # unhashable
+        # CRASHES getmodule(NULL)
+
+    def check_addmodule(self, add_module, accept_nonstr=False):
+        # create a new module
+        names = ['nonexistent']
+        if accept_nonstr:
+            names.append(b'\xff')  # non-UTF-8
+            # PyImport_AddModuleObject() accepts non-string names
+            names.append(tuple(['hashable non-string']))
+        for name in names:
+            with self.subTest(name=name):
+                self.assertNotIn(name, sys.modules)
+                try:
+                    module = add_module(name)
+                    self.assertIsInstance(module, types.ModuleType)
+                    self.assertEqual(module.__name__, name)
+                    self.assertIs(module, sys.modules[name])
+                finally:
+                    sys.modules.pop(name, None)
+
+        # get an existing module
+        self.check_import_loaded_module(add_module)
+
+    def test_addmoduleobject(self):
+        # Test PyImport_AddModuleObject()
+        addmoduleobject = _testlimitedcapi.PyImport_AddModuleObject
+        self.check_addmodule(addmoduleobject, accept_nonstr=True)
+
+        self.assertRaises(TypeError, addmoduleobject, [])  # unhashable
+        # CRASHES addmoduleobject(NULL)
+
+    def test_addmodule(self):
+        # Test PyImport_AddModule()
+        addmodule = _testlimitedcapi.PyImport_AddModule
+        self.check_addmodule(addmodule)
+
+        self.assertRaises(UnicodeDecodeError, addmodule, b'\xff')
+        # CRASHES addmodule(NULL)
+
+    def test_addmoduleref(self):
+        # Test PyImport_AddModuleRef()
+        addmoduleref = _testlimitedcapi.PyImport_AddModuleRef
+        self.check_addmodule(addmoduleref)
+
+        self.assertRaises(UnicodeDecodeError, addmoduleref, b'\xff')
+        # CRASHES addmoduleref(NULL)
+
+    def check_import_func(self, import_module):
+        self.check_import_loaded_module(import_module)
+        self.check_import_fresh_module(import_module)
+        self.assertRaises(ModuleNotFoundError, import_module, 'nonexistent')
+        self.assertRaises(ValueError, import_module, '')
+
+    def test_import(self):
+        # Test PyImport_Import()
+        import_ = _testlimitedcapi.PyImport_Import
+        self.check_import_func(import_)
+
+        self.assertRaises(TypeError, import_, b'os')
+        self.assertRaises(SystemError, import_, NULL)
+
+    def test_importmodule(self):
+        # Test PyImport_ImportModule()
+        importmodule = _testlimitedcapi.PyImport_ImportModule
+        self.check_import_func(importmodule)
+
+        self.assertRaises(UnicodeDecodeError, importmodule, b'\xff')
+        # CRASHES importmodule(NULL)
+
+    def test_importmodulenoblock(self):
+        # Test deprecated PyImport_ImportModuleNoBlock()
+        importmodulenoblock = _testlimitedcapi.PyImport_ImportModuleNoBlock
+        with check_warnings(('', DeprecationWarning)):
+            self.check_import_func(importmodulenoblock)
+            self.assertRaises(UnicodeDecodeError, importmodulenoblock, b'\xff')
+
+        # CRASHES importmodulenoblock(NULL)
+
+    def check_frozen_import(self, import_frozen_module):
+        # Importing a frozen module executes its code, so start by unloading
+        # the module to execute the code in a new (temporary) module.
+        old_zipimport = sys.modules.pop('zipimport')
+        try:
+            self.assertEqual(import_frozen_module('zipimport'), 1)
+
+            # import zipimport again
+            self.assertEqual(import_frozen_module('zipimport'), 1)
+        finally:
+            sys.modules['zipimport'] = old_zipimport
+
+        # not a frozen module
+        self.assertEqual(import_frozen_module('sys'), 0)
+        self.assertEqual(import_frozen_module('nonexistent'), 0)
+        self.assertEqual(import_frozen_module(''), 0)
+
+    def test_importfrozenmodule(self):
+        # Test PyImport_ImportFrozenModule()
+        importfrozenmodule = _testlimitedcapi.PyImport_ImportFrozenModule
+        self.check_frozen_import(importfrozenmodule)
+
+        self.assertRaises(UnicodeDecodeError, importfrozenmodule, b'\xff')
+        # CRASHES importfrozenmodule(NULL)
+
+    def test_importfrozenmoduleobject(self):
+        # Test PyImport_ImportFrozenModuleObject()
+        importfrozenmoduleobject = _testlimitedcapi.PyImport_ImportFrozenModuleObject
+        self.check_frozen_import(importfrozenmoduleobject)
+        self.assertEqual(importfrozenmoduleobject(b'zipimport'), 0)
+        self.assertEqual(importfrozenmoduleobject(NULL), 0)
+
+    def test_importmoduleex(self):
+        # Test PyImport_ImportModuleEx()
+        importmoduleex = _testlimitedcapi.PyImport_ImportModuleEx
+        self.check_import_func(lambda name: importmoduleex(name, NULL, NULL, NULL))
+
+        self.assertRaises(ModuleNotFoundError, importmoduleex, 'nonexistent', NULL, NULL, NULL)
+        self.assertRaises(ValueError, importmoduleex, '', NULL, NULL, NULL)
+        self.assertRaises(UnicodeDecodeError, importmoduleex, b'\xff', NULL, NULL, NULL)
+        # CRASHES importmoduleex(NULL, NULL, NULL, NULL)
+
+    def check_importmodulelevel(self, importmodulelevel):
+        self.check_import_func(lambda name: importmodulelevel(name, NULL, NULL, NULL, 0))
+
+        self.assertRaises(ModuleNotFoundError, importmodulelevel, 'nonexistent', NULL, NULL, NULL, 0)
+        self.assertRaises(ValueError, importmodulelevel, '', NULL, NULL, NULL, 0)
+
+        if __package__:
+            self.assertIs(importmodulelevel('test_import', globals(), NULL, NULL, 1),
+                          sys.modules['test.test_capi.test_import'])
+            self.assertIs(importmodulelevel('test_capi', globals(), NULL, NULL, 2),
+                          sys.modules['test.test_capi'])
+        self.assertRaises(ValueError, importmodulelevel, 'os', NULL, NULL, NULL, -1)
+        with self.assertWarns(ImportWarning):
+            self.assertRaises(KeyError, importmodulelevel, 'test_import', {}, NULL, NULL, 1)
+        self.assertRaises(TypeError, importmodulelevel, 'test_import', [], NULL, NULL, 1)
+
+    def test_importmodulelevel(self):
+        # Test PyImport_ImportModuleLevel()
+        importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevel
+        self.check_importmodulelevel(importmodulelevel)
+
+        self.assertRaises(UnicodeDecodeError, importmodulelevel, b'\xff', NULL, NULL, NULL, 0)
+        # CRASHES importmodulelevel(NULL, NULL, NULL, NULL, 0)
+
+    def test_importmodulelevelobject(self):
+        # Test PyImport_ImportModuleLevelObject()
+        importmodulelevel = _testlimitedcapi.PyImport_ImportModuleLevelObject
+        self.check_importmodulelevel(importmodulelevel)
+
+        self.assertRaises(TypeError, importmodulelevel, b'os', NULL, NULL, NULL, 0)
+        self.assertRaises(ValueError, importmodulelevel, NULL, NULL, NULL, NULL, 0)
+
+    def check_executecodemodule(self, execute_code, *args):
+        name = 'test_import_executecode'
+        try:
+            # Create a temporary module where the code will be executed
+            self.assertNotIn(name, sys.modules)
+            module = _testlimitedcapi.PyImport_AddModuleRef(name)
+            self.assertNotHasAttr(module, 'attr')
+
+            # Execute the code
+            code = compile('attr = 1', '<test>', 'exec')
+            module2 = execute_code(name, code, *args)
+            self.assertIs(module2, module)
+
+            # Check the function side effects
+            self.assertEqual(module.attr, 1)
+        finally:
+            sys.modules.pop(name, None)
+        return module.__spec__.origin
+
+    def test_executecodemodule(self):
+        # Test PyImport_ExecCodeModule()
+        execcodemodule = _testlimitedcapi.PyImport_ExecCodeModule
+        self.check_executecodemodule(execcodemodule)
+
+        code = compile('attr = 1', '<test>', 'exec')
+        self.assertRaises(UnicodeDecodeError, execcodemodule, b'\xff', code)
+        # CRASHES execcodemodule(NULL, code)
+        # CRASHES execcodemodule(name, NULL)
+
+    def test_executecodemoduleex(self):
+        # Test PyImport_ExecCodeModuleEx()
+        execcodemoduleex = _testlimitedcapi.PyImport_ExecCodeModuleEx
+
+        # Test NULL path (it should not crash)
+        self.check_executecodemodule(execcodemoduleex, NULL)
+
+        # Test non-NULL path
+        pathname = b'pathname'
+        origin = self.check_executecodemodule(execcodemoduleex, pathname)
+        self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname)))
+
+        pathname = os_helper.TESTFN_UNDECODABLE
+        if pathname:
+            origin = self.check_executecodemodule(execcodemoduleex, pathname)
+            self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname)))
+
+        code = compile('attr = 1', '<test>', 'exec')
+        self.assertRaises(UnicodeDecodeError, execcodemoduleex, b'\xff', code, NULL)
+        # CRASHES execcodemoduleex(NULL, code, NULL)
+        # CRASHES execcodemoduleex(name, NULL, NULL)
+
+    def check_executecode_pathnames(self, execute_code_func, object=False):
+        # Test non-NULL pathname and NULL cpathname
+
+        # Test NULL paths (it should not crash)
+        self.check_executecodemodule(execute_code_func, NULL, NULL)
+
+        pathname = 'pathname'
+        origin = self.check_executecodemodule(execute_code_func, pathname, NULL)
+        self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname)))
+        origin = self.check_executecodemodule(execute_code_func, NULL, pathname)
+        if not object:
+            self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname)))
+
+        pathname = os_helper.TESTFN_UNDECODABLE
+        if pathname:
+            if object:
+                pathname = os.fsdecode(pathname)
+            origin = self.check_executecodemodule(execute_code_func, pathname, NULL)
+            self.assertEqual(origin, os.path.abspath(os.fsdecode(pathname)))
+            self.check_executecodemodule(execute_code_func, NULL, pathname)
+
+        # Test NULL pathname and non-NULL cpathname
+        pyc_filename = importlib.util.cache_from_source(__file__)
+        py_filename = importlib.util.source_from_cache(pyc_filename)
+        origin = self.check_executecodemodule(execute_code_func, NULL, pyc_filename)
+        if not object:
+            self.assertEqual(origin, py_filename)
+
+    def test_executecodemodulewithpathnames(self):
+        # Test PyImport_ExecCodeModuleWithPathnames()
+        execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleWithPathnames
+        self.check_executecode_pathnames(execute_code_func)
+
+        code = compile('attr = 1', '<test>', 'exec')
+        self.assertRaises(UnicodeDecodeError, execute_code_func, b'\xff', code, NULL, NULL)
+        # CRASHES execute_code_func(NULL, code, NULL, NULL)
+        # CRASHES execute_code_func(name, NULL, NULL, NULL)
+
+    def test_executecodemoduleobject(self):
+        # Test PyImport_ExecCodeModuleObject()
+        execute_code_func = _testlimitedcapi.PyImport_ExecCodeModuleObject
+        self.check_executecode_pathnames(execute_code_func, object=True)
+
+        code = compile('attr = 1', '<test>', 'exec')
+        self.assertRaises(TypeError, execute_code_func, [], code, NULL, NULL)
+        nonstring = tuple(['hashable non-string'])
+        self.assertRaises(AttributeError, execute_code_func, nonstring, code, NULL, NULL)
+        sys.modules.pop(nonstring, None)
+        # CRASHES execute_code_func(NULL, code, NULL, NULL)
+        # CRASHES execute_code_func(name, NULL, NULL, NULL)
+
+    # TODO: test PyImport_GetImporter()
+    # TODO: test PyImport_ReloadModule()
+    # TODO: test PyImport_ExtendInittab()
+    # PyImport_AppendInittab() is tested by test_embed
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_import/__init__.py

@@ -3311,30 +3311,6 @@ def test_basic_multiple_interpreters_reset_each(self):
         #  * module's global state was initialized, not reset
 
 
-@cpython_only
-class CAPITests(unittest.TestCase):
-    def test_pyimport_addmodule(self):
-        # gh-105922: Test PyImport_AddModuleRef(), PyImport_AddModule()
-        # and PyImport_AddModuleObject()
-        _testcapi = import_module("_testcapi")
-        for name in (
-            'sys',     # frozen module
-            'test',    # package
-            __name__,  # package.module
-        ):
-            _testcapi.check_pyimport_addmodule(name)
-
-    def test_pyimport_addmodule_create(self):
-        # gh-105922: Test PyImport_AddModuleRef(), create a new module
-        _testcapi = import_module("_testcapi")
-        name = 'dontexist'
-        self.assertNotIn(name, sys.modules)
-        self.addCleanup(unload, name)
-
-        mod = _testcapi.check_pyimport_addmodule(name)
-        self.assertIs(mod, sys.modules[name])
-
-
 @cpython_only
 class TestMagicNumber(unittest.TestCase):
     def test_magic_number_endianness(self):

Modules/Setup.stdlib.in

@@ -163,7 +163,7 @@
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c
 @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c  _testcapi/config.c
-@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c
+@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c
 @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c
 @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c