Merge branch 'main' into webloop-browser-unsupported-methods

Author: ryanking13

cpython/Makefile

@@ -15,7 +15,7 @@ LIBFFIREPO=https://github.com/libffi/libffi
 LIBFFI_COMMIT=f08493d249d2067c8b3207ba46693dd858f95db3
 
 HIWIREBUILD=$(ROOT)/build/hiwire
-HIWIREREPO=https://github.com/hoodmane/hiwire
+HIWIREREPO=https://github.com/pyodide/hiwire
 HIWIRE_COMMIT=6a1e67280a15d929ebeceee54a6358c9c8d5f697
 
 ifdef CPYTHON_DEBUG

src/js/emscripten-settings.ts

@@ -3,7 +3,7 @@
 import { ConfigType } from "./pyodide";
 import { initializeNativeFS } from "./nativefs";
 import { loadBinaryFile, getBinaryResponse } from "./compat";
-import { API, PreRunFunc, type Module, type FSType } from "./types";
+import { API, PreRunFunc, type PyodideModule, type FSType } from "./types";
 import { getSentinelImport } from "generated/sentinel";
 
 /**
@@ -122,7 +122,7 @@ function callFsInitHook(
   ];
 }
 
-function computeVersionTuple(Module: Module): [number, number, number] {
+function computeVersionTuple(Module: PyodideModule): [number, number, number] {
   const versionInt = Module.HEAPU32[Module._Py_Version >>> 2];
   const major = (versionInt >>> 24) & 0xff;
   const minor = (versionInt >>> 16) & 0xff;
@@ -144,7 +144,7 @@ function computeVersionTuple(Module: Module): [number, number, number] {
  */
 function installStdlib(stdlibURL: string): PreRunFunc {
   const stdlibPromise: Promise<Uint8Array> = loadBinaryFile(stdlibURL);
-  return async (Module: Module) => {
+  return async (Module: PyodideModule) => {
     Module.API.pyVersionTuple = computeVersionTuple(Module);
     const [pymajor, pyminor] = Module.API.pyVersionTuple;
     Module.FS.mkdirTree("/lib");

src/js/nativefs.ts

@@ -1,9 +1,9 @@
-import { Module } from "./types";
+import { PyodideModule } from "./types";
 
 /**
  * @private
  */
-async function syncfs(m: Module, direction: boolean): Promise<void> {
+async function syncfs(m: PyodideModule, direction: boolean): Promise<void> {
   return new Promise((resolve, reject) => {
     m.FS.syncfs(direction, (err: any) => {
       if (err) {
@@ -18,21 +18,21 @@ async function syncfs(m: Module, direction: boolean): Promise<void> {
 /**
  * @private
  */
-export async function syncLocalToRemote(m: Module): Promise<void> {
+export async function syncLocalToRemote(m: PyodideModule): Promise<void> {
   return await syncfs(m, false);
 }
 
 /**
  * @private
  */
-export async function syncRemoteToLocal(m: Module): Promise<void> {
+export async function syncRemoteToLocal(m: PyodideModule): Promise<void> {
   return await syncfs(m, true);
 }
 
 /**
  * @private
  */
-export function initializeNativeFS(module: Module) {
+export function initializeNativeFS(module: PyodideModule) {
   const FS = module.FS;
   const MEMFS = module.FS.filesystems.MEMFS;
   const PATH = module.PATH;

src/js/pyodide.ts

@@ -16,7 +16,7 @@ import { version as version_ } from "./version";
 import type { PyodideAPI } from "./api.js";
 import type {
   TypedArray,
-  Module,
+  PyodideModule,
   PackageData,
   FSType,
   Lockfile,
@@ -40,7 +40,7 @@ export const version: string = version_;
 
 declare function _createPyodideModule(
   settings: EmscriptenSettings,
-): Promise<Module>;
+): Promise<PyodideModule>;
 
 // BUILD_ID is generated from hashing together pyodide.asm.js and
 // pyodide.asm.wasm in esbuild.config.outer.mjs

src/js/types.ts

@@ -21,7 +21,7 @@ export type TypedArray =
   | Float64Array;
 
 declare global {
-  export var Module: Module;
+  export var Module: PyodideModule;
   export var API: API;
 }
 
@@ -263,7 +263,7 @@ interface PyodideFSType {
 export type FSType = Omit<typeof FS, "lookupPath"> & PyodideFSType;
 
 /** @hidden */
-export type PreRunFunc = (Module: Module) => void;
+export type PreRunFunc = (Module: PyodideModule) => void;
 
 type DSO = any;
 
@@ -274,13 +274,8 @@ export interface LDSO {
   };
 }
 
-/**
- * TODO: consider renaming the type to ModuleType to avoid name collisions
- * between Module and ModuleType?
- * @hidden
- */
-export interface Module {
-  API: API;
+/** @hidden */
+export interface EmscriptenModule {
   locateFile: (file: string) => string;
   exited?: { toThrow: any };
   ENV: { [key: string]: string };
@@ -298,6 +293,36 @@ export interface Module {
   ERRNO_CODES: { [k: string]: number };
   stringToNewUTF8(x: string): number;
   stringToUTF8OnStack: (str: string) => number;
+  HEAP8: Uint8Array;
+  HEAPU32: Uint32Array;
+  getExceptionMessage(e: number): [string, string];
+  exitCode: number | undefined;
+  ExitStatus: { new (exitCode: number): Error };
+  _free: (ptr: number) => void;
+  stackSave: () => number;
+  stackRestore: (ptr: number) => void;
+  promiseMap: {
+    free(id: number): void;
+  };
+  _emscripten_dlopen_promise(lib: number, flags: number): number;
+  _dlerror(): number;
+  UTF8ToString: (
+    ptr: number,
+    maxBytesToRead: number,
+    ignoreNul?: boolean,
+  ) => string;
+}
+
+/** @hidden */
+export interface PythonModule extends EmscriptenModule {
+  _Py_EMSCRIPTEN_SIGNAL_HANDLING: number;
+  Py_EmscriptenSignalBuffer: TypedArray;
+  _Py_Version: number;
+}
+
+/** @hidden */
+export interface PyodideModule extends PythonModule {
+  API: API;
   _compat_to_string_repr: number;
   _compat_null_to_none: number;
   js2python_convert: (
@@ -312,36 +337,16 @@ export interface Module {
     },
   ) => any;
   _PropagatePythonError: typeof Error;
-  _Py_EMSCRIPTEN_SIGNAL_HANDLING: number;
-  Py_EmscriptenSignalBuffer: TypedArray;
-  HEAP8: Uint8Array;
-  HEAPU32: Uint32Array;
   __hiwire_get(a: number): any;
   __hiwire_set(a: number, b: any): void;
   __hiwire_immortal_add(a: any): void;
   _jslib_init(): number;
   _init_pyodide_proxy(): number;
-  getExceptionMessage(e: number): [string, string];
+
   handle_js_error(e: any): void;
-  exitCode: number | undefined;
-  ExitStatus: { new (exitCode: number): Error };
-  _Py_Version: number;
   _print_stdout: (ptr: number) => void;
   _print_stderr: (ptr: number) => void;
-  _free: (ptr: number) => void;
-  stackSave: () => number;
-  stackRestore: (ptr: number) => void;
-  promiseMap: {
-    free(id: number): void;
-  };
-  _emscripten_dlopen_promise(lib: number, flags: number): number;
   getPromise(p: number): Promise<any>;
-  _dlerror(): number;
-  UTF8ToString: (
-    ptr: number,
-    maxBytesToRead: number,
-    ignoreNul?: boolean,
-  ) => string;
 }
 
 /**
@@ -579,7 +584,7 @@ export type PackageManagerAPI = Pick<
  * @hidden
  */
 export type PackageManagerModule = Pick<
-  Module,
+  PyodideModule,
   | "PATH"
   | "LDSO"
   | "stringToNewUTF8"

src/tests/test_asyncio.py

@@ -332,20 +332,18 @@ def test_eval_code_await_error(selenium):
         )
 
 
+@run_in_pyodide
 def test_ensure_future_memleak(selenium):
-    selenium.run_js(
-        """
-        self.o = { "xxx" : 777 };
-        pyodide.runPython(`
-            import asyncio
-            from js import o
-            async def test():
-                return o
-            asyncio.ensure_future(test())
-            None
-        `);
-        """
-    )
+    import asyncio
+
+    from pyodide.code import run_js
+
+    o = run_js("() => ({ xxx: 777 })")()
+
+    async def test():
+        return o
+
+    asyncio.ensure_future(test())
 
 
 def test_await_pyproxy_eval_async(selenium):

src/tests/test_filesystem.py

@@ -28,23 +28,23 @@ def test_idbfs_persist_code(selenium_standalone):
         pyodide.FS.mount(pyodide.FS.filesystems.{fstype}, {{root : "."}}, mountDir);
         """
     )
-    # create file in mount
-    selenium.run_js(
-        f"""
-        pyodide.runPython(`
-            import pathlib
-            p = pathlib.Path('{mount_dir}/test_idbfs/__init__.py')
-            p.parent.mkdir(exist_ok=True, parents=True)
-            p.write_text("def test(): return 7")
-            from importlib import invalidate_caches
-            invalidate_caches()
-            import sys
-            sys.path.append('{mount_dir}')
-            from test_idbfs import test
-            assert test() == 7
-        `);
-        """
-    )
+
+    @run_in_pyodide
+    def create_test_file(selenium_module, mount_dir):
+        import sys
+        from importlib import invalidate_caches
+        from pathlib import Path
+
+        p = Path(f"{mount_dir}/test_idbfs/__init__.py")
+        p.parent.mkdir(exist_ok=True, parents=True)
+        p.write_text("def test(): return 7")
+        invalidate_caches()
+        sys.path.append(mount_dir)
+        from test_idbfs import test
+
+        assert test() == 7
+
+    create_test_file(selenium, mount_dir)
     # sync TO idbfs
     selenium.run_js(
         """