Enable generators + coroutines in JIT

Summary: With this remaining grab bag of fixes for tests we can now enable JIT compilation of generators and coroutines in 3.14.

Reviewed By: alexmalyshev

Differential Revision: D83003857

fbshipit-source-id: 467b7bbfd01d0aa0b90c21c7dfba2b63321d1257

Author: jbower-fb

cinderx/Jit/generators_core.cpp

@@ -80,7 +80,11 @@ PyObject* JitCoro_GetAwaitableIter(PyObject* o) {
 
   PyErr_Format(
       PyExc_TypeError,
+#if PY_VERSION_HEX >= 0x030E0000
+      "'%.100s' object can't be awaited",
+#else
       "object %.100s can't be used in 'await' expression",
+#endif
       ot->tp_name);
   return nullptr;
 }

cinderx/Jit/generators_rt.cpp

@@ -209,7 +209,13 @@ PySendResult jitgen_am_send(PyObject* obj, PyObject* arg, PyObject** presult) {
   Py_CLEAR(gen->gi_ci_awaiter);
 #endif
 
+#if PY_VERSION_HEX < 0x030E0000
   _PyErr_ClearExcState(&gen->gi_exc_state);
+#else
+  JIT_DCHECK(
+      gen->gi_exc_state.exc_value == nullptr,
+      "Should not have an exception by now");
+#endif
   JIT_DCHECK(gen->gi_frame_state == FRAME_CLEARED, "Frame not cleared");
 
   *presult = result;

cinderx/Jit/pyjit.cpp

@@ -3316,10 +3316,8 @@ Context::CompilationResult compilePreloaderImpl(
         preloader.fullname());
     return {nullptr, PYJIT_RESULT_CANNOT_SPECIALIZE};
   }
-  constexpr int forbidden_flags = PY_VERSION_HEX >= 0x030C0000
-      ? CO_ASYNC_GENERATOR |
-          (PY_VERSION_HEX >= 0x030E0000 ? CO_GENERATOR | CO_COROUTINE : 0)
-      : 0;
+  constexpr int forbidden_flags =
+      PY_VERSION_HEX >= 0x030C0000 ? CO_ASYNC_GENERATOR : 0;
   if (code->co_flags & forbidden_flags) {
     JIT_DLOG(
         "Cannot JIT compile {} as it has prohibited code flags: 0x{:x}",

cinderx/Jit/runtime_support.cpp

@@ -23,7 +23,17 @@ PyObject g_iterDoneSentinel = {
     nullptr};
 
 PyObject* invokeIterNext(PyObject* iterator) {
-  PyObject* val = (*iterator->ob_type->tp_iternext)(iterator);
+  iternextfunc internext_f = Py_TYPE(iterator)->tp_iternext;
+  // This check was introduced in 3.14+ but looks like it would be legit in all
+  // versions. I'm surprised it wasn't backported.
+  if (internext_f == nullptr) {
+    PyErr_Format(
+        PyExc_TypeError,
+        "'%.100s' object is not an iterator",
+        Py_TYPE(iterator)->tp_name);
+    return nullptr;
+  }
+  PyObject* val = internext_f(iterator);
   if (val != nullptr) {
     return val;
   }

cinderx/TestScripts/3.14-opt-failures.txt

@@ -77,11 +77,8 @@ test_cinderx.test_compiler_sbs_stdlib_9
 test_cinderx.test_cpython_overrides.test_dis
 test_cinderx.test_immortalize
 test_cinderx.test_jit_attr_cache
-test_cinderx.test_jit_coroutines
 test_cinderx.test_jit_disable
-test_cinderx.test_jit_exception
 test_cinderx.test_jit_frame
-test_cinderx.test_jit_generators
 test_cinderx.test_jit_global_cache
 test_cinderx.test_jit_perf_map
 test_cinderx.test_jit_preload

cinderx/TestScripts/cinder_jit_ignore_tests.txt

@@ -154,3 +154,11 @@ test.test_descr.ClassPropertiesAndMethods.test_dir
 # + {'self': <test.test_capi.test_eval.Tests testMethod=test_eval_getlocals>,
 # +  'x': 1}
 test.test_capi.test_eval.Tests.test_eval_getlocals
+
+# The JIT doesn't follow changes to local variables via the frame objects
+test.test_generators.ModifyUnderlyingIterableTest.test_modify_f_locals
+
+# TASK(T239036547) This relies on GC of a function's globals created in a call
+# to exec(). However, we keep all "globals" objects alive via the JIT's
+# PreloaderManager.
+test.test_generators.FinalizationTest.test_generator_resurrect