Introduce 3.14t CinderX build

Summary:
Gives us a working build of 3.14t + CinderX with no actual support for FT (we don't even provide the FT flag to the module yet). With this change we get to the point we can build + run a very basic smoke test with and without JIT.

The work is more than just adding Buck build targets because once we have `Py_GIL_DISABLED` things start breaking due to changes in internal features that we depend on. E.g. the reference counting fields of `PyObject` have changed. To help with this we now have a whole new `borrowed-3.14.free-threading.c.template` (which isn't called 3.14t for Buck-related reasons).

Some stuff here we were going to need to fix anyway, but most implementation is incomplete so I've filled tasks for all the todos.

Reviewed By: alexmalyshev

Differential Revision: D90686232

fbshipit-source-id: 03b2961422a8ab0c7565adf3ab2ef4872f34d3f3

Author: jbower-fb

cinderx/Common/py-portability.h

@@ -240,7 +240,7 @@ inline PyObject* frameExecutable(_PyInterpreterFrame* frame) {
     *_tmp_dst_ptr = PyStackRef_FromPyObjectSteal(VAL); \
     PyStackRef_XCLOSE(_tmp_old_dst);                   \
   } while (0)
-#define Ci_STACK_NEWREF(VAL) _PyStackRef_FromPyObjectNew(VAL)
+#define Ci_STACK_NEWREF(VAL) PyStackRef_FromPyObjectNew(VAL)
 #define Ci_STACK_CLOSE(VAL) PyStackRef_CLOSE(VAL)
 #define Ci_STACK_XCLOSE(VAL) PyStackRef_XCLOSE(VAL)
 

cinderx/Common/string.h

@@ -6,17 +6,32 @@
 
 #include "cinderx/Common/py-portability.h"
 
-// Create a function static variable for Python string.
-// This string is explicitly immortalized, but not
-// interned because doing so will cause it to be released
-// by the runtime at shutdown.
+// Create a function static variable for Python string. This string is
+// explicitly immortalized, but not interned because doing so will cause it to
+// be released by the runtime at shutdown. We cannot use
+// _Py_SetImmortalUntracked() as this has an assertion to force the use of
+// _PyUnicode_InternImmortal() for strings.
+#if PY_VERSION_HEX >= 0x030E0000 && defined(Py_GIL_DISABLED)
+#define DEFINE_NAMED_STATIC_STRING(NAME, STR)                  \
+  static PyObject* NAME = NULL;                                \
+  if (NAME == NULL) {                                          \
+    PyObject* op = PyUnicode_FromString(STR);                  \
+    op->ob_tid = _Py_UNOWNED_TID;                              \
+    op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL;              \
+    op->ob_ref_shared = 0;                                     \
+    _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); \
+    _PyASCIIObject_CAST(op)->state.statically_allocated = 1;   \
+    NAME = op;                                                 \
+  }
+#else
 #define DEFINE_NAMED_STATIC_STRING(NAME, STR)      \
   static PyObject* NAME = NULL;                    \
   if (NAME == NULL) {                              \
     PyObject* new_str = PyUnicode_FromString(STR); \
     new_str->ob_refcnt = 0x3fffffff;               \
     NAME = new_str;                                \
   }
+#endif
 
 // Shorter variant of DEFINE_NAMED_STATIC_STRING.
 #define DEFINE_STATIC_STRING(STR) DEFINE_NAMED_STATIC_STRING(s_##STR, (#STR))

cinderx/Immortalize/immortalize.cpp

@@ -74,7 +74,13 @@ bool immortalize(PyObject* obj) {
 }
 
 #if PY_VERSION_HEX >= 0x030C0000
-PyObject* immortalize_heap(PyObject* mod) {
+PyObject* immortalize_heap([[maybe_unused]] PyObject* mod) {
+#ifdef Py_GIL_DISABLED
+  PyErr_SetString(
+      PyExc_RuntimeError,
+      "Immortalizing the heap is not yet supported in FT Python");
+#else
+  // TODO(T251571267): Low priority for now.
   /* Remove any dead objects to avoid immortalizing them */
   PyGC_Collect();
 
@@ -95,8 +101,10 @@ PyObject* immortalize_heap(PyObject* mod) {
     Py_TYPE(FROM_GC(gc))
         ->tp_traverse(FROM_GC(gc), immortalize_visitor, nullptr);
   }
+#endif
 
   Py_RETURN_NONE;
+}
 #else
 PyObject* immortalize_heap(PyObject* /* mod */) {
   // for 3.10.cinder, we fall back to the implementation that ships in the gc
@@ -112,5 +120,5 @@ PyObject* immortalize_heap(PyObject* /* mod */) {
     return nullptr;
   }
   return Ref<>::steal(PyObject_CallFunctionObjArgs(immortalize, nullptr));
-#endif
 }
+#endif

cinderx/Interpreter/3.14/Includes/generated_cases.c.h

@@ -9445,6 +9445,13 @@
             _PyStackRef v;
             v = stack_pointer[-1];
             list = stack_pointer[-2 - (oparg-1)];
+            #if Py_GIL_DISABLED
+            int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
+                PyStackRef_AsPyObjectSteal(v));
+            if (err < 0) {
+                JUMP_TO_LABEL(pop_1_error);
+            }
+            #else
             _PyFrame_SetStackPointer(frame, stack_pointer);
             int err = Ci_ListOrCheckedList_Append(
                 (PyListObject*)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectBorrow(v));
@@ -9457,6 +9464,10 @@
             if (err < 0) {
                 JUMP_TO_LABEL(error);
             }
+            stack_pointer += 1;
+            #endif
+            stack_pointer += -1;
+            assert(WITHIN_STACK_BOUNDS());
             DISPATCH();
         }
 
@@ -11556,6 +11567,20 @@
             value = stack_pointer[-1];
             key = stack_pointer[-2];
             dict_st = stack_pointer[-3 - (oparg - 1)];
+            #if Py_GIL_DISABLED
+            PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
+            assert(PyDict_CheckExact(dict));
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            int err = _PyDict_SetItem_Take2(
+                (PyDictObject *)dict,
+                PyStackRef_AsPyObjectSteal(key),
+                PyStackRef_AsPyObjectSteal(value)
+            );
+            stack_pointer = _PyFrame_GetStackPointer(frame);
+            if (err != 0) {
+                JUMP_TO_LABEL(pop_2_error);
+            }
+            #else
             PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
             _PyFrame_SetStackPointer(frame, stack_pointer);
             int err = Ci_DictOrChecked_SetItem(dict,
@@ -11572,9 +11597,13 @@
             _PyFrame_SetStackPointer(frame, stack_pointer);
             PyStackRef_CLOSE(key);
             stack_pointer = _PyFrame_GetStackPointer(frame);
+            stack_pointer += 2;
+            #endif
             if (err != 0) {
-                JUMP_TO_LABEL(error);
+                JUMP_TO_LABEL(pop_2_error);
             }
+            stack_pointer += -2;
+            assert(WITHIN_STACK_BOUNDS());
             DISPATCH();
         }
 

cinderx/Interpreter/3.14/cinder-bytecodes.c

@@ -466,21 +466,42 @@ dummy_func(
         }
 
         override inst(MAP_ADD, (dict_st, unused[oparg - 1], key, value -- dict_st, unused[oparg - 1])) {
+#ifdef Py_GIL_DISABLED
+            // T250369690: Need thread-safe checked collections
+            PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
+            assert(PyDict_CheckExact(dict));
+            /* dict[key] = value */
+            // Do not DECREF INPUTS because the function steals the references
+            int err = _PyDict_SetItem_Take2(
+                (PyDictObject *)dict,
+                PyStackRef_AsPyObjectSteal(key),
+                PyStackRef_AsPyObjectSteal(value)
+            );
+            ERROR_IF(err != 0);
+#else
             PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);
             /* dict[key] = value */
             int err = Ci_DictOrChecked_SetItem(dict,
                 PyStackRef_AsPyObjectBorrow(key),
                 PyStackRef_AsPyObjectBorrow(value));
             PyStackRef_CLOSE(value);
             PyStackRef_CLOSE(key);
+#endif
             ERROR_IF(err != 0);
         }
 
         override inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) {
+#ifdef Py_GIL_DISABLED
+            // T250369690: Need thread-safe checked collections
+            int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list),
+                                           PyStackRef_AsPyObjectSteal(v));
+            ERROR_IF(err < 0);
+#else
             int err = Ci_ListOrCheckedList_Append(
                 (PyListObject*)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectBorrow(v));
             PyStackRef_CLOSE(v);
             ERROR_IF(err < 0);
+#endif
         }
 
         override inst(EXTENDED_OPCODE, (args[oparg>>2] -- top[oparg&0x03])) {

cinderx/Jit/codegen/frame_asm.cpp

@@ -153,6 +153,8 @@ void FrameAsm::emitIncTotalRefCount(const arch::Gp& scratch_reg) {
 #endif
 }
 
+#ifdef ENABLE_LIGHTWEIGHT_FRAMES
+// TODO(T251571746): need to correctly implement FT incref.
 void FrameAsm::incRef(const arch::Gp& reg, const arch::Gp& scratch_reg) {
 #if defined(CINDER_X86_64)
   as_->mov(scratch_reg, x86::ptr(reg, offsetof(PyObject, ob_refcnt)));
@@ -171,6 +173,7 @@ void FrameAsm::incRef(const arch::Gp& reg, const arch::Gp& scratch_reg) {
   CINDER_UNSUPPORTED
 #endif
 }
+#endif
 
 bool FrameAsm::storeConst(
     const arch::Gp& reg,
@@ -358,6 +361,8 @@ void FrameAsm::linkLightWeightFunctionFrame(
     preserver.remap();
   }
 #else
+  // TODO(T251571746): For FTPython either need to set TLBC somewhere in here or
+  // in the reifier.
   throw std::runtime_error{
       "linkLightWeightFunctionFrame: Lightweight frames are not supported"};
 #endif

cinderx/Jit/config.h

@@ -145,7 +145,14 @@ struct Config {
   // dictionaries (but not their contents).
   bool stable_frame{true};
   // Use inline caches for attribute accesses.
-  bool attr_caches{true};
+  bool attr_caches{
+#ifdef Py_GIL_DISABLED
+      // TODO(T250369692): FT support for inline-caches.
+      false
+#else
+      true
+#endif
+  };
   // Collect stats information about attribute caches.
   bool collect_attr_cache_stats{false};
   // Use type annotations to create runtime checks.

cinderx/Jit/deopt.cpp

@@ -2,6 +2,12 @@
 
 #include "cinderx/Jit/deopt.h"
 
+// clang-tidy off
+#include "cinderx/UpstreamBorrow/borrowed.h"
+// clang-tidy on
+
+#include "internal/pycore_ceval.h"
+
 #include "cinderx/Common/py-portability.h"
 #include "cinderx/Common/util.h"
 #include "cinderx/Jit/bytecode_offsets.h"
@@ -313,11 +319,18 @@ static void reifyFrameImpl(
     const uint64_t* regs) {
 #if PY_VERSION_HEX >= 0x030E0000
   BorrowedRef<PyCodeObject> code_obj = frameCode(frame);
+#ifdef Py_GIL_DISABLED
+  PyThreadState* tstate = _PyThreadState_GET();
+  frame->instr_ptr = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
+  frame->tlbc_index = reinterpret_cast<_PyThreadStateImpl*>(tstate)->tlbc_index;
+#else
+  frame->instr_ptr = _PyCode_CODE(code_obj);
+#endif
   int cause_instr_idx = frame_meta.cause_instr_idx.value();
   // Resume with instr_ptr pointing to the cause instruction if we are entering
   // the interpreter to re-run a failed instruction, or implement an instruction
   // we don't JIT.
-  frame->instr_ptr = _PyCode_CODE(code_obj) + cause_instr_idx;
+  frame->instr_ptr += cause_instr_idx;
   if (&frame_meta != &meta.innermostFrame()) {
     // If we're not the inner most frame then we're always deopting
     // after the instruction that executed