Add optional + struct to UpstreamBorrow

Summary:
The ability to mark things as optional means we can have the script fail if non-optional things are missing. Otherwise it's very confusing when the script is okay but things are missing.

Things might be optional if they depend on compile time flags (like `Py_DEBUG` or `Py_GIL_DISABLED`). I can't think of a simple way of controlling things by the macro flags directly.

The ability to borrow struct types is just because I made the two things together for following diffs and it seems more work than it's worth to separate them.

Reviewed By: alexmalyshev

Differential Revision: D91197755

fbshipit-source-id: 5ef31e55f1e581d307801c0621296b85bed5c452

Author: jbower-fb

cinderx/Interpreter/3.14/borrowed-ceval.c.template

@@ -41,12 +41,14 @@ int _Py_CheckRecursiveCallPy(PyThreadState* tstate);
 // @Borrow var _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS from Python/ceval.c
 
 // @Borrow function do_raise from Python/ceval.c
-// @Borrow function dump_item from Python/ceval.c
-// @Borrow function dump_stack from Python/ceval.c
-// @Borrow function lltrace_instruction from Python/ceval.c
-// @Borrow function lltrace_resume_frame from Python/ceval.c
 
-// @Borrow function maybe_lltrace_resume_frame from Python/ceval.c
+// These are all optional because they are only present + used in debug builds
+// @Borrow optional function dump_item from Python/ceval.c
+// @Borrow optional function dump_stack from Python/ceval.c
+// @Borrow optional function lltrace_instruction from Python/ceval.c
+// @Borrow optional function lltrace_resume_frame from Python/ceval.c
+// @Borrow optional function maybe_lltrace_resume_frame from Python/ceval.c
+
 // @Borrow function do_monitor_exc from Python/ceval.c
 // @Borrow function no_tools_for_global_event from Python/ceval.c
 // @Borrow function no_tools_for_local_event from Python/ceval.c

cinderx/Interpreter/3.14/ceval.h

@@ -172,6 +172,8 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
     Py_XDECREF(cause);
     return 0;
 }
+
+// These are all optional because they are only present + used in debug builds
 static void
 dump_item(_PyStackRef item)
 {
@@ -279,7 +281,6 @@ lltrace_resume_frame(_PyInterpreterFrame *frame)
     fflush(stdout);
     PyErr_SetRaisedException(exc);
 }
-
 static int
 maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
 {
@@ -306,6 +307,7 @@ maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
     }
     return lltrace;
 }
+
 static int
 do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
                _Py_CODEUNIT *instr, int event)

cinderx/UpstreamBorrow/UpstreamBorrow.py

@@ -21,7 +21,8 @@
     r".*// @Borrow CPP directives noinclude from (\S+)" + VERSION_AND_TRANSFORM
 )
 BORROW_DECL_PATTERN: re.Pattern[str] = re.compile(
-    r"// @Borrow (function|typedef|var) (\S+) from (\S+)" + VERSION_AND_TRANSFORM
+    r"// @Borrow (optional )?(function|typedef|var|struct|enum) (\S+) from (\S+)"
+    + VERSION_AND_TRANSFORM
 )
 CPP_DIRECTIVE_PATTERN: re.Pattern[str] = re.compile(
     r"#\s*(define|undef|if|elif|else|endif|include)"
@@ -67,6 +68,7 @@ class Decl:
     source_file: str
     version: str | None
     transform: str | None
+    optional: bool = False
 
 
 @dataclass
@@ -80,11 +82,12 @@ def parse_borrow_info(input_string: str) -> Decl | None:
     if not match:
         return None
 
-    kind_str = match.group(1)
-    name = match.group(2)
-    source_file = match.group(3)
-    version = match.group(4)  # not used for anything, but generated by callgraph.py
-    transform = match.group(5)
+    optional = match.group(1) is not None
+    kind_str = match.group(2)
+    name = match.group(3)
+    source_file = match.group(4)
+    version = match.group(5)  # not used for anything, but generated by callgraph.py
+    transform = match.group(6)
 
     if kind_str == "function":
         # pyre-ignore[16]: `CursorKind` has no attribute `FUNCTION_DECL`.
@@ -95,13 +98,18 @@ def parse_borrow_info(input_string: str) -> Decl | None:
     elif kind_str == "var":
         # pyre-ignore[16]: `CursorKind` has no attribute `VAR_DECL`.
         kind = CursorKind.VAR_DECL
+    elif kind_str == "struct":
+        # pyre-ignore[16]: `CursorKind` has no attribute `STRUCT_DECL`.
+        kind = CursorKind.STRUCT_DECL
     elif kind_str == "enum":
         # pyre-ignore[16]: `CursorKind` has no attribute `ENUM_DECL`.
         kind = CursorKind.ENUM_DECL
     else:
         raise Exception(f"Unknown kind: {kind_str}")
 
-    return Decl(kind, name, source_file, version, transform[1:] if transform else None)
+    return Decl(
+        kind, name, source_file, version, transform[1:] if transform else None, optional
+    )
 
 
 def transform_eval_frame(lines: list[str]) -> list[str]:
@@ -217,11 +225,14 @@ def _generate_output(self) -> None:
         out = []
         for line in self.input_lines:
             match line:
-                case Decl(kind, name, source_file, _, transform):
+                case Decl(kind, name, source_file, _, transform, optional):
                     lines = self.decls[source_file].get(name)
                     if lines is None:
-                        print(f"Could not find {kind} for '{name}' in {source_file}")
-                        continue
+                        if optional:
+                            continue
+                        raise Exception(
+                            f"Could not find {kind} for '{name}' in {source_file}"
+                        )
                     # Apply custom transformation if one exists for this function
                     if transform in TRANSFORMS:
                         lines = TRANSFORMS[transform](lines)

cinderx/UpstreamBorrow/borrowed-3.15.c.template

@@ -140,14 +140,11 @@ PyDictKeysObject* ci_dict_empty_keys;
 // @Borrow function unicodekeys_lookup_unicode from Objects/dictobject.c [3.15]
 // @Borrow function unicodekeys_lookup_generic from Objects/dictobject.c [3.15]
 // @Borrow function dictkeys_generic_lookup from Objects/dictobject.c [3.15]
-// @Borrow function _Py_dict_lookup_keep_lazy from Objects/dictobject.c [3.15]
 // @Borrow function lookdict_index from Objects/dictobject.c [3.15]
 // @Borrow function delete_index_from_values from Objects/dictobject.c [3.15]
 // @Borrow function dictkeys_set_index from Objects/dictobject.c [3.15]
 // @Borrow function delitem_common from Objects/dictobject.c [3.15]
-// @Borrow function delitem_knownhash_lock_held from Objects/dictobject.c [3.15]
 // @Borrow function new_keys_object from Objects/dictobject.c [3.15]
-// @Borrow function lazy_import_verbose from Objects/dictobject.c [3.15]
 // @Borrow function insert_to_emptydict from Objects/dictobject.c [3.15]
 // @Borrow function ensure_shared_on_resize from Objects/dictobject.c [3.15]
 // @Borrow function build_indices_generic from Objects/dictobject.c [3.15]
@@ -526,8 +523,6 @@ LONG_FLOAT_ACTION(compactlong_float_true_div, /)
 // @Borrow function tb_create_raw from Python/traceback.c [3.15]
 // @Borrow function _PyTraceBack_FromFrame from Python/traceback.c [3.15]
 
-// @Borrow function _PyFloat_FromDouble_ConsumeInputs from Objects/floatobject.c [3.15]
-
 // Internal dependencies for gen_dealloc.
 // @Borrow function gen_clear_frame from Objects/genobject.c [3.15]
 // End internal dependencies.

cinderx/UpstreamBorrow/borrowed-3.15.gen_cached.c

@@ -5597,7 +5597,6 @@ _PyTraceBack_FromFrame(PyObject *tb_next, PyFrameObject *frame)
     return tb_create_raw((PyTracebackObject *)tb_next, frame, addr, -1);
 }
 
-
 // Internal dependencies for gen_dealloc.
 static void
 gen_clear_frame(PyGenObject *gen)