Initial sketch of a Rust API for CPython

This commit introduces a rough sketch of a Rust API for CPython. The overall design goals are
1. Should be very similar to PyO3 API
2. Threadstate should be passed explicitly to ensure safety
3. Should feel familiar to C API users
4. If a C API for something is missing, we can add new internal C APIs (e.g. an operation where we need to pass the threadstate explicitly where it is fetched implicitly from TLS)
5. Should be minimal to what we need, we are not re-implementing all of PyO3

This is still a work in progress so things are still subject to change significantly.

Author: emmatyping

Cargo.lock

@@ -1,6 +1,10 @@
 [workspace]
 resolver = "3"
 members = [
-    "Modules/_base64", "Modules/cpython-build-helper", "Modules/cpython-rust-staticlib",
-    "Modules/cpython-sys"
+    "Modules/_base64",
+    "Modules/cpython-api",
+    "Modules/cpython-api-macros",
+    "Modules/cpython-build-helper",
+    "Modules/cpython-rust-staticlib",
+    "Modules/cpython-sys",
 ]

Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
-cpython-sys ={ path = "../cpython-sys" }
+cpython-api = { path = "../cpython-api" }
 
 [build-dependencies]
 cpython-build-helper = { path = "../cpython-build-helper" }

Modules/_base64/Cargo.toml

@@ -1,249 +1,93 @@
-use std::cell::UnsafeCell;
-use std::ffi::{c_char, c_int, c_void};
-use std::mem::MaybeUninit;
-use std::ptr;
-use std::slice;
-
-use cpython_sys::METH_FASTCALL;
-use cpython_sys::Py_DecRef;
-use cpython_sys::Py_buffer;
-use cpython_sys::Py_ssize_t;
-use cpython_sys::PyBuffer_Release;
-use cpython_sys::PyBytes_AsString;
-use cpython_sys::PyBytes_FromStringAndSize;
-use cpython_sys::PyErr_NoMemory;
-use cpython_sys::PyErr_SetString;
-use cpython_sys::PyExc_TypeError;
-use cpython_sys::PyMethodDef;
-use cpython_sys::PyMethodDefFuncPointer;
-use cpython_sys::PyModuleDef;
-use cpython_sys::PyModuleDef_HEAD_INIT;
-use cpython_sys::PyModuleDef_Init;
-use cpython_sys::PyObject;
-use cpython_sys::PyObject_GetBuffer;
-
-const PYBUF_SIMPLE: c_int = 0;
-const PAD_BYTE: u8 = b'=';
-const ENCODE_TABLE: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-#[inline]
-fn encoded_output_len(input_len: usize) -> Option<usize> {
-    input_len
-        .checked_add(2)
-        .map(|n| n / 3)
-        .and_then(|blocks| blocks.checked_mul(4))
-}
-
-#[inline]
-fn encode_into(input: &[u8], output: &mut [u8]) -> usize {
-    let mut src_index = 0;
-    let mut dst_index = 0;
-    let len = input.len();
-
-    while src_index + 3 <= len {
-        let chunk = (u32::from(input[src_index]) << 16)
-            | (u32::from(input[src_index + 1]) << 8)
-            | u32::from(input[src_index + 2]);
-        output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize];
-        output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize];
-        output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize];
-        output[dst_index + 3] = ENCODE_TABLE[(chunk & 0x3f) as usize];
-        src_index += 3;
-        dst_index += 4;
-    }
-
-    match len - src_index {
-        0 => {}
-        1 => {
-            let chunk = u32::from(input[src_index]) << 16;
-            output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize];
-            output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize];
-            output[dst_index + 2] = PAD_BYTE;
-            output[dst_index + 3] = PAD_BYTE;
-            dst_index += 4;
-        }
-        2 => {
-            let chunk =
-                (u32::from(input[src_index]) << 16) | (u32::from(input[src_index + 1]) << 8);
-            output[dst_index] = ENCODE_TABLE[((chunk >> 18) & 0x3f) as usize];
-            output[dst_index + 1] = ENCODE_TABLE[((chunk >> 12) & 0x3f) as usize];
-            output[dst_index + 2] = ENCODE_TABLE[((chunk >> 6) & 0x3f) as usize];
-            output[dst_index + 3] = PAD_BYTE;
-            dst_index += 4;
-        }
-        _ => unreachable!("len - src_index cannot exceed 2"),
-    }
-
-    dst_index
-}
-
-struct BorrowedBuffer {
-    view: Py_buffer,
-}
+//! The `_base64` module, implemented against `cpython-api`.
 
-impl BorrowedBuffer {
-    fn from_object(obj: &PyObject) -> Result<Self, ()> {
-        let mut view = MaybeUninit::<Py_buffer>::uninit();
-        let buffer = unsafe {
-            if PyObject_GetBuffer(obj.as_raw(), view.as_mut_ptr(), PYBUF_SIMPLE) != 0 {
-                return Err(());
-            }
-            Self {
-                view: view.assume_init(),
-            }
-        };
-        Ok(buffer)
-    }
+use std::mem::MaybeUninit;
 
-    fn len(&self) -> Py_ssize_t {
-        self.view.len
-    }
+use cpython_api::prelude::*;
 
-    fn as_ptr(&self) -> *const u8 {
-        self.view.buf.cast::<u8>() as *const u8
-    }
-}
+/// Per-module state, empty since the base64 module has no state
+struct Base64State;
 
-impl Drop for BorrowedBuffer {
-    fn drop(&mut self) {
-        unsafe {
-            PyBuffer_Release(&mut self.view);
-        }
+impl ModuleState for Base64State {
+    fn new<'py>(_ts: &ThreadState<'py>, _module: &Bound<'py, PyModule>) -> PyResult<Self> {
+        Ok(Base64State)
     }
 }
 
-/// # Safety
-/// `module` must be a valid pointer of PyObject representing the module.
-/// `args` must be a valid pointer to an array of valid PyObject pointers with length `nargs`.
-pub unsafe extern "C" fn standard_b64encode(
-    _module: *mut PyObject,
-    args: *mut *mut PyObject,
-    nargs: Py_ssize_t,
-) -> *mut PyObject {
-    if nargs != 1 {
-        unsafe {
-            PyErr_SetString(
-                PyExc_TypeError,
-                c"standard_b64encode() takes exactly one argument".as_ptr(),
-            );
-        }
-        return ptr::null_mut();
-    }
-
-    let source = unsafe { &**args };
+const PAD_BYTE: u8 = b'=';
+const ENCODE_TABLE: [u8; 64] = *b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
-    // Safe cast by Safety
-    match standard_b64encode_impl(source) {
-        Ok(result) => result,
-        Err(_) => ptr::null_mut(),
-    }
+/// 4 output bytes per started block of 3 input bytes; `None` on overflow.
+#[inline]
+fn encoded_output_len(input_len: usize) -> Option<usize> {
+    input_len.div_ceil(3).checked_mul(4)
 }
 
-fn standard_b64encode_impl(source: &PyObject) -> Result<*mut PyObject, ()> {
-    let buffer = match BorrowedBuffer::from_object(source) {
-        Ok(buf) => buf,
-        Err(_) => return Err(()),
+/// Encode `input` into `output`, which must be exactly
+/// `encoded_output_len(input.len())` bytes and is fully initialized on
+/// return.
+fn encode_into(input: &[u8], output: &mut [MaybeUninit<u8>]) {
+    let mut chunks = input.chunks_exact(3);
+    let mut out = output.iter_mut();
+    let mut put = |b: u8| {
+        out.next()
+            .expect("output sized to encoded_output_len")
+            .write(b);
     };
 
-    let view_len = buffer.len();
-    if view_len < 0 {
-        unsafe {
-            PyErr_SetString(
-                PyExc_TypeError,
-                c"standard_b64encode() argument has negative length".as_ptr(),
-            );
-        }
-        return Err(());
+    for chunk in &mut chunks {
+        let group = (u32::from(chunk[0]) << 16) | (u32::from(chunk[1]) << 8) | u32::from(chunk[2]);
+        put(ENCODE_TABLE[(group >> 18 & 0x3f) as usize]);
+        put(ENCODE_TABLE[(group >> 12 & 0x3f) as usize]);
+        put(ENCODE_TABLE[(group >> 6 & 0x3f) as usize]);
+        put(ENCODE_TABLE[(group & 0x3f) as usize]);
     }
 
-    let input_len = view_len as usize;
-    let input = unsafe { slice::from_raw_parts(buffer.as_ptr(), input_len) };
-
-    let Some(output_len) = encoded_output_len(input_len) else {
-        unsafe {
-            PyErr_NoMemory();
+    match *chunks.remainder() {
+        [] => {}
+        [a] => {
+            let group = u32::from(a) << 16;
+            put(ENCODE_TABLE[(group >> 18 & 0x3f) as usize]);
+            put(ENCODE_TABLE[(group >> 12 & 0x3f) as usize]);
+            put(PAD_BYTE);
+            put(PAD_BYTE);
         }
-        return Err(());
-    };
-
-    if output_len > isize::MAX as usize {
-        unsafe {
-            PyErr_NoMemory();
+        [a, b] => {
+            let group = (u32::from(a) << 16) | (u32::from(b) << 8);
+            put(ENCODE_TABLE[(group >> 18 & 0x3f) as usize]);
+            put(ENCODE_TABLE[(group >> 12 & 0x3f) as usize]);
+            put(ENCODE_TABLE[(group >> 6 & 0x3f) as usize]);
+            put(PAD_BYTE);
         }
-        return Err(());
+        _ => unreachable!("chunks_exact(3) remainder is at most 2 bytes"),
     }
-
-    let result = unsafe { PyBytes_FromStringAndSize(ptr::null(), output_len as Py_ssize_t) };
-    if result.is_null() {
-        return Err(());
-    }
-
-    let dest_ptr = unsafe { PyBytes_AsString(result) };
-    if dest_ptr.is_null() {
-        unsafe {
-            Py_DecRef(result);
-        }
-        return Err(());
-    }
-    let dest = unsafe { slice::from_raw_parts_mut(dest_ptr.cast::<u8>(), output_len) };
-
-    let written = encode_into(input, dest);
-    debug_assert_eq!(written, output_len);
-    Ok(result)
-}
-
-pub extern "C" fn _base64_clear(_obj: *mut PyObject) -> c_int {
-    //TODO
-    0
-}
-
-pub extern "C" fn _base64_free(_o: *mut c_void) {
-    //TODO
 }
 
-pub struct ModuleDef {
-    ffi: UnsafeCell<PyModuleDef>,
+/// Encode a bytes-like object with the standard Base64 alphabet.
+#[pyfunction(signature = (data, /))]
+fn standard_b64encode<'py>(
+    ts: &ThreadState<'py>,
+    _state: &Base64State,
+    data: PyBuffer<'py>,
+) -> PyResult<Bound<'py, PyBytes>> {
+    let input = data.as_bytes();
+    let Some(output_len) = encoded_output_len(input.len()) else {
+        return Err(PyMemoryError::raise(ts, "encoded result too long"));
+    };
+    // Write straight into the bytes object's buffer — no intermediate Vec.
+    PyBytes::new_with(ts, output_len, |output| {
+        encode_into(input, output);
+        Ok(())
+    })
 }
 
-impl ModuleDef {
-    fn init_multi_phase(&'static self) -> *mut PyObject {
-        unsafe { PyModuleDef_Init(self.ffi.get()) }
-    }
+fn base64_exec<'py>(_ts: &ThreadState<'py>, _module: &Bound<'py, PyModule>) -> PyResult<()> {
+    Ok(())
 }
 
-unsafe impl Sync for ModuleDef {}
-
-pub static _BASE64_MODULE_METHODS: [PyMethodDef; 2] = {
-    [
-        PyMethodDef {
-            ml_name: c"standard_b64encode".as_ptr() as *mut c_char,
-            ml_meth: PyMethodDefFuncPointer {
-                PyCFunctionFast: standard_b64encode,
-            },
-            ml_flags: METH_FASTCALL,
-            ml_doc: c"Demo for the _base64 module".as_ptr() as *mut c_char,
-        },
-        PyMethodDef::zeroed(),
-    ]
-};
-
-pub static _BASE64_MODULE: ModuleDef = {
-    ModuleDef {
-        ffi: UnsafeCell::new(PyModuleDef {
-            m_base: PyModuleDef_HEAD_INIT,
-            m_name: c"_base64".as_ptr() as *mut _,
-            m_doc: c"A test Rust module".as_ptr() as *mut _,
-            m_size: 0,
-            m_methods: &_BASE64_MODULE_METHODS as *const PyMethodDef as *mut _,
-            m_slots: std::ptr::null_mut(),
-            m_traverse: None,
-            m_clear: Some(_base64_clear),
-            m_free: Some(_base64_free),
-        }),
-    }
-};
-
-#[unsafe(no_mangle)]
-pub extern "C" fn PyInit__base64() -> *mut PyObject {
-    _BASE64_MODULE.init_multi_phase()
+export_module! {
+    name: _base64,
+    doc: c"Base64 encoding implemented in Rust",
+    state: Base64State,
+    methods: [standard_b64encode],
+    exec: base64_exec,
 }

Modules/_base64/src/lib.rs

@@ -0,0 +1,12 @@
+[package]
+name = "cpython-api-macros"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+quote = "1"
+syn = { version = "2", features = ["full"] }