Column Storage: Add storage interfaces to SDK

Introduce storage-related reference types and define a column
storage interface for extensions. This extends the SDK to allow
column data to be managed externally by extensions.

Author: villagedeb

unittest/gunit/villagesql/abi_v2_check-t.cc

@@ -51,8 +51,9 @@
 //   const char *decode_vdf_name;                    // +88  (protocol >= 2)
 //   const char *compare_vdf_name;                   // +96  (protocol >= 2)
 //   const char *hash_vdf_name;                      // +104 (protocol >= 2)
+//   vef_type_storage_intf_t storage_intf;           // +112 (protocol >= 2)
 // ---------------------------------------------------------------------------
-static_assert(sizeof(vef_type_desc_t) == 112,
+static_assert(sizeof(vef_type_desc_t) == 168,
               "ABI v2 break: vef_type_desc_t size changed");
 static_assert(offsetof(vef_type_desc_t, int_to_params) == 64,
               "ABI v2 break: vef_type_desc_t::int_to_params offset changed");
@@ -66,6 +67,8 @@ static_assert(offsetof(vef_type_desc_t, compare_vdf_name) == 96,
               "ABI v2 break: vef_type_desc_t::compare_vdf_name offset changed");
 static_assert(offsetof(vef_type_desc_t, hash_vdf_name) == 104,
               "ABI v2 break: vef_type_desc_t::hash_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, storage_intf) == 112,
+              "ABI v2 break: vef_type_desc_t::storage_intf offset changed");
 
 // Placeholder test so the binary links and runs.
 TEST(AbiV2Check, StaticAssertsPass) {}

villagesql/sdk/include/villagesql/abi/types.h

@@ -159,6 +159,7 @@ typedef enum : unsigned int {
                    // + Add deterministic VDF attribute.
                    // + Add encode/decode/compare/hash VDF name fields to
                    //   vef_type_desc_t.
+                   // + Add storage interface to vef_type_desc_t.
 } vef_protocol_t;
 
 // Max length of error messages in caller-provided buffers.
@@ -541,6 +542,170 @@ typedef bool (*vef_type_resolve_params_func_t)(
     const vef_type_param_t *params, size_t param_count,
     vef_type_resolved_params_t *result, char *error_msg);
 
+// =============================================================================
+// Storage Types and functions for column data stored by the extension.
+// =============================================================================
+// InnoDB tablespace reference.
+typedef uint32_t vef_storage_space_ref_t;
+
+// InnoDB storage reference: space and root page reference.
+typedef uint64_t vef_storage_ref_t;
+
+// InnoDB column reference: page reference and offset. The column is always
+// stored in the tablespace referred by the storage reference.
+typedef uint64_t vef_storage_col_ref_t;
+
+// InnoDB transaction reference.
+typedef uint64_t vef_storage_trx_ref_t;
+
+// Mini-transaction context that can be passed to InnoDB storage interface
+// acquiring pages.
+typedef void *vef_storage_mtr_ref_t;
+
+// Column data buffer and length
+typedef struct {
+  const unsigned char *data;
+  uint32_t length;
+} vef_storage_col_data_t;
+
+// Empty column reference.
+#define VEF_STORAGE_EMPTY_COLUMN_REF 0
+// Alignment requirement for arena allocators
+#define VEF_STORAGE_ALLOCATOR_ALIGNMENT 8
+
+// Arena allocator callback type
+// The allocator MUST return memory aligned to VEF_STORAGE_ALLOCATOR_ALIGNMENT.
+// Author can use the Allocator callback to allocate memory for context. The
+// allocated memory would be tracked and freed automatically by InnoDB after
+// the storage is dropped.
+typedef void *(*vef_storage_arena_func_t)(void *arena, uint32_t size);
+
+// Partially opaque storage context with visible storage reference. Author may
+// allocate a context object (using Arena allocator) of larger size with this
+// structure as prefix to store additional private state during creation. The
+// same context is passed to subsequent operations (insert, delete, etc.).
+typedef struct {
+  // Column storage reference
+  vef_storage_ref_t ref;
+} vef_storage_ctx_t;
+
+// Storage function pointer types
+
+// Create storage for column data.
+// Parameters:
+//   space_ref   - Tablespace in which to create the storage
+//   trx_ref     - Transaction under which the creation runs
+//   col_len     - Column storage length
+//   arena_ctx   - Opaque arena handle passed to arena_alloc
+//   arena_alloc - Allocator for the storage context; memory is freed when
+//                 storage is dropped
+//   storage     - Output: newly created storage context
+//   error_msg   - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_create_func_t)(
+    vef_storage_space_ref_t space_ref, vef_storage_trx_ref_t trx_ref,
+    uint32_t col_len, void *arena_ctx, vef_storage_arena_func_t arena_alloc,
+    vef_storage_ctx_t **storage, char *error_msg);
+
+// Drop storage for column data.
+// Parameters:
+//   storage   - Storage context to drop
+//   trx_ref   - Transaction under which the drop runs
+//   error_msg - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_drop_func_t)(vef_storage_ctx_t *storage,
+                                             vef_storage_trx_ref_t trx_ref,
+                                             char *error_msg);
+
+// Load existing storage from a storage reference.
+// Parameters:
+//   storage_ref - Reference to the existing storage to load
+//   arena_ctx   - Opaque arena handle passed to arena_alloc
+//   arena_alloc - Allocator for the storage context; memory is freed when
+//                 storage is dropped
+//   storage     - Output: loaded storage context
+//   error_msg   - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_load_func_t)(
+    vef_storage_ref_t storage_ref, void *arena_ctx,
+    vef_storage_arena_func_t arena_alloc, vef_storage_ctx_t **storage,
+    char *error_msg);
+
+// Insert column data into column storage along with transaction reference.
+// Parameters:
+//   storage      - Storage context to insert into
+//   mctx         - Mini-transaction context
+//   trx_ref      - Transaction performing the insert
+//   col_data     - Column data to store
+//   rowid_prefix - Row identifier prefix
+//   col_ref      - Output: reference to the newly inserted column entry
+//   error_msg    - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_insert_func_t)(
+    vef_storage_ctx_t *storage, vef_storage_mtr_ref_t mctx,
+    vef_storage_trx_ref_t trx_ref, vef_storage_col_data_t col_data,
+    vef_storage_col_data_t rowid_prefix, vef_storage_col_ref_t *col_ref,
+    char *error_msg);
+
+// Fetch column data reference pointer acquiring appropriate latch.
+// Parameters:
+//   storage        - Storage context to read from
+//   mctx           - Mini-transaction context
+//   col_ref        - Reference to the column entry to fetch
+//   col_data       - Output: fetched column data
+//   rowid_prefix   - Output: row identifier prefix
+//   trx_ref        - Output: transaction that last wrote this entry
+//   delete_marked  - Output: whether the entry is marked for deletion
+//   error_msg      - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_select_func_t)(
+    vef_storage_ctx_t *storage, vef_storage_mtr_ref_t mctx,
+    vef_storage_col_ref_t col_ref, vef_storage_col_data_t *col_data,
+    vef_storage_col_data_t *rowid_prefix, vef_storage_trx_ref_t *trx_ref,
+    bool *delete_marked, char *error_msg);
+
+// Mark or unmark column as deleted by the transaction.
+// Parameters:
+//   storage     - Storage context
+//   mctx        - Mini-transaction context
+//   trx_ref     - Transaction performing the operation
+//   col_ref     - Reference to the column entry to mark
+//   delete_mark - true to mark as deleted, false to unmark
+//   error_msg   - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_mark_delete_func_t)(
+    vef_storage_ctx_t *storage, vef_storage_mtr_ref_t mctx,
+    vef_storage_trx_ref_t trx_ref, vef_storage_col_ref_t col_ref,
+    bool delete_mark, char *error_msg);
+
+// Remove column only if it has the matching transaction reference.
+// Parameters:
+//   storage   - Storage context
+//   mctx      - Mini-transaction context
+//   trx_ref   - Transaction to match with the record to purge
+//   col_ref   - Reference to the column entry to purge
+//   error_msg - Output: error description on failure (VEF_MAX_ERROR_LEN)
+// Returns false on success, true on error (writes to error_msg).
+typedef bool (*vef_type_storage_purge_func_t)(vef_storage_ctx_t *storage,
+                                              vef_storage_mtr_ref_t mctx,
+                                              vef_storage_trx_ref_t trx_ref,
+                                              vef_storage_col_ref_t col_ref,
+                                              char *error_msg);
+
+// Note: This structure cannot be modified or extended since it is a
+// sub-structure of vef_type_desc_t. If a new interface needs to be added,
+// in any future ABI version, we should create another structure like
+// vef_type_storage_intf_v2_t and append it to vef_type_desc_t.
+typedef struct {
+  vef_type_storage_create_func_t create;
+  vef_type_storage_drop_func_t drop;
+  vef_type_storage_load_func_t load;
+  vef_type_storage_insert_func_t insert;
+  vef_type_storage_select_func_t select;
+  vef_type_storage_mark_delete_func_t mark_delete;
+  vef_type_storage_purge_func_t purge;
+} vef_type_storage_intf_t;
+
 typedef struct {
   // protocol >= VEF_PROTOCOL_1
   vef_protocol_t protocol;
@@ -589,6 +754,10 @@ typedef struct {
   const char *decode_vdf_name;
   const char *compare_vdf_name;
   const char *hash_vdf_name;  // OPTIONAL (like hash_func)
+
+  // OPTIONAL: Storage interfaces. Implement if the columns of this type are
+  // stored by the extension.
+  vef_type_storage_intf_t storage_intf;
 } vef_type_desc_t;
 
 typedef struct {