vef: allow VDFs as type operations (#59)

Extend vef_type_desc_t (protocol 2) with encode/decode/compare/hash_vdf_name
fields so extensions can name a registered VDF as the implementation of each
type operation instead of providing a raw function pointer.

Server-side: resolve VDF names at extension registration, validate signatures
(including rejecting prerun/postrun hooks), and encapsulate fn-ptr vs VDF
dispatch in new EncodeOp/DecodeOp/CompareOp/HashOp wrapper classes.

These wrappers are registered in the TypeDescriptor; all call sites use
invoke() which hide the protocol differences

SDK: add overloaded .encode()/.decode()/.compare()/.hash() builder
methods, that take the VDF.  Convert vsql-complex and vsql-tvector
examples to the new style.

Author: malone-at-work

sql/sql_executor.cc

@@ -3981,9 +3981,12 @@ static bool cmp_field_value(Field *field, ptrdiff_t diff) {
   }
 
   // Use custom comparison function for custom types
-  if (auto cmp_func = villagesql::GetCompareFunc(*field)) {
-    return cmp_func(field->data_ptr(), value1_length, field->data_ptr() + diff,
-                    value2_length) != 0;
+  if (field->has_type_context()) {
+    if (auto result = villagesql::TryCompareCustomType(
+            *field->get_type_context(), field->data_ptr(), value1_length,
+            field->data_ptr() + diff, value2_length)) {
+      return *result != 0;
+    }
   }
 
   // Trailing space can't be skipped and length is different
@@ -4060,15 +4063,16 @@ ulonglong calc_field_hash(const Field *field, ulonglong *hash_val) {
     const Field_json *json_field = down_cast<const Field_json *>(field);
 
     crc = json_field->make_hash_key(*hash_val);
-  } else if (auto hash_fn = villagesql::GetHashFunc(*field)) {
+  } else if (auto hash = villagesql::TryComputeHash(*field, field->data_ptr(),
+                                                    field->data_length())) {
     // Custom type with custom hash function - use it.
     // The hash function may return a constant (for comparison-based dedup)
     // or canonicalize on the fly before hashing.
     // If we didn't use this here, the key_type being VARBINARY* would
     // cause us to fall to the else case and use the raw binary, which
     // would not work for some custom types that have multiple binary
     // representations for equivalent values (such as -0 and 0).
-    my_hash_combine(crc, hash_fn(field->data_ptr(), field->data_length()));
+    my_hash_combine(crc, *hash);
   } else if (field->key_type() == HA_KEYTYPE_TEXT ||
              field->key_type() == HA_KEYTYPE_VARTEXT1 ||
              field->key_type() == HA_KEYTYPE_VARTEXT2) {

storage/innobase/rem/rem0cmp.cc

@@ -416,7 +416,7 @@ inline int cmp_data(ulint mtype, ulint prtype, bool is_asc, const byte *data1,
   }
 
   if (custom_column) {
-    int ret = custom_column->compare()(data1, len1, data2, len2);
+    int ret = custom_column->compare(data1, len1, data2, len2);
     return (is_asc ? ret : -ret);
   }
 

storage/innobase/villagesql/custom_column.cc

@@ -25,6 +25,7 @@
 #include "storage/innobase/include/dict0mem.h"
 #include "storage/innobase/include/ha_prototypes.h"
 #include "villagesql/include/error.h"
+#include "villagesql/schema/descriptor/type_descriptor.h"
 #include "villagesql/schema/victionary_client.h"
 #include "villagesql/types/util.h"
 
@@ -48,20 +49,25 @@ Custom_column::Info Custom_column::get_from_position(const dict_index_t *index,
   return get_from_field(index->get_field(position));
 }
 
+int Custom_column::compare(const unsigned char *data1, size_t len1,
+                           const unsigned char *data2, size_t len2) const {
+  return compare_op_.invoke(data1, len1, data2, len2);
+}
+
 void Custom_column::load(dict_table_t *table, dict_col_t *col,
                          const Field *sql_field, const dd::Column *) {
   ut_ad(!col->custom_column);
-  auto *custom_compare = villagesql::GetCompareFunc(*sql_field);
 
-  if (!custom_compare) return;
+  if (!sql_field->has_type_context()) return;
 
   void *mem = static_cast<Custom_column *>(
       mem_heap_zalloc(table->heap, sizeof(Custom_column)));
 
   static_assert(std::is_trivially_destructible<Custom_column>::value,
                 "Custom_column must be trivially destructible");
 
-  col->custom_column = new (mem) Custom_column(custom_compare);
+  col->custom_column = new (mem)
+      Custom_column(sql_field->get_type_context()->descriptor()->compare_op());
 }
 
 void Custom_column::load_all(dict_table_t *table) {

storage/innobase/villagesql/custom_column.h

@@ -18,7 +18,7 @@
 #define STORAGE_INNOBASE_VILLAGESQL_CUSTOM_COLUMN_H_
 
 #include <utility>
-#include "villagesql/sdk/include/villagesql/abi/types.h"
+#include "villagesql/types/type_op.h"
 
 // Forward declarations
 struct dict_table_t;
@@ -38,11 +38,14 @@ namespace innodb {
 
 class Custom_column {
  public:
-  using Compare = vef_compare_func_t;
   using Info = std::pair<Custom_column *, bool>;
 
-  Custom_column(Compare compare_fn) : compare_fn_(compare_fn) {}
-  inline Compare compare() const { return compare_fn_; }
+  explicit Custom_column(CompareOp compare_op)
+      : compare_op_(std::move(compare_op)) {}
+
+  // Compare two values using the registered compare implementation.
+  int compare(const unsigned char *data1, size_t len1,
+              const unsigned char *data2, size_t len2) const;
 
   // Get custom column descriptor and ascending flag from index position.
   static Info get_from_position(const dict_index_t *index, size_t position);
@@ -71,8 +74,7 @@ class Custom_column {
   static void load_all(dict_table_t *table);
 
  private:
-  // Custom column type comparison function defined by the extension.
-  Compare compare_fn_{nullptr};
+  CompareOp compare_op_;
 };
 }  // namespace innodb
 }  // namespace villagesql

storage/temptable/include/temptable/cell_calculator.h

@@ -197,10 +197,11 @@ inline size_t Cell_calculator::hash(const Cell &cell) const {
   }
 
   // Custom types: check for custom hash function.
-  // If hash_func is provided, use it. If nullptr, binary hash is safe
+  // If provided, use it. If absent, binary hash is safe
   // (encode canonicalizes equivalent values like -0.0 → +0.0).
-  if (auto hash_fn = villagesql::GetHashFunc(*m_mysql_field)) {
-    return hash_fn(data, data_length);
+  if (auto hash =
+          villagesql::TryComputeHash(*m_mysql_field, data, data_length)) {
+    return *hash;
   }
 
   /*

unittest/gunit/villagesql/CMakeLists.txt

@@ -18,6 +18,7 @@
 # List of all VillageSQL unit tests
 SET(VILLAGESQL_UNIT_TESTS
   abi_v1_check-t
+  abi_v2_check-t
   semver-t
   type_context-t
   type_descriptor-t
@@ -58,6 +59,7 @@ ENDMACRO()
 
 # Add all VillageSQL unit tests
 ADD_VILLAGESQL_TEST(abi_v1_check-t)
+ADD_VILLAGESQL_TEST(abi_v2_check-t)
 ADD_VILLAGESQL_TEST(semver-t)
 ADD_VILLAGESQL_TEST(type_context-t)
 ADD_VILLAGESQL_TEST(type_descriptor-t)

unittest/gunit/villagesql/abi_v2_check-t.cc

@@ -0,0 +1,71 @@
+/* Copyright (c) 2026 VillageSQL Contributors
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ */
+
+// ABI v2 Compile-time Layout Checks
+//
+// Verifies the binary layout of protocol-2 fields in the main SDK headers.
+// A failure here means the server has broken ABI compatibility with extensions
+// compiled against VEF_PROTOCOL_2 headers.
+//
+// Sizes and offsets were derived from the 64-bit LP64 layout (Linux/macOS
+// x86-64 and ARM64) where:
+//   pointer/size_t = 8 bytes (align 8), int/unsigned int = 4 bytes (align 4),
+//   bool = 1 byte, double = 8 bytes, long long = 8 bytes,
+//   enum : int / enum : unsigned int = 4 bytes (align 4).
+
+#include <gtest/gtest.h>
+
+#include <cstddef>
+
+#include "villagesql/sdk/include/villagesql/abi/types.h"
+
+// ---------------------------------------------------------------------------
+// vef_type_desc_t (protocol >= VEF_PROTOCOL_2 fields)
+//
+// Full layout including v1 and v2 fields:
+//   vef_protocol_t protocol;                        // +0
+//   [4 bytes padding]
+//   const char *name;                               // +8
+//   int64_t persisted_length;                       // +16
+//   int64_t max_decode_buffer_length;               // +24
+//   vef_encode_func_t encode_func;                  // +32  (protocol >= 1)
+//   vef_decode_func_t decode_func;                  // +40  (protocol >= 1)
+//   vef_compare_func_t compare_func;                // +48  (protocol >= 1)
+//   vef_hash_func_t hash_func;                      // +56  (protocol >= 1)
+//   vef_type_int_to_params_func_t int_to_params;    // +64  (protocol >= 2)
+//   vef_type_resolve_params_func_t resolve_params;  // +72  (protocol >= 2)
+//   const char *encode_vdf_name;                    // +80  (protocol >= 2)
+//   const char *decode_vdf_name;                    // +88  (protocol >= 2)
+//   const char *compare_vdf_name;                   // +96  (protocol >= 2)
+//   const char *hash_vdf_name;                      // +104 (protocol >= 2)
+// ---------------------------------------------------------------------------
+static_assert(sizeof(vef_type_desc_t) == 112,
+              "ABI v2 break: vef_type_desc_t size changed");
+static_assert(offsetof(vef_type_desc_t, int_to_params) == 64,
+              "ABI v2 break: vef_type_desc_t::int_to_params offset changed");
+static_assert(offsetof(vef_type_desc_t, resolve_params) == 72,
+              "ABI v2 break: vef_type_desc_t::resolve_params offset changed");
+static_assert(offsetof(vef_type_desc_t, encode_vdf_name) == 80,
+              "ABI v2 break: vef_type_desc_t::encode_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, decode_vdf_name) == 88,
+              "ABI v2 break: vef_type_desc_t::decode_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, compare_vdf_name) == 96,
+              "ABI v2 break: vef_type_desc_t::compare_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, hash_vdf_name) == 104,
+              "ABI v2 break: vef_type_desc_t::hash_vdf_name offset changed");
+
+// Placeholder test so the binary links and runs.
+TEST(AbiV2Check, StaticAssertsPass) {}

unittest/gunit/villagesql/type_context-t.cc

@@ -159,7 +159,8 @@ class TypeContextTest : public ::testing::Test {
 TEST_F(TypeContextTest, FixedLengthTypeUsesDescriptorValues) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("COMPLEX", "test_ext", "1.0.0"), 1, 16, 256,
-      dummy_encode, dummy_decode, dummy_compare);
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare));
   villagesql::TypeContextKey key("COMPLEX", "test_ext", "1.0.0");
   villagesql::TypeContext ctx(key, &desc);
 
@@ -170,7 +171,8 @@ TEST_F(TypeContextTest, FixedLengthTypeUsesDescriptorValues) {
 TEST_F(TypeContextTest, ParameterizedTypeUsesResolvedValues) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("VVECTOR", "test_ext", "1.0.0"), 1, -1, 0,
-      dummy_encode, dummy_decode, dummy_compare, nullptr, nullptr,
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare), std::nullopt, nullptr,
       resolve_params_ok);
   villagesql::TypeParameters params({{"dimension", "1536"}});
   villagesql::TypeContextKey key(
@@ -185,7 +187,8 @@ TEST_F(TypeContextTest, ParameterizedTypeUsesResolvedValues) {
 TEST_F(TypeContextTest, ResolveParamsFailureFallsBackToDescriptor) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("VVECTOR", "test_ext", "1.0.0"), 1, -1, 0,
-      dummy_encode, dummy_decode, dummy_compare, nullptr, nullptr,
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare), std::nullopt, nullptr,
       resolve_params_fail);
   villagesql::TypeParameters params({{"dimension", "1536"}});
   villagesql::TypeContextKey key(
@@ -200,7 +203,8 @@ TEST_F(TypeContextTest, ResolveParamsFailureFallsBackToDescriptor) {
 TEST_F(TypeContextTest, EmptyParamsSkipsResolveCallback) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("VVECTOR", "test_ext", "1.0.0"), 1, -1, 0,
-      dummy_encode, dummy_decode, dummy_compare, nullptr, nullptr,
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare), std::nullopt, nullptr,
       resolve_params_fail);
   // No parameters — should use descriptor values directly, not call
   // resolve_params (which would fail)

unittest/gunit/villagesql/type_descriptor-t.cc

@@ -83,7 +83,8 @@ TEST_F(TypeDescriptorTest, Construction) {
       1,    // implementation_type
       16,   // persisted_length
       256,  // max_decode_buffer_length
-      dummy_encode, dummy_decode, dummy_compare, dummy_hash);
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare), villagesql::HashOp(dummy_hash));
 
   // Check identity fields
   EXPECT_EQ(desc.type_name(), "MYTYPE");
@@ -98,23 +99,20 @@ TEST_F(TypeDescriptorTest, Construction) {
   EXPECT_EQ(desc.persisted_length(), 16);
   EXPECT_EQ(desc.max_decode_buffer_length(), 256);
 
-  // Check function pointers
-  EXPECT_EQ(desc.encode(), dummy_encode);
-  EXPECT_EQ(desc.decode(), dummy_decode);
-  EXPECT_EQ(desc.compare(), dummy_compare);
-  EXPECT_EQ(desc.hash(), dummy_hash);
+  // Verify ops are set and dispatch to wrapped functions
+  EXPECT_EQ(desc.compare_op().invoke(nullptr, 0, nullptr, 0), 0);
+  EXPECT_EQ(desc.hash_op()->invoke(nullptr, 0), 42u);
 }
 
 // Test TypeDescriptor with nullptr hash (optional)
 TEST_F(TypeDescriptorTest, ConstructionWithNullHash) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("NOHASH", "ext", "1.0"), 0, 8, 64,
-      dummy_encode, dummy_decode, dummy_compare, nullptr);
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare));
 
-  EXPECT_EQ(desc.hash(), nullptr);
-  EXPECT_NE(desc.encode(), nullptr);
-  EXPECT_NE(desc.decode(), nullptr);
-  EXPECT_NE(desc.compare(), nullptr);
+  EXPECT_FALSE(desc.hash_op().has_value());
+  EXPECT_EQ(desc.compare_op().invoke(nullptr, 0, nullptr, 0), 0);
   EXPECT_EQ(desc.int_to_params(), nullptr);
   EXPECT_EQ(desc.resolve_params(), nullptr);
 }
@@ -124,7 +122,8 @@ TEST_F(TypeDescriptorTest, ConstructionWithNullHash) {
 TEST_F(TypeDescriptorTest, KeyTypeCompatibility) {
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("TEST", "ext", "1.0"), 0, 8, 64,
-      dummy_encode, dummy_decode, dummy_compare);
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare));
 
   // Verify key_type is TypeDescriptorKey
   static_assert(std::is_same_v<villagesql::TypeDescriptor::key_type,
@@ -171,7 +170,8 @@ TEST_F(TypeDescriptorTest, ConstructionWithParamFunctions) {
       1,   // implementation_type
       -1,  // persisted_length (variable-length)
       0,   // max_decode_buffer_length (determined by params)
-      dummy_encode, dummy_decode, dummy_compare, dummy_hash,
+      villagesql::EncodeOp(dummy_encode), villagesql::DecodeOp(dummy_decode),
+      villagesql::CompareOp(dummy_compare), villagesql::HashOp(dummy_hash),
       dummy_int_to_params, dummy_resolve_params);
 
   EXPECT_EQ(desc.persisted_length(), -1);

unittest/gunit/villagesql/victionary_client-t.cc

@@ -1657,7 +1657,8 @@ TEST_F(VictionaryClientTest, TypeDescriptorOperations) {
       1,    // implementation_type
       16,   // persisted_length
       256,  // max_decode_buffer_length
-      test_encode, test_decode, test_compare);
+      villagesql::EncodeOp(test_encode), villagesql::DecodeOp(test_decode),
+      villagesql::CompareOp(test_compare));
 
   {
     auto guard = client_->get_write_lock();
@@ -1697,9 +1698,6 @@ TEST_F(VictionaryClientTest, TypeDescriptorOperations) {
     EXPECT_EQ(committed->implementation_type(), 1);
     EXPECT_EQ(committed->persisted_length(), 16);
     EXPECT_EQ(committed->max_decode_buffer_length(), 256);
-    EXPECT_EQ(committed->encode(), test_encode);
-    EXPECT_EQ(committed->decode(), test_decode);
-    EXPECT_EQ(committed->compare(), test_compare);
   }
 }
 
@@ -1709,7 +1707,8 @@ TEST_F(VictionaryClientTest, TypeDescriptorRollback) {
 
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("ROLLBACK_TYPE", "ext", "1.0"), 0, 8, 64,
-      test_encode, test_decode, test_compare);
+      villagesql::EncodeOp(test_encode), villagesql::DecodeOp(test_decode),
+      villagesql::CompareOp(test_compare));
 
   {
     auto guard = client_->get_write_lock();
@@ -1788,7 +1787,8 @@ TEST_F(VictionaryClientTest, AcquireKeepsEntryAlive) {
 
   // Create and commit a TypeDescriptor entry
   TypeDescriptorKey key("REFCOUNT_TYPE", "ext", "1.0");
-  TypeDescriptor entry(key, 0, 42, 0, nullptr, nullptr, nullptr, nullptr);
+  TypeDescriptor entry(key, 0, 42, 0, EncodeOp(test_encode),
+                       DecodeOp(test_decode), CompareOp(test_compare));
 
   {
     auto guard = client_->get_write_lock();
@@ -1912,9 +1912,7 @@ TEST_F(VictionaryClientTest, AcquireOrCreateNew) {
   THD *fake_thd = reinterpret_cast<THD *>(0xA0C1);
 
   // Create a TypeDescriptor for the type
-  TypeDescriptor type_desc(
-      TypeDescriptorKey("AOTEST_TYPE", "test_ext", "1.0.0"), 0, 16, 0, nullptr,
-      nullptr, nullptr, nullptr);
+  TypeDescriptor type_desc(TypeDescriptorKey("AOTEST_TYPE", "test_ext", "1.0.0"));
 
   {
     auto guard = client_->get_write_lock();
@@ -1962,9 +1960,7 @@ TEST_F(VictionaryClientTest, AcquireOrCreateExisting) {
   THD *fake_thd = reinterpret_cast<THD *>(0xA0C2);
 
   // Create a TypeDescriptor for the type
-  TypeDescriptor type_desc(
-      TypeDescriptorKey("AOTEST_TYPE2", "test_ext", "1.0.0"), 0, 32, 0, nullptr,
-      nullptr, nullptr, nullptr);
+  TypeDescriptor type_desc(TypeDescriptorKey("AOTEST_TYPE2", "test_ext", "1.0.0"));
 
   {
     auto guard = client_->get_write_lock();
@@ -2053,8 +2049,7 @@ TEST_F(VictionaryClientTest, AcquireOrCreateWithParameters) {
   THD *fake_thd = reinterpret_cast<THD *>(0xA0C3);
 
   // Create a TypeDescriptor for the type
-  TypeDescriptor type_desc(TypeDescriptorKey("VECTOR", "vector_ext", "2.0.0"),
-                           0, 64, 0, nullptr, nullptr, nullptr, nullptr);
+  TypeDescriptor type_desc(TypeDescriptorKey("VECTOR", "vector_ext", "2.0.0"));
 
   {
     auto guard = client_->get_write_lock();