vef: allow VDFs as type functions

Extend vef_type_desc_t (protocol 2) with encode/decode/compare/hash_vdf_name
fields so extensions can name a registered VDF as the implementation of each
type operation instead of providing a raw function pointer.

Server-side: resolve VDF names at extension registration, validate signatures
(including rejecting prerun/postrun hooks), and encapsulate fn-ptr vs VDF
dispatch in new EncodeOp/DecodeOp/CompareOp/HashOp wrapper classes.

These wrappers are registered in the TypeDescriptor; all call sites use
invoke() which hide the protocol differences

SDK: add overloaded .encode()/.decode()/.compare()/.hash() builder
methods, that take the VDF.  Convert vsql-complex and vsql-tvector
examples to the new style.

Author: malone-at-work

sql/sql_executor.cc

@@ -3981,9 +3981,12 @@ static bool cmp_field_value(Field *field, ptrdiff_t diff) {
   }
 
   // Use custom comparison function for custom types
-  if (auto cmp_func = villagesql::GetCompareFunc(*field)) {
-    return cmp_func(field->data_ptr(), value1_length, field->data_ptr() + diff,
-                    value2_length) != 0;
+  if (field->has_type_context()) {
+    if (auto result = villagesql::TryCompareCustomType(
+            *field->get_type_context(), field->data_ptr(), value1_length,
+            field->data_ptr() + diff, value2_length)) {
+      return *result != 0;
+    }
   }
 
   // Trailing space can't be skipped and length is different
@@ -4060,15 +4063,16 @@ ulonglong calc_field_hash(const Field *field, ulonglong *hash_val) {
     const Field_json *json_field = down_cast<const Field_json *>(field);
 
     crc = json_field->make_hash_key(*hash_val);
-  } else if (auto hash_fn = villagesql::GetHashFunc(*field)) {
+  } else if (auto hash = villagesql::TryComputeHash(*field, field->data_ptr(),
+                                                    field->data_length())) {
     // Custom type with custom hash function - use it.
     // The hash function may return a constant (for comparison-based dedup)
     // or canonicalize on the fly before hashing.
     // If we didn't use this here, the key_type being VARBINARY* would
     // cause us to fall to the else case and use the raw binary, which
     // would not work for some custom types that have multiple binary
     // representations for equivalent values (such as -0 and 0).
-    my_hash_combine(crc, hash_fn(field->data_ptr(), field->data_length()));
+    my_hash_combine(crc, *hash);
   } else if (field->key_type() == HA_KEYTYPE_TEXT ||
              field->key_type() == HA_KEYTYPE_VARTEXT1 ||
              field->key_type() == HA_KEYTYPE_VARTEXT2) {

storage/innobase/rem/rem0cmp.cc

@@ -416,7 +416,7 @@ inline int cmp_data(ulint mtype, ulint prtype, bool is_asc, const byte *data1,
   }
 
   if (custom_column) {
-    int ret = custom_column->compare()(data1, len1, data2, len2);
+    int ret = custom_column->compare(data1, len1, data2, len2);
     return (is_asc ? ret : -ret);
   }
 

storage/innobase/villagesql/custom_column.cc

@@ -25,6 +25,7 @@
 #include "storage/innobase/include/dict0mem.h"
 #include "storage/innobase/include/ha_prototypes.h"
 #include "villagesql/include/error.h"
+#include "villagesql/schema/descriptor/type_descriptor.h"
 #include "villagesql/schema/victionary_client.h"
 #include "villagesql/types/util.h"
 
@@ -48,20 +49,25 @@ Custom_column::Info Custom_column::get_from_position(const dict_index_t *index,
   return get_from_field(index->get_field(position));
 }
 
+int Custom_column::compare(const unsigned char *data1, size_t len1,
+                           const unsigned char *data2, size_t len2) const {
+  return compare_op_.invoke(data1, len1, data2, len2);
+}
+
 void Custom_column::load(dict_table_t *table, dict_col_t *col,
                          const Field *sql_field, const dd::Column *) {
   ut_ad(!col->custom_column);
-  auto *custom_compare = villagesql::GetCompareFunc(*sql_field);
 
-  if (!custom_compare) return;
+  if (!sql_field->has_type_context()) return;
 
   void *mem = static_cast<Custom_column *>(
       mem_heap_zalloc(table->heap, sizeof(Custom_column)));
 
   static_assert(std::is_trivially_destructible<Custom_column>::value,
                 "Custom_column must be trivially destructible");
 
-  col->custom_column = new (mem) Custom_column(custom_compare);
+  col->custom_column = new (mem)
+      Custom_column(sql_field->get_type_context()->descriptor()->compare_op());
 }
 
 void Custom_column::load_all(dict_table_t *table) {

storage/innobase/villagesql/custom_column.h

@@ -18,7 +18,7 @@
 #define STORAGE_INNOBASE_VILLAGESQL_CUSTOM_COLUMN_H_
 
 #include <utility>
-#include "villagesql/sdk/include/villagesql/abi/types.h"
+#include "villagesql/types/type_op.h"
 
 // Forward declarations
 struct dict_table_t;
@@ -38,11 +38,14 @@ namespace innodb {
 
 class Custom_column {
  public:
-  using Compare = vef_compare_func_t;
   using Info = std::pair<Custom_column *, bool>;
 
-  Custom_column(Compare compare_fn) : compare_fn_(compare_fn) {}
-  inline Compare compare() const { return compare_fn_; }
+  explicit Custom_column(CompareOp compare_op)
+      : compare_op_(std::move(compare_op)) {}
+
+  // Compare two values using the registered compare implementation.
+  int compare(const unsigned char *data1, size_t len1,
+              const unsigned char *data2, size_t len2) const;
 
   // Get custom column descriptor and ascending flag from index position.
   static Info get_from_position(const dict_index_t *index, size_t position);
@@ -71,8 +74,7 @@ class Custom_column {
   static void load_all(dict_table_t *table);
 
  private:
-  // Custom column type comparison function defined by the extension.
-  Compare compare_fn_{nullptr};
+  CompareOp compare_op_;
 };
 }  // namespace innodb
 }  // namespace villagesql

storage/temptable/include/temptable/cell_calculator.h

@@ -197,10 +197,11 @@ inline size_t Cell_calculator::hash(const Cell &cell) const {
   }
 
   // Custom types: check for custom hash function.
-  // If hash_func is provided, use it. If nullptr, binary hash is safe
+  // If provided, use it. If absent, binary hash is safe
   // (encode canonicalizes equivalent values like -0.0 → +0.0).
-  if (auto hash_fn = villagesql::GetHashFunc(*m_mysql_field)) {
-    return hash_fn(data, data_length);
+  if (auto hash =
+          villagesql::TryComputeHash(*m_mysql_field, data, data_length)) {
+    return *hash;
   }
 
   /*

unittest/gunit/villagesql/CMakeLists.txt

@@ -18,6 +18,7 @@
 # List of all VillageSQL unit tests
 SET(VILLAGESQL_UNIT_TESTS
   abi_v1_check-t
+  abi_v2_check-t
   semver-t
   type_context-t
   type_descriptor-t
@@ -58,6 +59,7 @@ ENDMACRO()
 
 # Add all VillageSQL unit tests
 ADD_VILLAGESQL_TEST(abi_v1_check-t)
+ADD_VILLAGESQL_TEST(abi_v2_check-t)
 ADD_VILLAGESQL_TEST(semver-t)
 ADD_VILLAGESQL_TEST(type_context-t)
 ADD_VILLAGESQL_TEST(type_descriptor-t)

unittest/gunit/villagesql/abi_v2_check-t.cc

@@ -0,0 +1,71 @@
+/* Copyright (c) 2026 VillageSQL Contributors
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ */
+
+// ABI v2 Compile-time Layout Checks
+//
+// Verifies the binary layout of protocol-2 fields in the main SDK headers.
+// A failure here means the server has broken ABI compatibility with extensions
+// compiled against VEF_PROTOCOL_2 headers.
+//
+// Sizes and offsets were derived from the 64-bit LP64 layout (Linux/macOS
+// x86-64 and ARM64) where:
+//   pointer/size_t = 8 bytes (align 8), int/unsigned int = 4 bytes (align 4),
+//   bool = 1 byte, double = 8 bytes, long long = 8 bytes,
+//   enum : int / enum : unsigned int = 4 bytes (align 4).
+
+#include <gtest/gtest.h>
+
+#include <cstddef>
+
+#include "villagesql/sdk/include/villagesql/abi/types.h"
+
+// ---------------------------------------------------------------------------
+// vef_type_desc_t (protocol >= VEF_PROTOCOL_2 fields)
+//
+// Full layout including v1 and v2 fields:
+//   vef_protocol_t protocol;                        // +0
+//   [4 bytes padding]
+//   const char *name;                               // +8
+//   int64_t persisted_length;                       // +16
+//   int64_t max_decode_buffer_length;               // +24
+//   vef_encode_func_t encode_func;                  // +32  (protocol >= 1)
+//   vef_decode_func_t decode_func;                  // +40  (protocol >= 1)
+//   vef_compare_func_t compare_func;                // +48  (protocol >= 1)
+//   vef_hash_func_t hash_func;                      // +56  (protocol >= 1)
+//   vef_type_int_to_params_func_t int_to_params;    // +64  (protocol >= 2)
+//   vef_type_resolve_params_func_t resolve_params;  // +72  (protocol >= 2)
+//   const char *encode_vdf_name;                    // +80  (protocol >= 2)
+//   const char *decode_vdf_name;                    // +88  (protocol >= 2)
+//   const char *compare_vdf_name;                   // +96  (protocol >= 2)
+//   const char *hash_vdf_name;                      // +104 (protocol >= 2)
+// ---------------------------------------------------------------------------
+static_assert(sizeof(vef_type_desc_t) == 112,
+              "ABI v2 break: vef_type_desc_t size changed");
+static_assert(offsetof(vef_type_desc_t, int_to_params) == 64,
+              "ABI v2 break: vef_type_desc_t::int_to_params offset changed");
+static_assert(offsetof(vef_type_desc_t, resolve_params) == 72,
+              "ABI v2 break: vef_type_desc_t::resolve_params offset changed");
+static_assert(offsetof(vef_type_desc_t, encode_vdf_name) == 80,
+              "ABI v2 break: vef_type_desc_t::encode_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, decode_vdf_name) == 88,
+              "ABI v2 break: vef_type_desc_t::decode_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, compare_vdf_name) == 96,
+              "ABI v2 break: vef_type_desc_t::compare_vdf_name offset changed");
+static_assert(offsetof(vef_type_desc_t, hash_vdf_name) == 104,
+              "ABI v2 break: vef_type_desc_t::hash_vdf_name offset changed");
+
+// Placeholder test so the binary links and runs.
+TEST(AbiV2Check, StaticAssertsPass) {}

unittest/gunit/villagesql/victionary_client-t.cc

@@ -1657,7 +1657,8 @@ TEST_F(VictionaryClientTest, TypeDescriptorOperations) {
       1,    // implementation_type
       16,   // persisted_length
       256,  // max_decode_buffer_length
-      test_encode, test_decode, test_compare);
+      villagesql::EncodeOp(test_encode), villagesql::DecodeOp(test_decode),
+      villagesql::CompareOp(test_compare));
 
   {
     auto guard = client_->get_write_lock();
@@ -1697,9 +1698,6 @@ TEST_F(VictionaryClientTest, TypeDescriptorOperations) {
     EXPECT_EQ(committed->implementation_type(), 1);
     EXPECT_EQ(committed->persisted_length(), 16);
     EXPECT_EQ(committed->max_decode_buffer_length(), 256);
-    EXPECT_EQ(committed->encode(), test_encode);
-    EXPECT_EQ(committed->decode(), test_decode);
-    EXPECT_EQ(committed->compare(), test_compare);
   }
 }
 
@@ -1709,7 +1707,8 @@ TEST_F(VictionaryClientTest, TypeDescriptorRollback) {
 
   villagesql::TypeDescriptor desc(
       villagesql::TypeDescriptorKey("ROLLBACK_TYPE", "ext", "1.0"), 0, 8, 64,
-      test_encode, test_decode, test_compare);
+      villagesql::EncodeOp(test_encode), villagesql::DecodeOp(test_decode),
+      villagesql::CompareOp(test_compare));
 
   {
     auto guard = client_->get_write_lock();
@@ -1788,7 +1787,7 @@ TEST_F(VictionaryClientTest, AcquireKeepsEntryAlive) {
 
   // Create and commit a TypeDescriptor entry
   TypeDescriptorKey key("REFCOUNT_TYPE", "ext", "1.0");
-  TypeDescriptor entry(key, 0, 42, 0, nullptr, nullptr, nullptr, nullptr);
+  TypeDescriptor entry(key);
 
   {
     auto guard = client_->get_write_lock();

villagesql/examples/vsql-complex/src/complex.cc

@@ -214,6 +214,22 @@ size_t hash_complex2(const unsigned char *data, size_t len) {
   return fnv1a_hash(canonical, kComplexSize);
 }
 
+// Compare VDF: complex_compare(a, b) -> INT
+void complex_compare_impl(vef_context_t *ctx, vef_invalue_t *in_l,
+                          vef_invalue_t *in_r, vef_vdf_result_t *out) {
+  out->int_value = cmp_complex(in_l->bin_value, in_l->bin_len, in_r->bin_value,
+                               in_r->bin_len);
+  out->type = VEF_RESULT_VALUE;
+}
+
+// Hash VDF: complex2_hash(c) -> INT
+void complex2_hash_impl(vef_context_t *ctx, vef_invalue_t *in,
+                        vef_vdf_result_t *out) {
+  out->int_value =
+      static_cast<long long>(hash_complex2(in->bin_value, in->bin_len));
+  out->type = VEF_RESULT_VALUE;
+}
+
 std::optional<Complex> TryLoadFromInValue(const vef_invalue_t *v) {
   if (v->bin_len != kComplexSize) {
     return std::nullopt;
@@ -446,21 +462,21 @@ VEF_GENERATE_ENTRY_POINTS(
         .type(make_type(COMPLEX)
                   .persisted_length(kComplexSize)
                   .max_decode_buffer_length(64)
-                  .encode(&encode_complex)
-                  .decode(&decode_complex)
-                  .compare(&cmp_complex)
+                  .encode("complex_from_string")
+                  .decode("complex_to_string")
+                  .compare("complex_compare")
                   .build())
         // COMPLEX2 type without canonicalization (preserves -0.0)
         // Requires custom hash that canonicalizes -0 to +0 before hashing
         .type(make_type(COMPLEX2)
                   .persisted_length(kComplexSize)
                   .max_decode_buffer_length(64)
-                  .encode(&encode_complex2)
-                  .decode(&decode_complex)
-                  .compare(&cmp_complex)
-                  .hash(&hash_complex2)
+                  .encode("complex2_from_string")
+                  .decode("complex2_to_string")
+                  .compare("complex2_compare")
+                  .hash("complex2_hash")
                   .build())
-        // Type conversion functions
+        // Type conversion functions (also serve as encode/decode VDFs)
         .func(make_func("complex_from_string")
                   .from_string<&encode_complex>(COMPLEX))
         .func(
@@ -469,6 +485,24 @@ VEF_GENERATE_ENTRY_POINTS(
                   .from_string<&encode_complex2>(COMPLEX2))
         .func(make_func("complex2_to_string")
                   .to_string<&decode_complex>(COMPLEX2))
+        // Compare and hash VDFs
+        .func(make_func<&complex_compare_impl>("complex_compare")
+                  .returns(INT)
+                  .param(COMPLEX)
+                  .param(COMPLEX)
+                  .deterministic()
+                  .build())
+        .func(make_func<&complex_compare_impl>("complex2_compare")
+                  .returns(INT)
+                  .param(COMPLEX2)
+                  .param(COMPLEX2)
+                  .deterministic()
+                  .build())
+        .func(make_func<&complex2_hash_impl>("complex2_hash")
+                  .returns(INT)
+                  .param(COMPLEX2)
+                  .deterministic()
+                  .build())
         // Arithmetic functions
         .func(make_func<&complex_add_impl>("complex_add")
                   .returns(COMPLEX)

villagesql/examples/vsql-tvector/src/tvector.cc

@@ -224,13 +224,34 @@ int tvector_compare(const unsigned char *data1, size_t len1,
   return 0;
 }
 
-VEF_GENERATE_ENTRY_POINTS(make_extension("vsql_tvector", "0.0.1")
-                              .type(make_type("TVECTOR")
-                                        .persisted_length(-1)
-                                        .max_decode_buffer_length(16)
-                                        .encode(&tvector_encode)
-                                        .decode(&tvector_decode)
-                                        .compare(&tvector_compare)
-                                        .int_to_params(&tvector_int_to_params)
-                                        .resolve_params(&tvector_resolve_params)
-                                        .build()))
+// Compare VDF: tvector_compare(a, b) -> INT
+void tvector_compare_impl(vef_context_t *ctx, vef_invalue_t *in_l,
+                          vef_invalue_t *in_r, vef_vdf_result_t *out) {
+  out->int_value = tvector_compare(in_l->bin_value, in_l->bin_len,
+                                   in_r->bin_value, in_r->bin_len);
+  out->type = VEF_RESULT_VALUE;
+}
+
+constexpr const char *TVECTOR = "TVECTOR";
+
+VEF_GENERATE_ENTRY_POINTS(
+    make_extension("vsql_tvector", "0.0.1")
+        .type(make_type(TVECTOR)
+                  .persisted_length(-1)
+                  .max_decode_buffer_length(16)
+                  .encode("tvector_from_string")
+                  .decode("tvector_to_string")
+                  .compare("tvector_compare")
+                  .int_to_params(&tvector_int_to_params)
+                  .resolve_params(&tvector_resolve_params)
+                  .build())
+        .func(make_func("tvector_from_string")
+                  .from_string<&tvector_encode>(TVECTOR))
+        .func(
+            make_func("tvector_to_string").to_string<&tvector_decode>(TVECTOR))
+        .func(make_func<&tvector_compare_impl>("tvector_compare")
+                  .returns(INT)
+                  .param(TVECTOR)
+                  .param(TVECTOR)
+                  .deterministic()
+                  .build()))