Use ReadOnlyNativeBuffer<byte> for AllocateTokenizer()

Author: budgetdevv

Native/src/lib.rs

@@ -231,11 +231,10 @@ impl TokenizeOutputOverflowedToken
 
 #[no_mangle]
 pub unsafe extern "C" fn allocate_tokenizer(
-    json_bytes_ptr: *const u8,
-    json_bytes_length: usize,
-) -> *mut Tokenizer
+    json_bytes: ReadOnlyBuffer<u8>)
+    -> *mut Tokenizer
 {
-    let json_bytes = slice::from_raw_parts(json_bytes_ptr, json_bytes_length);
+    let json_bytes = json_bytes.as_slice();
 
     let tokenizer = Tokenizer::from_bytes(json_bytes).unwrap();
 

Tokenizers.NET/Tokenizer.cs

@@ -256,14 +256,11 @@ public bool Truncate
 
         public Tokenizer()
         {
-            var rawTokenizerData = Config.RawTokenizerData.Buffer;
+            var rawTokenizerData = Config.RawTokenizerData.Buffer.AsReadOnly();
 
             Allocator = new();
 
-            TokenizerHandle = TokenizerNativeMethods.AllocateTokenizer(
-                rawTokenizerData.Ptr,
-                rawTokenizerData.Length
-            );
+            TokenizerHandle = TokenizerNativeMethods.AllocateTokenizer(rawTokenizerData);
         }
 
         [SkipLocalsInit]

Tokenizers.NET/TokenizerNativeMethods.cs

@@ -9,7 +9,7 @@ internal static unsafe partial class TokenizerNativeMethods
         private const string DLL_NAME = "tokenizers_net";
 
         [LibraryImport(DLL_NAME, EntryPoint = "allocate_tokenizer")]
-        public static partial nint AllocateTokenizer(byte* jsonBytesPtr, nuint jsonBytesLength);
+        public static partial nint AllocateTokenizer(ReadOnlyNativeBuffer<byte> jsonBytes);
 
         [LibraryImport(DLL_NAME, EntryPoint = "free_tokenizer")]
         public static partial void FreeTokenizer(nint tokenizerHandle);