Set tiktoken-c version to 0.10.0

Author: kojix2

Cargo.toml

@@ -18,7 +18,7 @@ default = []
 logging = ["log", "simple_logger"]  
 
 [dependencies]
-tiktoken-rs = "0.9"
+tiktoken-rs = "0.10.0"
 log = { version = "0.4", optional = true }  
 simple_logger = { version = "5.0", optional = true }
 libc = "0.2"

README.md

@@ -5,7 +5,7 @@
 [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/kojix2/tiktoken-c)
 
 - C API for [Tiktoken](https://github.com/openai/tiktoken), OpenAI's tokenizer
-- Compatible with [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) 0.9.1
+- Compatible with [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) 0.10.0
 
 ## Installation
 

src/corebpe.rs

@@ -83,10 +83,10 @@ pub extern "C" fn tiktoken_get_bpe_from_model(model: *const c_char) -> *mut Core
             }
         }
     };
-    let bpe = tiktoken_rs::get_bpe_from_model(model);
+    let bpe = tiktoken_rs::bpe_for_model(model);
     match bpe {
         Ok(bpe) => {
-            let boxed = Box::new(bpe);
+            let boxed = Box::new(bpe.clone());
             Box::into_raw(boxed)
         }
         Err(_) => {

src/lib.rs

@@ -61,7 +61,7 @@ pub extern "C" fn tiktoken_get_completion_max_tokens(
             }
         }
     };
-    match tiktoken_rs::get_completion_max_tokens(model, prompt) {
+    match tiktoken_rs::get_text_completion_max_tokens(model, prompt) {
         Ok(max_tokens) => max_tokens,
         Err(_) => {
             #[cfg(feature = "logging")]
@@ -162,6 +162,8 @@ pub extern "C" fn tiktoken_num_tokens_from_messages(
                 content: content,
                 name: name,
                 function_call: function_call,
+                tool_calls: Vec::new(),
+                refusal: None,
             });
         }
         messages_vec
@@ -258,6 +260,8 @@ pub extern "C" fn tiktoken_get_chat_completion_max_tokens(
                 content: content,
                 name: name,
                 function_call: function_call,
+                tool_calls: Vec::new(),
+                refusal: None,
             });
         }
         messages_vec
@@ -425,7 +429,6 @@ pub extern "C" fn tiktoken_corebpe_decode(
         return std::ptr::null_mut();
     }
     let tokens = unsafe { std::slice::from_raw_parts(tokens, num_tokens) };
-    let tokens = tokens.to_vec();
 
     let corebpe = unsafe { &mut *ptr };
     let decoded = corebpe.decode(tokens);