Fix access violation exception

apsonawane · apsonawane · commit 0896cef741af · 2025-11-21T13:47:31.000-08:00
diff --git a/operators/tokenizer/ugm_kernels.hpp b/operators/tokenizer/ugm_kernels.hpp
@@ -522,7 +522,7 @@ struct SpmUgmTokenizer {
         ORTX_CXX_API_THROW("[UgmTok]Index out of array bounds in precompiled charsmap!", ORT_RUNTIME_EXCEPTION);
       }
       const char* prefix_replacement = &prefix_replacements_[longest_prefix_offset];
-      return {prefix_replacement, static_cast<int>(longest_prefix_length)};
+      return {prefix_replacement, static_cast<int>(strlen(prefix_replacement))};
     } else {
       // if yes, return this sequence unmodified
       size_t prefix_offset = ustring::UTF8Len(input_view[0]);

Original file line number	Diff line number	Diff line change
`@@ -522,7 +522,7 @@ struct SpmUgmTokenizer {`
`522`	`522`	`ORTX_CXX_API_THROW("[UgmTok]Index out of array bounds in precompiled charsmap!", ORT_RUNTIME_EXCEPTION);`
`523`	`523`	`}`
`524`	`524`	`const char* prefix_replacement = &prefix_replacements_[longest_prefix_offset];`
`525`		`- return {prefix_replacement, static_cast<int>(longest_prefix_length)};`
	`525`	`+ return {prefix_replacement, static_cast<int>(strlen(prefix_replacement))};`
`526`	`526`	`} else {`
`527`	`527`	`// if yes, return this sequence unmodified`
`528`	`528`	`size_t prefix_offset = ustring::UTF8Len(input_view[0]);`