Update onnxruntime-extensions to include InternLM2Tokenizer

- Bump extensions commit to 087953cd (includes InternLM2Tokenizer support)
- Remove local patch now that support is upstream

Ref: microsoft/onnxruntime-extensions#1023

Author: amdrajeevp1

cmake/deps.txt

@@ -14,7 +14,7 @@ pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.zip;f78029
 googletest;https://github.com/google/googletest/archive/530d5c8c84abd2a46f38583ee817743c9b3a42b4.zip;5e3a61db2aa975cfd0f97ba92c818744e7fa7034
 microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5
 directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v1.613.1.zip;47653509a3371eabb156360f42faf582f314bf2e
-onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;2fbe0ebbb3eb21199ab74c92b6edf3804d827998
+onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;087953cde6149e423c6848c40c3791264272706c
 
 # These two dependencies are for the optional constrained decoding feature (USE_GUIDANCE)
 llguidance;https://github.com/microsoft/llguidance.git;94fa39128ef184ffeda33845f6d333f332a34b4d

cmake/external/onnxruntime_external_deps.cmake

@@ -88,22 +88,6 @@ FetchContent_Declare(
 set(OCOS_BUILD_PRESET ort_genai)
 onnxruntime_fetchcontent_makeavailable(onnxruntime_extensions)
 
-# Add InternLM2Tokenizer support: same BPE/LLaMA format as LlamaTokenizer (see Hugging Face model card).
-# This allows exported models to keep the official tokenizer_config.json (InternLM2Tokenizer) per
-# https://huggingface.co/internlm/internlm2-1_8b/blob/main/tokenizer_config.json
-FetchContent_GetProperties(onnxruntime_extensions)
-if(onnxruntime_extensions_POPULATED)
-  set(ORTX_TOKENIZER_CONFIG "${onnxruntime_extensions_SOURCE_DIR}/operators/tokenizer/tokenizer_jsconfig.hpp")
-  if(EXISTS "${ORTX_TOKENIZER_CONFIG}")
-    file(READ "${ORTX_TOKENIZER_CONFIG}" ORTX_TOKENIZER_CONFIG_CONTENT)
-    string(REPLACE "{\"LlamaTokenizer\", TokenType::kBPE},"
-      "{\"LlamaTokenizer\", TokenType::kBPE},\n  {\"InternLM2Tokenizer\", TokenType::kBPE},"
-      ORTX_TOKENIZER_CONFIG_CONTENT "${ORTX_TOKENIZER_CONFIG_CONTENT}")
-    file(WRITE "${ORTX_TOKENIZER_CONFIG}" "${ORTX_TOKENIZER_CONFIG_CONTENT}")
-    message(STATUS "Patched onnxruntime_extensions: added InternLM2Tokenizer support")
-  endif()
-endif()
-
 list(APPEND EXTERNAL_LIBRARIES
   onnxruntime_extensions
   ocos_operators