Replace deprecated std::wstring_convert with manual UTF-8 codec (k2-fsa#3215)

* Replace deprecated std::istrstream with std::istringstream

std::istrstream (from <strstream>) has been deprecated since C++98 and
produces -Wdeprecated-declarations warnings on GCC 14+ and Clang 18+.

Replace all 37 usages across 23 files with std::istringstream (from
<sstream>). The zero-copy property of istrstream is not needed at any
of these call sites — all source buffers outlive the stream and the
data sizes are small (symbol tables, lexicons, WAV chunks, FST rules).

For files that already include <sstream> (symbol-table.cc, lexicon.cc,
audio-tagging-label-file.cc, piper-phonemize-lexicon.cc,
offline-tts-character-frontend.cc, matcha-tts-lexicon.cc,
homophone-replacer.cc, melo-tts-lexicon.cc, character-lexicon.cc,
kokoro-multi-lang-lexicon.cc), the redundant <strstream> include is
simply removed.

* Replace deprecated std::wstring_convert with manual UTF-8 codec

std::wstring_convert and std::codecvt_utf8 were deprecated in C++17
and trigger -Wdeprecated-declarations warnings with GCC 11+.  This
commit removes all uses across 6 files:

text-utils.cc  – Rewrites Utf8ToUtf32(), Utf32ToUtf8(), ToWideString(),
                 and ToString() using manual RFC 3629 encoding/decoding.
                 Adds a new single-codepoint Utf32ToUtf8(char32_t) overload
                 (promoted from the local copy in matcha-tts-lexicon.cc).

text-utils.h   – Declares the new Utf32ToUtf8(char32_t) overload.

kokoro-multi-lang-lexicon.cc – Replaces wstring_convert with calls to
                 Utf32ToUtf8() and Utf8ToUtf32() from text-utils.

piper-phonemize-lexicon.cc – Same replacement; adds text-utils.h include.

offline-tts-character-frontend.cc – Same replacement; adds text-utils.h
                 include.

matcha-tts-lexicon.cc – Removes the local static Utf32ToUtf8(char32_t)
                 that is now shared via text-utils.

The MSVC-only #include <codecvt> in onnx-utils.h is intentionally left
alone — it is guarded by #ifdef _MSC_VER, does not warn on Linux, and
may have hidden transitive dependencies on Windows.

Tested: full build with -DSHERPA_ONNX_ENABLE_TTS=ON on GCC 15.2 with
-Wdeprecated-declarations produces zero wstring_convert/codecvt warnings.

---------

Co-authored-by: John Suykerbuyk <bd770i@suykerbuyk.org>

Author: suykerbuyk

sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc

@@ -4,7 +4,6 @@
 
 #include "sherpa-onnx/csrc/kokoro-multi-lang-lexicon.h"
 
-#include <codecvt>
 #include <fstream>
 #include <regex>
 #include <sstream>
@@ -377,12 +376,10 @@ class KokoroMultiLangLexicon::Impl {
         // Note phonemes[i] contains a vector of unicode codepoints;
         // we need to convert them to utf8
 
-        std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
-
         std::vector<int32_t> ids;
         for (const auto &v : phonemes) {
           for (const auto p : v) {
-            auto token = conv.to_bytes(p);
+            auto token = Utf32ToUtf8(p);
             if (token2id_.count(token)) {
               ids.push_back(token2id_.at(token));
             } else {
@@ -450,10 +447,9 @@ class KokoroMultiLangLexicon::Impl {
       }
     }
 
-    std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
     std::u32string s;
     for (const auto &p : token2id_) {
-      s = conv.from_bytes(p.first);
+      s = Utf8ToUtf32(p.first);
 
       if (s.size() != 1) {
         SHERPA_ONNX_LOGE("Error for token %s with id %d", p.first.c_str(),

sherpa-onnx/csrc/matcha-tts-lexicon.cc

@@ -56,28 +56,6 @@ static const std::vector<std::pair<std::string, std::string>> kReplacements = {
     {"e", "ɛ"},
 };
 
-std::string Utf32ToUtf8(char32_t cp) {
-  std::string out;
-
-  if (cp <= 0x7F) {
-    out.push_back(static_cast<char>(cp));
-  } else if (cp <= 0x7FF) {
-    out.push_back(static_cast<char>(0xC0 | (cp >> 6)));
-    out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
-  } else if (cp <= 0xFFFF) {
-    out.push_back(static_cast<char>(0xE0 | (cp >> 12)));
-    out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
-    out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
-  } else {
-    out.push_back(static_cast<char>(0xF0 | (cp >> 18)));
-    out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
-    out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
-    out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
-  }
-
-  return out;
-}
-
 std::vector<std::string> ConvertPhonemesToUTF8(
     const std::vector<std::vector<char32_t>> &phonemes) {
   std::vector<std::string> out;

sherpa-onnx/csrc/offline-tts-character-frontend.cc

@@ -4,7 +4,6 @@
 
 #include <algorithm>
 #include <cctype>
-#include <codecvt>
 #include <fstream>
 #include <locale>
 #include <memory>
@@ -26,11 +25,11 @@
 #include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
 #include "sherpa-onnx/csrc/offline-tts-character-frontend.h"
+#include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {
 
 static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
-  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
   std::unordered_map<char32_t, int32_t> token2id;
 
   std::string line;
@@ -61,7 +60,7 @@ static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
       continue;
     }
 
-    s = conv.from_bytes(sym);
+    s = Utf8ToUtf32(sym);
     if (s.size() != 1) {
       SHERPA_ONNX_LOGE("Error when reading tokens at Line %s. size: %d",
                        line.c_str(), static_cast<int32_t>(s.size()));
@@ -113,8 +112,7 @@ std::vector<TokenIDs> OfflineTtsCharacterFrontend::ConvertTextToTokenIds(
   std::transform(_text.begin(), _text.end(), text.begin(),
                  [](auto c) { return std::tolower(c); });
 
-  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
-  std::u32string s = conv.from_bytes(text);
+  std::u32string s = Utf8ToUtf32(text);
 
   std::vector<TokenIDs> ans;
 

sherpa-onnx/csrc/piper-phonemize-lexicon.cc

@@ -4,7 +4,6 @@
 
 #include "sherpa-onnx/csrc/piper-phonemize-lexicon.h"
 
-#include <codecvt>
 #include <fstream>
 #include <locale>
 #include <map>
@@ -29,6 +28,7 @@
 #include "phonemize.hpp"    // NOLINT
 #include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
+#include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {
 
@@ -70,7 +70,6 @@ void CallPhonemizeEspeak(const std::string &text,
 }
 
 static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
-  std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
   std::unordered_map<char32_t, int32_t> token2id;
 
   std::string line;
@@ -95,7 +94,7 @@ static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
       SHERPA_ONNX_EXIT(-1);
     }
 
-    s = conv.from_bytes(sym);
+    s = Utf8ToUtf32(sym);
     if (s.size() != 1) {
       // for tokens.txt from coqui-ai/TTS, the last token is <BLNK>
       if (s.size() == 6 && s[0] == '<' && s[1] == 'B' && s[2] == 'L' &&