Refactor text-utils (k2-fsa#2855)

This pull request refactors text utility functions by introducing a generic Join function in text-utils and removing a specialized JoinTokensNoSpace function.

Author: csukuangfj

sherpa-onnx/csrc/matcha-tts-lexicon.cc

@@ -92,14 +92,6 @@ std::vector<std::string> ConvertPhonemesToUTF8(
   return out;
 }
 
-std::string JoinTokensNoSpace(const std::vector<std::string> &tokens) {
-  std::string out;
-  for (const auto &t : tokens) {
-    out += t;
-  }
-  return out;
-}
-
 std::string ApplyReplacements(std::string s) {
   for (const auto &p : kReplacements) {
     const std::string &from = p.first;
@@ -131,7 +123,7 @@ std::vector<std::string> SplitTokensUTF8(const std::string &s) {
 std::vector<std::string> ProcessPhonemes(
     const std::vector<std::vector<char32_t>> &phonemes) {
   auto tokens = ConvertPhonemesToUTF8(phonemes);
-  std::string joined = JoinTokensNoSpace(tokens);
+  std::string joined = Join(tokens);
   std::string replaced = ApplyReplacements(joined);
   return SplitTokensUTF8(replaced);
 }

sherpa-onnx/csrc/text-utils.cc

@@ -733,6 +733,17 @@ std::vector<std::string> SplitString(const std::string &s, int32_t chunk_size) {
   return ans;
 }
 
+std::string Join(const std::vector<std::string> &ss, const std::string &delim) {
+  std::ostringstream oss;
+  if (!ss.empty()) {
+    oss << ss[0];
+    for (size_t i = 1; i < ss.size(); ++i) {
+      oss << delim << ss[i];
+    }
+  }
+  return oss.str();
+}
+
 std::u32string Utf8ToUtf32(const std::string &str) {
   std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
   return conv.from_bytes(str);

sherpa-onnx/csrc/text-utils.h

@@ -151,6 +151,9 @@ bool Contains(const std::string &haystack, const std::string &needle);
 
 std::vector<std::string> SplitString(const std::string &s, int32_t chunk_size);
 
+std::string Join(const std::vector<std::string> &ss,
+                 const std::string &delim = "");
+
 // Converts a UTF-8 std::string to a UTF-32 std::u32string
 std::u32string Utf8ToUtf32(const std::string &str);