Skip to content

Commit 9544e7e

Browse files
Wasser1462Wasser1462
committed
update code after review
1 parent 5783675 commit 9544e7e

15 files changed

Lines changed: 295 additions & 353 deletions

‎sherpa-onnx/csrc/funasr-nano-tokenizer.cc‎

Lines changed: 31 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,15 @@
44

55
#include "sherpa-onnx/csrc/funasr-nano-tokenizer.h"
66

7+
#if __ANDROID_API__ >= 9
8+
#include "android/asset_manager.h"
9+
#include "android/asset_manager_jni.h"
10+
#endif
11+
12+
#if __OHOS__
13+
#include "rawfile/raw_file_manager.h"
14+
#endif
15+
716
#include <algorithm>
817
#include <cctype>
918
#include <cstdint>
@@ -1129,19 +1138,11 @@ FunASRNanoTokenizer::FunASRNanoTokenizer(const std::string &tokenizer_dir) {
11291138
Init(tokenizer_dir);
11301139
}
11311140

1132-
#if __ANDROID_API__ >= 9
1133-
FunASRNanoTokenizer::FunASRNanoTokenizer(AAssetManager *mgr,
1134-
const std::string &tokenizer_dir) {
1135-
Init(mgr, tokenizer_dir);
1136-
}
1137-
#endif
1138-
1139-
#if __OHOS__
1140-
FunASRNanoTokenizer::FunASRNanoTokenizer(NativeResourceManager *mgr,
1141+
template <typename Manager>
1142+
FunASRNanoTokenizer::FunASRNanoTokenizer(Manager *mgr,
11411143
const std::string &tokenizer_dir) {
11421144
Init(mgr, tokenizer_dir);
11431145
}
1144-
#endif
11451146

11461147
void FunASRNanoTokenizer::Init(const std::string &tokenizer_dir) {
11471148
std::string tok_json = FindTokenizerJson(tokenizer_dir);
@@ -1197,50 +1198,8 @@ void FunASRNanoTokenizer::Init(const std::string &tokenizer_dir) {
11971198
FinalizeSpecialIds();
11981199
}
11991200

1200-
#if __ANDROID_API__ >= 9
1201-
void FunASRNanoTokenizer::Init(AAssetManager *mgr,
1202-
const std::string &tokenizer_dir) {
1203-
std::string tok_json = tokenizer_dir + "/tokenizer.json";
1204-
std::string vocab_json = tokenizer_dir + "/vocab.json";
1205-
std::string merges_txt = tokenizer_dir + "/merges.txt";
1206-
1207-
const std::string tok_blob = LoadBytesFromFile(mgr, tok_json);
1208-
const std::string vocab_blob = LoadBytesFromFile(mgr, vocab_json);
1209-
const std::string merges_blob = LoadBytesFromFile(mgr, merges_txt);
1210-
1211-
if (tok_blob.empty() || vocab_blob.empty() || merges_blob.empty()) {
1212-
SHERPA_ONNX_LOGE("Failed to read tokenizer files from assets: %s",
1213-
tokenizer_dir.c_str());
1214-
SHERPA_ONNX_EXIT(-1);
1215-
}
1216-
1217-
BuildBytesToUnicode(byte_to_unicode_, &unicode_to_byte_);
1218-
1219-
if (!ParseVocabJson(vocab_blob, &token2id_)) {
1220-
SHERPA_ONNX_LOGE("Failed to parse vocab.json from assets: %s",
1221-
vocab_json.c_str());
1222-
SHERPA_ONNX_EXIT(-1);
1223-
}
1224-
if (!ParseMergesTxt(merges_blob, &merges_rank_)) {
1225-
SHERPA_ONNX_LOGE("Failed to parse merges.txt from assets: %s",
1226-
merges_txt.c_str());
1227-
SHERPA_ONNX_EXIT(-1);
1228-
}
1229-
1230-
if (!ParseAddedTokensFromTokenizerJson(tok_blob, &added_tokens_)) {
1231-
SHERPA_ONNX_LOGE("Failed to parse added_tokens from assets tokenizer.json");
1232-
SHERPA_ONNX_EXIT(-1);
1233-
}
1234-
MergeVocabAndAddedTokens(&token2id_, added_tokens_, &added_token_contents_);
1235-
BuildIdToToken(token2id_, added_token_contents_, &id2token_);
1236-
BuildAddedTokensTrie(added_tokens_, &trie_);
1237-
FinalizeSpecialIds();
1238-
}
1239-
#endif
1240-
1241-
#if __OHOS__
1242-
void FunASRNanoTokenizer::Init(NativeResourceManager *mgr,
1243-
const std::string &tokenizer_dir) {
1201+
template <typename Manager>
1202+
void FunASRNanoTokenizer::Init(Manager *mgr, const std::string &tokenizer_dir) {
12441203
std::string tok_json = tokenizer_dir + "/tokenizer.json";
12451204
std::string vocab_json = tokenizer_dir + "/vocab.json";
12461205
std::string merges_txt = tokenizer_dir + "/merges.txt";
@@ -1250,35 +1209,34 @@ void FunASRNanoTokenizer::Init(NativeResourceManager *mgr,
12501209
const std::string merges_blob = LoadBytesFromFile(mgr, merges_txt);
12511210

12521211
if (tok_blob.empty() || vocab_blob.empty() || merges_blob.empty()) {
1253-
SHERPA_ONNX_LOGE("Failed to read tokenizer files from rawfile: %s",
1254-
tokenizer_dir.c_str());
1212+
SHERPA_ONNX_LOGE(
1213+
"Failed to read tokenizer files via resource manager "
1214+
"(tokenizer_dir=%s)",
1215+
tokenizer_dir.c_str());
12551216
SHERPA_ONNX_EXIT(-1);
12561217
}
12571218

12581219
BuildBytesToUnicode(byte_to_unicode_, &unicode_to_byte_);
12591220

12601221
if (!ParseVocabJson(vocab_blob, &token2id_)) {
1261-
SHERPA_ONNX_LOGE("Failed to parse vocab.json from rawfile: %s",
1262-
vocab_json.c_str());
1222+
SHERPA_ONNX_LOGE("Failed to parse vocab.json: %s", vocab_json.c_str());
12631223
SHERPA_ONNX_EXIT(-1);
12641224
}
12651225
if (!ParseMergesTxt(merges_blob, &merges_rank_)) {
1266-
SHERPA_ONNX_LOGE("Failed to parse merges.txt from rawfile: %s",
1267-
merges_txt.c_str());
1226+
SHERPA_ONNX_LOGE("Failed to parse merges.txt: %s", merges_txt.c_str());
12681227
SHERPA_ONNX_EXIT(-1);
12691228
}
12701229

12711230
if (!ParseAddedTokensFromTokenizerJson(tok_blob, &added_tokens_)) {
1272-
SHERPA_ONNX_LOGE(
1273-
"Failed to parse added_tokens from rawfile tokenizer.json");
1231+
SHERPA_ONNX_LOGE("Failed to parse added_tokens from tokenizer.json: %s",
1232+
tok_json.c_str());
12741233
SHERPA_ONNX_EXIT(-1);
12751234
}
12761235
MergeVocabAndAddedTokens(&token2id_, added_tokens_, &added_token_contents_);
12771236
BuildIdToToken(token2id_, added_token_contents_, &id2token_);
12781237
BuildAddedTokensTrie(added_tokens_, &trie_);
12791238
FinalizeSpecialIds();
12801239
}
1281-
#endif
12821240

12831241
void FunASRNanoTokenizer::FinalizeSpecialIds() {
12841242
im_end_token_id_ = TokenToIdOrDefault(token2id_, "<|im_end|>", 151645);
@@ -1564,4 +1522,14 @@ std::string FunASRNanoTokenizer::Decode(const std::vector<int64_t> &token_ids) {
15641522
return out;
15651523
}
15661524

1525+
#if __ANDROID_API__ >= 9
1526+
template FunASRNanoTokenizer::FunASRNanoTokenizer(
1527+
AAssetManager *mgr, const std::string &tokenizer_dir);
1528+
#endif
1529+
1530+
#if __OHOS__
1531+
template FunASRNanoTokenizer::FunASRNanoTokenizer(
1532+
NativeResourceManager *mgr, const std::string &tokenizer_dir);
1533+
#endif
1534+
15671535
} // namespace sherpa_onnx

‎sherpa-onnx/csrc/funasr-nano-tokenizer.h‎

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,14 @@
1717
#include <unordered_set>
1818
#include <vector>
1919

20-
#if __ANDROID_API__ >= 9
21-
#include <android/asset_manager.h>
22-
#endif
23-
24-
#if __OHOS__
25-
struct NativeResourceManager;
26-
#endif
27-
2820
namespace sherpa_onnx {
2921

3022
class FunASRNanoTokenizer {
3123
public:
3224
explicit FunASRNanoTokenizer(const std::string &tokenizer_dir);
3325

34-
#if __ANDROID_API__ >= 9
35-
FunASRNanoTokenizer(AAssetManager *mgr, const std::string &tokenizer_dir);
36-
#endif
37-
38-
#if __OHOS__
39-
FunASRNanoTokenizer(NativeResourceManager *mgr,
40-
const std::string &tokenizer_dir);
41-
#endif
26+
template <typename Manager>
27+
FunASRNanoTokenizer(Manager *mgr, const std::string &tokenizer_dir);
4228

4329
std::vector<int64_t> Encode(const std::string &text);
4430
std::string Decode(const std::vector<int64_t> &token_ids);
@@ -68,13 +54,8 @@ class FunASRNanoTokenizer {
6854
private:
6955
void Init(const std::string &tokenizer_dir);
7056

71-
#if __ANDROID_API__ >= 9
72-
void Init(AAssetManager *mgr, const std::string &tokenizer_dir);
73-
#endif
74-
75-
#if __OHOS__
76-
void Init(NativeResourceManager *mgr, const std::string &tokenizer_dir);
77-
#endif
57+
template <typename Manager>
58+
void Init(Manager *mgr, const std::string &tokenizer_dir);
7859

7960
void FinalizeSpecialIds();
8061

‎sherpa-onnx/csrc/offline-qwen3-asr-model-config.cc‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,14 @@ bool OfflineQwen3ASRModelConfig::Validate() const {
9494
return false;
9595
}
9696

97+
if (!FileExists(tokenizer + "/tokenizer_config.json")) {
98+
SHERPA_ONNX_LOGE(
99+
"'%s/tokenizer_config.json' does not exist. Please check "
100+
"--qwen3-asr-tokenizer",
101+
tokenizer.c_str());
102+
return false;
103+
}
104+
97105
if (max_total_len <= 0) {
98106
SHERPA_ONNX_LOGE("--qwen3-asr-max-total-len should be > 0. Given: %d",
99107
max_total_len);

‎sherpa-onnx/csrc/offline-qwen3-asr-model-config.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,22 @@ struct OfflineQwen3ASRModelConfig {
4141

4242
OfflineQwen3ASRModelConfig() = default;
4343

44+
OfflineQwen3ASRModelConfig(const std::string &conv_frontend,
45+
const std::string &encoder,
46+
const std::string &decoder,
47+
const std::string &tokenizer,
48+
int32_t max_total_len, int32_t max_new_tokens,
49+
float temperature, float top_p, int32_t seed)
50+
: conv_frontend(conv_frontend),
51+
encoder(encoder),
52+
decoder(decoder),
53+
tokenizer(tokenizer),
54+
max_total_len(max_total_len),
55+
max_new_tokens(max_new_tokens),
56+
temperature(temperature),
57+
top_p(top_p),
58+
seed(seed) {}
59+
4460
void Register(ParseOptions *po);
4561
bool Validate() const;
4662

0 commit comments

Comments
 (0)