Skip to content

Commit 1b8d95d

Browse files
csukuangfjcsukuangfj
authored
Fix file related operations on Windows by using wide characters (k2-fsa#3710)
1 parent 768fd7a commit 1b8d95d

35 files changed

Lines changed: 121 additions & 114 deletions

‎sherpa-onnx/csrc/audio-tagging-label-file.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,11 @@
2020
namespace sherpa_onnx {
2121

2222
AudioTaggingLabels::AudioTaggingLabels(const std::string &filename) {
23-
std::ifstream is(filename);
23+
auto is = OpenInputFile(filename);
24+
if (!is) {
25+
SHERPA_ONNX_LOGE("Open label file failed: '%s'", filename.c_str());
26+
SHERPA_ONNX_EXIT(-1);
27+
}
2428
Init(is);
2529
}
2630

‎sherpa-onnx/csrc/character-lexicon.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,12 @@ class CharacterLexicon::Impl {
4444
}
4545

4646
{
47-
std::ifstream is(tokens);
47+
auto is = OpenInputFile(tokens);
4848
InitTokens(is);
4949
}
5050

5151
{
52-
std::ifstream is(lexicon);
52+
auto is = OpenInputFile(lexicon);
5353
InitLexicon(is);
5454
}
5555
}

‎sherpa-onnx/csrc/file-utils.cc‎

Lines changed: 37 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
#include "sherpa-onnx/csrc/file-utils.h"
66

77
#include <fstream>
8-
#include <memory>
98
#include <sstream>
109
#include <string>
1110
#include <vector>
@@ -14,25 +13,41 @@
1413
#include <windows.h>
1514
#else
1615
#include <sys/stat.h>
17-
#include <unistd.h>
1816
#include <limits.h>
1917
#include <stdlib.h>
2018
#endif
2119

2220
#include "sherpa-onnx/csrc/macros.h"
21+
#include "sherpa-onnx/csrc/text-utils.h"
2322

2423
namespace sherpa_onnx {
25-
std::wstring ToWideString(const std::string &s);
26-
std::string ToString(const std::wstring &s);
24+
25+
std::ifstream OpenInputFile(const std::string &filename,
26+
std::ios_base::openmode mode) {
27+
#ifdef _WIN32
28+
return std::ifstream(ToWideString(filename).c_str(), mode);
29+
#else
30+
return std::ifstream(filename, mode);
31+
#endif
32+
}
33+
34+
std::ofstream OpenOutputFile(const std::string &filename,
35+
std::ios_base::openmode mode) {
36+
#ifdef _WIN32
37+
return std::ofstream(ToWideString(filename).c_str(), mode);
38+
#else
39+
return std::ofstream(filename, mode);
40+
#endif
41+
}
2742

2843
bool FileExists(const std::string &filename) {
2944
#ifdef _WIN32
30-
DWORD attributes = GetFileAttributesW(ToWideString(filename).c_str());
31-
32-
return attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY);
45+
DWORD attributes = GetFileAttributesW(ToWideString(filename).c_str());
46+
return attributes != INVALID_FILE_ATTRIBUTES &&
47+
!(attributes & FILE_ATTRIBUTE_DIRECTORY);
3348
#else
34-
struct stat file_stat;
35-
return stat(filename.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode);
49+
struct stat file_stat;
50+
return stat(filename.c_str(), &file_stat) == 0 && S_ISREG(file_stat.st_mode);
3651
#endif
3752
}
3853

@@ -47,67 +62,24 @@ std::vector<char> ReadFile(const std::string &filename) {
4762
if (filename.empty()) {
4863
return {};
4964
}
50-
try {
51-
#ifdef _WIN32
52-
HANDLE hFile = CreateFileW(
53-
ToWideString(filename).c_str(),
54-
GENERIC_READ,
55-
FILE_SHARE_READ,
56-
nullptr,
57-
OPEN_EXISTING,
58-
FILE_ATTRIBUTE_NORMAL,
59-
nullptr
60-
);
61-
62-
if (hFile == INVALID_HANDLE_VALUE) {
63-
return {};
64-
}
65-
66-
std::unique_ptr<void, decltype(&CloseHandle)> file_guard(
67-
hFile, CloseHandle);
68-
69-
LARGE_INTEGER file_size;
70-
if (!GetFileSizeEx(hFile, &file_size) || file_size.QuadPart > SIZE_MAX) {
71-
return {};
72-
}
73-
74-
std::vector<char> buffer(static_cast<size_t>(file_size.QuadPart));
7565

76-
DWORD bytes_read = 0;
77-
bool read_success = ::ReadFile(
78-
hFile,
79-
buffer.data(),
80-
static_cast<DWORD>(buffer.size()),
81-
&bytes_read,
82-
nullptr
83-
);
84-
if (!read_success || bytes_read != buffer.size()) {
85-
return {};
86-
}
87-
88-
return buffer;
89-
#else
90-
std::ifstream file(filename, std::ios::binary | std::ios::ate);
91-
if (!file.is_open()) {
92-
return {};
93-
}
94-
95-
std::streamsize size = file.tellg();
96-
if (size < 0) {
97-
return {};
98-
}
99-
file.seekg(0, std::ios::beg);
66+
std::ifstream file = OpenInputFile(filename, std::ios::binary | std::ios::ate);
67+
if (!file.is_open()) {
68+
return {};
69+
}
10070

101-
std::vector<char> buffer(static_cast<size_t>(size));
102-
if (!file.read(buffer.data(), size)) {
103-
return {};
104-
}
71+
std::streamsize size = file.tellg();
72+
if (size < 0) {
73+
return {};
74+
}
75+
file.seekg(0, std::ios::beg);
10576

106-
return buffer;
107-
#endif
108-
} catch (const std::exception&) {
77+
std::vector<char> buffer(static_cast<size_t>(size));
78+
if (!file.read(buffer.data(), size)) {
10979
return {};
11080
}
81+
82+
return buffer;
11183
}
11284

11385
#if __ANDROID_API__ >= 9

‎sherpa-onnx/csrc/file-utils.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@ std::vector<char> ReadFile(NativeResourceManager *mgr,
4646

4747
std::string ResolveAbsolutePath(const std::string &path);
4848

49+
std::ifstream OpenInputFile(const std::string &filename,
50+
std::ios_base::openmode mode = std::ios_base::in);
51+
52+
std::ofstream OpenOutputFile(const std::string &filename,
53+
std::ios_base::openmode mode = std::ios_base::out);
54+
4955
} // namespace sherpa_onnx
5056

5157
#endif // SHERPA_ONNX_CSRC_FILE_UTILS_H_

‎sherpa-onnx/csrc/fst-utils.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "fst/extensions/far/far.h"
1313
#include "kaldifst/csrc/kaldi-fst-io.h"
14+
#include "sherpa-onnx/csrc/file-utils.h"
1415
#include "sherpa-onnx/csrc/macros.h"
1516

1617
namespace sherpa_onnx {
@@ -22,7 +23,7 @@ namespace sherpa_onnx {
2223
// avoid memory leak.
2324
fst::Fst<fst::StdArc> *ReadGraph(const std::string &filename) {
2425
// read decoding network FST
25-
std::ifstream is(filename, std::ios::binary);
26+
auto is = OpenInputFile(filename, std::ios::binary);
2627
if (!is.good()) {
2728
SHERPA_ONNX_LOGE("Could not open decoding-graph FST %s", filename.c_str());
2829
}

‎sherpa-onnx/csrc/homophone-replacer.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ class HomophoneReplacer::Impl {
8989
public:
9090
explicit Impl(const HomophoneReplacerConfig &config) : config_(config) {
9191
{
92-
std::ifstream is(config.lexicon);
92+
auto is = OpenInputFile(config.lexicon);
9393
InitLexicon(is);
9494
}
9595

‎sherpa-onnx/csrc/keyword-spotter-transducer-impl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
300300
InitKeywords(is);
301301
#else
302302
// each line in keywords_file contains space-separated words
303-
std::ifstream is(config_.keywords_file);
303+
auto is = OpenInputFile(config_.keywords_file);
304304
if (!is) {
305305
#if __OHOS__
306306
SHERPA_ONNX_LOGE("Open keywords file failed: '%{public}s'",

‎sherpa-onnx/csrc/keyword-spotter.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "rawfile/raw_file_manager.h"
2424
#endif
2525

26+
#include "sherpa-onnx/csrc/file-utils.h"
2627
#include "sherpa-onnx/csrc/keyword-spotter-impl.h"
2728

2829
namespace sherpa_onnx {
@@ -118,7 +119,7 @@ bool KeywordSpotterConfig::Validate() const {
118119
// keywords file will be packaged into the sherpa-onnx-wasm-kws-main.data file
119120
// Solution: take keyword_file variable is directly
120121
// parsed as a string of keywords
121-
if (keywords_buf.empty() && !std::ifstream(keywords_file.c_str()).good()) {
122+
if (keywords_buf.empty() && !FileExists(keywords_file)) {
122123
SHERPA_ONNX_LOGE("Keywords file '%s' does not exist.",
123124
keywords_file.c_str());
124125
return false;

‎sherpa-onnx/csrc/kokoro-multi-lang-lexicon.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ class KokoroMultiLangLexicon::Impl {
427427
}
428428

429429
void InitTokens(const std::string &tokens) {
430-
std::ifstream is(tokens);
430+
auto is = OpenInputFile(tokens);
431431
InitTokens(is);
432432
}
433433

@@ -471,7 +471,7 @@ class KokoroMultiLangLexicon::Impl {
471471
std::vector<std::string> files;
472472
SplitStringToVector(lexicon, ",", false, &files);
473473
for (const auto &f : files) {
474-
std::ifstream is(f);
474+
auto is = OpenInputFile(f);
475475
InitLexicon(is);
476476
}
477477
}

‎sherpa-onnx/csrc/lexicon.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,12 @@ Lexicon::Lexicon(const std::string &lexicon, const std::string &tokens,
108108
InitLanguage(language);
109109

110110
{
111-
std::ifstream is(tokens);
111+
auto is = OpenInputFile(tokens);
112112
InitTokens(is);
113113
}
114114

115115
{
116-
std::ifstream is(lexicon);
116+
auto is = OpenInputFile(lexicon);
117117
InitLexicon(is);
118118
}
119119

0 commit comments

Comments
 (0)