Skip to content

Commit c31bda1

Browse files
bab2minbab2min
committed
Fix compilation errors
1 parent 0ad9387 commit c31bda1

14 files changed

+44
-97
lines changed

Diff for: ‎src/MathFunc.hpp

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#pragma once
22
#include <cmath>
3+
#include <algorithm>
4+
#include <stdexcept>
35
#include "MathFunc.h"
46
#include "SIMD.hpp"
57

Diff for: ‎src/PathEvaluator.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -358,8 +358,8 @@ namespace kiwi
358358
{
359359
for (auto& curMorph : cands)
360360
{
361-
if (splitComplex && curMorph->getCombined()->complex) continue;
362-
if (blocklist && blocklist->count(curMorph->getCombined())) continue;
361+
if (splitComplex && curMorph->hasComplex()) continue;
362+
if (blocklist && curMorph->hasMorpheme(*blocklist)) continue;
363363

364364
// 덧붙은 받침(zCoda)을 위한 지름길
365365
if (curMorph->tag == POSTag::z_coda)

Diff for: ‎src/SkipBigramModel.hpp

+29-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include <kiwi/ArchUtils.h>
77
#include "ArchAvailable.h"
88
#include "Knlm.hpp"
9+
#include "MathFunc.h"
910
#include "search.h"
1011

1112
namespace kiwi
@@ -109,7 +110,34 @@ namespace kiwi
109110
return !!vocabValidness[k];
110111
}
111112

112-
float evaluate(const KeyType* history, size_t cnt, KeyType next, float base) const;
113+
float evaluate(const KeyType* history, size_t cnt, KeyType next, float base) const
114+
{
115+
if (!cnt) return base;
116+
if (!vocabValidness[next]) return base;
117+
118+
#if defined(__GNUC__) && __GNUC__ < 5
119+
alignas(256) float arr[windowSize * 2];
120+
#else
121+
alignas(ArchInfo<arch>::alignment) float arr[windowSize * 2];
122+
#endif
123+
std::fill(arr, arr + windowSize, base);
124+
std::fill(arr + windowSize, arr + windowSize * 2, -INFINITY);
125+
126+
size_t b = ptrs[next], e = ptrs[next + 1];
127+
size_t size = e - b;
128+
129+
for (size_t i = 0; i < cnt; ++i)
130+
{
131+
arr[i] = discnts[history[i]] + base;
132+
float out;
133+
if (nst::search<arch>(&keyData[b], &compensations[b], size, history[i], out))
134+
{
135+
arr[i + windowSize] = out;
136+
}
137+
}
138+
return logSumExp<arch>(arr, windowSize * 2) - logWindowSize;
139+
}
140+
113141
};
114142

115143
template<size_t windowSize, ArchType _arch, class VocabTy>

Diff for: ‎src/SkipBigramModelImpl.hpp

-40
This file was deleted.

Diff for: ‎src/archImpl/avx2.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -27,11 +27,6 @@ namespace kiwi
2727
{
2828
namespace lm
2929
{
30-
template class SkipBigramModel<ArchType::avx2, uint8_t, 8>;
31-
template class SkipBigramModel<ArchType::avx2, uint16_t, 8>;
32-
template class SkipBigramModel<ArchType::avx2, uint32_t, 8>;
33-
template class SkipBigramModel<ArchType::avx2, uint64_t, 8>;
34-
3530
template float logSumExp<ArchType::avx2>(const float* arr, size_t size);
3631
template void logSumExpTransposed<ArchType::avx2>(float* arr, size_t size, size_t batchSize, size_t stride);
3732
template void logSoftmax<ArchType::avx2>(float* arr, size_t size);

Diff for: ‎src/archImpl/avx512bw.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -27,11 +27,6 @@ namespace kiwi
2727
{
2828
namespace lm
2929
{
30-
template class SkipBigramModel<ArchType::avx512bw, uint8_t, 8>;
31-
template class SkipBigramModel<ArchType::avx512bw, uint16_t, 8>;
32-
template class SkipBigramModel<ArchType::avx512bw, uint32_t, 8>;
33-
template class SkipBigramModel<ArchType::avx512bw, uint64_t, 8>;
34-
3530
template<>
3631
float logSumExp<ArchType::avx512bw>(const float* arr, size_t size)
3732
{

Diff for: ‎src/archImpl/avx512vnni.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -9,11 +9,6 @@ namespace kiwi
99
{
1010
namespace lm
1111
{
12-
template class SkipBigramModel<ArchType::avx512vnni, uint8_t, 8>;
13-
template class SkipBigramModel<ArchType::avx512vnni, uint16_t, 8>;
14-
template class SkipBigramModel<ArchType::avx512vnni, uint32_t, 8>;
15-
template class SkipBigramModel<ArchType::avx512vnni, uint64_t, 8>;
16-
1712
template<>
1813
float logSumExp<ArchType::avx512vnni>(const float* arr, size_t size)
1914
{

Diff for: ‎src/archImpl/avx_vnni.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -9,11 +9,6 @@ namespace kiwi
99
{
1010
namespace lm
1111
{
12-
template class SkipBigramModel<ArchType::avx_vnni, uint8_t, 8>;
13-
template class SkipBigramModel<ArchType::avx_vnni, uint16_t, 8>;
14-
template class SkipBigramModel<ArchType::avx_vnni, uint32_t, 8>;
15-
template class SkipBigramModel<ArchType::avx_vnni, uint64_t, 8>;
16-
1712
template float logSumExp<ArchType::avx_vnni>(const float* arr, size_t size);
1813
template void logSumExpTransposed<ArchType::avx_vnni>(float* arr, size_t size, size_t batchSize, size_t stride);
1914
template void logSoftmax<ArchType::avx_vnni>(float* arr, size_t size);

Diff for: ‎src/archImpl/neon.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -9,11 +9,6 @@ namespace kiwi
99
{
1010
namespace lm
1111
{
12-
template class SkipBigramModel<ArchType::neon, uint8_t, 8>;
13-
template class SkipBigramModel<ArchType::neon, uint16_t, 8>;
14-
template class SkipBigramModel<ArchType::neon, uint32_t, 8>;
15-
template class SkipBigramModel<ArchType::neon, uint64_t, 8>;
16-
1712
template float logSumExp<ArchType::neon>(const float* arr, size_t size);
1813
template void logSumExpTransposed<ArchType::neon>(float* arr, size_t size, size_t batchSize, size_t stride);
1914
template void logSoftmax<ArchType::neon>(float* arr, size_t size);

Diff for: ‎src/archImpl/none.cpp

+1-11
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../gemm.h"
33

44
#include <Eigen/Dense>
@@ -7,21 +7,11 @@ namespace kiwi
77
{
88
namespace lm
99
{
10-
template class SkipBigramModel<ArchType::none, uint8_t, 8>;
11-
template class SkipBigramModel<ArchType::none, uint16_t, 8>;
12-
template class SkipBigramModel<ArchType::none, uint32_t, 8>;
13-
template class SkipBigramModel<ArchType::none, uint64_t, 8>;
14-
1510
template float logSumExp<ArchType::none>(const float* arr, size_t size);
1611
template void logSumExpTransposed<ArchType::none>(float* arr, size_t size, size_t batchSize, size_t stride);
1712
template void logSoftmax<ArchType::none>(float* arr, size_t size);
1813
template void logSoftmaxTransposed<ArchType::none>(float* arr, size_t size, size_t batchSize, size_t stride);
1914

20-
template class SkipBigramModel<ArchType::balanced, uint8_t, 8>;
21-
template class SkipBigramModel<ArchType::balanced, uint16_t, 8>;
22-
template class SkipBigramModel<ArchType::balanced, uint32_t, 8>;
23-
template class SkipBigramModel<ArchType::balanced, uint64_t, 8>;
24-
2515
template float logSumExp<ArchType::balanced>(const float* arr, size_t size);
2616
template void logSumExpTransposed<ArchType::balanced>(float* arr, size_t size, size_t batchSize, size_t stride);
2717
template void logSoftmax<ArchType::balanced>(float* arr, size_t size);

Diff for: ‎src/archImpl/sse2.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../gemm.h"
33

44
#define Eigen EigenSSE2
@@ -8,11 +8,6 @@ namespace kiwi
88
{
99
namespace lm
1010
{
11-
template class SkipBigramModel<ArchType::sse2, uint8_t, 8>;
12-
template class SkipBigramModel<ArchType::sse2, uint16_t, 8>;
13-
template class SkipBigramModel<ArchType::sse2, uint32_t, 8>;
14-
template class SkipBigramModel<ArchType::sse2, uint64_t, 8>;
15-
1611
template float logSumExp<ArchType::sse2>(const float* arr, size_t size);
1712
template void logSumExpTransposed<ArchType::sse2>(float* arr, size_t size, size_t batchSize, size_t stride);
1813
template void logSoftmax<ArchType::sse2>(float* arr, size_t size);

Diff for: ‎src/archImpl/sse4_1.cpp

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#include "../SkipBigramModelImpl.hpp"
1+
#include "../MathFunc.hpp"
22
#include "../qgemm.hpp"
33
#include "../gemm.h"
44

@@ -9,11 +9,6 @@ namespace kiwi
99
{
1010
namespace lm
1111
{
12-
template class SkipBigramModel<ArchType::sse4_1, uint8_t, 8>;
13-
template class SkipBigramModel<ArchType::sse4_1, uint16_t, 8>;
14-
template class SkipBigramModel<ArchType::sse4_1, uint32_t, 8>;
15-
template class SkipBigramModel<ArchType::sse4_1, uint64_t, 8>;
16-
1712
template float logSumExp<ArchType::sse4_1>(const float* arr, size_t size);
1813
template void logSumExpTransposed<ArchType::sse4_1>(float* arr, size_t size, size_t batchSize, size_t stride);
1914
template void logSoftmax<ArchType::sse4_1>(float* arr, size_t size);

Diff for: ‎src/qgemm.hpp

+2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#pragma once
2+
#include <cstring>
3+
#include <kiwi/Types.h>
24
#include "qgemm.h"
35
#include "SIMD.hpp"
46

Diff for: ‎tools/Evaluator.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ inline TokenInfo parseWordPOS(const u16string& str)
4444
{
4545
auto p = str.rfind('/');
4646
if (p == str.npos) return {};
47-
u16string form = replace(nonstd::u16string_view(str.data(), p), u"_", u" ");
47+
u16string form = replace(std::u16string_view(str.data(), p), u"_", u" ");
4848
if (str[p + 1] == 'E')
4949
{
5050
if (form[0] == u'아' || form[0] == u'여') form[0] = u'어';

0 commit comments

Comments
 (0)