Skip to content

Commit 9aaff2a

Browse files
authored
Merge pull request #189 from bab2min/dev/model
Refactor model path
2 parents 93f3188 + 9f921bf commit 9aaff2a

26 files changed

+63
-63
lines changed

Diff for: .github/workflows/arm64_centos7.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,15 @@ jobs:
3434
- name: Test
3535
run: |
3636
./build/test/kiwi-test
37-
mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
37+
mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
3838
cp -r build /artifacts/
3939
cp -r eval_results /artifacts/
4040
- name: Benchmark
4141
run: |
4242
curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
43-
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
44-
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
45-
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
43+
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
44+
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
45+
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
4646
- name: Archive binaries
4747
uses: actions/upload-artifact@v3
4848
with:

Diff for: .github/workflows/centos7.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ jobs:
4040
- name: Run Evaluator
4141
run: |
4242
mkdir eval_results
43-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
44-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
43+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
44+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
4545
- run: tar -zcvf arts.tgz build/*kiwi* build/test/*kiwi* eval_results/*.txt build/bindings/java/*.jar
4646
- name: Archive binaries
4747
uses: actions/upload-artifact@v3

Diff for: .github/workflows/macos.yml

+13-13
Original file line numberDiff line numberDiff line change
@@ -60,24 +60,24 @@ jobs:
6060
- name: Run Evaluator
6161
run: |
6262
mkdir eval_results
63-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
64-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
63+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
64+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
6565
- name: Run Benchmark
6666
run: |
6767
curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
68-
./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
69-
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
70-
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
68+
./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
69+
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
70+
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
7171
if [ "${{ matrix.arch }}" = "x86_64" ]; then
72-
KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
73-
KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
74-
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
75-
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
76-
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
72+
KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
73+
KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
74+
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
75+
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
76+
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
7777
else
78-
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
79-
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
80-
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
78+
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
79+
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
80+
KIWI_ARCH_TYPE=neon ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
8181
fi
8282
- name: Archive binaries
8383
uses: actions/upload-artifact@v3

Diff for: .github/workflows/ppc64le_centos7.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 ..
2929
make -j2 && popd
3030
./build/test/kiwi-test
31-
mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
31+
mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
3232
cp -r build /artifacts/
3333
cp -r eval_results /artifacts/
3434
- name: Archive binaries

Diff for: .github/workflows/release.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ jobs:
114114
asset_name: kiwi-java-${{ steps.get_release.outputs.tag_name }}-mac-${{ matrix.arch }}.jar
115115
asset_content_type: application/octet-stream
116116
- if: matrix.arch == 'arm64'
117-
run: tar -zcvf model.tgz ModelGenerator/sj.* ModelGenerator/extract.mdl ModelGenerator/*.dict ModelGenerator/combiningRule.txt ModelGenerator/skipbigram.mdl
117+
run: tar -zcvf model.tgz models/base/sj.* models/base/extract.mdl models/base/*.dict models/base/combiningRule.txt models/base/skipbigram.mdl
118118
- name: Upload release binary
119119
if: matrix.arch == 'arm64'
120120
uses: actions/[email protected]
@@ -214,7 +214,7 @@ jobs:
214214
mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 ..
215215
make -j2 && popd
216216
./build/test/kiwi-test
217-
mkdir eval_results && ./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
217+
mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
218218
cd build
219219
mkdir include && mkdir lib && mkdir bin
220220
mv libkiwi* lib/

Diff for: .github/workflows/ubuntu.yml

+11-11
Original file line numberDiff line numberDiff line change
@@ -60,20 +60,20 @@ jobs:
6060
- name: Run Evaluator
6161
run: |
6262
mkdir eval_results
63-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt -o eval_results/
64-
./build/kiwi-evaluator -m ./ModelGenerator eval_data/*.txt --sbg -o eval_results/
63+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
64+
./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
6565
- name: Run Benchmark
6666
run: |
6767
curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
68-
./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
69-
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
70-
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
71-
KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
72-
KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
73-
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
74-
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out kowiki1000.txt
75-
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --sbg kowiki1000.txt
76-
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
68+
./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
69+
KIWI_ARCH_TYPE=none ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
70+
KIWI_ARCH_TYPE=balanced ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
71+
KIWI_ARCH_TYPE=sse2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
72+
KIWI_ARCH_TYPE=sse4_1 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
73+
KIWI_ARCH_TYPE=avx2 ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
74+
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out kowiki1000.txt
75+
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --sbg kowiki1000.txt
76+
KIWI_ARCH_TYPE=avx512bw ./build/kiwi-cli-* -m ./models/base -e -o test.out --typos 6 kowiki1000.txt
7777
- name: Archive binaries
7878
uses: actions/upload-artifact@v3
7979
with:

Diff for: .github/workflows/windows.yml

+11-11
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ jobs:
3535
- name: Run Evaluator
3636
run: |
3737
mkdir eval_results
38-
.\build\Release\kiwi-evaluator.exe -m .\ModelGenerator (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
39-
.\build\Release\kiwi-evaluator.exe -m .\ModelGenerator --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
38+
.\build\Release\kiwi-evaluator.exe -m .\models\base (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
39+
.\build\Release\kiwi-evaluator.exe -m .\models\base --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
4040
- name: Archive binaries
4141
uses: actions/upload-artifact@v3
4242
with:
@@ -59,21 +59,21 @@ jobs:
5959
- name: Run Benchmark
6060
run: |
6161
curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt
62-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
62+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
6363
$env:KIWI_ARCH_TYPE = 'none'
64-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
64+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
6565
$env:KIWI_ARCH_TYPE = 'balanced'
66-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
66+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
6767
$env:KIWI_ARCH_TYPE = 'sse2'
68-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
68+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
6969
if ( '${{ matrix.arch }}' -eq 'x64' )
7070
{
7171
$env:KIWI_ARCH_TYPE = 'sse4_1'
72-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
72+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
7373
$env:KIWI_ARCH_TYPE = 'avx2'
74-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
74+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
7575
$env:KIWI_ARCH_TYPE = 'avx512bw'
76-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out kowiki1000.txt
76+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out kowiki1000.txt
7777
}
78-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --sbg kowiki1000.txt
79-
.\build\Release\kiwi-cli-*.exe -m .\ModelGenerator -e -o test.out --typos 6 kowiki1000.txt
78+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --sbg kowiki1000.txt
79+
.\build\Release\kiwi-cli-*.exe -m .\models\base -e -o test.out --typos 6 kowiki1000.txt

Diff for: ModelGenerator/sj.knlm

-3
This file was deleted.

Diff for: ModelGenerator/sj.morph

-3
This file was deleted.

Diff for: ModelGenerator/skipbigram.mdl

-3
This file was deleted.

Diff for: README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ $ ldconfig
8080

8181
설치가 잘 됐는지 확인하기 위해서는 `kiwi-evaluator`를 실행해봅니다.
8282
```console
83-
$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg
83+
$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg
8484
Loading Time : 981.745 ms
8585
ArchType : avx2
8686
LM Size : 34.1853 MB
@@ -110,7 +110,7 @@ Avg Score
110110

111111
0.13.0 버전부터 추가된 오타 교정 기능이 잘 작동하는지 확인하기 위해서는 다음과 같이 실행합니다.
112112
```console
113-
$ ./kiwi-evaluator --model ../ModelGenerator ../eval_data/* --sbg --typo 6
113+
$ ./kiwi-evaluator --model ../models/base ../eval_data/* --sbg --typo 6
114114
Loading Time : 9414.45 ms
115115
ArchType : avx2
116116
LM Size : 34.1853 MB

Diff for: bindings/java/README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ KiwiJava의 jar 파일은 자체적으로 실행 가능한 main함수를 가지
1212

1313
# 모델 파일 다운로드
1414
$ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.0/kiwi_model_v0.16.0_base.tgz
15-
$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 ModelGenerator라는 폴더가 생성됨
15+
$ tar -zxvf kiwi_model_v0.16.0_base.tgz # 압축 해제. 모델을 포함한 models/base라는 폴더가 생성됨
1616

1717
# KiwiJava 다운로드
1818
$ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16.1-lnx-x86-64.jar
@@ -21,7 +21,7 @@ $ wget https://github.com/bab2min/Kiwi/releases/download/v0.16.1/kiwi-java-v0.16
2121
# 즉, kiwi_model_v0.16.0은 kiwi-java-v0.16.* 전부에서 사용가능합니다.
2222

2323
# jar 구동. 구동 인자로 모델 경로를 입력해주어야 함
24-
$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar ModelGenerator/
24+
$ java -jar kiwi-java-v0.16.1-lnx-x86-64.jar models/base/
2525
Kiwi 0.16.1 is loaded!
2626
>> 안녕하세요?
2727
[Token(form=안녕, tag=NNG, position=0, length=2),

Diff for: bindings/java/kr/pe/bab2min/KiwiTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
public class KiwiTest {
1313

14-
private static String modelPath = "../../ModelGenerator";
14+
private static String modelPath = "../../models/base";
1515
private static Kiwi reusableInst = null;
1616

1717
Kiwi getReusableKiwi() throws Exception {

Diff for: bindings/wasm/package-demo/vite.config.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { viteStaticCopy } from 'vite-plugin-static-copy';
33
import { requiredModelFiles } from './src/modelFiles';
44

55
const targets = requiredModelFiles.map((file) => ({
6-
src: '../../../ModelGenerator/' + file,
6+
src: '../../../models/base/' + file,
77
dest: 'model',
88
}));
99

Diff for: include/kiwi/capi.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ DECL_DLL int kiwi_typo_close(kiwi_typo_h handle);
442442
/**
443443
* @brief KiwiBuilder를 거치지 않고 바로 Kiwi instance를 생성합니다.
444444
*
445-
* @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./ModelGenerator).
445+
* @param model_path 모델이 들어있는 디렉토리 경로 (e.g., ./models/base).
446446
* @param num_threads 사용할 쓰레드의 수 (0일 경우, 자동으로 설정).
447447
* @param options 생성 옵션. KIWI_BUILD_* 참조.
448448
* @return Kiwi의 핸들.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.

Diff for: models/base/sj.knlm

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:52d50761ed4aeea82e3be9f6fbb4724b75f526e56368bdc76dd530049ef9a07e
3+
size 35828400

Diff for: models/base/sj.morph

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:1c8a9999841059a00ef2c76dc190f651e698c0516209da524930c9818f01279c
3+
size 3581294

Diff for: models/base/skipbigram.mdl

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
version https://git-lfs.github.com/spec/v1
2+
oid sha256:ae4f0ee268f516bf4dfe282337f8773ab900ce081fefc1167b5c29259b1c465b
3+
size 3186444
File renamed without changes.

Diff for: test/common.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#include <iostream>
55
#include <fstream>
66

7-
#define MODEL_PATH "./ModelGenerator"
7+
#define MODEL_PATH "./models/base"
88
#define KWORD u"킼윜"
99
#define KWORD8 u8"킼윜"
1010

Diff for: test/test_cpp.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ TEST(KiwiCpp, HSDataset)
365365
{
366366
KiwiBuilder kw{ MODEL_PATH, 0, BuildOption::default_, };
367367
std::vector<std::string> data;
368-
data.emplace_back(MODEL_PATH "/testHSDataset.txt");
368+
data.emplace_back("./ModelGenerator/testHSDataset.txt");
369369

370370
static constexpr size_t batchSize = 32, windowSize = 8;
371371

Diff for: tools/evaluator_main.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ int main(int argc, const char* argv[])
125125
{
126126
CmdLine cmd{ "Kiwi evaluator" };
127127

128-
ValueArg<string> model{ "m", "model", "Kiwi model path", false, "ModelGenerator", "string" };
128+
ValueArg<string> model{ "m", "model", "Kiwi model path", false, "models/base", "string" };
129129
ValueArg<string> output{ "o", "output", "output dir for evaluation errors", false, "", "string" };
130130
SwitchArg noNormCoda{ "", "no-normcoda", "without normalizing coda", false };
131131
SwitchArg noZCoda{ "", "no-zcoda", "without z-coda", false };

0 commit comments

Comments
 (0)