kiwipiepy/.github/workflows/pull_request_test.yml at ee09a33fb5b149c65832297fdab1fdedda132072 · bab2min/kiwipiepy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
name: Pull Request Test

on:
  pull_request:
    branches:
      - main

jobs:
  build_manylinux2014:
    name: Build for manylinux2014
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        cp: [cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313]

    steps:
    - uses: actions/checkout@v3
      with:
        submodules: recursive
        lfs: true
    - uses: bab2min/run-on-arch-action@multiple-step-v2
      id: runcmd
      with:
        image: quay.io/pypa/manylinux2014_x86_64
        githubToken: ${{ github.token }}
        multipleRun: |
          - name: Build
            run: |
              mv Kiwi/models/base/sj.* model/kiwipiepy_model/
              mv Kiwi/models/base/extract.mdl model/kiwipiepy_model/
              mv Kiwi/models/base/*.dict model/kiwipiepy_model/
              mv Kiwi/models/base/combiningRule.txt model/kiwipiepy_model/
              mv Kiwi/models/base/skipbigram.mdl model/kiwipiepy_model/

              /opt/python/${{ matrix.cp }}/bin/pip install "cmake<4"
              rm /usr/local/bin/cmake || true
              ln -s /opt/python/${{ matrix.cp }}/bin/cmake /usr/local/bin/cmake

              /opt/python/${{ matrix.cp }}/bin/python -m pip install setuptools numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py` tqdm

              cd model
              /opt/python/${{ matrix.cp }}/bin/python setup.py build install
              cd ..
              /opt/python/${{ matrix.cp }}/bin/python setup.py build install
              /opt/python/${{ matrix.cp }}/bin/python -m pip install numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py v1` || true
          - name: Test kiwipiepy
            run: |
              /opt/python/${{ matrix.cp }}/bin/python -m pip install pytest
              /opt/python/${{ matrix.cp }}/bin/python -m pytest -svv test/test_kiwipiepy.py
          - name: Test transformers_addon
            run: |
              for v in {12..46}
              do
                echo "Test with transformers 4.$v ..."
                if /opt/python/${{ matrix.cp }}/bin/python -m pip install -U "transformers<4.$(($v+1))"; then
                  /opt/python/${{ matrix.cp }}/bin/python -m pytest --verbose test/test_transformers_addon.py
                fi
              done
    - run: tar -zcvf build.tgz build
    - name: Archive binary
      uses: actions/upload-artifact@v4
      with:
        name: Linux Binary ${{ matrix.cp }}
        path: build.tgz

  build_macos_13:
    name: Build for macOS 13
    runs-on: macOS-13
    strategy:
      max-parallel: 4
      matrix:
        python-version: ["3.10", 3.11, 3.12, 3.13]

    steps:
    - uses: actions/checkout@v2
      with:
        submodules: recursive
        lfs: true
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
    - name: Build
      run: |
        mv Kiwi/models/base/sj.* model/kiwipiepy_model/
        mv Kiwi/models/base/extract.mdl model/kiwipiepy_model/
        mv Kiwi/models/base/*.dict model/kiwipiepy_model/
        mv Kiwi/models/base/combiningRule.txt model/kiwipiepy_model/
        mv Kiwi/models/base/skipbigram.mdl model/kiwipiepy_model/

        python -m pip install numpy==`python .github/workflows/numpy_version.py` setuptools tqdm

        cd model
        python setup.py build install
        cd ..

        MACOSX_DEPLOYMENT_TARGET=10.14 KIWI_CPU_ARCH=arm64 USE_MIMALLOC=1 python setup.py build
        MACOSX_DEPLOYMENT_TARGET=10.14 KIWI_CPU_ARCH=x86_64 USE_MIMALLOC=1 python setup.py build install
        python -m pip install numpy==`python .github/workflows/numpy_version.py v1` || true
    - name: Archive binary
      uses: actions/upload-artifact@v4
      with:
        name: macOS Binary ${{ matrix.python-version }}
        path: |
          build/*
    - name: Test kiwipiepy
      run: |
        python -m pip install pytest
        python -m pytest -svv test/test_kiwipiepy.py

  build_windows:
    name: Build for Windows
    runs-on: windows-2019
    strategy:
      max-parallel: 5
      matrix:
        python-version: ["3.10", 3.11, 3.12, 3.13]
        architecture: [x86, x64]

    steps:
    - uses: actions/checkout@v2
      with:
        submodules: recursive
        lfs: true
    - name: Set up Python ${{ matrix.python-version }} ${{ matrix.architecture }}
      uses: actions/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
        architecture: ${{ matrix.architecture }}
    - name: Build
      run: |
        python -m pip install --upgrade pip setuptools tqdm
        python -m pip install numpy==$(python .github/workflows/numpy_version.py)

        mv Kiwi/models/base/sj.* model/kiwipiepy_model/
        mv Kiwi/models/base/extract.mdl model/kiwipiepy_model/
        mv Kiwi/models/base/*.dict model/kiwipiepy_model/
        mv Kiwi/models/base/combiningRule.txt model/kiwipiepy_model/
        mv Kiwi/models/base/skipbigram.mdl model/kiwipiepy_model/

        cd model
        python setup.py build install
        cd ..
        $env:USE_MIMALLOC = 1; python setup.py build install
        Try { python -m pip install numpy==$(python .github/workflows/numpy_version.py v1) } Catch {}
    - name: Archive binary
      uses: actions/upload-artifact@v4
      with:
        name: Windows Binary ${{ matrix.python-version }} ${{ matrix.architecture }}
        path: |
          build/*
    - name: Test kiwipiepy
      run: |
        python -m pip install pytest
        python -m pytest -vv test/test_kiwipiepy.py

  build_other_arch:
    name: Build for manylinux (other arch)
    runs-on: ubuntu-24.04-arm
    strategy:
      max-parallel: 8
      matrix:
        cp: [cp310-cp310, cp311-cp311, cp312-cp312, cp313-cp313]
        arch: [aarch64]

    steps:
    - uses: actions/checkout@v2
      with:
        submodules: recursive
        lfs: true
    - uses: bab2min/run-on-arch-action@multiple-step-v2
      id: runcmd
      with:
        image: quay.io/pypa/manylinux2014_${{ matrix.arch }}
        githubToken: ${{ github.token }}
        setup: |
          mkdir -p "${PWD}/artifacts"
        dockerRunArgs: |
          --volume "${PWD}/artifacts:/artifacts"
        multipleRun: |
          - name: Copy Model files
            run: |
              mv Kiwi/models/base/sj.* model/kiwipiepy_model/
              mv Kiwi/models/base/extract.mdl model/kiwipiepy_model/
              mv Kiwi/models/base/*.dict model/kiwipiepy_model/
              mv Kiwi/models/base/combiningRule.txt model/kiwipiepy_model/
              mv Kiwi/models/base/skipbigram.mdl model/kiwipiepy_model/
          - name: Install dependencies
            run: |
              /opt/python/${{ matrix.cp }}/bin/pip install "cmake<4"
              rm /usr/local/bin/cmake || true
              ln -s /opt/python/${{ matrix.cp }}/bin/cmake /usr/local/bin/cmake

              /opt/python/${{ matrix.cp }}/bin/python -m pip install wheel setuptools numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py` tqdm
          - name: Build model
            run: |
              cd model
              /opt/python/${{ matrix.cp }}/bin/python setup.py build install
              cd ..
          - name: Build
            run: |
              /opt/python/${{ matrix.cp }}/bin/python setup.py build install bdist_wheel
              /opt/python/${{ matrix.cp }}/bin/python -m pip install numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py v1` || true
              tar -zcvf /artifacts/build.tgz build/*
              cp -r dist /artifacts/
          - name: Test
            run: |
              /opt/python/${{ matrix.cp }}/bin/python -m pip install pytest
              /opt/python/${{ matrix.cp }}/bin/python -m pytest -svv test/test_kiwipiepy.py

    - name: Archive binary
      uses: actions/upload-artifact@v4
      with:
        name: Linux ${{ matrix.arch }} Binary ${{ matrix.cp }}
        path: artifacts/build.tgz
    - name: Archive binary
      uses: actions/upload-artifact@v4
      with:
        name: Linux ${{ matrix.arch }} Wheel ${{ matrix.cp }}
        path: artifacts/dist/*.whl