Skip to content

Commit 3f81328

Browse files
bab2minbab2min
committed
Add NUM_AVAILABLE_CPU_CORES environment variable support for parallel builds
1 parent 411a4ee commit 3f81328

4 files changed

Lines changed: 15 additions & 12 deletions

File tree

‎.github/workflows/deploy.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,12 @@ jobs:
4747
/opt/python/cp311-cp311/bin/python -m pip install twine wheel numpy
4848
/opt/python/cp311-cp311/bin/python setup.py sdist
4949
/opt/python/cp311-cp311/bin/python -m twine upload dist/*.tar.gz
50-
USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
50+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
5151
auditwheel repair dist/*-linux_x86_64.whl
5252
for cp in cp314-cp314t
5353
do
5454
/opt/python/${cp}/bin/python -m pip install wheel setuptools numpy==`/opt/python/${cp}/bin/python .github/workflows/numpy_version.py`
55-
USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
55+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
5656
auditwheel repair dist/*-${cp}-linux_x86_64.whl
5757
done
5858
/opt/python/cp311-cp311/bin/python -m twine upload wheelhouse/*.whl
@@ -242,12 +242,12 @@ jobs:
242242
rm /usr/local/bin/cmake || true
243243
ln -s /opt/python/cp311-cp311/bin/cmake /usr/local/bin/cmake
244244
/opt/python/cp311-cp311/bin/pip install -U wheel setuptools numpy twine
245-
USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
245+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
246246
auditwheel repair dist/*.whl
247247
for cp in cp314-cp314t
248248
do
249249
/opt/python/${cp}/bin/pip install -U wheel setuptools numpy==`/opt/python/${cp}/bin/python .github/workflows/numpy_version.py`
250-
USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
250+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
251251
auditwheel repair dist/*-${cp}-*.whl
252252
done
253253
/opt/python/cp311-cp311/bin/twine upload wheelhouse/*.whl

‎.github/workflows/deploy_test.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,12 @@ jobs:
4444
/opt/python/cp311-cp311/bin/python -m pip install twine wheel numpy
4545
/opt/python/cp311-cp311/bin/python setup.py sdist
4646
/opt/python/cp311-cp311/bin/python -m twine upload --repository testpypi dist/*.tar.gz
47-
USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
47+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
4848
auditwheel repair dist/*-linux_x86_64.whl
4949
for cp in cp314-cp314t
5050
do
5151
/opt/python/${cp}/bin/python -m pip install wheel setuptools numpy==`/opt/python/${cp}/bin/python .github/workflows/numpy_version.py`
52-
USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
52+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
5353
auditwheel repair dist/*-${cp}-linux_x86_64.whl
5454
done
5555
/opt/python/cp311-cp311/bin/python -m twine upload --repository testpypi wheelhouse/*.whl
@@ -239,12 +239,12 @@ jobs:
239239
rm /usr/local/bin/cmake || true
240240
ln -s /opt/python/cp311-cp311/bin/cmake /usr/local/bin/cmake
241241
/opt/python/cp311-cp311/bin/pip install -U wheel setuptools numpy twine
242-
USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
242+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 Py_LIMITED_API=1 /opt/python/cp311-cp311/bin/python setup.py build bdist_wheel
243243
auditwheel repair dist/*.whl
244244
for cp in cp314-cp314t
245245
do
246246
/opt/python/${cp}/bin/pip install -U wheel setuptools numpy==`/opt/python/${cp}/bin/python .github/workflows/numpy_version.py`
247-
USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
247+
NUM_AVAILABLE_CPU_CORES=2 USE_MIMALLOC=1 /opt/python/${cp}/bin/python setup.py build bdist_wheel
248248
auditwheel repair dist/*-${cp}-*.whl
249249
done
250250
/opt/python/cp311-cp311/bin/twine upload --repository testpypi wheelhouse/*.whl

‎.github/workflows/pull_request_test.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
cd model
4242
/opt/python/${{ matrix.cp }}/bin/python setup.py build install
4343
cd ..
44-
/opt/python/${{ matrix.cp }}/bin/python setup.py build install
44+
NUM_AVAILABLE_CPU_CORES=2 /opt/python/${{ matrix.cp }}/bin/python setup.py build install
4545
/opt/python/${{ matrix.cp }}/bin/python -m pip install numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py v1` || true
4646
- name: Test kiwipiepy
4747
run: |
@@ -102,7 +102,7 @@ jobs:
102102
cd model
103103
/opt/python/cp39-cp39/bin/python setup.py bdist_wheel
104104
cd ..
105-
Py_LIMITED_API=1 /opt/python/cp39-cp39/bin/python setup.py bdist_wheel
105+
NUM_AVAILABLE_CPU_CORES=2 Py_LIMITED_API=1 /opt/python/cp39-cp39/bin/python setup.py bdist_wheel
106106
- name: Test kiwipiepy
107107
run: |
108108
/opt/python/${{ matrix.cp }}/bin/python -m pip install model/dist/*.whl
@@ -458,7 +458,7 @@ jobs:
458458
cd ..
459459
- name: Build
460460
run: |
461-
/opt/python/${{ matrix.cp }}/bin/python setup.py build install bdist_wheel
461+
NUM_AVAILABLE_CPU_CORES=2 /opt/python/${{ matrix.cp }}/bin/python setup.py build install bdist_wheel
462462
/opt/python/${{ matrix.cp }}/bin/python -m pip install numpy==`/opt/python/${{ matrix.cp }}/bin/python .github/workflows/numpy_version.py v1` || true
463463
tar -zcvf /artifacts/build.tgz build/*
464464
cp -r dist /artifacts/
@@ -522,7 +522,7 @@ jobs:
522522
cd ..
523523
- name: Build
524524
run: |
525-
Py_LIMITED_API=1 /opt/python/cp39-cp39/bin/python setup.py bdist_wheel
525+
NUM_AVAILABLE_CPU_CORES=2 Py_LIMITED_API=1 /opt/python/cp39-cp39/bin/python setup.py bdist_wheel
526526
tar -zcvf /artifacts/build.tgz build/*
527527
cp -r dist /artifacts/
528528
- name: Test

‎setup.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,9 @@ def get_extra_cmake_options():
7777
return _cmake_extra_options, _clean_build_folder
7878

7979
def num_available_cpu_cores(ram_per_build_process_in_gb):
80+
if os.environ.get('NUM_AVAILABLE_CPU_CORES'):
81+
return int(os.environ.get('NUM_AVAILABLE_CPU_CORES'))
82+
8083
import multiprocessing
8184
try:
8285
mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')

0 commit comments

Comments
 (0)