Skip to content

Commit ea31bec

Browse files
jameslambStrikerRUS
jameslamb
and
StrikerRUS
authored
[ci] move more Python jobs to GitHub Actions (#7096)
* [ci] move more Python jobs to GitHub Actions * produce sdist artifacts * download-artifacts updates * maybe the bdist jobs need to be run on the CUDA runners for test_dual.py? * test GPU-and-CPU dual code on GitHub runners * add code testing dual support * try again * installing the library would help * test the hardward on Azure DevOps too * run clang-17 bdist jobs on a machine with a real GPU * typo * set CUDA version * fix workflow file * container.options cannot be an empty string * remove nvidia-smi * yamllint * restore appveyor * remove bdist jobs that were added for debugging * container-options is not used * Apply suggestions from code review Co-authored-by: Nikita Titov <[email protected]> * one more 'method: wheel' --------- Co-authored-by: Nikita Titov <[email protected]>
1 parent 09272a0 commit ea31bec

File tree

3 files changed

+75
-37
lines changed

3 files changed

+75
-37
lines changed

‎.ci/download-artifacts.sh‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@
1111
# Based on the list from https://github.com/microsoft/LightGBM/releases/tag/v4.6.0.
1212
# The following are not yet handled by this script.
1313
#
14-
# - lib_lightgbm.so
15-
# - lightgbm-4.6.0-py3-none-manylinux_2_28_x86_64.whl
16-
# - lightgbm-4.6.0.tar.gz
1714
# - LightGBM.4.6.0.nupkg
1815
#
1916

‎.github/workflows/python_package.yml‎

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ env:
2222
CMAKE_BUILD_PARALLEL_LEVEL: 4
2323
# avoid interactive prompts in Debian-based distributions
2424
DEBIAN_FRONTEND: noninteractive
25+
# On runners using nvidia-container-runtime with Docker, ensure GPUs are available to running processes.
26+
#
27+
# ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/docker-specialized.html
28+
NVIDIA_VISIBLE_DEVICES: 'all'
2529
SKBUILD_STRICT_CONFIG: true
2630

2731
jobs:
@@ -97,12 +101,43 @@ jobs:
97101
python_version: '3.12'
98102
- os: macos-15-intel
99103
task: bdist
104+
method: wheel
100105
compiler: gcc
101106
python_version: '3.9'
102107
- os: windows-2022
103108
task: sdist
104109
compiler: MSVC
105110
python_version: '3.10'
111+
- os: ubuntu-latest
112+
container: &ubuntu_latest_image ubuntu:22.04
113+
os-display-name: &ubuntu_latest_display_name 'ubuntu22.04'
114+
task: regular
115+
compiler: clang-17
116+
python_version: '3.13'
117+
in-ubuntu-base-container: 'true'
118+
- os: ubuntu-latest
119+
container: *ubuntu_latest_image
120+
os-display-name: *ubuntu_latest_display_name
121+
task: sdist
122+
compiler: clang-17
123+
python_version: '3.13'
124+
in-ubuntu-base-container: 'true'
125+
#########################
126+
# OpenCL-based GPU jobs #
127+
#########################
128+
# clang-17 jobs are run on machines with actual GPUs because
129+
# the package built with that compiler toolchain can't target the
130+
# CPUs on GitHub's runners.
131+
#
132+
# ref: https://github.com/microsoft/LightGBM/pull/7096/files#r2590879590
133+
- os: LightGBM-GPU
134+
container: nvcr.io/nvidia/cuda:12.9.1-devel-ubuntu22.04
135+
os-display-name: ubuntu22.04-with-gpu
136+
task: bdist
137+
compiler: clang-17
138+
method: wheel
139+
python_version: '3.11'
140+
in-ubuntu-base-container: 'true'
106141
############
107142
# MPI jobs #
108143
############
@@ -122,17 +157,17 @@ jobs:
122157
method: wheel
123158
python_version: '3.10'
124159
- os: ubuntu-latest
125-
container: lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64
126-
os-display-name: 'manylinux_2_28'
160+
container: &manylinux_amd64_image lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64
161+
os-display-name: &manylinux_amd64_display_name 'manylinux_2_28'
127162
task: mpi
128163
compiler: gcc
129164
method: source
130165
python_version: '3.10'
131166
in-ubuntu-base-container: 'false'
132167
setup-conda: 'false'
133168
- os: ubuntu-latest
134-
container: &ubuntu_latest_image ubuntu:22.04
135-
os-display-name: &ubuntu_latest_display_name 'ubuntu22.04'
169+
container: *ubuntu_latest_image
170+
os-display-name: *ubuntu_latest_display_name
136171
task: mpi
137172
compiler: clang-17
138173
method: source
@@ -178,6 +213,37 @@ jobs:
178213
produces-artifacts: 'true'
179214
artifact-name: 'macosx-amd64-liblightgbm'
180215
python_version: '3.11'
216+
- os: ubuntu-latest
217+
container: *manylinux_amd64_image
218+
os-display-name: *manylinux_amd64_display_name
219+
task: regular
220+
compiler: gcc
221+
python_version: '3.11'
222+
in-ubuntu-base-container: 'false'
223+
setup-conda: 'false'
224+
produces-artifacts: 'true'
225+
artifact-name: 'linux-amd64-liblightgbm'
226+
- os: ubuntu-latest
227+
container: *manylinux_amd64_image
228+
os-display-name: *manylinux_amd64_display_name
229+
task: bdist
230+
compiler: gcc
231+
method: wheel
232+
python_version: '3.10'
233+
in-ubuntu-base-container: 'false'
234+
setup-conda: 'false'
235+
produces-artifacts: 'true'
236+
artifact-name: 'linux-amd64-wheel'
237+
- os: ubuntu-latest
238+
container: *manylinux_amd64_image
239+
os-display-name: *manylinux_amd64_display_name
240+
task: sdist
241+
compiler: gcc
242+
python_version: '3.9'
243+
in-ubuntu-base-container: 'false'
244+
setup-conda: 'false'
245+
produces-artifacts: 'true'
246+
artifact-name: 'sdist'
181247
- os: windows-2022
182248
task: bdist
183249
compiler: MSVC
@@ -223,7 +289,7 @@ jobs:
223289
persist-credentials: false
224290
submodules: true
225291
- name: Setup and run tests on Linux and macOS
226-
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu')
292+
if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') || matrix.os == 'LightGBM-GPU'
227293
shell: bash
228294
run: |
229295
export COMPILER="${{ matrix.compiler }}"
@@ -233,7 +299,7 @@ jobs:
233299
export METHOD="${{ matrix.method }}"
234300
if [[ "${{ matrix.os }}" =~ ^macos ]]; then
235301
export OS_NAME="macos"
236-
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
302+
elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]] || [[ "${{ matrix.os }}" == "LightGBM-GPU" ]]; then
237303
export OS_NAME="linux"
238304
export PATH="/usr/lib64/openmpi/bin:${PATH}"
239305
fi
@@ -279,6 +345,8 @@ jobs:
279345
artifacts/*.dll
280346
artifacts/*.dylib
281347
artifacts/*.exe
348+
artifacts/*.so
349+
artifacts/*.tar.gz
282350
artifacts/*.whl
283351
if-no-files-found: error
284352
test-latest-versions:

‎.vsts-ci.yml‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -78,20 +78,11 @@ jobs:
7878
COMPILER: gcc
7979
SETUP_CONDA: 'false'
8080
OS_NAME: 'linux'
81-
PRODUCES_ARTIFACTS: 'true'
81+
PRODUCES_ARTIFACTS: 'false'
8282
pool: lightgbm_agent_pool_ado
8383
container: linux-artifact-builder
8484
strategy:
8585
matrix:
86-
regular:
87-
TASK: regular
88-
PYTHON_VERSION: '3.11'
89-
sdist:
90-
TASK: sdist
91-
PYTHON_VERSION: '3.9'
92-
bdist:
93-
TASK: bdist
94-
PYTHON_VERSION: '3.10'
9586
gpu_source:
9687
TASK: gpu
9788
METHOD: source
@@ -113,17 +104,6 @@ jobs:
113104
inputs:
114105
filePath: $(Build.SourcesDirectory)/.ci/test.sh
115106
targetType: filePath
116-
- task: PublishBuildArtifacts@1
117-
condition: >
118-
and(
119-
succeeded(),
120-
in(variables['TASK'], 'regular', 'sdist', 'bdist'),
121-
not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
122-
)
123-
inputs:
124-
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
125-
artifactName: PackageAssets
126-
artifactType: container
127107
################
128108
# Linux_latest #
129109
################
@@ -138,13 +118,6 @@ jobs:
138118
container: ubuntu-latest
139119
strategy:
140120
matrix:
141-
regular:
142-
TASK: regular
143-
sdist:
144-
TASK: sdist
145-
bdist:
146-
TASK: bdist
147-
PYTHON_VERSION: '3.11'
148121
gpu_source:
149122
TASK: gpu
150123
METHOD: source

0 commit comments

Comments
 (0)