-
Notifications
You must be signed in to change notification settings - Fork 51
/
Copy pathtox.ini
168 lines (157 loc) · 7.2 KB
/
tox.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
; For all tests that run in Github Actions, see
; .github/workflows/cpu-ci.yml for the workflow definition.
[tox]
envlist = gpu,gpu-cu11,multi-gpu,horovod-cpu,nvtabular-cpu,systems-cpu,transformers4rec-cpu,docs,docs-multi
[testenv]
commands =
pip install --upgrade pip
pip install -e .[all]
[testenv:gpu-cu11]
; Runs in: GitHub Actions
; Runs GPU-based tests.
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
PIP_EXTRA_INDEX_URL=https://pypi.nvidia.com
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
allowlist_externals =
bash
passenv =
CUDA_VISIBLE_DEVICES
deps =
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH}
nvidia-cudnn-cu11~=8.6.0
cudf-cu11=={env:RAPIDS_VERSION}
dask-cudf-cu11=={env:RAPIDS_VERSION}
commands =
bash -c 'python -m pytest --cov-report term --cov merlin -m "{env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
[testenv:gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
allowlist_externals =
bash
cp
deps =
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/systems.git@{env:MERLIN_BRANCH}
passenv =
OPAL_PREFIX
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
sitepackages=true
commands =
; copy system libs into virtualenv path (e.g. XGBoost)
bash -c 'cp $(python -c "import sys; print(sys.base_prefix)")/lib/*.so* $(python -c "import sys; print(sys.prefix)")/lib'
bash -c 'python -m pytest --cov-report term --cov merlin -m "{env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
[testenv:horovod-gpu]
; Runs in: Github Actions
; Runs GPU-based tests.
allowlist_externals =
horovodrun
sh
bash
passenv =
OPAL_PREFIX
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
CPATH={env:CPATH}{:}{envdir}/hugectr/include
LD_LIBRARY_PATH=${envdir}/hugectr/include/lib{:}/usr/local/lib/python3.10/dist-packages/tensorflow{:}{env:LD_LIBRARY_PATH}
LIBRARY_PATH=${envdir}/hugectr/lib{:}{env:LIBRARY_PATH}
sitepackages=true
deps =
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
commands =
sh examples/usecases/multi-gpu/install_sparse_operation_kit.sh {envdir}
bash -c 'horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh python -m pytest -m "unit and horovod {env:PYTEST_MARKERS}" -rxs {posargs:tests} || ([ $? = 5 ] && exit 0 || exit $?)'
[testenv:horovod-cpu]
setenv =
HOROVOD_WITH_MPI=1
HOROVOD_WITH_TENSORFLOW=1
PATH={env:PATH}{:}{envdir}/env/bin
LD_LIBRARY_PATH={env:LD_LIBRARY_PATH}{:}{envdir}/env/lib
deps =
git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH}
git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH}
commands =
conda update --yes --name base --channel defaults conda
conda env create --prefix {envdir}/env --file requirements/horovod-cpu-environment.yml --force
{envdir}/env/bin/python -m pip install 'horovod==0.27.0' --no-cache-dir
{envdir}/env/bin/horovodrun --check-build
{envdir}/env/bin/horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m "unit and horovod {env:PYTEST_MARKERS}" -rxs {posargs:tests}
[testenv:nvtabular-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/NVTabular.git nvtabular-{env:GIT_COMMIT}
python -m pip install --upgrade "./nvtabular-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./nvtabular-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest nvtabular-{env:GIT_COMMIT}/tests/unit
[testenv:systems-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
deps =
-rrequirements/base.txt
-rrequirements/dev.txt
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/systems.git systems-{env:GIT_COMMIT}
python -m pip install --upgrade "./systems-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./systems-{env:GIT_COMMIT}/requirements/test-cpu.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest -m "not notebook" systems-{env:GIT_COMMIT}/tests/unit
[testenv:transformers4rec-cpu]
passenv=GIT_COMMIT
allowlist_externals = git
commands =
; the GIT_COMMIT env is the current commit of the models repo
git clone --depth 1 --branch {env:MERLIN_BRANCH:main} https://github.com/NVIDIA-Merlin/Transformers4Rec.git Transformers4Rec-{env:GIT_COMMIT}
python -m pip install --upgrade -r "./Transformers4Rec-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade "./Transformers4Rec-{env:GIT_COMMIT}"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{env:MERLIN_BRANCH:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{env:MERLIN_BRANCH:main}
python -m pip install .
python -m pytest Transformers4Rec-{env:GIT_COMMIT}/tests/unit
[testenv:docs]
; Runs in: Github Actions
; Generates documentation with sphinx. There are other steps in the Github Actions workflow
; to publish the documentation on release.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git
git+https://github.com/NVIDIA-Merlin/dataloader.git
git+https://github.com/NVIDIA-Merlin/NVTabular.git
commands =
python -m sphinx.cmd.build -E -P -b html docs/source docs/build/html
[testenv:docs-multi]
; Run the multi-version build that is shown on GitHub Pages.
changedir = {toxinidir}
deps =
-rrequirements/docs.txt
-rrequirements/test.txt
git+https://github.com/NVIDIA-Merlin/core.git
git+https://github.com/NVIDIA-Merlin/dataloader.git
git+https://github.com/NVIDIA-Merlin/NVTabular.git
commands =
sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
sphinx-multiversion docs/source docs/build/html