Skip to content

Commit 31fa404

Browse files
BrianPeiBrianPei
committed
restore nv original ymls
1 parent cfa308f commit 31fa404

21 files changed

Lines changed: 2090 additions & 0 deletions

‎.github/CODEOWNERS‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
megatron/core/ @NVIDIA/core-adlr @NVIDIA/core-nemo
2+
3+
megatron/core/models/gpt/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/gpt
4+
5+
megatron/core/models/multimodal/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/multi-modal
6+
7+
megatron/core/models/mamba/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
8+
megatron/core/ssm/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
9+
10+
megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
11+
12+
megatron/core/distributed/fsdp/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
13+
14+
megatron/core/transformer/fsdp_dtensor_checkpoint.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
15+
16+
megatron/core/dist_checkpointing/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-checkpointing
17+
18+
megatron/core/optimizer/distrib_optimizer/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/dist-optimizer
19+
20+
megatron/core/inference/modelopt_support @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/quantization-and-inference
21+
22+
megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
23+
24+
megatron/core/pipeline_parallel/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/pipeline-parallelism
25+
26+
megatron/core/transformer/ @NVIDIA/core-adlr @NVIDIA/core-nemo
27+
28+
megatron/core/transformer/moe/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/mixture-of-experts-adlr @NVIDIA/mixture-of-experts-devtech
29+
30+
megatron/core/inference/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/inference
31+
32+
megatron/core/parallel_state.py @NVIDIA/core-adlr @NVIDIA/core-nemo
33+
34+
megatron/core/post_training/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/post-training
35+
36+
megatron/post_training/ @NVIDIA/post-training
37+
38+
.gitlab/ @NVIDIA/ci
39+
.github/ @NVIDIA/ci
40+
.gitlab-ci.yml @NVIDIA/ci
41+
docker/ @NVIDIA/ci
42+
tests/functional_tests/python_test_utils/ @NVIDIA/ci
43+
tests/functional_tests/shell_test_utils/ @NVIDIA/ci
44+
tests/test_utils/recipes/ @NVIDIA/ci
45+
tests/unit_tests/run_ci_test.sh @NVIDIA/ci
46+
47+
megatron/rl/ @NVIDIA/reinforcement-learning
48+
examples/rl/ @NVIDIA/reinforcement-learning
49+
test/unit_tests/test_rl_utils.py @NVIDIA/reinforcement-learning
50+
train_rl.py @NVIDIA/reinforcement-learning

‎.github/copy-pr-bot.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
enabled: true
2+
auto_sync_draft: false
3+
auto_sync_ready: true
4+
trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "gautham-kollu", "guyueh1", "hxbai", "jaredcasper", "jiemingz", "jkamalu", "jon-barker", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mehraakash", "mkhona-nvidia", "pablo-garay", "parthmannan", "pthombre", "rogerwaleffe", "sanandaraj5597", "santhnm2", "sbak5", "shanmugamr1992", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"]

‎.github/workflows/_build_test_publish_wheel.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
ref:
5+
required: false
6+
description: Ref (SHA or branch) to release
7+
type: string
8+
default: ${{ github.sha }}
9+
dry-run:
10+
required: false
11+
description: Upload to PyPy Test instance
12+
type: boolean
13+
default: true
14+
no-publish:
15+
required: false
16+
description: Do not publish the wheel
17+
type: boolean
18+
default: true
19+
secrets:
20+
TWINE_USERNAME:
21+
required: true
22+
TWINE_PASSWORD:
23+
required: true
24+
25+
jobs:
26+
build-and-test-wheels:
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
include:
31+
- PACKAGE: megatron-core
32+
PLATFORM: arm64
33+
IMAGE: quay.io/pypa/manylinux_2_28_aarch64
34+
- PACKAGE: megatron-core
35+
PLATFORM: amd64
36+
IMAGE: quay.io/pypa/manylinux_2_28_x86_64
37+
- PACKAGE: megatron-fsdp
38+
IMAGE: quay.io/pypa/manylinux_2_28_x86_64
39+
PLATFORM: amd64
40+
runs-on: ${{ matrix.PLATFORM == 'amd64' && 'ubuntu-22.04' || 'ubuntu-22.04-arm' }}
41+
env:
42+
PACKAGE: ${{ matrix.PACKAGE }}
43+
IMAGE: ${{ matrix.IMAGE }}
44+
PLATFORM: ${{ matrix.PLATFORM }}
45+
PUBLISH_DRYRUN: ${{ inputs.dry-run }}
46+
steps:
47+
- name: Checkout repository
48+
uses: actions/checkout@v4
49+
with:
50+
ref: ${{ inputs.ref }}
51+
52+
- name: Build wheel
53+
id: build-wheel
54+
run: |
55+
set -x
56+
57+
if [ "$PACKAGE" = "megatron-core" ]; then
58+
ROOTDIR="megatron/core"
59+
BUILD_DIR="."
60+
elif [ "$PACKAGE" = "megatron-fsdp" ]; then
61+
ROOTDIR="megatron/core/distributed/fsdp/src/megatron_fsdp"
62+
BUILD_DIR="megatron/core/distributed/fsdp/src"
63+
else
64+
echo Unknown package: $PACKAGE
65+
exit 1
66+
fi
67+
68+
if [ "$PUBLISH_DRYRUN" = "true" ]; then
69+
PRE_RELEASE=$(sed -n "s/.*PRE_RELEASE = '\(.*\)'/\1/p" $ROOTDIR/package_info.py)
70+
sed -i "/^PRE_RELEASE/c\PRE_RELEASE = '${PRE_RELEASE}.dev$((RANDOM % 900000 + 100000))'" $ROOTDIR/package_info.py
71+
fi
72+
73+
pushd $BUILD_DIR
74+
rm LICENSE || true
75+
docker run --rm -v $(pwd):/workspace -w /workspace $IMAGE bash -c '\
76+
for python_version in cp310 cp311 cp312 cp313; do \
77+
/opt/python/${python_version}-${python_version}/bin/pip install --upgrade "setuptools>=80.0.0" build; \
78+
done && \
79+
for python_version in cp310 cp311 cp312 cp313; do \
80+
/opt/python/${python_version}-${python_version}/bin/python -m build; \
81+
done \
82+
'
83+
84+
PLATFORM_WHEELS=$(find dist -name "*.whl" -not -name "*-none-any.whl")
85+
if [ -n "$PLATFORM_WHEELS" ]; then
86+
echo "Found platform wheels to repair: $PLATFORM_WHEELS"
87+
docker run --rm -v $(pwd):/workspace -w /workspace $IMAGE auditwheel repair $PLATFORM_WHEELS
88+
docker run --rm -v $(pwd):/workspace -w /workspace $IMAGE rm -rf dist/*.whl
89+
docker run --rm -v $(pwd):/workspace -w /workspace $IMAGE cp -a wheelhouse/* dist/
90+
fi
91+
popd
92+
93+
pushd $ROOTDIR
94+
EXPECTED_RELEASE_NUMBER=$(python -c "import package_info; print(package_info.__version__)")
95+
popd
96+
97+
echo "expected-release-number=$EXPECTED_RELEASE_NUMBER" | tee -a "${GITHUB_OUTPUT}"
98+
99+
if [ "$PACKAGE" = "megatron-fsdp" ]; then
100+
mkdir -p dist/
101+
cp -a megatron/core/distributed/fsdp/src/dist/* dist/
102+
fi
103+
104+
ls -al dist/
105+
106+
- name: Test wheels
107+
run: |
108+
ls -al dist/
109+
110+
if [ "$PACKAGE" = "megatron-core" ]; then
111+
ROOTPATH="megatron.core"
112+
WHEEL_PREFIX="megatron_core"
113+
elif [ "$PACKAGE" = "megatron-fsdp" ]; then
114+
ROOTPATH="megatron_fsdp"
115+
WHEEL_PREFIX="megatron_fsdp"
116+
else
117+
echo Unknown package: $PACKAGE
118+
exit 1
119+
fi
120+
121+
if [ "$PACKAGE" = "megatron-core" ]; then
122+
if [[ "$PLATFORM" == "arm64" ]]; then
123+
for file in dist/$WHEEL_PREFIX*cp310*aarch64.whl; do
124+
pip install --no-cache-dir "$file"
125+
done
126+
else
127+
for file in dist/$WHEEL_PREFIX*cp310*x86_64.whl; do
128+
pip install --no-cache-dir "$file"
129+
done
130+
fi
131+
else
132+
pip install --no-cache-dir dist/$WHEEL_PREFIX*.whl
133+
fi
134+
135+
sudo rm -rf megatron/
136+
137+
RELEASE_NUMBER=$(python -c "import $ROOTPATH; print($ROOTPATH.__version__)")
138+
test "${{ steps.build-wheel.outputs.expected-release-number }}" == "$RELEASE_NUMBER"
139+
140+
- name: Upload wheels
141+
uses: actions/upload-artifact@v4
142+
with:
143+
name: wheels-${{ matrix.PACKAGE }}-${{ inputs.dry-run && 'dry-run' || 'release' }}
144+
path: dist/
145+
146+
publish-wheels:
147+
needs: [build-and-test-wheels]
148+
runs-on: ubuntu-latest
149+
if: inputs.no-publish == false
150+
environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && 'main' || 'public' }}
151+
strategy:
152+
fail-fast: false
153+
matrix:
154+
include:
155+
- PACKAGE: megatron_core
156+
- PACKAGE: megatron_fsdp
157+
env:
158+
PACKAGE: ${{ matrix.PACKAGE }}
159+
steps:
160+
- name: Download wheels
161+
uses: actions/download-artifact@v4
162+
with:
163+
name: wheels-${{ matrix.PACKAGE }}-${{ inputs.dry-run && 'dry-run' || 'release' }}
164+
path: dist/
165+
merge-multiple: true
166+
167+
- name: Publish wheels
168+
env:
169+
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
170+
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
171+
TWINE_REPOSITORY: ${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) && 'pypi' || 'testpypi' }}
172+
run: |
173+
ls -al dist/$PACKAGE*
174+
pip install twine
175+
twine upload -r $TWINE_REPOSITORY -u $TWINE_USERNAME -p $TWINE_PASSWORD dist/$PACKAGE*

0 commit comments

Comments
 (0)