Skip to content

Commit 70fb444

Browse files
Darryl233Darryl233
authored
Merge branch 'flagos-ai:main' into main
2 parents ec79f86 + acced6d commit 70fb444

89 files changed

Lines changed: 10625 additions & 1899 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/blossom-ci.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
# See LICENSE for license information.
44

55
# A workflow to trigger ci on hybrid infra (github + self hosted runner)
6+
7+
# DISABLED in FlagOS
68
name: Blossom-CI
79
on:
810
issue_comment:
9-
types: [created]
11+
types: [__disabled_do_not_remove__]
1012
workflow_dispatch:
1113
inputs:
1214
platform:

‎.github/workflows/build.yml‎

Lines changed: 14 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -8,90 +8,30 @@ on:
88
pull_request:
99
workflow_dispatch:
1010
jobs:
11-
core:
12-
name: 'Core'
13-
runs-on: ubuntu-latest
14-
container:
15-
image: nvcr.io/nvidia/cuda:12.1.0-devel-ubuntu22.04
16-
options: --user root
17-
steps:
18-
- name: 'Dependencies'
19-
run: |
20-
apt-get update
21-
apt-get install -y git python3.9 pip cudnn9-cuda-12
22-
pip install cmake==3.21.0 pybind11[global] ninja nvidia-mathdx==25.1.1
23-
- name: 'Checkout'
24-
uses: actions/checkout@v3
25-
with:
26-
submodules: recursive
27-
- name: 'Build'
28-
run: pip install --no-build-isolation . -v
29-
env:
30-
NVTE_FRAMEWORK: none
31-
MAX_JOBS: 1
32-
- name: 'Sanity check'
33-
run: python3 -c "import transformer_engine"
34-
working-directory: /
3511
pytorch:
3612
name: 'PyTorch'
37-
runs-on: ubuntu-latest
13+
runs-on: [ self-hosted, Linux, X64, nvidia, gpu-8 ]
14+
defaults:
15+
run:
16+
shell: bash
3817
container:
39-
image: nvcr.io/nvidia/cuda:12.8.0-devel-ubuntu22.04
18+
image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
4019
options: --user root
4120
steps:
42-
- name: 'Dependencies'
43-
run: |
44-
apt-get update
45-
apt-get install -y git python3.9 pip cudnn9-cuda-12
46-
pip install cmake torch ninja pydantic importlib-metadata>=1.0 packaging pybind11 numpy einops onnxscript nvidia-mathdx==25.1.1
4721
- name: 'Checkout'
4822
uses: actions/checkout@v3
4923
with:
5024
submodules: recursive
5125
- name: 'Build'
52-
run: pip install --no-build-isolation . -v --no-deps
26+
run:
27+
source /opt/miniconda3/etc/profile.d/conda.sh
28+
conda activate flagscale-train
29+
pip install --no-build-isolation . -v --no-deps
5330
env:
5431
NVTE_FRAMEWORK: pytorch
55-
MAX_JOBS: 1
56-
- name: 'Sanity check'
57-
run: python3 tests/pytorch/test_sanity_import.py
58-
jax:
59-
name: 'JAX'
60-
runs-on: ubuntu-latest
61-
container:
62-
image: ghcr.io/nvidia/jax:jax
63-
options: --user root
64-
steps:
65-
- name: 'Dependencies'
66-
run: pip install pybind11[global] nvidia-mathdx==25.1.1
67-
- name: 'Checkout'
68-
uses: actions/checkout@v3
69-
with:
70-
submodules: recursive
71-
- name: 'Build'
72-
run: pip install --no-build-isolation . -v
73-
env:
74-
NVTE_FRAMEWORK: jax
75-
MAX_JOBS: 1
76-
- name: 'Sanity check'
77-
run: python3 tests/jax/test_sanity_import.py
78-
all:
79-
name: 'All'
80-
runs-on: ubuntu-latest
81-
container:
82-
image: ghcr.io/nvidia/jax:jax
83-
options: --user root
84-
steps:
85-
- name: 'Dependencies'
86-
run: pip install torch pybind11[global] einops onnxscript nvidia-mathdx==25.1.1
87-
- name: 'Checkout'
88-
uses: actions/checkout@v3
89-
with:
90-
submodules: recursive
91-
- name: 'Build'
92-
run: pip install --no-build-isolation . -v --no-deps
93-
env:
94-
NVTE_FRAMEWORK: all
95-
MAX_JOBS: 1
32+
TE_WITH_NCCL: 1
9633
- name: 'Sanity check'
97-
run: python3 tests/pytorch/test_sanity_import.py && python3 tests/jax/test_sanity_import.py
34+
run:
35+
source /opt/miniconda3/etc/profile.d/conda.sh
36+
conda activate flagscale-train
37+
python3 tests/pytorch/test_sanity_import.py

‎.github/workflows/deploy_nightly_docs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
name: Deploy nightly docs
77
on:
88
push:
9-
branches: [ "main" ]
9+
branches: [ "__disabled_do_not_remove__" ]
1010
jobs:
1111
build:
1212
uses: ./.github/workflows/docs.yml

‎.github/workflows/license.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# A workflow to trigger the TE license check on GitHub
66
name: 'License'
77
on:
8-
pull_request:
8+
pull_request: [__disabled_do_not_remove__]
99
workflow_dispatch:
1010
jobs:
1111
check:

‎.github/workflows/qa-format.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: format_check
2+
3+
on:
4+
pull_request:
5+
branches: [ "main" ]
6+
types: [opened, synchronize, reopened]
7+
8+
jobs:
9+
format:
10+
runs-on: ubuntu-22.04
11+
env:
12+
PRID: ${{ github.event.pull_request.number }}
13+
BRANCH: main
14+
15+
steps:
16+
- name: Checkout code
17+
uses: actions/checkout@v4
18+
with:
19+
ref: ${{ github.event.pull_request.base.ref }}
20+
21+
- name: Merge PR to sub-branch
22+
run: |
23+
git fetch origin pull/${PRID}/merge
24+
git checkout -b test FETCH_HEAD
25+
26+
- name: Set up Python
27+
uses: actions/setup-python@v5
28+
with:
29+
python-version: "3.10"
30+
31+
- name: Run pre-commit
32+
run: bash ./qa/format.sh

‎.github/workflows/qa-l0-pytorch-wheel.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: QA Pytorch Wheel
2+
3+
on:
4+
push:
5+
branches:
6+
- __disabled_do_not_remove__
7+
pull_request:
8+
branches:
9+
- __disabled_do_not_remove__
10+
11+
concurrency:
12+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.actor }}
13+
cancel-in-progress: true
14+
15+
jobs:
16+
qa-l0-pytorch-wheel:
17+
runs-on: [ self-hosted, Linux, X64, nvidia, gpu-8 ]
18+
defaults:
19+
run:
20+
shell: bash
21+
container:
22+
image: harbor.baai.ac.cn/flagscale/cuda12.8.1-torch2.7.1-python3.10-te2.9:20260209
23+
ports:
24+
- 80:80
25+
options: >-
26+
--gpus all
27+
--shm-size=500g
28+
--privileged
29+
--ipc=host
30+
--ulimit memlock=-1
31+
--ulimit stack=67108864
32+
--ulimit nofile=65535:65535
33+
--user root
34+
--pull always
35+
36+
steps:
37+
- name: Checkout Code
38+
uses: actions/checkout@v6.0.1
39+
with:
40+
repository: ${{ github.event.pull_request.head.repo.full_name }}
41+
ref: ${{ github.event.pull_request.head.ref }}
42+
ssh-strict: true
43+
ssh-user: git
44+
persist-credentials: true
45+
clean: true
46+
sparse-checkout-cone-mode: true
47+
fetch-tags: false
48+
show-progress: true
49+
lfs: false
50+
submodules: recursive
51+
set-safe-directory: true
52+
53+
- name: L0 Pytorch Wheel
54+
id: L0_pytoech_wheel
55+
# timeout-minutes: 50
56+
env:
57+
TE_PATH: .
58+
RUN_LOG: /logs/pytorch/wheel
59+
run: |
60+
echo "TE_PATH: ${TE_PATH}"
61+
sed -i "s/^cd transformer_engine\/pytorch\s*$/pushd transformer_engine\/pytorch/" qa/L0_pytorch_wheel/test.sh
62+
sed -i '44 s/^cd \s*\$TE_PATH\s*$/popd/' qa/L0_pytorch_wheel/test.sh
63+
64+
cat qa/L0_pytorch_wheel/test.sh
65+
source /opt/miniconda3/etc/profile.d/conda.sh
66+
conda activate flagscale-train
67+
pip uninstall -y transformer_engine
68+
69+
bash qa/L0_pytorch_wheel/test.sh | tee ${RUN_LOG}/pytorch_wheel-${{ github.run_id }}.log
70+
71+
- name: Upload Installation Logs
72+
if: always() && steps.L0_pytoech_wheel.outcome == 'failure'
73+
uses: actions/upload-artifact@v4
74+
with:
75+
name: L0-pytorch-logs-${{ github.run_id }}
76+
path: /logs/pytorch/wheel
77+
retention-days: 7
78+
if-no-files-found: warn

0 commit comments

Comments
 (0)