Skip to content

Commit a652e3d

Browse files
zihugithubzihugithub
authored and
LiJunscs
committed
[CICD] add CLI validation workflow for FlagScale CLI (flagos-ai#1144)
### PR Category CICD ### PR Types New Features ### PR Description - Add functional_tests_cli.yml for CLI install and validation - Validate flagscale --version, --help, and all subcommand --help - Run CLI unit tests (pytest tests/unit_tests/test_cli.py) - Integrate as a gate job before unit_tests in all_tests_common.yml - Fix checkout step to support both push and pull_request triggers --------- Co-authored-by: zihugithub <fbye@baai.ac.cn>
1 parent 253d0b4 commit a652e3d

2 files changed

Lines changed: 196 additions & 1 deletion

File tree

‎.github/workflows/all_tests_common.yml‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,22 @@ jobs:
9999
# Load configuration and group tests by task
100100
load_platform_config "$PLATFORM"
101101
102-
unit_tests:
102+
# CLI validation runs first (outside virtual env) as a gate for all subsequent tests
103+
cli_validation:
103104
needs: checkout_and_config
105+
uses: ./.github/workflows/functional_tests_cli.yml
106+
with:
107+
platform: ${{ inputs.platform }}
108+
image: ${{ needs.checkout_and_config.outputs.ci_train_image }}
109+
runs_on: ${{ needs.checkout_and_config.outputs.runs_on }}
110+
container_volumes: ${{ needs.checkout_and_config.outputs.container_volumes }}
111+
container_options: ${{ needs.checkout_and_config.outputs.container_options }}
112+
source_artifact: flagscale-source-${{ github.sha }}
113+
114+
unit_tests:
115+
needs:
116+
- checkout_and_config
117+
- cli_validation
104118
strategy:
105119
fail-fast: false
106120
matrix:
@@ -216,6 +230,7 @@ jobs:
216230
shell: bash
217231
needs:
218232
- checkout_and_config
233+
- cli_validation
219234
- unit_tests
220235
- functional_tests_train
221236
- functional_tests_hetero_train
@@ -236,6 +251,11 @@ jobs:
236251
failed=true
237252
fi
238253
254+
if [ "${{ needs.cli_validation.result }}" != "success" ]; then
255+
echo "❌ CLI validation failed"
256+
failed=true
257+
fi
258+
239259
# Only check functional tests if they ran
240260
if [ "${{ needs.functional_tests_train.result }}" != "success" ] && \
241261
[ "${{ needs.functional_tests_train.result }}" != "skipped" ]; then

‎.github/workflows/functional_tests_cli.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
name: Common Functional Tests - CLI
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
platform:
7+
required: true
8+
type: string
9+
description: Platform name (e.g., cuda, default)
10+
image:
11+
required: true
12+
type: string
13+
runs_on:
14+
required: true
15+
type: string
16+
container_volumes:
17+
required: true
18+
type: string
19+
container_options:
20+
required: true
21+
type: string
22+
source_artifact:
23+
required: true
24+
type: string
25+
description: Name of the artifact containing source code
26+
27+
jobs:
28+
functional_test_cli:
29+
defaults:
30+
run:
31+
shell: bash
32+
env:
33+
PROJECT_ROOT: /tmp/FlagScale
34+
runs-on: ${{ fromJson(inputs.runs_on) }}
35+
container:
36+
image: ${{ inputs.image }}
37+
ports:
38+
- 80
39+
volumes: ${{ fromJson(inputs.container_volumes) }}
40+
options: ${{ inputs.container_options }}
41+
42+
steps:
43+
- name: Download source code artifact (attempt 1)
44+
uses: actions/download-artifact@v4
45+
continue-on-error: true
46+
id: download_attempt_1
47+
with:
48+
name: ${{ inputs.source_artifact }}
49+
path: /tmp
50+
51+
- name: Download source code artifact (attempt 2)
52+
if: steps.download_attempt_1.outcome == 'failure'
53+
uses: actions/download-artifact@v4
54+
continue-on-error: true
55+
id: download_attempt_2
56+
with:
57+
name: ${{ inputs.source_artifact }}
58+
path: /tmp
59+
60+
- name: Download source code artifact (attempt 3)
61+
if: steps.download_attempt_2.outcome == 'failure'
62+
uses: actions/download-artifact@v4
63+
id: download_attempt_3
64+
with:
65+
name: ${{ inputs.source_artifact }}
66+
path: /tmp
67+
68+
- name: Verify artifact download
69+
run: |
70+
if [ "${{ steps.download_attempt_1.outcome }}" == "success" ]; then
71+
echo "✅ Artifact downloaded successfully on attempt 1"
72+
elif [ "${{ steps.download_attempt_2.outcome }}" == "success" ]; then
73+
echo "✅ Artifact downloaded successfully on attempt 2 (retried once)"
74+
elif [ "${{ steps.download_attempt_3.outcome }}" == "success" ]; then
75+
echo "✅ Artifact downloaded successfully on attempt 3 (retried twice)"
76+
else
77+
echo "❌ Error: All 3 download attempts failed"
78+
echo "Artifact name: ${{ inputs.source_artifact }}"
79+
exit 1
80+
fi
81+
82+
- name: Extract source code
83+
run: |
84+
mkdir -p $PROJECT_ROOT
85+
tar -xzf /tmp/flagscale-source.tar.gz -C $PROJECT_ROOT
86+
87+
- name: Set safe directory
88+
run: |
89+
git config --global --add safe.directory $PROJECT_ROOT
90+
91+
- name: Install FlagScale CLI
92+
run: |
93+
set -euo pipefail
94+
cd $PROJECT_ROOT
95+
96+
echo "Installing FlagScale CLI (outside virtual environment)"
97+
echo "Python location: $(which python)"
98+
echo "Python version: $(python --version)"
99+
100+
# Install FlagScale CLI using system pip
101+
pip install . --no-build-isolation --root-user-action=ignore || { echo "❌ FlagScale CLI install failed"; exit 1; }
102+
103+
# Verify installation
104+
command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; }
105+
echo "✅ FlagScale CLI installed successfully"
106+
timeout-minutes: 10
107+
108+
- name: Validate CLI functionality
109+
id: cli_validation
110+
run: |
111+
set -euo pipefail
112+
cd $PROJECT_ROOT
113+
114+
echo "Validating CLI (outside virtual environment)"
115+
echo "Python location: $(which python)"
116+
echo "Python version: $(python --version)"
117+
118+
failed=false
119+
120+
# 1. Validate flagscale --version
121+
echo "=========================================="
122+
echo "Test: flagscale --version"
123+
echo "=========================================="
124+
if flagscale --version; then
125+
echo "✅ flagscale --version passed"
126+
else
127+
echo "❌ flagscale --version failed"
128+
failed=true
129+
fi
130+
131+
# 2. Validate flagscale --help
132+
echo "=========================================="
133+
echo "Test: flagscale --help"
134+
echo "=========================================="
135+
if flagscale --help; then
136+
echo "✅ flagscale --help passed"
137+
else
138+
echo "❌ flagscale --help failed"
139+
failed=true
140+
fi
141+
142+
# 3. Validate each subcommand --help
143+
SUBCOMMANDS="train serve inference rl compress install test pull run"
144+
for cmd in $SUBCOMMANDS; do
145+
echo "=========================================="
146+
echo "Test: flagscale $cmd --help"
147+
echo "=========================================="
148+
if flagscale $cmd --help; then
149+
echo "✅ flagscale $cmd --help passed"
150+
else
151+
echo "❌ flagscale $cmd --help failed"
152+
failed=true
153+
fi
154+
done
155+
156+
# 4. Run CLI unit tests
157+
echo "=========================================="
158+
echo "Test: CLI unit tests (pytest)"
159+
echo "=========================================="
160+
pip install pytest pytest-mock --root-user-action=ignore || { echo "❌ pytest install failed"; exit 1; }
161+
if pytest tests/unit_tests/test_cli.py -v; then
162+
echo "✅ CLI unit tests passed"
163+
else
164+
echo "❌ CLI unit tests failed"
165+
failed=true
166+
fi
167+
168+
# Final result
169+
echo "=========================================="
170+
if [ "$failed" = "true" ]; then
171+
echo "❌ CLI validation failed"
172+
exit 1
173+
fi
174+
echo "✅ All CLI validations passed!"
175+
timeout-minutes: 10

0 commit comments

Comments
 (0)