Skip to content

Commit d3de545

Browse files
shnizzedyshnizzedy
authored
👷 Automate lite integration and regression tests (#2162)
2 parents fd49ded + 09d78c0 commit d3de545

File tree

10 files changed

+261
-209
lines changed

10 files changed

+261
-209
lines changed

‎.github/README/README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ flowchart LR
4646
subgraph build_C-PAC.yml
4747
bCPAC[[C-PAC]]
4848
end
49-
subgraph build_and_test.yml
49+
subgraph build_and_test.yaml
5050
ubuntu[[Ubnutu]]-->stages[[stages]]-->build-base[[build-base]]-->build-base-standard[[build-base-standard]]
5151
5252
Circle_tests[[Circle_tests]]
@@ -65,7 +65,7 @@ flowchart LR
6565
smoke-tests-participant[[smoke-tests-participant]]
6666
end
6767
68-
on_push.yml-->build_and_test.yml
68+
on_push.yaml-->build_and_test.yaml
6969
7070
delete_images.yml
7171
end
@@ -79,8 +79,8 @@ flowchart LR
7979
8080
Circle_tests-->CircleCI((Run tests on Circle CI))
8181
82-
on_push.yml<-->get_pr_base_shas
83-
on_push.yml-->update_all_preconfigs
82+
on_push.yaml<-->get_pr_base_shas
83+
on_push.yaml-->update_all_preconfigs
8484
8585
cpacdockerfiles<-->C-PAC
8686
@@ -94,7 +94,7 @@ flowchart LR
9494
bCPAC<-->local_ghcr
9595
stages<-->local_ghcr
9696
97-
push>git push]-->on_push.yml
97+
push>git push]-->on_push.yaml
9898
9999
smoke-tests-participant-->smoke_test_human
100100
smoke-tests-participant-->smoke_test_nhp

‎.github/workflows/build_and_test.yml‎ renamed to ‎.github/workflows/build_and_test.yaml‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ on:
4646
description: 'third phase of staging images to rebuild (base images)'
4747
type: string
4848
required: true
49+
test_mode:
50+
description: 'lite or full?'
51+
type: string
52+
default: None
4953

5054
jobs:
5155
Ubuntu:
@@ -186,7 +190,7 @@ jobs:
186190
run: |
187191
sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/${{ matrix.variant }}.txt
188192
echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt
189-
echo '.github/workflows/build_and_test.yml' >> .github/stage_requirements/${{ matrix.variant }}.txt
193+
echo '.github/workflows/build_and_test.yaml' >> .github/stage_requirements/${{ matrix.variant }}.txt
190194
echo '.github/stage_requirements/${{ matrix.variant }}.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt
191195
- name: Set tag & see if it exists
192196
continue-on-error: true
@@ -258,7 +262,7 @@ jobs:
258262
run: |
259263
sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/standard.txt
260264
echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/standard.txt
261-
echo '.github/workflows/build_and_test.yml' >> .github/stage_requirements/standard.txt
265+
echo '.github/workflows/build_and_test.yaml' >> .github/stage_requirements/standard.txt
262266
echo '.github/stage_requirements/standard.txt' >> .github/stage_requirements/standard.txt
263267
- name: Set tag & see if it exists
264268
continue-on-error: true
@@ -327,19 +331,21 @@ jobs:
327331
if: github.ref_name == 'develop' || github.ref_name == 'main'
328332
uses: ./.github/workflows/smoke_test_participant.yml
329333

330-
regtest-lite:
331-
name: Run lite regression test
334+
check_test_mode:
335+
name: check_test_mode
336+
runs-on: ubuntu-latest
337+
steps:
338+
- run: echo ${{ inputs.test_mode }}
339+
340+
regtest:
341+
name: Run regression and integration test
332342
needs:
333343
- C-PAC
334344
secrets: inherit
335-
if: contains(github.event.head_commit.message, '[run reg-suite]')
336-
uses: ./.github/workflows/regression_test_lite.yml
337-
338-
regtest-full:
339-
name: Run full regression test
340-
needs:
341-
- smoke-tests-participant
342-
uses: ./.github/workflows/regression_test_full.yml
345+
if: inputs.test_mode == 'lite'
346+
uses: ./.github/workflows/regtest.yaml
347+
with:
348+
test_mode: ${{ inputs.test_mode }}
343349

344350
Circle_tests:
345351
name: Run tests on CircleCI

‎.github/workflows/on_push.yml‎ renamed to ‎.github/workflows/on_push.yaml‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,10 +98,36 @@ jobs:
9898
echo "phase_three=${phase_three}" >> $GITHUB_OUTPUT
9999
echo "rebuild_phase_three=${rebuild_phase_three}" >> $GITHUB_OUTPUT
100100
101+
check_pr:
102+
runs-on: ubuntu-latest
103+
outputs:
104+
test_mode: ${{ steps.check_pr.outputs.test_mode }}
105+
steps:
106+
- name: Check out C-PAC
107+
uses: actions/checkout@v3
108+
with:
109+
fetch-depth: 2
110+
- name: Check if commit is in a PR to develop
111+
id: check_pr
112+
run: |
113+
TEST_MODE=none
114+
if echo "${{ github.event.head_commit.message }}" | grep -q '\[run reg-suite lite\]'
115+
then
116+
TEST_MODE=lite
117+
elif gh pr list --base develop --json number,state,draft | jq 'any(.[]; .state == "OPEN" or .draft == true)'; then
118+
TEST_MODE=lite
119+
elif gh pr list --base main --json number,state,draft | jq 'any(.[]; .state == "OPEN" or .draft == true)'; then
120+
TEST_MODE=full
121+
fi
122+
echo "test_mode=${TEST_MODE}"
123+
echo "test_mode=${TEST_MODE}" >> $GITHUB_OUTPUT
124+
101125
build-stages:
102126
name: Build multistage image stages
103-
needs: check-updated-preconfigs
104-
uses: ./.github/workflows/build_and_test.yml
127+
needs:
128+
- check_pr
129+
- check-updated-preconfigs
130+
uses: ./.github/workflows/build_and_test.yaml
105131
secrets: inherit
106132
with:
107133
phase_one: ${{ needs.check-updated-preconfigs.outputs.phase_one }}
@@ -110,3 +136,4 @@ jobs:
110136
rebuild_phase_two: ${{ needs.check-updated-preconfigs.outputs.rebuild_phase_two }}
111137
phase_three: ${{ needs.check-updated-preconfigs.outputs.phase_three }}
112138
rebuild_phase_three: ${{ needs.check-updated-preconfigs.outputs.rebuild_phase_three }}
139+
test_mode: ${{ needs.check_pr.outputs.test_mode }}

‎.github/workflows/regression_test_full.yml‎

Lines changed: 0 additions & 41 deletions
This file was deleted.

‎.github/workflows/regression_test_lite.yml‎

Lines changed: 0 additions & 88 deletions
This file was deleted.

‎.github/workflows/regtest.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
name: Launch regression test
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- develop
7+
types:
8+
- opened
9+
- ready_for_review
10+
- reopened
11+
workflow_call:
12+
inputs:
13+
test_mode:
14+
type: string
15+
required: true
16+
secrets:
17+
GH_CLI_BIN_PATH:
18+
description: 'path to directory containing GitHub CLI binary if not on default $PATH'
19+
required: false
20+
SSH_PRIVATE_KEY:
21+
required: true
22+
SSH_USER:
23+
required: true
24+
SSH_HOST:
25+
required: true
26+
SSH_WORK_DIR:
27+
required: true
28+
workflow_dispatch:
29+
inputs:
30+
test_mode:
31+
type: string
32+
required: true
33+
34+
jobs:
35+
test:
36+
name: Regression Test - ${{ inputs.test_mode }}
37+
environment: ACCESS
38+
env:
39+
COMPARISON_PATH: ${{ secrets.COMPARISON_PATH }}
40+
DASHBOARD_REPO: ${{ vars.DASHBOARD_REPO}}
41+
DOCKER_TAG:
42+
GH_CLI_BIN_PATH: ${{ secrets.GH_CLI_BIN_PATH }}
43+
SLURM_TESTING_BRANCH: ${{ vars.SLURM_TESTING_BRANCH }}
44+
SLURM_TESTING_PACKAGE: ${{ vars.SLURM_TESTING_PACKAGE }}
45+
SLURM_TESTING_REPO: ${{ vars.SLURM_TESTING_REPO }}
46+
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
47+
SSH_HOST: ${{ secrets.SSH_HOST }}
48+
SSH_USER: ${{ secrets.SSH_USER }}
49+
SSH_WORK_DIR: ${{ secrets.SSH_WORK_DIR }}
50+
TOKEN_FILE: ${{ secrets.TOKEN_FILE }}
51+
if: |
52+
${{ github.env.SSH_PRIVATE_KEY != '' &&
53+
(github.event_name == 'workflow_dispatch' ||
54+
(github.event_name == 'pull_request' &&
55+
github.event.repository.fork == 'false')) }}
56+
runs-on: ubuntu-latest
57+
steps:
58+
- name: Get C-PAC branch
59+
run: |
60+
if [[ ! $GITHUB_REF_NAME == 'main' ]] && [[ ! $GITHUB_REF_NAME == 'develop' ]]
61+
then
62+
TAG=${GITHUB_REF_NAME//\//_}
63+
elif [[ $GITHUB_REF_NAME == 'develop' ]]
64+
then
65+
TAG=nightly
66+
elif [[ $GITHUB_REF_NAME == 'main' ]]
67+
then
68+
TAG=latest
69+
fi
70+
TAG=$TAG$VARIANT
71+
echo DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}" | tr '[:upper:]' '[:lower:]'):$TAG >> $GITHUB_ENV
72+
cat $GITHUB_ENV
73+
74+
- name: Install SSH Keys
75+
run: |
76+
mkdir -p ~/.ssh/
77+
echo "${{ env.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
78+
chmod 600 ~/.ssh/id_rsa
79+
ssh-keyscan -H -t rsa "${{ env.SSH_HOST }}" > ~/.ssh/known_hosts
80+
81+
- name: Connect and Run Regression Test ${{ inputs.test_mode }}
82+
uses: appleboy/[email protected]
83+
with:
84+
host: ${{ env.SSH_HOST }}
85+
username: ${{ env.SSH_USER }}
86+
key: ${{ env.SSH_PRIVATE_KEY }}
87+
command_timeout: 200m
88+
script: |
89+
set -x
90+
cd ${{ env.SSH_WORK_DIR }}
91+
if pip show "${{ env.SLURM_TESTING_PACKAGE }}" > /dev/null 2>&1; then
92+
# If the package is installed, upgrade it
93+
python3 -m pip install --user --upgrade --force-reinstall "https://github.com/${{ env.SLURM_TESTING_REPO }}/archive/${{ env.SLURM_TESTING_BRANCH }}.zip"
94+
else
95+
# If the package is not installed, install it
96+
python3 -m pip install --user "https://github.com/${{ env.SLURM_TESTING_REPO }}/archive/${{ env.SLURM_TESTING_BRANCH }}.zip"
97+
fi
98+
_CPAC_SLURM_TESTING_WD="${{ env.SSH_WORK_DIR }}/automatic_tests/${{ inputs.test_mode }}/${{ github.sha }}"
99+
mkdir -p "${_CPAC_SLURM_TESTING_WD}"
100+
sbatch cpac-slurm-status ${{ inputs.test_mode }} launch \
101+
--wd="${_CPAC_SLURM_TESTING_WD}" \
102+
--comparison-path="${{ env.COMPARISON_PATH }}" \
103+
--dashboard-repo="${{ env.DASHBOARD_REPO}}" \
104+
--home-dir="${{ env.SSH_WORK_DIR }}" \
105+
--image="${{ env.DOCKER_TAG }}" \
106+
--owner="${{ github.repository_owner }}" \
107+
--path-extra="${{ env.GH_CLI_BIN_PATH }}" \
108+
--repo="${{ github.repository }}" \
109+
--sha="${{ github.sha }}" \
110+
--slurm-testing-branch="${{ env.SLURM_TESTING_BRANCH }}" \
111+
--slurm-testing-repo="${{ env.SLURM_TESTING_REPO }}" \
112+
--token-file="${{ env.TOKEN_FILE }}"
113+
114+
- name: Cleanup SSH
115+
run: |
116+
rm -rf ~/.ssh

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
4545
- Input `desc-brain_bold` to `desc-preproc_bold` for `sbref` generation nodeblock `coregistration_prep_vol`.
4646
- Turned `generate_xcpqc_files` on for all preconfigurations except `blank`.
4747
- Introduced specific switch `restore_t1w_intensity` for `correct_restore_brain_intensity_abcd` nodeblock, enabling it by default only in `abcd-options` pre-config.
48+
- Updated GitHub Actions to run automated integration and regression tests on HPC.
4849

4950
### Fixed
5051

0 commit comments

Comments
 (0)