Skip to content

Commit 22264e9

Browse files
dgomezTTdgomezTT
authored
Implement pipeline to run accuracy tests (#864)
* Fix pipeline for accuracy-tests * removing that hardcoded array * Fixed a bunch of problems with the pipeline and created an action for test file calculation * fixed the refactor in the common_model_test * :bye ugly hardcoded array, setting some schedule to run accuracy tests * Fixed an small typo in the config of before_merge.yaml * Addressed the logic for before_merge * Improved how we detect the accuracy tests * Fixed merge issue * fix docker for accuracy tests * Fixed how we pass the accuracy test between steps * Added mock test to speed up the process * Added mock test to speed up the process * Added mock test to speed up the process * Added mock test to speed up the process * fixed some docker configuration in the accuracy test * Improved the way to select what tests to run * fix filter group * fix rebase error * added the option to configure the docker image and fixed a cache issue * fixed download artifacts * fixed download artifacts * fixed download artifacts
1 parent 0706762 commit 22264e9

File tree

5 files changed

+168
-55
lines changed

5 files changed

+168
-55
lines changed

‎.github/actions/common_model_tests/action.yaml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
name: 'Run Model tests'
22
description: 'Run Model tests'
3+
inputs:
4+
splits:
5+
description: 'Number of splits for test distribution'
6+
required: true
7+
commit_report:
8+
description: 'Commit report input (None, Docs, All)'
9+
required: false
10+
default: 'None'
11+
matrix_group:
12+
description: 'Matrix group index for splitting tests'
13+
required: true
314
runs:
415
using: "composite"
516
steps:
@@ -12,7 +23,7 @@ runs:
1223
else
1324
num_iterations=5
1425
fi
15-
python3 -m pytest --github-report tests/models/ --report_nth_iteration=$num_iterations --gen_op_accuracy_tests --splits 40 --group ${{ matrix.group }} -s
26+
python3 -m pytest --github-report tests/models/ --report_nth_iteration=$num_iterations --gen_op_accuracy_tests --splits ${{ inputs.splits }} --group ${{ matrix.group }} -s
1627
exit_code=$? # Capture the exit code
1728
if [ $exit_code -eq 5 ]; then
1829
if [ ${{ matrix.group }} -eq 0 ]; then

‎.github/actions/count_test_files/action.yaml

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
name: 'Count Test Files'
2+
description: 'Counts the number of valid test files in a directory'
3+
4+
inputs:
5+
test_directory:
6+
description: 'Directory containing the test files'
7+
required: true
8+
default: 'tests/models/'
9+
10+
outputs:
11+
num_files:
12+
description: 'The number of valid test files'
13+
value: ${{ steps.count-files.outputs.num_files }}
14+
15+
matrix:
16+
description: 'A JSON array of test groups'
17+
value: ${{ steps.generate-matrix.outputs.matrix }}
18+
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Count Test Files
23+
id: count-files
24+
shell: bash
25+
run: |
26+
# Count the number of valid test files
27+
num_tests=$(find "${{ inputs.test_directory }}" -type f \( -name "test_*.py" -o -name "*_test.py" \) | wc -l)
28+
if [ "$num_tests" -eq 0 ]; then
29+
echo "Error: No test files found in the directory '${{ inputs.test_directory }}'."
30+
exit 1
31+
fi
32+
echo "num_files=$num_tests" >> $GITHUB_OUTPUT
33+
34+
- name: Generate Matrix
35+
id: generate-matrix
36+
shell: bash
37+
run: |
38+
if [ -z "$MOCK_RUN" ]; then
39+
# Generate matrix based on the number of test files
40+
num_files="${{ steps.count-files.outputs.num_files }}"
41+
matrix_tmp=$(seq 1 $num_files | jq -c --slurp '.')
42+
else
43+
# Use MOCK_RUN directly as the matrix
44+
matrix_tmp="$MOCK_RUN"
45+
echo "Mock mode enabled. Using MOCK_RUN as matrix: $matrix_tmp"
46+
fi
47+
echo "matrix=$matrix_tmp" >> $GITHUB_OUTPUT
48+
49+
- name: Debug Outputs
50+
shell: bash
51+
run: |
52+
echo "Number of files: ${{ steps.count-files.outputs.num_files }}"
53+
echo "Matrix: ${{ steps.generate-matrix.outputs.matrix }}"

‎.github/workflows/before_merge.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ jobs:
1111
docker_tag: 'ghcr.io/tenstorrent/pytorch2.0_ttnn/ubuntu-22.04-amd64:latest'
1212
commit_report: 'None'
1313

14-
1514
validate-pr:
1615
if: ${{ always() }}
1716
runs-on: ubuntu-latest

‎.github/workflows/run-accuracy-tests.yaml

Lines changed: 78 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,18 @@ name: "Accuracy Tests"
22

33
on:
44
workflow_dispatch:
5-
5+
inputs:
6+
MockRun:
7+
description: "Array of test groups to run. If empty, all tests will be run."
8+
required: false
9+
default: ""
10+
docker_tag:
11+
description: 'Docker container tag to use'
12+
required: false
13+
type: string
14+
default: 'ghcr.io/tenstorrent/pytorch2.0_ttnn/ubuntu-22.04-amd64:latest'
15+
schedule:
16+
- cron: "0 21 * * 6" # Run every Saturday at 9:00 PM UTC
617
permissions:
718
actions: read
819
contents: write
@@ -11,30 +22,66 @@ permissions:
1122
pull-requests: read
1223

1324
jobs:
14-
tools-tests:
25+
count-test-files:
26+
runs-on: ubuntu-latest
27+
outputs:
28+
matrix: ${{ steps.count-files.outputs.matrix }}
29+
num_files: ${{ steps.count-files.outputs.num_files }}
1530
env:
16-
pytest_verbosity: 2
17-
pytest_report_title: "⭐️ Tools Tests"
18-
runs-on: ["in-service"]
31+
MOCK_RUN: ${{ github.event.inputs.MockRun }}
32+
steps:
33+
- name: Checkout Code
34+
uses: actions/checkout@v4
35+
36+
- name: Count Test Files
37+
id: count-files
38+
uses: ./.github/actions/count_test_files
39+
with:
40+
test_directory: 'tests/models/'
41+
model-tests:
42+
needs: [count-test-files]
43+
runs-on: ["in-service", "nfs"]
1944
container:
20-
image: ghcr.io/tenstorrent/pytorch2.0_ttnn/ubuntu-22.04-amd64:latest
45+
image: ${{ inputs.docker_tag }}
2146
credentials:
2247
username: ${{ github.actor }}
2348
password: ${{ secrets.GH_TOKEN }}
2449
options: >-
2550
--rm -v /dev/hugepages-1G:/dev/hugepages-1G --device /dev/tenstorrent
2651
-v ${{ github.workspace }}:${{ github.workspace }} -w ${{ github.workspace }}
27-
steps:
52+
-v /mnt/tt-metal-pytorch-cache/.cache:/root/.cache
53+
env:
54+
pytest_verbosity: 0
55+
TORCH_HOME: /mnt/tt-metal-pytorch-cache/.cache/torch
56+
HF_HOME: /mnt/tt-metal-pytorch-cache/.cache/huggingface
57+
strategy:
58+
matrix:
59+
group: ${{ fromJson(needs.count-test-files.outputs.matrix) }}
60+
steps:
2861
- uses: actions/checkout@v4
29-
- uses: ./.github/actions/common_repo_setup
30-
- name: Run Tools Tests
62+
with:
63+
lfs: true
64+
fetch-depth: 0
65+
- name: docker-cleanup
3166
run: |
32-
python3 -m pytest --github-report tests/tools/ -s
67+
docker system prune -a -f --volumes
68+
df -h # Debug space
69+
- uses: ./.github/actions/common_model_tests
70+
with:
71+
splits: ${{ needs.count-test-files.outputs.num_files }}
72+
matrix_group: ${{ matrix.group }}
73+
commit_report: ${{ github.event.inputs.commit_report }}
74+
- name: Upload Accuracy Tests Artifact
75+
uses: actions/upload-artifact@v4
76+
with:
77+
name: autogen-accuracy-tests-group-${{ matrix.group }}
78+
path: tests/autogen_accuracy_tests/
3379

3480
gen-model-accuracy:
81+
needs: [model-tests]
3582
runs-on: ["in-service"]
3683
container:
37-
image: ghcr.io/tenstorrent/pytorch2.0_ttnn/ubuntu-22.04-amd64:latest
84+
image: ${{ inputs.docker_tag }}
3885
credentials:
3986
username: ${{ github.actor }}
4087
password: ${{ secrets.GH_TOKEN }}
@@ -48,73 +95,68 @@ jobs:
4895
steps:
4996
- uses: actions/checkout@v4
5097
- uses: ./.github/actions/common_repo_setup
51-
98+
5299
- name: Download All Accuracy Tests and Inputs Artifacts
53100
uses: actions/download-artifact@v4
54101
with:
55-
pattern: model-accuracy-tests-group-*
102+
pattern: autogen-accuracy-tests-group-*
56103
merge-multiple: true
57104
path: tests/autogen_accuracy_tests/
58105

59106
- name: Calculate Number of Groups
60107
id: calculate-groups
61108
run: |
62109
# Count the number of test files
63-
num_files=$(find tests/autogen_accuracy_tests -type f -name "*.py" | wc -l)
64-
65-
# Get the number of available CPUs (or set a default if not available)
66-
num_cpus=$(nproc || echo 4)
67-
68-
# Calculate the number of groups (e.g., 1 group per CPU, or adjust as needed)
69-
num_groups=$((num_files < num_cpus ? num_files : num_cpus))
70-
110+
num_groups=$(find tests/autogen_accuracy_tests -type f -name "*.py" | wc -l)
71111
# Ensure at least 1 group
72112
num_groups=$((num_groups > 0 ? num_groups : 1))
73-
74113
# Generate the list of groups as JSON
75-
groups=$(seq 1 $num_groups | jq -c '.')
114+
groups=$(seq 1 $num_groups | jq -c --slurp '.')
115+
echo "Found: $groups"
76116
echo "groups=$groups" >> $GITHUB_OUTPUT
77-
outputs:
78-
groups: ${{ steps.calculate-groups.outputs.groups }}
79-
80-
- name: Upload Accuracy Tests Artifact
81-
uses: actions/upload-artifact@v4
82-
with:
83-
name: autogen-accuracy-tests
84-
path: tests/autogen_accuracy_tests/
85117
86118
test-model-accuracy:
87119
needs: [gen-model-accuracy]
88-
runs-on: ["in-service"]
120+
runs-on: ["in-service", "nfs"]
89121
container:
90-
image: ghcr.io/tenstorrent/pytorch2.0_ttnn/ubuntu-22.04-amd64:latest
122+
image: ${{ inputs.docker_tag }}
91123
credentials:
92124
username: ${{ github.actor }}
93125
password: ${{ secrets.GH_TOKEN }}
94126
options: >-
95127
--rm -v /dev/hugepages-1G:/dev/hugepages-1G --device /dev/tenstorrent
96128
-v ${{ github.workspace }}:${{ github.workspace }} -w ${{ github.workspace }}
129+
-v /mnt/tt-metal-pytorch-cache/.cache:/root/.cache
97130
env:
98131
PYTHONPATH: ${{ github.workspace }}
132+
TORCH_HOME: /mnt/tt-metal-pytorch-cache/.cache/torch
133+
HF_HOME: /mnt/tt-metal-pytorch-cache/.cache/huggingface
99134
strategy:
100135
matrix:
101136
group: ${{ fromJson(needs.gen-model-accuracy.outputs.groups) }}
102137
steps:
103138
- uses: actions/checkout@v4
104139
- uses: ./.github/actions/common_repo_setup
105-
140+
- name: docker-cleanup
141+
run: |
142+
docker system prune -a -f --volumes
143+
df -h # Debug space
106144
- name: Download Accuracy Tests Artifact
107145
uses: actions/download-artifact@v4
108146
with:
109-
name: autogen-accuracy-tests
147+
pattern: autogen-accuracy-tests-group-*
110148
path: tests/autogen_accuracy_tests/
111149

112150
- name: Run Accuracy Tests
113151
run: |
114152
cd tests/autogen_accuracy_tests
115-
set +e
116153
test_file=$(find . -type f -name "*.py" | sed -n "${{ matrix.group }}p")
117154
python3 -m pytest $test_file -s
155+
exit_code=$?
156+
if [ $exit_code -ne 0 ]; then
157+
echo "Tests failed with exit code $exit_code"
158+
exit $exit_code
159+
fi
118160
exit 0;
119161
shell: bash
120162

‎.github/workflows/run-tests.yaml

Lines changed: 25 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
name: "Run Tests"
2-
32
on:
43
workflow_call:
54
inputs:
@@ -79,8 +78,23 @@ jobs:
7978
- uses: ./.github/actions/common_repo_setup
8079
- uses: ./.github/actions/common_lowering_tests
8180

81+
count-test-files:
82+
runs-on: ubuntu-latest
83+
outputs:
84+
matrix: ${{ steps.count-files.outputs.matrix }}
85+
num_files: ${{ steps.count-files.outputs.num_files }}
86+
steps:
87+
- name: Checkout Code
88+
uses: actions/checkout@v4
89+
90+
- name: Count Test Files
91+
id: count-files
92+
uses: ./.github/actions/count_test_files
93+
with:
94+
test_directory: 'tests/models/'
95+
8296
model-tests:
83-
needs: lowering-tests
97+
needs: [count-test-files, lowering-tests]
8498
runs-on: ["in-service", "nfs"]
8599
container:
86100
image: ${{ inputs.docker_tag }}
@@ -97,34 +111,29 @@ jobs:
97111
TORCH_HOME: /mnt/tt-metal-pytorch-cache/.cache/torch
98112
HF_HOME: /mnt/tt-metal-pytorch-cache/.cache/huggingface
99113
strategy:
100-
matrix: # Need to find a way to replace this with a generator
101-
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40]
114+
matrix:
115+
group: ${{ fromJson(needs.count-test-files.outputs.matrix) }}
102116
steps:
103117
- uses: actions/checkout@v4
104118
with:
105119
lfs: true
106120
fetch-depth: 0
107-
- uses: ./.github/actions/common_repo_setup
108121
- name: docker-cleanup
109122
run: |
110123
docker system prune -a -f --volumes
111124
df -h # Debug space
112125
- uses: ./.github/actions/common_model_tests
113-
126+
with:
127+
splits: ${{ needs.count-test-files.outputs.num_files }}
128+
matrix_group: ${{ matrix.group }}
129+
commit_report: ${{ github.event.inputs.commit_report }}
114130
- name: Upload Metrics Artifact
115131
if: success() # Only run if tests passed
116132
uses: actions/upload-artifact@v4
117133
with:
118134
name: model-tests-metrics-group-${{ matrix.group }}
119135
path: metrics/
120136

121-
- name: Upload All Accuracy Tests and Inputs Artifacts
122-
if: success() # Only run if tests passed
123-
uses: actions/upload-artifact@v4
124-
with:
125-
name: model-accuracy-tests-group-${{ matrix.group }}
126-
path: tests/autogen_accuracy_tests/
127-
128137
push-autogen-op-tests:
129138
needs: [model-tests]
130139
if: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_report != 'None'}}
@@ -176,7 +185,7 @@ jobs:
176185
fi
177186
178187
model-autogen-op-tests:
179-
needs: [push-autogen-op-tests]
188+
needs: [push-autogen-op-tests, count-test-files]
180189
if: ${{ github.event_name == 'workflow_dispatch' && inputs.commit_report != 'None'}}
181190
runs-on: ["in-service"]
182191
container:
@@ -188,9 +197,8 @@ jobs:
188197
--rm -v /dev/hugepages-1G:/dev/hugepages-1G --device /dev/tenstorrent
189198
-v ${{ github.workspace }}:${{ github.workspace }} -w ${{ github.workspace }}
190199
strategy:
191-
matrix: # Need to find a way to replace this with a generator
192-
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40]
193-
200+
matrix:
201+
group: ${{ fromJson(needs.count-test-files.outputs.matrix) }}
194202
env:
195203
pytest_verbosity: 0
196204
pytest_report_title: "⭐️ Model Input Variations Tests - Group ${{ matrix.group }}"

0 commit comments

Comments
 (0)