Skip to content

Commit e2708e7

Browse files
authored
Split training tests to nightly and weekly (#2273)
### Problem description Most of the training tests were `NOT_SUPPORTED_SKIP`. This made difficult to run the tests as it needed to be changed to either `EXPECTED_PASSING` or `KNOWN_FAILURE_XFAIL` ### What's changed - Removed bringup_status from models that are now `KNOWN_FAILURE_XFAIL` - Removed reason from `KNOWN_FAILURE_XFAIL` and which reasons can now be parsed automaticly - tagged all models with `nightly`/`weekly - `nightly` models are for now models that are used in tt-blacksmith and in CI have bad PCC - `weekly` all other models - Updated *.json-s to run inference and training as separete querry - Fixed bug in `model-test-xfail.json` where it was running also weekly xfail tests
1 parent 051ebb2 commit e2708e7

File tree

3 files changed

+715
-853
lines changed

3 files changed

+715
-853
lines changed
Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
[
2-
{ "runs-on": "n150", "name": "run_forge_models_torch", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "weekly and n150 and (known_failure_xfail or not_supported_skip)", "parallel-groups": 1 },
3-
{ "runs-on": "p150", "name": "run_forge_models_torch", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "weekly and p150 and (known_failure_xfail or not_supported_skip)", "parallel-groups": 1 }
2+
{ "runs-on": "n150", "name": "run_forge_models_torch", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "inference and weekly and n150 and (known_failure_xfail or not_supported_skip)", "parallel-groups": 1 },
3+
{ "runs-on": "p150", "name": "run_forge_models_torch", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "inference and weekly and p150 and (known_failure_xfail or not_supported_skip)", "parallel-groups": 1 },
4+
{ "runs-on": "n150", "name": "run_forge_models_torch", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "training and weekly and n150 and (known_failure_xfail or not_supported_skip)", "parallel-groups": 20 }
45
]
Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
[
2-
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "n150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
3-
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "p150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
4-
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "n150 and (known_failure_xfail or not_supported_skip) and large", "parallel-groups": 1, "shared-runners": "true" },
5-
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "p150 and (known_failure_xfail or not_supported_skip) and large", "parallel-groups": 1, "shared-runners": "true" },
6-
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "n150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
7-
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "p150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
2+
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "not weekly and n150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
3+
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "not weekly and p150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
4+
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "not weekly and n150 and (known_failure_xfail or not_supported_skip) and large", "parallel-groups": 1, "shared-runners": "true" },
5+
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_jax", "test-mark": "not weekly and p150 and (known_failure_xfail or not_supported_skip) and large", "parallel-groups": 1, "shared-runners": "true" },
6+
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "inference and not weekly and n150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
7+
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "inference and not weekly and p150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 3 },
8+
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "training and nightly and n150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 1 },
9+
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_all_models_torch", "test-mark": "training and nightly and p150 and (known_failure_xfail or not_supported_skip) and not large", "parallel-groups": 1 },
810
{ "runs-on": "p150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_placeholder_models", "test-mark": "placeholder", "parallel-groups": 1 },
911
{ "runs-on": "n150", "name": "run_forge_models", "dir": "./tests/runner/test_models.py::test_placeholder_models", "test-mark": "placeholder", "parallel-groups": 1 }
1012
]

0 commit comments

Comments
 (0)