Add one second test check to CI (#1033)

* add one second test check to CI

* pyproject add tests-report as an available testenv

* fix typo

* exclude the option of tox parsing weird

* 5 seconds for system tests

* tox is a pain

* add two different codecov file names

---------

Co-authored-by: Dominic Oram <[email protected]>

Author: stan-dot

.github/scripts/check_test_durations.py

@@ -0,0 +1,30 @@
+import json
+import sys
+
+# Get report filename and threshold from command-line arguments
+REPORT_FILE = sys.argv[1] if len(sys.argv) > 1 else "report.json"
+THRESHOLD = float(sys.argv[2]) if len(sys.argv) > 2 else 1.0
+
+try:
+    with open(REPORT_FILE) as f:
+        data = json.load(f)
+except FileNotFoundError:
+    print(f"❌ Error: Report file '{REPORT_FILE}' not found.")
+    sys.exit(1)
+except json.JSONDecodeError:
+    print(f"❌ Error: Failed to parse JSON in '{REPORT_FILE}'.")
+    sys.exit(1)
+
+slow_tests = [
+    (t["nodeid"], t["call"]["duration"])
+    for t in data.get("tests", [])
+    if "call" in t and t["call"]["duration"] > THRESHOLD
+]
+
+if slow_tests:
+    print(f"❌ The following tests exceeded the {THRESHOLD:.2f}s threshold:")
+    for test, duration in slow_tests:
+        print(f" - {test}: {duration:.2f}s")
+    sys.exit(1)
+
+print("✅ All tests ran within the acceptable time limit.")

.github/workflows/_test.yml

@@ -50,13 +50,24 @@ jobs:
           python-version: ${{ inputs.python-version }}
           pip-install: ".[dev]"
 
-      - name: Run tests
-        run: tox -e tests
+      - name: Run unit tests
+        run: tox -e unit-report
+
+      - name: Check unit test durations
+        run: |
+          python .github/scripts/check_test_durations.py unit-report.json 1
+
+      - name: Run system tests
+        run: tox -e system-report
+
+      - name: Check system test durations
+        run: |
+          python .github/scripts/check_test_durations.py system-report.json 5
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           name: ${{ inputs.python-version }}/${{ inputs.runs-on }}
-          files: cov.xml
+          files: ./unit_cov.xml,./system_cov.xml
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -74,3 +74,7 @@ lockfiles/
 
 # pycharm project files
 .idea/
+
+
+# custom CI setup 
+report.json

pyproject.toml

@@ -58,6 +58,7 @@ dev = [
     "pytest",
     "pytest-asyncio",
     "pytest-cov",
+    "pytest-json-report",
     "pytest-random-order",
     "ruff",
     "sphinx<7.4.6",              #  pinned due to https://github.com/sphinx-doc/sphinx/issues/12660
@@ -141,7 +142,7 @@ legacy_tox_ini = """
 [tox]
 skipsdist=True
 
-[testenv:{pre-commit,type-checking,tests,docs}]
+[testenv:{pre-commit,type-checking,tests,docs,unit-report,system-report}]
 # Don't create a virtualenv for the command, requires tox-direct plugin
 direct = True
 passenv = *
@@ -156,6 +157,8 @@ commands =
     type-checking: pyright src tests {posargs}
     pre-commit: pre-commit run --all-files --show-diff-on-failure {posargs}
     docs: sphinx-{posargs:build -E} -T docs build/html
+    unit-report: pytest -m 'not (s03 or adsim)' --cov=dodal --cov-report term --cov-report xml:unit_cov.xml --json-report --json-report-file=unit-report.json tests {posargs}
+    system-report: pytest -m 'not (s03 or adsim)' --cov=dodal --cov-report term --cov-report xml:system_cov.xml --json-report --json-report-file=system-report.json system_tests {posargs}
 """
 
 [tool.ruff]