workflows: Fix test selection logic

Signed-off-by: Jorgen Kvalvaag <jorgen.kvalvaag@nordicsemi.no>

Author: jorgenmk

.github/workflows/build-and-target-test.yml

@@ -3,10 +3,11 @@ name: Build and Test
 on:
   workflow_dispatch:
     inputs:
-      pytest_marker:
-        type: string
+      build_all:
+        description: Build all configurations
+        type: boolean
         required: false
-        default: '-m "not slow"'
+        default: false
 
   schedule:
     - cron: "0 0 * * *"
@@ -19,6 +20,8 @@ jobs:
   build:
     uses: ./.github/workflows/build.yml
     secrets: inherit
+    with:
+      build_all: ${{ inputs.build_all || false }}
   test:
     permissions:
       actions: read
@@ -30,4 +33,4 @@ jobs:
     with:
       artifact_fw_version: ${{ needs.build.outputs.version }}
       artifact_run_id: ${{ needs.build.outputs.run_id }}
-      pytest_marker: ${{ inputs.pytest_marker }}
+      test_all: ${{ inputs.build_all || false }}

.github/workflows/build.yml

@@ -169,6 +169,7 @@ jobs:
 
       # Asset Tracker Template debug firmware build
       - name: Build thingy91x debug firmware
+        if: ${{ inputs.build_all }}
         uses: ./asset-tracker-template/.github/actions/build-step
         with:
           memfault_project_key: ${{ secrets.MEMFAULT_PROJECT_KEY }}
@@ -201,7 +202,7 @@ jobs:
         working-directory: asset-tracker-template
 
       - name: Build patched thingy91x firmware
-        if: ${{ github.event_name != 'pull_request' }}
+        if: ${{ inputs.build_all }}
         uses: ./asset-tracker-template/.github/actions/build-step
         with:
           board: thingy91x/nrf9151/ns

.github/workflows/target-test.yml

@@ -9,9 +9,10 @@ on:
       artifact_fw_version:
         type: string
         required: true
-      pytest_marker:
-        type: string
+      test_all:
+        type: boolean
         required: false
+        default: false
       pytest_path:
         type: string
         required: false
@@ -32,10 +33,11 @@ on:
         description: The run ID of the workflow to fetch artifacts from
         type: string
         required: true
-      pytest_marker:
-        description: The pytest marker to run
-        type: string
+      test_all:
+        description: Run all tests
+        type: boolean
         required: false
+        default: false
       pytest_path:
         description: Select test execution path
         type: string
@@ -52,17 +54,28 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       device_matrix: ${{ steps.set-matrix.outputs.device_matrix }}
+      pytest_marker: ${{ steps.set-matrix.outputs.pytest_marker }}
     steps:
       - id: set-matrix
         run: |
-          if [[ ${{ github.event_name == 'schedule' }} == "true" ]]; then
-              matrix='["thingy91x","nrf9151dk","ppk_thingy91x"]'
+          if [[ ${{ github.event_name == 'schedule' }} == true ]]; then
+            marker=''
+            matrix='["thingy91x","nrf9151dk","ppk_thingy91x"]'
+          elif [[ ${{ inputs.test_all }} == true ]]; then
+            marker=''
+            matrix='["thingy91x","nrf9151dk","ppk_thingy91x"]'
+          elif [[ ${{ github.event_name == 'push' }} == true ]]; then
+            marker='-m "not slow"'
+            matrix='["thingy91x"]'
           else
-            matrix=${{ inputs.devices }}
+            marker='-m "not slow"'
+            matrix='["thingy91x"]'
           fi
 
-          echo "Matrix as string: $matrix"
+          echo Matrix as string: $matrix
+          echo Pytest marker as string: $marker
           echo "device_matrix=$matrix" >> $GITHUB_OUTPUT
+          echo "pytest_marker=$marker" >> $GITHUB_OUTPUT
 
   target_test:
     # This will create multiple jobs, one for each target defined in the matrix
@@ -119,7 +132,7 @@ jobs:
           nrfutil install trace
 
       - name: Upload symbol file to Memfault
-        if: ${{ matrix.device == 'thingy91x' }}
+        if: ${{ matrix.device == 'thingy91x' && inputs.test_all }}
         working-directory: asset-tracker-template/tests/on_target/artifacts
         run: |
           memfault \
@@ -131,7 +144,7 @@ jobs:
               asset-tracker-template-${{ inputs.artifact_fw_version }}-debug-${{ vars.DUT_DEVICE_TYPE }}-nrf91.elf
 
       - name: Generate and Push RAM and FLASH Badges
-        if: ${{ matrix.device == 'thingy91x' && inputs.is_scheduled }}
+        if: ${{ matrix.device == 'thingy91x' && github.event_name == 'schedule' }}
         continue-on-error: true
         working-directory: asset-tracker-template
         env:
@@ -144,12 +157,12 @@ jobs:
           mkdir -p results
           if [[ '${{ matrix.device }}' == 'ppk_thingy91x' ]]; then
             # For PPK device, only run tests/test_ppk
-            pytest -v ${{ inputs.pytest_marker }} \
+            pytest -v ${{ needs.set-matrix.outputs.pytest_marker }} \
               --junit-xml=results/test-results.xml \
               --html=results/test-results.html --self-contained-html \
               tests/test_ppk
           else
-            pytest -v ${{ inputs.pytest_marker }} \
+            pytest -v ${{ needs.set-matrix.outputs.pytest_marker }} \
               --junit-xml=results/test-results.xml \
               --html=results/test-results.html --self-contained-html \
               --ignore=tests/test_ppk \

tests/on_target/tests/pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
-    mqtt: marks tests that require MQTT functionality

tests/on_target/tests/test_functional/test_memfault.py

@@ -88,6 +88,7 @@ def timestamp(event):
         event["captured_date"], "%Y-%m-%dT%H:%M:%S.%f%z"
     )
 
+@pytest.mark.slow
 def test_memfault(dut_board, debug_hex_file):
     # Save timestamp of latest coredump
     coredumps = get_latest_coredump_traces(IMEI)

tests/on_target/tests/test_functional/test_mqtt.py

@@ -13,7 +13,6 @@
 
 logger = get_logger()
 
-@pytest.mark.mqtt
 @pytest.mark.slow
 def test_mqtt_firmware(dut_board, hex_file_mqtt):
     """Test the firmware with cloud MQTT module."""

tests/on_target/tests/test_functional/test_patched.py

@@ -4,16 +4,16 @@
 ##########################################################################################
 
 import os
-from utils.flash_tools import flash_device, reset_device
-import sys
 import pytest
 from utils.flash_tools import flash_device, reset_device
+import sys
 
 sys.path.append(os.getcwd())
 from utils.logger import get_logger
 
 logger = get_logger()
 
+@pytest.mark.slow
 def test_patched_firmware(dut_board, hex_file_patched):
     """Test the patched firmware with magnetometer functionality."""
     # Only run this test for thingy91x devices