test_native.yml

name: Run Tests on Native platform

on:
  workflow_call:
  workflow_dispatch:

permissions: {}

env:
  # Only pushes to the default branch (develop) populate the cache; PR / merge_group runs
  # restore it but never save, so they stop filling up the repo's Actions cache storage.
  SAVE_CACHE: ${{ github.event_name == 'push' && github.ref_name == github.event.repository.default_branch }}
  LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"

jobs:
  # Guard the registered native-suite total. `platformio test` discovers and runs whatever
  # test_* directories exist, so it never notices when test/native-suite-count drifts from the
  # actual directory count (a suite added without registering it, or the file left stale). That
  # reconciliation only lives in bin/run-tests.sh, which CI does not invoke - so mirror the exact
  # check here and fail the PR on a mismatch, keeping the manual count honest.
  suite-count-check:
    name: Native Suite Count
    runs-on: ubuntu-slim
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Reconcile native-suite-count with test/ directories
        shell: bash
        run: |
          set -euo pipefail
          count_file="test/native-suite-count"
          if [[ ! -f $count_file ]]; then
            echo "::error title=Missing native-suite-count::$count_file not found - it must record the number of test_* suite directories."
            exit 1
          fi
          # Same canonical set as bin/run-tests.sh: directories named test_* directly under test/.
          expected_count=$(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | wc -l)
          canonical_count=$(tr -d '[:space:]' <"$count_file")
          if ! [[ $canonical_count =~ ^[0-9]+$ ]]; then
            echo "::error title=Invalid native-suite-count::$count_file must contain a single integer, got '$canonical_count'."
            exit 1
          fi
          echo "test/ directories: $expected_count"
          echo "native-suite-count: $canonical_count"
          if [[ $expected_count -ne $canonical_count ]]; then
            if [[ $expected_count -gt $canonical_count ]]; then
              hint="a suite was added - bump $count_file to $expected_count"
            else
              hint="a suite was removed - lower $count_file to $expected_count"
            fi
            echo "::error title=native-suite-count mismatch::test/ has $expected_count suite directories but $count_file says $canonical_count ($hint)."
            exit 1
          fi
          echo "native-suite-count matches the $expected_count suite directories."

  # Reject naive deadline comparisons against the 32-bit uptime clocks. `millis() > deadline` and
  # `deadline < millis()` invert while the deadline sits on the far side of the 32-bit wrap: the
  # action fires immediately, or blocks for about the interval it should have waited. The correct
  # forms are
  # Throttle::isWithinTimespanMs / hasElapsed (elapsed since a stored event) and
  # Throttle::deadlinePassed (an absolute deadline). See .github/copilot-instructions.md.
  millis-deadline-check:
    # Name is load-bearing: upstream branch protection matches the check by name. Widen the guard,
    # not this string.
    name: Naive millis() Deadline Compare
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Reject 32-bit uptime clocks used directly in a deadline comparison
        shell: bash
        run: |
          set -euo pipefail
          allowlist=".github/millis-deadline-allowlist.txt"

          # Flag millis() or its Time::getMillis() wrapper directly adjacent to a comparison
          # operator, in either order. The correct idioms subtract first, so they are not matched.
          #
          # Line comments are stripped before matching, so prose may name the broken idiom (this
          # guard's own documentation does). Block comments are not stripped; keep `millis() >` out
          # of /* */ blocks. mawk-compatible - ubuntu-latest has no gawk.
          find src -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.hpp' -o -name '*.ino' \) \
               ! -path 'src/mesh/generated/*' -print0 |
            xargs -0 awk '
              {
                line = $0
                sub(/\/\/.*/, "", line)
                if (line ~ /((millis|getMillis)\(\)[ \t]*[<>]=?)|([<>]=?[ \t]*(millis|getMillis)\(\))/) {
                  code = line
                  sub(/^[ \t]+/, "", code); sub(/[ \t]+$/, "", code)
                  printf "%s\t%s\t%s\n", FILENAME, FNR, code
                }
              }' > /tmp/millis-hits.tsv

          # Allowlisted entries are keyed on file + exact source text, deliberately without a line
          # number, so unrelated edits above them do not invalidate the entry.
          : > /tmp/millis-allowed.tsv
          if [[ -f $allowlist ]]; then
            grep -vE '^[[:space:]]*(#|$)' "$allowlist" > /tmp/millis-allowed.tsv || true
          fi

          violations=0
          while IFS=$'\t' read -r file line code; do
            [[ -n ${file:-} ]] || continue
            if grep -qxF "$(printf '%s\t%s' "$file" "$code")" /tmp/millis-allowed.tsv; then
              continue
            fi
            echo "$file:$line: $code"
            violations=$((violations + 1))
          done < /tmp/millis-hits.tsv

          if [[ $violations -gt 0 ]]; then
            echo "::error title=Naive uptime deadline compare::$violations line(s) compare a 32-bit uptime clock directly, which inverts while the deadline is on the far side of the 32-bit wrap - the action fires immediately, or blocks for about the interval it should have waited. Use Throttle::deadlinePassed(deadline) for a stored absolute deadline, or Throttle::hasElapsed(lastEvent, intervalMs) for an interval. If a match genuinely is not a deadline test (an uptime threshold, say), add it to $allowlist with a reason."
            exit 1
          fi
          echo "No naive 32-bit uptime deadline comparisons in src/ (allowlist: $(wc -l < /tmp/millis-allowed.tsv) entr(y/ies))."

  simulator-tests:
    name: Native Simulator Tests
    runs-on: ubuntu-24.04-arm
    needs: suite-count-check
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Install simulator dependencies
        run: pip install -U dotmap

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-simulator-tests-

      # We now run integration test before other build steps (to quickly see runtime failures)
      - name: Build for native/coverage
        run: platformio run -e coverage

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Capture initial coverage information
        shell: bash
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
          sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.

      - name: Config check tests
        # Drives the same binary against test/fixtures/portduino-config: asserts that
        # `--check` reports each planted fault, and that a normal run still refuses the
        # configs it should. Runs before the simulator test because it is seconds long
        # and a failure here explains a lot of downstream weirdness.
        timeout-minutes: 5
        run: ./bin/test-config-check.sh .pio/build/coverage/meshtasticd

      - name: Integration test
        # Cap the whole step: if the simulator ever fails to exit (e.g. the
        # exit_simulator admin path regresses again) the job must fail fast,
        # not run to GitHub's 6-hour limit.
        timeout-minutes: 5
        run: |
          .pio/build/coverage/meshtasticd -s &
          PID=$!
          trap 'kill "$PID" 2>/dev/null || true' EXIT
          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
          echo "Simulator started, launching python test..."
          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
          # The Python harness sends exit_simulator and exits; the simulator is
          # expected to terminate on its own. Give it a moment, then verify.
          # If it is still alive the exit handshake is broken - fail loudly and
          # do NOT fall through to `wait`, which would otherwise block until the
          # job's hard timeout.
          for i in $(seq 1 10); do
            kill -0 "$PID" 2>/dev/null || break
            sleep 1
          done
          if kill -0 "$PID" 2>/dev/null; then
            echo "::error title=Simulator did not exit::meshtasticd ignored exit_simulator and is still running after the integration test. The exit_simulator admin path is broken (see AdminModule::handleReceivedProtobuf, ARCH_PORTDUINO bypass). Killing it to avoid a 6-hour CI overrun."
            kill -9 "$PID" 2>/dev/null || true
            wait "$PID" 2>/dev/null || true
            exit 1
          fi
          wait "$PID" 2>/dev/null || true

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
          sed -i -e "s#${PWD}#.#" coverage_integration.info # Make paths relative.

      - name: Get release version string
        if: always() # run this step even if previous step failed
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  platformio-tests:
    name: Native PlatformIO Tests
    runs-on: ubuntu-24.04-arm
    needs: suite-count-check
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      # Disable (comment-out) BUILD_EPOCH. It causes a full rebuild between tests and resets the
      # coverage information each time.
      - name: Disable BUILD_EPOCH
        run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-coverage-tests-

      - name: Build test programs once
        # One shared build of src + every test program. This is the single source build; gcov then
        # accumulates coverage counts into this shared .pio/build/coverage/src as the chunks run.
        run: platformio test -e coverage --without-testing

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Run tests one area at a time
        shell: bash
        run: |
          set -uo pipefail
          # One runner, no matrix, no concurrency. Group the test_* suites by area and run each
          # area sequentially, reusing the single build above (--without-building). Each area gets
          # its own JUnit report and its own collapsible log, so a failure lands in a small named
          # section instead of being buried past the log limit. Sequential runs share one build
          # dir, so gcov coverage accumulates and the single capture in the next step has the union.

          # Ordered area rules "name:ERE"; first match wins. Anything unmatched falls to "misc", so
          # a newly added suite always runs even before it is placed. Add a suite to an area by
          # extending that area's regex; add a new area by inserting a rule line.
          area_rules=(
            "admin:^test_(admin|pki)_"
            "crypto:^test_(crypto|packet_signing)$"
            "routing:^test_(mesh|nexthop|traceroute|hop|traffic|nodedb|warm)_"
            "position:^test_position_"
            "fuzz:^test_fuzz_"
            "packets:^test_(packet|transmit|meshpacket)_"
            "io:^test_(serial|stream|xmodem|http|mqtt)"
          )

          mapfile -t suites < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
          declare -A group
          for s in "${suites[@]}"; do
            a="misc"
            for rule in "${area_rules[@]}"; do
              if [[ "$s" =~ ${rule#*:} ]]; then a="${rule%%:*}"; break; fi
            done
            group[$a]="${group[$a]:-} -f $s"
          done

          run_order=()
          for rule in "${area_rules[@]}"; do run_order+=("${rule%%:*}"); done
          run_order+=("misc")

          fail=0
          for a in "${run_order[@]}"; do
            [ -n "${group[$a]:-}" ] || continue
            echo "::group::area $a (${group[$a]# })"
            # Capture platformio's real exit status (not grep's) via a log file, then show the log
            # with the noisy per-variant SKIPPED rows filtered out.
            if ! platformio test -e coverage --without-building -v ${group[$a]# } \
                 --junit-output-path "testreport-$a.xml" > "area-$a.log" 2>&1; then
              fail=1
              echo "::error::area $a had test failures"
            fi
            grep -v "[[:space:]]SKIPPED$" "area-$a.log" || true
            echo "::endgroup::"
          done
          exit $fail

      - name: Merge per-area reports into testreport.xml
        # Preserve the single-file JUnit contract that downstream consumers rely on
        # (pr_tests.yml's summary and generate-reports' Test Report both read testreport.xml).
        # The per-area split is only for readable logs; the report stays consolidated.
        if: always() # run even when a chunk failed, so the report captures the failures
        shell: bash
        run: |
          python3 - <<'PY'
          import glob, xml.etree.ElementTree as ET
          out = ET.Element('testsuites')
          for f in sorted(glob.glob('testreport-*.xml')):
              try:
                  root = ET.parse(f).getroot()
              except ET.ParseError:
                  continue
              # PlatformIO writes a <testsuites> root; fold in a bare <testsuite> too, just in case.
              out.extend(root.findall('testsuite') if root.tag == 'testsuites' else [root])
          ET.ElementTree(out).write('testreport.xml', encoding='utf-8', xml_declaration=True)
          PY

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.

      - name: Save test results
        if: always() # run this step even if previous step failed
        uses: actions/upload-artifact@v7
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./testreport.xml

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  generate-reports:
    name: Generate Test Reports
    runs-on: ubuntu-latest
    permissions: # Needed for dorny/test-reporter.
      contents: read
      actions: read
      checks: write
    needs:
      - simulator-tests
      - platformio-tests
    # Run this job even if the previous jobs failed, but skip if the workflow was cancelled.
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Download test artifacts
        uses: actions/download-artifact@v8
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          merge-multiple: true

      - name: Drop no-status testsuites from the report
        # PlatformIO emits a self-closing <testsuite tests="0"/> row for every test_* dir
        # crossed with every hardware variant it cannot run on the native host (~4900 rows).
        # They carry no pass/fail/skip status and bury the suites that actually ran. Strip
        # them so the Test Report lists only suites with a real status. Only the copy the
        # reporter renders is trimmed; the uploaded artifact keeps the full XML.
        run: sed -i -E 's#<testsuite [^>]*tests="0"[^>]*/>##g' testreport.xml

      - name: Test Report
        uses: dorny/test-reporter@v3.0.0
        with:
          name: PlatformIO Tests
          path: testreport.xml
          reporter: java-junit

      - name: Download coverage artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.long }}
          path: code-coverage-report
          merge-multiple: true

      - name: Generate Code Coverage Report
        # Merge every tracefile the jobs produced: coverage_base.info (zeroed baseline),
        # coverage_integration.info, and one coverage_tests_<chunk>.info per chunk. lcov
        # sums hit counts across them, so the merged report is the union of all chunks -
        # identical to running the whole suite in one job.
        run: |
          sudo apt-get install -y lcov
          args=()
          for f in code-coverage-report/coverage_*.info; do
            args+=(--add-tracefile "$f")
          done
          lcov --quiet "${args[@]}" --output-file code-coverage-report/coverage_src.info
          genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report

      - name: Save Code Coverage Report
        uses: actions/upload-artifact@v7
        with:
          name: code-coverage-report-${{ steps.version.outputs.long }}
          path: code-coverage-report