firmware/.github/workflows/test_native.yml

name: Run Tests on Native platform

on:
  workflow_call:
    inputs:
      suite_order_seed:
        description: >-
          Seed for shuffling the test-area order. Empty (the default) means: fixed declared order on
          pull_request, so a contributor's PR never turns red because of an order they did not
          choose; commit-SHA-derived elsewhere. Set a number to force that exact order anywhere -
          that is how you replay a shuffled failure.
        type: string
        required: false
        default: ""
  workflow_dispatch:

permissions: {}

env:
  # Only pushes to the default branch (develop) populate the cache; PR / merge_group runs
  # restore it but never save, so they stop filling up the repo's Actions cache storage.
  SAVE_CACHE: ${{ github.event_name == 'push' && github.ref_name == github.event.repository.default_branch }}
  LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"

jobs:
  # Tripwire against the native suite set shrinking by accident. `platformio test` discovers and
  # runs whatever test_* directories exist, and bin/run-tests.sh derives its expected count from
  # the same walk - so a suite directory lost in a bad rebase or an overzealous cleanup just means
  # fewer suites run, and every remaining check stays green. Compare the test_* directory list
  # against the PR's merge base and fail when a suite vanished without the PR saying so: a removed
  # suite's name must appear in the PR title, the PR body, or a commit message in the PR's range.
  # A deliberate removal satisfies that by stating what it removes; an accidental loss cannot.
  # Only pull_request runs have a base to compare against (and PRs are where accidents arrive);
  # every other event skips. No job depends on this one: a skipped job would skip its dependents,
  # and the expensive jobs should not wait on a full-history clone.
  suite-shrinkage-check:
    name: Native Suite Shrinkage
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-slim
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
          # Full history: the merge base must be computed, not guessed from a possibly stale
          # event payload, and the acknowledgment scan reads every commit message in the range.
          fetch-depth: 0

      - name: Fail if a test_* suite vanished unacknowledged
        shell: bash
        # PR title/body are attacker-controlled text; they reach the script through env: only,
        # never spliced into the shell source (same rule as the suite-order seed below).
        env:
          BASE_REF: ${{ github.base_ref }}
          PR_TITLE: ${{ github.event.pull_request.title }}
          PR_BODY: ${{ github.event.pull_request.body }}
        run: |
          set -euo pipefail
          git fetch --quiet origin "$BASE_REF"
          base=$(git merge-base FETCH_HEAD HEAD)
          # Same canonical set every other consumer derives: directories named test_* directly
          # under test/, read from the git trees so the comparison is exact at both endpoints.
          list_suites() { git ls-tree -d --name-only "$1" test/ | sed 's#^test/##' | grep '^test_' | sort; }
          removed=$(comm -23 <(list_suites "$base") <(list_suites HEAD))
          if [[ -z $removed ]]; then
            echo "No suite removed: $(list_suites HEAD | wc -l) test_* directories, none lost since merge base ${base:0:8}."
            exit 0
          fi
          messages=$(git log --format=%B "$base..HEAD")
          fail=0
          while IFS= read -r suite; do
            if printf '%s\n%s\n%s\n' "$PR_TITLE" "$PR_BODY" "$messages" | grep -qF "$suite"; then
              echo "Removed suite $suite is named in the PR title/body or a commit message - acknowledged."
            else
              echo "::error title=Native suite vanished::test/$suite exists on the merge base but is gone from this PR, and nothing in the PR title, body, or commit messages mentions it. If the removal is deliberate, name $suite in the PR description or a commit message; if not, restore the directory - platformio test would silently run without it."
              fail=1
            fi
          done <<<"$removed"
          exit $fail

  # Reject naive deadline comparisons against the 32-bit uptime clocks. `millis() > deadline` and
  # `deadline < millis()` invert while the deadline sits on the far side of the 32-bit wrap: the
  # action fires immediately, or blocks for about the interval it should have waited. The correct
  # forms are
  # Throttle::isWithinTimespanMs / hasElapsed (elapsed since a stored event) and
  # Throttle::deadlinePassed (an absolute deadline). See .github/copilot-instructions.md.
  millis-deadline-check:
    # Name is load-bearing: upstream branch protection matches the check by name. Widen the guard,
    # not this string.
    name: Naive millis() Deadline Compare
    runs-on: ubuntu-latest
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false

      - name: Reject 32-bit uptime clocks used directly in a deadline comparison
        shell: bash
        run: |
          set -euo pipefail
          allowlist=".github/millis-deadline-allowlist.txt"

          # Flag millis() or its Time::getMillis() wrapper directly adjacent to a comparison
          # operator, in either order. The correct idioms subtract first, so they are not matched.
          #
          # Line comments are stripped before matching, so prose may name the broken idiom (this
          # guard's own documentation does). Block comments are not stripped; keep `millis() >` out
          # of /* */ blocks. mawk-compatible - ubuntu-latest has no gawk.
          find src -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.hpp' -o -name '*.ino' \) \
               ! -path 'src/mesh/generated/*' -print0 |
            xargs -0 awk '
              {
                line = $0
                sub(/\/\/.*/, "", line)
                if (line ~ /((millis|getMillis)\(\)[ \t]*[<>]=?)|([<>]=?[ \t]*(millis|getMillis)\(\))/) {
                  code = line
                  sub(/^[ \t]+/, "", code); sub(/[ \t]+$/, "", code)
                  printf "%s\t%s\t%s\n", FILENAME, FNR, code
                }
              }' > /tmp/millis-hits.tsv

          # Allowlisted entries are keyed on file + exact source text, deliberately without a line
          # number, so unrelated edits above them do not invalidate the entry.
          : > /tmp/millis-allowed.tsv
          if [[ -f $allowlist ]]; then
            grep -vE '^[[:space:]]*(#|$)' "$allowlist" > /tmp/millis-allowed.tsv || true
          fi

          violations=0
          while IFS=$'\t' read -r file line code; do
            [[ -n ${file:-} ]] || continue
            if grep -qxF "$(printf '%s\t%s' "$file" "$code")" /tmp/millis-allowed.tsv; then
              continue
            fi
            echo "$file:$line: $code"
            violations=$((violations + 1))
          done < /tmp/millis-hits.tsv

          if [[ $violations -gt 0 ]]; then
            echo "::error title=Naive uptime deadline compare::$violations line(s) compare a 32-bit uptime clock directly, which inverts while the deadline is on the far side of the 32-bit wrap - the action fires immediately, or blocks for about the interval it should have waited. Use Throttle::deadlinePassed(deadline) for a stored absolute deadline, or Throttle::hasElapsed(lastEvent, intervalMs) for an interval. If a match genuinely is not a deadline test (an uptime threshold, say), add it to $allowlist with a reason."
            exit 1
          fi
          echo "No naive 32-bit uptime deadline comparisons in src/ (allowlist: $(wc -l < /tmp/millis-allowed.tsv) entr(y/ies))."

  simulator-tests:
    name: Native Simulator Tests
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Install simulator dependencies
        run: pip install -U dotmap

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-simulator-tests-

      # We now run integration test before other build steps (to quickly see runtime failures)
      - name: Build for native/coverage
        run: platformio run -e coverage

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Capture initial coverage information
        shell: bash
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
          sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.

      - name: Config check tests
        # Drives the same binary against test/fixtures/portduino-config: asserts that
        # `--check` reports each planted fault, and that a normal run still refuses the
        # configs it should. Runs before the simulator test because it is seconds long
        # and a failure here explains a lot of downstream weirdness.
        timeout-minutes: 5
        run: ./bin/test-config-check.sh .pio/build/coverage/meshtasticd

      - name: Shared-state checker self-test
        # Fixtures that write nothing / exactly what they declare / something undeclared /
        # a declared write they never make, asserting CLEAN / CLEAN / DIRTY / MISSING. A
        # checker that has silently stopped matching looks identical to a clean codebase.
        timeout-minutes: 5
        run: ./bin/test-state-check.sh

      - name: Integration test
        # Cap the whole step: if the simulator ever fails to exit (e.g. the
        # exit_simulator admin path regresses again) the job must fail fast,
        # not run to GitHub's 6-hour limit.
        timeout-minutes: 5
        run: |
          .pio/build/coverage/meshtasticd -s &
          PID=$!
          trap 'kill "$PID" 2>/dev/null || true' EXIT
          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
          echo "Simulator started, launching python test..."
          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
          # The Python harness sends exit_simulator and exits; the simulator is
          # expected to terminate on its own. Give it a moment, then verify.
          # If it is still alive the exit handshake is broken - fail loudly and
          # do NOT fall through to `wait`, which would otherwise block until the
          # job's hard timeout.
          for i in $(seq 1 10); do
            kill -0 "$PID" 2>/dev/null || break
            sleep 1
          done
          if kill -0 "$PID" 2>/dev/null; then
            echo "::error title=Simulator did not exit::meshtasticd ignored exit_simulator and is still running after the integration test. The exit_simulator admin path is broken (see AdminModule::handleReceivedProtobuf, ARCH_PORTDUINO bypass). Killing it to avoid a 6-hour CI overrun."
            kill -9 "$PID" 2>/dev/null || true
            wait "$PID" 2>/dev/null || true
            exit 1
          fi
          wait "$PID" 2>/dev/null || true

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
          sed -i -e "s#${PWD}#.#" coverage_integration.info # Make paths relative.

      - name: Get release version string
        if: always() # run this step even if previous step failed
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  platformio-tests:
    name: Native PlatformIO Tests
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      # Disable (comment-out) BUILD_EPOCH. It causes a full rebuild between tests and resets the
      # coverage information each time.
      - name: Disable BUILD_EPOCH
        run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-coverage-tests-

      - name: Warm the shared test build
        # Compiles src + every test program once so no single area absorbs the whole src build in
        # its reported duration; gcov then accumulates counts into this shared
        # .pio/build/coverage/src as the areas run. NOT a substitute for building in the run step:
        # PlatformIO links every test program to the one .pio/build/coverage/meshtasticd path, so a
        # --without-building run executes whichever suite was linked last under every suite's name.
        run: platformio test -e coverage --without-testing

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Run tests one area at a time
        shell: bash
        # Both values reach the script through env: rather than ${{ }} inside run:, so nothing from
        # the event payload is ever spliced into the shell text.
        env:
          SUITE_ORDER_SEED: ${{ inputs.suite_order_seed }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          set -uo pipefail
          # One runner, no matrix, no concurrency. Group the test_* suites by area and run each
          # area sequentially, reusing the single build above (--without-building). Each area gets
          # its own JUnit report and its own collapsible log, so a failure lands in a small named
          # section instead of being buried past the log limit. Sequential runs share one build
          # dir, so gcov coverage accumulates and the single capture in the next step has the union.

          # Ordered area rules "name:ERE"; first match wins. Anything unmatched falls to "misc", so
          # a newly added suite always runs even before it is placed. Add a suite to an area by
          # extending that area's regex; add a new area by inserting a rule line.
          area_rules=(
            "admin:^test_(admin|pki)_"
            "crypto:^test_(crypto|packet_signing)$"
            "routing:^test_(mesh|nexthop|traceroute|hop|traffic|nodedb|warm)_"
            "position:^test_position_"
            "fuzz:^test_fuzz_"
            "packets:^test_(packet|transmit|meshpacket)_"
            "io:^test_(serial|stream|xmodem|http|mqtt)"
          )

          mapfile -t suites < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
          declare -A group
          for s in "${suites[@]}"; do
            a="misc"
            for rule in "${area_rules[@]}"; do
              if [[ "$s" =~ ${rule#*:} ]]; then a="${rule%%:*}"; break; fi
            done
            group[$a]="${group[$a]:-} -f $s"
          done

          run_order=()
          for rule in "${area_rules[@]}"; do run_order+=("${rule%%:*}"); done
          run_order+=("misc")

          # Area order. The rule order above is an accident of how the areas were written, and
          # running it fixed forever means order dependence between areas is never observed - but
          # randomising it on a contributor's PR would turn their run red for an order they did not
          # choose, which is how a randomisation gets reverted instead of the coupling fixed.
          #
          # So: pull_request keeps the fixed declared order. Everywhere else (push, schedule) the
          # order is shuffled, seeded from the commit SHA - deterministic per commit, replayable,
          # attributable, and it never blocks someone else's PR. An explicit seed input overrides
          # both, which is how you replay a specific failing order anywhere.
          #
          # Intra-area order stays PlatformIO's: filters select suites, they do not order them
          # (list_test_names() walks test/ with os.walk()), so controlling it needs one invocation
          # per suite. bin/run-tests.sh --shuffle does exactly that locally.
          seed_input="${SUITE_ORDER_SEED:-}"
          if [ -n "$seed_input" ]; then
            seed="$seed_input"
            echo "area order: shuffled with explicitly supplied seed $seed"
          elif [ "${EVENT_NAME:-}" = "pull_request" ]; then
            seed=""
            echo "area order: fixed declared order (pull_request) - ${run_order[*]}"
            echo "  to exercise a different order, re-run this workflow with a suite_order_seed input"
          else
            seed=$((16#${GITHUB_SHA:0:8}))
            echo "area order: shuffled with seed $seed (from ${GITHUB_SHA:0:8})"
          fi

          if [ -n "$seed" ]; then
            # Same shuffle_suites() bin/run-tests.sh uses, so the replay hint below is true by
            # construction rather than by two copies happening to agree.
            source bin/lib/shuffle.sh
            mapfile -t run_order < <(shuffle_suites "$seed" "${run_order[@]}")
            echo "area order: ${run_order[*]}"
            echo "  replay locally: ./bin/run-tests.sh --shuffle --seed $seed"
          fi

          fail=0
          for a in "${run_order[@]}"; do
            [ -n "${group[$a]:-}" ] || continue
            echo "::group::area $a (${group[$a]# })"
            # Capture platformio's real exit status (not grep's) via a log file, then show the log
            # with the noisy per-variant SKIPPED rows filtered out.
            if ! platformio test -e coverage -v ${group[$a]# } \
                 --junit-output-path "testreport-$a.xml" > "area-$a.log" 2>&1; then
              fail=1
              echo "::error::area $a had test failures"
            fi
            # Suites outside this area are reported SKIPPED by design (PlatformIO lists every suite
            # in the env and marks the unselected ones finished), so those rows are noise here. The
            # attribution check below is what catches a suite that was selected and did not run.
            grep -v "[[:space:]]SKIPPED$" "area-$a.log" || true
            # Per area, so a mismatch names the area it happened in rather than the whole run.
            if ! ./bin/check-test-attribution.py --label "area $a" \
                 --expect "${group[$a]# }" "testreport-$a.xml"; then
              fail=1
              echo "::error::area $a ran suites that did not match their own test binaries"
            fi
            echo "::endgroup::"
          done
          exit $fail

      - name: Merge per-area reports into testreport.xml
        # Preserve the single-file JUnit contract that downstream consumers rely on
        # (pr_tests.yml's summary and generate-reports' Test Report both read testreport.xml).
        # The per-area split is only for readable logs; the report stays consolidated.
        if: always() # run even when a chunk failed, so the report captures the failures
        shell: bash
        run: |
          python3 - <<'PY'
          import glob, xml.etree.ElementTree as ET
          out = ET.Element('testsuites')
          for f in sorted(glob.glob('testreport-*.xml')):
              try:
                  root = ET.parse(f).getroot()
              except ET.ParseError:
                  continue
              # PlatformIO writes a <testsuites> root; fold in a bare <testsuite> too, just in case.
              out.extend(root.findall('testsuite') if root.tag == 'testsuites' else [root])
          ET.ElementTree(out).write('testreport.xml', encoding='utf-8', xml_declaration=True)
          PY

      - name: Verify every suite ran its own tests
        # Whole-run gate over the merged report: every test_* directory must appear with at least
        # one test case, and every case must come from the suite that reported it. The per-area
        # check above cannot see an area that never executed - this can.
        if: always() # a suite going missing is the finding; do not hide it behind an earlier failure
        shell: bash
        run: |
          set -euo pipefail
          mapfile -t suites < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
          ./bin/check-test-attribution.py --label "coverage (all areas)" \
            --expect "${suites[*]}" testreport.xml

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.

      - name: Attribution canary
        # Guards the guard above: runs two suites the broken way (--without-building, so PlatformIO
        # does not relink and both execute the same leftover binary) and requires the checker to
        # catch it. Fails if the checker regressed, or if the reproduction stops reproducing - in
        # which case the reason both harnesses stopped passing that flag no longer holds.
        #
        # Lives in this job, not simulator-tests: it relinks $BUILD_DIR/$PROGNAME, and there that
        # replaced the daemon binary with a test suite, so the integration test waited for a socket
        # a test binary never opens. Here the binary is already per-suite and nothing later needs it.
        timeout-minutes: 15
        run: ./bin/test-attribution-canary.sh -e coverage

      - name: Event channel policy tests
        run: platformio test -e coverage-event-policy -v --junit-output-path event-policy-testreport.xml

      - name: Verify the event-policy suites ran their own tests
        # Expected set read through PlatformIO's own config parser, so it cannot drift from the
        # env's test_filter the way a second hand-maintained list would.
        run: |
          set -euo pipefail
          expect=$(python3 -c "from platformio.project.config import ProjectConfig; \
            print(' '.join(ProjectConfig().get('env:coverage-event-policy', 'test_filter', [])))")
          ./bin/check-test-attribution.py --label coverage-event-policy \
            --expect "$expect" event-policy-testreport.xml

      - name: Save test results
        if: always() # run this step even if previous step failed
        uses: actions/upload-artifact@v7
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./*testreport.xml

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  generate-reports:
    name: Generate Test Reports
    runs-on: ubuntu-latest
    permissions: # Needed for dorny/test-reporter.
      contents: read
      actions: read
      checks: write
    needs:
      - simulator-tests
      - platformio-tests
    # Run this job even if the previous jobs failed, but skip if the workflow was cancelled.
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Download test artifacts
        uses: actions/download-artifact@v8
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          merge-multiple: true

      - name: Drop no-status testsuites from the report
        # PlatformIO emits a self-closing <testsuite tests="0"/> row for every test_* dir
        # crossed with every hardware variant it cannot run on the native host (~4900 rows).
        # They carry no pass/fail/skip status and bury the suites that actually ran. Strip
        # them so the Test Report lists only suites with a real status. Only the copy the
        # reporter renders is trimmed; the uploaded artifact keeps the full XML.
        run: sed -i -E 's#<testsuite [^>]*tests="0"[^>]*/>##g' testreport.xml

      - name: Test Report
        uses: dorny/test-reporter@v3.0.0
        with:
          name: PlatformIO Tests
          path: testreport.xml
          reporter: java-junit

      - name: Download coverage artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.long }}
          path: code-coverage-report
          merge-multiple: true

      - name: Generate Code Coverage Report
        # Merge every tracefile the jobs produced: coverage_base.info (zeroed baseline),
        # coverage_integration.info, and one coverage_tests_<chunk>.info per chunk. lcov
        # sums hit counts across them, so the merged report is the union of all chunks -
        # identical to running the whole suite in one job.
        run: |
          sudo apt-get install -y lcov
          args=()
          for f in code-coverage-report/coverage_*.info; do
            args+=(--add-tracefile "$f")
          done
          lcov --quiet "${args[@]}" --output-file code-coverage-report/coverage_src.info
          genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report

      - name: Save Code Coverage Report
        uses: actions/upload-artifact@v7
        with:
          name: code-coverage-report-${{ steps.version.outputs.long }}
          path: code-coverage-report