diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 91eb2690e90..a758d26ad6d 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -55,7 +55,11 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + with: + # Nothing in this job runs git after the checkout, so there is no reason to + # leave the token in .git/config while it executes scripts from the branch. + persist-credentials: false + - uses: actions/setup-python@v7 with: python-version: 3.x cache: pip @@ -73,6 +77,11 @@ jobs: echo "Name: $GITHUB_REF_NAME Base: $GITHUB_BASE_REF Ref: $GITHUB_REF" echo "all=$TARGETS" >> $GITHUB_OUTPUT echo "$TARGETS" >> $GITHUB_STEP_SUMMARY + + # Static config check, like the board_level validation above: no toolchain, and + # it covers every ESP32 env, not just the subset this event builds. + - name: Check ESP32 flash geometry + run: python3 bin/check_partition_sizes.py --fetch-board-manifests "$RUNNER_TEMP/esp32-boards" outputs: all: ${{ steps.jsonStep.outputs.all }} diff --git a/bin/check_partition_sizes.py b/bin/check_partition_sizes.py new file mode 100644 index 00000000000..01969866a49 --- /dev/null +++ b/bin/check_partition_sizes.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 + +"""Cross-check ESP32 flash geometry across the three places it is declared. + +For every ESP32 PlatformIO environment this reconciles: + + * ``upload.maximum_size`` from the resolved board manifest (the board's flash size), + * the end of the last partition in the selected partition CSV, + * ``custom_meshtastic_partition_scheme``, the flash-size tier published in the + ``.mt.json`` manifest (bin/platformio-custom.py) and consumed by the flasher. + +Nothing at build time reconciles them: the espressif32 builder overwrites +``upload.maximum_size`` with the app-slot size from the partition table +(``_update_max_upload_size``), so a variant pointing at a 16MB table on an 8MB +board still links and only fails when it is flashed. This script is that +reconciliation, run as a static config check before anything is built. + +Board resolution mirrors the builder: the ``board_build.partitions`` project +option wins, otherwise ``build.partitions`` from the board manifest. Board +manifests are looked up in the project ``boards/`` directory first, then in the +installed platform packages under the PlatformIO core directory. + +Exit status is 0 when every environment is consistent, 1 otherwise. +""" + +import argparse +import glob +import io +import json +import os +import re +import shutil +import sys +import urllib.error +import urllib.parse +import urllib.request +import zipfile + +from platformio.project.config import ProjectConfig + +# Board manifests are fetched as data, never via 'pio pkg install', which +# exec_module()s the downloaded platform.py. Owner allowlist, not just host. +ALLOWED_MANIFEST_HOSTS = ("github.com", "codeload.github.com", "raw.githubusercontent.com") +ALLOWED_MANIFEST_OWNERS = ("meshtastic", "pioarduino", "platformio") +MANIFEST_FETCH_TIMEOUT = 120 +MAX_ARCHIVE_BYTES = 128 * 1024 * 1024 +MAX_MANIFEST_BYTES = 1024 * 1024 + +# Flash-size tiers a board can ship; used to turn a byte count into the tier that +# 'custom_meshtastic_partition_scheme' names (e.g. 8388608 -> "8MB"). +FLASH_TIERS = ( + 1 * 1024 * 1024, + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, + 64 * 1024 * 1024, +) + +# Partition table sits at 0x8000 and fills a 0x1000 sector, so an omitted first +# offset starts right after it. Same constant the builder uses. +FIRST_PARTITION_OFFSET = 0x9000 + + +def parse_size(value): + """Parse a partition CSV size/offset: decimal, 0x hex, or a K/M suffix.""" + if isinstance(value, int): + return value + value = str(value).strip() + if value.isdigit(): + return int(value) + if value.lower().startswith("0x"): + return int(value, 16) + if value and value[-1].upper() in ("K", "M"): + base = 1024 if value[-1].upper() == "K" else 1024 * 1024 + return int(value[:-1]) * base + raise ValueError(f"unparseable size '{value}'") + + +def parse_partitions(csv_path): + """Parse a partition CSV into [(name, offset, size)], resolving omitted offsets. + + Mirrors _parse_partitions() in the espressif32 builder: app partitions align + to 0x10000, everything else to 4 bytes. + """ + rows = [] + next_offset = FIRST_PARTITION_OFFSET + with open(csv_path, encoding="utf-8") as fp: + for lineno, line in enumerate(fp, start=1): + line = line.strip() + if not line or line.startswith("#"): + continue + tokens = [t.strip() for t in line.split(",")] + if len(tokens) < 5: + continue + bound = 0x10000 if tokens[1] in ("0", "app") else 4 + calculated_offset = (next_offset + bound - 1) & ~(bound - 1) + try: + offset = parse_size(tokens[3]) if tokens[3] else calculated_offset + size = parse_size(tokens[4]) + except ValueError as exc: + raise ValueError(f"{csv_path}:{lineno}: {exc}") from exc + rows.append((tokens[0], offset, size)) + next_offset = offset + size + if not rows: + raise ValueError(f"{csv_path}: no partition entries") + return rows + + +def flash_tier(size_bytes): + """Smallest standard flash size that can hold 'size_bytes', or None.""" + for tier in FLASH_TIERS: + if size_bytes <= tier: + return tier + return None + + +def format_tier(size_bytes): + return f"{size_bytes // (1024 * 1024)}MB" + + +def parse_scheme(scheme): + """Parse a 'custom_meshtastic_partition_scheme' value ('8MB') into bytes.""" + match = re.fullmatch(r"\s*(\d+)\s*([KM])B?\s*", scheme, re.IGNORECASE) + if not match: + return None + base = 1024 if match.group(2).upper() == "K" else 1024 * 1024 + return int(match.group(1)) * base + + +def find_board_manifest(board_id, project_dir, core_dir, fetched_dir=None): + """Locate a board manifest: project boards/, then fetched, then installed platforms.""" + local = os.path.join(project_dir, "boards", f"{board_id}.json") + if os.path.isfile(local): + return local + if fetched_dir: + fetched = os.path.join(fetched_dir, f"{board_id}.json") + if os.path.isfile(fetched): + return fetched + # Only espressif32*, so a same-named board in another installed platform can never + # answer; its versions unpack side by side, and sorted() keeps the pick deterministic. + pattern = os.path.join(core_dir, "platforms", "espressif32*", "boards", f"{board_id}.json") + installed = sorted(glob.glob(pattern)) + return installed[0] if installed else None + + +def check_manifest_url(url): + """Reject a platform archive URL that is not an allowlisted https location. + + The URL is read out of the checkout, so on a fork PR it is attacker-controlled. + Restricting it to repositories owned by the projects this firmware actually + tracks keeps a PR from redirecting the fetch at content it authored. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https": + return f"'{url}' is not https" + if parsed.hostname not in ALLOWED_MANIFEST_HOSTS: + return f"host '{parsed.hostname}' is not one of {', '.join(ALLOWED_MANIFEST_HOSTS)}" + owner = parsed.path.lstrip("/").split("/")[0] + if owner not in ALLOWED_MANIFEST_OWNERS: + return f"owner '{owner}' is not one of {', '.join(ALLOWED_MANIFEST_OWNERS)}" + return None + + +class AllowlistedRedirectHandler(urllib.request.HTTPRedirectHandler): + """Re-check the allowlist on each redirect target; the default opener follows + redirects blind, and this fetch does redirect (github.com to codeload.github.com).""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + problem = check_manifest_url(newurl) + if problem: + raise urllib.error.HTTPError(newurl, code, f"refusing redirect: {problem}", headers, fp) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def fetch_board_manifests(url, dest_dir): + """Download a platform archive and extract only its boards/*.json, as data. + + Nothing from the archive is imported or executed, and every file is written to + dest_dir under its basename alone, so a crafted member name cannot escape it. + """ + problem = check_manifest_url(url) + if problem: + raise ValueError(f"refusing to fetch board manifests: {problem}") + + # check_manifest_url() gates the initial URL and every redirect target, so file:// + # and untrusted hosts cannot reach the fetch. + request = urllib.request.Request(url, headers={"User-Agent": "meshtastic-firmware-ci"}) # noqa: S310 + opener = urllib.request.build_opener(AllowlistedRedirectHandler) + # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected + with opener.open(request, timeout=MANIFEST_FETCH_TIMEOUT) as response: + payload = response.read(MAX_ARCHIVE_BYTES + 1) + if len(payload) > MAX_ARCHIVE_BYTES: + raise ValueError(f"archive at {url} exceeds {MAX_ARCHIVE_BYTES} bytes") + + os.makedirs(dest_dir, exist_ok=True) + written = 0 + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + for entry in archive.infolist(): + parts = entry.filename.split("/") + if entry.is_dir() or len(parts) < 2 or parts[-2] != "boards": + continue + if not parts[-1].endswith(".json") or entry.file_size > MAX_MANIFEST_BYTES: + continue + # basename only: never join a path taken from the archive. + target = os.path.join(dest_dir, os.path.basename(parts[-1])) + with archive.open(entry) as src, open(target, "wb") as dst: + shutil.copyfileobj(src, dst, length=64 * 1024) + written += 1 + if not written: + raise ValueError(f"archive at {url} contained no boards/*.json") + return written + + +def board_option(manifest, dotted_key): + """Read a dotted key ('build.partitions') out of a board manifest.""" + node = manifest + for part in dotted_key.split("."): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def esp32_envs(cfg, only_env=None): + """Yield (env_name, platform_dir) for ESP32 environments. + + Platform comes from the '-I variants/