Merge branch 'feature/support_base64_logs' into 'master'

feat(binlog): Support binary log format expansion in monitor

See merge request espressif/esp-idf-monitor!24

Author: peterdragun

.pre-commit-config.yaml

@@ -7,11 +7,14 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.4.0
     hooks:
-      - id: trailing-whitespace
       - id: end-of-file-fixer
+        exclude: ^test/host_test/inputs
+      - id: trailing-whitespace
+        exclude: ^test/host_test/inputs
       - id: check-executables-have-shebangs
       - id: mixed-line-ending
         args: ['-f=lf']
+        exclude: ^test/host_test/inputs
       - id: double-quote-string-fixer
 
   - repo: https://github.com/pycqa/flake8

esp_idf_monitor/base/binlog.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import hashlib
@@ -12,6 +12,7 @@
 from esp_idf_panic_decoder import PanicOutputDecoder
 from serial.tools import miniterm  # noqa: F401
 
+from .binlog import BinaryLog
 from .console_parser import (ConsoleParser, key_description,  # noqa: F401
                              prompt_next_action)
 from .console_reader import ConsoleReader  # noqa: F401
@@ -84,21 +85,35 @@ def __init__(self, last_line_part, serial_check_exit, logger, decode_panic, read
         self.reset = Reset(serial_instance, target)
         self.panic_handler = PanicOutputDecoder(toolchain_prefix, elf_files, target)
         self.disable_auto_color = disable_auto_color
+        self.binlog = BinaryLog(elf_files)
+        self.binary_log_detected = False
 
     def splitdata(self, data):  # type: (bytes) -> List[bytes]
         """
         Split data into lines, while keeping newlines, and move unfinished line for future processing
         """
         # if data is empty fallback to empty string for easier concatenation with last line
-        sp = data.splitlines(keepends=True) or [b'']
-        if self._last_line_part != b'':
-            # add unprocessed part from previous "data" to the first line
-            sp[0] = self._last_line_part + sp[0]
-            self._last_line_part = b''
-        if not sp[-1].endswith(b'\n'):
-            # last part is not a full line
-            self._last_line_part = sp.pop()
-        return sp
+        self.binary_log_detected = (
+            self.binlog.detected(self._last_line_part[0])
+            if self._last_line_part
+            else self.binlog.detected(data[0]) if data
+            else False
+        )
+        if self.binary_log_detected:
+            if self._last_line_part != b'':
+                data = self._last_line_part + data
+                self._last_line_part = b''
+            return [data]
+        else:
+            sp = data.splitlines(keepends=True) or [b'']
+            if self._last_line_part != b'':
+                # add unprocessed part from previous "data" to the first line
+                sp[0] = self._last_line_part + sp[0]
+                self._last_line_part = b''
+            if not sp[-1].endswith(b'\n'):
+                # last part is not a full line
+                self._last_line_part = sp.pop()
+            return sp
 
     def print_colored(self, line: bytes) -> None:
         if self.disable_auto_color:
@@ -148,6 +163,12 @@ def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_m
                 data = data[(pos + 1):]
 
         sp = self.splitdata(data)
+        if self.binary_log_detected:
+            text_lines, self._last_line_part = self.binlog.convert_to_text(sp[0])
+            for line in text_lines:
+                self.print_colored(line)
+            return
+
         for line in sp:
             line_strip = line.strip()
             if self._serial_check_exit and line_strip == EXIT_KEY.encode('latin-1'):

esp_idf_monitor/base/serial_handler.py

@@ -30,6 +30,7 @@ dependencies = [
     "pyserial>=3.3",
     "esp-coredump~=1.2",
     "esp-idf-panic-decoder~=1.2",
+    "pyelftools",
 ]
 
 [project.readme]

pyproject.toml

@@ -0,0 +1,131 @@
+Serial port /dev/ttyUSB0
+Connecting....
+ets Jul 29 2019 12:21:46
+
+rst:0x1 (POWERON_RESET),boot:0x13 (SPI_FAST_FLASH_BOOT)
+configsip: 0, SPIWP:0xee
+clk_drv:0x00,q_drv:0x00,d_drv:0x00,cs0_drv:0x00,hd_drv:0x00,wp_drv:0x00
+mode:DIO, clock div:2
+load:0x3fff0040,len:2524
+ho 0 tail 12 room 4
+load:0x40078000,len:17096
+ho 0 tail 12 room 4
+load:0x40080400,len:3860
+entry 0x4008068c
+I (31) boot: ESP-IDF v5.5-dev-2410-g306d67599fd-dirt 2nd stage bootloader
+I (31) boot: compile time Mar 20 2025 11:15:24
+I (31) boot: Multicore bootloader
+I (32) boot: chip revision: v3.0
+I (32) boot.esp32: SPI Speed      : 40MHz
+I (35) boot.esp32: SPI Mode       : DIO
+I (38) boot.esp32: SPI Flash Size : 2MB
+I (40) boot: Enabling RNG early entropy source...
+I (42) boot: Partition Table:
+I (43) boot: ## Label            Usage          Type ST Offset   Length
+I (45) boot:  0 nvs              WiFi data        01 02 00009000 00006000
+I (49) boot:  1 phy_init         RF data          01 01 0000f000 00001000
+I (54) boot:  2 factory          factory app      00 00 00010000 00100000
+I (60) boot: End of partition table
+I (61) esp_image: segment 0: paddr=00010020 vaddr=3f400020 size=06a2ch ( 27180) map
+I (75) esp_image: segment 1: paddr=00016a54 vaddr=3ff80000 size=0001ch (    28) load
+I (75) esp_image: segment 2: paddr=00016a78 vaddr=3ffb0000 size=02050h (  8272) load
+I (78) esp_image: segment 3: paddr=00018ad0 vaddr=40080000 size=07548h ( 30024) load
+I (90) esp_image: segment 4: paddr=00020020 vaddr=400d0020 size=0ff14h ( 65300) map
+I (113) esp_image: segment 5: paddr=0002ff3c vaddr=40087548 size=05858h ( 22616) load
+I (128) boot: Loaded app from partition at offset 0x10000
+I (129) boot: Disabling RNG early entropy source...
+I (133) cpu_start: Multicore app
+I (141) cpu_start: Pro cpu start user code
+I (141) cpu_start: cpu freq: 160000000 Hz
+I (141) app_init: Application information:
+I (141) app_init: Project name:     log
+I (141) app_init: App version:      v5.5-dev-2410-g306d67599fd-dirt
+I (141) app_init: Compile time:     Mar 20 2025 11:15:15
+I (141) app_init: ELF file SHA256:  5cd3b3293...
+I (142) app_init: ESP-IDF:          v5.5-dev-2410-g306d67599fd-dirt
+I (144) efuse_init: Min chip rev:     v0.0
+I (146) efuse_init: Max chip rev:     v3.99 
+I (148) efuse_init: Chip rev:         v3.0
+I (151) heap_init: Initializing. RAM available for dynamic allocation:
+I (152) heap_init: At 3FFAE6E0 len 00001920 (6 KiB): DRAM
+I (155) heap_init: At 3FFB28A8 len 0002D758 (181 KiB): DRAM
+I (158) heap_init: At 3FFE0440 len 00003AE0 (14 KiB): D/IRAM
+I (160) heap_init: At 3FFE4350 len 0001BCB0 (111 KiB): D/IRAM
+I (163) heap_init: At 4008CDA0 len 00013260 (76 KiB): IRAM
+I (167) spi_flash: detected chip: generic
+I (169) spi_flash: flash io: dio
+W (171) spi_flash: Detected size(4096k) larger than the size in the binary image header(2048k). Using the size in the binary image header.
+I (174) main_task: Started on CPU0
+I (184) main_task: Calling app_main()
+I (184) example: Start
+I (184) example: char_var A 0x41
+I (184) example: var8 240 -120
+I (184) example: var16 65303 -16954 0xff17
+I (184) example: const int var -1024
+I (184) example: const long long unsigned var 2095
+I (184) example: int_var 4095 -4095
+I (184) example: var32 9000 -2000
+I (194) example: var64 10700 -29468
+I (194) example: 64 32 64 vars 10700 9000 10800
+I (194) example: var_size 96843 4
+I (194) example: float var 1.600000
+I (204) example: double var -70.199997
+I (204) example: int, double, var32, float, float, char 4095 -70.199997 9000 1.600000 1.600000 A
+I (204) example: str_array 0x3ffb3a20
+I (214) example: str_ptr 0x3f402b38
+I (214) example: 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
+I (214) example: 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f
+I (214) example: 20 21 21 21 21 21 21 21 21 21 21 21 21 21 21 21
+I (214) example: 30 31 32 33 34 35 36 37 38 39 30 31 32 33 34 35
+I (214) example: 40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f
+I (214) example: 50 51 52 53 54 55 56 57 58 59
+I (224) example: 0x3ffb3a2a   00 01 02 03 04 05 06 07  08 09 0A 0B 0C 0D 0E 0F  |................|
+I (224) example: 0x3ffb3a3a   10 11 12 13 14 15 16 17  18 19 1A 1B 1C 1D 1E 1F  |................|
+I (224) example: 0x3ffb3a4a   20 21 21 21 21 21 21 21  21 21 21 21 21 21 21 21  | !!!!!!!!!!!!!!!|
+I (224) example: 0x3ffb3a5a   30 31 32 33 34 35 36 37  38 39 30 31 32 33 34 35  |0123456789012345|
+I (224) example: 0x3ffb3a6a   40 41 42 43 44 45 46 47  48 49 4A 4B 4C 4D 4E 4F  |@ABCDEFGHIJKLMNO|
+I (224) example: 0x3ffb3a7a   50 51 52 53 54 55 56 57  58 59                    |PQRSTUVWXY|
+I (234) example: Text1:The way to
+I (234) example:  get started is 
+I (234) example: to quit talking 
+I (234) example: and begin doing.
+I (234) example:  - Walt Disney
+I (244) example: Text2:The way to
+I (244) example:  get started is 
+I (244) example: to quit talking 
+I (244) example: and begin doing.
+I (244) example:  - Walt Disney
+I (244) example: buffer2 0x3ffb3ad4
+I (244) example: 0x3ffb3ad4   00 00 00 00 01 00 00 00  02 00 00 00 03 00 00 00  |................|
+I (244) example: 0x3ffb3ae4   04 00 00 00 05 00 00 00  06 00 00 00 07 00 00 00  |................|
+I (244) example: 0x3ffb3af4   08 00 00 00 09 00 00 00  0A 00 00 00 0B 00 00 00  |................|
+I (244) example: 0x3ffb3b04   0C 00 00 00 0D 00 00 00  0E 00 00 00 0F 00 00 00  |................|
+I (244) example: 0x3ffb3b14   10 00 00 00 11 00 00 00  12 00 00 00 13 00 00 00  |................|
+I (254) example: buffer3 0x3ffb3b28
+I (254) example: 0x3ffb3b28   00 00 00 00 00 00 00 00  01 00 00 00 00 00 00 00  |................|
+I (254) example: 0x3ffb3b38   02 00 00 00 00 00 00 00  03 00 00 00 00 00 00 00  |................|
+I (254) example: 0x3ffb3b48   04 00 00 00 00 00 00 00  05 00 00 00 00 00 00 00  |................|
+I (254) example: 0x3ffb3b58   06 00 00 00 00 00 00 00  07 00 00 00 00 00 00 00  |................|
+I (254) example: 0x3ffb3b68   08 00 00 00 00 00 00 00  09 00 00 00 00 00 00 00  |................|
+I (254) example: 0x3ffb3b78   0A 00 00 00 00 00 00 00                           |........|
+I (264) example: Precision and Width Tests
+I (274) example: int width |   42| |00042| |42   |
+I (274) example: int precision |   42| |00042| |00042|
+I (274) example: float width |  3.141590| |3.141590  | |3.141590|
+I (274) example: double width |-123.456789| |-123.456789| |-123.456789|
+I (284) example: string width |ESP32     | |ESP32     | |ESP32 Test|
+I (284) example: longs |1234567890| |123456789012345|
+I (284) example: shorts |ffff| |ff|
+I (294) example: Precision and Width Tests Complete
+I (294) example: Flag and Specifier Tests
+I (294) example: + flag |+42| |-42|
+I (294) example:   flag | 42| |-42|
+I (294) example: # flag |0xff| |0XFF| |0377|
+I (304) example: # flag long |0x499602d2| |011145401322|
+I (304) example: Octal format |377| |0377|
+I (304) example: Octal short |777| |0777|
+I (304) example: Combined Flags |       +42| |        42| |      0x2a| |       052|
+I (314) example: Left-aligned Combined Flags |+42       | | 42       | |0x2a      | |052       |
+I (314) example: Flag and Specifier Tests Complete
+I (314) example: Success
+I (314) main_task: Returned from app_main()

test/host_test/inputs/binlog

@@ -345,6 +345,23 @@ def test_wrong_elf_file(self):
             stderr = f_err.read()
         assert "--- Warning: ELF file 'non_existing.elf' does not exist" in stderr
 
+    def test_binary_logging(self):
+        args = [IN_DIR + '/log.elf', IN_DIR + '/bootloader.elf']
+        out, err = self.run_monitor(args, 'binlog', timeout=10)
+        with open(err, 'r') as f_err:
+            stderr = f_err.read()
+            assert 'Stopping condition has been received' in stderr
+
+        ansi_regex = re.compile(r'\x1B\[\d+(;\d+){0,2}m')
+        with open(out, 'r') as f_out, open(os.path.join(IN_DIR, 'binlog_out.txt'), 'r') as f_expected:
+            for line_out, line_expected in zip(f_out, f_expected):
+                if os.name == 'nt':
+                    line_expected = ansi_regex.sub('', line_expected)
+                    line_out = ansi_regex.sub('', line_out)
+                line_out = line_out.strip()
+                line_expected = line_expected.strip()
+                assert line_out == line_expected, f'Mismatch: {line_out} != {line_expected}'
+
 
 @pytest.mark.skipif(os.name == 'nt', reason='Linux/MacOS only')
 class TestConfig(TestBaseClass):