mount: refactor connection protol

Author: pavelrevak

README_API.md

@@ -96,7 +96,7 @@ conn = mpytool.ConnSocket(address='192.168.1.100:8266')
 
 ### Common Connection Methods
 
-All connection types (including `ConnIntercept` used by mount) share these methods:
+All connection types share these methods:
 
 #### read(timeout=0)
 
@@ -131,7 +131,7 @@ Property indicating if the connection is busy with an internal protocol exchange
 conn.busy  # True during VFS dispatch, False otherwise
 ```
 
-Always `False` on plain `ConnSerial`/`ConnSocket`. On `ConnIntercept` (after mount), `True` while servicing a VFS request from the device.
+Always `False` when no mount is active. After `mount()`, `True` while servicing a VFS request from the device.
 
 #### fd
 
@@ -661,9 +661,8 @@ mpy.mount(local_path, mount_point='/remote', log=None, writable=False, mpy_cross
 The device can then read, import and execute files from the local directory
 without uploading to flash. A MicroPython agent is injected into the device
 that forwards filesystem requests (stat, listdir, open, read, close, and
-optionally write, mkdir, remove) to the PC over the serial link. The connection
-is wrapped in a transparent proxy (`ConnIntercept`) that intercepts VFS protocol
-messages while passing REPL I/O through.
+optionally write, mkdir, remove, rename, seek) to the PC over the serial link.
+VFS protocol messages are intercepted transparently while REPL I/O passes through.
 
 **Write support:** If `writable=True`, the device can create, modify and delete
 files in the mounted directory. All changes are written directly to the local

README_mpremote.md

@@ -126,8 +126,8 @@ Detailed comparison between [mpytool](https://github.com/pavelrevak/mpytool) and
 | VFS SEEK | 🟢 | 🟢 |
 | VFS READLINE | 🟢 | 🟢 |
 | File iteration | 🟢 | 🟢 |
-| Iterative listdir | 🟢 | 🟢 |
-| Agent size | 🟢 4.2KB raw | 🔴 5.5KB compressed |
+| Directory listing | 🟢 batch (1 RTT) | 🔴 iterative (buggy recursion) |
+| Agent size | 🟢 4.3KB raw | 🔴 5.5KB compressed |
 
 ## Summary
 
@@ -140,7 +140,8 @@ Detailed comparison between [mpytool](https://github.com/pavelrevak/mpytool) and
 - Shell completion with remote path support
 - CWD and sys.path tracking across commands
 - No auto soft-reset (preserves device state between commands)
-- Smaller VFS agent (4.2KB vs 5.5KB) — 24% less RAM, faster mount
+- Smaller VFS agent (4.3KB vs 5.5KB) — 22% less RAM, faster mount
+- Batch directory listing (1 RTT) — mpremote has recursion bug with shared state
 - Minimalist design (blocking I/O, simpler code)
 
 **mpremote advantages:**

mpytool/conn.py

@@ -3,6 +3,9 @@
 import time as _time
 import select as _select
 
+# VFS escape byte - signals start of VFS command from device
+ESCAPE = 0x18
+
 
 class ConnError(Exception):
     """General connection error"""
@@ -15,15 +18,36 @@ class Timeout(ConnError):
 class Conn():
     def __init__(self, log=None):
         self._log = log
-        self._buffer = bytearray(b'')
+        self._buffer = bytearray()
+        self._escape_handlers = {}  # {escape_byte: handler}
+        self._active_handler = None  # Currently processing handler
 
     @property
     def fd(self):
         """Return file descriptor for select()"""
         return None
 
+    def register_escape_handler(self, escape, handler):
+        """Register handler for escape byte.
+
+        handler.process(data) returns:
+            None  - need more data
+            b''   - done
+            bytes - done with leftovers
+        """
+        self._escape_handlers[escape] = handler
+
+    def unregister_escape_handler(self, escape):
+        """Unregister handler for escape byte."""
+        self._escape_handlers.pop(escape, None)
+        if self._active_handler is self._escape_handlers.get(escape):
+            self._active_handler = None
+
     def _has_data(self, timeout=0):
         """Check if data is available to read using select()"""
+        if self._active_handler and hasattr(self._active_handler, 'pending'):
+            if self._active_handler.pending:
+                return True
         fd = self.fd
         if fd is None:
             return False
@@ -38,6 +62,62 @@ def _write_raw(self, data):
         """Write data to device, return bytes written (must be implemented by subclass)"""
         raise NotImplementedError
 
+    def _process_data(self, data):
+        """Process incoming data, intercept escape sequences.
+
+        Detects registered escape bytes and routes data to handlers.
+        Returns output bytes (may be empty if all data was for handler).
+
+        Handler protocol:
+            handler.process(data) returns:
+            - None: handler needs more data, keep routing to it
+            - b'': handler done, no leftover data
+            - bytes: handler done, these are leftover bytes
+        """
+        if not data:
+            return data
+
+        output = b''
+
+        while data:
+            if self._active_handler:
+                # Route data to active handler
+                result = self._active_handler.process(data)
+                if result is None:
+                    # Handler needs more data
+                    return output or None
+                # Handler done
+                self._active_handler = None
+                data = result  # May be b'' or leftover bytes
+            else:
+                # Find registered escape byte in data
+                earliest_pos = len(data)
+                earliest_handler = None
+                for esc, handler in self._escape_handlers.items():
+                    try:
+                        pos = data.index(esc)
+                        if pos < earliest_pos:
+                            earliest_pos = pos
+                            earliest_handler = handler
+                    except ValueError:
+                        pass
+
+                if earliest_handler:
+                    output += data[:earliest_pos]
+                    self._active_handler = earliest_handler
+                    data = data[earliest_pos:]  # Include escape, handler verifies
+                else:
+                    output += data
+                    break
+
+        # Check for soft reboot in output (VFS-specific)
+        if output and self._escape_handlers.get(ESCAPE):
+            handler = self._escape_handlers[ESCAPE]
+            if hasattr(handler, 'check_reboot'):
+                handler.check_reboot(output)
+
+        return output or b''
+
     def _read_to_buffer(self, wait_timeout=0):
         """Read available data into buffer
 
@@ -46,6 +126,7 @@ def _read_to_buffer(self, wait_timeout=0):
         """
         if self._has_data(wait_timeout):
             data = self._read_available()
+            data = self._process_data(data)
             if data:
                 self._buffer += data
                 return True
@@ -60,19 +141,20 @@ def flush(self):
     @property
     def busy(self):
         """True if connection is busy with internal protocol exchange"""
-        return False
+        return self._active_handler is not None
 
     def read(self, timeout=0):
         """Read available data from device (non-blocking by default).
 
         Returns device output bytes, or None if no data available.
-        On ConnIntercept, also services VFS requests transparently.
+        When escape handler is active, services requests transparently.
 
         Arguments:
             timeout: how long to wait for data (0 = non-blocking)
         """
         if self._has_data(timeout):
-            return self._read_available()
+            data = self._read_available()
+            return self._process_data(data)
         return None
 
     def read_bytes(self, count, timeout=1):
@@ -88,22 +170,16 @@ def read_bytes(self, count, timeout=1):
                 raise Timeout("No data received")
         data = bytes(self._buffer[:count])
         del self._buffer[:count]
-        if self._log:
-            self._log.debug("rd: %s", data)
         return data
 
     def write(self, data):
         """Write data to device"""
-        if self._log:
-            self._log.debug("wr: %s", bytes(data))
         while data:
             count = self._write_raw(data)
             data = data[count:]
 
     def read_until(self, end, timeout=1):
         """Read until end marker is found"""
-        if self._log:
-            self._log.debug("wait for %s", end)
         start_time = _time.time()
         while True:
             # Use select() with 1ms timeout instead of sleep - wakes immediately on data
@@ -119,8 +195,6 @@ def read_until(self, end, timeout=1):
         index = self._buffer.index(end)
         data = self._buffer[:index]
         del self._buffer[:index + len(end)]
-        if self._log:
-            self._log.debug("rd: %s", bytes(data + end))
         return data
 
     def read_line(self, timeout=None):

mpytool/conn_serial.py

@@ -55,7 +55,10 @@ def _read_available(self):
         try:
             in_waiting = self._serial.in_waiting
             if in_waiting > 0:
-                return self._serial.read(in_waiting)
+                data = self._serial.read(in_waiting)
+                if data and self._log:
+                    self._log.debug("RX: %r", data)
+                return data
             return None
         except OSError as err:
             raise _conn.ConnError(f"Connection lost: {err}") from err
@@ -64,6 +67,8 @@ def _write_raw(self, data):
         """Write data to serial port"""
         if self._serial is None:
             raise _conn.ConnError("Not connected")
+        if self._log:
+            self._log.debug("TX: %r", data)
         try:
             return self._serial.write(data)
         except OSError as err:

mpytool/conn_socket.py

@@ -49,6 +49,8 @@ def _read_available(self):
         try:
             data = self._socket.recv(4096)
             if data:
+                if self._log:
+                    self._log.debug("RX: %r", data)
                 return data
         except BlockingIOError:
             pass
@@ -60,6 +62,8 @@ def _write_raw(self, data):
         """Write data to socket"""
         if self._socket is None:
             raise _conn.ConnError("Not connected")
+        if self._log:
+            self._log.debug("TX: %r", data)
         try:
             return self._socket.send(data)
         except OSError as err: