ble stability

Author: HamzaYslmn

README.md

@@ -184,6 +184,26 @@ with espbridge.connect_all() as boards:    # or just open all of them
     boards.by_name("relays").gpio.write(2, 1)
 ```
 
+## Troubleshooting
+
+Errors name the command and say what to check (`I2C_WRITE (0x4003) failed:
+IO — no ACK on the wire — check wiring, power, device address and pull-ups`).
+A timeout additionally pings the board so the message tells you whether the
+link itself died or a single frame got lost. Useful knobs:
+
+```python
+esp = Bridge(retries=1)         # default: re-send safe commands once on timeout
+esp.free_heap()                 # heap + dropped-frame counters from the firmware
+```
+
+```bash
+ESPBRIDGE_DEBUG=1 python app.py   # trace every request/response with names
+```
+
+Lost frames on a busy link are also *prevented* now: pipelined bursts
+(OLED frames, NeoPixel updates) are automatically throttled to what the
+firmware's link buffer can absorb, on both USB serial and Bluetooth.
+
 ## Repo layout
 
 | path | what |

docs/PROTOCOL.md

@@ -101,6 +101,24 @@ compiled in via `BRIDGE_PASSWORD` at the top of `firmware.ino`; empty string
 `SYS_READY` banner to that client, so the handshake proceeds exactly like
 USB. Auth state is per-connection and resets on disconnect.
 
+## Link flow control (fire-and-forget bursts)
+
+Fire-and-forget requests (`seq = 0`) get no reply, so nothing paces them. The
+firmware buffers inbound bytes (UART ring `SERIAL_RX_BUF` = 4096 B; BLE link
+stream buffer `LINK_RX_BUF` = 6144 B) and drains them only as fast as handlers
+execute — a pipelined burst (an OLED frame push is ~9 KB of `I2C_WRITE`s, each
+1 KB page taking ~23 ms on a 400 kHz bus) can overrun the buffer, and overrun
+bytes are **dropped** (CRC then discards the mangled frames; the host-visible
+symptom is a timeout on the next *waited* request).
+
+Hosts must therefore cap un-acknowledged fire-and-forget bytes and insert any
+cheap request (`SYS_PING`) as a **fence**: frames are consumed in arrival
+order, so its reply proves the firmware drained every byte sent before it.
+The Python library does this automatically (transport `burst_window`:
+3072 serial / 4096 BLE). The firmware counts BLE-side overflow drops and
+reports them in `SYS_FREE_HEAP` (5th u32, fw ≥ 0.3.2) alongside a
+rate-limited `SYS_LOG` warning — drops are never silent.
+
 ## NET flow control (socket proxy)
 
 The serial link is slower than Wi-Fi, so each proxied TCP socket has a credit

examples/uv.lock

@@ -76,9 +76,12 @@ void setup() {
   uint8_t info[64];
   uint16_t n = sys_build_info(info);
   proto_send_event(SYS_READY, info, n);
-}
 
-void loop() {
-  // Everything runs in dedicated FreeRTOS tasks; park the Arduino loop task.
-  vTaskDelay(portMAX_DELAY);
+  // Everything runs in dedicated FreeRTOS tasks — delete the Arduino loop
+  // task instead of parking it: its 8 KB stack goes back to the heap, which
+  // a classic ESP32 running Wi-Fi + Bluedroid needs badly (Bluedroid stops
+  // delivering notifications below ~8 KB free, killing the BLE link).
+  vTaskDelete(nullptr);  // never returns
 }
+
+void loop() {}  // never runs (loopTask deleted at end of setup)

firmware/firmware.ino

@@ -6,7 +6,7 @@
 #define PROTOCOL_VERSION 1
 #define FW_VERSION_MAJOR 0
 #define FW_VERSION_MINOR 3
-#define FW_VERSION_PATCH 1
+#define FW_VERSION_PATCH 2
 
 // Frame (logical, pre-COBS):
 //   flags u8 | seq u8 | cmd u16 BE | payload .. | crc16 BE

firmware/src/espbridge/commands.h

@@ -35,6 +35,7 @@ void bt_prepare_ble_only();
 bool link_ble_enabled();    // link_ble_init() succeeded
 bool link_ble_connected();  // a BLE central is currently connected
 bool link_ble_authed();     // ...and presented the correct password
+uint32_t link_ble_rx_dropped();  // bytes lost to RX buffer overflow (diagnostics)
 void link_ble_set_authed(bool v);
 const char* link_ble_password();
 

firmware/src/espbridge/link.h

@@ -223,8 +223,36 @@ struct RxState {
 static RxState rxstate[2];  // [LINK_USB], [LINK_BLE]
 static uint8_t rxframe[MAX_FRAME];
 
+// Wedge watchdog: some IDF calls can block rx_task forever with no panic
+// and no error (seen: i2c driver install at exhausted heap). tx_task stays
+// alive, so a watcher can still tell the host WHERE the firmware is stuck
+// instead of leaving it guessing at timeouts.
+static volatile uint16_t cur_cmd = 0xFFFF;  // command rx_task is inside, if any
+static volatile uint32_t disp_n = 0;
+
+static void watch_task(void*) {
+  uint32_t last_n = 0;
+  uint8_t stuck = 0;
+  for (;;) {
+    vTaskDelay(pdMS_TO_TICKS(1000));
+    uint16_t c = cur_cmd;
+    if (c != 0xFFFF && disp_n == last_n) {
+      if (++stuck >= 2 && (stuck & 1) == 0) {  // after 2 s, then every 2 s
+        char msg[56];
+        snprintf(msg, sizeof(msg), "rx_task stuck in cmd 0x%04X for %u s", c, stuck);
+        proto_log(2, msg);
+      }
+    } else {
+      stuck = 0;
+    }
+    last_n = disp_n;
+  }
+}
+
 static void dispatch(uint8_t seq, uint16_t cmd, const uint8_t* p, uint16_t len) {
   uint8_t mod = cmd >> 8, op = cmd & 0xFF;
+  cur_cmd = cmd;
+  disp_n++;
   switch (mod) {
     // Fast handlers: run inline on rx_task.
     case MOD_SYS:   sys_handle(op, seq, p, len); break;
@@ -253,6 +281,7 @@ static void dispatch(uint8_t seq, uint16_t cmd, const uint8_t* p, uint16_t len)
     case MOD_CAM:   net_enqueue(cmd, seq, p, len); break;
     default:        proto_reply_err(seq, cmd, ST_UNKNOWN_CMD); break;
   }
+  cur_cmd = 0xFFFF;
 }
 
 // Constant-time-ish password compare (no early exit on mismatch).
@@ -366,4 +395,6 @@ void proto_start() {
   xTaskCreatePinnedToCore(tx_task, "bridge_tx", 4096, nullptr, 12, nullptr, BRIDGE_CORE);
   xTaskCreatePinnedToCore(rx_task, "bridge_rx", 8192, nullptr, 10, nullptr, BRIDGE_CORE);
   xTaskCreatePinnedToCore(net_task, "bridge_net", 8192, nullptr, 9, nullptr, BRIDGE_CORE);
+  // Above rx_task so a wedged handler can't starve it; sleeps 1 s between looks.
+  xTaskCreatePinnedToCore(watch_task, "bridge_watch", 2048, nullptr, 11, nullptr, BRIDGE_CORE);
 }

firmware/src/espbridge/protocol.cpp

@@ -53,6 +53,7 @@ static const char* link_password = "";
 static BLEServer* server = nullptr;
 static BLECharacteristic* tx_chr = nullptr;
 static StreamBufferHandle_t rx_buf = nullptr;
+static volatile uint32_t rx_dropped = 0;  // bytes lost to RX buffer overflow
 
 // Hands-off link policy (lesson from Esp-WiFi-BLE-Now, which runs all three
 // radios stably): the peripheral issues NO link-layer control procedures —
@@ -89,7 +90,21 @@ class LinkRxCb : public BLECharacteristicCallbacks {
     // Bluedroid callbacks run on a BT task: plain (non-ISR) send is fine.
     // Allow a short block so pipelined bursts backpressure the BT task
     // instead of dropping bytes; past that the host times out & retries.
-    xStreamBufferSend(rx_buf, c->getData(), c->getLength(), pdMS_TO_TICKS(50));
+    size_t n = xStreamBufferSend(rx_buf, c->getData(), c->getLength(),
+                                 pdMS_TO_TICKS(50));
+    if (n < c->getLength()) {
+      // Never drop silently — corrupted frames look like random timeouts on
+      // the host. Count (SYS_FREE_HEAP reports it) and warn, rate-limited.
+      rx_dropped += c->getLength() - n;
+      static uint32_t last_warn = 0;
+      uint32_t now = millis();
+      if (now - last_warn > 1000) {
+        last_warn = now;
+        proto_log(2, "ble: rx overflow — frames lost (host writing faster "
+                     "than commands execute; update python-esp-bridge for "
+                     "burst throttling)");
+      }
+    }
   }
 };
 static LinkRxCb link_rx_cb;
@@ -164,6 +179,7 @@ void link_ble_init(const char* password) {
 
 bool link_ble_enabled() { return enabled; }
 bool link_ble_connected() { return connected; }
+uint32_t link_ble_rx_dropped() { return rx_dropped; }
 bool link_ble_authed() { return connected && authed; }
 void link_ble_set_authed(bool v) { authed = v; }
 const char* link_ble_password() { return link_password; }
@@ -203,6 +219,7 @@ void link_ble_init(const char*) {}
 void bt_prepare_ble_only() {}
 bool link_ble_enabled() { return false; }
 bool link_ble_connected() { return false; }
+uint32_t link_ble_rx_dropped() { return 0; }
 bool link_ble_authed() { return false; }
 void link_ble_set_authed(bool) {}
 const char* link_ble_password() { return ""; }

firmware/src/link_ble.cpp

@@ -2,6 +2,7 @@
 #include "espbridge/protocol.h"
 #include "espbridge/modules.h"
 #include <Wire.h>
+#include <esp_heap_caps.h>
 
 static bool i2c_inited[2];
 
@@ -32,11 +33,28 @@ void i2c_handle(uint8_t op, uint8_t seq, const uint8_t* p, uint16_t len) {
       // BLE link connected runs within a few KB of — so fall back to
       // smaller buffers instead of failing, and report the size that stuck
       // (hosts chunk their writes to it).
-      static const uint16_t sizes[] = {MAX_PAYLOAD, 512, 128};
+      //
+      // The fallback must be heap-aware, not try-and-see (all measured on
+      // classic ESP32 + BLE link, core 3.3.6):
+      //  - the IDF i2c driver install after the buffer allocation eats
+      //    ~2 KB and WEDGES rx_task silently below ~7 KB free (no panic,
+      //    no error return — the board just stops answering);
+      //  - the radio stacks need ~6.5 KB free *for the rest of the
+      //    session*, or Bluedroid starts dropping replies under load
+      //    (6.7 KB rest = solid; 5.9 KB rest = host-visible timeouts).
+      // So a buffer above the 128-byte floor must leave 2*size + ~9 KB,
+      // plus a contiguous chunk to spare.
+      static const uint16_t sizes[] = {MAX_PAYLOAD, 1024, 512, 256, 128};
       uint16_t got = 0;
-      for (uint8_t i = 0; i < 3 && !got; i++)
+      for (uint8_t i = 0; i < 5 && !got; i++) {
+        if (sizes[i] > 128) {
+          if (ESP.getFreeHeap() < 2u * sizes[i] + 9216) continue;
+          if (heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)
+              < (size_t)sizes[i] + 2048) continue;
+        }
         if (w->setBufferSize(sizes[i]) && w->begin(p[1], p[2], rd32(p + 3)))
           got = sizes[i];
+      }
       if (!got) {
         proto_log_heap("i2c: init failed");  // ST_IO alone is opaque
         proto_reply_err(seq, cmd, ST_IO);
@@ -67,12 +85,29 @@ void i2c_handle(uint8_t op, uint8_t seq, const uint8_t* p, uint16_t len) {
       w->beginTransmission(p[1]);
       // write() returns bytes buffered — short means TX buffer overflow,
       // which would silently corrupt the transfer (drops the tail bytes)
+      uint8_t st = ST_OK;
       if (len > 2 && w->write(p + 2, len - 2) != (size_t)(len - 2)) {
         w->endTransmission();
-        proto_reply_err(seq, cmd, ST_BAD_ARGS);
+        st = ST_BAD_ARGS;
+      } else if (w->endTransmission() != 0) {
+        st = ST_IO;
+      }
+      if (st != ST_OK) {
+        // Fire-and-forget (seq 0) gets no error reply, so a failed write
+        // in a pipelined burst (OLED frame push) would vanish without a
+        // trace — visible only as display corruption. Warn, rate-limited.
+        if (seq == 0) {
+          static uint32_t last_warn = 0;
+          uint32_t now = millis();
+          if (now - last_warn > 1000) {
+            last_warn = now;
+            proto_log(2, st == ST_IO ? "i2c: unacked write failed (NACK/bus)"
+                                     : "i2c: unacked write exceeds wire buffer");
+          }
+        }
+        proto_reply_err(seq, cmd, st);
         return;
       }
-      if (w->endTransmission() != 0) { proto_reply_err(seq, cmd, ST_IO); return; }
       proto_reply_ok(seq, cmd);
       break;
     }

firmware/src/mod_i2c.cpp

@@ -211,13 +211,14 @@ void sys_handle(uint8_t op, uint8_t seq, const uint8_t* p, uint16_t len) {
       break;
 #endif
 
-    case 0x05: {  // FREE_HEAP
-      uint8_t buf[16];
+    case 0x05: {  // FREE_HEAP -> free, min_free, largest, dropped_evts, rx_dropped
+      uint8_t buf[20];
       wr32(buf, ESP.getFreeHeap());
       wr32(buf + 4, ESP.getMinFreeHeap());
       wr32(buf + 8, heap_caps_get_largest_free_block(MALLOC_CAP_8BIT));
       wr32(buf + 12, proto_dropped_events());
-      proto_reply(seq, cmd, buf, 16);
+      wr32(buf + 16, link_ble_rx_dropped());  // fw >= 0.3.2 (host len-checks)
+      proto_reply(seq, cmd, buf, 20);
       break;
     }