Merge remote-tracking branch 'origin/main' into claude/compassionate-nash-116395

Author: bwhitman

tools/amyboardctl/amyboardctl/link.py

@@ -52,6 +52,7 @@
 TAG_ACK = 0x41       # 'A' (followed by 'K')
 TAG_OK = 0x4F        # 'O' (followed by 'K') — zI pong
 TAG_ERROR = 0x58     # 'X'
+TAG_OVERLOAD = 0x4C  # 'L' — AMY overload failsafe tripped (b64 load percent)
 TAG_VERSION = 0x56   # 'V'
 TAG_DUMP_ONE = 0x30  # '0' single-frame dump
 TAG_DUMP_MID = 0x43  # 'C' chunk, more follow
@@ -83,6 +84,7 @@ def __init__(self, port_match=DEFAULT_PORT_MATCH, verbose=True, backend="auto",
         self._dump = bytearray()
         self._dump_done = threading.Event()
         self.errors = []      # decoded sketch-error tracebacks since last clear
+        self.overloads = []   # load percents from 'L' overload-failsafe frames
         self.version = None
         self.in_name = None
         self.out_name = None
@@ -126,6 +128,18 @@ def _on_sysex(self, data):
                 text = "<undecodable sketch error frame>"
             self.errors.append(text)
             self._log("  ⚠ sketch error frame received (%d bytes)" % len(text))
+        elif tag == TAG_OVERLOAD:
+            # amyboard.py's _on_amy_overload: AMY's CPU failsafe tripped on the
+            # board (synth reset + bleep already happened, sketch stopped).
+            # Always loud — a run that hits this has stopped making the sound
+            # under test, whatever the harness thinks it's recording.
+            try:
+                pct = base64.b64decode(bytes(data[4:])).decode("ascii", "replace").strip()
+            except Exception:
+                pct = "?"
+            self.overloads.append(pct)
+            print("[amyboardctl] ⚠ AMY OVERLOAD failsafe tripped on the board "
+                  "(load %s%%) — sketch stopped" % pct, file=sys.stderr)
         elif tag == TAG_VERSION:
             self.version = bytes(data[4:]).decode("ascii", "replace").strip()
             self._version_evt.set()

tools/amyboardctl/amyboardctl/transport.py

@@ -157,6 +157,7 @@ def __init__(self, port=DEFAULT_PORT_MATCH):
         self.ack_available = False
         self.in_name = self.out_name = None
         self._reader = None
+        self._closing = False
 
     def open(self):
         self.port = self.port_match if self.port_match.startswith("hw:") \
@@ -165,21 +166,25 @@ def open(self):
             raise RuntimeError("Could not find an AMYboard ALSA MIDI device "
                                "(match=%r); see `amidi -l`." % self.port_match)
         self.in_name = self.out_name = self.port
-        # Persistent receive stream: line-buffered so frames arrive in real time
-        # (block buffering would sit on the board's ACKs). Coexists fine with the
-        # one-shot `amidi -S` sends on the same rawmidi device.
         try:
-            self._reader = subprocess.Popen(
-                ["stdbuf", "-oL", "amidi", "-p", self.port, "-d"],
-                stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
-            threading.Thread(target=self._read_loop, daemon=True).start()
-            self.ack_available = True
+            self._spawn_reader()
         except Exception as e:
             print("[amyboardctl] no amidi read stream (%s); send-only" % e,
                   file=sys.stderr)
         return self
 
+    def _spawn_reader(self):
+        # Persistent receive stream: line-buffered so frames arrive in real time
+        # (block buffering would sit on the board's ACKs). Coexists fine with the
+        # one-shot `amidi -S` sends on the same rawmidi device.
+        self._reader = subprocess.Popen(
+            ["stdbuf", "-oL", "amidi", "-p", self.port, "-d"],
+            stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
+        threading.Thread(target=self._read_loop, daemon=True).start()
+        self.ack_available = True
+
     def close(self):
+        self._closing = True
         if self._reader:
             try:
                 self._reader.terminate()
@@ -191,9 +196,10 @@ def close(self):
     def _read_loop(self):
         """Parse the `amidi -d` hex dump into SysEx frames and dispatch each
         frame's payload (the bytes between F0 and F7) to on_sysex."""
+        reader = self._reader
         buf, insx = [], False
         try:
-            for line in self._reader.stdout:
+            for line in reader.stdout:
                 for tok in line.split():
                     try:
                         b = int(tok, 16)
@@ -209,6 +215,33 @@ def _read_loop(self):
                         buf.append(b)
         except Exception:
             pass
+        # `amidi -d` dies when the board re-enumerates (its rawmidi node goes
+        # away). That used to be silent — every later send_and_ack just timed
+        # out and transfers aborted with no trace. Respawn it once the device
+        # is back; if it never comes back, drop to send-only pacing and say so.
+        if self._closing:
+            return
+        print("[amyboardctl] amidi read stream died (rc=%s); respawning"
+              % reader.poll(), file=sys.stderr)
+        self.ack_available = False
+        deadline = time.time() + 15.0
+        while not self._closing and time.time() < deadline:
+            time.sleep(1.0)
+            port = self.port_match if self.port_match.startswith("hw:") \
+                else _alsa_find(self.port_match)
+            if not port:
+                continue
+            self.port = self.in_name = self.out_name = port
+            try:
+                self._spawn_reader()
+                print("[amyboardctl] amidi read stream back on %s" % port,
+                      file=sys.stderr)
+                return
+            except Exception:
+                continue
+        if not self._closing:
+            print("[amyboardctl] amidi read stream gone for good; "
+                  "continuing send-only (paced)", file=sys.stderr)
 
     def send(self, data):
         cmd = ["amidi", "-p", self.port, "-S", " ".join("%02X" % b for b in data)]

tulip/amyboard/hwci/hwci.py

@@ -290,7 +290,20 @@ def load_world_sketch(self, text, settle=1.5):
                 pass
             self.link.reset_amy()
             time.sleep(0.3)
-            self.link.transfer_file(text)
+            # An unACKed transfer means the sketch may not actually be on the
+            # board — recording it anyway just measures 10s of silence with a
+            # clean log (the 2026-07-08 woodpiano incident). Retry once, then
+            # fail this sketch loudly instead of recording nothing.
+            for attempt in (1, 2):
+                sent, acked = self.link.transfer_file(text)
+                if acked == sent:
+                    break
+                print(f"[world] TRANSFER NOT ACKED ({acked}/{sent} frames), "
+                      f"attempt {attempt}/2")
+                if attempt == 2:
+                    raise RuntimeError(
+                        f"sketch transfer failed: {acked}/{sent} frames acked")
+                time.sleep(1.0)
             self.link.environment_transfer_done()
             time.sleep(settle)
         finally:
@@ -365,14 +378,9 @@ def __init__(self, port, baud, label):
     def start(self):
         self._t.start()
 
-    def _run(self):
-        try:
-            import serial  # pyserial (ships with esptool)
-        except ImportError:
-            print(f"[log] pyserial missing; cannot capture {self.port}")
-            return
+    def _open(self, serial, window_s):
         ser = None
-        deadline = time.monotonic() + 8.0
+        deadline = time.monotonic() + window_s
         while ser is None and not self._stop.is_set() and time.monotonic() < deadline:
             try:
                 ser = serial.Serial()
@@ -385,21 +393,53 @@ def _run(self):
             except Exception:
                 ser = None
                 time.sleep(0.3)
-        if ser is None:
-            print(f"[log] could not open {self.port} ({self.label})")
+        return ser
+
+    def _run(self):
+        try:
+            import serial  # pyserial (ships with esptool)
+        except ImportError:
+            print(f"[log] pyserial missing; cannot capture {self.port}")
             return
-        self.opened = True
+        first = True
         while not self._stop.is_set():
+            # A read error mid-run usually means the board re-enumerated its
+            # USB (reboot / self-heal). Dying silently here truncated the log
+            # with no trace (the 2026-07-08 woodpiano incident) — instead say
+            # so, mark the log, and reopen so the post-reboot console (boot
+            # log, tracebacks) is captured too. 15s covers a re-enumeration.
+            ser = self._open(serial, 8.0 if first else 15.0)
+            if ser is None:
+                if self._stop.is_set():
+                    return
+                if first:
+                    print(f"[log] could not open {self.port} ({self.label})")
+                else:
+                    print(f"[log] {self.port} ({self.label}) did not come back; "
+                          "capture stops here")
+                    self.buf.extend(b"\n===== [log] port did not come back; "
+                                    b"capture truncated =====\n")
+                return
+            self.opened = True
+            if not first:
+                print(f"[log] {self.port} ({self.label}) reopened")
+                self.buf.extend(b"\n===== [log] reopened after read error =====\n")
+            first = False
+            while not self._stop.is_set():
+                try:
+                    data = ser.read(4096)
+                except Exception as e:
+                    print(f"[log] {self.port} ({self.label}) read error: {e}; "
+                          "reopening")
+                    self.buf.extend(f"\n===== [log] read error: {e}; "
+                                    "reopening =====\n".encode())
+                    break
+                if data:
+                    self.buf.extend(data)
             try:
-                data = ser.read(4096)
+                ser.close()
             except Exception:
-                break
-            if data:
-                self.buf.extend(data)
-        try:
-            ser.close()
-        except Exception:
-            pass
+                pass
 
     def stop(self):
         self._stop.set()