fix(biometrics): bypass cbor2 C extension that silently skips records

The cbor2 C extension (_cbor2) reads files in internal 4096-byte chunks,
advancing f.tell() by 4096 regardless of actual record size. Since RAW
records are 17-5000 bytes, cbor2.load(f) silently skips most records.
On Pod 5, piezo-dual records are ~2700 bytes so nearly every other record
was lost, severely degrading vitals accuracy.

Additionally, Pod 5 firmware writes empty placeholder records (data=b'')
as sequence markers. cbor2.loads(b'') raises CBORDecodeEOF (a subclass of
EOFError), which the read loop caught as end-of-file, terminating reads
mid-file with valid data remaining.

Fix: Replace cbor2.load(f) with _read_raw_record(f) that manually parses
the outer {seq, data} CBOR wrapper byte-by-byte, keeping f.tell() accurate.
Empty placeholders return None and are skipped. On read errors, seek back
to last known good position instead of breaking the loop.

Applied to both piezo-processor and sleep-detector modules.

Ref: throwaway31265/free-sleep#46

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: ng

modules/piezo-processor/main.py

@@ -27,6 +27,8 @@
 from collections import deque
 from typing import Optional
 
+import struct
+
 import cbor2
 import numpy as np
 from scipy.signal import butter, filtfilt, welch
@@ -172,6 +174,78 @@ def compute_breathing_rate(samples: np.ndarray, fs: float = SAMPLE_RATE) -> Opti
         log.debug("Breathing rate computation failed: %s", e)
         return None
 
+# ---------------------------------------------------------------------------
+# CBOR record reader
+# ---------------------------------------------------------------------------
+
+def _read_raw_record(f):
+    """
+    Manually parse one outer {seq, data} CBOR record using f.read().
+
+    The cbor2 C extension (_cbor2) reads files in internal 4096-byte chunks,
+    so cbor2.load(f) advances f.tell() by 4096 bytes regardless of the actual
+    record size. Since RAW file records are typically 17-5000 bytes, this causes
+    nearly every record to be skipped silently.
+
+    This function parses the outer {seq: uint, data: bytes} wrapper byte-by-byte
+    using f.read(), keeping f.tell() accurate after each record.
+
+    Returns the raw inner data bytes, or None for empty placeholder records
+    (which the Pod firmware writes as sequence number markers with data=b'').
+    Raises EOFError at end of file, ValueError on malformed data.
+    """
+    b = f.read(1)
+    if not b:
+        raise EOFError
+    if b[0] != 0xa2:
+        raise ValueError('Expected outer map 0xa2, got 0x%02x' % b[0])
+    if f.read(4) != b'\x63\x73\x65\x71':
+        raise ValueError('Expected seq key')
+    hdr = f.read(1)
+    if not hdr:
+        raise EOFError
+    if hdr[0] == 0x1a:
+        seq_bytes = f.read(4)
+        if len(seq_bytes) < 4:
+            raise EOFError
+    elif hdr[0] == 0x1b:
+        seq_bytes = f.read(8)
+        if len(seq_bytes) < 8:
+            raise EOFError
+    else:
+        raise ValueError('Unexpected seq encoding: 0x%02x' % hdr[0])
+    if f.read(5) != b'\x64\x64\x61\x74\x61':
+        raise ValueError('Expected data key')
+    bs = f.read(1)
+    if not bs:
+        raise EOFError
+    ai = bs[0] & 0x1f
+    if ai <= 23:
+        length = ai
+    elif ai == 24:
+        lb = f.read(1)
+        if not lb:
+            raise EOFError
+        length = lb[0]
+    elif ai == 25:
+        lb = f.read(2)
+        if len(lb) < 2:
+            raise EOFError
+        length = struct.unpack('>H', lb)[0]
+    elif ai == 26:
+        lb = f.read(4)
+        if len(lb) < 4:
+            raise EOFError
+        length = struct.unpack('>I', lb)[0]
+    else:
+        raise ValueError('Unsupported length encoding: %d' % ai)
+    data = f.read(length)
+    if len(data) < length:
+        raise EOFError
+    if not data:
+        return None  # empty placeholder record, caller should skip
+    return data
+
 # ---------------------------------------------------------------------------
 # RAW file follower
 # ---------------------------------------------------------------------------
@@ -186,6 +260,7 @@ def __init__(self, data_dir: Path):
         self.data_dir = data_dir
         self._file = None
         self._path = None
+        self._last_pos = 0
 
     def _find_latest(self) -> Optional[Path]:
         candidates = sorted(self.data_dir.glob("*.RAW"), key=lambda p: p.stat().st_mtime, reverse=True)
@@ -205,16 +280,22 @@ def read_records(self):
                     self._file.close()
                 self._file = open(latest, "rb")
                 self._path = latest
+                self._last_pos = 0
 
             try:
-                record = cbor2.load(self._file)
-                inner = cbor2.loads(record["data"])
+                data_bytes = _read_raw_record(self._file)
+                if data_bytes is None:
+                    self._last_pos = self._file.tell()
+                    continue  # empty placeholder record
+                inner = cbor2.loads(data_bytes)
+                self._last_pos = self._file.tell()
                 yield inner
             except EOFError:
                 # No new data yet — poll
                 time.sleep(0.01)
             except Exception as e:
                 log.warning("Error reading RAW record: %s", e)
+                self._file.seek(self._last_pos)
                 time.sleep(1)
 
 # ---------------------------------------------------------------------------

modules/sleep-detector/main.py

@@ -29,6 +29,8 @@
 from dataclasses import dataclass, field
 from typing import Optional
 
+import struct
+
 import cbor2
 import numpy as np
 
@@ -255,6 +257,78 @@ def _flush_movement(self, ts: float) -> None:
             self._movement_buf = []
         self._last_movement_write = ts
 
+# ---------------------------------------------------------------------------
+# CBOR record reader
+# ---------------------------------------------------------------------------
+
+def _read_raw_record(f):
+    """
+    Manually parse one outer {seq, data} CBOR record using f.read().
+
+    The cbor2 C extension (_cbor2) reads files in internal 4096-byte chunks,
+    so cbor2.load(f) advances f.tell() by 4096 bytes regardless of the actual
+    record size. Since RAW file records are typically 17-5000 bytes, this causes
+    nearly every record to be skipped silently.
+
+    This function parses the outer {seq: uint, data: bytes} wrapper byte-by-byte
+    using f.read(), keeping f.tell() accurate after each record.
+
+    Returns the raw inner data bytes, or None for empty placeholder records
+    (which the Pod firmware writes as sequence number markers with data=b'').
+    Raises EOFError at end of file, ValueError on malformed data.
+    """
+    b = f.read(1)
+    if not b:
+        raise EOFError
+    if b[0] != 0xa2:
+        raise ValueError('Expected outer map 0xa2, got 0x%02x' % b[0])
+    if f.read(4) != b'\x63\x73\x65\x71':
+        raise ValueError('Expected seq key')
+    hdr = f.read(1)
+    if not hdr:
+        raise EOFError
+    if hdr[0] == 0x1a:
+        seq_bytes = f.read(4)
+        if len(seq_bytes) < 4:
+            raise EOFError
+    elif hdr[0] == 0x1b:
+        seq_bytes = f.read(8)
+        if len(seq_bytes) < 8:
+            raise EOFError
+    else:
+        raise ValueError('Unexpected seq encoding: 0x%02x' % hdr[0])
+    if f.read(5) != b'\x64\x64\x61\x74\x61':
+        raise ValueError('Expected data key')
+    bs = f.read(1)
+    if not bs:
+        raise EOFError
+    ai = bs[0] & 0x1f
+    if ai <= 23:
+        length = ai
+    elif ai == 24:
+        lb = f.read(1)
+        if not lb:
+            raise EOFError
+        length = lb[0]
+    elif ai == 25:
+        lb = f.read(2)
+        if len(lb) < 2:
+            raise EOFError
+        length = struct.unpack('>H', lb)[0]
+    elif ai == 26:
+        lb = f.read(4)
+        if len(lb) < 4:
+            raise EOFError
+        length = struct.unpack('>I', lb)[0]
+    else:
+        raise ValueError('Unsupported length encoding: %d' % ai)
+    data = f.read(length)
+    if len(data) < length:
+        raise EOFError
+    if not data:
+        return None  # empty placeholder record, caller should skip
+    return data
+
 # ---------------------------------------------------------------------------
 # RAW file follower (same pattern as piezo-processor)
 # ---------------------------------------------------------------------------
@@ -264,6 +338,7 @@ def __init__(self, data_dir: Path):
         self.data_dir = data_dir
         self._file = None
         self._path = None
+        self._last_pos = 0
 
     def _find_latest(self):
         candidates = sorted(self.data_dir.glob("*.RAW"), key=lambda p: p.stat().st_mtime, reverse=True)
@@ -282,15 +357,21 @@ def read_records(self):
                     self._file.close()
                 self._file = open(latest, "rb")
                 self._path = latest
+                self._last_pos = 0
 
             try:
-                record = cbor2.load(self._file)
-                inner = cbor2.loads(record["data"])
+                data_bytes = _read_raw_record(self._file)
+                if data_bytes is None:
+                    self._last_pos = self._file.tell()
+                    continue  # empty placeholder record
+                inner = cbor2.loads(data_bytes)
+                self._last_pos = self._file.tell()
                 yield inner
             except EOFError:
                 time.sleep(0.5)
             except Exception as e:
                 log.warning("Error reading RAW record: %s", e)
+                self._file.seek(self._last_pos)
                 time.sleep(1)
 
         # Clean up file handle on shutdown