comma-sync.sh

#!/usr/bin/env bash
#
# comma-sync.sh — pull new dashcam footage off a comma/openpilot device and
#                 stitch each drive into playable MP4s.
#
# What it does, end to end:
#   1. rsync ONLY the new raw *.hevc segments from the device into:
#         Comma Footage/Raw HEVC Chunks/      (visible, so you can browse them)
#      Already-processed drives are skipped and never re-downloaded.
#   2. Groups segments by "drive" (route) and concatenates them in order.
#   3. Produces one playable .mp4 per camera per drive, named by recording start
#      time (so they sort chronologically), each in its own folder. Microphone
#      audio (stored in qcamera.ts) is muxed in when it was recorded:
#         Comma Footage/2026-06-19_23-59-31/2026-06-19_23-59-31__road.mp4
#         Comma Footage/2026-06-19_23-59-31/2026-06-19_23-59-31__wide.mp4
#         Comma Footage/2026-06-19_23-59-31/2026-06-19_23-59-31__driver.mp4
#   4. Records the drive as done. The raw chunks are KEPT in "Raw HEVC Chunks"
#      so you can clean them up yourself whenever you like — deleting them will
#      NOT cause a re-download (the ledger tracks what's processed).
#
# To purge processed raw chunks automatically, run:  CLEAN_RAW=1 ./comma-sync.sh
# Or just delete the "Raw HEVC Chunks" folder by hand anytime.
#
# Safe to run as often as you like. Just run:  ./comma-sync.sh
#
set -euo pipefail

# ---- config -----------------------------------------------------------------
COMMA_IP="${COMMA_IP:-}"                        # optional fixed IP. Leave EMPTY to always
                                               # auto-discover — the device's DHCP IP usually
                                               # changes on every connect, so a fixed guess
                                               # just wastes time.
AUTO_DISCOVER="${AUTO_DISCOVER:-1}"            # 1 = scan the network to find the device
                                               # 0 = only use COMMA_IP (lock to one address)
REMOTE_PORT="${REMOTE_PORT:-22}"               # comma SSH port (older devices use 8022)
SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519}"    # private key whose .pub is on your GitHub
ROOT="${ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/Comma Footage}"  # stitched output
CHUNKS_DIR="${CHUNKS_DIR:-}"                     # where to keep raw chunks (empty = a
                                               # "Raw HEVC Chunks" folder inside ROOT)
FPS="${FPS:-20}"                               # openpilot cameras record at 20fps
WITH_AUDIO="${WITH_AUDIO:-1}"                   # 1 = mux microphone audio (from qcamera.ts)
                                               #     into the video when it was recorded
WITH_COMBINED="${WITH_COMBINED:-0}"            # 1 = also make one combined multi-angle video
PRIMARY_CAM="${PRIMARY_CAM:-road}"             # combined-video roles, by label (road|wide|driver):
SECONDARY_CAM="${SECONDARY_CAM:-wide}"         #   3 cams -> primary fills the top 2/3, secondary
TERTIARY_CAM="${TERTIARY_CAM:-driver}"         #   bottom-left, tertiary bottom-right; 2 -> side by side
SKIP_LATEST="${SKIP_LATEST:-1}"                # 1 = hold back a drive only if it's still
                                               #     being recorded; 0 = stitch everything now.
MIN_AGE_SECS="${MIN_AGE_SECS:-120}"            # a drive whose newest chunk was written less
                                               #     than this many seconds ago is treated as
                                               #     "still recording" and skipped until next run.
CLEAN_RAW="${CLEAN_RAW:-0}"                     # 0 = keep raw chunks for manual cleanup
                                               # 1 = delete a drive's raw chunks after stitching
USE_USB="${USE_USB:-0}"                          # 1 = transfer over a USB/ADB link instead of WiFi.
                                               #     A fallback for when WiFi isn't available —
                                               #     it is NOT faster than WiFi on the comma 3X.
USB_PORT="${USB_PORT:-2222}"                     # local port used to tunnel SSH over ADB
BWLIMIT="${BWLIMIT:-}"                            # optional rsync transfer cap (e.g. 3m = 3 MB/s).
                                               #     Lowers the comma's power draw — set this if the
                                               #     device browns out / reboots mid-transfer.
SSH_CIPHER="${SSH_CIPHER:-aes128-gcm@openssh.com}" # HW-accelerated on the comma 3X (ARMv8 crypto):
                                               #     ~20% faster + less CPU/heat than the default
                                               #     chacha20. Set empty to use SSH's default.
REMOTE_USER="comma"
REMOTE_PATH="/data/media/0/realdata/"
RSYNC="$(command -v /opt/homebrew/bin/rsync || command -v rsync)"
# -----------------------------------------------------------------------------

STAGING="${CHUNKS_DIR:-$ROOT/Raw HEVC Chunks}"  # visible: browse/clean these yourself
LEDGER="$ROOT/.processed_routes"
IPCACHE="${HOME}/.cache/comma-sync/last_ip"   # global (not per-folder) so discovery
                                              # works the first time on any new location

# One cleanup hook for temp files and any USB port-forward.
cleanup() {
  [ -n "${EXCLUDES:-}" ] && rm -f "$EXCLUDES"
  [ -n "${DEVCOUNTS:-}" ] && rm -f "$DEVCOUNTS"
  [ "$USE_USB" = "1" ] && adb forward --remove "tcp:${USB_PORT}" >/dev/null 2>&1
  [ -n "${CAFFEINATE_PID:-}" ] && kill "$CAFFEINATE_PID" >/dev/null 2>&1
  return 0
}
trap cleanup EXIT

# Optional USB/ADB transport. Tunnels SSH over the comma's ADB gadget and points the
# whole pipeline at localhost. Same speed or slower than WiFi on the comma 3X — only
# worth it when no WiFi is available. Requires ADB enabled on the device (Developer menu).
if [ "$USE_USB" = "1" ]; then
  command -v adb >/dev/null 2>&1 || { echo "!! USE_USB=1 needs adb (macOS: brew install --cask android-platform-tools · Linux: apt install android-tools-adb)"; exit 1; }
  adb get-state >/dev/null 2>&1 || { echo "!! No ADB device. Enable ADB on the comma (Settings -> Developer) and connect USB."; exit 1; }
  echo "==> Using USB (ADB) link. NOTE: not faster than WiFi on the comma 3X — use only when WiFi is unavailable."
  adb forward "tcp:${USB_PORT}" "tcp:${REMOTE_PORT}" >/dev/null
  COMMA_IP="127.0.0.1"; REMOTE_PORT="${USB_PORT}"; AUTO_DISCOVER=0
fi

# Ignore host keys for this LAN device — they change on every reflash, which would
# otherwise trigger a "host identification changed" error and block the connection.
SSH_BASE="-i ${SSH_KEY} -p ${REMOTE_PORT} ${SSH_CIPHER:+-c $SSH_CIPHER} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
SSH_OPTS="ssh ${SSH_BASE} -o ConnectTimeout=10 -o ServerAliveInterval=10 -o ServerAliveCountMax=6"

mkdir -p "$ROOT" "$STAGING"
mkdir -p "$(dirname "$IPCACHE")" 2>/dev/null || true
touch "$LEDGER"

# Sweep scratch left by a previously-interrupted stitch (a kill/crash mid-render skips
# the inline cleanup and can strand a multi-GB temp that would otherwise wedge the next
# run). Only touch our own "comma_*" entries older than a minute, so a concurrent run's
# fresh scratch is never removed.
find "${TMPDIR:-/tmp}" -maxdepth 1 -name 'comma_*' -mmin +1 -exec rm -rf {} + 2>/dev/null || true

# ---- portability shims (macOS vs Linux) -------------------------------------
case "$(uname -s)" in Darwin) IS_MAC=1 ;; *) IS_MAC=0 ;; esac

# Keep the Mac awake for the whole sync + stitch. A long drive (hours of footage)
# takes a while to stitch; if the Mac idle-sleeps part way through, ffmpeg is killed
# and leaves half-written, unplayable MP4s (no moov atom). `caffeinate -i -m -w $$`
# holds an idle-sleep + disk-idle assertion that lasts until THIS script exits, then
# releases on its own. Works on battery, not just AC. (cleanup() also kills it.)
CAFFEINATE_PID=""
if [ "$IS_MAC" = 1 ] && command -v caffeinate >/dev/null 2>&1; then
  caffeinate -i -m -w $$ >/dev/null 2>&1 &
  CAFFEINATE_PID=$!
fi

# epoch mtime of a file
_mtime() { if [ "$IS_MAC" = 1 ]; then stat -f %m "$1"; else stat -c %Y "$1"; fi; }
# byte size of a file
_size() { if [ "$IS_MAC" = 1 ]; then stat -f %z "$1"; else stat -c %s "$1"; fi; }
# format an epoch with a strftime string (e.g. "+%Y-%m-%d_%H-%M-%S")
_fmtdate() { if [ "$IS_MAC" = 1 ]; then date -r "$1" "$2"; else date -d "@$1" "$2"; fi; }
# Is an MP4 actually finished/playable? A render that was interrupted (e.g. the Mac
# slept mid-stitch) leaves a file with no moov atom and no readable duration. We only
# trust / reuse an existing video if ffprobe reports a positive duration for it.
mp4_ok() {
  [ -f "$1" ] || return 1
  local d
  d="$(ffprobe -v error -show_entries format=duration -of default=nk=1:nw=1 "$1" 2>/dev/null | head -1)"
  awk -v d="$d" 'BEGIN{ exit !(d+0 > 0) }'
}
# TCP connect probe (~1s). BSD nc uses -G for connect timeout; GNU/ncat don't.
_ncz() {
  if [ "$IS_MAC" = 1 ]; then nc -z -G 1 -w 1 "$1" "$2" >/dev/null 2>&1
  else nc -z -w 1 "$1" "$2" >/dev/null 2>&1; fi
}
# this machine's LAN IP (for the subnet scan)
_localip() {
  if [ "$IS_MAC" = 1 ]; then
    local iface; iface="$(route -n get default 2>/dev/null | awk '/interface:/{print $2; exit}')"
    ipconfig getifaddr "${iface:-en0}" 2>/dev/null || ipconfig getifaddr en1 2>/dev/null || true
  else
    ip route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="src"){print $(i+1); exit}}' \
      || hostname -I 2>/dev/null | awk '{print $1}'
  fi
}
# -----------------------------------------------------------------------------

# Map a camera file to a friendly label.
label_for() {
  case "$1" in
    fcamera.hevc) echo "road" ;;
    ecamera.hevc) echo "wide" ;;
    dcamera.hevc) echo "driver" ;;
    *)            echo "${1%.hevc}" ;;
  esac
}

# Recording start stamp for a route, from the earliest segment's file mtime
# (rsync -a preserves the device's mtimes). Prints YYYY-MM-DD_HH-MM-SS.
route_start_stamp() {
  local route="$1" seg0 epoch f
  seg0="$(find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec basename {} \; \
          | while IFS= read -r d; do printf '%s\t%s\n' "${d##*--}" "$d"; done \
          | sort -n -k1,1 | head -1 | cut -f2)"
  [ -n "$seg0" ] || return 1
  epoch="$(for f in "$STAGING/$seg0"/*.hevc; do [ -e "$f" ] || continue; _mtime "$f"; done | sort -n | head -1)"
  [ -n "$epoch" ] || return 1
  _fmtdate "$epoch" "+%Y-%m-%d_%H-%M-%S"
}

# Newest chunk mtime for a route (epoch) — used to tell if a drive is still recording.
route_newest_mtime() {
  local route="$1" s f
  for s in "$STAGING/${route}--"*; do
    [ -d "$s" ] || continue
    for f in "$s"/*.hevc; do [ -e "$f" ] || continue; _mtime "$f"; done
  done | sort -n | tail -1
}

# Quick TCP check so a stale/expected IP fails fast (~1s) instead of a long SSH timeout.
port_open() { _ncz "$1" "$REMOTE_PORT"; }

# Is the host at $1 actually our comma? (SSH in with our key, look for the
# openpilot install — present on every comma even before any drives are recorded)
is_comma() {
  ssh $SSH_BASE -o BatchMode=yes -o ConnectTimeout=4 \
    "${REMOTE_USER}@$1" 'test -d /data/openpilot' >/dev/null 2>&1
}

# Find the comma via mDNS/Bonjour — no subnet scan, gentle on WiFi, and it keeps
# working when the DHCP IP changes. The comma's avahi advertises an _ssh._tcp
# service named like "comma SSH - tizi - [comma-XXXX]". Echoes its .local hostname.
discover_mdns() {
  local host="" tmp p
  if command -v dns-sd >/dev/null 2>&1; then          # macOS
    tmp="$(mktemp)"
    dns-sd -B _ssh._tcp local >"$tmp" 2>/dev/null &
    p=$!; sleep 2; kill "$p" 2>/dev/null || true; wait "$p" 2>/dev/null || true
    host="$(sed -nE 's/.*\[(comma-[A-Za-z0-9]+)\].*/\1/p' "$tmp" | head -1)"
    rm -f "$tmp"
  elif command -v avahi-browse >/dev/null 2>&1; then   # Linux
    host="$(avahi-browse -rtp _ssh._tcp 2>/dev/null | awk -F';' 'tolower($0) ~ /comma/ && $1=="=" {print $7; exit}')"
  fi
  [ -n "$host" ] || return 1
  case "$host" in *.local|*.local.) host="${host%.}" ;; *) host="${host}.local" ;; esac
  printf '%s\n' "$host"
}

# Find the comma's current IP: try preferred + cached first, then scan the subnet.
# On success, sets COMMA_IP and caches it. Returns non-zero if nothing is found.
resolve_comma_ip() {
  local ip cached
  cached="$(cat "$IPCACHE" 2>/dev/null || true)"

  # Fast path: a fixed COMMA_IP (if set) or the last-known IP — but only bother
  # with a full SSH probe when the port is actually open, so a stale IP is cheap.
  for ip in "$COMMA_IP" "$cached"; do
    [ -n "$ip" ] || continue
    if port_open "$ip" && is_comma "$ip"; then
      COMMA_IP="$ip"; echo "$ip" > "$IPCACHE"
      echo "==> Comma found at ${ip}" >&2; return 0
    fi
  done

  if [ "$AUTO_DISCOVER" != "1" ]; then
    echo "!! Couldn't reach the comma at ${COMMA_IP} (AUTO_DISCOVER is off)." >&2; return 1
  fi

  # Preferred: mDNS (no scan, gentle on WiFi, survives DHCP changes).
  local mhost
  mhost="$(discover_mdns 2>/dev/null || true)"
  if [ -n "$mhost" ] && is_comma "$mhost"; then
    COMMA_IP="$mhost"; echo "$mhost" > "$IPCACHE"
    echo "==> Comma found via mDNS (${mhost})" >&2; return 0
  fi

  if ! command -v nc >/dev/null 2>&1; then
    echo "!! 'nc' not found, can't scan the network. Set COMMA_IP manually." >&2; return 1
  fi

  # Find our active interface's subnet (e.g. 192.168.1).
  local iface localip base i tmp
  localip="$(_localip)"
  if [ -z "$localip" ]; then
    echo "!! Couldn't determine this Mac's network. Set COMMA_IP manually." >&2; return 1
  fi
  base="${localip%.*}"

  echo "==> Scanning ${base}.0/24 for the comma (port ${REMOTE_PORT})..." >&2
  tmp="$(mktemp)"
  # Parallel TCP scan, throttled in batches so we don't spawn 254 procs at once.
  for i in $(seq 1 254); do
    ( _ncz "${base}.${i}" "$REMOTE_PORT" && echo "${base}.${i}" >> "$tmp" ) &
    if [ $((i % 64)) -eq 0 ]; then wait; fi
  done
  wait

  # Probe each host with an open port; first real comma wins.
  while IFS= read -r ip; do
    [ "$ip" = "$localip" ] && continue
    if is_comma "$ip"; then
      COMMA_IP="$ip"; echo "$ip" > "$IPCACHE"; rm -f "$tmp"
      echo "==> Comma found at ${ip}" >&2; return 0
    fi
  done < <(sort -t. -k4 -n "$tmp")
  rm -f "$tmp"

  echo "!! No comma device found on ${base}.0/24." >&2
  echo "   Make sure it's powered on and on the same WiFi, then re-run." >&2
  return 1
}

# One-shot remote scan: per route still on the comma, print
#   route|earliest_mtime_epoch|cam1.hevc,cam2.hevc|segments|sizeKB
device_drives_remote() {
  resolve_comma_ip || return 1
  ssh $SSH_BASE -o BatchMode=yes -o ConnectTimeout=8 "${REMOTE_USER}@${COMMA_IP}" '
    cd /data/media/0/realdata 2>/dev/null || exit 0
    for r in $(ls -1d *--*/ 2>/dev/null | sed -E "s#--[0-9]+/##" | sort -u); do
      [ "$r" = "boot" ] && continue
      cnt=$(ls -1d ${r}--*/ 2>/dev/null | wc -l | tr -d " ")
      [ "$cnt" = "0" ] && continue
      mt=$(for f in ${r}--*/*.hevc; do [ -e "$f" ] || continue; stat -c %Y "$f"; done | sort -n | head -1)
      cams=$(ls ${r}--*/*.hevc 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -u | tr "\n" "," | sed "s/,$//")
      sz=$(du -sk ${r}--* 2>/dev/null | awk "{s+=\$1} END{print s+0}")
      echo "${r}|${mt}|${cams}|${cnt}|${sz}"
    done
  ' 2>/dev/null
}

# Pull just one route (hevc + qcamera.ts) off the comma into the chunks folder.
pull_route() {
  local route="$1"
  resolve_comma_ip || return 1
  echo "   downloading ${route} from ${COMMA_IP}..."
  local attempt=0 max="${MAX_ATTEMPTS:-40}"
  until "$RSYNC" -a --info=progress2 --no-inc-recursive --partial --append-verify --timeout=120 --prune-empty-dirs \
    --rsync-path="nice -n 19 rsync" ${BWLIMIT:+--bwlimit=$BWLIMIT} \
    --include="${route}--*/" \
    --include="${route}--*/*.hevc" \
    --include="${route}--*/qcamera.ts" \
    --exclude='*' \
    -e "$SSH_OPTS" \
    "${REMOTE_USER}@${COMMA_IP}:${REMOTE_PATH}" "$STAGING/"; do
    attempt=$((attempt + 1))
    [ "$attempt" -ge "$max" ] && { echo "   !! gave up after ${max} reconnect attempts (re-run to resume)"; return 1; }
    echo "   ...dropped (the comma may be rebooting); reconnecting (attempt ${attempt}/${max})..."
    sleep 8
    if [ $((attempt % 6)) -eq 0 ] && ! port_open "$COMMA_IP" "$REMOTE_PORT"; then
      resolve_comma_ip >/dev/null 2>&1 || true
    fi
  done
}

# Comma-separated camera filenames -> friendly labels (ecamera.hevc,fcamera.hevc -> wide,road)
labels_csv() {
  local IFS=',' cam out=""
  for cam in $1; do
    [ -n "$cam" ] || continue
    out="${out:+$out,}$(label_for "$cam")"
  done
  echo "$out"
}

# Is a route fully downloaded? 0 = complete (segments are contiguous 0..max, and
# match the device's count when known), 1 = still partial. Empty dirs are pruned
# before this runs, so only segments with footage are counted.
route_complete() {
  local route="$1" nums cnt max local_b exp_b s f
  nums="$(find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec basename {} \; 2>/dev/null \
          | sed -E 's/.*--//' | sort -n)"
  [ -z "$nums" ] && return 1

  # Authoritative when the device was reachable: compare total hevc BYTES. This
  # catches both missing segments AND truncated/partial files (a reboot can leave
  # a half-downloaded segment that the count check would wrongly pass).
  exp_b="$(awk -v r="$route" '$1==r{print $3}' "${DEVCOUNTS:-/dev/null}" 2>/dev/null)"
  if [ -n "$exp_b" ] && [ "$exp_b" -gt 0 ] 2>/dev/null; then
    local_b="$(for s in "$STAGING/${route}--"*; do
                 for f in "$s"/*.hevc; do [ -e "$f" ] || continue; _size "$f"; done
               done | awk '{s+=$1} END{print s+0}')"
    [ "${local_b:-0}" -lt "$exp_b" ] && return 1
    return 0
  fi

  # Fallback (device offline): contiguous segments 0..max, no gaps.
  cnt="$(printf '%s\n' "$nums" | grep -c .)"
  max="$(printf '%s\n' "$nums" | tail -1)"
  [ "$cnt" -ne "$((max + 1))" ] && return 1
  return 0
}

# Stitch one drive's local chunks into MP4s. $2=1 enables collision-safe naming
# combine_video <outdir> <stamp> <suffix>: build one optional multi-angle MP4 from
# the per-camera MP4s just produced. 2 cameras -> side by side at native aspect; 3 ->
# the primary across the top 2/3 with the secondary (bottom-left) and tertiary
# (bottom-right) in the bottom third. Roles come from PRIMARY_CAM/SECONDARY_CAM/
# TERTIARY_CAM (labels road|wide|driver); missing or duplicate roles are skipped, and
# with fewer than 2 cameras nothing is made. Re-encodes (HW on macOS).
combine_video() {
  local outdir="$1" stamp="$2" suffix="$3"
  local r rmp4 used="" roles=()
  for r in "$PRIMARY_CAM" "$SECONDARY_CAM" "$TERTIARY_CAM"; do
    case " $used " in *" $r "*) continue ;; esac          # skip a duplicate role
    rmp4="$outdir/${stamp}__${r}${suffix}.mp4"
    mp4_ok "$rmp4" && { roles+=("$rmp4"); used="$used $r"; }
  done
  local nc="${#roles[@]}"
  [ "$nc" -ge 2 ] || return 0                             # 1 camera (or 0) -> nothing to combine

  local cout="$outdir/${stamp}__combined${suffix}.mp4" ctmp fc rc=0
  ctmp="${cout}.part"; rm -f "$ctmp"
  local venc=() inargs=()
  if [ "$IS_MAC" = "1" ]; then venc=(-c:v h264_videotoolbox -b:v 14M)
  else venc=(-c:v libx264 -preset veryfast -crf 22 -pix_fmt yuv420p); fi
  # On macOS, hardware-decode the HEVC inputs on the VideoToolbox media engine (same
  # silicon Final Cut uses). We already HW-encode; this offloads the decode too —
  # measured ~24% faster and ~1/3 the CPU on a 3-cam combine, with identical output.
  # It auto-falls back to software if a stream isn't HW-decodable. Linux/Windows stay
  # on the software path (HW decode there is GPU-vendor-specific).
  for rmp4 in "${roles[@]}"; do
    [ "$IS_MAC" = "1" ] && inargs+=(-hwaccel videotoolbox)
    inargs+=(-i "$rmp4")
  done

  if [ "$nc" -eq 2 ]; then
    fc="[0:v]scale=-2:1208[a];[1:v]scale=-2:1208[b];[a][b]hstack=inputs=2[v]"
  else
    fc="[0:v]scale=1920:1200:force_original_aspect_ratio=decrease,pad=1920:1200:(ow-iw)/2:(oh-ih)/2[p];"
    fc+="[1:v]scale=960:600:force_original_aspect_ratio=decrease,pad=960:600:(ow-iw)/2:(oh-ih)/2[s];"
    fc+="[2:v]scale=960:600:force_original_aspect_ratio=decrease,pad=960:600:(ow-iw)/2:(oh-ih)/2[t];"
    fc+="color=c=black:s=1920x1800:r=${FPS}[bg];[bg][p]overlay=0:0[b1];[b1][s]overlay=0:1200[b2];[b2][t]overlay=960:1200:shortest=1[v]"
  fi

  ffmpeg -y -loglevel error "${inargs[@]}" -filter_complex "$fc" \
    -map "[v]" -map "0:a?" "${venc[@]}" -c:a copy -movflags +faststart -f mp4 "$ctmp" || rc=$?
  if [ "$rc" -eq 0 ] && mp4_ok "$ctmp"; then
    mv -f "$ctmp" "$cout"
    echo "      combined (${nc} cams): $(basename "$cout")"
  else
    rm -f "$ctmp"
    echo "      !! combined video failed for ${stamp}"
  fi
}

# (append " (N)" so a re-sync into an existing folder never overwrites old files).
# Returns 0 only if every camera stitched. Does NOT touch the ledger or chunks.
stitch_route() {
  local route="$1" collision="${2:-0}" stamp outdir line s f
  stamp="$(route_start_stamp "$route" 2>/dev/null || true)"
  [ -n "$stamp" ] || stamp="$route"
  outdir="$ROOT/$stamp"
  mkdir -p "$outdir"

  # Segment dirs for this route, sorted by trailing segment number (numeric).
  local segs=()
  while IFS= read -r line; do segs+=("$line"); done < <(
    find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec basename {} \; \
      | while IFS= read -r d; do printf '%s\t%s\n' "${d##*--}" "$d"; done \
      | sort -n -k1,1 | cut -f2
  )
  if [ "${#segs[@]}" -eq 0 ]; then echo "   !! no raw chunks for ${route}"; return 1; fi

  # Cameras present (iterate the glob directly so paths with spaces don't split).
  local cams=()
  while IFS= read -r line; do cams+=("$line"); done < <(
    for s in "${segs[@]}"; do
      for f in "$STAGING/$s"/*.hevc; do [ -e "$f" ] || continue; basename "$f"; done
    done | sort -u
  )
  if [ "${#cams[@]}" -eq 0 ]; then echo "   !! no camera footage downloaded for ${route} yet"; return 1; fi

  # If the per-camera videos for this drive are already in the output folder, reuse
  # them for the combined video instead of re-rendering everything (just builds the
  # multi-angle file from the existing MP4s; no re-stitch of the individuals).
  if [ "$WITH_COMBINED" = "1" ]; then
    local have_all=1 _l _c
    for _c in "${cams[@]}"; do
      _l="$(label_for "$_c")"
      mp4_ok "$ROOT/$stamp/${stamp}__${_l}.mp4" || { have_all=0; break; }
    done
    if [ "$have_all" = "1" ]; then
      if [ -f "$ROOT/$stamp/${stamp}__combined.mp4" ]; then
        echo "==> ${stamp}: individual + combined videos already exist — nothing to do"
        return 0
      fi
      echo "==> ${stamp}: using the existing individual videos for the combined (no re-render)"
      combine_video "$ROOT/$stamp" "$stamp" ""
      return $?
    fi
  fi

  # Collision-safe suffix: lowest N (1 = none) where no camera output exists yet.
  local suffix="" cam lbl
  if [ "$collision" = "1" ]; then
    local n=1 sfx hit
    while :; do
      [ "$n" -eq 1 ] && sfx="" || sfx=" ($n)"
      hit=0
      for cam in "${cams[@]}"; do
        lbl="$(label_for "$cam")"
        [ -e "$outdir/${stamp}__${lbl}${sfx}.mp4" ] && { hit=1; break; }
      done
      [ "$hit" -eq 0 ] && { suffix="$sfx"; break; }
      n=$((n + 1))
    done
  fi

  echo "==> Stitching drive ${route}  ->  ${stamp}${suffix}"

  # One scratch dir for this drive's big temp files (concatenated HEVC + audio PCM),
  # with fixed names inside a UNIQUE dir. (BSD/macOS mktemp only randomizes a trailing
  # run of X's, so "comma_XXXXXX.hevc" is NOT unique — it's a fixed name that an
  # interrupted run can leave behind and block the next run. A unique dir avoids that,
  # and one `rm -rf` cleans everything even if a camera fails.)
  local tmpd
  tmpd="$(mktemp -d "${TMPDIR:-/tmp}/comma_XXXXXX")"

  # Microphone audio, locked to the video segment-by-segment. The comma's mic starts
  # a few seconds AFTER the camera at the start of a drive (the first qcamera.ts has a
  # leading gap — e.g. audio doesn't begin until ~10s in), and segments can have small
  # gaps too. If we just decode the samples and concatenate, every later segment plays
  # early and the audio runs ahead of the video (more so the longer the drive). So for
  # each segment we decode its audio *preserving timing* (aresample fills the leading
  # gap with silence) and fit it to that segment's exact video length, so audio and
  # video realign at every segment boundary and never drift. The comma mic is 16 kHz
  # mono. Segments with no audio get silence so they don't shift everything after them.
  local audio_pcm="" audio_dur=0 spcm rate=16000 segframes segdur tbytes cur had_audio=0
  if [ "$WITH_AUDIO" != "0" ]; then
    audio_pcm="$tmpd/aud.pcm"; : > "$audio_pcm"
    spcm="$tmpd/seg.pcm"
    for s in "${segs[@]}"; do
      # This segment's video length, from its frame count (any camera — they're synced).
      segframes=""
      for f in "$STAGING/$s"/*.hevc; do
        [ -e "$f" ] || continue
        segframes="$(ffprobe -v error -select_streams v -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "$f" 2>/dev/null | head -1 | tr -d '[:space:]')"
        [ -n "$segframes" ] && break
      done
      [ -n "$segframes" ] || continue
      segdur="$(awk -v f="$segframes" -v r="$FPS" 'BEGIN{printf "%.4f", f/r}')"
      tbytes="$(awk -v d="$segdur" -v r="$rate" 'BEGIN{printf "%d", int(d*r+0.5)*2}')"
      if [ -f "$STAGING/$s/qcamera.ts" ] && \
         ffmpeg -y -v error -i "$STAGING/$s/qcamera.ts" -map 0:a:0 \
           -af "aresample=async=1:first_pts=0,apad" -t "$segdur" \
           -ar "$rate" -ac 1 -f s16le "$spcm" 2>/dev/null && [ -s "$spcm" ]; then
        had_audio=1
        cur="$(_size "$spcm")"
        if [ "$cur" -ge "$tbytes" ]; then
          head -c "$tbytes" "$spcm" >> "$audio_pcm"
        else
          cat "$spcm" >> "$audio_pcm"
          head -c "$((tbytes - cur))" /dev/zero >> "$audio_pcm"
        fi
      else
        head -c "$tbytes" /dev/zero >> "$audio_pcm"   # keep sync when a segment has no audio
      fi
    done
    rm -f "$spcm"
    if [ "$had_audio" = "1" ] && [ -s "$audio_pcm" ]; then
      audio_dur="$(awk -v b="$(_size "$audio_pcm")" 'BEGIN{printf "%.4f",(b/2)/16000}')"
    else
      rm -f "$audio_pcm"; audio_pcm=""
    fi
  fi

  local ok=1 out vout combined rc atag frames vdur tempo
  for cam in "${cams[@]}"; do
    lbl="$(label_for "$cam")"
    out="$outdir/${stamp}__${lbl}${suffix}.mp4"
    # Render to a temp file and only move it into place on success, so an interrupted
    # render (Mac sleep, kill, disk full) never leaves an unplayable file at the real
    # path that later runs would mistake for a finished video.
    vout="${out}.part"
    rm -f "$vout"
    combined="$tmpd/vid.hevc"; rm -f "$combined"
    for s in "${segs[@]}"; do
      [ -f "$STAGING/$s/$cam" ] && cat "$STAGING/$s/$cam" >> "$combined"
    done
    rc=0
    if [ -n "$audio_pcm" ]; then
      # Stretch the audio to exactly fill this camera's video so it stays in sync
      # and doesn't end early. Count frames cheaply (packets, no decode).
      frames="$(ffprobe -v error -select_streams v -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "$combined" 2>/dev/null)"
      vdur="$(awk -v f="${frames:-0}" -v r="$FPS" 'BEGIN{ if(r>0 && f>0) printf "%.4f", f/r; else print 0 }')"
      tempo="$(awk -v a="$audio_dur" -v v="$vdur" 'BEGIN{ if(v>0){t=a/v; if(t<0.5)t=0.5; if(t>2.0)t=2.0; printf "%.6f",t} else print 1 }')"
      ffmpeg -y -loglevel error -framerate "$FPS" -i "$combined" -f s16le -ar 16000 -ac 1 -i "$audio_pcm" \
        -map 0:v:0 -map 1:a:0 -c:v copy -filter:a "atempo=${tempo}" -c:a aac -b:a 96k -tag:v hvc1 -f mp4 "$vout" || rc=$?
    else
      ffmpeg -y -loglevel error -framerate "$FPS" -i "$combined" \
        -c copy -tag:v hvc1 -f mp4 "$vout" || rc=$?
    fi
    if [ "$rc" -eq 0 ] && mp4_ok "$vout"; then
      mv -f "$vout" "$out"
      [ -n "$audio_pcm" ] && atag=" +audio" || atag=""
      echo "      ${lbl}${atag}: $(basename "$out")"
    else
      rm -f "$vout"
      echo "      !! ffmpeg failed for ${lbl} on ${route}"; ok=0
    fi
    rm -f "$combined"
  done
  if [ "$ok" = "1" ] && [ "$WITH_COMBINED" = "1" ]; then
    combine_video "$outdir" "$stamp" "$suffix"
  fi
  rm -rf "$tmpd"
  [ "$ok" = "1" ]
}

# Does a drive have audio? (check its last segment's qcamera.ts) -> echoes 1/0
drive_has_audio() {
  local route="$1" last
  last="$(find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec basename {} \; \
          | while IFS= read -r d; do printf '%s\t%s\n' "${d##*--}" "$d"; done \
          | sort -n -k1,1 | tail -1 | cut -f2)"
  if [ -n "$last" ] && [ -f "$STAGING/$last/qcamera.ts" ] \
     && ffprobe -v error -select_streams a -show_entries stream=codec_type -of csv=p=0 \
        "$STAGING/$last/qcamera.ts" 2>/dev/null | grep -q audio; then
    echo 1
  else
    echo 0
  fi
}

# --list: one TSV row per drive, merging chunks on this Mac with drives still on
# the comma (re-downloadable). Columns:
#   route <tab> stamp <tab> cameras(csv) <tab> hasAudio(0/1/-) <tab> sizeKB <tab> segments <tab> location
# location is "local" (chunks on disk) or "device" (only on the comma).
cmd_list() {
  local route stamp cams_csv audio sizek segcount s f
  local seen; seen="$(mktemp)"

  # 1) Drives whose chunks are on this Mac.
  while IFS= read -r route; do
    [ -n "$route" ] || continue
    echo "$route" >> "$seen"
    stamp="$(route_start_stamp "$route" 2>/dev/null || echo "$route")"
    cams_csv="$(
      for s in $(find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec basename {} \;); do
        for f in "$STAGING/$s"/*.hevc; do [ -e "$f" ] || continue; basename "$f"; done
      done | sort -u | while IFS= read -r c; do label_for "$c"; done | paste -sd ',' -
    )"
    audio="$(drive_has_audio "$route")"
    segcount="$(find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" | wc -l | tr -d ' ')"
    sizek="$(du -sk "$STAGING"/${route}--* 2>/dev/null | awk '{s+=$1} END{print s+0}' || true)"
    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$route" "$stamp" "$cams_csv" "$audio" "$sizek" "$segcount" "local"
  done < <(
    find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name '*--*' -exec basename {} \; \
      | sed -E 's/--[0-9]+$//' | sort -u
  )

  # 2) Drives still on the comma that aren't already on this Mac (re-downloadable).
  local r mt cams cnt sz dstamp
  while IFS='|' read -r r mt cams cnt sz; do
    [ -n "$r" ] || continue
    grep -qxF "$r" "$seen" && continue
    if [ -n "$mt" ]; then dstamp="$(_fmtdate "$mt" "+%Y-%m-%d_%H-%M-%S")"; else dstamp="$r"; fi
    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$r" "$dstamp" "$(labels_csv "$cams")" "-" "${sz:-0}" "${cnt:-0}" "device"
  done < <(device_drives_remote || true)

  rm -f "$seen"
}

# --restitch <route>: re-stitch a drive with the current settings, never
# overwriting existing output. Re-downloads the chunks from the comma if needed.
cmd_restitch() {
  local route="$1"
  [ -n "$route" ] || { echo "!! usage: --restitch <route>"; return 2; }
  if ! find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" 2>/dev/null | grep -q .; then
    echo "==> No local chunks for ${route} — fetching from the comma..."
    pull_route "$route" || {
      echo "!! Couldn't download ${route} (comma offline, or the drive is no longer on it)."
      return 1
    }
    if ! find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" 2>/dev/null | grep -q .; then
      echo "!! ${route} is not on the comma anymore."
      return 1
    fi
  fi
  stitch_route "$route" 1 || return 1
  echo
  echo "==> Re-sync complete. Output in: ${ROOT}/"
}

# Subcommands used by the app (list synced drives / re-sync one). Default = sync.
case "${1:-}" in
  --list)     cmd_list; exit 0 ;;
  --restitch) cmd_restitch "${2:-}" && exit 0 || exit 1 ;;
esac

# --- 1. Pull only new *.hevc, skipping drives we've already processed ---------
EXCLUDES="$(mktemp)"
# (EXCLUDES is removed by the global cleanup() trap set above)
# Turn each processed route into an rsync exclude pattern so it's never re-pulled.
while IFS= read -r route; do
  [ -n "$route" ] && printf '%s--*\n' "$route" >> "$EXCLUDES"
done < "$LEDGER"

if resolve_comma_ip; then
  echo "==> Syncing new footage from ${REMOTE_USER}@${COMMA_IP} ..."
  # comma devices drop WiFi often, so keep partial files and retry on disconnects.
  # --partial/--append-verify resumes interrupted files; the ledger (above) is what
  # prevents re-downloading already-processed drives, so --ignore-existing isn't needed.
  # The comma may reboot repeatedly mid-transfer (e.g. on weak power). rsync's
  # --partial/--append-verify resumes each interrupted file (finished files are
  # skipped, so nothing already downloaded is re-transferred), and the ledger keeps
  # done drives out entirely. We just need to outlast the reboots and re-find the
  # device if it comes back on a new IP — so retry generously and re-discover.
  attempt=0; max_attempts="${MAX_ATTEMPTS:-40}"
  until "$RSYNC" -a --info=progress2 --no-inc-recursive --partial --append-verify --timeout=120 --prune-empty-dirs \
    --rsync-path="nice -n 19 rsync" ${BWLIMIT:+--bwlimit=$BWLIMIT} \
    --exclude-from="$EXCLUDES" \
    --include='*/' --include='*.hevc' --include='qcamera.ts' --exclude='*' \
    -e "$SSH_OPTS" \
    "${REMOTE_USER}@${COMMA_IP}:${REMOTE_PATH}" \
    "$STAGING/"; do
    attempt=$((attempt + 1))
    if [ "$attempt" -ge "$max_attempts" ]; then
      echo "!! Paused after ${max_attempts} reconnect attempts. Just run it again —"
      echo "   finished files are kept and resume, so nothing already downloaded re-transfers."
      break
    fi
    echo "   ...dropped (the comma may be rebooting); waiting for it to come back (attempt ${attempt}/${max_attempts})..."
    sleep 8
    # Usually it returns on the same IP (cheap retry). Every few tries, re-discover
    # in case a reboot moved it to a new address — rsync resumes either way.
    if [ $((attempt % 6)) -eq 0 ] && ! port_open "$COMMA_IP" "$REMOTE_PORT"; then
      resolve_comma_ip >/dev/null 2>&1 || true
    fi
  done
else
  echo "==> Skipping download; stitching whatever is already in ${STAGING}/."
fi

# --- 1b. Clean up partial-transfer debris + learn which drives finished -------
# Interrupted transfers (the comma rebooting) leave empty segment dirs — the dir
# is created before its hevc is pulled. Remove them so only complete segments count.
find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name '*--*' 2>/dev/null | while IFS= read -r d; do
  [ -z "$(find "$d" -maxdepth 1 -name '*.hevc' -print -quit 2>/dev/null)" ] && rm -rf "$d"
done

# Authoritative segment count per route from the device (only if it's reachable
# right now — no extra scan), so we skip drives that are still downloading.
DEVCOUNTS="$(mktemp)"
if [ -n "${COMMA_IP:-}" ] && port_open "$COMMA_IP" "$REMOTE_PORT"; then
  ssh $SSH_BASE -o BatchMode=yes -o ConnectTimeout=8 "${REMOTE_USER}@${COMMA_IP}" '
    cd /data/media/0/realdata 2>/dev/null || exit 0
    for r in $(ls -1d *--*/ 2>/dev/null | sed -E "s#--[0-9]+/##" | sort -u); do
      [ "$r" = "boot" ] && continue
      cnt=$(ls -1d ${r}--*/ 2>/dev/null | wc -l | tr -d " ")
      hb=$(for f in ${r}--*/*.hevc; do [ -e "$f" ] || continue; stat -c %s "$f"; done | awk "{s+=\$1} END{print s+0}")
      echo "${r} ${cnt} ${hb}"
    done' 2>/dev/null > "$DEVCOUNTS" || true
fi

# --- 2. Figure out which drives (routes) are staged --------------------------
ROUTES=()
while IFS= read -r line; do ROUTES+=("$line"); done < <(
  find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name '*--*' -exec basename {} \; \
    | sed -E 's/--[0-9]+$//' | sort -u
)

if [ "${#ROUTES[@]}" -eq 0 ]; then
  echo "==> No new footage to process."
  exit 0
fi

# --- 3. Stitch each drive ----------------------------------------------------
for route in "${ROUTES[@]}"; do
  # Already stitched on a previous run? Skip (its raw chunks are just being kept).
  if grep -qxF "$route" "$LEDGER"; then
    continue
  fi
  # Not fully downloaded yet (the comma may still be rebooting)? Leave it for the
  # next run rather than stitching a partial drive.
  if ! route_complete "$route"; then
    echo "==> Skipping ${route}: still downloading (incomplete) — re-run to finish it."
    continue
  fi
  # Hold back a drive only if it looks like it's still being recorded right now
  # (its newest chunk was written within the last MIN_AGE_SECS seconds).
  if [ "$SKIP_LATEST" != "0" ] && [ "$MIN_AGE_SECS" -gt 0 ]; then
    newest="$(route_newest_mtime "$route")"
    if [ -n "$newest" ]; then
      age=$(( $(date +%s) - newest ))
      if [ "$age" -lt "$MIN_AGE_SECS" ]; then
        echo "==> Skipping ${route}: last chunk written ${age}s ago (still recording?). Re-run later."
        continue
      fi
    fi
  fi

  # --- 4. Stitch; on full success record the drive (+ optionally purge chunks) -
  if stitch_route "$route" 0; then
    grep -qxF "$route" "$LEDGER" || echo "$route" >> "$LEDGER"
    if [ "$CLEAN_RAW" = "1" ]; then
      find "$STAGING" -mindepth 1 -maxdepth 1 -type d -name "${route}--*" -exec rm -rf {} +
      echo "      purged raw chunks for ${route}"
    else
      echo "      raw chunks kept in: ${STAGING}"
    fi
  fi
done

echo
echo "==> Done. Stitched drives are in: ${ROOT}/"
echo "    Raw chunks are in:         ${STAGING}/"
echo "    Delete that folder anytime to reclaim space (won't trigger re-downloads)."