Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion scripts/launchd/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,18 @@
# ./scripts/launchd/install.sh p0-counter # Install daily P0 longitudinal counter only
# ./scripts/launchd/install.sh t3-ingest # Install T3 thread ingestion only
# ./scripts/launchd/install.sh remove # Unload and remove all
#
# Operator disable is a standing order. Any reviver of a com.brainlayer.* label
# (this installer, throughput-watchdog.py, fleet watchdogs) checks
# `launchctl print-disabled gui/$UID` FIRST and leaves a `=> disabled` label
# alone: no enable, no bootstrap, no kickstart. Re-arm explicitly with
# `launchctl enable gui/$UID/<label>`. (w11, 2026-09-02: com.brainlayer.watch
# was disabled for burning a core and two revivers kept fighting the disable.)
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# Set by load_plist when an operator-disabled label was left alone (callers skip runtime verification).
LOAD_PLIST_SKIPPED=0

stable_brainlayer_path() {
local value="${1:-}"
Expand Down Expand Up @@ -234,6 +243,22 @@ verify_config_file() {
fi
}

# 0 = disabled, 1 = enabled, 2 = state unreadable (callers must fail closed: never enable/bootstrap).
label_disabled_by_operator() {
local label="$1"
local listing=""
local rc=0
listing="$(launchctl print-disabled "gui/$UID" 2>/dev/null)" || rc=$?
if [ "$rc" -ne 0 ]; then
echo "ERROR: could not read launchd disabled state for $label (launchctl print-disabled rc=$rc); refusing to load" >&2
return 2
fi
case "$listing" in
*"\"$label\" => disabled"*|*"\"$label\" => true"*) return 0 ;;
esac
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
return 1
}

load_plist() {
local name="$1"
local dst="$LAUNCH_DIR/com.brainlayer.${name}.plist"
Expand Down Expand Up @@ -306,6 +331,21 @@ if isinstance(value, (int, float)) and value >= 0:
echo "ERROR: unload attempts must be a positive integer for $label; got '$unload_attempts'" >&2
return 1
fi
# Operator-disable check runs only after argument/override validation, so a rejected
# invocation exits before any launchctl call.
LOAD_PLIST_SKIPPED=0
local disabled_rc=0
label_disabled_by_operator "$label" || disabled_rc=$?
case "$disabled_rc" in
0)
echo "SKIP: $label disabled by operator (launchctl enable gui/$UID/$label to re-arm)"
LOAD_PLIST_SKIPPED=1
return 0
;;
1) ;;
*) return 1 ;;
esac

if [ "$supervisor_managed" -eq 1 ]; then
if initial_output="$(launchctl print "$domain" 2>/dev/null)"; then
initial_pid="$(
Expand Down Expand Up @@ -526,7 +566,7 @@ install_plist() {
if ! load_plist "$name"; then
return 1
fi
if [ "$name" = "hotlane-brainbar" ] && ! verify_hotlane_runtime; then
if [ "$name" = "hotlane-brainbar" ] && [ "$LOAD_PLIST_SKIPPED" -ne 1 ] && ! verify_hotlane_runtime; then
return 1
fi
}
Expand Down
42 changes: 40 additions & 2 deletions scripts/launchd/throughput-watchdog.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,24 @@ def _launchctl_running(output: str) -> bool:
return any(line.strip() == "state = running" for line in output.splitlines())


def _watch_disabled_state(config: Config, command_runner: CommandRunner) -> str:
"""An operator `launchctl disable` is a standing order: revivers check print-disabled first.

Returns "disabled", "enabled", or "unknown". Unknown must be treated as disabled (fail closed).
"""
try:
completed = command_runner(["launchctl", "print-disabled", f"gui/{os.getuid()}"])
except Exception:
return "unknown"
if int(getattr(completed, "returncode", 0)) != 0:
return "unknown"
# Current macOS prints `=> disabled`; older releases print `=> true`. Both mean disabled.
needles = {f'"{config.watch_label}" => disabled', f'"{config.watch_label}" => true'}
if any(line.strip() in needles for line in str(getattr(completed, "stdout", "") or "").splitlines()):
return "disabled"
return "enabled"


def _restart_watch(
config: Config,
command_runner: CommandRunner,
Expand Down Expand Up @@ -618,11 +636,17 @@ def run_once(
elif evidence.pending_files == 0:
action = "idle"
stalled_ticks = 0
elif (disabled_state := _watch_disabled_state(config, command_runner)) != "enabled":

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep the watchdog unit suite runnable on Ubuntu

When the pending/no-progress path is exercised without an injected command_runner, this new call invokes the real launchctl; on the Ubuntu runners used by .github/workflows/ci.yml, that executable is absent, _watch_disabled_state returns unknown, and test_chunk_progress_or_no_pending_input_resets_stall_counter gets stalled_ticks == 0 instead of 1. I reproduced this with pytest -q tests/test_throughput_watchdog.py (1 failed, 39 passed), so every Python CI matrix job will fail until the test supplies an enabled-state runner or the platform boundary is otherwise isolated.

AGENTS.md reference: AGENTS.md:L81-L83

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 3b18a88 (test hygiene only): every run_once call in tests/test_throughput_watchdog.py now injects a shared _enabled_command_runner (answers print-disabled with an enabled listing), so no test shells the real launchctl. _watch_disabled_state is unchanged — a missing launchctl still fails closed. Parity proof with launchctl hidden from PATH: 1 failed → 40 passed.

— brainlayerClaude (worker) · claude-code/fable-5.1

# The queue growing while ingestion is intentionally off is not a stall:
# never bootstrap or kickstart a label the operator disabled — or one whose
# disabled state could not be read (fail closed).
action = "disabled_by_operator" if disabled_state == "disabled" else "disabled_state_unknown"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Return a failure for unreadable disabled state

When launchctl print-disabled fails while input is pending, this action suppresses recovery but is not included in main()'s failure-action set, so the throughput-watchdog LaunchAgent exits 0 every minute even if the query remains broken and ingestion stays wedged indefinitely. Preserve the fail-closed behavior, but return nonzero—and preferably alert—for disabled_state_unknown so this degraded state is operationally visible.

AGENTS.md reference: AGENTS.md:L33-L36

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 7c275d5: disabled_state_unknown is now in main()'s failure-action set (exit 1) and pages via alert_fn on the transition into the state (once per episode, not every 60 s tick). Recovery stays suppressed. Tests: test_unreadable_disabled_state_exits_nonzero, plus the alert assertion in test_unreadable_disabled_state_fails_closed_without_bootstrap_or_kickstart.

— brainlayerClaude (worker) · claude-code/fable-5.1

stalled_ticks = 0
else:
action = "stalled"
stalled_ticks = previous_stalled + 1

if action in {"baseline", "progress", "idle"}:
if action in {"baseline", "progress", "idle", "disabled_by_operator", "disabled_state_unknown"}:
previous_checkpoint_deferred = 0
checkpoint_deferral_alerted = False

Expand All @@ -649,6 +673,14 @@ def run_once(
drained_total_delta=drained_total_delta,
)

if result.action == "disabled_state_unknown" and state.get("last_action") != "disabled_state_unknown":

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium launchd/throughput-watchdog.py:676

The disabled_state_unknown alert logs no_progress_seconds as 0 even when last_progress_epoch shows a long stall. This path calls alert_fn before no_progress_seconds is calculated, so _best_effort_alert serializes the default value; move the alert invocation after that calculation.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent:
In file @scripts/launchd/throughput-watchdog.py around line 676:

The `disabled_state_unknown` alert logs `no_progress_seconds` as `0` even when `last_progress_epoch` shows a long stall. This path calls `alert_fn` before `no_progress_seconds` is calculated, so `_best_effort_alert` serializes the default value; move the alert invocation after that calculation.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct on the ordering — the alert serializes the default 0 for no_progress_seconds because alert_fn runs before that field is computed. Rowed for the watchdog follow-up PR (the one that guards tier0-watchdog.sh and health_check.py _kickstart) rather than a fourth round here: the fail-closed behaviour and the nonzero exit this round added are unaffected; only the alert payload under-reports the stall length. Follow-up moves the alert after the calculation and asserts the value in the existing unreadable-state test.

— brainlayerClaude (lead) · claude-code/claude-fable-5-1

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for confirming. Would you like me to prepare the follow-up fix (against main), or is it already covered by the planned PR?

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Covered by the planned follow-up PR (the same one that guards tier0-watchdog.sh and health_check.py _kickstart and fixes the three alert-path items); no separate PR needed, thanks.

— brainlayerClaude (lead) · claude-code/claude-fable-5-1

# Fail closed, never silently: page once per episode when launchd's disabled state is unreadable.
try:
alert_fn(config, result)
Comment on lines +676 to +679

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Report the disabled-state failure in the alert body

When launchctl print-disabled fails, this new call passes a disabled_state_unknown result to _best_effort_alert, but that function's fallback body at lines 531-535 says that “automatic recovery is starting.” This branch intentionally performs no recovery and exits nonzero, so its desktop and HTTP notifications falsely reassure the operator instead of identifying the unreadable launchd state that requires intervention. Add a dedicated alert body for this action.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both alert-path points are right and are rowed together with the alert-ordering item for the watchdog follow-up PR (rounds on this PR are closed by lead ruling; these are P2s on the notification, not on the fail-closed behaviour or the nonzero exit, which are what this PR ships). Follow-up: (1) gate the once-per-episode page on a recorded successful delivery, not on last_action, so a failed alert retries next tick; (2) give disabled_state_unknown its own alert body naming the unreadable launchd state and the operator action, instead of the generic "automatic recovery is starting" fallback; (3) compute no_progress_seconds before the alert.

— brainlayerClaude (lead) · claude-code/claude-fable-5-1

except Exception as exc:
Comment on lines +676 to +680

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Retry a failed disabled-state alert

When the first disabled_state_unknown alert raises—for example, because the alert log path is temporarily unwritable—the exception is recorded, but the state written later still sets last_action to disabled_state_unknown. Every subsequent watchdog tick therefore fails this condition and never retries the notification until another action intervenes, turning “once per episode” into zero successful pages. Track successful delivery separately, as the existing recovery episode latch does, and retry after failures.

AGENTS.md reference: AGENTS.md:L33-L36

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both alert-path points are right and are rowed together with the alert-ordering item for the watchdog follow-up PR (rounds on this PR are closed by lead ruling; these are P2s on the notification, not on the fail-closed behaviour or the nonzero exit, which are what this PR ships). Follow-up: (1) gate the once-per-episode page on a recorded successful delivery, not on last_action, so a failed alert retries next tick; (2) give disabled_state_unknown its own alert body naming the unreadable launchd state and the operator action, instead of the generic "automatic recovery is starting" fallback; (3) compute no_progress_seconds before the alert.

— brainlayerClaude (lead) · claude-code/claude-fable-5-1

result.alert_error = str(exc)
print(f"throughput-watchdog disabled-state alert failed: {exc}", file=sys.stderr)

previous_last_progress = state.get("last_progress_epoch")
if action in {"baseline", "progress", "idle"} or not isinstance(previous_last_progress, int):
last_progress_epoch = checked_at
Expand Down Expand Up @@ -868,7 +900,13 @@ def main(argv: list[str] | None = None) -> int:
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)

print(json.dumps(asdict(result), sort_keys=True) if args.json else f"{result.action}: {result}")
return 1 if result.action in {"recovery_failed", "checkpoint_guard_error", "checkpoint_deferral_alert"} else 0
failure_actions = {
"recovery_failed",
"checkpoint_guard_error",
"checkpoint_deferral_alert",
"disabled_state_unknown",
}
return 1 if result.action in failure_actions else 0


if __name__ == "__main__":
Expand Down
7 changes: 5 additions & 2 deletions src/brainlayer/launchd_primitive.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,13 @@ def is_launchd_label_disabled(
result = command_runner(["launchctl", "print-disabled", domain])
if _command_returncode(result) != 0:
return None
match = re.search(rf'["\']?{re.escape(label)}["\']?\s*=>\s*(true|false)', _command_stdout(result), re.I)
# launchctl prints `=> disabled|enabled` on current macOS and `=> true|false` on older releases.
match = re.search(
rf'["\']?{re.escape(label)}["\']?\s*=>\s*(true|false|disabled|enabled)', _command_stdout(result), re.I
)
if match is None:
return False
return match.group(1).lower() == "true"
return match.group(1).lower() in {"true", "disabled"}


def verify_launchd_label_loaded(
Expand Down
1 change: 1 addition & 0 deletions tests/test_installable_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -2817,6 +2817,7 @@ def test_launchd_enable_missing_service_is_retried_after_bootstrap(tmp_path: Pat
commands = launchctl_log.read_text(encoding="utf-8").splitlines()
assert result.returncode == 0, result.stdout + result.stderr
assert [command.split()[0] for command in commands] == [
"print-disabled",
"bootout",
"print",
"enable",
Expand Down
170 changes: 170 additions & 0 deletions tests/test_launchd_hygiene.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import subprocess
from pathlib import Path

import pytest
from typer.testing import CliRunner

from brainlayer.cli import app
Expand Down Expand Up @@ -565,3 +566,172 @@ def test_launchd_installer_uses_bootstrap_not_legacy_load_unload():
assert "launchctl print" in load_plist_body
assert "launchctl load" not in load_plist_body
assert "launchctl unload" not in load_plist_body


@pytest.mark.parametrize("form", ["disabled", "true"])
@pytest.mark.parametrize("name", ["watch", "hotlane-brainbar", "enrichment"])
def test_launchd_installer_load_plist_skips_operator_disabled_label(tmp_path, name, form):
"""An operator `launchctl disable` is a standing order: load_plist must not enable/bootstrap it."""
install_source = (REPO_ROOT / "scripts/launchd/install.sh").read_text(encoding="utf-8")
load_plist_body = (
"load_plist() {" + install_source.split("\nload_plist() {", 1)[1].split("\nunload_plist() {", 1)[0]
)
helper_name = "label_disabled_by_operator() {"
assert helper_name in install_source, "install.sh must define label_disabled_by_operator()"
helper_body = helper_name + install_source.split("\n" + helper_name, 1)[1].split("\n}\n", 1)[0] + "\n}\n"

fake_bin = tmp_path / "bin"
fake_bin.mkdir()
launchctl_log = tmp_path / "launchctl.log"
fake_launchctl = fake_bin / "launchctl"
fake_launchctl.write_text(
"#!/usr/bin/env bash\n"
'printf "%s\\n" "$*" >> "$LAUNCHCTL_LOG"\n'
'[ "$1" = "print-disabled" ] && printf "%s\\n" "$LAUNCHCTL_LISTING"\n'
"exit 0\n",
encoding="utf-8",
)
listing = (
"\tdisabled services = {\n"
f'\t\t"com.brainlayer.watch" => {form}\n\t\t"com.brainlayer.hotlane-brainbar" => {form}\n'
f'\t\t"com.brainlayer.enrichment" => {form}\n\t\t"com.brainlayer.drain" => enabled\n'
"\t}"
)
fake_launchctl.chmod(0o755)
harness = tmp_path / "harness.sh"
harness.write_text(
"set -euo pipefail\n"
f'LAUNCH_DIR="{tmp_path}"\nPYTHON_BIN=/usr/bin/true\nLOAD_PLIST_SKIPPED=0\n'
+ helper_body
+ load_plist_body
+ '\nload_plist "$1"\n',
encoding="utf-8",
)
env = {
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"LAUNCHCTL_LOG": str(launchctl_log),
"LAUNCHCTL_LISTING": listing,
}

result = subprocess.run(["/bin/bash", str(harness), name], env=env, capture_output=True, text=True, check=False)

assert result.returncode == 0, result.stderr
uid = os.getuid()
assert (
f"SKIP: com.brainlayer.{name} disabled by operator (launchctl enable gui/{uid}/com.brainlayer.{name} to re-arm)"
in result.stdout
)
calls = launchctl_log.read_text(encoding="utf-8").splitlines()
assert calls == [f"print-disabled gui/{uid}"], calls


def test_launchd_primitive_reads_current_print_disabled_vocabulary():
"""macOS 14+ prints `=> disabled|enabled`; older releases print `=> true|false`. Both must be honored."""
from types import SimpleNamespace

from brainlayer.launchd_primitive import is_launchd_label_disabled

listing = (
"\tdisabled services = {\n"
'\t\t"com.brainlayer.watch" => disabled\n'
'\t\t"com.brainlayer.drain" => enabled\n'
'\t\t"com.legacy.off" => true\n'
'\t\t"com.legacy.on" => false\n'
"\t}\n"
)
runner = lambda _args: SimpleNamespace(returncode=0, stdout=listing, stderr="") # noqa: E731

assert is_launchd_label_disabled("com.brainlayer.watch", command_runner=runner) is True
assert is_launchd_label_disabled("com.brainlayer.drain", command_runner=runner) is False
assert is_launchd_label_disabled("com.legacy.off", command_runner=runner) is True
assert is_launchd_label_disabled("com.legacy.on", command_runner=runner) is False
assert is_launchd_label_disabled("com.absent", command_runner=runner) is False


def test_launchd_installer_hotlane_skip_bypasses_runtime_verification(tmp_path):
"""install.sh hotlane-brainbar on an operator-disabled label: rc 0, no verify_hotlane_runtime, no bootout."""
install_source = (REPO_ROOT / "scripts/launchd/install.sh").read_text(encoding="utf-8")

def _fn(name: str) -> str:
head = name + "() {"
return head + install_source.split("\n" + head, 1)[1].split("\n}\n", 1)[0] + "\n}\n"

fake_bin = tmp_path / "bin"
fake_bin.mkdir()
launchctl_log = tmp_path / "launchctl.log"
(fake_bin / "launchctl").write_text(
"#!/usr/bin/env bash\n"
'printf "%s\\n" "$*" >> "$LAUNCHCTL_LOG"\n'
'[ "$1" = "print-disabled" ] && printf \'\\t"com.brainlayer.hotlane-brainbar" => disabled\\n\'\n'
"exit 0\n",
encoding="utf-8",
)
(fake_bin / "launchctl").chmod(0o755)
script_dir = tmp_path / "launchd"
script_dir.mkdir()
(script_dir / "com.brainlayer.hotlane-brainbar.plist").write_text("<plist/>\n", encoding="utf-8")
verify_marker = tmp_path / "verify-ran"
harness = tmp_path / "harness.sh"
harness.write_text(
"set -euo pipefail\n"
f'SCRIPT_DIR="{script_dir}"\nLAUNCH_DIR="{tmp_path}"\nLOG_DIR="{tmp_path}"\nBRAINLAYER_LOG_DIR="{tmp_path}"\n'
"PYTHON_BIN=/usr/bin/true\nBRAINLAYER_BIN=x\nBRAINLAYER_DIR=x\nBRAINLAYER_LAUNCHD_DIR=x\nBRAINLAYER_PYTHON=x\n"
"BRAINLAYER_ENV_FILE=x\nBRAINLAYER_ENV_RUN=x\nHOTLANE_BRAINBAR_DST=x\nLOAD_PLIST_SKIPPED=0\n"
"install_hotlane_brainbar_daemon() { :; }\ninstall_env_runner() { :; }\nverify_config_file() { :; }\n"
f'verify_hotlane_runtime() {{ touch "{verify_marker}"; return 1; }}\n'
+ _fn("label_disabled_by_operator")
+ _fn("load_plist")
+ _fn("install_plist")
+ "\ninstall_plist hotlane-brainbar\n",
encoding="utf-8",
)
env = {**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}", "LAUNCHCTL_LOG": str(launchctl_log)}

result = subprocess.run(["/bin/bash", str(harness)], env=env, capture_output=True, text=True, check=False)

assert result.returncode == 0, result.stderr
assert "SKIP: com.brainlayer.hotlane-brainbar disabled by operator" in result.stdout
assert not verify_marker.exists(), "verify_hotlane_runtime must not run after an operator-disable skip"
assert launchctl_log.read_text(encoding="utf-8").splitlines() == [f"print-disabled gui/{os.getuid()}"]


def test_launchd_installer_refuses_to_load_when_disabled_state_is_unreadable(tmp_path):
"""`launchctl print-disabled` failing must fail closed: no enable, no bootstrap, rc 1 with a clear error."""
install_source = (REPO_ROOT / "scripts/launchd/install.sh").read_text(encoding="utf-8")

def _fn(name: str) -> str:
head = name + "() {"
return head + install_source.split("\n" + head, 1)[1].split("\n}\n", 1)[0] + "\n}\n"

fake_bin = tmp_path / "bin"
fake_bin.mkdir()
launchctl_log = tmp_path / "launchctl.log"
(fake_bin / "launchctl").write_text(
"#!/usr/bin/env bash\n"
'printf "%s\\n" "$*" >> "$LAUNCHCTL_LOG"\n'
'[ "$1" = "print-disabled" ] && { echo "Could not find domain" >&2; exit 1; }\n'
"exit 0\n",
encoding="utf-8",
)
(fake_bin / "launchctl").chmod(0o755)
harness = tmp_path / "harness.sh"
harness.write_text(
"set -euo pipefail\n"
f'LAUNCH_DIR="{tmp_path}"\nPYTHON_BIN=/usr/bin/true\nLOAD_PLIST_SKIPPED=0\n'
+ _fn("label_disabled_by_operator")
+ _fn("load_plist")
+ "\nload_plist watch\n",
encoding="utf-8",
)
env = {**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}", "LAUNCHCTL_LOG": str(launchctl_log)}

result = subprocess.run(["/bin/bash", str(harness)], env=env, capture_output=True, text=True, check=False)

assert result.returncode == 1
assert (
"ERROR: could not read launchd disabled state for com.brainlayer.watch "
"(launchctl print-disabled rc=1); refusing to load"
) in result.stderr
assert "SKIP:" not in result.stdout
assert launchctl_log.read_text(encoding="utf-8").splitlines() == [f"print-disabled gui/{os.getuid()}"]
Loading
Loading