-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathbot.sh
More file actions
executable file
·2023 lines (1899 loc) · 96 KB
/
Copy pathbot.sh
File metadata and controls
executable file
·2023 lines (1899 loc) · 96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# jarvis-harness: Lark/Feishu bot + heartbeat-driven personal AI agent
#
# - Listens for user messages on Lark
# - Heartbeat loop runs scheduled tasks (feed triage, memory consolidation, etc.)
# - Claude Code handles conversation with full memory injection
#
# Configuration: jarvis.yaml (copy from jarvis.example.yaml)
set -uo pipefail
# Ensure UTF-8 encoding for Chinese content processing
export LANG="${LANG:-en_US.UTF-8}"
export LC_ALL="${LC_ALL:-en_US.UTF-8}"
JARVIS_DIR="$(cd "$(dirname "$0")" && pwd)"
export JARVIS_DIR
# Ensure the native-installer `claude` (~/.local/bin/claude → versions/<x>) is on
# PATH for EVERY child — most importantly the heartbeat loop, which shells out to
# `claude` each cycle. launchd starts the daemon with a minimal PATH that omits
# ~/.local/bin, so this MUST be exported before any child is spawned. Previously
# this lived further down (after the heartbeat launch), so on a launchd cold-start
# the heartbeat inherited the minimal PATH and every claude_call died with
# "Claude CLI not found", tripping non-priority circuits until the next restart.
export PATH="$HOME/.local/bin:$PATH"
# Anchor CWD to JARVIS_DIR. The bg helpers run as `python3 -m core.X`, which
# resolves `core/` from the CWD — if bot.sh is launched (or self-exec'd) from
# any other directory (e.g. a restart kicked off from WORK_DIR), every helper
# dies with `ModuleNotFoundError: No module named 'core'`, taking heartbeat and
# the ef-stream down and spiralling into a restart loop. Anchoring here makes
# that impossible regardless of how/where we were launched.
cd "$JARVIS_DIR" || { echo "FATAL: cannot cd to JARVIS_DIR ($JARVIS_DIR)" >&2; exit 1; }
# ── Single-instance lock (prevent duplicate replies) ────────────────
# PID file format: "PID BOOT_TIMESTAMP" — validates both PID liveness AND
# that the PID belongs to the same boot cycle (guards against PID reuse).
PIDFILE="$JARVIS_DIR/.bot.pid"
_BOOT_TS=$(date +%s)
if [ -f "$PIDFILE" ]; then
old_pid=$(awk '{print $1}' "$PIDFILE" 2>/dev/null)
old_ts=$(awk '{print $2}' "$PIDFILE" 2>/dev/null)
if [ -n "$old_pid" ] && kill -0 "$old_pid" 2>/dev/null; then
# Extra check: if PID file is older than 7 days, it's likely a reused PID
if [ -n "$old_ts" ] && [ "$(($(date +%s) - old_ts))" -lt 604800 ]; then
echo "bot.sh is already running (PID $old_pid, started $(( ($(date +%s) - old_ts) / 60 ))m ago). Exiting." >&2
exit 1
fi
# PID file too old — likely stale (PID reused by another process)
echo "WARN: Stale PID file (PID $old_pid, age > 7 days) — overriding" >&2
fi
rm -f "$PIDFILE"
fi
echo "$$ $_BOOT_TS" > "$PIDFILE"
# ── Process conflict detection ──────────────────────────────────────
# Detect competing eigenflux stream processes from openclaw-gateway or
# stale bot instances. Multiple streams cause "Connection replaced" loops.
# Match only actual eigenflux stream processes (not Claude prompts containing the string)
_competing_streams=$(ps -eo pid,comm,args | awk '$2 == "eigenflux" && $4 == "stream" {print $1}' | wc -l | tr -d ' ')
if [ "$_competing_streams" -gt 0 ]; then
echo "WARN: Found $_competing_streams competing eigenflux stream process(es) — killing" >&2
ps -eo pid,comm,args | awk '$2 == "eigenflux" && $4 == "stream" {print $1}' | xargs kill 2>/dev/null || true
sleep 1
fi
LOG_FILE="$JARVIS_DIR/jarvis.log"
LOG_MAX_BYTES=500000 # 500KB — rotate on startup if exceeded
MEMORY_CACHE_FILE="$JARVIS_DIR/.memory_cache" # last-known-good memory snapshot
# ── Log rotation (on startup) ────────────────────────────────────────
# Archive 3 generations before truncating — destroyed history made
# failure-rate audits impossible.
if [ -f "$LOG_FILE" ] && [ "$(stat -f%z "$LOG_FILE" 2>/dev/null || stat -c%s "$LOG_FILE" 2>/dev/null || echo 0)" -gt "$LOG_MAX_BYTES" ]; then
mv -f "$LOG_FILE.2" "$LOG_FILE.3" 2>/dev/null || true
mv -f "$LOG_FILE.1" "$LOG_FILE.2" 2>/dev/null || true
cp -f "$LOG_FILE" "$LOG_FILE.1" 2>/dev/null || true
tail -500 "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"
fi
# ── Logging ──────────────────────────────────────────────────────────
# All log messages go to jarvis.log AND stderr. They NEVER go to stdout,
# which is reserved for user-facing output that may be sent to Lark.
log() {
local level="$1"; shift
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
echo "$msg" >&2
echo "$msg" >> "$LOG_FILE"
}
log_err() { log ERROR "$@"; }
log_warn() { log WARN "$@"; }
log_info() { log INFO "$@"; }
# ── Portable timeout (macOS has no 'timeout' by default) ─────────────
if command -v timeout >/dev/null 2>&1; then
TIMEOUT_CMD="timeout"
elif command -v gtimeout >/dev/null 2>&1; then
TIMEOUT_CMD="gtimeout"
else
TIMEOUT_CMD=""
fi
# Run a command with a timeout. Usage: with_timeout <seconds> <cmd> <args...>
# On macOS without coreutils, falls back to no timeout (Python layer handles it).
with_timeout() {
local secs="$1"; shift
if [ -n "$TIMEOUT_CMD" ]; then
"$TIMEOUT_CMD" "$secs" "$@"
else
# Pure-bash fallback. macOS ships no coreutils timeout, so this branch
# used to run the command with NO limit — a placebo for every caller
# (6000s bg jobs, 12s narration; a hung narration could wedge the
# watchdog loop). Run in bg, kill past the deadline.
# The killer's stdout MUST be detached: inside $(...) substitution it
# would otherwise hold the pipe open for the full sleep.
"$@" &
local _wt_cmd_pid=$!
( sleep "$secs"; kill "$_wt_cmd_pid" 2>/dev/null ) >/dev/null 2>&1 &
local _wt_killer_pid=$!
wait "$_wt_cmd_pid"
local _wt_rc=$?
kill "$_wt_killer_pid" 2>/dev/null
wait "$_wt_killer_pid" 2>/dev/null
return $_wt_rc
fi
}
# ── Prerequisite check ──────────────────────────────────────────────
# Fail early with actionable errors instead of cryptic failures deep in
# the Python eval or lark-cli subshell.
missing_dep() {
echo "ERROR: required dependency not found: $1" >&2
echo " install: $2" >&2
exit 1
}
command -v python3 >/dev/null 2>&1 || missing_dep "python3" "brew install python3 (macOS) / apt install python3 (Linux)"
command -v jq >/dev/null 2>&1 || missing_dep "jq" "brew install jq (macOS) / apt install jq (Linux)"
python3 -c "import yaml" >/dev/null 2>&1 || missing_dep "python3 yaml module" \
"pip install pyyaml (or run: ./setup.sh)"
# ── Load config via core/config.py (single source of truth) ──────────
# Uses shlex.quote() so config values are never shell-interpolated.
CONFIG_FILE="$JARVIS_DIR/jarvis.yaml"
if [ ! -f "$CONFIG_FILE" ]; then
echo "ERROR: jarvis.yaml not found." >&2
echo " Run: cp jarvis.example.yaml jarvis.yaml && \$EDITOR jarvis.yaml" >&2
echo " Or: ./setup.sh (does it + more)" >&2
exit 1
fi
CONFIG_VARS=$(JARVIS_DIR="$JARVIS_DIR" CONFIG_FILE="$CONFIG_FILE" python3 - <<'PYEOF'
import os, shlex, sys
sys.path.insert(0, os.environ["JARVIS_DIR"])
from core.config import Config
c = Config(os.environ["CONFIG_FILE"])
def emit(name, value):
print(f"{name}={shlex.quote(str(value))}")
emit("USER_ID", c.lark.get("user_id", ""))
emit("APP_ID", c.lark.get("app_id", ""))
# Event backend switch + sidecar credentials (jarvis.yaml is gitignored, so
# the secret never reaches the repo). Empty values keep the lark-cli path.
emit("JARVIS_EVENT_BACKEND", c.lark.get("event_backend", ""))
emit("LARK_APP_SECRET", c.lark.get("app_secret", ""))
emit("DATA_DIR", c.data_dir)
emit("WORK_DIR", c.work_dir)
emit("MEMORY_DIR", c.memory_dir)
emit("MAX_SESSION_SIZE", c.claude.get("max_session_size", 512000))
emit("MAIN_MODEL", c.claude.get("main_model", "opus") or "opus")
emit("HEARTBEAT_MODEL", c.claude.get("heartbeat_model", "opus"))
emit("HEARTBEAT_TIMEOUT", c.claude.get("heartbeat_timeout", 600))
emit("CLAUDE_BACKUP_ENABLED", str(bool(c.claude.get("backup_enabled", True))).lower())
emit("CLAUDE_BACKUP_AUTH_TOKEN", c.claude.get("backup_auth_token", ""))
emit("CLAUDE_BACKUP_BASE_URL", c.claude.get("backup_base_url", ""))
emit("OPENAI_FALLBACK_ENABLED", str(bool(c.openai.get("fallback_enabled", True))).lower())
emit("OPENAI_FALLBACK_MODEL", c.openai.get("fallback_model", "gpt-5.2"))
emit("OPENAI_API_KEY_CONFIG", c.openai.get("api_key", ""))
emit("OPENAI_BASE_URL", c.openai.get("base_url", "https://api.openai.com/v1"))
emit("OPENAI_USER_AGENT", c.openai.get("user_agent", ""))
emit("OPENAI_FALLBACK_TIMEOUT", c.openai.get("timeout", 120))
emit("OPENAI_FALLBACK_MAX_OUTPUT_TOKENS", c.openai.get("max_output_tokens", 4096))
emit("CHECK_INTERVAL", c.heartbeat.get("check_interval", 10))
emit("ADMIN_ENABLED", str(bool(c.admin.get("enabled", False))).lower())
emit("ADMIN_HOST", c.admin.get("host", "127.0.0.1"))
emit("ADMIN_PORT", c.admin.get("port", 3456))
PYEOF
)
# shellcheck disable=SC1090
eval "$CONFIG_VARS"
# Never let the main conversation inherit the account's DEFAULT claude model:
# that default can be a banned model (Fable). Pin to opus (= Opus 4.8) so an
# empty/missing config or a changed account default can never break the bot.
: "${MAIN_MODEL:=opus}"
# Claude Code stores sessions in ~/.claude/projects/<slug>/, where <slug>
# is the absolute cwd with every '/' replaced by '-' (leading dash kept).
CLAUDE_PROJECT_DIR="$HOME/.claude/projects/$(echo "$WORK_DIR" | sed 's|/|-|g')"
SESSION_TRACKER="$JARVIS_DIR/active_sessions.json"
HEARTBEAT_TRIGGER="/tmp/jarvis-heartbeat-trigger"
export MEMORY_DIR WORK_DIR CLAUDE_PROJECT_DIR USER_ID LOG_FILE MAIN_MODEL HEARTBEAT_MODEL HEARTBEAT_TIMEOUT CHECK_INTERVAL
export CLAUDE_BACKUP_ENABLED CLAUDE_BACKUP_AUTH_TOKEN CLAUDE_BACKUP_BASE_URL
export OPENAI_FALLBACK_ENABLED OPENAI_FALLBACK_MODEL OPENAI_BASE_URL OPENAI_USER_AGENT OPENAI_FALLBACK_TIMEOUT OPENAI_FALLBACK_MAX_OUTPUT_TOKENS
if [ -z "${OPENAI_API_KEY:-}" ] && [ -n "${OPENAI_API_KEY_CONFIG:-}" ]; then
export OPENAI_API_KEY="$OPENAI_API_KEY_CONFIG"
fi
unset OPENAI_API_KEY_CONFIG
# Sidecar event backend (empty = lark-cli default; see plugins/lark/client.sh)
export JARVIS_EVENT_BACKEND LARK_APP_SECRET
log_info "Starting jarvis-harness..."
log_info " JARVIS_DIR: $JARVIS_DIR"
log_info " DATA_DIR: $DATA_DIR"
log_info " WORK_DIR: $WORK_DIR"
log_info " CLAUDE_DIR: $CLAUDE_PROJECT_DIR"
log_info " USER_ID: ${USER_ID:-(not set, IM disabled)}"
log_info " TIMEOUT: ${TIMEOUT_CMD:-(unavailable, Python layer handles)}"
# Lark is configured but lark-cli missing → log a warning but don't abort.
# The bot will still run heartbeat-only.
if [ -n "$USER_ID" ] && ! command -v lark-cli >/dev/null 2>&1; then
log_warn "lark.user_id is set but lark-cli is not installed (install: npm i -g @larksuite/cli)"
log_warn "Bot will run in heartbeat-only mode. Configure lark-cli or remove lark.user_id."
USER_ID="" # degrade to headless
fi
JOBS_DIR="$JARVIS_DIR/jobs"
MAX_HANDLERS=5 # max concurrent message handlers
mkdir -p "$DATA_DIR" "$MEMORY_DIR" "$MEMORY_DIR/hot" "$MEMORY_DIR/warm" \
"$MEMORY_DIR/timeline" "$MEMORY_DIR/system" "$JARVIS_DIR/eigenflux" \
"$JOBS_DIR" "$JARVIS_DIR/session_compacts"
[ -f "$SESSION_TRACKER" ] || echo "{}" > "$SESSION_TRACKER"
# Clean up stale session locks from previous crashes/restarts
rm -f "$JARVIS_DIR"/.session_lock_* 2>/dev/null
# Clean up old Claude session files (>30 days, prevents unbounded disk growth)
if [ -d "$CLAUDE_PROJECT_DIR" ]; then
_old_sessions=$(find "$CLAUDE_PROJECT_DIR" -name "*.jsonl" -mtime +30 2>/dev/null | wc -l | tr -d ' ')
if [ "$_old_sessions" -gt 0 ]; then
find "$CLAUDE_PROJECT_DIR" -name "*.jsonl" -mtime +30 -delete 2>/dev/null
log_info "Cleaned $_old_sessions session files older than 30 days"
fi
fi
# ── Load built-in plugins ────────────────────────────────────────────
# Lark (Feishu) — shell helpers for IM. Other plugins (e.g. eigenflux)
# are Python libraries loaded directly by tasks/*.py scripts.
# shellcheck source=plugins/lark/client.sh
. "$JARVIS_DIR/plugins/lark/client.sh"
# ── Session management ────────────────────────────────────────────────
# Passes conv_key via env to avoid shell-injection into the Python source.
# Delegates to core.session.SessionManager: the old embedded copy did an
# UNLOCKED, non-atomic read-modify-write of the tracker, racing
# force_rotate's flock+atomic writes (a concurrent rotation could be
# clobbered, and a torn write parsed as {} reset every counter). Same
# uuid5(NAMESPACE, key-counter) scheme, so session ids are unchanged.
get_session_id() {
local conv_key="$1"
JV_TRACKER="$SESSION_TRACKER" JV_SDIR="$CLAUDE_PROJECT_DIR" \
JV_MAX="$MAX_SESSION_SIZE" JV_KEY="$conv_key" python3 <<'PYEOF'
import os, sys
sys.path.insert(0, os.environ["JARVIS_DIR"])
from core.session import SessionManager
sm = SessionManager(os.environ["JV_TRACKER"], os.environ["JV_SDIR"],
max_size=int(os.environ["JV_MAX"]))
sid, rotated = sm.get_session(os.environ["JV_KEY"])
if rotated:
print("ROTATED", file=sys.stderr)
print(sid)
PYEOF
}
# ── Memory loader with last-known-good cache ─────────────────────────
# If Python fails, we reuse the cached snapshot so Claude never gets an
# empty memory string (which causes the bot to "forget" everything).
load_memory() {
local fresh
fresh=$(python3 -c "
import os, sys; sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.memory import load_tiered_memory
print(load_tiered_memory(os.environ['MEMORY_DIR']))
" 2>>"$LOG_FILE")
if [ -n "$fresh" ]; then
printf '%s' "$fresh" > "$MEMORY_CACHE_FILE"
printf '%s' "$fresh"
elif [ -s "$MEMORY_CACHE_FILE" ]; then
log_warn "Memory load returned empty — using cached snapshot"
cat "$MEMORY_CACHE_FILE"
else
log_err "Memory load failed and no cache available"
printf '%s' "## Memory unavailable\n(memory loader failed — Claude is operating without context)"
fi
}
# ── Send to Lark (only user-facing content — never errors) ────────────
# Thin wrapper around the Lark plugin's lark_send, with a local log line.
send_to_lark() {
local content="$1"
[ -z "$content" ] && return
if ! lark_send "$content"; then
log_warn "Failed to send message to Lark"
fi
}
# ── Check if Claude output looks like an error (reuses core/safety.py) ─
looks_like_error() {
JV_TEXT="$1" python3 -c "
import os, sys
sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.safety import looks_like_error
sys.exit(0 if looks_like_error(os.environ.get('JV_TEXT','')) else 1)
" 2>/dev/null
}
# ── Heartbeat Loop (background, Python) ─────────────────────────────
# The heartbeat loop is now in Python (core/heartbeat_loop.py) where it
# can be tested. bot.sh only launches it as a background process.
# All output routing, card/text splitting, outbox writing, engagement
# tracking, and restart detection is in Python — no more bash set -u traps.
# ── Replay dropped messages from restart ────────────────────────────
# If a restart killed in-flight handlers, notify the user so they know
# to resend. We can't replay the original content (it's lost with the
# process), but we CAN tell them which sessions were interrupted.
_queue_file="$JARVIS_DIR/.message_queue"
if [ -f "$_queue_file" ] && [ -s "$_queue_file" ] && [ -n "$USER_ID" ]; then
_dropped=$(wc -l < "$_queue_file" | tr -d ' ')
log_warn "Found $_dropped interrupted message(s) from restart — notifying user"
lark_send "⚠️ 重启中断了 ${_dropped} 条正在处理的消息,请重新发送。" 2>/dev/null || true
rm -f "$_queue_file"
fi
sleep 3 # let config load settle
python3 -m core.heartbeat_loop 2>>"$LOG_FILE" &
HEARTBEAT_PID=$!
# ── EigenFlux Real-Time Stream (background, Python) ─────────────────
# The stream loop is now in Python (core/ef_stream_loop.py) where it
# can be tested. Handles reconnect, backoff, message delivery, analysis.
# PATH (incl. ~/.local/bin) is exported at the top so every child — heartbeat,
# this stream, admin — can find the native-installer `claude`.
# Identify jarvis to EigenFlux server telemetry (same contract as client.sh).
# The `eigenflux stream` child inherits these via the Python process env.
EIGENFLUX_HOST="${EIGENFLUX_HOST:-jarvis}" EIGENFLUX_CHANNEL="${EIGENFLUX_CHANNEL:-lark}" \
LOG_FILE="$LOG_FILE" python3 -m core.ef_stream_loop 2>>"$LOG_FILE" &
STREAM_PID=$!
log_info "Heartbeat started (PID: $HEARTBEAT_PID)"
log_info "EigenFlux stream started (PID: $STREAM_PID)"
# ── Admin Console (optional, background) ─────────────────────────────
ADMIN_PID=""
if [ "$ADMIN_ENABLED" = "true" ]; then
python3 "$JARVIS_DIR/admin.py" >>"$LOG_FILE" 2>&1 &
ADMIN_PID=$!
log_info "Admin started (PID: $ADMIN_PID) — http://$ADMIN_HOST:$ADMIN_PORT"
else
log_info "Admin disabled (set admin.enabled: true in jarvis.yaml to enable)"
fi
# ── Action Post-Processor ────────────────────────────────────────────
# Scans Claude's reply for [ACTION:...] markers, executes them, and
# returns the cleaned reply with any action results appended.
# Usage: cleaned_reply=$(process_actions "$reply" "$conv_key" "$message_id")
process_actions() {
local reply="$1" conv_key="$2" message_id="$3"
local action_results=""
# Extract all action markers
local actions
actions=$(echo "$reply" | grep -o '\[ACTION:[^]]*\]' 2>/dev/null || true)
if [ -z "$actions" ]; then
printf '%s' "$reply"
return
fi
# ── Python-handled actions (calendar, task, intent, feed, watchlater, etc.) ──
# Delegate to core/actions.py for all non-process-control actions.
# This handles: feed_search, watchlater, heartbeat, calendar_*, task_*,
# praxis_*, intent_*, schedule_task. Returns cleaned reply with results.
reply=$(JV_REPLY="$reply" JV_LOG_FILE="$LOG_FILE" python3 -c "
import os, sys; sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.actions import ActionProcessor
ap = ActionProcessor(os.environ['JARVIS_DIR'], os.environ['MEMORY_DIR'],
os.environ.get('JV_JOBS_DIR', 'jobs'), os.environ.get('JV_LOG_FILE', ''))
print(ap.process(os.environ['JV_REPLY']))
" 2>>"$LOG_FILE" || printf '%s' "$reply")
# ── Bash-handled actions (need process control: &, wait, PID tracking) ──
# Re-extract remaining markers (Python already stripped the ones it handled)
actions=$(echo "$reply" | grep -o '\[ACTION:[^]]*\]' 2>/dev/null || true)
if [ -z "$actions" ]; then
printf '%s' "$reply"
return
fi
while IFS= read -r marker; do
[ -z "$marker" ] && continue
local action_body="${marker#\[ACTION:}"
action_body="${action_body%\]}"
local action_type="${action_body%%|*}"
local action_params="${action_body#*|}"
[ "$action_params" = "$action_type" ] && action_params=""
case "$action_type" in
bg)
local bg_prompt="${action_params#prompt=}"
local bg_desc="${bg_prompt:0:80}"
local job_id
job_id=$(JV_JOBS_DIR="$JOBS_DIR" JV_CONV_KEY="$conv_key" \
JV_DESC="$bg_desc" JV_MSG_ID="$message_id" \
python3 "$JARVIS_DIR/core/jobs.py" create 2>>"$LOG_FILE" || echo "")
if [ -n "$job_id" ]; then
log_info "[bg:$job_id] Started via action: $bg_desc"
run_background_job "$job_id" "$conv_key" "$bg_prompt" "$message_id" &
# Push a "started" card immediately so the user SEES the bg job kick
# off (the completion card comes later from run_background_job).
# Body via JV_BODY env to avoid shell expansion of backticks/$ —
# same safe pattern as the completion card below.
local _bg_start_body _bg_start_card
_bg_start_body="正在后台独立运行,不占用我们的对话;跑完我会把结果卡片发给你。
**任务**:$bg_desc
**Job ID**:\`$job_id\`
查进度发「jobs」,查结果发「job output $job_id」,取消发「cancel $job_id」"
_bg_start_card=$(JV_BODY="$_bg_start_body" python3 -c "
import os, sys; sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.card import build_card
print(build_card('🚀 后台任务已启动', os.environ['JV_BODY']))
" 2>>"$LOG_FILE") || _bg_start_card=""
if [ -n "$_bg_start_card" ]; then
lark_send_card "$_bg_start_card"
else
send_to_lark "🚀 后台任务已启动:$bg_desc (Job $job_id)"
fi
action_results="${action_results}
🚀 已在后台启动:$bg_desc"
fi
;;
jobs)
local jobs_output
jobs_output=$(JV_JOBS_DIR="$JOBS_DIR" JV_CONV_KEY="$conv_key" \
python3 "$JARVIS_DIR/core/jobs.py" list 2>>"$LOG_FILE" || echo "Failed to list jobs")
action_results="${action_results}
${jobs_output}"
;;
job_cancel)
local cancel_id="${action_params#id=}"
local cancel_result
cancel_result=$(JV_JOBS_DIR="$JOBS_DIR" \
python3 "$JARVIS_DIR/core/jobs.py" cancel "$cancel_id" 2>>"$LOG_FILE" || echo "error")
if [ "$cancel_result" = "cancelled" ]; then
action_results="${action_results}
Job $cancel_id cancelled."
else
action_results="${action_results}
Job not found or not running: $cancel_id"
fi
;;
job_output)
local out_id="${action_params#id=}"
local out_file="$JOBS_DIR/${out_id}/output.md"
if [ -f "$out_file" ]; then
local out_content
out_content=$(cat "$out_file" 2>/dev/null)
if [ ${#out_content} -gt 4000 ]; then
out_content="${out_content:0:4000}
... (truncated)"
fi
action_results="${action_results}
${out_content}"
else
action_results="${action_results}
No output found for job: $out_id"
fi
;;
# All other action types (heartbeat, calendar_*, task_*, praxis_*, intent_*, etc.)
# are handled by core/actions.py above. If any marker reaches here, it's unknown.
*)
log_warn "[action] Unknown action type in bash fallback: $action_type"
;;
esac
done <<< "$actions"
# Strip all [ACTION:...] markers and collapse blank lines (macOS-safe)
local cleaned
cleaned=$(echo "$reply" | sed 's/\[ACTION:[^]]*\]//g' | grep -v '^[[:space:]]*$' || true)
# Append action results if any
if [ -n "$action_results" ]; then
printf '%s\n%s' "$cleaned" "$action_results"
else
printf '%s' "$cleaned"
fi
}
# ── Message Handler (runs in background subshell) ────────────────────
# Extracted from the main loop so different conversations run in parallel.
# Same-session messages serialize via the existing lock file mechanism.
handle_message() {
local conv_key="$1" content="$2" message_id="$3" session_id="$4"
local reaction_id="$5"
# Prepend an authoritative current-time line to the user's message body.
# The system prompt already carries 'Current time', but the message body
# itself has none — in a long all-day thread Claude can anchor on a stale
# in-conversation timestamp. Single line at home (Shanghai), dual when abroad.
local msg_ts
msg_ts=$(python3 -c "import os,sys; sys.path.insert(0,os.environ['JARVIS_DIR']); from core.timeutil import msg_timestamp_prefix; print(msg_timestamp_prefix())" 2>>"$LOG_FILE")
if [ -n "$msg_ts" ]; then
content="$msg_ts
$content"
fi
# Build system prompt with memory + recent turns (delegated to core/prompt.py)
local sys_prompt
sys_prompt=$(JV_TRACKER="$SESSION_TRACKER" JV_KEY="$conv_key" \
JV_SID="$session_id" JV_SDIR="$CLAUDE_PROJECT_DIR" \
python3 -c "
import os, sys; sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.prompt import build_system_prompt
from core.timeutil import now_local_str
print(build_system_prompt(
jarvis_dir=os.environ['JARVIS_DIR'],
memory_dir=os.environ['MEMORY_DIR'],
session_dir=os.environ.get('JV_SDIR', ''),
session_id=os.environ.get('JV_SID', ''),
conv_key=os.environ.get('JV_KEY', ''),
now_ts=now_local_str('%Y-%m-%d %H:%M %A'),
tracker_path=os.environ.get('JV_TRACKER', 'active_sessions.json'),
))
" 2>>"$LOG_FILE")
if [ -z "$sys_prompt" ]; then
log_warn "[$session_id] System prompt build failed — using fallback"
sys_prompt="You are a personal assistant. Reply in the same language the user uses.
$(load_memory)"
fi
# Call Claude (runs in WORK_DIR, with optional timeout)
# Wait for any existing session lock (previous message still processing)
local LOCK_FILE="$JARVIS_DIR/.session_lock_${session_id}"
local ANSWER_FILE
ANSWER_FILE=$(mktemp /tmp/jarvis-answer-XXXXXX)
local SYS_PROMPT_FILE="${ANSWER_FILE}.system_prompt"
printf '%s' "$sys_prompt" > "$SYS_PROMPT_FILE"
# Lock ownership token: every write to the lock file carries it, and every
# destructive operation (overwrite at spawn, cleanup rm) first verifies the
# token is still ours. Without ownership checks, a waiter that legitimately
# reclaimed a stale lock could be silently dispossessed by the original
# handler's blind writes (the 2026-06-11 review found three such races).
# Lock file format: "<pid-or-'acquiring'> <token>" — readers that kill the
# holder (restart.sh, /stop, staleness checks) take the FIRST field.
local _lock_token
_lock_token="$$.$(date +%s).$RANDOM"
local waited=0
local _busy_notice_sent=0
while :; do
# Acquire atomically (noclobber) BEFORE spawning Claude. Stale-lock
# policy: a NUMERIC dead pid → reclaim immediately; an alive holder is
# waited out up to 6200s (the 6000s watchdog ceiling governs in-flight
# calls — a shorter cutoff here used to break the lock of legitimately
# long-running handlers).
until (set -C; printf 'acquiring %s' "$_lock_token" > "$LOCK_FILE") 2>/dev/null; do
_lock_holder=$(awk '{print $1}' "$LOCK_FILE" 2>/dev/null)
case "$_lock_holder" in
''|*[!0-9]*) : ;; # placeholder — treat as alive (just acquired)
*)
if ! kill -0 "$_lock_holder" 2>/dev/null; then
log_warn "[$session_id] Lock holder PID $_lock_holder is dead — reclaiming stale lock"
rm -f "$LOCK_FILE"
waited=0
continue
fi ;;
esac
if [ "$waited" -ge 6200 ]; then
log_warn "[$session_id] Lock wait exceeded watchdog ceiling (${waited}s) — force-clearing"
rm -f "$LOCK_FILE"
waited=0
continue
fi
if [ "$((waited % 30))" -eq 0 ] && [ "$waited" -gt 0 ]; then
log_info "[$session_id] Session busy, waiting... (${waited}s)"
if [ "$_busy_notice_sent" -eq 0 ]; then
_busy_notice_sent=1
lark_reply_text "$message_id" "前一条还在处理,我已把这条排队;轮到它时会继续,不需要重发。" >/dev/null 2>&1 || true
fi
fi
sleep 5
waited=$((waited + 5))
done
# Re-resolve after acquiring: the conversation may have rotated to a new
# session while we waited (background-job auto-promotion does this). Our
# session_id was resolved at dispatch time — resuming it now would write
# into the transcript the promoted Claude is still appending to.
_cur_sid=$(JV_TRACKER="$SESSION_TRACKER" JV_KEY="$conv_key" python3 -c "
import json, os
try:
print(json.load(open(os.environ['JV_TRACKER'])).get(os.environ['JV_KEY'], {}).get('session_id', ''))
except Exception:
print('')
" 2>/dev/null || echo "")
if [ -n "$_cur_sid" ] && [ "$_cur_sid" != "$session_id" ]; then
log_info "[$session_id] Conversation rotated to $_cur_sid while waiting — switching"
rm -f "$LOCK_FILE"
session_id="$_cur_sid"
LOCK_FILE="$JARVIS_DIR/.session_lock_${session_id}"
continue
fi
break
done
# Run Claude with automatic retry on empty response
local session_file="$CLAUDE_PROJECT_DIR/${session_id}.jsonl"
local _claude_pid
local answer=""
local _attempt
local _watchdog_killed=0 # set to 1 only when the 6000s watchdog did the kill
local _use_claude_backup=0
local _claude_backup_tried=0
local _openai_tried=0
local _answer_provider=""
local _answer_model=""
# REQ-77: the model to use this attempt. Degrades (opus→sonnet→haiku) if a
# spawn fails with a model-unavailable / spend-limit stderr, instead of
# looping to empty death ("Continue / No response requested").
local _cur_model="$MAIN_MODEL"
for _attempt in 1 2 3 4; do
if [ "$_attempt" -gt 1 ]; then
log_info "[$session_id] Retry attempt $_attempt after empty response (sleeping 3s)"
sleep 3
fi
# Ownership check before every spawn: if the lock no longer carries our
# token (a waiter reclaimed it as stale, or promotion released it and
# someone else acquired), we must NOT touch this session again.
if ! grep -q "$_lock_token" "$LOCK_FILE" 2>/dev/null; then
log_warn "[$session_id] Lost lock ownership before attempt $_attempt — aborting retries"
answer=""
break
fi
if [ -f "$session_file" ]; then
[ "$_attempt" -eq 1 ] && log_info "[$session_id] Resuming session"
if [ "$_use_claude_backup" -eq 1 ]; then
log_warn "[$session_id] Calling Claude Code backup provider model=$_cur_model"
(cd "$WORK_DIR" && printf '%s' "$content" | env \
ANTHROPIC_AUTH_TOKEN="$CLAUDE_BACKUP_AUTH_TOKEN" \
ANTHROPIC_BASE_URL="$CLAUDE_BACKUP_BASE_URL" \
claude -p \
--resume "$session_id" \
--model "$_cur_model" \
--append-system-prompt "$sys_prompt" \
--dangerously-skip-permissions \
--output-format json \
2>"${ANSWER_FILE}.stderr" > "$ANSWER_FILE") &
else
log_info "[$session_id] Calling primary Claude Code model=$_cur_model"
(cd "$WORK_DIR" && printf '%s' "$content" | claude -p \
--resume "$session_id" \
--model "$_cur_model" \
--append-system-prompt "$sys_prompt" \
--dangerously-skip-permissions \
--output-format json \
2>"${ANSWER_FILE}.stderr" > "$ANSWER_FILE") &
fi
else
[ "$_attempt" -eq 1 ] && log_info "[$session_id] New session"
if [ "$_use_claude_backup" -eq 1 ]; then
log_warn "[$session_id] Calling Claude Code backup provider model=$_cur_model"
(cd "$WORK_DIR" && printf '%s' "$content" | env \
ANTHROPIC_AUTH_TOKEN="$CLAUDE_BACKUP_AUTH_TOKEN" \
ANTHROPIC_BASE_URL="$CLAUDE_BACKUP_BASE_URL" \
claude -p \
--session-id "$session_id" \
--model "$_cur_model" \
--append-system-prompt "$sys_prompt" \
--dangerously-skip-permissions \
--output-format json \
2>"${ANSWER_FILE}.stderr" > "$ANSWER_FILE") &
else
log_info "[$session_id] Calling primary Claude Code model=$_cur_model"
(cd "$WORK_DIR" && printf '%s' "$content" | claude -p \
--session-id "$session_id" \
--model "$_cur_model" \
--append-system-prompt "$sys_prompt" \
--dangerously-skip-permissions \
--output-format json \
2>"${ANSWER_FILE}.stderr" > "$ANSWER_FILE") &
fi
fi
_claude_pid=$!
printf '%s %s' "$_claude_pid" "$_lock_token" > "$LOCK_FILE"
# Live activity stream: poll session file every 20s, send new tool calls to user
# Also acts as watchdog: kills Claude after 6000s
(_session_jsonl="$CLAUDE_PROJECT_DIR/${session_id}.jsonl"
# Snapshot current tool count so we only report NEW tools from this call
_last_tool_count=$(python3 -c "
import json
n=0
try:
with open('$CLAUDE_PROJECT_DIR/${session_id}.jsonl') as f:
for line in f:
for b in (json.loads(line).get('message',{}).get('content',[]) or []):
if isinstance(b,dict) and b.get('type')=='tool_use': n+=1
except: pass
print(n)
" 2>/dev/null || echo 0)
_elapsed=0
# Responsiveness policy is single-sourced + tested in core/responsiveness
# (REQ-59). Pull the tuned constants here; fall back to literals if the
# module call ever fails so the loop never breaks.
eval "$(python3 -m core.responsiveness env 2>/dev/null)"
: "${JV_POLL_FIRST:=6}" "${JV_POLL_STEADY:=20}"
: "${JV_THINKING_ACK:=💭 收到了,正在想。}"
# First poll fast (~6s) so the user sees a sign of life quickly, then
# settle to 20s to avoid spam. The instant "Typing" reaction already
# fired at dispatch; this loop adds the FIRST textual feedback within
# ~6s — either a tool narration (🔧) or, when opus is just thinking with
# no tool calls, a one-time "received, thinking" note so the long
# generation (median ~100s on opus) isn't dead silence.
_poll="$JV_POLL_FIRST"
_thinking_sent=0
while [ "$_elapsed" -lt 6000 ]; do
sleep "$_poll"
_elapsed=$((_elapsed + _poll))
_poll="$JV_POLL_STEADY"
if ! kill -0 $_claude_pid 2>/dev/null; then break; fi
# Extract tool call descriptions, compare with last snapshot
_new_tools=$(python3 -c "
import json, sys
descs = []
try:
with open('$_session_jsonl') as f:
for line in f:
obj = json.loads(line)
for block in (obj.get('message',{}).get('content',[]) or []):
if isinstance(block,dict) and block.get('type')=='tool_use':
inp = block.get('input',{})
d = inp.get('description','')
if not d:
name = block.get('name','')
path = inp.get('file_path','')
cmd = inp.get('command','')[:50]
pattern = inp.get('pattern','')[:30]
if path: d = f'{name}: {path.split(\"/\")[-1]}'
elif cmd: d = f'{name}: {cmd}'
elif pattern: d = f'{name}: {pattern}'
else: d = name
descs.append(d[:60])
except: pass
# Output: total_count then new descriptions (after offset)
offset = int(sys.argv[1]) if len(sys.argv)>1 else 0
print(len(descs))
for d in descs[offset:]:
print(d)
" "$_last_tool_count" 2>/dev/null)
_new_count=$(echo "$_new_tools" | head -1)
_new_descs=$(echo "$_new_tools" | tail -n +2)
if [ -n "$_new_descs" ] && [ "$_new_count" -gt "$_last_tool_count" ] 2>/dev/null; then
_thinking_sent=1 # tool narration IS the sign of life — suppress the thinking note
_formatted=$(echo "$_new_descs" | while IFS= read -r _d; do
[ -n "$_d" ] && echo "• $_d"
done)
if [ -n "$_formatted" ]; then
# REQ-18 (user-designed 5/29): narrate progress with a fast cheap
# model instead of dumping raw tool names — "它搜了什么网站、有没有
# 真的去看,还是在幻觉,对用户是非常重要的信息". Throttled to ≥60s
# between narrations; raw list is the fallback on any failure.
_now_s=$(date +%s)
if [ $((_now_s - ${_last_narrate:-0})) -ge 60 ]; then
_last_narrate=$_now_s
# Narrate in a BACKGROUND fork: an inline call (even with a real
# timeout) blocks this poll loop, delaying the 120s promotion
# check and the 6000s watchdog by up to 12s per narration.
( _n=$(with_timeout 12 claude -p \
"下面是 AI 助手正在执行的工具调用列表。用一句中文(≤40字)向用户转述它正在做什么、信息来自哪里。只输出那一句话,不要前缀。
$_formatted" \
--model haiku --no-session-persistence --disable-slash-commands \
--dangerously-skip-permissions </dev/null 2>/dev/null | head -2 | tr '\n' ' ')
if [ -n "$_n" ] && [ "${#_n}" -lt 200 ] && ! looks_like_error "$_n"; then
lark_reply_text "$message_id" "🔧 $_n" >/dev/null 2>&1
else
lark_reply_text "$message_id" "🔧 $_formatted" >/dev/null 2>&1
fi ) >/dev/null 2>&1 &
else
lark_reply_text "$message_id" "🔧 $_formatted" >/dev/null 2>&1 || true
fi
fi
_last_tool_count="$_new_count"
elif [ "$_thinking_sent" -eq 0 ]; then
# No tool calls yet and the reply isn't back — opus is "just
# thinking". Send ONE textual ack (the reaction alone left a long
# silent gap). Fast replies (<6s) never reach here: the kill -0
# check above broke the loop. Fires at most once per turn.
_thinking_sent=1
lark_reply_text "$message_id" "$JV_THINKING_ACK" >/dev/null 2>&1 || true
fi
# ── Auto-promotion (REQ-16 MVP-2): a call running >120s becomes a
# background job. Release the conversation instead of blocking it —
# "一跑跑3个小时我就用不了这个机器人" was the single harshest complaint
# in the interaction audit. The conversation rotates to a fresh session
# so new messages never resume the transcript this Claude still writes;
# the result comes back via the normal reply + pending_merge.
if [ "$_elapsed" -ge 120 ] && [ ! -f "${ANSWER_FILE}.promoted" ] && kill -0 $_claude_pid 2>/dev/null; then
_bg_job_id=$(JV_JOBS_DIR="$JOBS_DIR" JV_CONV_KEY="$conv_key" \
JV_DESC="auto-promoted: ${content:0:120}" JV_MSG_ID="$message_id" \
python3 "$JARVIS_DIR/core/jobs.py" create 2>>"$LOG_FILE")
if [ -n "$_bg_job_id" ]; then
JV_JOBS_DIR="$JOBS_DIR" python3 "$JARVIS_DIR/core/jobs.py" set-pid "$_bg_job_id" "$_claude_pid" \
2>>"$LOG_FILE" || true
printf '%s' "$_bg_job_id" > "${ANSWER_FILE}.promoted"
if JV_TRACKER="$SESSION_TRACKER" JV_KEY="$conv_key" JV_SDIR="$CLAUDE_PROJECT_DIR" python3 -c "
import os, sys; sys.path.insert(0, os.environ['JARVIS_DIR'])
from core.session import SessionManager
sm = SessionManager(os.environ['JV_TRACKER'], os.environ['JV_SDIR'])
sm.force_rotate(os.environ['JV_KEY'])
" 2>>"$LOG_FILE"; then
# Release only OUR lock (ownership may already have moved)
if grep -q "$_lock_token" "$LOCK_FILE" 2>/dev/null; then
rm -f "$LOCK_FILE"
fi
lark_reply_text "$message_id" "⏳ 这个任务跑得比较久,已自动转后台(job \`$_bg_job_id\`)。会话已释放,可以继续找我聊别的;做完我会把结果发回来。发 jobs 可查进度,发 cancel $_bg_job_id 可取消。" >/dev/null 2>&1 || true
log_info "[$session_id] Promoted to background job $_bg_job_id after ${_elapsed}s — lock released"
else
# Rotation failed: keep the lock (conversation stays busy) so a
# follow-up message can't resume the transcript mid-write.
rm -f "${ANSWER_FILE}.promoted"
log_warn "[$session_id] Promotion rotate failed — keeping lock, job $_bg_job_id orphaned to sweeper"
fi
fi
fi
done
# Watchdog timeout. Drop a marker so the parent can tell a genuine 6000s
# timeout apart from a SIGTERM that came from a restart / external kill —
# both surface as exit 143, but only the real timeout should tell the user
# to resume with 「继续」.
if kill -0 $_claude_pid 2>/dev/null; then
: > "${ANSWER_FILE}.watchdog"
kill $_claude_pid 2>/dev/null
log_warn "[$session_id] Claude killed by 6000s watchdog"
fi
) &
_watchdog_pid=$!
wait $_claude_pid 2>/dev/null
local _exit_code=$?
kill $_watchdog_pid 2>/dev/null 2>&1; wait $_watchdog_pid 2>/dev/null
# Reset the lock to placeholder while we own it: between attempts the
# file would otherwise hold a DEAD claude pid, which a waiter's staleness
# check reads as "handler crashed" and reclaims mid-retry.
if grep -q "$_lock_token" "$LOCK_FILE" 2>/dev/null; then
printf 'acquiring %s' "$_lock_token" > "$LOCK_FILE"
fi
# Did the 6000s watchdog do this kill, or was it a restart / external SIGTERM?
[ -f "${ANSWER_FILE}.watchdog" ] && _watchdog_killed=1
# Extract the final assistant text from the --output-format json envelope:
# one object {"result": "<final text>", "subtype": "success", ...}. Parsing
# .result is immune to stdout pollution from a task-notification / sub-agent
# resume dump — the root cause of the 124k-char leak on 2026-05-30 (a
# background task completed mid-resume and its full envelope went to stdout,
# which the old `cat` blasted at the user). On any parse failure or non-
# success subtype we yield "" so the empty-answer retry path takes over.
answer=$(JV_AF="$ANSWER_FILE" python3 -c "
import json, os, sys
try:
obj = json.load(open(os.environ['JV_AF']))
r = obj.get('result')
if obj.get('subtype') == 'success' and isinstance(r, str):
sys.stdout.write(r)
elif isinstance(r, str):
sys.stdout.write(r) # surface error text → looks_like_error filters it
except Exception:
pass
" 2>/dev/null)
local _stderr_content
_stderr_content=$(head -5 "${ANSWER_FILE}.stderr" 2>/dev/null | tr '\n' ' ')
local _answer_is_error=0
if [ -n "$answer" ] && looks_like_error "$answer"; then
_answer_is_error=1
fi
local _model_error_text="${_stderr_content:-}"
if [ "$_answer_is_error" -eq 1 ]; then
_model_error_text="${_model_error_text} ${answer}"
fi
if [ -z "$answer" ] || [ "$_answer_is_error" -eq 1 ]; then
if [ "$_answer_is_error" -eq 1 ]; then
log_warn "[$session_id] Error-looking answer from Claude (attempt $_attempt, exit=$_exit_code, content=${answer:0:180})"
else
log_warn "[$session_id] Empty answer from Claude (attempt $_attempt, exit=$_exit_code, stderr=${_stderr_content:-none})"
fi
# Promoted to background: never retry — the conversation has moved on
# (rotated session), so a silent re-run would race the new session's
# traffic and double-bill a task the user already saw go background.
if [ -f "${ANSWER_FILE}.promoted" ]; then
break
fi
# exit 143 = SIGTERM: either the 6000s watchdog (task ran long) or a
# restart/external kill. Either way, retrying in-loop is pointless (the
# process is already gone), so stop now. The user-facing message below
# branches on whether the watchdog marker is present.
if [ "${_exit_code:-0}" -eq 143 ]; then
break
fi
# REQ-77: if the empty answer was a MODEL error (unavailable / banned /
# spend limit) rather than a transient blip, degrade the model for the
# next attempt instead of retrying the same broken model to death.
_fallback=$(printf '%s' "$_model_error_text" | python3 -m core.model_fallback "$_cur_model" 2>/dev/null)
if [ -n "$_fallback" ]; then
log_warn "[$session_id] Model error on $_cur_model → degrading to $_fallback (REQ-77)"
_cur_model="$_fallback"
elif [ "$_use_claude_backup" -eq 0 ] \
&& [ "${CLAUDE_BACKUP_ENABLED:-true}" = "true" ] \
&& [ "$_claude_backup_tried" -eq 0 ] \
&& [ -n "${CLAUDE_BACKUP_AUTH_TOKEN:-}" ] \
&& [ -n "${CLAUDE_BACKUP_BASE_URL:-}" ] \
&& printf '%s' "$_model_error_text" | python3 -m core.model_fallback --is-model-error 2>/dev/null; then
_claude_backup_tried=1
_use_claude_backup=1
_cur_model="$MAIN_MODEL"
log_warn "[$session_id] Primary Claude exhausted on $_cur_model → trying Claude Code backup provider"
elif [ "${OPENAI_FALLBACK_ENABLED:-true}" = "true" ] \
&& [ -n "${OPENAI_API_KEY:-}" ] \
&& [ "$_openai_tried" -eq 0 ] \
&& printf '%s' "$_model_error_text" | python3 -m core.model_fallback --is-model-error 2>/dev/null; then
_openai_tried=1
log_warn "[$session_id] Claude model chain exhausted on $_cur_model → trying OpenAI fallback (${OPENAI_FALLBACK_MODEL:-gpt-5.2})"
answer=$(printf '%s' "$content" | JV_SYSTEM_PROMPT_FILE="$SYS_PROMPT_FILE" \
python3 -m core.openai_fallback \
2>"${ANSWER_FILE}.openai.stderr")
_openai_exit=$?
if [ "$_openai_exit" -eq 0 ] && [ -n "$answer" ]; then
_answer_provider="GPT fallback"
_answer_model="${OPENAI_FALLBACK_MODEL:-gpt-5.2}"
log_warn "[$session_id] OpenAI fallback succeeded (${#answer} chars)"
break
fi
_openai_err=$(head -5 "${ANSWER_FILE}.openai.stderr" 2>/dev/null | tr '\n' ' ')
log_warn "[$session_id] OpenAI fallback failed (exit=$_openai_exit, stderr=${_openai_err:-none})"
fi
# On first failure, session file may have been created — update for retry
session_file="$CLAUDE_PROJECT_DIR/${session_id}.jsonl"
else
if [ "$_use_claude_backup" -eq 1 ]; then
_answer_provider="Claude backup"
else
_answer_provider="Claude primary"
fi
_answer_model="$_cur_model"
break
fi
done
local _promoted_job=""
[ -f "${ANSWER_FILE}.promoted" ] && _promoted_job=$(cat "${ANSWER_FILE}.promoted" 2>/dev/null)
rm -f "$ANSWER_FILE" "${ANSWER_FILE}.stderr" "${ANSWER_FILE}.watchdog" \
"${ANSWER_FILE}.promoted" "${ANSWER_FILE}.openai.stderr" "$SYS_PROMPT_FILE"
# Remove the lock ONLY if we still own it: after promotion released it (or
# a staleness reclaim), it may belong to another live handler — an
# unconditional rm here silently unlocked their in-flight session.
if grep -q "$_lock_token" "$LOCK_FILE" 2>/dev/null; then
rm -f "$LOCK_FILE"
fi
# Filter error-like answers — never send them to the user as the "real" reply
local reply=""
if [ -n "$answer" ] && ! looks_like_error "$answer"; then
reply="$answer"
fi
# Promoted-job bookkeeping (REQ-16 MVP-2): close the registry entry and, on
# success, queue the result for merge into the conversation's NEW session
# (it rotated away at promotion time and hasn't seen this answer).
if [ -n "$_promoted_job" ]; then
if [ -n "$reply" ]; then
JV_JOBS_DIR="$JOBS_DIR" python3 "$JARVIS_DIR/core/jobs.py" finish "$_promoted_job" completed \
2>>"$LOG_FILE" || true
jq -cn --arg key "$conv_key" --arg job "$_promoted_job" \
--arg ts "$(date '+%Y-%m-%d %H:%M')" --arg summary "${reply:0:1500}" \
'{conv_key:$key,job_id:$job,ts:$ts,summary:$summary}' \
>> "$JOBS_DIR/pending_merge.jsonl" 2>>"$LOG_FILE" || true
log_info "[$session_id] Promoted job $_promoted_job completed (${#reply} chars)"
else
JV_JOBS_DIR="$JOBS_DIR" python3 "$JARVIS_DIR/core/jobs.py" finish "$_promoted_job" failed \
2>>"$LOG_FILE" || true