added SIGKILL unit tests. bump to forkrun v3.1.2

Author: jkool702

DOCS/DESIGN.md

@@ -1,6 +1,6 @@
 ### `DESIGN.md`
 
-# forkrun Ring Architecture – Design Overview (v9.8.0-NUMA Golden Master)
+# forkrun Ring Architecture – Design Overview
 
 ## 1. Purpose
 

META

@@ -1,2 +1,2 @@
 NAME: forkrun
-VERSION: v3.1.1
+VERSION: v3.1.2

UNIT_TESTS/RESULTS.md

@@ -549,7 +549,7 @@ run_test_regex "Dry run (-N)" \
 
 run_test "Version (-V)" \
   "frun -V" \
-  "forkrun v3.1.1"
+  "forkrun v3.1.2"
 
 # FIXED: Check for USAGE string explicitly without exact match
 run_test_regex "Help (--help)" \
@@ -888,15 +888,15 @@ run_test "Worker transient failure mid-batch" \
 
 # Test 3: Worker command crashes once mid-batch
 # If a specific input crashes only once it should recover and the rest of the file should still process.
-\rm ./.crash
+[[ -f ./.crash ]] && \rm ./.crash &>/dev/null
 run_test "Worker one-time transient failure mid-batch" \
   "crash_func() { for arg in \"\$@\"; do if [[ \"\$arg\" == \"3\" ]] && ! [[ -f ./.crash  ]]; then : >./.crash; exit 1; else echo \"\$arg\"; fi; done; }; seq 1 5 | FORKRUN_EXTRA_FUNCS='crash_func' frun -l 1 -k crash_func" \
   "1
 2
 3
 4
 5"
-\rm ./.crash
+\rm ./.crash &>/dev/null
 
 print_section "Misc additional tests"
 

UNIT_TESTS/test_frun.sh

@@ -549,7 +549,7 @@ run_test_regex "Dry run (-N)" \
 
 run_test "Version (-V)" \
   "frun -V" \
-  "forkrun v3.1.1"
+  "forkrun v3.1.2"
 
 # FIXED: Check for USAGE string explicitly without exact match
 run_test_regex "Help (--help)" \

UNIT_TESTS/test_frun.sh.txt

@@ -1,4 +1,4 @@
-// forkrun_ring.c v3.1.1
+// forkrun_ring.c v3.1.2
 // ======================================================================================
 // ARCHITECTURE OVERVIEW:
 //
@@ -298,7 +298,7 @@ static inline char *try_simd_scan(char *p, char *safe_end, uint64_t target,
 #define DAMPING_OFFSET 6
 
 #ifndef FORKRUN_RING_VERSION
-#define FORKRUN_RING_VERSION "v3.1.1"
+#define FORKRUN_RING_VERSION "v3.1.2"
 #endif
 
 #define atomic_load_acquire(ptr) __atomic_load_n(ptr, __ATOMIC_ACQUIRE)

UNIT_TESTS/test_frun_comprehensive.sh

@@ -280,7 +280,7 @@ EOF
             # help system
             -h|-\?|--help|--help=*|--usage)  _frun_displayHelp "$1";  return 0  ;;
 
-            -V|--version|--VERSION)           echo 'forkrun v3.1.1';  return 0  ;;
+            -V|--version|--VERSION)           echo 'forkrun v3.1.2';  return 0  ;;
 
             --) shift; break ;;
 
@@ -706,6 +706,7 @@ W_NODE[$3]=$2
 
         local -A trap_ack_pending
         local _poll_timer_cmd=0
+        local _timer_armed=false
 
         for ((i=0; i<FORKRUN_NUM_NODES; i++)); do node_workers[i]=0; done
         node_worker_max=$(( nWorkersMax / FORKRUN_NUM_NODES ))
@@ -723,6 +724,7 @@ W_NODE[$3]=$2
             case "$POLL_EVENT" in
                 IGNORE) ;;
                 TIMEOUT)
+                    _timer_armed=false
                     if (( ${#trap_ack_pending[@]} > 0 )); then
                         echo "forkrun [FATAL]: Worker(s) [ ${!trap_ack_pending[@]} ] exited non-zero and EXIT trap did not confirm recovery within 3s grace period. Aborting." >&2
                         ring_abort
@@ -740,6 +742,7 @@ W_NODE[$3]=$2
 
                     if (( ${#trap_ack_pending[@]} == 0 )); then
                         _poll_timer_cmd=-1 # Cancel timer cleanly
+                        _timer_armed=false
                     fi
                     ;;
                 SPAWN)
@@ -780,8 +783,11 @@ W_NODE[$3]=$2
                             unset 'trap_ack_pending[$wID]'
                         elif (( trap_ack_pending[$wID] > 0 )); then
                             # Death arrived before TRAP_ACK. Arm the 3-second deadline.
-                            _poll_timer_cmd=3000
-                            echo "forkrun [WARN]: Worker $wID (node ${node_idx}) exited with signal status $status. Waiting up to 3s for EXIT trap confirmation." >&2
+                            if ! $_timer_armed; then
+                                _poll_timer_cmd=3000
+                                _timer_armed=true
+                                echo "forkrun [WARN]: Worker $wID (node ${node_idx}) exited with status $status. Waiting up to 3s for EXIT trap confirmation." >&2
+                            fi
                         fi
 
                         # Unconditionally respawn replacement worker

UNIT_TESTS/test_frun_comprehensive.sh.txt

@@ -280,7 +280,7 @@ EOF
             # help system
             -h|-\?|--help|--help=*|--usage)  _frun_displayHelp "$1";  return 0  ;;
 
-            -V|--version|--VERSION)           echo 'forkrun v3.1.1';  return 0  ;;
+            -V|--version|--VERSION)           echo 'forkrun v3.1.2';  return 0  ;;
 
             --) shift; break ;;
 
@@ -477,22 +477,23 @@ toc() { :; }
 
         # --- 2 & 3. THE PRODUCER PLUMBING ---
             declare -a fd_scan_death_r fd_scan_death_w SCANNER_P
+            ring_pipe fd_trap_ack_r fd_trap_ack_w
 
             if (( FORKRUN_NUM_NODES > 1 )); then
-                # NUMA TOPOLOGICAL PIPELINE (No Pipes, Lock-Free Meta Ring)
+                # NUMA TOPOLOGICAL PIPELINE
                 ordered_flag=0
                 [[ "${order_mode}" == "ordered" ]] && ordered_flag=1
 
-                ( ring_numa_ingest ${fd0} ${fd_write} $FORKRUN_NUM_NODES $ordered_flag ) &
+                ( exec {fd_trap_ack_w}>&-; ring_numa_ingest ${fd0} ${fd_write} $FORKRUN_NUM_NODES $ordered_flag ) &
 
                 for (( i=0; i<FORKRUN_NUM_NODES; i++ )); do
-                    ( ring_indexer_numa ${fd_scan} $i ) &
+                    ( exec {fd_trap_ack_w}>&-; ring_indexer_numa ${fd_scan} $i ) &
                 done
 
                 for (( i=0; i<FORKRUN_NUM_NODES; i++ )); do
                     ring_pipe fd_scan_death_r[$i] fd_scan_death_w[$i]
                     (
-                        exec {fd_fallow_w}>&-
+                        exec {fd_fallow_w}>&- {fd_trap_ack_w}>&-
                         ring_numa_scanner ${fd_scan} $i $fd_spawn_w $FORKRUN_NUM_NODES
                     ) &
                     SCANNER_P[$i]=$!
@@ -501,10 +502,10 @@ toc() { :; }
 
             else
                 # LEGACY FLAT PIPELINE
-                ( ring_copy ${fd_write} ${fd0}; ring_signal ) &
+                ( exec {fd_trap_ack_w}>&-; ring_copy ${fd_write} ${fd0}; ring_signal ) &
                 ring_pipe fd_scan_death_r[0] fd_scan_death_w[0]
                 (
-                    exec {fd_spawn_r}<&-
+                    exec {fd_spawn_r}<&- {fd_trap_ack_w}>&-
                     ring_scanner ${fd_scan} ${fd_spawn_w}
                 ) &
                 SCANNER_P[0]=$!
@@ -514,7 +515,7 @@ toc() { :; }
         exec {fd_spawn_w}>&-
 
         (
-            exec {fd_fallow_w}>&-
+            exec {fd_fallow_w}>&- {fd_trap_ack_w}>&-
             fallow_args=( "${fd_fallow_r}" "${fd_write}" )
             if (( FORKRUN_NUM_NODES > 1 )); then
                 ring_fallow_phys "${fallow_args[@]}"
@@ -528,7 +529,7 @@ toc() { :; }
         [[ "${order_mode}" == "realtime" ]] || {
             ring_pipe fd_order_r fd_order_w
             (
-                exec {fd_order_w}>&-
+                exec {fd_order_w}>&- {fd_trap_ack_w}>&-
 
                 order_args=( "${fd_order_r}" 'memfd' )
                 [[ "${order_mode}" == "buffered" ]] && order_args+=( "unordered" )
@@ -637,26 +638,26 @@ toc() { :; }
   set +m
   export RING_NODE_ID="$2"
   export RING_WID="$3"
+  export FD_TRAP_ACK_W="$4"
   
   _ring_registered=false
   
   trap '"'"'
     status=$?
     ${_ring_registered} && { ring_worker dec; ring_cleanup_waiter; }
     
-    # Only attempt recovery if we actually held an active batch
     if (( status != 0 && RING_BATCH_SLOTS > 0 )); then
         (( RING_NUM_KILLS++ ))
-        
-        # Revert partial corrupted output from this failed run (unless realtime)
         if [[ "$order_mode" != "realtime" && -n "${fd_out[$RING_WID]:-}" ]]; then
             ring_revert_output "${fd_out[$RING_WID]}"
         fi
-
-        # Always throw it back into escrow. 
-        # If kills >= LIMIT, the NEXT worker will see RING_POISONED=1 and safely skip & ack it.
         ring_escrow_put "$RING_NODE_ID" "$RING_BATCH_IDX" "$RING_BATCH_SLOTS" "$RING_NUM_KILLS"
     fi
+    
+    # Notify parent that the trap successfully fired
+    if (( status != 0 )); then
+        echo "$RING_WID" >&"${FD_TRAP_ACK_W}" 2>/dev/null
+    fi
     exit $status
   '"'"' EXIT
 
@@ -672,7 +673,7 @@ toc() { :; }
     '
        ${insert_id_flag:-false} && worker_func_src+='W_BATCH=0
     '
-        worker_func_src+='shift 3
+        worker_func_src+='shift 4
     ring_worker inc $fd_read
     _ring_registered=true
     while ring_claim; do
@@ -702,6 +703,11 @@ W_NODE[$3]=$2
         # --- SPAWN LOOP REACTOR ---
         nWorkers=0
         local -a node_workers W_NODE fd_worker_r fd_worker_w P wID_free
+        
+        local -A trap_ack_pending
+        local _poll_timer_cmd=0
+        local _timer_armed=false
+
         for ((i=0; i<FORKRUN_NUM_NODES; i++)); do node_workers[i]=0; done
         node_worker_max=$(( nWorkersMax / FORKRUN_NUM_NODES ))
         (( node_worker_max < 1 )) && node_worker_max=1
@@ -712,9 +718,33 @@ W_NODE[$3]=$2
             wID_free[$nn]=''
         done
 
-        while ring_poll "$fd_spawn_arg" fd_scan_death_r fd_worker_r; do
+        while ring_poll "$fd_spawn_arg" fd_scan_death_r fd_worker_r "$_poll_timer_cmd" "$fd_trap_ack_r"; do
+            _poll_timer_cmd=0
+
             case "$POLL_EVENT" in
                 IGNORE) ;;
+                TIMEOUT)
+                    _timer_armed=false
+                    if (( ${#trap_ack_pending[@]} > 0 )); then
+                        echo "forkrun [FATAL]: Worker(s) [ ${!trap_ack_pending[@]} ] exited non-zero and EXIT trap did not confirm recovery within 3s grace period. Aborting." >&2
+                        ring_abort
+                        exit 1
+                    fi
+                    ;;
+                TRAP_ACK)
+                    wID=$POLL_ARG1
+                    (( trap_ack_pending[$wID]-- ))
+                    
+                    # If it balanced out to 0 (DEATH arrived first), clean it up
+                    if (( trap_ack_pending[$wID] == 0 )); then
+                        unset 'trap_ack_pending[$wID]'
+                    fi
+                    
+                    if (( ${#trap_ack_pending[@]} == 0 )); then
+                        _poll_timer_cmd=-1 # Cancel timer cleanly
+                        _timer_armed=false
+                    fi
+                    ;;
                 SPAWN)
                     spawn_count=$POLL_ARG1
                     node_idx=$POLL_ARG2
@@ -723,28 +753,20 @@ W_NODE[$3]=$2
                     (( target > node_worker_max )) && target=$node_worker_max
 
                     for (( ; node_workers[node_idx] < target; node_workers[node_idx]++ )); do
-                        # Find first empty wID slot
                         for wID in "${!wID_free[@]}"; do break; done
                         unset 'wID_free[$wID]'
 
                         ring_pipe fd_worker_r[$wID] fd_worker_w[$wID]
-                        spawn_worker "$wID" "$node_idx" "$wID"
+                        spawn_worker "$wID" "$node_idx" "$wID" "${fd_trap_ack_w}"
                         exec {fd_worker_w[$wID]}>&-
                         ((nWorkers++))
                     done
                     ;;
-                    
                 WORKER_DEATH)
                     wID=$POLL_ARG1
                     wait "${P[$wID]}" 2>/dev/null
                     status=$?
 
-                    if (( status > 128 && status != 143 )); then
-                        echo "forkrun [FATAL]: Worker $wID killed by signal $((status - 128)). Aborting." >&2
-                        ring_abort
-                        exit 1
-                    fi
-                    
                     exec {fd_worker_r[$wID]}<&-
                     unset 'fd_worker_r[$wID]' 'fd_worker_w[$wID]' 'P[$wID]'
 
@@ -753,18 +775,31 @@ W_NODE[$3]=$2
 
                     (( node_workers[node_idx]-- ))
 
-                    # ONLY respawn if the worker crashed. Clean exit (0) means no work left.
                     if (( status != 0 )); then
+                        (( trap_ack_pending[$wID]++ ))
+                        
+                        if (( trap_ack_pending[$wID] == 0 )); then
+                            # TRAP_ACK already arrived! Clean up safely.
+                            unset 'trap_ack_pending[$wID]'
+                        elif (( trap_ack_pending[$wID] > 0 )); then
+                            # Death arrived before TRAP_ACK. Arm the 3-second deadline.
+                            if ! $_timer_armed; then
+                                _poll_timer_cmd=3000
+                                _timer_armed=true
+                                echo "forkrun [WARN]: Worker $wID (node ${node_idx}) exited with status $status. Waiting up to 3s for EXIT trap confirmation." >&2
+                            fi
+                        fi
+                        
+                        # Unconditionally respawn replacement worker
                         ring_pipe fd_worker_r[$wID] fd_worker_w[$wID]
-                        spawn_worker "$wID" "$node_idx" "$wID"
+                        spawn_worker "$wID" "$node_idx" "$wID" "${fd_trap_ack_w}"
                         exec {fd_worker_w[$wID]}>&-
                         ((nWorkers++))
                         (( node_workers[node_idx]++ ))
                     else
                         wID_free[$wID]=''
                     fi
                     ;;
-                    
                 SCAN_DEATH)
                     sID=$POLL_ARG1
                     wait "${SCANNER_P[$sID]}" 2>/dev/null
@@ -779,7 +814,6 @@ W_NODE[$3]=$2
                     exec {fd_scan_death_r[$sID]}<&-
                     unset 'fd_scan_death_r[$sID]' 'SCANNER_P[$sID]'
                     ;;
-                    
                 EOF)
                     fd_spawn_arg="-1"
                     ;;
@@ -789,7 +823,7 @@ W_NODE[$3]=$2
         ${verbose_flag} && printf '\nSPAWNED %s workers\n' "${nWorkers}" >&2
 
         # --- SHUTDOWN ---
-        exec {fd_spawn_r}<&- {fd_fallow_w}>&-
+        exec {fd_spawn_r}<&- {fd_fallow_w}>&- {fd_trap_ack_r}<&- {fd_trap_ack_w}>&-
         [[ "${order_mode}" == "realtime" ]] || exec {fd_order_w}>&-
 
         wait