feat(recovery): add confidence threshold to filter false positives

recovery/config_test.go

@@ -10,8 +10,8 @@ func TestDefaultRecoveryConfig(t *testing.T) {
 	if cfg.MaxRetries != 1 {
 		t.Errorf("MaxRetries = %d, want 1", cfg.MaxRetries)
 	}
-	if cfg.MinConfidence != 0.4 {
-		t.Errorf("MinConfidence = %f, want 0.4", cfg.MinConfidence)
+	if cfg.MinConfidence != defaultRecoveryMinConfidence {
+		t.Errorf("MinConfidence = %f, want %f", cfg.MinConfidence, defaultRecoveryMinConfidence)
 	}
 	if cfg.PreferHighConfidence {
 		t.Error("PreferHighConfidence should be false by default")

recovery/engine.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"strings"
 	"time"
+	"unicode"
 
 	"github.com/pinchtab/semantic"
 )
@@ -18,7 +20,7 @@ type RecoveryConfig struct {
 	MaxRetries int
 
 	// MinConfidence is the minimum score the semantic re-match must
-	// achieve for the recovery attempt to proceed. Default 0.4.
+	// achieve for the recovery attempt to proceed. Default 0.52.
 	MinConfidence float64
 
 	// PreferHighConfidence when true will only auto-recover if the
@@ -27,11 +29,28 @@ type RecoveryConfig struct {
 	PreferHighConfidence bool
 }
 
+const defaultRecoveryMinConfidence = 0.52
+
+var recoveryRoleKeywords = map[string]bool{
+	"button":   true,
+	"input":    true,
+	"link":     true,
+	"textbox":  true,
+	"checkbox": true,
+	"radio":    true,
+	"select":   true,
+	"option":   true,
+	"tab":      true,
+	"menu":     true,
+	"form":     true,
+	"search":   true,
+}
+
 func DefaultRecoveryConfig() RecoveryConfig {
 	return RecoveryConfig{
 		Enabled:              true,
 		MaxRetries:           1,
-		MinConfidence:        0.4,
+		MinConfidence:        defaultRecoveryMinConfidence,
 		PreferHighConfidence: false,
 	}
 }
@@ -358,11 +377,36 @@ func (re *RecoveryEngine) reconstructQuery(tabID, ref string) string {
 		return ""
 	}
 	if entry.Query != "" {
-		return entry.Query
+		return enrichRecoveryQuery(entry)
 	}
 	return entry.Descriptor.Composite()
 }
 
+func enrichRecoveryQuery(entry IntentEntry) string {
+	query := strings.TrimSpace(entry.Query)
+	if query == "" {
+		return entry.Descriptor.Composite()
+	}
+
+	role := strings.TrimSpace(entry.Descriptor.Role)
+	if role == "" || queryHasRoleKeyword(query) {
+		return query
+	}
+
+	return query + " " + role
+}
+
+func queryHasRoleKeyword(query string) bool {
+	for _, token := range strings.FieldsFunc(strings.ToLower(query), func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	}) {
+		if recoveryRoleKeywords[token] {
+			return true
+		}
+	}
+	return false
+}
+
 func (re *RecoveryEngine) RecordIntent(tabID, ref string, entry IntentEntry) {
 	if re.IntentCache != nil {
 		re.IntentCache.Store(tabID, ref, entry)

recovery/engine_recovery_test.go

@@ -182,7 +182,7 @@ func TestRecoveryEngine_Attempt_ScoreBelowThreshold(t *testing.T) {
 		findFn: func(_ context.Context, _ string, _ []semantic.ElementDescriptor, _ semantic.FindOptions) (semantic.FindResult, error) {
 			return semantic.FindResult{
 				BestRef:   "e2",
-				BestScore: 0.25, // Below default MinConfidence (0.4)
+				BestScore: 0.25, // Below the configured recovery threshold.
 			}, nil
 		},
 	}
@@ -433,6 +433,7 @@ func TestRecoveryEngine_PreferHighConfidence_RejectsLow(t *testing.T) {
 	}
 
 	cfg := DefaultRecoveryConfig()
+	cfg.MinConfidence = 0.4
 	cfg.PreferHighConfidence = true
 
 	re := NewRecoveryEngine(
@@ -507,6 +508,52 @@ func TestRecoveryEngine_ReconstructQuery_FallbackToComposite(t *testing.T) {
 	}
 }
 
+func TestRecoveryEngine_ReconstructQuery_AppendsRoleWhenQueryOmitsIt(t *testing.T) {
+	cache := NewIntentCache(100, 5*time.Minute)
+	cache.Store("tab1", "e1", IntentEntry{
+		Query:      "log out",
+		Descriptor: semantic.ElementDescriptor{Ref: "e1", Role: "button", Name: "Log Out"},
+	})
+
+	querySeen := ""
+	matcher := &mockMatcher{
+		findFn: func(_ context.Context, query string, _ []semantic.ElementDescriptor, _ semantic.FindOptions) (semantic.FindResult, error) {
+			querySeen = query
+			return semantic.FindResult{
+				BestRef:   "e2",
+				BestScore: 0.9,
+				Strategy:  "combined",
+			}, nil
+		},
+	}
+
+	re := NewRecoveryEngine(
+		DefaultRecoveryConfig(),
+		matcher,
+		cache,
+		func(_ context.Context, _ string) error { return nil },
+		func(_, ref string) (int64, bool) {
+			if ref == "e2" {
+				return 22, true
+			}
+			return 0, false
+		},
+		func(_ string) []semantic.ElementDescriptor {
+			return []semantic.ElementDescriptor{{Ref: "e2", Role: "button", Name: "Logout"}}
+		},
+	)
+
+	_, _, _ = re.Attempt(context.Background(), "tab1", "e1", "click",
+		func(_ context.Context, _ string, _ int64) (map[string]any, error) {
+			return map[string]any{"ok": true}, nil
+		},
+	)
+
+	if querySeen != "log out button" {
+		t.Errorf("reconstructed query = %q, want %q", querySeen, "log out button")
+	}
+}
+
 func TestRecoveryEngine_RecordIntent(t *testing.T) {
 	cache := NewIntentCache(100, 5*time.Minute)
 	re := NewRecoveryEngine(

recovery/engine_scenario_test.go

@@ -3,6 +3,7 @@ package recovery
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -345,6 +346,111 @@ func TestRecovery_Scenario_CMSNavigationLink(t *testing.T) {
 	}
 }
 
+func TestRecovery_Scenario_RenamedLogoutButton(t *testing.T) {
+	cache := NewIntentCache(100, 5*time.Minute)
+	cache.Store("tab-account", "e5", IntentEntry{
+		Query:      "log out",
+		Descriptor: semantic.ElementDescriptor{Ref: "e5", Role: "button", Name: "Log Out"},
+	})
+
+	freshDescs := []semantic.ElementDescriptor{
+		{Ref: "e5", Role: "button", Name: "Logout"},
+	}
+
+	matcher := semantic.NewCombinedMatcher(semantic.NewHashingEmbedder(128))
+
+	re := NewRecoveryEngine(
+		DefaultRecoveryConfig(),
+		matcher,
+		cache,
+		func(_ context.Context, _ string) error { return nil },
+		func(_, ref string) (int64, bool) {
+			if ref == "e5" {
+				return 500, true
+			}
+			return 0, false
+		},
+		func(_ string) []semantic.ElementDescriptor { return freshDescs },
+	)
+
+	rr, res, err := re.AttemptWithClassification(
+		context.Background(), "tab-account", "e5", "click",
+		FailureElementStale,
+		func(_ context.Context, _ string, nodeID int64) (map[string]any, error) {
+			if nodeID != 500 {
+				return nil, fmt.Errorf("wrong button, nodeID=%d want 500", nodeID)
+			}
+			return map[string]any{"clicked": true}, nil
+		},
+	)
+
+	if err != nil {
+		t.Fatalf("recovery failed: %v", err)
+	}
+	if !rr.Recovered {
+		t.Fatal("should recover for 'log out' -> 'Logout'")
+	}
+	if rr.NewRef != "e5" {
+		t.Errorf("NewRef = %q, want e5", rr.NewRef)
+	}
+	if rr.Score < defaultRecoveryMinConfidence {
+		t.Errorf("Score = %f, want >= %f", rr.Score, defaultRecoveryMinConfidence)
+	}
+	if res["clicked"] != true {
+		t.Error("action result should contain clicked=true")
+	}
+}
+
+func TestRecovery_Scenario_RemovedDeleteButton_NoFalsePositive(t *testing.T) {
+	cache := NewIntentCache(100, 5*time.Minute)
+	cache.Store("tab-items", "e3", IntentEntry{
+		Query:      "delete button",
+		Descriptor: semantic.ElementDescriptor{Ref: "e3", Role: "button", Name: "Delete"},
+	})
+
+	freshDescs := []semantic.ElementDescriptor{
+		{Ref: "e1", Role: "text", Name: "Item 1"},
+		{Ref: "e2", Role: "button", Name: "Edit"},
+		{Ref: "e3", Role: "button", Name: "Archive"},
+	}
+
+	matcher := semantic.NewCombinedMatcher(semantic.NewHashingEmbedder(128))
+
+	re := NewRecoveryEngine(
+		DefaultRecoveryConfig(),
+		matcher,
+		cache,
+		func(_ context.Context, _ string) error { return nil },
+		func(_, _ string) (int64, bool) {
+			t.Fatal("resolver should not be called when no candidate clears the recovery threshold")
+			return 0, false
+		},
+		func(_ string) []semantic.ElementDescriptor { return freshDescs },
+	)
+
+	rr, _, err := re.AttemptWithClassification(
+		context.Background(), "tab-items", "e3", "click",
+		FailureElementNotFound,
+		func(_ context.Context, _ string, _ int64) (map[string]any, error) {
+			t.Fatal("action executor should not run for a low-confidence recovery")
+			return nil, nil
+		},
+	)
+
+	if err == nil {
+		t.Fatal("expected recovery to reject low-confidence false positive")
+	}
+	if rr.Recovered {
+		t.Fatal("should not recover when only low-confidence alternatives remain")
+	}
+	if !strings.Contains(rr.Error, "no match above threshold") {
+		t.Fatalf("Error = %q, want threshold rejection", rr.Error)
+	}
+	if rr.NewRef != "" {
+		t.Errorf("NewRef = %q, want empty", rr.NewRef)
+	}
+}
+
 func TestRecovery_Scenario_NetworkError_NoAttempt(t *testing.T) {
 	re := &RecoveryEngine{Config: DefaultRecoveryConfig()}
 

tests/benchmark/scripts/run-full-benchmark.sh

@@ -21,6 +21,26 @@ SEMANTIC="${BENCHMARK_DIR}/semantic"
 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
 REPORT_FILE="${RESULTS_DIR}/full_benchmark_${TIMESTAMP}.json"
 
+has_role_keyword() {
+    local query="$1"
+    echo "$query" | grep -Eiq '(^|[^[:alnum:]])(button|input|link|textbox|checkbox|radio|select|option|tab|menu|form|search)([^[:alnum:]]|$)'
+}
+
+enrich_recovery_query() {
+    local query="$1"
+    local role="$2"
+
+    if [[ -z "$query" || -z "$role" ]]; then
+        printf '%s' "$query"
+        return
+    fi
+    if has_role_keyword "$query"; then
+        printf '%s' "$query"
+        return
+    fi
+    printf '%s %s' "$query" "$role"
+}
+
 # Initialize report
 jq -n \
     --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
@@ -65,7 +85,10 @@ if [[ -f "$SCENARIOS_FILE" ]]; then
     for i in $(seq 0 $((SCENARIO_COUNT - 1))); do
         ID=$(jq -r ".[$i].id" "$SCENARIOS_FILE")
         NAME=$(jq -r ".[$i].name" "$SCENARIOS_FILE")
-        QUERY=$(jq -r ".[$i].original_query" "$SCENARIOS_FILE")
+        RAW_QUERY=$(jq -r ".[$i].original_query" "$SCENARIOS_FILE")
+        ORIGINAL_REF=$(jq -r ".[$i].original_ref // empty" "$SCENARIOS_FILE")
+        ORIGINAL_ROLE=$(jq -r ".[$i].before[]? | select(.ref == \"$ORIGINAL_REF\") | .role // empty" "$SCENARIOS_FILE")
+        QUERY=$(enrich_recovery_query "$RAW_QUERY" "$ORIGINAL_ROLE")
         EXPECTED=$(jq -r ".[$i].expected_ref // empty" "$SCENARIOS_FILE")
         EXPECTED_ALT=$(jq -r ".[$i].expected_alt // [] | join(\",\")" "$SCENARIOS_FILE")
         EXPECT_NO_MATCH=$(jq -r ".[$i].expect_no_match // false" "$SCENARIOS_FILE")
@@ -74,10 +97,10 @@ if [[ -f "$SCENARIOS_FILE" ]]; then
         AFTER_FILE=$(mktemp)
         jq ".[$i].after" "$SCENARIOS_FILE" > "$AFTER_FILE"
 
-        # Run semantic find on after snapshot
-        RESULT=$("${SEMANTIC}" find "$QUERY" --snapshot "$AFTER_FILE" --format json --threshold 0.2 2>/dev/null || echo '{"matches":[]}')
+        # Run semantic find on after snapshot with the same minimum score
+        # enforced by DefaultRecoveryConfig in the recovery engine.
+        RESULT=$("${SEMANTIC}" find "$QUERY" --snapshot "$AFTER_FILE" --format json --threshold 0.52 2>/dev/null || echo '{"matches":[]}')
         BEST_REF=$(echo "$RESULT" | jq -r '.best_ref // ""')
-        BEST_SCORE=$(echo "$RESULT" | jq -r '.best_score // 0')
 
         rm -f "$AFTER_FILE"
 

tests/e2e/cases/13-recovery.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+CASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CASE_DIR}/../lib.sh"
+
+echo "  -- Recovery: Confidence Threshold --"
+
+# Recovery should accept renamed controls that still clear the real threshold.
+# "log out button" vs "Logout"
+result=$(echo '[{"ref":"e5","role":"button","name":"Logout"}]' | semantic find "log out button" --format json --threshold 0.52)
+assert_json_field "$result" ".best_ref" "e5" "recovery: 'log out' matches 'Logout'"
+assert_json_gte "$result" ".best_score" "0.52" "recovery: log out score >= 0.52"
+
+# "sign in button" vs "Login" - synonym phrase match
+result=$(echo '[{"ref":"e1","role":"button","name":"Login"}]' | semantic find "sign in button" --format json --threshold 0.52)
+assert_json_field "$result" ".best_ref" "e1" "recovery: 'sign in' matches 'Login'"
+assert_json_gte "$result" ".best_score" "0.52" "recovery: sign in score >= 0.52"
+
+# Element renamed: "Submit" to "Send" - should still match
+result=$(echo '[{"ref":"e3","role":"button","name":"Send"}]' | semantic find "submit button" --format json --threshold 0.52)
+assert_json_field "$result" ".best_ref" "e3" "recovery: 'submit button' matches 'Send'"
+
+# Element removed: "delete button" with no Delete - should NOT match Edit
+result=$(echo '[{"ref":"e2","role":"button","name":"Edit"},{"ref":"e3","role":"button","name":"Archive"}]' | semantic find "delete button" --format json --threshold 0.52)
+BEST_REF=$(echo "$result" | jq -r '.best_ref // ""')
+BEST_SCORE=$(echo "$result" | jq -r '.best_score // 0')
+# With threshold 0.52, Edit (0.50) should be filtered out
+if [[ -z "$BEST_REF" ]] || [[ "$BEST_REF" == "null" ]]; then
+    pass "recovery: 'delete button' correctly returns no match when element is removed"
+else
+    # If there's a match, it should only be Archive (not Edit)
+    if [[ "$BEST_REF" == "e2" ]]; then
+        fail "recovery: should not match Edit for 'delete button'" "got e2 (Edit)"
+    else
+        pass "recovery: 'delete button' returns non-Edit match: $BEST_REF"
+    fi
+fi
+
+summary "recovery"