Make match ranking deterministic and strengthen visual-hint regressions

README.md

@@ -103,6 +103,7 @@ Implementations are internal — consumers use the `ElementMatcher` interface an
 ## Features
 
 - **Synonym expansion** — 54 UI synonym groups ("sign in" ↔ "log in", "cart" ↔ "basket", "preferences" ↔ "settings", etc.)
+- **Visual position hints** — Understand layout cues like `top`, `bottom`, `left`, `right`, and `above`/`below` anchors
 - **Confidence calibration** — Scores mapped to high (≥ 0.8) / medium (≥ 0.6) / low labels
 - **Error classification** — Classify browser errors (CDP, chromedp) as recoverable or not
 - **Self-healing recovery** — Re-locate stale elements after DOM changes via callback interfaces
@@ -184,6 +185,11 @@ semantic find "login" --snapshot page.json --format json    # machine-readable
 semantic find "login" --snapshot page.json --format table   # human-readable
 semantic find "login" --snapshot page.json --format refs    # just refs
 
+# Visual position hints
+semantic find "button in top right corner" --snapshot page.json
+semantic find "link below the search box" --snapshot page.json
+semantic find "sidebar on the left" --snapshot page.json
+
 # Score a specific element
 semantic match "login" e4 --snapshot page.json
 

cmd/semantic/main.go

@@ -62,10 +62,16 @@ Flags (find/match):
 
 // snapshotElement is the JSON shape from pinchtab's /snapshot endpoint.
 type snapshotPositional struct {
-	Depth        int    `json:"depth"`
-	SiblingIndex int    `json:"sibling_index"`
-	SiblingCount int    `json:"sibling_count"`
-	LabelledBy   string `json:"labelled_by"`
+	Depth        int     `json:"depth"`
+	SiblingIndex int     `json:"sibling_index"`
+	SiblingCount int     `json:"sibling_count"`
+	LabelledBy   string  `json:"labelled_by"`
+	X            float64 `json:"x"`
+	Y            float64 `json:"y"`
+	Top          float64 `json:"top"`
+	Left         float64 `json:"left"`
+	Width        float64 `json:"width"`
+	Height       float64 `json:"height"`
 }
 
 type snapshotElement struct {
@@ -80,6 +86,12 @@ type snapshotElement struct {
 	SiblingIdx  int                 `json:"sibling_index"`
 	SiblingCnt  int                 `json:"sibling_count"`
 	LabelledBy  string              `json:"labelled_by"`
+	X           float64             `json:"x"`
+	Y           float64             `json:"y"`
+	Top         float64             `json:"top"`
+	Left        float64             `json:"left"`
+	Width       float64             `json:"width"`
+	Height      float64             `json:"height"`
 	Positional  *snapshotPositional `json:"positional"`
 }
 
@@ -112,6 +124,16 @@ func loadSnapshot(path string) ([]semantic.ElementDescriptor, error) {
 		depth := e.Depth
 		siblingIdx := e.SiblingIdx
 		siblingCnt := e.SiblingCnt
+		x := e.X
+		y := e.Y
+		if x == 0 && e.Left != 0 {
+			x = e.Left
+		}
+		if y == 0 && e.Top != 0 {
+			y = e.Top
+		}
+		width := e.Width
+		height := e.Height
 		if e.Positional != nil {
 			if e.Positional.Depth != 0 {
 				depth = e.Positional.Depth
@@ -125,6 +147,23 @@ func loadSnapshot(path string) ([]semantic.ElementDescriptor, error) {
 			if e.Positional.LabelledBy != "" {
 				labelledBy = e.Positional.LabelledBy
 			}
+
+			hasHorizontal := e.Positional.X != 0 || e.Positional.Left != 0 || e.Positional.Width > 0
+			hasVertical := e.Positional.Y != 0 || e.Positional.Top != 0 || e.Positional.Height > 0
+			if hasHorizontal {
+				x = e.Positional.X
+				if x == 0 && e.Positional.Left != 0 {
+					x = e.Positional.Left
+				}
+				width = e.Positional.Width
+			}
+			if hasVertical {
+				y = e.Positional.Y
+				if y == 0 && e.Positional.Top != 0 {
+					y = e.Positional.Top
+				}
+				height = e.Positional.Height
+			}
 		}
 
 		descs[i] = semantic.ElementDescriptor{
@@ -140,6 +179,10 @@ func loadSnapshot(path string) ([]semantic.ElementDescriptor, error) {
 				SiblingIndex: siblingIdx,
 				SiblingCount: siblingCnt,
 				LabelledBy:   labelledBy,
+				X:            x,
+				Y:            y,
+				Width:        width,
+				Height:       height,
 			},
 		}
 	}

cmd/semantic/main_test.go

@@ -12,8 +12,8 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) {
 	}
 
 	json := `[
-		{"ref":"e1","role":"button","name":"Submit","interactive":true,"parent":"Login form","section":"Authentication","depth":3,"sibling_index":1,"sibling_count":2,"labelled_by":"Primary Action"},
-		{"ref":"e2","role":"text","name":"Submit","interactive":false,"parent":"Payment form","section":"Checkout","positional":{"depth":2,"sibling_index":0,"sibling_count":1,"labelled_by":"Secondary Action"}}
+		{"ref":"e1","role":"button","name":"Submit","interactive":true,"parent":"Login form","section":"Authentication","depth":3,"sibling_index":1,"sibling_count":2,"labelled_by":"Primary Action","x":20,"y":40,"width":120,"height":30},
+		{"ref":"e2","role":"text","name":"Submit","interactive":false,"parent":"Payment form","section":"Checkout","positional":{"depth":2,"sibling_index":0,"sibling_count":1,"labelled_by":"Secondary Action","left":300,"top":640,"width":200,"height":44}}
 	]`
 	if _, err := f.WriteString(json); err != nil {
 		t.Fatalf("WriteString failed: %v", err)
@@ -50,6 +50,12 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) {
 	if descs[0].Positional.LabelledBy != "Primary Action" {
 		t.Fatalf("expected first descriptor labelled_by=Primary Action, got %q", descs[0].Positional.LabelledBy)
 	}
+	if descs[0].Positional.X != 20 || descs[0].Positional.Y != 40 {
+		t.Fatalf("expected first descriptor x/y=20/40, got %f/%f", descs[0].Positional.X, descs[0].Positional.Y)
+	}
+	if descs[0].Positional.Width != 120 || descs[0].Positional.Height != 30 {
+		t.Fatalf("expected first descriptor width/height=120/30, got %f/%f", descs[0].Positional.Width, descs[0].Positional.Height)
+	}
 	if descs[1].Interactive {
 		t.Fatalf("expected second descriptor interactive=false")
 	}
@@ -71,4 +77,10 @@ func TestLoadSnapshot_PropagatesInteractiveFlag(t *testing.T) {
 	if descs[1].Positional.LabelledBy != "Secondary Action" {
 		t.Fatalf("expected second descriptor labelled_by=Secondary Action, got %q", descs[1].Positional.LabelledBy)
 	}
+	if descs[1].Positional.X != 300 || descs[1].Positional.Y != 640 {
+		t.Fatalf("expected second descriptor x/y=300/640, got %f/%f", descs[1].Positional.X, descs[1].Positional.Y)
+	}
+	if descs[1].Positional.Width != 200 || descs[1].Positional.Height != 44 {
+		t.Fatalf("expected second descriptor width/height=200/44, got %f/%f", descs[1].Positional.Width, descs[1].Positional.Height)
+	}
 }

docs/reference/cli.md

@@ -33,6 +33,11 @@ semantic find "login" --snapshot page.json --format json
 
 # Just refs (for piping)
 semantic find "submit" --snapshot page.json --format refs
+
+# Visual layout hints
+semantic find "button in top right corner" --snapshot page.json
+semantic find "link below the search box" --snapshot page.json
+semantic find "sidebar on the left" --snapshot page.json
 ```
 
 ### `semantic match`
@@ -81,8 +86,34 @@ The CLI expects a JSON array of element descriptors:
 
 ```json
 [
-  {"ref": "e0", "role": "button", "name": "Sign In"},
-  {"ref": "e1", "role": "textbox", "name": "Email"},
-  {"ref": "e2", "role": "link", "name": "Forgot Password"}
+  {
+    "ref": "e0",
+    "role": "button",
+    "name": "Sign In",
+    "interactive": true,
+    "parent": "Auth card",
+    "section": "Header",
+    "x": 920,
+    "y": 16,
+    "width": 96,
+    "height": 32
+  },
+  {
+    "ref": "e1",
+    "role": "textbox",
+    "name": "Email",
+    "positional": {
+      "depth": 3,
+      "sibling_index": 1,
+      "sibling_count": 2,
+      "labelled_by": "Email",
+      "left": 120,
+      "top": 240,
+      "width": 320,
+      "height": 36
+    }
+  }
 ]
 ```
+
+Top-level geometry (`x`, `y`, `top`, `left`, `width`, `height`) and nested `positional` fields are both supported. Supplying coordinates improves results for visual hints such as `top right`, `below`, and `left`.

internal/engine/combined.go

@@ -44,14 +44,26 @@ func (c *CombinedMatcher) Find(ctx context.Context, query string, elements []typ
 		opts.TopK = 3
 	}
 
+	visualHints := parseVisualQueryHints(query)
+	effectiveQuery := query
+	if visualHints.baseQuery != "" {
+		effectiveQuery = visualHints.baseQuery
+	}
+
+	mergeOpts := opts
+	if visualHints.hasHints {
+		mergeOpts.TopK = len(elements)
+	}
+
 	lexW, embW := c.weights(opts)
 
-	lexResult, embResult, err := c.runBoth(ctx, query, elements, opts)
+	lexResult, embResult, err := c.runBoth(ctx, effectiveQuery, elements, opts)
 	if err != nil {
 		return types.FindResult{}, err
 	}
 
-	return c.mergeResults(lexResult, embResult, elements, opts, lexW, embW), nil
+	merged := c.mergeResults(lexResult, embResult, elements, mergeOpts, lexW, embW)
+	return applyVisualHintBoost(merged, visualHints, elements, opts.TopK), nil
 }
 
 func (c *CombinedMatcher) weights(opts types.FindOptions) (float64, float64) {
@@ -110,6 +122,7 @@ type scored struct {
 	ref      string
 	score    float64
 	el       types.ElementDescriptor
+	order    int
 	lexScore float64
 	embScore float64
 }
@@ -133,20 +146,48 @@ func (c *CombinedMatcher) mergeResults(lexResult, embResult types.FindResult, el
 	}
 
 	candidates := make([]scored, 0, len(allRefs))
-	for ref := range allRefs {
+	appendCandidate := func(ref string, el types.ElementDescriptor, order int) {
 		combined := lexW*lexScores[ref] + embW*embScores[ref]
-		if combined >= opts.Threshold {
-			s := scored{ref: ref, score: combined, el: refToElem[ref]}
-			if opts.Explain {
-				s.lexScore = lexW * lexScores[ref]
-				s.embScore = embW * embScores[ref]
-			}
-			candidates = append(candidates, s)
+		if combined < opts.Threshold {
+			return
+		}
+
+		s := scored{ref: ref, score: combined, el: el, order: order}
+		if opts.Explain {
+			s.lexScore = lexW * lexScores[ref]
+			s.embScore = embW * embScores[ref]
+		}
+		candidates = append(candidates, s)
+	}
+
+	for i, el := range elements {
+		if !allRefs[el.Ref] {
+			continue
+		}
+		appendCandidate(el.Ref, el, i)
+		delete(allRefs, el.Ref)
+	}
+
+	if len(allRefs) > 0 {
+		extraRefs := make([]string, 0, len(allRefs))
+		for ref := range allRefs {
+			extraRefs = append(extraRefs, ref)
+		}
+		sort.Strings(extraRefs)
+		for i, ref := range extraRefs {
+			appendCandidate(ref, refToElem[ref], len(elements)+i)
 		}
 	}
 
 	sort.Slice(candidates, func(i, j int) bool {
-		return candidates[i].score > candidates[j].score
+		return rankedMatchLess(
+			candidates[i].score,
+			candidates[i].el,
+			candidates[i].order,
+			candidates[j].score,
+			candidates[j].el,
+			candidates[j].order,
+		)
 	})
 	if len(candidates) > opts.TopK {
 		candidates = candidates[:opts.TopK]

internal/engine/combined_test.go

@@ -97,6 +97,24 @@ func TestCombinedMatcher_TopK(t *testing.T) {
 	}
 }
 
+func TestCombinedMatcher_DeterministicTieBreak(t *testing.T) {
+	m := NewCombinedMatcher(NewHashingEmbedder(128))
+	elements := []types.ElementDescriptor{
+		{Ref: "first", Role: "button", Name: "Open", Positional: types.PositionalHints{Depth: 2, SiblingIndex: 0}},
+		{Ref: "second", Role: "button", Name: "Open", Positional: types.PositionalHints{Depth: 2, SiblingIndex: 0}},
+	}
+
+	for i := 0; i < 100; i++ {
+		result, err := m.Find(context.Background(), "open button", elements, types.FindOptions{Threshold: 0, TopK: 2})
+		if err != nil {
+			t.Fatalf("Find returned error: %v", err)
+		}
+		if result.BestRef != "first" {
+			t.Fatalf("run %d: expected BestRef=first, got %s", i, result.BestRef)
+		}
+	}
+}
+
 func TestCombinedMatcher_ScoresDescending(t *testing.T) {
 	m := NewCombinedMatcher(NewHashingEmbedder(128))
 

internal/engine/embedding.go

@@ -75,18 +75,26 @@ func (m *EmbeddingMatcher) Find(_ context.Context, query string, elements []type
 	type scored struct {
 		desc  types.ElementDescriptor
 		score float64
+		order int
 	}
 
 	var candidates []scored
 	for i, el := range elements {
 		sim := CosineSimilarity(queryVec, contextVecs[i])
 		if sim >= opts.Threshold {
-			candidates = append(candidates, scored{desc: el, score: sim})
+			candidates = append(candidates, scored{desc: el, score: sim, order: i})
 		}
 	}
 
 	sort.Slice(candidates, func(i, j int) bool {
-		return candidates[i].score > candidates[j].score
+		return rankedMatchLess(
+			candidates[i].score,
+			candidates[i].desc,
+			candidates[i].order,
+			candidates[j].score,
+			candidates[j].desc,
+			candidates[j].order,
+		)
 	})
 
 	if len(candidates) > opts.TopK {

internal/engine/embedding_test.go

@@ -187,6 +187,28 @@ func TestEmbeddingMatcher_ThresholdFiltering(t *testing.T) {
 	}
 }
 
+func TestEmbeddingMatcher_TieBreaksByPositionalHints(t *testing.T) {
+	e := newScriptedEmbedder(map[string][]float32{
+		"open button":  {1, 0, 0},
+		"button: Open": {1, 0, 0},
+	})
+	m := NewEmbeddingMatcherWithNeighborWeight(e, 0)
+
+	elements := []types.ElementDescriptor{
+		{Ref: "shallow", Role: "button", Name: "Open", Positional: types.PositionalHints{Depth: 1, SiblingIndex: 1}},
+		{Ref: "deep-left", Role: "button", Name: "Open", Positional: types.PositionalHints{Depth: 3, SiblingIndex: 0}},
+		{Ref: "deep-right", Role: "button", Name: "Open", Positional: types.PositionalHints{Depth: 3, SiblingIndex: 2}},
+	}
+
+	res, err := m.Find(context.Background(), "open button", elements, types.FindOptions{Threshold: 0, TopK: 3})
+	if err != nil {
+		t.Fatalf("Find failed: %v", err)
+	}
+	if res.BestRef != "deep-left" {
+		t.Fatalf("expected deep-left to win tie-break, got %s", res.BestRef)
+	}
+}
+
 func TestEmbeddingMatcher_NeighborContextDisambiguatesRealWorldButtons(t *testing.T) {
 	e := newScriptedEmbedder(map[string][]float32{
 		"laptop add to cart":         {1, 1, 0},