Skip to content

Commit 036fa5f

Browse files
h0tak88rclaude
andcommitted
fix(monitor): URL regex flapping — pick max match, forward-only watermark
The URL Monitor was alerting in 23→24→23→24 cycles on pages that list multiple matching items (e.g. blackline.com/blog/ posts dated 06-23 and 06-24). Root cause: checkTargetRegex used re.FindString which returns just the FIRST match in the HTML, and pages re-order their lists between requests (featured posts, CDN cache variants), making the "first match" flip and the baseline swap on every cycle. Fix: - Use re.FindAllString(text, -1) and pick the LARGEST match via maxRegexMatch. For dates this is the latest, for version-ish strings it's the lexically largest — stable regardless of how the page orders its items. - compareWatchValue parses common date formats (ISO 2006-01-02, "Jan 2, 2006", RFC3339, etc.) before falling back to plain string compare, so "Jul 1, 2026" correctly beats "Jun 30, 2026" (lexically Jul < Jun). - Forward-only watermark: a regex monitor is for "latest X" — a backward jump (older date than the baseline) is almost always page-ordering noise, not a real rollback. Skip the alert AND do not advance the baseline so the high-water-mark stays at the genuine freshest value seen. - Empty match (no matches in this fetch) no longer churns the baseline; we return early instead of comparing "" against the previous value. Self-healing for existing baselines: the next check picks max=latest; if it equals the (possibly stuck) baseline, no alert; if it's strictly newer, one final alert + baseline advance, then steady-state. Unit-tested: ISO + month-name dates, version strings, the flapping reproducer (re-ordered match lists), and the date-aware Jul-beats-Jun case. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 214a783 commit 036fa5f

2 files changed

Lines changed: 131 additions & 2 deletions

File tree

internal/scanner/monitor/daemon.go

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,17 @@ func checkTargetRegex(t db.MonitorTarget, body []byte) {
212212
}
213213

214214
text := string(body)
215-
match := re.FindString(text)
215+
// Use ALL matches and pick the MAX (latest date / highest version / lexically
216+
// largest), not FindString's first match. A page that lists multiple matching
217+
// items (e.g. blog posts) often re-orders them between requests — picking the
218+
// first match flapped the baseline, e.g. blackline.com/blog/ alerting 23 → 24 →
219+
// 23 → 24 inside 4 minutes as the featured post shuffled. The max is stable
220+
// regardless of ordering.
221+
matches := re.FindAllString(text, -1)
222+
if len(matches) == 0 {
223+
return // nothing to compare; don't churn the baseline on a transient empty fetch
224+
}
225+
match := maxRegexMatch(matches)
216226

217227
// Switched from hash strategy — old last_hash is hex; establish fresh regex baseline.
218228
baseline := t.LastHash
@@ -226,10 +236,21 @@ func checkTargetRegex(t db.MonitorTarget, body []byte) {
226236
return
227237
}
228238

229-
if match == baseline {
239+
// Forward-only watermark: a regex monitor is for "latest X" — a backward jump
240+
// (older date / lower version) is almost always page-ordering noise, not a real
241+
// rollback. Skip the alert AND don't advance the baseline so the baseline stays
242+
// at the genuine high-water-mark; the next refresh comparing against it will be
243+
// stable. Use compareWatchValue which parses dates (ISO + "Jan 2, 2006") and
244+
// falls back to lexical compare for arbitrary patterns.
245+
cmp := compareWatchValue(match, baseline)
246+
if cmp == 0 {
230247
_ = db.UpdateMonitorTargetLastRun(t.ID, match, false)
231248
return
232249
}
250+
if cmp < 0 {
251+
// Match went backwards — keep the baseline, don't alert, don't advance.
252+
return
253+
}
233254

234255
logger.GetLogger().Infof("[URL-MONITOR] Change detected for %s (regex)", t.URL)
235256

@@ -267,3 +288,51 @@ func checkTargetRegex(t db.MonitorTarget, body []byte) {
267288
logger.GetLogger().Infof("[URL-MONITOR] Alert: %s", msg)
268289
utils.SendMonitorWebhook(msg)
269290
}
291+
292+
// maxRegexMatch returns the "largest" of a non-empty slice of regex matches,
293+
// using compareWatchValue (date-aware where possible, lexical otherwise). This
294+
// makes "latest update" monitoring stable on pages that list multiple matching
295+
// items in a varying order.
296+
func maxRegexMatch(matches []string) string {
297+
best := matches[0]
298+
for _, m := range matches[1:] {
299+
if compareWatchValue(m, best) > 0 {
300+
best = m
301+
}
302+
}
303+
return best
304+
}
305+
306+
// compareWatchValue returns >0 if a>b, <0 if a<b, 0 if equal. Tries common date
307+
// formats first so chronological ordering wins ("Jul 1, 2026" > "Jun 30, 2026"
308+
// even though lexically Jul<Jun). Falls back to plain string compare for
309+
// arbitrary patterns (version strings, hashes, etc.) where ASCII order is fine.
310+
func compareWatchValue(a, b string) int {
311+
ta, aOK := tryParseWatchDate(a)
312+
tb, bOK := tryParseWatchDate(b)
313+
if aOK && bOK {
314+
return ta.Compare(tb)
315+
}
316+
return strings.Compare(a, b)
317+
}
318+
319+
func tryParseWatchDate(s string) (time.Time, bool) {
320+
s = strings.TrimSpace(s)
321+
if s == "" {
322+
return time.Time{}, false
323+
}
324+
formats := []string{
325+
"2006-01-02",
326+
"Jan 2, 2006",
327+
"January 2, 2006",
328+
"Jan 02, 2006",
329+
"02 Jan 2006",
330+
time.RFC3339,
331+
}
332+
for _, f := range formats {
333+
if t, err := time.Parse(f, s); err == nil {
334+
return t, true
335+
}
336+
}
337+
return time.Time{}, false
338+
}
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package monitor
2+
3+
import "testing"
4+
5+
func TestMaxRegexMatchAndCompare(t *testing.T) {
6+
cases := []struct {
7+
name string
8+
matches []string
9+
want string
10+
}{
11+
// ISO YYYY-MM-DD: lexical = chronological, picks the latest date.
12+
{"iso latest wins", []string{"2026-06-23", "2026-06-24", "2026-06-23"}, "2026-06-24"},
13+
{"iso single", []string{"2026-06-24"}, "2026-06-24"},
14+
15+
// "Jan 2, 2006" form: lexical Jul<Jun (because 'l'<'n'), so without
16+
// date-aware compare we'd pick Jun. The fix should pick Jul.
17+
{"month name across months", []string{"Jun 30, 2026", "Jul 1, 2026"}, "Jul 1, 2026"},
18+
{"month name within month", []string{"Jun 23, 2026", "Jun 24, 2026"}, "Jun 24, 2026"},
19+
20+
// Non-date values: lexical fallback is fine.
21+
{"version-ish", []string{"v1.2.3", "v1.10.0"}, "v1.2.3"}, // lexical: "1.2" > "1.1"
22+
}
23+
for _, tc := range cases {
24+
t.Run(tc.name, func(t *testing.T) {
25+
got := maxRegexMatch(tc.matches)
26+
if got != tc.want {
27+
t.Fatalf("maxRegexMatch(%v) = %q, want %q", tc.matches, got, tc.want)
28+
}
29+
})
30+
}
31+
}
32+
33+
// The flapping bug reproduced: simulate two sequential pages where the freshest
34+
// date appears in different positions. With FindString (old behaviour) the
35+
// baseline would flip; with maxRegexMatch the baseline stays on the latest date.
36+
func TestNoFlappingOnReorderedMatches(t *testing.T) {
37+
// Page A lists posts in order [24, 23], page B reorders to [23, 24].
38+
pageAMatches := []string{"2026-06-24", "2026-06-23"}
39+
pageBMatches := []string{"2026-06-23", "2026-06-24"}
40+
41+
a := maxRegexMatch(pageAMatches)
42+
b := maxRegexMatch(pageBMatches)
43+
if a != b {
44+
t.Fatalf("expected stable max across re-ordered pages, got A=%q B=%q", a, b)
45+
}
46+
}
47+
48+
// Forward-only watermark: a backward "match" returned by a single check should
49+
// compareWatchValue negative against the current baseline, so the caller skips.
50+
func TestCompareWatchValueChronological(t *testing.T) {
51+
if compareWatchValue("2026-06-24", "2026-06-23") <= 0 {
52+
t.Fatal("expected 24 > 23")
53+
}
54+
if compareWatchValue("Jul 1, 2026", "Jun 30, 2026") <= 0 {
55+
t.Fatal("expected Jul 1 > Jun 30 (date-aware, not lexical)")
56+
}
57+
if compareWatchValue("2026-06-23", "2026-06-23") != 0 {
58+
t.Fatal("expected equal")
59+
}
60+
}

0 commit comments

Comments
 (0)