Merge branch 'fix/s3-region-full-url'

Author: h0tak88r

internal/api/program_monitor.go

@@ -140,6 +140,25 @@ func collectProgramsForMonitor() []ProgramSummary {
 	return all
 }
 
+// programMonitorKey returns a unique-per-program key for the program_assets store.
+// p.ID (the platform's stable UUID/ID) is preferred when present so a future handle
+// parsing bug can't collapse multiple programs into one bucket — that exact mistake
+// is what caused the Intigriti "/detail" flood: every IT program ended up with
+// Handle="detail" and shared a single program_assets row.
+func programMonitorKey(p ProgramSummary) string {
+	id := strings.TrimSpace(p.ID)
+	if id != "" {
+		return strings.ToLower(p.Platform) + ":id:" + id
+	}
+	return strings.ToLower(p.Platform) + ":" + p.Handle
+}
+
+// floodSuspectThreshold caps a single program's "new asset" alert. Real scope
+// additions are 1-5 assets at a time; a huge "diff" on a watched catalogue is
+// almost always an identifier-collision/key bug. Above this we silently re-baseline
+// and log a warning instead of flooding Discord.
+const floodSuspectThreshold = 20
+
 // sweepProgram fetches one program's current in-scope assets and alerts on new ones.
 func sweepProgram(p ProgramSummary) {
 	assets := fetchProgramAssets(p)
@@ -148,7 +167,7 @@ func sweepProgram(p ProgramSummary) {
 		// otherwise the next successful fetch would report every asset as "new".
 		return
 	}
-	key := programScopeCacheKey(p.Platform, p.Handle)
+	key := programMonitorKey(p)
 	newAssets, firstRun, err := db.RecordProgramScopeAssets(key, assets)
 	if err != nil {
 		logger.GetLogger().Infof("[PROGRAM-MONITOR] record failed for %s: %v", key, err)
@@ -157,6 +176,14 @@ func sweepProgram(p ProgramSummary) {
 	if firstRun || len(newAssets) == 0 {
 		return // baseline run, or nothing new
 	}
+	// Safety cap: a real scope change is incremental. A huge sudden delta almost
+	// always means a key collision (multiple programs mapped to one row) or a
+	// platform API change. Skip the alert rather than spam.
+	if len(newAssets) >= floodSuspectThreshold || len(newAssets) == len(assets) {
+		logger.GetLogger().Infof("[PROGRAM-MONITOR] WARN suspicious delta for %s (%s): %d/%d assets reported new — silently re-baselining, no alert sent",
+			p.Handle, p.Platform, len(newAssets), len(assets))
+		return
+	}
 	alertNewAssets(p, newAssets)
 }
 

internal/api/programs_api.go

@@ -821,9 +821,17 @@ func fetchITPrograms(bbpOnly, includeScope bool) ([]ProgramSummary, error) {
 			if i := strings.Index(detail, "="); i >= 0 && i+1 < len(detail) {
 				programPath = detail[i+1:]
 			}
-			handle := programPath
-			if i := strings.LastIndex(strings.TrimRight(programPath, "/"), "/"); i >= 0 {
-				handle = strings.TrimRight(programPath, "/")[i+1:]
+			// Intigriti's webLinks.detail paths look like
+			//   ".../programs/<company>/<program-handle>/detail"
+			// — the LAST segment is the literal word "detail", not the handle.
+			// Strip a trailing "/detail" first, then take the last segment.
+			// (Without this every IT program gets handle="detail", which collides
+			// into a single program_assets bucket and floods the scope monitor.)
+			handlePath := strings.TrimRight(programPath, "/")
+			handlePath = strings.TrimSuffix(handlePath, "/detail")
+			handle := handlePath
+			if i := strings.LastIndex(handlePath, "/"); i >= 0 {
+				handle = handlePath[i+1:]
 			}
 			name := strings.TrimSpace(rec.Get("name").Str)
 			if name == "" {

internal/app/app.go

@@ -73,6 +73,12 @@ func StartAPI() error {
 	// the persisted state is truthful and monitoring continues after a Docker restart.
 	resumeMonitorsOnStartup()
 
+	// One-shot cleanup: a previous build mis-parsed Intigriti program handles as the
+	// literal "detail" path segment, collapsing every IT program into a single
+	// program_assets bucket and producing a Discord alert flood. Wipe that row so the
+	// monitor re-baselines under the now-correct keys.
+	cleanupCorruptedProgramAssets()
+
 	// Bug-bounty scope-change monitor: slow rolling sweep of all programs that alerts
 	// to MONITOR_WEBHOOK_URL (Discord) whenever a new in-scope asset appears. No-op when
 	// no webhook is configured or PROGRAM_MONITOR=off.
@@ -133,6 +139,21 @@ func reconcileStaleScansOnStartup() {
 	}
 }
 
+// cleanupCorruptedProgramAssets drops program_assets rows poisoned by past
+// identifier-collision bugs (currently: every Intigriti program shared the key
+// "it:detail" because the handle parser took "/detail" as the handle). Safe to call
+// every boot — it only deletes the known-bad keys; healthy rows are untouched.
+func cleanupCorruptedProgramAssets() {
+	if err := db.Init(); err != nil {
+		return
+	}
+	for _, key := range []string{"it:detail"} {
+		if n, err := db.DeleteProgramScopeAssetsByKey(key); err == nil && n > 0 {
+			log.Printf("[INFO] Cleaned %d poisoned program_assets row(s) under key %q.", n, key)
+		}
+	}
+}
+
 // resumeMonitorsOnStartup restarts the monitor daemon goroutines (which don't survive
 // a process restart) when the DB still has monitor targets marked is_running, so polling
 // actually resumes instead of the dashboard merely showing a stale "running" flag.

internal/db/db.go

@@ -162,6 +162,16 @@ func RecordProgramScopeAssets(programKey string, assets []string) ([]string, boo
 	return dbInstance.RecordProgramScopeAssets(programKey, assets)
 }
 
+// DeleteProgramScopeAssetsByKey clears stored assets for a program key.
+func DeleteProgramScopeAssetsByKey(programKey string) (int64, error) {
+	if dbInstance == nil {
+		if err := Init(); err != nil {
+			return 0, err
+		}
+	}
+	return dbInstance.DeleteProgramScopeAssetsByKey(programKey)
+}
+
 // InsertKeyhackTemplate inserts or updates a KeyHack template
 func InsertKeyhackTemplate(keyname, commandTemplate, method, url, header, body, notes, description string) error {
 	if dbInstance == nil {

internal/db/postgres.go

@@ -718,6 +718,15 @@ func (p *PostgresDB) ListProgramScopeAssets(programKey string) ([]string, error)
 	return out, rows.Err()
 }
 
+// DeleteProgramScopeAssetsByKey clears every stored asset for a program key.
+func (p *PostgresDB) DeleteProgramScopeAssetsByKey(programKey string) (int64, error) {
+	tag, err := p.pool.Exec(p.ctx, `DELETE FROM program_assets WHERE program_key = $1;`, programKey)
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete program_assets: %v", err)
+	}
+	return tag.RowsAffected(), nil
+}
+
 // RecordProgramScopeAssets diffs+stores the current asset set, returning newly-seen assets.
 func (p *PostgresDB) RecordProgramScopeAssets(programKey string, assets []string) ([]string, bool, error) {
 	existing, err := p.ListProgramScopeAssets(programKey)

internal/db/sqlite.go

@@ -595,6 +595,16 @@ func (s *SQLiteDB) ListProgramScopeAssets(programKey string) ([]string, error) {
 	return out, rows.Err()
 }
 
+// DeleteProgramScopeAssetsByKey clears every stored asset for a program key.
+func (s *SQLiteDB) DeleteProgramScopeAssetsByKey(programKey string) (int64, error) {
+	res, err := s.db.Exec(`DELETE FROM program_assets WHERE program_key = ?;`, programKey)
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete program_assets: %v", err)
+	}
+	n, _ := res.RowsAffected()
+	return n, nil
+}
+
 // RecordProgramScopeAssets diffs+stores the current asset set, returning newly-seen assets.
 func (s *SQLiteDB) RecordProgramScopeAssets(programKey string, assets []string) ([]string, bool, error) {
 	existing, err := s.ListProgramScopeAssets(programKey)

internal/db/types.go

@@ -121,6 +121,10 @@ type DB interface {
 	// newly-seen assets. firstRun is true when the program had no stored assets yet — in
 	// that case the assets are baselined silently and newAssets is empty (no alert).
 	RecordProgramScopeAssets(programKey string, assets []string) (newAssets []string, firstRun bool, err error)
+	// DeleteProgramScopeAssetsByKey removes every asset row stored under the given key.
+	// Used at boot to clean rows poisoned by past identifier-collision bugs (e.g. all
+	// Intigriti programs ending up under "it:detail").
+	DeleteProgramScopeAssetsByKey(programKey string) (int64, error)
 
 	// DNS Takeover Providers
 	ListVulnerableDNSProviders() (map[string]string, error)