Merge branch 'fix/s3-region-full-url' into testing

Author: h0tak88r

internal/api/programs_api.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+	"github.com/h0tak88r/AutoAR/internal/db"
 	"github.com/tidwall/gjson"
 )
 
@@ -297,7 +298,11 @@ func apiProgramScopeSummaries(c *gin.Context) {
 			case "bc", "bugcrowd":
 				if bcToken != "" {
 					summary = fetchBCScopeSummary(item.Handle, item.URL, bcToken)
-					fetched = true
+					// fetchBCScopeSummary doesn't return ok — all 8 failure paths
+					// return a zero-valued summary. Treat any empty result as a
+					// failed fetch so we don't overwrite persisted good data.
+					// (Matches the enrichBCScopeCounts guard.)
+					fetched = summary.ScopeTargets > 0 || summary.LatestTarget != ""
 				}
 			}
 
@@ -307,6 +312,13 @@ func apiProgramScopeSummaries(c *gin.Context) {
 			// hide the program's real scope until the TTL expires.
 			if fetched {
 				setCachedProgramScope(cacheKey, summary)
+				// Persist last-known-good in the DB — so a search-driven refresh
+				// for a program that the warmer keeps rate-limiting still wins.
+				_ = db.UpsertProgramScope(db.PersistedProgramScope{
+					Platform: summary.Platform, Handle: summary.Handle,
+					ScopeTargets: summary.ScopeTargets, LatestTarget: summary.LatestTarget,
+					LatestTargetUpdatedAt: summary.LatestTargetUpdatedAt, LatestTargetBrief: summary.LatestTargetBrief,
+				})
 				// User-initiated force-fetch (Programs page search) → also feed the
 				// scope-update watch so a genuinely-newer-than-watermark program
 				// alerts Discord immediately, instead of waiting for the next
@@ -494,6 +506,13 @@ func enrichH1ScopeCounts(programs []ProgramSummary, auth string) {
 				p.LatestTarget = summary.LatestTarget
 				p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
 				p.LatestTargetBrief = summary.LatestTargetBrief
+				// Persist last-known-good — failed/rate-limited fetches don't
+				// reach here, so a transient 429 never overwrites real data.
+				_ = db.UpsertProgramScope(db.PersistedProgramScope{
+					Platform: p.Platform, Handle: p.Handle,
+					ScopeTargets: p.ScopeTargets, LatestTarget: p.LatestTarget,
+					LatestTargetUpdatedAt: p.LatestTargetUpdatedAt, LatestTargetBrief: p.LatestTargetBrief,
+				})
 			}
 		}(&programs[i])
 	}
@@ -679,10 +698,19 @@ func enrichBCScopeCounts(programs []ProgramSummary, token string) {
 			time.Sleep(500 * time.Millisecond) // extra spacing for BC
 
 			summary := fetchBCScopeSummary(p.Handle, p.URL, token)
-			p.ScopeTargets = summary.ScopeTargets
-			p.LatestTarget = summary.LatestTarget
-			p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
-			p.LatestTargetBrief = summary.LatestTargetBrief
+			// Only persist (and overwrite the in-memory row) when fetch actually
+			// returned scope — empty/transient failures leave prior data intact.
+			if summary.ScopeTargets > 0 || summary.LatestTarget != "" {
+				p.ScopeTargets = summary.ScopeTargets
+				p.LatestTarget = summary.LatestTarget
+				p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
+				p.LatestTargetBrief = summary.LatestTargetBrief
+				_ = db.UpsertProgramScope(db.PersistedProgramScope{
+					Platform: p.Platform, Handle: p.Handle,
+					ScopeTargets: p.ScopeTargets, LatestTarget: p.LatestTarget,
+					LatestTargetUpdatedAt: p.LatestTargetUpdatedAt, LatestTargetBrief: p.LatestTargetBrief,
+				})
+			}
 		}(&programs[i])
 	}
 	wg.Wait()
@@ -916,10 +944,26 @@ func enrichITScopeCounts(programs []ProgramSummary, token string) {
 			defer func() { <-sem }()
 
 			summary := fetchITScopeSummary(client, token, p.ID)
-			p.ScopeTargets = summary.ScopeTargets
-			if p.LatestTarget == "" {
-				p.LatestTarget = summary.LatestTarget
-				p.LatestTargetBrief = summary.LatestTargetBrief
+			// Only persist on a real scope payload — IT often rate-limits and the
+			// fetcher swallows that as an empty summary; leaving the in-memory row
+			// alone preserves whatever was already there (e.g. from the merged DB).
+			if summary.ScopeTargets > 0 || summary.LatestTarget != "" {
+				p.ScopeTargets = summary.ScopeTargets
+				if p.LatestTarget == "" {
+					p.LatestTarget = summary.LatestTarget
+					p.LatestTargetBrief = summary.LatestTargetBrief
+				}
+				// Overwrite LatestTargetUpdatedAt with the asset-level timestamp from
+				// fetchITScopeSummary (may be empty if IT didn't expose it). DROP the
+				// program-level lastUpdatedAt that fetchITPrograms set — otherwise the
+				// scope-update watch fires on every program edit (bounty bump, etc.)
+				// not just on real scope changes.
+				p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
+				_ = db.UpsertProgramScope(db.PersistedProgramScope{
+					Platform: p.Platform, Handle: p.Handle,
+					ScopeTargets: p.ScopeTargets, LatestTarget: p.LatestTarget,
+					LatestTargetUpdatedAt: p.LatestTargetUpdatedAt, LatestTargetBrief: p.LatestTargetBrief,
+				})
 			}
 		}(&programs[i])
 	}
@@ -967,9 +1011,16 @@ func fetchITScopeSummary(client *http.Client, token, programID string) ProgramSu
 			if target != "" {
 				summary.Assets = append(summary.Assets, target)
 			}
-			if target != "" && summary.LatestTarget == "" {
+			// Per-asset timestamp (try common IT field names). Use the LATEST one
+			// across all assets so the watch fires only when scope genuinely changes
+			// — NOT on unrelated program edits (bounty bump, description change, …)
+			// which would otherwise look like scope updates if we used the program
+			// list's program-level lastUpdatedAt.
+			updatedAt := firstGJSONString(v, "updatedAt", "lastUpdatedAt", "modifiedAt", "addedAt", "createdAt")
+			if target != "" && (summary.LatestTarget == "" || isNewerProgramTime(updatedAt, summary.LatestTargetUpdatedAt)) {
 				summary.LatestTarget = target
 				summary.LatestTargetBrief = firstGJSONString(v, "description", "type.value")
+				summary.LatestTargetUpdatedAt = updatedAt
 			}
 			return true
 		})

internal/api/programs_cache.go

@@ -134,6 +134,13 @@ func buildProgramsPayload() programsCachePayload {
 
 	wg.Wait()
 
+	// Overlay last-known-good scope from the program_scope DB table. This is the
+	// key persistence guarantee: a program previously fetched successfully keeps
+	// its real values on the dashboard even when this refresh's enrichment came
+	// back empty (e.g. rate-limited). The enrich functions ONLY upsert on success,
+	// so anything in the DB is real data — never an empty value masquerading.
+	overlayPersistedProgramScope(all)
+
 	return programsCachePayload{
 		Programs:    all,
 		HasH1Token:  os.Getenv("H1_USERNAME") != "" && os.Getenv("H1_TOKEN") != "",
@@ -143,6 +150,42 @@ func buildProgramsPayload() programsCachePayload {
 	}
 }
 
+// overlayPersistedProgramScope fills any program row whose fresh enrichment left
+// scope empty (ScopeTargets==0 AND no LatestTarget) from the program_scope DB
+// table. Rows where this refresh DID get fresh scope are left alone — the latest
+// successful fetch always wins. Mutates in place.
+func overlayPersistedProgramScope(programs []ProgramSummary) {
+	persisted, err := db.LoadProgramScopes()
+	if err != nil || len(persisted) == 0 {
+		return
+	}
+	restored := 0
+	for i := range programs {
+		p := &programs[i]
+		// Only treat scope as "fresh this refresh" when we actually got a target
+		// or a count. LatestTargetUpdatedAt can be populated from the program-list
+		// API (e.g. IT's program-level lastUpdatedAt) even when scope enrich
+		// returned nothing — checking it here would suppress the overlay and
+		// leave the dashboard showing "—".
+		if p.ScopeTargets > 0 || p.LatestTarget != "" {
+			continue // this refresh enriched scope — keep the fresh value
+		}
+		key := strings.ToLower(p.Platform) + ":" + p.Handle
+		s, ok := persisted[key]
+		if !ok || (s.ScopeTargets == 0 && s.LatestTarget == "") {
+			continue
+		}
+		p.ScopeTargets = s.ScopeTargets
+		p.LatestTarget = s.LatestTarget
+		p.LatestTargetUpdatedAt = s.LatestTargetUpdatedAt
+		p.LatestTargetBrief = s.LatestTargetBrief
+		restored++
+	}
+	if restored > 0 {
+		log.Printf("[PROGRAMS] preserved last-known-good scope for %d program(s) where this refresh came back empty", restored)
+	}
+}
+
 // refreshProgramsCache rebuilds and persists the cache. It is single-flight:
 // if a refresh is already running, it returns false immediately instead of
 // launching a second concurrent fetch. Returns true when it persisted a build.

internal/db/db.go

@@ -192,6 +192,26 @@ func DeleteMonitorChangesByType(changeType string) (int64, error) {
 	return dbInstance.DeleteMonitorChangesByType(changeType)
 }
 
+// UpsertProgramScope persists last-known-good scope for one program.
+func UpsertProgramScope(s PersistedProgramScope) error {
+	if dbInstance == nil {
+		if err := Init(); err != nil {
+			return err
+		}
+	}
+	return dbInstance.UpsertProgramScope(s)
+}
+
+// LoadProgramScopes returns every persisted scope row keyed by platform:handle.
+func LoadProgramScopes() (map[string]PersistedProgramScope, error) {
+	if dbInstance == nil {
+		if err := Init(); err != nil {
+			return nil, err
+		}
+	}
+	return dbInstance.LoadProgramScopes()
+}
+
 // InsertKeyhackTemplate inserts or updates a KeyHack template
 func InsertKeyhackTemplate(keyname, commandTemplate, method, url, header, body, notes, description string) error {
 	if dbInstance == nil {

internal/db/postgres.go

@@ -246,6 +246,20 @@ func (p *PostgresDB) InitSchema() error {
 	);
 	CREATE INDEX IF NOT EXISTS idx_program_assets_key ON program_assets(program_key);
 
+	-- Persistent last-known-good scope per program. Catalogue payload overlays
+	-- these onto the freshly-fetched program list — a rate-limited refresh never
+	-- wipes a program's previously-known scope.
+	CREATE TABLE IF NOT EXISTS program_scope (
+		platform VARCHAR(16) NOT NULL,
+		handle VARCHAR(255) NOT NULL,
+		scope_targets INTEGER DEFAULT 0,
+		latest_target VARCHAR(1024) DEFAULT '',
+		latest_target_updated_at VARCHAR(64) DEFAULT '',
+		latest_target_brief VARCHAR(2048) DEFAULT '',
+		fetched_at TIMESTAMP DEFAULT NOW(),
+		PRIMARY KEY (platform, handle)
+	);
+
 	-- Create keyhack_templates table with proper constraints
 	CREATE TABLE IF NOT EXISTS keyhack_templates (
 		id SERIAL PRIMARY KEY,
@@ -727,6 +741,52 @@ func (p *PostgresDB) DeleteProgramScopeAssetsByKey(programKey string) (int64, er
 	return tag.RowsAffected(), nil
 }
 
+// UpsertProgramScope writes the last-known-good scope for one program. Only
+// called on a SUCCESSFUL fetch — failed/rate-limited fetches don't touch the
+// row, so prior data is preserved.
+func (p *PostgresDB) UpsertProgramScope(s PersistedProgramScope) error {
+	platform := strings.ToLower(strings.TrimSpace(s.Platform))
+	handle := strings.TrimSpace(s.Handle)
+	if platform == "" || handle == "" {
+		return nil
+	}
+	_, err := p.pool.Exec(p.ctx, `
+		INSERT INTO program_scope (platform, handle, scope_targets, latest_target, latest_target_updated_at, latest_target_brief, fetched_at)
+		VALUES ($1, $2, $3, $4, $5, $6, NOW())
+		ON CONFLICT (platform, handle) DO UPDATE SET
+			scope_targets = EXCLUDED.scope_targets,
+			latest_target = EXCLUDED.latest_target,
+			latest_target_updated_at = EXCLUDED.latest_target_updated_at,
+			latest_target_brief = EXCLUDED.latest_target_brief,
+			fetched_at = EXCLUDED.fetched_at;
+	`, platform, handle, s.ScopeTargets, s.LatestTarget, s.LatestTargetUpdatedAt, s.LatestTargetBrief)
+	if err != nil {
+		return fmt.Errorf("failed to upsert program_scope: %v", err)
+	}
+	return nil
+}
+
+// LoadProgramScopes returns every persisted scope row keyed by "<platform>:<handle>".
+func (p *PostgresDB) LoadProgramScopes() (map[string]PersistedProgramScope, error) {
+	rows, err := p.pool.Query(p.ctx, `
+		SELECT platform, handle, scope_targets, latest_target, latest_target_updated_at, latest_target_brief, fetched_at
+		FROM program_scope;
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query program_scope: %v", err)
+	}
+	defer rows.Close()
+	out := make(map[string]PersistedProgramScope)
+	for rows.Next() {
+		var s PersistedProgramScope
+		if err := rows.Scan(&s.Platform, &s.Handle, &s.ScopeTargets, &s.LatestTarget, &s.LatestTargetUpdatedAt, &s.LatestTargetBrief, &s.FetchedAt); err != nil {
+			return nil, fmt.Errorf("failed to scan program_scope: %v", err)
+		}
+		out[strings.ToLower(s.Platform)+":"+s.Handle] = s
+	}
+	return out, rows.Err()
+}
+
 // TruncateProgramScopeAssets wipes every program_assets row.
 func (p *PostgresDB) TruncateProgramScopeAssets() (int64, error) {
 	tag, err := p.pool.Exec(p.ctx, `DELETE FROM program_assets;`)

internal/db/sqlite.go

@@ -138,6 +138,20 @@ func (s *SQLiteDB) InitSchema() error {
 	);
 	CREATE INDEX IF NOT EXISTS idx_program_assets_key ON program_assets(program_key);
 
+	-- Persistent last-known-good scope per program. The catalogue payload overlays
+	-- these on top of the freshly-fetched program list, so a rate-limited refresh
+	-- never wipes a program's previously-known scope from the dashboard.
+	CREATE TABLE IF NOT EXISTS program_scope (
+		platform TEXT NOT NULL,
+		handle TEXT NOT NULL,
+		scope_targets INTEGER DEFAULT 0,
+		latest_target TEXT DEFAULT '',
+		latest_target_updated_at TEXT DEFAULT '',
+		latest_target_brief TEXT DEFAULT '',
+		fetched_at TIMESTAMP DEFAULT (datetime('now')),
+		PRIMARY KEY (platform, handle)
+	);
+
 	-- Create keyhack_templates table
 	CREATE TABLE IF NOT EXISTS keyhack_templates (
 		id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -605,6 +619,52 @@ func (s *SQLiteDB) DeleteProgramScopeAssetsByKey(programKey string) (int64, erro
 	return n, nil
 }
 
+// UpsertProgramScope writes the last-known-good scope for one program. Called
+// only on a SUCCESSFUL fetch — failed fetches don't touch the table, so prior
+// data is preserved.
+func (s *SQLiteDB) UpsertProgramScope(p PersistedProgramScope) error {
+	platform := strings.ToLower(strings.TrimSpace(p.Platform))
+	handle := strings.TrimSpace(p.Handle)
+	if platform == "" || handle == "" {
+		return nil
+	}
+	_, err := s.db.Exec(`
+		INSERT INTO program_scope (platform, handle, scope_targets, latest_target, latest_target_updated_at, latest_target_brief, fetched_at)
+		VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
+		ON CONFLICT (platform, handle) DO UPDATE SET
+			scope_targets = excluded.scope_targets,
+			latest_target = excluded.latest_target,
+			latest_target_updated_at = excluded.latest_target_updated_at,
+			latest_target_brief = excluded.latest_target_brief,
+			fetched_at = excluded.fetched_at;
+	`, platform, handle, p.ScopeTargets, p.LatestTarget, p.LatestTargetUpdatedAt, p.LatestTargetBrief)
+	if err != nil {
+		return fmt.Errorf("failed to upsert program_scope: %v", err)
+	}
+	return nil
+}
+
+// LoadProgramScopes returns every persisted scope row keyed by "<platform>:<handle>".
+func (s *SQLiteDB) LoadProgramScopes() (map[string]PersistedProgramScope, error) {
+	rows, err := s.db.Query(`
+		SELECT platform, handle, scope_targets, latest_target, latest_target_updated_at, latest_target_brief, fetched_at
+		FROM program_scope;
+	`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query program_scope: %v", err)
+	}
+	defer rows.Close()
+	out := make(map[string]PersistedProgramScope)
+	for rows.Next() {
+		var p PersistedProgramScope
+		if err := rows.Scan(&p.Platform, &p.Handle, &p.ScopeTargets, &p.LatestTarget, &p.LatestTargetUpdatedAt, &p.LatestTargetBrief, &p.FetchedAt); err != nil {
+			return nil, fmt.Errorf("failed to scan program_scope: %v", err)
+		}
+		out[strings.ToLower(p.Platform)+":"+p.Handle] = p
+	}
+	return out, rows.Err()
+}
+
 // TruncateProgramScopeAssets wipes every program_assets row.
 func (s *SQLiteDB) TruncateProgramScopeAssets() (int64, error) {
 	res, err := s.db.Exec(`DELETE FROM program_assets;`)

internal/db/types.go

@@ -132,6 +132,15 @@ type DB interface {
 	// Used to scrub the historical flood of false "new_program_asset" entries.
 	DeleteMonitorChangesByType(changeType string) (int64, error)
 
+	// UpsertProgramScope persists the last-known-good scope summary for a single
+	// program. Only called on SUCCESSFUL fetches — a failed/rate-limited fetch must
+	// NOT call this, otherwise we'd overwrite good data with empty values.
+	UpsertProgramScope(s PersistedProgramScope) error
+	// LoadProgramScopes returns every persisted scope row keyed by
+	// "<lower(platform)>:<handle>" so the warmer can overlay it onto a fresh
+	// catalogue payload (preserving prior scope when a refresh's enrichment fails).
+	LoadProgramScopes() (map[string]PersistedProgramScope, error)
+
 	// DNS Takeover Providers
 	ListVulnerableDNSProviders() (map[string]string, error)
 	AddVulnerableDNSProvider(name, fingerprint string) error
@@ -279,6 +288,21 @@ type JSEndpoint struct {
 	SourceJS string // the JS URL it was found in (best-effort)
 }
 
+// PersistedProgramScope is the last-known-good scope summary for a single
+// bug-bounty program. UpsertProgramScope writes one of these PER successful
+// fetch; a failed/rate-limited fetch must never call upsert. The serving layer
+// overlays this on top of the freshly-built catalogue so the dashboard keeps
+// showing real values even when this refresh's enrichment came back empty.
+type PersistedProgramScope struct {
+	Platform              string // "h1" | "bc" | "it"
+	Handle                string // platform handle (e.g. "ryan-bbp")
+	ScopeTargets          int
+	LatestTarget          string
+	LatestTargetUpdatedAt string
+	LatestTargetBrief     string
+	FetchedAt             time.Time
+}
+
 // MonitorChange records a detected change for history/querying
 type MonitorChange struct {
 	ID          int