Merge branch 'fix/s3-region-full-url'

Author: h0tak88r

internal/api/programs_api.go

@@ -53,6 +53,7 @@ type ProgramStats struct {
 
 type programScopeRequest struct {
 	Programs []programScopeRequestItem `json:"programs"`
+	Force    bool                      `json:"force"` // bypass the scope cache (manual Refresh)
 }
 
 type programScopeRequestItem struct {
@@ -270,11 +271,13 @@ func apiProgramScopeSummaries(c *gin.Context) {
 		}
 
 		cacheKey := programScopeCacheKey(item.Platform, item.Handle)
-		if summary, ok := getCachedProgramScope(cacheKey); ok {
-			mu.Lock()
-			summaries[cacheKey] = summary
-			mu.Unlock()
-			continue
+		if !req.Force {
+			if summary, ok := getCachedProgramScope(cacheKey); ok {
+				mu.Lock()
+				summaries[cacheKey] = summary
+				mu.Unlock()
+				continue
+			}
 		}
 
 		wg.Add(1)
@@ -288,8 +291,7 @@ func apiProgramScopeSummaries(c *gin.Context) {
 			switch item.Platform {
 			case "h1", "hackerone":
 				if h1Auth != "" {
-					summary = fetchH1ScopeSummary(item.Handle, h1Auth)
-					fetched = true
+					summary, fetched = fetchH1ScopeSummary(item.Handle, h1Auth)
 				}
 			case "bc", "bugcrowd":
 				if bcToken != "" {
@@ -300,6 +302,8 @@ func apiProgramScopeSummaries(c *gin.Context) {
 
 			summary.Platform = item.Platform
 			summary.Handle = item.Handle
+			// Only cache a successful fetch — caching a rate-limited/empty result would
+			// hide the program's real scope until the TTL expires.
 			if fetched {
 				setCachedProgramScope(cacheKey, summary)
 			}
@@ -476,33 +480,45 @@ func enrichH1ScopeCounts(programs []ProgramSummary, auth string) {
 			defer wg.Done()
 			defer func() { <-sem }()
 
-			summary := fetchH1ScopeSummary(p.Handle, auth)
-			p.ScopeTargets = summary.ScopeTargets
-			p.LatestTarget = summary.LatestTarget
-			p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
-			p.LatestTargetBrief = summary.LatestTargetBrief
+			summary, ok := fetchH1ScopeSummary(p.Handle, auth)
+			if ok {
+				p.ScopeTargets = summary.ScopeTargets
+				p.LatestTarget = summary.LatestTarget
+				p.LatestTargetUpdatedAt = summary.LatestTargetUpdatedAt
+				p.LatestTargetBrief = summary.LatestTargetBrief
+			}
 		}(&programs[i])
 	}
 	wg.Wait()
 }
 
-func fetchH1ScopeSummary(handle, auth string) ProgramSummary {
+// fetchH1ScopeSummary returns the scope summary and an ok flag. ok is false when the
+// request fails or H1 returns a non-200 (e.g. 429 rate-limit / 403) — the caller must
+// NOT cache a !ok result, otherwise a transient failure would hide a program's real
+// scope (showing "—") for the whole cache TTL.
+func fetchH1ScopeSummary(handle, auth string) (ProgramSummary, bool) {
 	url := fmt.Sprintf("https://api.hackerone.com/v1/hackers/programs/%s/structured_scopes?page%%5Bsize%%5D=100", handle)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		return ProgramSummary{}
+		return ProgramSummary{}, false
 	}
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("Authorization", "Basic "+auth)
 
 	client := &http.Client{Timeout: 15 * time.Second}
 	resp, err := client.Do(req)
 	if err != nil {
-		return ProgramSummary{}
+		return ProgramSummary{}, false
 	}
 	body, _ := io.ReadAll(resp.Body)
 	resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		// 429 (rate-limited by hammering the API), 403, 401, etc. — signal failure so
+		// the empty result isn't cached and the next refresh retries.
+		return ProgramSummary{}, false
+	}
+
 	scopes := gjson.Get(string(body), "data")
 	summary := ProgramSummary{Platform: "h1", Handle: handle}
 	for _, s := range scopes.Array() {
@@ -518,7 +534,7 @@ func fetchH1ScopeSummary(handle, auth string) ProgramSummary {
 			}
 		}
 	}
-	return summary
+	return summary, true
 }
 
 // ─────────────────────────────────────────────────────────────────────────────

internal/api/ui/index.html

@@ -424,7 +424,7 @@ <h1 class="view-title">Programs</h1>
             <option value="bc">Bugcrowd</option>
             <option value="it">Intigriti</option>
           </select>
-          <input class="search-input" id="programs-search" placeholder="Search programs…" autocomplete="off" oninput="window.ProgramsPage.renderPrograms()" />
+          <input class="search-input" id="programs-search" placeholder="Search programs…" autocomplete="off" oninput="window.ProgramsPage.onSearch()" />
         </div>
 
         <div id="programs-stats-bar" style="display:flex;gap:16px;margin-bottom:12px;font-size:12px;color:rgba(255,255,255,0.5);"></div>

internal/api/ui/pages/programs.js

@@ -184,6 +184,55 @@
     renderStats(false);
   }
 
+  function matchesProgram(p, q) {
+    return (p.name || '').toLowerCase().includes(q) ||
+      (p.handle || '').toLowerCase().includes(q) ||
+      (p.latest_target || '').toLowerCase().includes(q) ||
+      (p.latest_target_brief || '').toLowerCase().includes(q);
+  }
+
+  // When the user searches, the warm program list often carries an empty/rate-limited
+  // scope for the match (the background warmer can't enrich ~800 H1 programs without
+  // hitting the API rate limit). Fetch the matched programs' scope on demand with
+  // force=true so a stale/empty cached value can't keep showing "—".
+  let scopeSearchDebounce;
+  async function hydrateMatchingScope() {
+    const q = (document.getElementById('programs-search')?.value || '').toLowerCase().trim();
+    if (q.length < 2) return;
+    const need = programsData.filter(p =>
+      p.platform && p.handle && !p._scope_loading && !p._scope_fresh &&
+      !(Number(p.scope_targets) > 0) && !p.latest_target && matchesProgram(p, q)
+    ).slice(0, 25);
+    if (!need.length) return;
+    const mySeq = loadSeq;
+    need.forEach(p => { p._scope_loading = true; });
+    renderPrograms();
+    try {
+      const res = await window.apiPost('/api/scope/program-summaries', {
+        programs: need.map(p => ({ platform: p.platform, handle: p.handle, url: p.url })),
+        force: true,
+      });
+      if (mySeq !== loadSeq) return;
+      const summaries = res.summaries || {};
+      need.forEach(p => {
+        const summary = summaries[programKey(p)] || summaries[`${p.platform}:${p.handle}`];
+        if (summary) mergeScopeSummary(p, summary);
+        p._scope_loading = false;
+        p._scope_fresh = true; // attempted a live fetch; don't re-hit on every keystroke
+      });
+    } catch (e) {
+      need.forEach(p => { p._scope_loading = false; });
+    }
+    renderPrograms();
+  }
+
+  // Wired to the search box: filter immediately, then live-fetch scope for matches.
+  function onSearch() {
+    renderPrograms();
+    clearTimeout(scopeSearchDebounce);
+    scopeSearchDebounce = setTimeout(hydrateMatchingScope, 350);
+  }
+
   async function loadPrograms(force = false) {
     const seq = ++loadSeq;
     const platform = document.getElementById('programs-platform-filter')?.value || 'all';
@@ -230,6 +279,8 @@
   // the fresh data once it has likely finished.
   function refreshNow() {
     if (window.showToast) window.showToast('info', 'Refreshing', 'Rebuilding the program cache in the background…');
+    // Allow the on-search live scope fetch to re-attempt after a manual refresh.
+    programsData.forEach(p => { p._scope_fresh = false; });
     loadPrograms(true);
     setTimeout(() => { if (window.state && window.state.view === 'programs') loadPrograms(false); }, 30000);
   }
@@ -354,5 +405,5 @@
     }
   }
 
-  window.ProgramsPage = { loadPrograms, renderPrograms, setSort, refreshNow };
+  window.ProgramsPage = { loadPrograms, renderPrograms, onSearch, setSort, refreshNow };
 })();