feat(scans): reuse stored httpx URLs to skip re-probing in nuclei scans

Live subdomains are already persisted with their resolved scheme (http_url/
https_url columns, set by the httpx step). Expose that so nuclei scans can target
the scheme-prefixed URL directly and skip httpx:

- db: SubdomainStatus.BestURL() picks the responding scheme (https first);
  db.ListLiveSubdomainURLs(domain) returns every live subdomain as https://host.
- pipeline: fast-path — if a root already has stored live URLs, reuse them and
  skip enumeration + httpx entirely; otherwise enumerate → httpx (which persists
  the scheme URLs) → feed those stored URLs to nuclei.
- global nuclei scan: dump the stored scheme-prefixed URL for probed-live subs
  (bare host fallback for unprobed) so it no longer re-probes.
- nuclei's target sanitizer already preserves http/https schemes, so the URLs
  reach the engine intact.
- test: BestURL scheme-preference table.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: h0tak88r

internal/api/pipeline_api.go

@@ -114,6 +114,19 @@ func runRootPipeline(scanID, template string, newOnly bool, threads, maxRoots in
 		go func(root string) {
 			defer wg.Done()
 			defer func() { <-sem }()
+
+			// Fast path: if this root already has probed live URLs stored, reuse them
+			// and skip enumeration + httpx entirely — the schemes were resolved when
+			// the subdomains were first probed, so nuclei can target them directly.
+			if stored, serr := db.ListLiveSubdomainURLs(root); serr == nil && len(stored) > 0 {
+				mu.Lock()
+				done++
+				allSubs = append(allSubs, stored...)
+				stdLog(scanID, "[OK] %s → %d stored live URLs, skipped httpx (%d/%d roots)", root, len(stored), done, len(targetRoots))
+				mu.Unlock()
+				return
+			}
+
 			subs, err := subdomains.EnumerateSubdomains(root, threads)
 			if err != nil {
 				mu.Lock()
@@ -127,21 +140,20 @@ func runRootPipeline(scanID, template string, newOnly bool, threads, maxRoots in
 					stdLog(scanID, "[WARN] store %s subs failed: %v", root, ierr)
 				}
 			}
-			// httpx: probe for live hosts (marks live in the DB) and scan only those.
-			// Falls back to the raw subdomains if the probe finds nothing (nuclei does
-			// its own probing anyway, so no host is lost).
-			targets := subs
-			liveN := 0
-			if res, herr := livehosts.FilterLiveHosts(root, threads, true); herr == nil && res != nil && res.LiveSubsFile != "" {
-				if live, rerr := utils.ReadLines(res.LiveSubsFile); rerr == nil && len(live) > 0 {
-					targets = live
-					liveN = len(live)
-				}
+			// httpx: probe for live hosts — marks them live AND stores the resolved
+			// scheme-prefixed URL per host in the DB, so future runs skip httpx.
+			_, _ = livehosts.FilterLiveHosts(root, threads, true)
+			// Use the stored scheme-prefixed live URLs as nuclei targets (nuclei then
+			// skips its own probing). Fall back to raw subs if nothing came back live.
+			targets, _ := db.ListLiveSubdomainURLs(root)
+			liveN := len(targets)
+			if liveN == 0 {
+				targets = subs // fallback: nuclei probes these itself
 			}
 			mu.Lock()
 			done++
 			allSubs = append(allSubs, targets...)
-			stdLog(scanID, "[OK] %s → %d subs, %d live (%d/%d roots)", root, len(subs), liveN, done, len(targetRoots))
+			stdLog(scanID, "[OK] %s → %d subs, %d live URLs (%d/%d roots)", root, len(subs), liveN, done, len(targetRoots))
 			mu.Unlock()
 		}(r)
 	}

internal/api/ui_api.go

@@ -676,8 +676,17 @@ func apiRunGlobalNuclei(c *gin.Context) {
 				break
 			}
 			for _, s := range subs {
-				if s.Subdomain != "" {
-					tmpFile.WriteString(s.Subdomain + "\n")
+				// For subdomains already probed live, use the stored scheme-prefixed
+				// URL (https://host) so nuclei skips its own httpx probing. Unprobed
+				// hosts fall back to the bare name (nuclei resolves the scheme itself).
+				target := s.Subdomain
+				if s.IsLive {
+					if u := s.BestURL(); u != "" {
+						target = u
+					}
+				}
+				if target != "" {
+					tmpFile.WriteString(target + "\n")
 					totalSubs++
 				}
 			}

internal/db/db.go

@@ -312,6 +312,28 @@ func ListSubdomainsWithStatus(domain string) ([]SubdomainStatus, error) {
 	return dbInstance.ListSubdomainsWithStatus(domain)
 }
 
+// ListLiveSubdomainURLs returns the stored, scheme-prefixed URL for every live
+// subdomain of a domain (e.g. https://api.example.com). Using these as nuclei
+// targets lets a scan skip the httpx probing step, since the scheme was already
+// resolved when the subdomain was probed. Returns nothing if the domain has no
+// live subdomains stored yet (caller should then run httpx).
+func ListLiveSubdomainURLs(domain string) ([]string, error) {
+	subs, err := ListSubdomainsWithStatus(domain)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]string, 0, len(subs))
+	for _, s := range subs {
+		if !s.IsLive {
+			continue
+		}
+		if u := s.BestURL(); u != "" {
+			out = append(out, u)
+		}
+	}
+	return out, nil
+}
+
 // ListAllSubdomainsPaginated returns a paginated global list of subdomains matching a search.
 func ListAllSubdomainsPaginated(search, techFilter, cnameFilter string, statusFilter int, liveOnly bool, limit, offset int) ([]GlobalSubdomain, int, error) {
 	if dbInstance == nil {

internal/db/types.go

@@ -384,6 +384,26 @@ type SubdomainStatus struct {
 	CNAMEs      string `json:"cnames,omitempty"`
 }
 
+// BestURL returns the scheme-prefixed URL to use for this subdomain, preferring
+// the scheme that actually responded (https first). Used so downstream nuclei
+// scans can target the already-probed URL and skip re-running httpx.
+func (s SubdomainStatus) BestURL() string {
+	switch {
+	case s.HTTPSStatus > 0 && s.HTTPSURL != "":
+		return s.HTTPSURL
+	case s.HTTPStatus > 0 && s.HTTPURL != "":
+		return s.HTTPURL
+	case s.HTTPSURL != "":
+		return s.HTTPSURL
+	case s.HTTPURL != "":
+		return s.HTTPURL
+	case s.Subdomain != "":
+		return "https://" + s.Subdomain
+	default:
+		return ""
+	}
+}
+
 // GlobalSubdomain extends SubdomainStatus with the root domain
 type GlobalSubdomain struct {
 	SubdomainStatus

internal/db/types_test.go

@@ -0,0 +1,44 @@
+package db
+
+import "testing"
+
+func TestSubdomainStatusBestURL(t *testing.T) {
+	cases := []struct {
+		name string
+		in   SubdomainStatus
+		want string
+	}{
+		{
+			name: "https responded → prefer https",
+			in:   SubdomainStatus{Subdomain: "a.example.com", HTTPURL: "http://a.example.com", HTTPSURL: "https://a.example.com", HTTPStatus: 200, HTTPSStatus: 200},
+			want: "https://a.example.com",
+		},
+		{
+			name: "only http responded → use http",
+			in:   SubdomainStatus{Subdomain: "b.example.com", HTTPURL: "http://b.example.com", HTTPSURL: "https://b.example.com", HTTPStatus: 200, HTTPSStatus: 0},
+			want: "http://b.example.com",
+		},
+		{
+			name: "no status but https url present → https",
+			in:   SubdomainStatus{Subdomain: "c.example.com", HTTPSURL: "https://c.example.com"},
+			want: "https://c.example.com",
+		},
+		{
+			name: "bare host only → construct https",
+			in:   SubdomainStatus{Subdomain: "d.example.com"},
+			want: "https://d.example.com",
+		},
+		{
+			name: "empty → empty",
+			in:   SubdomainStatus{},
+			want: "",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := tc.in.BestURL(); got != tc.want {
+				t.Errorf("BestURL() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}