Recon increment 2: URL/endpoint harvesting (urls/extract/fingerprint)

exegol/my-resources/bin/recon-extract

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+: "${DOMAIN:?load an engagement first (DOMAIN unset) — use: dl <target>}"
+OUT="${WORKSPACE:-$PWD}/recon"; mkdir -p "$OUT"
+
+# Extract endpoints + secrets from the crawled JS files; feed endpoints back
+# into the URL base.
+[[ -s "$OUT/js.txt" ]] || { echo "[!] run recon-crawl first"; exit 1; }
+: > "$OUT/endpoints.txt"; : > "$OUT/secrets.txt"
+while read -r js; do
+    [[ -n "$js" ]] || continue
+    if command -v xnLinkFinder >/dev/null 2>&1; then
+        if xnLinkFinder -i "$js" -o "$OUT/_xn.txt" 2>/dev/null; then cat "$OUT/_xn.txt" >> "$OUT/endpoints.txt"; fi
+    fi
+    if command -v linkfinder >/dev/null 2>&1; then linkfinder -i "$js" -o cli 2>/dev/null >> "$OUT/endpoints.txt" || true; fi
+    if command -v secretfinder >/dev/null 2>&1; then secretfinder -i "$js" -o cli 2>/dev/null >> "$OUT/secrets.txt" || true; fi
+done < "$OUT/js.txt"
+rm -f "$OUT/_xn.txt"
+sort -u "$OUT/endpoints.txt" -o "$OUT/endpoints.txt" 2>/dev/null || true
+if [[ -f "$OUT/urls_all.txt" ]]; then
+    cat "$OUT/endpoints.txt" >> "$OUT/urls_all.txt"
+    sort -u "$OUT/urls_all.txt" -o "$OUT/urls_all.txt"
+fi
+echo "[+] $(wc -l < "$OUT/endpoints.txt") endpoints, $(wc -l < "$OUT/secrets.txt") secret hits"

exegol/my-resources/bin/recon-fingerprint

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+: "${DOMAIN:?load an engagement first (DOMAIN unset) — use: dl <target>}"
+OUT="${WORKSPACE:-$PWD}/recon"; mkdir -p "$OUT"
+
+# Technology fingerprint of the alive hosts.
+[[ -s "$OUT/subdomains_alive.txt" ]] || { echo "[!] run recon-alive first"; exit 1; }
+echo "[*] tech fingerprint"
+httpx-toolkit -l "$OUT/subdomains_alive.txt" -td -silent 2>/dev/null > "$OUT/tech.txt" || true
+if command -v whatweb >/dev/null 2>&1; then whatweb -i "$OUT/subdomains_alive.txt" --no-errors >> "$OUT/tech.txt" 2>/dev/null || true; fi
+echo "[+] tech → $OUT/tech.txt"

exegol/my-resources/bin/recon-full

@@ -7,7 +7,10 @@ mkdir -p "$OUT"
 echo "[*] full recon pipeline for $DOMAIN"
 recon-subs
 recon-alive
+recon-fingerprint || echo "[i] fingerprint step skipped"
 recon-crawl
+recon-urls || echo "[i] urls step skipped"
 recon-loot || echo "[i] loot step skipped"
+recon-extract || echo "[i] extract step skipped"
 recon-sourcemaps || echo "[i] sourcemaps step skipped"
 echo "[+] done → $OUT"

exegol/my-resources/bin/recon-urls

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+: "${DOMAIN:?load an engagement first (DOMAIN unset) — use: dl <target>}"
+OUT="${WORKSPACE:-$PWD}/recon"; mkdir -p "$OUT"
+
+# Aggregate all passive URL sources into one deduplicated base.
+echo "[*] aggregating url sources for $DOMAIN"
+{
+    [[ -f "$OUT/allurls.txt" ]] && cat "$OUT/allurls.txt"
+    gau "$DOMAIN" 2>/dev/null || true
+    if command -v waybackurls >/dev/null 2>&1; then printf '%s\n' "$DOMAIN" | waybackurls 2>/dev/null || true; fi
+    if command -v urlfinder   >/dev/null 2>&1; then urlfinder -d "$DOMAIN" -silent 2>/dev/null || true; fi
+    if command -v waymore     >/dev/null 2>&1; then waymore -i "$DOMAIN" -mode U 2>/dev/null || true; fi
+} | sort -u > "$OUT/urls_all.txt"
+echo "[+] $(wc -l < "$OUT/urls_all.txt") urls → $OUT/urls_all.txt"

exegol/my-resources/fragments/load_user_setup.dotsec.sh

@@ -9,3 +9,7 @@ fi
 if ! command -v unwebpack-sourcemap >/dev/null 2>&1; then
     npm install -g unwebpack-sourcemap 2>/dev/null || true
 fi
+# url/endpoint harvesting helpers (ProjectDiscovery + LinkFinder are already in Exegol)
+if ! command -v waybackurls >/dev/null 2>&1; then
+    go install github.com/tomnomnom/waybackurls@latest 2>/dev/null || true
+fi

tests/exegol.bats

@@ -50,7 +50,7 @@ run_deploy() { bash "${DOTSEC_HOME}/exegol/my-resources/deploy.sh"; }
 }
 
 @test "recon scripts fail fast when DOMAIN is unset" {
-    for s in recon-subs recon-alive recon-crawl recon-loot recon-sourcemaps recon-full; do
+    for s in recon-subs recon-alive recon-crawl recon-loot recon-sourcemaps recon-full recon-urls recon-extract recon-fingerprint; do
         run env -u DOMAIN bash "${DOTSEC_HOME}/exegol/my-resources/bin/$s"
         [ "$status" -ne 0 ] || { echo "$s did not guard DOMAIN"; false; }
     done