Skip to content

Commit 16cb112

Browse files
h0tak88rh0tak88r
committed
feat: Enhanced lite scan, nuclei integration, and external URL collection
- Fixed nuclei scan results not being sent (path mismatch fix) - Increased concurrency for misconfig (200) and nuclei (500) scans - Optimized all lite scan phases to use live hosts from Step 1 - Added external URL collection from VirusTotal, Wayback, URLScan, OTX, Common Crawl - Enhanced URLs module with API key support from .env - Fixed nuclei scan to use live hosts file and scan both public and custom templates - Added real-time file sending via webhook for all lite scan phases - Improved concurrency across all scan phases (cnames, dns, misconfig, reflection) - Added dangling IP detection to DNS takeover module - Enhanced backup scan to use live hosts file - Fixed various race conditions and resource leaks
1 parent 5ec7a51 commit 16cb112

21 files changed

Lines changed: 2950 additions & 191 deletions

File tree

‎cmd/autoar/main.go‎

Lines changed: 456 additions & 33 deletions
Large diffs are not rendered by default.

‎go.mod‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,15 @@ require (
1313
github.com/h0tak88r/confused2 v0.0.0-20251024120607-be290e443862
1414
github.com/hahwul/dalfox/v2 v2.12.0
1515
github.com/jackc/pgx/v5 v5.7.5
16+
github.com/joeguo/tldextract v0.0.0-20220507100122-d83daa6adef8
1617
github.com/juju/persistent-cookiejar v1.0.0
1718
github.com/majd/ipatool/v2 v2.2.0
1819
github.com/projectdiscovery/dnsx v1.2.2
1920
github.com/projectdiscovery/goflags v0.1.74
2021
github.com/projectdiscovery/httpx v1.7.4
2122
github.com/projectdiscovery/naabu/v2 v2.3.7
2223
github.com/projectdiscovery/subfinder/v2 v2.6.7
24+
github.com/sw33tLie/bbscope v0.0.0-20251113222800-c453973e83dd
2325
golang.org/x/time v0.14.0
2426
gopkg.in/yaml.v3 v3.0.1
2527
modernc.org/sqlite v1.40.1
@@ -38,6 +40,7 @@ require (
3840
github.com/STARRY-S/zip v0.2.3 // indirect
3941
github.com/Ullaakut/nmap/v3 v3.0.6 // indirect
4042
github.com/VividCortex/ewma v1.2.0 // indirect
43+
github.com/adrg/xdg v0.4.0 // indirect
4144
github.com/akrylysov/pogreb v0.10.2 // indirect
4245
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
4346
github.com/andybalholm/brotli v1.2.0 // indirect
@@ -93,6 +96,8 @@ require (
9396
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
9497
github.com/dustin/go-humanize v1.0.1 // indirect
9598
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
99+
github.com/ffuf/ffuf/v2 v2.1.0 // indirect
100+
github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693 // indirect
96101
github.com/fsnotify/fsnotify v1.7.0 // indirect
97102
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
98103
github.com/gaissmai/bart v0.26.0 // indirect
@@ -137,7 +142,6 @@ require (
137142
github.com/jackc/pgpassfile v1.0.0 // indirect
138143
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
139144
github.com/jackc/puddle/v2 v2.2.2 // indirect
140-
github.com/joeguo/tldextract v0.0.0-20220507100122-d83daa6adef8 // indirect
141145
github.com/json-iterator/go v1.1.12 // indirect
142146
github.com/juju/go4 v0.0.0-20160222163258-40d72ab9641a // indirect
143147
github.com/kataras/jwt v0.1.10 // indirect
@@ -174,6 +178,7 @@ require (
174178
github.com/olekukonko/errors v1.1.0 // indirect
175179
github.com/olekukonko/ll v0.0.9 // indirect
176180
github.com/olekukonko/tablewriter v1.0.8 // indirect
181+
github.com/pelletier/go-toml v1.9.5 // indirect
177182
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
178183
github.com/pierrec/lz4/v4 v4.1.22 // indirect
179184
github.com/pkg/errors v0.9.1 // indirect
@@ -227,7 +232,6 @@ require (
227232
github.com/spf13/pflag v1.0.6 // indirect
228233
github.com/spf13/viper v1.18.2 // indirect
229234
github.com/subosito/gotenv v1.6.0 // indirect
230-
github.com/sw33tLie/bbscope v0.0.0-20251113222800-c453973e83dd // indirect
231235
github.com/syndtr/goleveldb v1.0.0 // indirect
232236
github.com/tidwall/btree v1.7.0 // indirect
233237
github.com/tidwall/buntdb v1.3.1 // indirect
@@ -284,5 +288,3 @@ require (
284288
modernc.org/mathutil v1.7.1 // indirect
285289
modernc.org/memory v1.11.0 // indirect
286290
)
287-
288-
replace github.com/sw33tLie/bbscope => ./bbscope

‎go.sum‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ github.com/Ullaakut/nmap/v3 v3.0.6 h1:ZCQ70TQp97f/YqIFhlzFMDi5xVDeA0CwMbNeJZGA//
7070
github.com/Ullaakut/nmap/v3 v3.0.6/go.mod h1:dd5K68P7LHc5nKrFwQx6EdTt61O9UN5x3zn1R4SLcco=
7171
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
7272
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
73+
github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
74+
github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
7375
github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78=
7476
github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
7577
github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE=
@@ -253,6 +255,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
253255
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
254256
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
255257
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
258+
github.com/ffuf/ffuf/v2 v2.1.0 h1:0w+MBVMhc52tb5y5onzdkNJVt+hkStH6rVGRdorxd+o=
259+
github.com/ffuf/ffuf/v2 v2.1.0/go.mod h1:R+WpzI2JMHxiWtq8f/lGy72rOgmWC1O+OoxfEeg0yOc=
260+
github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693 h1:fdlgw33oLPzRpoHa4ppDFX5EcmzHHychPrO5xXmzxqc=
261+
github.com/ffuf/pencode v0.0.0-20230421231718-2cea7e60a693/go.mod h1:Qmgn2URTRtZ5wMntUke1+/G7z8rofTFHG1EvN3addNY=
256262
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
257263
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
258264
github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20=
@@ -433,6 +439,8 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng
433439
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
434440
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
435441
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
442+
github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
443+
github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
436444
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
437445
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
438446
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
@@ -629,6 +637,8 @@ github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhA
629637
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
630638
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
631639
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
640+
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
641+
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
632642
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
633643
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
634644
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
@@ -817,6 +827,8 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
817827
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
818828
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
819829
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
830+
github.com/sw33tLie/bbscope v0.0.0-20251113222800-c453973e83dd h1:kXtyUNyBqC245GlK3bR7Y/vFDprLo9LZfcOyktNFSoM=
831+
github.com/sw33tLie/bbscope v0.0.0-20251113222800-c453973e83dd/go.mod h1:3okqkgLXMMus8LdkWv7nq8S4hmU7Nhke5kCNV0jhKro=
820832
github.com/sw33tLie/fleex v0.0.0-20210708174758-524c14fa45e5/go.mod h1:coJoqdGdu/N4M3dDko4LZVf8ntOYG/Ar922UJ2J8tow=
821833
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
822834
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -1161,6 +1173,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
11611173
golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11621174
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11631175
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1176+
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11641177
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11651178
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
11661179
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

‎internal/modules/cnames/cnames.go‎

Lines changed: 97 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ package cnames
22

33
import (
44
"bufio"
5+
"context"
56
"fmt"
67
"log"
78
"os"
89
"path/filepath"
910
"strings"
11+
"sync"
12+
"time"
1013

1114
"github.com/h0tak88r/AutoAR/internal/modules/subdomains"
1215
"github.com/h0tak88r/AutoAR/internal/modules/utils"
@@ -20,15 +23,36 @@ type Result struct {
2023
OutputFile string
2124
}
2225

26+
// Options for CNAME collection
27+
type Options struct {
28+
Domain string
29+
Threads int
30+
Timeout time.Duration
31+
}
32+
2333
// CollectCNAMEs mirrors the behaviour of modules/cnames.sh using Go.
2434
// It ensures subdomains, then uses dnsx (if available) to resolve CNAME records.
2535
func CollectCNAMEs(domain string) (*Result, error) {
26-
if domain == "" {
36+
return CollectCNAMEsWithOptions(Options{
37+
Domain: domain,
38+
Threads: 100, // Default 100 concurrent DNS queries
39+
Timeout: 5 * time.Minute, // 5 minute timeout
40+
})
41+
}
42+
43+
// CollectCNAMEsWithOptions collects CNAME records with custom options
44+
func CollectCNAMEsWithOptions(opts Options) (*Result, error) {
45+
if opts.Domain == "" {
2746
return nil, fmt.Errorf("domain is required")
2847
}
29-
48+
if opts.Threads <= 0 {
49+
opts.Threads = 100
50+
}
51+
if opts.Timeout <= 0 {
52+
opts.Timeout = 5 * time.Minute
53+
}
3054
resultsDir := utils.GetResultsDir()
31-
domainDir := filepath.Join(resultsDir, domain)
55+
domainDir := filepath.Join(resultsDir, opts.Domain)
3256
subsDir := filepath.Join(domainDir, "subs")
3357
if err := utils.EnsureDir(subsDir); err != nil {
3458
return nil, fmt.Errorf("failed to create subs dir: %w", err)
@@ -38,8 +62,8 @@ func CollectCNAMEs(domain string) (*Result, error) {
3862

3963
// Ensure we have subdomains – reuse Go subdomains module
4064
if _, err := os.Stat(allSubs); err != nil {
41-
log.Printf("[INFO] all-subs.txt missing, enumerating subdomains for %s", domain)
42-
subs, err := subdomains.EnumerateSubdomains(domain, 100)
65+
log.Printf("[INFO] all-subs.txt missing, enumerating subdomains for %s", opts.Domain)
66+
subs, err := subdomains.EnumerateSubdomains(opts.Domain, 100)
4367
if err != nil {
4468
return nil, fmt.Errorf("failed to enumerate subdomains: %w", err)
4569
}
@@ -70,39 +94,91 @@ func CollectCNAMEs(domain string) (*Result, error) {
7094
}
7195

7296
if len(targets) == 0 {
73-
log.Printf("[WARN] No subdomains found; creating empty CNAME file for %s", domain)
97+
log.Printf("[WARN] No subdomains found; creating empty CNAME file for %s", opts.Domain)
7498
if err := writeLines(out, nil); err != nil {
7599
return nil, fmt.Errorf("failed to initialise %s: %w", out, err)
76100
}
77101
return &Result{
78-
Domain: domain,
102+
Domain: opts.Domain,
79103
Records: 0,
80104
OutputFile: out,
81105
}, nil
82106
}
83107

84-
log.Printf("[INFO] Collecting CNAME records for %s via dnsx library", domain)
108+
log.Printf("[INFO] Collecting CNAME records for %s via dnsx library (threads: %d)", opts.Domain, opts.Threads)
85109

86110
// Initialize dnsx client
87111
dnsClient, err := dnsx.New(dnsx.DefaultOptions)
88112
if err != nil {
89113
return nil, fmt.Errorf("failed to create dnsx client: %w", err)
90114
}
91115

92-
// Collect CNAME records
116+
// Create context with timeout
117+
ctx, cancel := context.WithTimeout(context.Background(), opts.Timeout)
118+
defer cancel()
119+
120+
// Collect CNAME records with concurrency
93121
var cnameRecords []string
94-
for _, target := range targets {
95-
// Query CNAME record
96-
results, err := dnsClient.QueryOne(target)
97-
if err != nil {
98-
continue
99-
}
100-
// Extract CNAME from response
101-
if results != nil && len(results.CNAME) > 0 {
102-
for _, cname := range results.CNAME {
103-
cnameRecords = append(cnameRecords, fmt.Sprintf("%s CNAME %s", target, cname))
122+
var recordsMutex sync.Mutex
123+
124+
// Worker pool
125+
jobs := make(chan string, len(targets))
126+
var wg sync.WaitGroup
127+
128+
// Start workers
129+
for i := 0; i < opts.Threads; i++ {
130+
wg.Add(1)
131+
go func() {
132+
defer wg.Done()
133+
for target := range jobs {
134+
// Check context cancellation
135+
select {
136+
case <-ctx.Done():
137+
return
138+
default:
139+
}
140+
141+
// Query CNAME record
142+
results, err := dnsClient.QueryOne(target)
143+
if err != nil {
144+
continue
145+
}
146+
// Extract CNAME from response
147+
if results != nil && len(results.CNAME) > 0 {
148+
recordsMutex.Lock()
149+
for _, cname := range results.CNAME {
150+
cnameRecords = append(cnameRecords, fmt.Sprintf("%s CNAME %s", target, cname))
151+
}
152+
recordsMutex.Unlock()
153+
}
154+
}
155+
}()
156+
}
157+
158+
// Send jobs
159+
go func() {
160+
defer close(jobs)
161+
for _, target := range targets {
162+
select {
163+
case jobs <- target:
164+
case <-ctx.Done():
165+
return
104166
}
105167
}
168+
}()
169+
170+
// Wait for completion
171+
done := make(chan struct{})
172+
go func() {
173+
wg.Wait()
174+
close(done)
175+
}()
176+
177+
select {
178+
case <-done:
179+
// Completed
180+
case <-ctx.Done():
181+
log.Printf("[WARN] CNAME collection timed out after %v", opts.Timeout)
106182
}
107183

108184
// Write results to file
@@ -111,10 +187,10 @@ func CollectCNAMEs(domain string) (*Result, error) {
111187
}
112188

113189
count, _ := countLines(out)
114-
log.Printf("[OK] Found %d CNAME records for %s", count, domain)
190+
log.Printf("[OK] Found %d CNAME records for %s", count, opts.Domain)
115191

116192
return &Result{
117-
Domain: domain,
193+
Domain: opts.Domain,
118194
Records: count,
119195
OutputFile: out,
120196
}, nil

‎internal/modules/db/db.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func ListSubdomainsWithStatus(domain string) ([]SubdomainStatus, error) {
152152
}
153153
}
154154
return dbInstance.ListSubdomainsWithStatus(domain)
155-
}
155+
}
156156

157157
// CountSubdomains returns the count of subdomains for a given domain.
158158
func CountSubdomains(domain string) (int, error) {

‎internal/modules/db/sqlite.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ func (s *SQLiteDB) Init() error {
4444
return fmt.Errorf("failed to open SQLite database: %v", err)
4545
}
4646

47+
// Configure connection pool settings for SQLite
48+
// SQLite doesn't support multiple writers well, so limit connections
49+
db.SetMaxOpenConns(1) // Only one connection at a time for SQLite
50+
db.SetMaxIdleConns(1) // Keep one idle connection
51+
db.SetConnMaxLifetime(time.Hour) // Close connections after 1 hour
52+
db.SetConnMaxIdleTime(time.Minute * 30) // Close idle connections after 30 minutes
53+
4754
// Test connection
4855
if err := db.Ping(); err != nil {
4956
return fmt.Errorf("failed to ping database: %v", err)

0 commit comments

Comments
 (0)