Skip to content

Commit 57686fd

Browse files
committed
refactor: overhaul scan result file handling by adding automated module/category inference and local directory traversal logic.
1 parent 7c3d919 commit 57686fd

35 files changed

Lines changed: 8579 additions & 1110 deletions
131 KB
Loading

cmd/migrate-supabase/main.go

Lines changed: 656 additions & 0 deletions
Large diffs are not rendered by default.

env.example

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ API_PORT=8000
1919
# ============================================================================
2020
# DASHBOARD AUTH (HTTP API + /ui) — optional; see docs/DASHBOARD_AUTH.md
2121
# ============================================================================
22+
# When SUPABASE_JWT_SECRET or SUPABASE_URL is set, /api/* requires Authorization: Bearer <access_token>.
23+
# For local Postgres without Supabase login, either unset both variables or:
24+
# AUTOAR_API_AUTH_DISABLED=true
25+
#
2226
# SUPABASE_URL=https://xxxx.supabase.co
2327
# SUPABASE_JWT_SECRET= # Legacy HS256 secret; JWKS used when URL-only ES256
2428
# SUPABASE_ANON_KEY= # Publishable key (exposed to browser via /api/config)
@@ -109,6 +113,7 @@ DB_TYPE=postgresql
109113
# For PostgreSQL:
110114
# Option 1: Use PostgreSQL connection string
111115
# DB_HOST=postgresql://username:password@host:port/database
116+
# (Alternatively set DATABASE_URL=postgresql://… if DB_HOST is empty — same effect.)
112117
#
113118
# Option 2: Use individual environment variables
114119
# DB_HOST=your_postgres_host

go.mod

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ go 1.24.2
44

55
require (
66
github.com/99designs/keyring v1.2.1
7-
github.com/MicahParks/keyfunc/v3 v3.8.0
87
github.com/aws/aws-sdk-go-v2 v1.41.0
98
github.com/aws/aws-sdk-go-v2/config v1.32.6
109
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
@@ -46,7 +45,6 @@ require (
4645
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.0 // indirect
4746
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
4847
github.com/Masterminds/semver/v3 v3.2.1 // indirect
49-
github.com/MicahParks/jwkset v0.11.0 // indirect
5048
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect
5149
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
5250
github.com/PuerkitoBio/goquery v1.11.0 // indirect
@@ -156,6 +154,7 @@ require (
156154
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
157155
github.com/jackc/puddle/v2 v2.2.2 // indirect
158156
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
157+
github.com/joho/godotenv v1.5.1 // indirect
159158
github.com/json-iterator/go v1.1.12 // indirect
160159
github.com/juju/go4 v0.0.0-20160222163258-40d72ab9641a // indirect
161160
github.com/kataras/jwt v0.1.10 // indirect

go.sum

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,6 @@ github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1
5454
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
5555
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
5656
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
57-
github.com/MicahParks/jwkset v0.11.0 h1:yc0zG+jCvZpWgFDFmvs8/8jqqVBG9oyIbmBtmjOhoyQ=
58-
github.com/MicahParks/jwkset v0.11.0/go.mod h1:U2oRhRaLgDCLjtpGL2GseNKGmZtLs/3O7p+OZaL5vo0=
59-
github.com/MicahParks/keyfunc/v3 v3.8.0 h1:Hx2dgIjAXGk9slakM6rV9BOeaWDPEXXZ4Us8guNBfds=
60-
github.com/MicahParks/keyfunc/v3 v3.8.0/go.mod h1:z66bkCviwqfg2YUp+Jcc/xRE9IXLcMq6DrgV/+Htru0=
6157
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE=
6258
github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4=
6359
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8=
@@ -524,6 +520,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i
524520
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
525521
github.com/joeguo/tldextract v0.0.0-20220507100122-d83daa6adef8 h1:Ig0ESdy6JtHI17vsb7L+UlUFpoZctKfvBZplcILeL6g=
526522
github.com/joeguo/tldextract v0.0.0-20220507100122-d83daa6adef8/go.mod h1:oGfutRjaB95239mjFVwofaOPTwuS3vb71ZLIGCEb36g=
523+
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
524+
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
527525
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
528526
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
529527
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

internal/modules/backup/backup.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"time"
1111

1212
fuzzulitool "github.com/h0tak88r/AutoAR/internal/tools/fuzzuli"
13+
"github.com/h0tak88r/AutoAR/internal/modules/utils"
1314
)
1415

1516
// Options controls how the backup scan runs.
@@ -222,6 +223,27 @@ func Run(opts Options) (*Result, error) {
222223
}
223224
}
224225

226+
// Write JSON results to scan directory (local-first)
227+
if scanID := os.Getenv("AUTOAR_CURRENT_SCAN_ID"); scanID != "" && res.FoundCount > 0 {
228+
data, readErr := os.ReadFile(res.ResultsFile)
229+
if readErr == nil {
230+
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
231+
var nonEmpty []string
232+
for _, l := range lines {
233+
if strings.TrimSpace(l) != "" {
234+
nonEmpty = append(nonEmpty, l)
235+
}
236+
}
237+
target := opts.Domain
238+
if target == "" {
239+
target = "backup-scan"
240+
}
241+
if err := utils.WriteLinesAsJSON(scanID, target, "backup", "backup-vulnerabilities.json", nonEmpty); err != nil {
242+
log.Printf("[WARN] Failed to write backup JSON: %v", err)
243+
}
244+
}
245+
}
246+
225247
return res, nil
226248
}
227249

internal/modules/cnames/cnames.go

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -238,16 +238,45 @@ func CollectCNAMEsWithOptions(opts Options) (*Result, error) {
238238
count, _ := countLines(out)
239239
log.Printf("[OK] Found %d CNAME records for %s", count, domain)
240240

241-
// Send result files to Discord webhook if configured (only when not running under bot)
242-
// When running under bot (AUTOAR_CURRENT_SCAN_ID is set), the bot handles R2 upload and zip link
241+
// Write JSON output to scan directory for dashboard
242+
if scanID := os.Getenv("AUTOAR_CURRENT_SCAN_ID"); scanID != "" && len(cnameRecords) > 0 {
243+
// Emit structured objects: {subdomain, cname, type}
244+
type cnameEntry struct {
245+
Subdomain string `json:"subdomain"`
246+
CNAME string `json:"cname"`
247+
Type string `json:"type"`
248+
}
249+
var entries []cnameEntry
250+
for _, rec := range cnameRecords {
251+
// format: "sub.example.com CNAME target.example.com"
252+
parts := strings.Fields(rec)
253+
if len(parts) >= 3 {
254+
entries = append(entries, cnameEntry{
255+
Subdomain: parts[0],
256+
CNAME: parts[2],
257+
Type: "CNAME",
258+
})
259+
}
260+
}
261+
if err := utils.WriteJSONToScanDir(scanID, "cname-records.json", map[string]interface{}{
262+
"scan_id": scanID,
263+
"target": domain,
264+
"scan_type": "cnames",
265+
"generated": fmt.Sprintf("%v", count),
266+
"records": entries,
267+
"count": len(entries),
268+
}); err != nil {
269+
log.Printf("[WARN] Failed to write CNAME JSON: %v", err)
270+
}
271+
}
272+
273+
// Send result files to Discord webhook (only when not running under bot)
243274
if os.Getenv("AUTOAR_CURRENT_SCAN_ID") == "" {
244275
webhookURL := os.Getenv("DISCORD_WEBHOOK")
245276
if webhookURL != "" {
246-
// Send CNAME output file if it exists and has content
247277
if info, err := os.Stat(out); err == nil && info.Size() > 0 {
248278
utils.SendWebhookFileAsync(out, fmt.Sprintf("CNAME Records: %d CNAME records found for %s", count, domain))
249279
} else if count == 0 {
250-
// Send "no findings" message if no CNAME records found
251280
utils.SendWebhookLogAsync(fmt.Sprintf("CNAME collection completed for %s: 0 CNAME records found", domain))
252281
}
253282
}

internal/modules/db/postgres.go

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,12 @@ type PostgresDB struct {
2323

2424
// Init initializes the PostgreSQL database connection pool
2525
func (p *PostgresDB) Init() error {
26-
// Parse PostgreSQL connection string if provided
27-
dbHostEnv := os.Getenv("DB_HOST")
26+
// Parse PostgreSQL connection string if provided.
27+
// Prefer DB_HOST; fall back to DATABASE_URL (common in hosting / copied .env files).
28+
dbHostEnv := strings.TrimSpace(os.Getenv("DB_HOST"))
29+
if dbHostEnv == "" {
30+
dbHostEnv = strings.TrimSpace(os.Getenv("DATABASE_URL"))
31+
}
2832
var connStr string
2933

3034
if strings.HasPrefix(dbHostEnv, "postgresql://") || strings.HasPrefix(dbHostEnv, "postgres://") {
@@ -387,6 +391,10 @@ func (p *PostgresDB) InitSchema() error {
387391
return fmt.Errorf("failed to migrate scans.result_url: %v", err)
388392
}
389393

394+
// Migrate scan_artifacts table: add module and category columns
395+
_, _ = p.pool.Exec(p.ctx, `ALTER TABLE scan_artifacts ADD COLUMN IF NOT EXISTS module TEXT`)
396+
_, _ = p.pool.Exec(p.ctx, `ALTER TABLE scan_artifacts ADD COLUMN IF NOT EXISTS category TEXT`)
397+
390398
// Deduplicate legacy artifact rows before enforcing uniqueness.
391399
_, _ = p.pool.Exec(p.ctx, `
392400
DELETE FROM scan_artifacts a
@@ -1477,17 +1485,19 @@ func (p *PostgresDB) AppendScanArtifact(artifact *ScanArtifact) error {
14771485
}
14781486
_, err := p.pool.Exec(p.ctx, `
14791487
INSERT INTO scan_artifacts (
1480-
scan_id, file_name, local_path, r2_key, public_url, size_bytes, line_count, content_type
1481-
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
1488+
scan_id, file_name, local_path, r2_key, public_url, size_bytes, line_count, content_type, module, category
1489+
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
14821490
ON CONFLICT (scan_id, r2_key) DO UPDATE SET
14831491
file_name=EXCLUDED.file_name,
14841492
local_path=EXCLUDED.local_path,
14851493
public_url=EXCLUDED.public_url,
14861494
size_bytes=EXCLUDED.size_bytes,
14871495
line_count=EXCLUDED.line_count,
14881496
content_type=EXCLUDED.content_type,
1497+
module=EXCLUDED.module,
1498+
category=EXCLUDED.category,
14891499
created_at=NOW()
1490-
`, artifact.ScanID, artifact.FileName, artifact.LocalPath, artifact.R2Key, artifact.PublicURL, artifact.SizeBytes, artifact.LineCount, artifact.ContentType)
1500+
`, artifact.ScanID, artifact.FileName, artifact.LocalPath, artifact.R2Key, artifact.PublicURL, artifact.SizeBytes, artifact.LineCount, artifact.ContentType, artifact.Module, artifact.Category)
14911501
if err != nil {
14921502
return fmt.Errorf("failed to append scan artifact: %v", err)
14931503
}
@@ -1499,7 +1509,7 @@ func (p *PostgresDB) ListScanArtifacts(scanID string) ([]*ScanArtifact, error) {
14991509
rows, err := p.pool.Query(p.ctx, `
15001510
SELECT id, scan_id, COALESCE(file_name, ''), COALESCE(local_path, ''), COALESCE(r2_key, ''),
15011511
COALESCE(public_url, ''), COALESCE(size_bytes, 0), COALESCE(line_count, 0),
1502-
COALESCE(content_type, ''), created_at
1512+
COALESCE(content_type, ''), COALESCE(module, ''), COALESCE(category, ''), created_at
15031513
FROM scan_artifacts
15041514
WHERE scan_id = $1
15051515
ORDER BY created_at DESC, id DESC
@@ -1512,7 +1522,7 @@ func (p *PostgresDB) ListScanArtifacts(scanID string) ([]*ScanArtifact, error) {
15121522
var out []*ScanArtifact
15131523
for rows.Next() {
15141524
var a ScanArtifact
1515-
if err := rows.Scan(&a.ID, &a.ScanID, &a.FileName, &a.LocalPath, &a.R2Key, &a.PublicURL, &a.SizeBytes, &a.LineCount, &a.ContentType, &a.CreatedAt); err != nil {
1525+
if err := rows.Scan(&a.ID, &a.ScanID, &a.FileName, &a.LocalPath, &a.R2Key, &a.PublicURL, &a.SizeBytes, &a.LineCount, &a.ContentType, &a.Module, &a.Category, &a.CreatedAt); err != nil {
15161526
return nil, fmt.Errorf("failed to scan artifact row: %v", err)
15171527
}
15181528
out = append(out, &a)

internal/modules/db/sqlite.go

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,19 @@ func (s *SQLiteDB) InitSchema() error {
263263
return fmt.Errorf("failed to migrate scans.result_url: %v", merr)
264264
}
265265
}
266+
// Migrate scan_artifacts table: add module and category columns
267+
if _, merr := s.db.Exec(`ALTER TABLE scan_artifacts ADD COLUMN module TEXT`); merr != nil {
268+
low := strings.ToLower(merr.Error())
269+
if !strings.Contains(low, "duplicate column") {
270+
log.Printf("[WARN] Failed to add module column: %v", merr)
271+
}
272+
}
273+
if _, merr := s.db.Exec(`ALTER TABLE scan_artifacts ADD COLUMN category TEXT`); merr != nil {
274+
low := strings.ToLower(merr.Error())
275+
if !strings.Contains(low, "duplicate column") {
276+
log.Printf("[WARN] Failed to add category column: %v", merr)
277+
}
278+
}
266279
// Deduplicate legacy artifact rows before enforcing uniqueness.
267280
_, _ = s.db.Exec(`
268281
DELETE FROM scan_artifacts
@@ -1420,17 +1433,19 @@ func (s *SQLiteDB) AppendScanArtifact(artifact *ScanArtifact) error {
14201433
}
14211434
_, err := s.db.Exec(`
14221435
INSERT INTO scan_artifacts (
1423-
scan_id, file_name, local_path, r2_key, public_url, size_bytes, line_count, content_type
1424-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
1436+
scan_id, file_name, local_path, r2_key, public_url, size_bytes, line_count, content_type, module, category
1437+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
14251438
ON CONFLICT(scan_id, r2_key) DO UPDATE SET
14261439
file_name=excluded.file_name,
14271440
local_path=excluded.local_path,
14281441
public_url=excluded.public_url,
14291442
size_bytes=excluded.size_bytes,
14301443
line_count=excluded.line_count,
14311444
content_type=excluded.content_type,
1445+
module=excluded.module,
1446+
category=excluded.category,
14321447
created_at=datetime('now')
1433-
`, artifact.ScanID, artifact.FileName, artifact.LocalPath, artifact.R2Key, artifact.PublicURL, artifact.SizeBytes, artifact.LineCount, artifact.ContentType)
1448+
`, artifact.ScanID, artifact.FileName, artifact.LocalPath, artifact.R2Key, artifact.PublicURL, artifact.SizeBytes, artifact.LineCount, artifact.ContentType, artifact.Module, artifact.Category)
14341449
if err != nil {
14351450
return fmt.Errorf("failed to append scan artifact: %v", err)
14361451
}
@@ -1442,7 +1457,7 @@ func (s *SQLiteDB) ListScanArtifacts(scanID string) ([]*ScanArtifact, error) {
14421457
rows, err := s.db.Query(`
14431458
SELECT id, scan_id, COALESCE(file_name, ''), COALESCE(local_path, ''), COALESCE(r2_key, ''),
14441459
COALESCE(public_url, ''), COALESCE(size_bytes, 0), COALESCE(line_count, 0),
1445-
COALESCE(content_type, ''), created_at
1460+
COALESCE(content_type, ''), COALESCE(module, ''), COALESCE(category, ''), created_at
14461461
FROM scan_artifacts
14471462
WHERE scan_id = ?
14481463
ORDER BY created_at DESC, id DESC
@@ -1455,7 +1470,7 @@ func (s *SQLiteDB) ListScanArtifacts(scanID string) ([]*ScanArtifact, error) {
14551470
var out []*ScanArtifact
14561471
for rows.Next() {
14571472
var a ScanArtifact
1458-
if err := rows.Scan(&a.ID, &a.ScanID, &a.FileName, &a.LocalPath, &a.R2Key, &a.PublicURL, &a.SizeBytes, &a.LineCount, &a.ContentType, &a.CreatedAt); err != nil {
1473+
if err := rows.Scan(&a.ID, &a.ScanID, &a.FileName, &a.LocalPath, &a.R2Key, &a.PublicURL, &a.SizeBytes, &a.LineCount, &a.ContentType, &a.Module, &a.Category, &a.CreatedAt); err != nil {
14591474
return nil, fmt.Errorf("failed to scan artifact row: %v", err)
14601475
}
14611476
out = append(out, &a)

internal/modules/db/types.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,8 @@ type ScanArtifact struct {
156156
SizeBytes int64 `json:"size_bytes"`
157157
LineCount int `json:"line_count"`
158158
ContentType string `json:"content_type,omitempty"`
159+
Module string `json:"module,omitempty"` // The module that produced this artifact (e.g., "nuclei", "subfinder", "httpx")
160+
Category string `json:"category,omitempty"` // Artifact category (e.g., "vulnerability", "recon", "config")
159161
CreatedAt time.Time `json:"created_at"`
160162
}
161163

0 commit comments

Comments
 (0)