Merge branch 'fix/s3-region-full-url'

Author: h0tak88r

internal/api/scan_handlers.go

@@ -706,6 +706,16 @@ func scanS3(c *gin.Context) {
 	if req.Region != nil {
 		opts.Region = *req.Region
 	}
+	// Accept a full S3 URL (e.g. bucket.s3.ap-south-1.amazonaws.com) in the bucket
+	// field: use the clean bucket name for the scan record/title and adopt an embedded
+	// region when none was supplied. handleScan re-parses + auto-detects too.
+	if b, r := s3mod.ParseBucketInput(bucket); b != "" {
+		bucket = b
+		opts.Bucket = b
+		if opts.Region == "" && r != "" {
+			opts.Region = r
+		}
+	}
 	scanID := generateScanID()
 	go RunScanInProcess(scanID, "s3", bucket, func() error {
 		return s3mod.Run(opts)

internal/api/ui/pages/launcher.js

@@ -20,7 +20,7 @@
     dns_dangling: { path: 'dns', modes: ['domain', 'domain_list'], extra: { dns_type: 'dangling-ip' }, placeholders: { domain: 'example.com', domain_list: 'one domain per line' } },
     dns_takeover: { path: 'dns-takeover', modes: ['domain', 'domain_list'], placeholders: { domain: 'example.com', domain_list: 'one domain per line' } },
     dns_cf1016: { path: 'dns-cf1016', modes: ['domain', 'subdomain', 'domain_list', 'subdomain_list'], placeholders: { domain: 'example.com', subdomain: 'api.example.com', domain_list: 'one domain per line', subdomain_list: 'one subdomain per line' } },
-    s3: { path: 's3', modes: ['bucket', 'bucket_list', 'domain', 'domain_list'], placeholders: { bucket: 'bucket-name', bucket_list: 'one bucket per line', domain: 'example.com', domain_list: 'one domain per line' } },
+    s3: { path: 's3', modes: ['bucket', 'bucket_list', 'domain', 'domain_list'], placeholders: { bucket: 'bucket-name or bucket.s3.ap-south-1.amazonaws.com', bucket_list: 'one bucket name or s3 URL per line', domain: 'example.com', domain_list: 'one domain per line' } },
     github: { path: 'github', modes: ['repo', 'repo_list'], placeholders: { repo: 'owner/repository or github.com/owner/repo', repo_list: 'one owner/repo per line' } },
     github_org: { path: 'github_org', modes: ['domain', 'domain_list'], placeholders: { domain: 'org-name or github.com/org', domain_list: 'one org per line' } },
     zerodays: { path: 'zerodays', modes: ['domain', 'domain_list'], placeholders: { domain: 'example.com', domain_list: 'one domain per line' } },
@@ -60,7 +60,7 @@
     dns: [{ key: 'dns_type', label: 'DNS type', type: 'select', options: ['takeover', 'dangling-ip'], default: 'takeover', advanced: false, help: 'takeover = CNAME/NS takeover checks; dangling-ip = dangling A-record checks.' }],
     dns_dangling: [{ key: 'dns_type', label: 'DNS type', type: 'select', options: ['dangling-ip', 'takeover'], default: 'dangling-ip', advanced: false, help: 'dangling-ip = dangling A-record checks.' }],
     s3: [
-      { key: 'region', label: 'Region (optional)', type: 'text', advanced: false, help: 'AWS region for the bucket, e.g. us-east-1. Leave empty to auto-detect.' },
+      { key: 'region', label: 'Region (optional)', type: 'text', advanced: false, help: 'AWS region, e.g. ap-south-1. Leave empty to auto-detect (parsed from the URL if you paste a full s3 URL, else read from the bucket itself).' },
       { key: 'threads', label: 'Threads (reserved)', type: 'number', min: 1, advanced: true, help: 'Reserved for future concurrent bucket probing.' },
     ],
     ffuf: [

internal/scanner/s3/s3.go

@@ -192,18 +192,13 @@ func handleDomainEnumAndScan(opts Options, resultsDir string) error {
 		if region == "" {
 			region = opts.Region
 		}
-		regions := []string{"us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"}
+		regions := []string{"us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1", "ap-south-1", "eu-central-1"}
 		if region != "" {
 			regions = []string{region}
 		}
 	out:
 		for _, r := range regions {
-			urls := []string{
-				fmt.Sprintf("https://%s.s3.amazonaws.com/?list-type=2", fb.name),
-				fmt.Sprintf("https://%s.s3-%s.amazonaws.com/?list-type=2", fb.name, r),
-				fmt.Sprintf("https://s3.amazonaws.com/%s/?list-type=2", fb.name),
-				fmt.Sprintf("https://s3-%s.amazonaws.com/%s/?list-type=2", r, fb.name),
-			}
+			urls := bucketListURLs(fb.name, r)
 			for _, u := range urls {
 				bucketDir := filepath.Join(outputDir, fb.name)
 				_ = os.MkdirAll(bucketDir, 0755)
@@ -455,6 +450,26 @@ func handleScan(opts Options, resultsDir string) error {
 		return fmt.Errorf("bucket name is required for scan action")
 	}
 
+	// Accept a full S3 URL (bucket.s3.region.amazonaws.com, path-style, with or without
+	// scheme) as well as a plain bucket name — extract the bucket and any embedded region.
+	if b, r := ParseBucketInput(opts.Bucket); b != "" {
+		opts.Bucket = b
+		if opts.Region == "" {
+			opts.Region = r
+		}
+	}
+	// Auto-detect the region when still unknown. Required for buckets outside us-east-1
+	// (e.g. ap-south-1) whose virtual-hosted endpoints differ from the global host; also
+	// fulfils the launcher's "leave empty to auto-detect" hint.
+	if opts.Region == "" {
+		if r := detectBucketRegion(opts.Bucket); r != "" {
+			opts.Region = r
+			logger.GetLogger().Infof("[INFO] S3 scan: auto-detected region %s for bucket %s", r, opts.Bucket)
+		} else {
+			logger.GetLogger().Infof("[INFO] S3 scan: could not auto-detect region for %s (bucket may not exist); falling back to multi-region probing", opts.Bucket)
+		}
+	}
+
 	outputDir := filepath.Join(resultsDir, "s3", utils.SanitizeTargetSegment(opts.Bucket))
 	if err := os.MkdirAll(outputDir, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %v", err)
@@ -552,22 +567,17 @@ func handleScan(opts Options, resultsDir string) error {
 		}
 		fmt.Printf("[INFO] Testing bucket %s without authentication (public access check)...\n", opts.Bucket)
 
-		regions := []string{"us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"}
+		regions := []string{"us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1", "ap-south-1", "eu-central-1"}
 		if opts.Region != "" {
 			regions = []string{opts.Region}
 		}
 		logger.GetLogger().Infof("[INFO] S3 scan: Testing %d region(s) for public access", len(regions))
-		
+
 		var foundURL string
 		totalURLsTested := 0
 		for _, region := range regions {
-			urls := []string{
-				fmt.Sprintf("https://%s.s3.amazonaws.com/?list-type=2", opts.Bucket),
-				fmt.Sprintf("https://%s.s3-%s.amazonaws.com/?list-type=2", opts.Bucket, region),
-				fmt.Sprintf("https://s3.amazonaws.com/%s/?list-type=2", opts.Bucket),
-				fmt.Sprintf("https://s3-%s.amazonaws.com/%s/?list-type=2", region, opts.Bucket),
-			}
-			
+			urls := bucketListURLs(opts.Bucket, region)
+
 			logger.GetLogger().Infof("[INFO] S3 scan: Testing region %s (%d URL format(s))", region, len(urls))
 			for _, testURL := range urls {
 				totalURLsTested++
@@ -674,13 +684,164 @@ func handleScan(opts Options, resultsDir string) error {
 	return nil
 }
 
+// ParseBucketInput accepts a plain bucket name OR any S3 URL form and returns the
+// bucket name and (when present) the AWS region. Recognised forms:
+//
+//	bucket-name
+//	bucket.s3.amazonaws.com                       (virtual-hosted, global/us-east-1)
+//	bucket.s3.ap-south-1.amazonaws.com            (virtual-hosted, modern dot)
+//	bucket.s3-ap-south-1.amazonaws.com            (virtual-hosted, legacy dash)
+//	bucket.s3.dualstack.ap-south-1.amazonaws.com  (virtual-hosted, dualstack)
+//	s3.amazonaws.com/bucket                       (path-style, global)
+//	s3.ap-south-1.amazonaws.com/bucket            (path-style, regional)
+//	https://…/…                                   (scheme, port and trailing path are stripped)
+func ParseBucketInput(input string) (bucket, region string) {
+	s := strings.TrimSpace(input)
+	if s == "" {
+		return "", ""
+	}
+	if i := strings.Index(s, "://"); i != -1 {
+		s = s[i+3:]
+	}
+	host := s
+	path := ""
+	if i := strings.IndexByte(s, '/'); i != -1 {
+		host = s[:i]
+		path = strings.Trim(s[i+1:], "/")
+	}
+	if i := strings.IndexByte(host, ':'); i != -1 { // strip port
+		host = host[:i]
+	}
+	host = strings.ToLower(strings.TrimSuffix(host, "."))
+
+	const sfx = ".amazonaws.com"
+	if !strings.HasSuffix(host, sfx) {
+		// Plain bucket name (no AWS host) — return as-is.
+		return host, ""
+	}
+
+	core := strings.TrimSuffix(host, sfx) // e.g. "bucket.s3.ap-south-1", "bucket.s3", "s3.ap-south-1", "s3"
+
+	// Path-style: host is "s3" / "s3.<region>" / "s3-<region>", bucket is first path segment.
+	switch {
+	case core == "s3":
+		return firstSegment(path), ""
+	case strings.HasPrefix(core, "s3."):
+		return firstSegment(path), cleanRegion(strings.TrimPrefix(core, "s3."))
+	case strings.HasPrefix(core, "s3-"):
+		return firstSegment(path), cleanRegion(strings.TrimPrefix(core, "s3-"))
+	}
+
+	// Virtual-hosted: "<bucket>.s3" / "<bucket>.s3.<region>" / "<bucket>.s3-<region>".
+	// LastIndex so bucket names that themselves contain ".s3" still parse correctly.
+	if strings.HasSuffix(core, ".s3") {
+		return strings.TrimSuffix(core, ".s3"), ""
+	}
+	if i := strings.LastIndex(core, ".s3."); i != -1 {
+		return core[:i], cleanRegion(core[i+len(".s3."):])
+	}
+	if i := strings.LastIndex(core, ".s3-"); i != -1 {
+		return core[:i], cleanRegion(core[i+len(".s3-"):])
+	}
+	// Unknown amazonaws host shape — fall back to the first label.
+	return firstSegment(core), ""
+}
+
+// firstSegment returns everything before the first '/' (a bucket name, which may
+// itself contain dots, so we deliberately do not split on '.').
+func firstSegment(s string) string {
+	if i := strings.IndexByte(s, '/'); i != -1 {
+		return s[:i]
+	}
+	return s
+}
+
+// cleanRegion strips dualstack/website prefixes and trailing labels, leaving the bare
+// region (e.g. "dualstack.ap-south-1" -> "ap-south-1"). Returns "" for non-regions.
+func cleanRegion(r string) string {
+	r = strings.TrimPrefix(r, "dualstack.")
+	r = strings.TrimPrefix(r, "dualstack-")
+	r = strings.TrimPrefix(r, "website-")
+	r = strings.TrimPrefix(r, "website.")
+	if i := strings.IndexByte(r, '.'); i != -1 { // region is a single label
+		r = r[:i]
+	}
+	switch r {
+	case "", "external-1", "website", "dualstack", "accelerate":
+		// Non-region S3 host keywords (transfer-acceleration, website, FIPS dualstack).
+		return ""
+	}
+	return r
+}
+
+// detectBucketRegion returns the bucket's AWS region from the x-amz-bucket-region
+// response header, which S3 sets on any request to the bucket — even unauthenticated
+// 301/403/400 responses. Returns "" if it can't be determined.
+func detectBucketRegion(bucketName string) string {
+	if bucketName == "" {
+		return ""
+	}
+	client := &http.Client{Timeout: 8 * time.Second}
+	for _, u := range []string{
+		"https://" + bucketName + ".s3.amazonaws.com",
+		"https://s3.amazonaws.com/" + bucketName,
+	} {
+		req, err := http.NewRequest("HEAD", u, nil)
+		if err != nil {
+			continue
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			continue
+		}
+		region := resp.Header.Get("x-amz-bucket-region")
+		resp.Body.Close()
+		if region != "" {
+			return region
+		}
+	}
+	return ""
+}
+
+// bucketListURLs returns the candidate list-objects endpoints for a bucket, covering
+// the modern dot-style regional host (required for buckets outside us-east-1, e.g.
+// ap-south-1), the legacy dash-style host, and both virtual-hosted and path-style forms.
+func bucketListURLs(bucketName, region string) []string {
+	const q = "/?list-type=2"
+	urls := []string{
+		fmt.Sprintf("https://%s.s3.amazonaws.com%s", bucketName, q),
+		fmt.Sprintf("https://s3.amazonaws.com/%s%s", bucketName, q),
+	}
+	if region != "" && region != "us-east-1" {
+		urls = append(urls,
+			fmt.Sprintf("https://%s.s3.%s.amazonaws.com%s", bucketName, region, q), // modern dot (virtual-hosted)
+			fmt.Sprintf("https://s3.%s.amazonaws.com/%s%s", region, bucketName, q),  // modern dot (path-style)
+			fmt.Sprintf("https://%s.s3-%s.amazonaws.com%s", bucketName, region, q),  // legacy dash (virtual-hosted)
+			fmt.Sprintf("https://s3-%s.amazonaws.com/%s%s", region, bucketName, q),  // legacy dash (path-style)
+		)
+	}
+	return urls
+}
+
 func probeBucketPermissions(bucketName, region string, logFile io.Writer) (canList, canRead, canWrite, canDelete bool) {
+	// Resolve the region if the caller didn't supply one, so write/read/delete probes
+	// hit the bucket's real regional endpoint instead of only the global host (which
+	// 301-redirects for buckets outside us-east-1, making every probe read false).
+	if region == "" {
+		region = detectBucketRegion(bucketName)
+	}
 	client := &http.Client{Timeout: 8 * time.Second}
 	urls := []string{
 		fmt.Sprintf("https://%s.s3.amazonaws.com", bucketName),
 	}
-	if region != "" {
-		urls = append(urls, fmt.Sprintf("https://%s.s3-%s.amazonaws.com", bucketName, region))
+	if region != "" && region != "us-east-1" {
+		// Modern dot-style host first — required for buckets outside us-east-1 (e.g.
+		// ap-south-1); the global host above returns a 301 redirect for those, so
+		// without this the permission probe always reads false. Legacy dash kept as fallback.
+		urls = append(urls,
+			fmt.Sprintf("https://%s.s3.%s.amazonaws.com", bucketName, region),
+			fmt.Sprintf("https://%s.s3-%s.amazonaws.com", bucketName, region),
+		)
 	}
 	testKey := fmt.Sprintf("autoar-perm-test-%d.txt", time.Now().UnixNano())
 	testBody := []byte("autoar permission test")

internal/scanner/s3/s3_test.go

@@ -4,6 +4,54 @@ import (
 	"testing"
 )
 
+func TestParseBucketInput(t *testing.T) {
+	cases := []struct {
+		in         string
+		wantBucket string
+		wantRegion string
+	}{
+		// Plain bucket name.
+		{"my-bucket", "my-bucket", ""},
+		{"unifyapps-sonydev-application-uploads-cloudstorage-2", "unifyapps-sonydev-application-uploads-cloudstorage-2", ""},
+		// Virtual-hosted, modern dot.
+		{"unifyapps-sonydev-application-uploads-cloudstorage-2.s3.ap-south-1.amazonaws.com", "unifyapps-sonydev-application-uploads-cloudstorage-2", "ap-south-1"},
+		{"my-bucket.s3.ap-south-1.amazonaws.com", "my-bucket", "ap-south-1"},
+		{"https://my-bucket.s3.eu-west-1.amazonaws.com/", "my-bucket", "eu-west-1"},
+		{"http://my-bucket.s3.us-west-2.amazonaws.com/some/key.txt", "my-bucket", "us-west-2"},
+		// Virtual-hosted, global (no region).
+		{"my-bucket.s3.amazonaws.com", "my-bucket", ""},
+		{"https://my-bucket.s3.amazonaws.com", "my-bucket", ""},
+		// Virtual-hosted, legacy dash.
+		{"my-bucket.s3-ap-south-1.amazonaws.com", "my-bucket", "ap-south-1"},
+		// Virtual-hosted, dualstack.
+		{"my-bucket.s3.dualstack.ap-south-1.amazonaws.com", "my-bucket", "ap-south-1"},
+		// Transfer-acceleration endpoint — not a region.
+		{"my-bucket.s3-accelerate.amazonaws.com", "my-bucket", ""},
+		{"my-bucket.s3-accelerate.dualstack.amazonaws.com", "my-bucket", ""},
+		// Website endpoint — region is the real region, "website" prefix stripped.
+		{"my-bucket.s3-website-us-east-1.amazonaws.com", "my-bucket", "us-east-1"},
+		// GovCloud region passes through unchanged.
+		{"my-bucket.s3.us-gov-west-1.amazonaws.com", "my-bucket", "us-gov-west-1"},
+		// Path-style.
+		{"s3.amazonaws.com/my-bucket", "my-bucket", ""},
+		{"s3.ap-south-1.amazonaws.com/my-bucket", "my-bucket", "ap-south-1"},
+		{"https://s3.ap-south-1.amazonaws.com/my-bucket/key", "my-bucket", "ap-south-1"},
+		{"s3-eu-west-1.amazonaws.com/my-bucket", "my-bucket", "eu-west-1"},
+		// Dotted bucket name in virtual-hosted form (LastIndex handling).
+		{"assets.media.s3.eu-central-1.amazonaws.com", "assets.media", "eu-central-1"},
+		// Whitespace + trailing dot/port.
+		{"  my-bucket.s3.ap-south-1.amazonaws.com  ", "my-bucket", "ap-south-1"},
+		// Empty.
+		{"", "", ""},
+	}
+	for _, c := range cases {
+		gotB, gotR := ParseBucketInput(c.in)
+		if gotB != c.wantBucket || gotR != c.wantRegion {
+			t.Errorf("ParseBucketInput(%q) = (%q, %q), want (%q, %q)", c.in, gotB, gotR, c.wantBucket, c.wantRegion)
+		}
+	}
+}
+
 func TestGenerateBucketNamesSimple(t *testing.T) {
 	names := generateBucketNames("example.com")
 	if len(names) != 22 {