@@ -12,20 +12,17 @@ import (
1212// Client checks for CDN based IPs which should be excluded
1313// during scans since they belong to third party firewalls.
1414type Client struct {
15- Options Options
16- Data map [string ]struct {}
17- ranger cidranger.Ranger
15+ Options * Options
16+ ranges map [string ][] string
17+ rangers map [ string ] cidranger.Ranger
1818}
1919
2020var defaultScrapers = map [string ]scraperFunc {
21- // "akamai": scrapeAkamai,
2221 "azure" : scrapeAzure ,
2322 "cloudflare" : scrapeCloudflare ,
2423 "cloudfront" : scrapeCloudFront ,
2524 "fastly" : scrapeFastly ,
2625 "incapsula" : scrapeIncapsula ,
27- // "sucuri": scrapeSucuri,
28- // "leaseweb": scrapeLeaseweb,
2926}
3027
3128var defaultScrapersWithOptions = map [string ]scraperWithOptionsFunc {
@@ -34,10 +31,6 @@ var defaultScrapersWithOptions = map[string]scraperWithOptionsFunc{
3431 "leaseweb" : scrapeLeaseweb ,
3532}
3633
37- var cachedScrapers = map [string ]scraperFunc {
38- "projectdiscovery" : scrapeProjectDiscovery ,
39- }
40-
4134// New creates a new firewall IP checking client.
4235func New () (* Client , error ) {
4336 return new (& Options {})
@@ -65,60 +58,95 @@ func new(options *Options) (*Client, error) {
6558 },
6659 Timeout : time .Duration (30 ) * time .Second ,
6760 }
68- client := & Client {}
61+ client := & Client {Options : options }
6962
63+ var err error
7064 if options .Cache {
71- for _ , scraper := range cachedScrapers {
72- cidrs , err := scraper (httpClient )
73- if err != nil {
74- return nil , err
75- }
76- client .parseCidrs (cidrs )
77- }
65+ err = client .getCDNDataFromCache (httpClient )
7866 } else {
79- for _ , scraper := range defaultScrapers {
80- cidrs , err := scraper (httpClient )
81- if err != nil {
82- return nil , err
83- }
84- client .parseCidrs (cidrs )
85- }
67+ err = client .getCDNData (httpClient )
8668 }
69+ if err != nil {
70+ return nil , err
71+ }
72+ return client , nil
73+ }
74+
75+ func (c * Client ) getCDNData (httpClient * http.Client ) error {
76+ c .ranges = make (map [string ][]string )
77+ c .rangers = make (map [string ]cidranger.Ranger )
78+
79+ for provider , scraper := range defaultScrapers {
80+ cidrs , err := scraper (httpClient )
81+ if err != nil {
82+ return err
83+ }
8784
88- if options .HasAuthInfo () {
89- for _ , scraper := range defaultScrapersWithOptions {
90- cidrs , err := scraper (httpClient , options )
85+ c .ranges [provider ] = cidrs
86+ ranger := cidranger .NewPCTrieRanger ()
87+ for _ , cidr := range cidrs {
88+ _ , network , err := net .ParseCIDR (cidr )
9189 if err != nil {
92- return nil , err
90+ continue
9391 }
94- client . parseCidrs ( cidrs )
92+ _ = ranger . Insert ( cidranger . NewBasicRangerEntry ( * network ) )
9593 }
9694 }
95+ if c .Options .HasAuthInfo () {
96+ for provider , scraper := range defaultScrapersWithOptions {
97+ cidrs , err := scraper (httpClient , c .Options )
98+ if err != nil {
99+ return err
100+ }
97101
98- ranger := cidranger .NewPCTrieRanger ()
99- for cidr := range client .Data {
100- _ , network , err := net .ParseCIDR (cidr )
101- if err != nil {
102- continue
102+ c .ranges [provider ] = cidrs
103+ ranger := cidranger .NewPCTrieRanger ()
104+ for _ , cidr := range cidrs {
105+ _ , network , err := net .ParseCIDR (cidr )
106+ if err != nil {
107+ continue
108+ }
109+ _ = ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
110+ }
103111 }
104- ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
105112 }
106- client .ranger = ranger
107-
108- return client , nil
113+ return nil
109114}
110115
111- // parseCidrs inserts the scraped cidrs to the internal structure
112- func (c * Client ) parseCidrs (cidrs []string ) {
113- if c .Data == nil {
114- c .Data = make (map [string ]struct {})
116+ func (c * Client ) getCDNDataFromCache (httpClient * http.Client ) error {
117+ var err error
118+ c .ranges , err = scrapeProjectDiscovery (httpClient )
119+ if err != nil {
120+ return err
115121 }
116- for _ , cidr := range cidrs {
117- c .Data [cidr ] = struct {}{}
122+
123+ c .rangers = make (map [string ]cidranger.Ranger )
124+ for provider , ranges := range c .ranges {
125+ ranger := cidranger .NewPCTrieRanger ()
126+
127+ for _ , cidr := range ranges {
128+ _ , network , err := net .ParseCIDR (cidr )
129+ if err != nil {
130+ continue
131+ }
132+ _ = ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
133+ }
134+ c .rangers [provider ] = ranger
118135 }
136+ return nil
119137}
120138
121139// Check checks if an IP is contained in the blacklist
122- func (c * Client ) Check (ip net.IP ) (bool , error ) {
123- return c .ranger .Contains (ip )
140+ func (c * Client ) Check (ip net.IP ) (bool , string , error ) {
141+ for provider , ranger := range c .rangers {
142+ if contains , err := ranger .Contains (ip ); contains {
143+ return true , provider , err
144+ }
145+ }
146+ return false , "" , nil
147+ }
148+
149+ // Ranges returns the providers and ranges for the cdn client
150+ func (c * Client ) Ranges () map [string ][]string {
151+ return c .ranges
124152}
0 commit comments