@@ -12,32 +12,41 @@ import (
1212// Client checks for CDN based IPs which should be excluded
1313// during scans since they belong to third party firewalls.
1414type Client struct {
15- Data map [string ]struct {}
16- ranger cidranger.Ranger
15+ Options * Options
16+ ranges map [string ][]string
17+ rangers map [string ]cidranger.Ranger
1718}
1819
1920var defaultScrapers = map [string ]scraperFunc {
20- "akamai " : scrapeAkamai ,
21+ "azure " : scrapeAzure ,
2122 "cloudflare" : scrapeCloudflare ,
23+ "cloudfront" : scrapeCloudFront ,
24+ "fastly" : scrapeFastly ,
2225 "incapsula" : scrapeIncapsula ,
23- "sucuri" : scrapeSucuri ,
2426}
2527
26- var cachedScrapers = map [string ]scraperFunc {
27- "projectdiscovery" : scrapeProjectDiscovery ,
28+ var defaultScrapersWithOptions = map [string ]scraperWithOptionsFunc {
29+ "akamai" : scrapeAkamai ,
30+ "sucuri" : scrapeSucuri ,
31+ "leaseweb" : scrapeLeaseweb ,
2832}
2933
3034// New creates a new firewall IP checking client.
3135func New () (* Client , error ) {
32- return new (false )
36+ return new (& Options {} )
3337}
3438
3539// NewWithCache creates a new firewall IP with cached data from project discovery (faster)
3640func NewWithCache () (* Client , error ) {
37- return new (true )
41+ return new (& Options { Cache : true } )
3842}
3943
40- func new (cache bool ) (* Client , error ) {
44+ // NewWithOptions creates a new instance with options
45+ func NewWithOptions (Options * Options ) (* Client , error ) {
46+ return new (Options )
47+ }
48+
49+ func new (options * Options ) (* Client , error ) {
4150 httpClient := & http.Client {
4251 Transport : & http.Transport {
4352 MaxIdleConns : 100 ,
@@ -49,40 +58,95 @@ func new(cache bool) (*Client, error) {
4958 },
5059 Timeout : time .Duration (30 ) * time .Second ,
5160 }
52- client := & Client {}
61+ client := & Client {Options : options }
5362
54- var scrapers map [ string ] scraperFunc
55- if cache {
56- scrapers = cachedScrapers
63+ var err error
64+ if options . Cache {
65+ err = client . getCDNDataFromCache ( httpClient )
5766 } else {
58- scrapers = defaultScrapers
67+ err = client .getCDNData (httpClient )
68+ }
69+ if err != nil {
70+ return nil , err
5971 }
72+ return client , nil
73+ }
6074
61- client .Data = make (map [string ]struct {})
62- for _ , scraper := range scrapers {
75+ func (c * Client ) getCDNData (httpClient * http.Client ) error {
76+ c .ranges = make (map [string ][]string )
77+ c .rangers = make (map [string ]cidranger.Ranger )
78+
79+ for provider , scraper := range defaultScrapers {
6380 cidrs , err := scraper (httpClient )
6481 if err != nil {
65- return nil , err
82+ return err
6683 }
84+
85+ c .ranges [provider ] = cidrs
86+ ranger := cidranger .NewPCTrieRanger ()
6787 for _ , cidr := range cidrs {
68- client .Data [cidr ] = struct {}{}
88+ _ , network , err := net .ParseCIDR (cidr )
89+ if err != nil {
90+ continue
91+ }
92+ _ = ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
6993 }
7094 }
95+ if c .Options .HasAuthInfo () {
96+ for provider , scraper := range defaultScrapersWithOptions {
97+ cidrs , err := scraper (httpClient , c .Options )
98+ if err != nil {
99+ return err
100+ }
71101
72- ranger := cidranger .NewPCTrieRanger ()
73- for cidr := range client .Data {
74- _ , network , err := net .ParseCIDR (cidr )
75- if err != nil {
76- continue
102+ c .ranges [provider ] = cidrs
103+ ranger := cidranger .NewPCTrieRanger ()
104+ for _ , cidr := range cidrs {
105+ _ , network , err := net .ParseCIDR (cidr )
106+ if err != nil {
107+ continue
108+ }
109+ _ = ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
110+ }
77111 }
78- ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
79112 }
80- client .ranger = ranger
113+ return nil
114+ }
81115
82- return client , nil
116+ func (c * Client ) getCDNDataFromCache (httpClient * http.Client ) error {
117+ var err error
118+ c .ranges , err = scrapeProjectDiscovery (httpClient )
119+ if err != nil {
120+ return err
121+ }
122+
123+ c .rangers = make (map [string ]cidranger.Ranger )
124+ for provider , ranges := range c .ranges {
125+ ranger := cidranger .NewPCTrieRanger ()
126+
127+ for _ , cidr := range ranges {
128+ _ , network , err := net .ParseCIDR (cidr )
129+ if err != nil {
130+ continue
131+ }
132+ _ = ranger .Insert (cidranger .NewBasicRangerEntry (* network ))
133+ }
134+ c .rangers [provider ] = ranger
135+ }
136+ return nil
83137}
84138
85139// Check checks if an IP is contained in the blacklist
86- func (c * Client ) Check (ip net.IP ) (bool , error ) {
87- return c .ranger .Contains (ip )
140+ func (c * Client ) Check (ip net.IP ) (bool , string , error ) {
141+ for provider , ranger := range c .rangers {
142+ if contains , err := ranger .Contains (ip ); contains {
143+ return true , provider , err
144+ }
145+ }
146+ return false , "" , nil
147+ }
148+
149+ // Ranges returns the providers and ranges for the cdn client
150+ func (c * Client ) Ranges () map [string ][]string {
151+ return c .ranges
88152}
0 commit comments