Add Shodan

Author: evalphobia

README.md

@@ -62,6 +62,8 @@ it provides both of cli binary and golang API.
 - [IPQualityScore](https://www.ipqualityscore.com/)
 - [ipregistry.co](https://ipregistry.co/)
 - [MaxMind minFraud](https://www.maxmind.com/en/solutions/minfraud-services/)
+- [Shodan](https://sodan.io/)
+
 
 # Quick Usage for binary
 
@@ -132,12 +134,15 @@ Exec api call of ip address fraud check providers from csv list file
 
 Options:
 
-  -h, --help       display help information
-  -p, --provider  *set types of api provider (space separated) --provider='ipdata ipinfo minfraud'
-  -i, --input     *input csv/tsv file path --input='./input.csv'
-  -o, --output    *output tsv file path --output='./output.tsv'
-      --route      set if you need route data from IRR (this might be slow) --route
-      --debug      set if you use HTTP debug feature --debug
+  -h, --help           display help information
+  -p, --provider      *set types of api provider (space separated) --provider='ipdata ipinfo minfraud'
+  -i, --input         *input csv/tsv file path --input='./input.csv'
+  -o, --output        *output tsv file path --output='./output.tsv'
+      --route          set if you need route data from IRR (this might be slow) --route
+      --interval       time interval after a API call to handle rate limit (ms=msec s=sec, m=min) --interval=1.5s
+  -m, --parallel[=2]   parallel number (multiple API calls) --parallel=2
+  -v, --verbose        set if you need detail logs --verbose
+      --debug          set if you use HTTP debug feature --debug
 ```
 
 For example, you can check ip address from csv list like below
@@ -171,6 +176,21 @@ ipdata.co	8.8.4.4		0.00000	Google LLC		15169	US			0.00000	0.00000	false	false	fa
 ipinfo.io	8.8.4.4	dns.google	0.00000		Google LLC	15169	US	Mountain View	California	37.40560	-122.07750	false	false	false	false	false	false	false	false
 ipdata.co	1.1.1.1		0.00000	Cloudflare, Inc.		13335	AU			0.00000	0.00000	false	false	false	false	false	false	false	false
 ipinfo.io	1.1.1.1	one.one.one.one	0.00000		Cloudflare, Inc.	13335	US	San Francisco	California37.76210	-122.39710	false	false	true	false	false	false	false	false
+
+
+
+# if provider has a rate limit, then use --interval and --parallel option.
+$ ./go-ip-fraud-check list -p 'shodan' -i ./input.csv -o ./output.tsv --interval=1.2s --parallel=1
+```
+
+
+### providers command
+
+`providers` command is used to see supported providers.
+
+```bash
+$ ./go-ip-fraud-check providers
+[bigdatacloud ip2proxy ipdata ipgeolocation ipinfo ipqualityscore ipregistry minfraud shodan]
 ```
 
 # Quick Usage for API
@@ -235,3 +255,4 @@ see example dir for more examples.
 | `IPREGISTRY_APIKEY` | [Ipregistry API Key](https://ipregistry.co/docs/authentication). |
 | `MINFRAUD_ACCOUNT_ID` | [MaxMind Account ID](https://support.maxmind.com/account-faq/license-keys/how-do-i-generate-a-license-key/). |
 | `MINFRAUD_LICENSE_KEY` | [MaxMind License Key](https://support.maxmind.com/account-faq/license-keys/how-do-i-generate-a-license-key/). |
+| `FRAUD_CHECK_SHODAN_APIKEY` | [Shodan API Key](https://developer.shodan.io/api/requirements). |

cmd/command_list.go

@@ -1,9 +1,14 @@
 package main
 
 import (
+	"fmt"
+	"os"
+	"os/signal"
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/mkideal/cli"
 
@@ -25,14 +30,15 @@ var outputHeader = []string{
 	"region",
 	"latitude",
 	"longitude",
-	"is_anonymous",
-	"is_anonymous_vpn",
+	"is_vpn",
 	"is_hosting",
 	"is_proxy",
 	"is_tor",
 	"is_bot",
 	"is_bogon",
 	"has_other_threat",
+	"threat_comment",
+	"error",
 	// "as_routes", append by code
 }
 
@@ -43,10 +49,19 @@ type listT struct {
 	InputCSV string `cli:"*i,input" usage:"input csv/tsv file path --input='./input.csv'"`
 	Output   string `cli:"*o,output" usage:"output tsv file path --output='./output.tsv'"`
 	UseRoute bool   `cli:"route" usage:"set if you need route data from IRR (this might be slow) --route"`
+	Interval string `cli:"interval" usage:"time interval after a API call to handle rate limit (ms=msec s=sec, m=min) --interval=1.5s"`
+	Parallel int    `cli:"m,parallel" usage:"parallel number (multiple API calls) --parallel=2" dft:"2"`
+	Verbose  bool   `cli:"v,verbose" usage:"set if you need detail logs --verbose"`
 	Debug    bool   `cli:"debug" usage:"set if you use HTTP debug feature --debug"`
 }
 
 func (a *listT) Validate(ctx *cli.Context) error {
+	if a.Interval != "" {
+		if _, err := time.ParseDuration(a.Interval); err != nil {
+			return fmt.Errorf("invalid 'interval' format: [%w]", err)
+		}
+	}
+
 	return validateProviderString(a.Provider)
 }
 
@@ -69,16 +84,24 @@ type ListRunner struct {
 	Provider string
 	InputCSV string
 	Output   string
+	Parallel int
 	UseRoute bool
+	Interval string
+	Verbose  bool
 	Debug    bool
+
+	logger log.Logger
 }
 
 func newListRunner(p listT) ListRunner {
 	return ListRunner{
 		Provider: p.Provider,
 		InputCSV: p.InputCSV,
 		Output:   p.Output,
+		Parallel: p.Parallel,
 		UseRoute: p.UseRoute,
+		Interval: p.Interval,
+		Verbose:  p.Verbose,
 		Debug:    p.Debug,
 	}
 }
@@ -99,17 +122,26 @@ func (r *ListRunner) Run() error {
 		return err
 	}
 
-	maxReqNum := 3
-	maxReq := make(chan struct{}, maxReqNum)
-
 	providerList, err := getProvidersFromString(r.Provider)
 	if err != nil {
 		panic(err)
 	}
 
+	// parse interval duration
+	var interval time.Duration
+	if r.Interval != "" {
+		v, err := time.ParseDuration(r.Interval)
+		if err != nil {
+			return err
+		}
+		interval = v
+	}
+
 	logger := &log.StdLogger{}
+	r.logger = logger
 	svc, err := ipfraudcheck.New(ipfraudcheck.Config{
 		UseRoute: r.UseRoute,
+		Interval: interval,
 		Debug:    r.Debug,
 		Logger:   logger,
 	}, providerList)
@@ -122,6 +154,20 @@ func (r *ListRunner) Run() error {
 
 	providerSize := len(providerList)
 	result := make([]string, len(lines)*providerSize)
+
+	// handle kill signal
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-sig
+		logger.Errorf("Stop signal detected!")
+		logger.Errorf("Saving intermediate results...")
+		result = append([]string{strings.Join(outputHeader, "\t")}, result...)
+		w.WriteAll(result)
+		os.Exit(2)
+	}()
+
+	maxReq := make(chan struct{}, r.Parallel)
 	var wg sync.WaitGroup
 	for i, line := range lines {
 		i = i * providerSize
@@ -133,38 +179,49 @@ func (r *ListRunner) Run() error {
 				wg.Done()
 			}()
 
-			logger.Infof("exec #: [%d]\n", i/providerSize)
-			rows, err := r.execAPI(svc, line)
+			num := i / providerSize
+			logger.Infof("exec #: [%d]\n", num)
+			rows, err := r.execAPI(svc, line, num)
 			if err != nil {
 				logger.Errorf("#: [%d]; err=[%v]\n", i, err)
 				return
 			}
 			for j, row := range rows {
 				result[i+j] = strings.Join(row, "\t")
 			}
+			svc.WaitInterval()
 		}(i, line)
 	}
 	wg.Wait()
 
 	result = append([]string{strings.Join(outputHeader, "\t")}, result...)
+	logger.Infof("Finished")
 	return w.WriteAll(result)
 }
 
-func (r *ListRunner) execAPI(svc *ipfraudcheck.Client, param map[string]string) ([][]string, error) {
+func (r *ListRunner) execAPI(svc *ipfraudcheck.Client, param map[string]string, num int) ([][]string, error) {
 	resp, err := svc.CheckIP(param["ip_address"])
 	if err != nil {
 		return nil, err
 	}
 
 	rows := make([][]string, len(resp.List))
-	for i, r := range resp.List {
+	for i, res := range resp.List {
 		row := make([]string, 0, len(outputHeader))
 		for _, v := range outputHeader {
 			if v == "as_routes" {
 				row = append(row, strings.Join(resp.ASPrefix, " "))
 				continue
 			}
-			row = append(row, getValue(param, r, v))
+			row = append(row, getValue(param, res, v))
+		}
+		if r.Verbose {
+			switch {
+			case res.Err != "":
+				r.logger.Errorf("#: [%d]; provider=[%s] ip=[%s]  err=[%v]\n", num, res.ServiceName, res.IP, res.Err)
+			default:
+				r.logger.Infof("#: [%d]; provider=[%s] ip=[%s]  row=[%v]\n", num, res.ServiceName, res.IP, row)
+			}
 		}
 		rows[i] = row
 	}
@@ -197,10 +254,8 @@ func getValue(param map[string]string, resp provider.FraudCheckResponse, name st
 		return strconv.FormatFloat(resp.Latitude, 'f', 5, 64)
 	case "longitude":
 		return strconv.FormatFloat(resp.Longitude, 'f', 5, 64)
-	case "is_anonymous":
-		return strconv.FormatBool(resp.IsAnonymous)
 	case "is_anonymous_vpn":
-		return strconv.FormatBool(resp.IsAnonymousVPN)
+		return strconv.FormatBool(resp.IsVPN)
 	case "is_hosting":
 		return strconv.FormatBool(resp.IsHosting)
 	case "is_proxy":
@@ -213,6 +268,10 @@ func getValue(param map[string]string, resp provider.FraudCheckResponse, name st
 		return strconv.FormatBool(resp.IsBogon)
 	case "has_other_threat":
 		return strconv.FormatBool(resp.HasOtherThreat)
+	case "threat_comment":
+		return resp.ThreatComment
+	case "error":
+		return resp.Err
 	}
 	return ""
 }

cmd/command_providers.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/mkideal/cli"
+)
+
+// parameters of 'providers' command.
+type providersT struct {
+	cli.Helper
+}
+
+func (a *providersT) Validate(ctx *cli.Context) error {
+	return nil
+}
+
+var providersC = &cli.Command{
+	Name: "providers",
+	Desc: "Show supported provider types",
+	Argv: func() interface{} { return new(providersT) },
+	Fn:   execShowProviders,
+}
+
+func execShowProviders(ctx *cli.Context) error {
+	argv := ctx.Argv().(*providersT)
+
+	r := newShowProvidersRunner(*argv)
+	return r.Run()
+}
+
+type ShowProvidersRunner struct{}
+
+func newShowProvidersRunner(p providersT) ShowProvidersRunner {
+	return ShowProvidersRunner{}
+}
+
+func (r *ShowProvidersRunner) Run() error {
+	providers := make([]string, 0, len(providerMap))
+	for key := range providerMap {
+		providers = append(providers, key)
+	}
+	sort.Strings(providers)
+	fmt.Printf("%v\n", providers)
+	return nil
+}

cmd/main.go

@@ -12,6 +12,7 @@ func main() {
 		cli.Tree(help),
 		cli.Tree(singleC),
 		cli.Tree(listC),
+		cli.Tree(providersC),
 	).Run(os.Args[1:]); err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)

cmd/providers.go

@@ -13,6 +13,7 @@ import (
 	"github.com/evalphobia/go-ip-fraud-check/provider/ipqualityscore"
 	"github.com/evalphobia/go-ip-fraud-check/provider/ipregistry"
 	"github.com/evalphobia/go-ip-fraud-check/provider/minfraud"
+	"github.com/evalphobia/go-ip-fraud-check/provider/shodan"
 )
 
 const (
@@ -24,6 +25,7 @@ const (
 	providerIPQS          = "ipqualityscore"
 	providerIPRegistry    = "ipregistry"
 	providerMinFraud      = "minfraud"
+	providerShodan        = "shodan"
 )
 
 var providerMap = map[string]provider.Provider{
@@ -35,6 +37,7 @@ var providerMap = map[string]provider.Provider{
 	providerIPQS:          &ipqualityscore.IPQualityScoreProvider{},
 	providerIPRegistry:    &ipregistry.IPRegistryProvider{},
 	providerMinFraud:      &minfraud.MinFraudProvider{},
+	providerShodan:        &shodan.ShodanProvider{},
 }
 
 func validateProviderString(s string) error {

ipfraudcheck/config.go

@@ -14,6 +14,7 @@ const (
 	envIPdatacoAPIKey      = "FRAUD_CHECK_IPDATACO_APIKEY"
 	envIPGeolocationAPIKey = "FRAUD_CHECK_IPGEOLOCATION_APIKEY"
 	envIPinfoioToken       = "FRAUD_CHECK_IPINFOIO_TOKEN"
+	envShodanAPIKey        = "FRAUD_CHECK_SHODAN_APIKEY"
 )
 
 const (
@@ -41,9 +42,12 @@ type Config struct {
 	// minFraud
 	MinFraudAccountID  string
 	MinFraudLicenseKey string
+	// shodan.io
+	ShodanAPIKey string
 
 	// common option
 	UseRoute bool
+	Interval time.Duration
 	Debug    bool
 	Timeout  time.Duration
 	Logger   log.Logger
@@ -100,3 +104,11 @@ func (c Config) GetIPinfoioToken() string {
 	}
 	return c.IPinfoioToken
 }
+
+func (c Config) GetShodanAPIKey() string {
+	s := os.Getenv(envShodanAPIKey)
+	if s != "" {
+		return s
+	}
+	return c.ShodanAPIKey
+}