feat: add flag to disable ssl verify

Author: rgmz

main.go

@@ -78,6 +78,7 @@ var (
 	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
 	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
 	jobReportFile        = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	SslVerify            = cli.Flag("ssl-verify", "Whether to verify the SSL certificates when making requests.").Default("true").Bool()
 
 	noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()
 
@@ -297,7 +298,7 @@ func init() {
 
 	cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
 
-	// Configure logging.
+	// Configure log level.
 	switch {
 	case *trace:
 		log.SetLevel(5)
@@ -318,6 +319,11 @@ func init() {
 			log.SetLevel(l)
 		}
 	}
+
+	// Disable certificate validation, if specified.
+	if !*SslVerify {
+		feature.NoVerifySsl.Store(true)
+	}
 }
 
 func main() {

pkg/common/http.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-retryablehttp"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 )
 
@@ -204,23 +205,31 @@ func RetryableHTTPClientTimeout(timeOutSeconds int64, opts ...ClientOption) *htt
 
 const DefaultResponseTimeout = 5 * time.Second
 
-var saneTransport = &http.Transport{
-	Proxy: http.ProxyFromEnvironment,
-	DialContext: (&net.Dialer{
-		Timeout:   2 * time.Second,
-		KeepAlive: 5 * time.Second,
-	}).DialContext,
-	MaxIdleConns:          5,
-	IdleConnTimeout:       5 * time.Second,
-	TLSHandshakeTimeout:   3 * time.Second,
-	ExpectContinueTimeout: 1 * time.Second,
+func saneTransport() *http.Transport {
+	t := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   2 * time.Second,
+			KeepAlive: 5 * time.Second,
+		}).DialContext,
+		MaxIdleConns:          5,
+		IdleConnTimeout:       5 * time.Second,
+		TLSHandshakeTimeout:   3 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	// Disable TLS certificate validation.
+	if feature.NoVerifySsl.Load() {
+		t.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	}
+	return t
 }
 
 func SaneHttpClient() *http.Client {
-	httpClient := &http.Client{}
-	httpClient.Timeout = DefaultResponseTimeout
-	httpClient.Transport = NewCustomTransport(saneTransport)
-	return httpClient
+	client := &http.Client{}
+	client.Timeout = DefaultResponseTimeout
+	client.Transport = NewCustomTransport(saneTransport())
+	return client
 }
 
 // SaneHttpClientTimeOut adds a custom timeout for some scanners

pkg/detectors/aha/aha.go

@@ -21,8 +21,6 @@ var (
 	// Ensure the Scanner satisfies the interface at compile time.
 	_ detectors.Detector = (*Scanner)(nil)
 
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`)
 	URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.aha\.io)`)
@@ -34,13 +32,6 @@ func (s Scanner) Keywords() []string {
 	return []string{"aha.io"}
 }
 
-func (s Scanner) getClient() *http.Client {
-	if s.client != nil {
-		return s.client
-	}
-	return defaultClient
-}
-
 // FromData will find and optionally verify Aha secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
@@ -61,8 +52,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			client := s.getClient()
-			isVerified, verificationErr := verifyAha(ctx, client, resMatch, resURLMatch)
+			if s.client == nil {
+				s.client = detectors.GetHttpClientWithNoLocalAddresses()
+			}
+			isVerified, verificationErr := verifyAha(ctx, s.client, resMatch, resURLMatch)
 			s1.Verified = isVerified
 			s1.SetVerificationError(verificationErr, resMatch)
 		}

pkg/detectors/apiflash/apiflash.go

@@ -14,15 +14,14 @@ import (
 )
 
 type Scanner struct {
+	client *http.Client
 	detectors.DefaultMultiPartCredentialProvider
 }
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiflash"}) + `\b([a-z0-9]{32})\b`)
 
@@ -51,7 +50,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			isVerified, verificationErr := verifyAPIFlash(ctx, client, key)
+			if s.client == nil {
+				s.client = detectors.GetHttpClientWithNoLocalAddresses()
+			}
+			isVerified, verificationErr := verifyAPIFlash(ctx, s.client, key)
 			s1.Verified = isVerified
 			s1.SetVerificationError(verificationErr, key)
 		}

pkg/detectors/artifactory/artifactory.go

@@ -23,8 +23,6 @@ var (
 	_ detectors.Detector           = (*Scanner)(nil)
 	_ detectors.EndpointCustomizer = (*Scanner)(nil)
 
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`)
 	URLPat = regexp.MustCompile(`\b([A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])\.jfrog\.io)`)
@@ -36,13 +34,6 @@ func (s Scanner) Keywords() []string {
 	return []string{"artifactory"}
 }
 
-func (s Scanner) getClient() *http.Client {
-	if s.client != nil {
-		return s.client
-	}
-	return defaultClient
-}
-
 // FromData will find and optionally verify Artifactory secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
@@ -57,8 +48,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	for _, match := range matches {
 		resMatch := strings.TrimSpace(match[1])
 
-		client := s.getClient()
-
 		for _, URL := range s.Endpoints(resURLMatch) {
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
@@ -67,7 +56,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch)
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithNoLocalAddresses()
+				}
+
+				isVerified, verificationErr := verifyArtifactory(ctx, s.client, URL, resMatch)
 				s1.Verified = isVerified
 				s1.SetVerificationError(verificationErr, resMatch)
 			}

pkg/detectors/auth0managementapitoken/auth0managementapitoken.go

@@ -13,6 +13,7 @@ import (
 )
 
 type Scanner struct {
+	client *http.Client
 	detectors.DefaultMultiPartCredentialProvider
 }
 
@@ -21,8 +22,6 @@ var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil)
 
 var (
-	client = detectors.DetectorHttpClientWithLocalAddresses
-
 	// long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days
 	// at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer
 	managementAPITokenPat = regexp.MustCompile(`\b(ey[a-zA-Z0-9._-]+)\b`)
@@ -61,6 +60,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithLocalAddresses()
+				}
 				/*
 				   curl -H "Authorization: Bearer $token" https://domain/api/v2/users
 				*/
@@ -70,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", managementAPITokenRes))
-				res, err := client.Do(req)
+				res, err := s.client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/azure_batch/azurebatch.go

@@ -26,7 +26,6 @@ var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
 
 var (
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	urlPat    = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`)
 	secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`)
@@ -61,9 +60,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithNoLocalAddresses()
 				}
 				url := fmt.Sprintf("%s/applications?api-version=2020-09-01.12.0", endpoint)
 				date := time.Now().UTC().Format(http.TimeFormat)
@@ -84,7 +82,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				req.Header.Set("Content-Type", "application/json")
 				req.Header.Set("Authorization", fmt.Sprintf("SharedKey %s:%s", accountName, signature))
 				req.Header.Set("Date", date)
-				resp, err := client.Do(req)
+				resp, err := s.client.Do(req)
 				if err != nil {
 					continue
 				}

pkg/detectors/azure_storage/storage.go

@@ -26,8 +26,6 @@ type Scanner struct {
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	namePat = regexp.MustCompile(`(?i:Account[_.-]?Name|Storage[_.-]?(?:Account|Name))(?:.|\s){0,20}?\b([a-z0-9]{3,24})\b|([a-z0-9]{3,24})(?i:\.blob\.core\.windows\.net)`) // Names can only be lowercase alphanumeric.
 	keyPat  = regexp.MustCompile(`(?i:(?:Access|Account|Storage)[_.-]?Key)(?:.|\s){0,25}?([a-zA-Z0-9+\/-]{86,88}={0,2})`)
 
@@ -105,12 +103,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithNoLocalAddresses()
 				}
 
-				isVerified, verificationErr := s.verifyMatch(ctx, client, name, key, s1.ExtraData)
+				isVerified, verificationErr := s.verifyMatch(ctx, s.client, name, key, s1.ExtraData)
 				s1.Verified = isVerified
 				s1.SetVerificationError(verificationErr, key)
 			}

pkg/detectors/azuresearchquerykey/azuresearchquerykey.go

@@ -21,7 +21,6 @@ type Scanner struct {
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`)
 	urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `https:\/\/([0-9a-z]{5,40})\.search\.windows\.net\/indexes\/([0-9a-z]{5,40})\b`)
@@ -53,17 +52,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				RawV2:        []byte(resMatch + resUrlMatch),
 			}
 			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithNoLocalAddresses()
 				}
 				req, err := http.NewRequestWithContext(ctx, "GET", resUrlMatch+"/docs/$count?api-version=2023-10-01-Preview", nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("api-key", resMatch)
 
-				res, err := client.Do(req)
+				res, err := s.client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/caspio/caspio.go

@@ -13,15 +13,14 @@ import (
 )
 
 type Scanner struct {
+	client *http.Client
 	detectors.DefaultMultiPartCredentialProvider
 }
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`)
 	idPat     = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`)
@@ -60,13 +59,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 
 				if verify {
+					if s.client == nil {
+						s.client = detectors.GetHttpClientWithNoLocalAddresses()
+					}
+
 					payload := strings.NewReader(fmt.Sprintf(`grant_type=client_credentials&client_id=%s&client_secret=%s`, resIdMatch, resMatch))
 					req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("https://%s.caspio.com/oauth/token", resDomainMatch), payload)
 					if err != nil {
 						continue
 					}
 					req.Header.Add("Content-Type", "text/plain")
-					res, err := client.Do(req)
+					res, err := s.client.Do(req)
 					if err == nil {
 						defer res.Body.Close()
 						if res.StatusCode >= 200 && res.StatusCode < 300 {

pkg/detectors/databrickstoken/databrickstoken.go

@@ -21,8 +21,6 @@ type Scanner struct {
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azuredatabricks\.net))\b`)
 	keyPat = regexp.MustCompile(`\b(dapi[0-9a-f]{32}(-\d)?)\b`)
@@ -54,16 +52,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-				client := s.client
-				if client == nil {
-					client = defaultClient
+				if s.client == nil {
+					s.client = detectors.GetHttpClientWithNoLocalAddresses()
 				}
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomainMatch+"/api/2.0/clusters/list", nil)
 				if err != nil {
 					continue
 				}
 				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
-				res, err := client.Do(req)
+				res, err := s.client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {