Skip to content

Commit 4d49e30

Browse files
rgmzrgmz
committed
build: optionally exclude sources
1 parent 12164e3 commit 4d49e30

File tree

100 files changed

+593
-186
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+593
-186
lines changed

‎main.go

+31-17
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,21 @@ import (
3030
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
3131
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
3232
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
33+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/circleci"
3334
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults"
35+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/docker"
36+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/elasticsearch"
37+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/filesystem"
38+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/gcs"
39+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/git"
40+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/github"
41+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/gitlab"
42+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/huggingface"
43+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/jenkins"
44+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/postman"
45+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/s3"
46+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/syslog"
47+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/travisci"
3448
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
3549
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
3650
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
@@ -704,7 +718,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
704718
Bare: *gitScanBare,
705719
ExcludeGlobs: *gitScanExcludeGlobs,
706720
}
707-
if ref, err = eng.ScanGit(ctx, gitCfg); err != nil {
721+
if ref, err = git.Scan(ctx, gitCfg, eng); err != nil {
708722
return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
709723
}
710724
case githubScan.FullCommand():
@@ -733,7 +747,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
733747
CommentsTimeframeDays: *githubCommentsTimeframeDays,
734748
Filter: filter,
735749
}
736-
if ref, err = eng.ScanGitHub(ctx, cfg); err != nil {
750+
if ref, err = github.Scan(ctx, cfg, eng); err != nil {
737751
return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
738752
}
739753
case githubExperimentalScan.FullCommand():
@@ -744,7 +758,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
744758
CollisionThreshold: *githubExperimentalCollisionThreshold,
745759
DeleteCachedData: *githubExperimentalDeleteCache,
746760
}
747-
if ref, err = eng.ScanGitHubExperimental(ctx, cfg); err != nil {
761+
if ref, err = github.ScanExperimental(ctx, cfg, eng); err != nil {
748762
return scanMetrics, fmt.Errorf("failed to scan using Github Experimental: %v", err)
749763
}
750764
case gitlabScan.FullCommand():
@@ -761,7 +775,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
761775
ExcludeRepos: *gitlabScanExcludeRepos,
762776
Filter: filter,
763777
}
764-
if ref, err = eng.ScanGitLab(ctx, cfg); err != nil {
778+
if ref, err = gitlab.Scan(ctx, cfg, eng); err != nil {
765779
return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
766780
}
767781
case filesystemScan.FullCommand():
@@ -776,7 +790,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
776790
IncludePathsFile: *filesystemScanIncludePaths,
777791
ExcludePathsFile: *filesystemScanExcludePaths,
778792
}
779-
if ref, err = eng.ScanFileSystem(ctx, cfg); err != nil {
793+
if ref, err = filesystem.Scan(ctx, cfg, eng); err != nil {
780794
return scanMetrics, fmt.Errorf("failed to scan filesystem: %v", err)
781795
}
782796
case s3Scan.FullCommand():
@@ -790,7 +804,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
790804
CloudCred: *s3ScanCloudEnv,
791805
MaxObjectSize: int64(*s3ScanMaxObjectSize),
792806
}
793-
if ref, err = eng.ScanS3(ctx, cfg); err != nil {
807+
if ref, err = s3.Scan(ctx, cfg, eng); err != nil {
794808
return scanMetrics, fmt.Errorf("failed to scan S3: %v", err)
795809
}
796810
case syslogScan.FullCommand():
@@ -802,15 +816,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
802816
KeyPath: *syslogTLSKey,
803817
Concurrency: *concurrency,
804818
}
805-
if ref, err = eng.ScanSyslog(ctx, cfg); err != nil {
819+
if ref, err = syslog.Scan(ctx, cfg, eng); err != nil {
806820
return scanMetrics, fmt.Errorf("failed to scan syslog: %v", err)
807821
}
808822
case circleCiScan.FullCommand():
809-
if ref, err = eng.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
823+
if ref, err = circleci.Scan(ctx, *circleCiScanToken, eng); err != nil {
810824
return scanMetrics, fmt.Errorf("failed to scan CircleCI: %v", err)
811825
}
812826
case travisCiScan.FullCommand():
813-
if ref, err = eng.ScanTravisCI(ctx, *travisCiScanToken); err != nil {
827+
if ref, err = travisci.Scan(ctx, *travisCiScanToken, eng); err != nil {
814828
return scanMetrics, fmt.Errorf("failed to scan TravisCI: %v", err)
815829
}
816830
case gcsScan.FullCommand():
@@ -827,7 +841,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
827841
Concurrency: *concurrency,
828842
MaxObjectSize: int64(*gcsMaxObjectSize),
829843
}
830-
if ref, err = eng.ScanGCS(ctx, cfg); err != nil {
844+
if ref, err = gcs.Scan(ctx, cfg, eng); err != nil {
831845
return scanMetrics, fmt.Errorf("failed to scan GCS: %v", err)
832846
}
833847
case dockerScan.FullCommand():
@@ -836,7 +850,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
836850
Images: *dockerScanImages,
837851
UseDockerKeychain: *dockerScanToken == "",
838852
}
839-
if ref, err = eng.ScanDocker(ctx, cfg); err != nil {
853+
if ref, err = docker.Scan(ctx, cfg, eng); err != nil {
840854
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
841855
}
842856
case postmanScan.FullCommand():
@@ -873,7 +887,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
873887
WorkspacePaths: *postmanWorkspacePaths,
874888
EnvironmentPaths: *postmanEnvironmentPaths,
875889
}
876-
if ref, err = eng.ScanPostman(ctx, cfg); err != nil {
890+
if ref, err = postman.Scan(ctx, cfg, eng); err != nil {
877891
return scanMetrics, fmt.Errorf("failed to scan Postman: %v", err)
878892
}
879893
case elasticsearchScan.FullCommand():
@@ -889,17 +903,17 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
889903
SinceTimestamp: *elasticsearchSinceTimestamp,
890904
BestEffortScan: *elasticsearchBestEffortScan,
891905
}
892-
if ref, err = eng.ScanElasticsearch(ctx, cfg); err != nil {
906+
if ref, err = elasticsearch.Scan(ctx, cfg, eng); err != nil {
893907
return scanMetrics, fmt.Errorf("failed to scan Elasticsearch: %v", err)
894908
}
895909
case jenkinsScan.FullCommand():
896-
cfg := engine.JenkinsConfig{
910+
cfg := sources.JenkinsConfig{
897911
Endpoint: *jenkinsURL,
898912
InsecureSkipVerifyTLS: *jenkinsInsecureSkipVerifyTLS,
899913
Username: *jenkinsUsername,
900914
Password: *jenkinsPassword,
901915
}
902-
if ref, err = eng.ScanJenkins(ctx, cfg); err != nil {
916+
if ref, err = jenkins.Scan(ctx, cfg, eng); err != nil {
903917
return scanMetrics, fmt.Errorf("failed to scan Jenkins: %v", err)
904918
}
905919
case huggingfaceScan.FullCommand():
@@ -911,7 +925,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
911925
return scanMetrics, fmt.Errorf("invalid config: you must specify at least one organization, user, model, space or dataset")
912926
}
913927

914-
cfg := engine.HuggingfaceConfig{
928+
cfg := sources.HuggingfaceConfig{
915929
Endpoint: *huggingfaceEndpoint,
916930
Models: *huggingfaceModels,
917931
Spaces: *huggingfaceSpaces,
@@ -932,7 +946,7 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
932946
IncludePrs: *huggingfaceIncludePrs,
933947
Concurrency: *concurrency,
934948
}
935-
if ref, err = eng.ScanHuggingface(ctx, cfg); err != nil {
949+
if ref, err = huggingface.Scan(ctx, cfg, eng); err != nil {
936950
return scanMetrics, fmt.Errorf("failed to scan HuggingFace: %v", err)
937951
}
938952
default:

‎pkg/engine/circleci.go ‎pkg/engine/circleci/circleci.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
package engine
1+
//go:build !no_circleci
2+
3+
package circleci
24

35
import (
46
"runtime"
@@ -7,13 +9,14 @@ import (
79
"google.golang.org/protobuf/types/known/anypb"
810

911
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
1013
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
1114
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
1215
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
1316
)
1417

15-
// ScanCircleCI scans CircleCI logs.
16-
func (e *Engine) ScanCircleCI(ctx context.Context, token string) (sources.JobProgressRef, error) {
18+
// Scan scans CircleCI logs.
19+
func Scan(ctx context.Context, token string, e *engine.Engine) (sources.JobProgressRef, error) {
1720
connection := &sourcespb.CircleCI{
1821
Credential: &sourcespb.CircleCI_Token{
1922
Token: token,
@@ -28,11 +31,11 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) (sources.JobPro
2831
}
2932

3033
sourceName := "trufflehog - Circle CI"
31-
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)
34+
sourceID, jobID, _ := e.SourceManager().GetIDs(ctx, sourceName, circleci.SourceType)
3235

3336
circleSource := &circleci.Source{}
3437
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
3538
return sources.JobProgressRef{}, err
3639
}
37-
return e.sourceManager.EnumerateAndScan(ctx, sourceName, circleSource)
40+
return e.SourceManager().EnumerateAndScan(ctx, sourceName, circleSource)
3841
}

‎pkg/engine/circleci/circleci_disabled.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//go:build no_circleci
2+
3+
package circleci
4+
5+
import (
6+
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
7+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
8+
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
9+
)
10+
11+
func Scan(_ context.Context, _ string, _ *engine.Engine) (sources.JobProgressRef, error) {
12+
return sources.JobProgressRef{}, engine.ErrSourceDisabled
13+
}

‎pkg/engine/docker.go ‎pkg/engine/docker/docker.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
package engine
1+
//go:build !no_docker
2+
3+
package docker
24

35
import (
46
"runtime"
@@ -7,13 +9,14 @@ import (
79
"google.golang.org/protobuf/types/known/anypb"
810

911
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
1013
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
1114
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
1215
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
1316
)
1417

15-
// ScanDocker scans a given docker connection.
16-
func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {
18+
// Scan scans a given docker connection.
19+
func Scan(ctx context.Context, c sources.DockerConfig, e *engine.Engine) (sources.JobProgressRef, error) {
1720
connection := &sourcespb.Docker{Images: c.Images}
1821

1922
switch {
@@ -33,11 +36,11 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (source
3336
}
3437

3538
sourceName := "trufflehog - docker"
36-
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)
39+
sourceID, jobID, _ := e.SourceManager().GetIDs(ctx, sourceName, docker.SourceType)
3740

3841
dockerSource := &docker.Source{}
3942
if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
4043
return sources.JobProgressRef{}, err
4144
}
42-
return e.sourceManager.EnumerateAndScan(ctx, sourceName, dockerSource)
45+
return e.SourceManager().EnumerateAndScan(ctx, sourceName, dockerSource)
4346
}

‎pkg/engine/docker/docker_disabled.go

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//go:build no_docker
2+
3+
package docker
4+
5+
import (
6+
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
7+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
8+
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
9+
)
10+
11+
func Scan(_ context.Context, _ sources.DockerConfig, _ *engine.Engine) (sources.JobProgressRef, error) {
12+
return sources.JobProgressRef{}, engine.ErrSourceDisabled
13+
}

‎pkg/engine/elasticsearch.go ‎pkg/engine/elasticsearch/elasticsearch.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
package engine
1+
//go:build !no_elasticsearch
2+
3+
package elasticsearch
24

35
import (
46
"runtime"
@@ -7,13 +9,14 @@ import (
79
"google.golang.org/protobuf/types/known/anypb"
810

911
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
1013
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
1114
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
1215
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/elasticsearch"
1316
)
1417

15-
// ScanElasticsearch scans a Elasticsearch installation.
16-
func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchConfig) (sources.JobProgressRef, error) {
18+
// Scan scans a Elasticsearch installation.
19+
func Scan(ctx context.Context, c sources.ElasticsearchConfig, e *engine.Engine) (sources.JobProgressRef, error) {
1720
connection := &sourcespb.Elasticsearch{
1821
Nodes: c.Nodes,
1922
Username: c.Username,
@@ -35,11 +38,11 @@ func (e *Engine) ScanElasticsearch(ctx context.Context, c sources.ElasticsearchC
3538
}
3639

3740
sourceName := "trufflehog - Elasticsearch"
38-
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, elasticsearch.SourceType)
41+
sourceID, jobID, _ := e.SourceManager().GetIDs(ctx, sourceName, elasticsearch.SourceType)
3942

4043
elasticsearchSource := &elasticsearch.Source{}
4144
if err := elasticsearchSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
4245
return sources.JobProgressRef{}, err
4346
}
44-
return e.sourceManager.EnumerateAndScan(ctx, sourceName, elasticsearchSource)
47+
return e.SourceManager().EnumerateAndScan(ctx, sourceName, elasticsearchSource)
4548
}

‎pkg/engine/elasticsearch/elasticsearch_disabled.go

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
//go:build no_elasticsearch
2+
3+
package elasticsearch
4+
5+
import (
6+
"context"
7+
8+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
9+
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
10+
)
11+
12+
func Scan(_ context.Context, _ sources.ElasticsearchConfig, _ *engine.Engine) (sources.JobProgressRef, error) {
13+
return sources.JobProgressRef{}, engine.ErrSourceDisabled
14+
}

‎pkg/engine/engine.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,12 @@ import (
3333

3434
var detectionTimeout = detectors.DefaultResponseTimeout
3535

36-
var errOverlap = errors.New(
37-
"More than one detector has found this result. For your safety, verification has been disabled." +
38-
"You can override this behavior by using the --allow-verification-overlap flag.",
36+
var (
37+
ErrSourceDisabled = errors.New("trufflehog was compiled without this source")
38+
errOverlap = errors.New(
39+
"More than one detector has found this result. For your safety, verification has been disabled." +
40+
"You can override this behavior by using the --allow-verification-overlap flag.",
41+
)
3942
)
4043

4144
// Metrics for the scan engine for external consumption.
@@ -219,6 +222,10 @@ type Engine struct {
219222
verificationOverlapWorkerMultiplier int
220223
}
221224

225+
func (e *Engine) SourceManager() *sources.SourceManager {
226+
return e.sourceManager
227+
}
228+
222229
// NewEngine creates a new Engine instance with the provided configuration.
223230
func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {
224231
verificationCache := verificationcache.New(cfg.VerificationResultCache, cfg.VerificationCacheMetrics)

0 commit comments

Comments
 (0)