Skip to content

Commit d237c9c

Browse files
abmussaniabmussani
authored
Merge branch 'main' into fix-postgres-detector-dbtype-bug
2 parents 4a39478 + 82ea62c commit d237c9c

20 files changed

+1565
-908
lines changed

‎pkg/analyzer/analyzers/privatekey/privatekey.go

+27
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"errors"
77
"fmt"
88
"os"
9+
"regexp"
910
"strings"
1011
"sync"
1112
"time"
@@ -130,6 +131,9 @@ func AnalyzeAndPrintPermissions(cfg *config.Config, key string) {
130131
return
131132
}
132133

134+
// key entered through command line may have spaces instead of newlines, replace them
135+
token = replaceSpacesWithNewlines(token)
136+
133137
info, err := AnalyzePermissions(context.Background(), cfg, token)
134138
if err != nil {
135139
color.Red("[x] Error: %s", err.Error())
@@ -301,3 +305,26 @@ func analyzeGithubUser(ctx context.Context, parsedKey any) (*string, error) {
301305
func analyzeGitlabUser(ctx context.Context, parsedKey any) (*string, error) {
302306
return privatekey.VerifyGitLabUser(ctx, parsedKey)
303307
}
308+
309+
// replaceSpacesWithNewlines extracts the base64 part, replaces spaces with newlines if needed, and reconstructs the key.
310+
func replaceSpacesWithNewlines(privateKey string) string {
311+
// Regex pattern to extract the key content
312+
re := regexp.MustCompile(`(?i)(-----\s*BEGIN[ A-Z0-9_-]*PRIVATE KEY\s*-----)\s*([\s\S]*?)\s*(-----\s*END[ A-Z0-9_-]*PRIVATE KEY\s*-----)`)
313+
314+
// Find matches
315+
matches := re.FindStringSubmatch(privateKey)
316+
if len(matches) != 4 {
317+
// no need to process
318+
return privateKey
319+
}
320+
321+
header := matches[1] // BEGIN line
322+
base64Part := matches[2] // Base64 content
323+
footer := matches[3] // END line
324+
325+
// Replace spaces with newlines
326+
formattedBase64 := strings.ReplaceAll(base64Part, " ", "\n")
327+
328+
// Reconstruct the private key
329+
return fmt.Sprintf("%s\n%s\n%s", header, formattedBase64, footer)
330+
}

‎pkg/detectors/sentryorgtoken/sentryorgtoken.go

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
package sentryorgtoken
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
9+
regexp "github.com/wasilibs/go-re2"
10+
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
12+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
13+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
14+
)
15+
16+
type Scanner struct {
17+
client *http.Client
18+
}
19+
20+
// Ensure the Scanner satisfies the interface at compile time.
21+
var _ detectors.Detector = (*Scanner)(nil)
22+
23+
var (
24+
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
25+
orgAuthTokenPat = regexp.MustCompile(`\b(sntrys_eyJ[a-zA-Z0-9=_+/]{197})\b`)
26+
)
27+
28+
// Keywords are used for efficiently pre-filtering chunks.
29+
// Use identifiers in the secret preferably, or the provider name.
30+
func (s Scanner) Keywords() []string {
31+
return []string{"sntrys_eyJ"}
32+
}
33+
34+
func (s Scanner) Type() detectorspb.DetectorType {
35+
return detectorspb.DetectorType_SentryOrgToken
36+
}
37+
38+
func (s Scanner) Description() string {
39+
return "Sentry is an error tracking service that helps developers monitor and fix crashes in real time. Sentry Organization Auth Tokens can be used in many places to interact with Sentry programmatically. For example, they can be used for sentry-cli, bundler plugins, or similar use cases."
40+
}
41+
42+
// FromData will find and optionally verify SentryToken secrets in a given set of bytes.
43+
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
44+
dataStr := string(data)
45+
46+
// find all unique org auth tokens
47+
var uniqueOrgTokens = make(map[string]struct{})
48+
49+
for _, orgToken := range orgAuthTokenPat.FindAllStringSubmatch(dataStr, -1) {
50+
uniqueOrgTokens[orgToken[1]] = struct{}{}
51+
}
52+
53+
for orgToken := range uniqueOrgTokens {
54+
s1 := detectors.Result{
55+
DetectorType: detectorspb.DetectorType_SentryOrgToken,
56+
Raw: []byte(orgToken),
57+
}
58+
59+
if verify {
60+
if s.client == nil {
61+
s.client = common.SaneHttpClient()
62+
}
63+
64+
isVerified, verificationErr := verifySentryOrgToken(ctx, s.client, orgToken)
65+
s1.Verified = isVerified
66+
s1.SetVerificationError(verificationErr, orgToken)
67+
}
68+
69+
results = append(results, s1)
70+
}
71+
72+
return results, nil
73+
}
74+
75+
// docs: https://docs.sentry.io/account/auth-tokens/#organization-auth-tokens
76+
func verifySentryOrgToken(ctx context.Context, client *http.Client, token string) (bool, error) {
77+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/auth/validate", nil)
78+
if err != nil {
79+
return false, err
80+
}
81+
82+
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
83+
84+
resp, err := client.Do(req)
85+
if err != nil {
86+
return false, err
87+
}
88+
defer func() {
89+
_, _ = io.Copy(io.Discard, resp.Body)
90+
_ = resp.Body.Close()
91+
}()
92+
93+
switch resp.StatusCode {
94+
case http.StatusOK:
95+
return true, nil
96+
case http.StatusForbidden, http.StatusUnauthorized:
97+
return false, nil
98+
default:
99+
return false, fmt.Errorf("unexpected HTTP response status %d", resp.StatusCode)
100+
}
101+
}

‎pkg/detectors/sentryorgtoken/sentryorgtoken_integration_test.go

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
//go:build detectors
2+
// +build detectors
3+
4+
package sentryorgtoken
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/google/go-cmp/cmp/cmpopts"
14+
15+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
16+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
17+
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
18+
)
19+
20+
func TestSentryOrgToken_FromChunk(t *testing.T) {
21+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
22+
defer cancel()
23+
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
24+
if err != nil {
25+
t.Fatalf("could not get test secrets from GCP: %s", err)
26+
}
27+
secret := testSecrets.MustGetField("SENTRY_ORG_TOKEN")
28+
inactiveSecret := testSecrets.MustGetField("SENTRY_ORG_TOKEN_INACTIVE")
29+
30+
type args struct {
31+
ctx context.Context
32+
data []byte
33+
verify bool
34+
}
35+
tests := []struct {
36+
name string
37+
s Scanner
38+
args args
39+
want []detectors.Result
40+
wantErr bool
41+
wantVerificationErr bool
42+
}{
43+
{
44+
name: "found, verified",
45+
s: Scanner{},
46+
args: args{
47+
ctx: context.Background(),
48+
data: []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),
49+
verify: true,
50+
},
51+
want: []detectors.Result{
52+
{
53+
DetectorType: detectorspb.DetectorType_SentryOrgToken,
54+
Verified: true,
55+
},
56+
},
57+
wantErr: false,
58+
},
59+
{
60+
name: "found, unverified",
61+
s: Scanner{},
62+
args: args{
63+
ctx: context.Background(),
64+
data: []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
65+
verify: true,
66+
},
67+
want: []detectors.Result{
68+
{
69+
DetectorType: detectorspb.DetectorType_SentryOrgToken,
70+
Verified: false,
71+
},
72+
},
73+
wantErr: false,
74+
},
75+
{
76+
name: "not found",
77+
s: Scanner{},
78+
args: args{
79+
ctx: context.Background(),
80+
data: []byte("You cannot find the secret within"),
81+
verify: true,
82+
},
83+
want: nil,
84+
wantErr: false,
85+
},
86+
}
87+
88+
for _, tt := range tests {
89+
t.Run(tt.name, func(t *testing.T) {
90+
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
91+
if (err != nil) != tt.wantErr {
92+
t.Errorf("SentryOrgToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
93+
return
94+
}
95+
for i := range got {
96+
if len(got[i].Raw) == 0 {
97+
t.Fatal("no raw secret present")
98+
}
99+
if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
100+
t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
101+
}
102+
}
103+
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
104+
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
105+
t.Errorf("SentryOrgToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
106+
}
107+
})
108+
}
109+
}
110+
111+
func BenchmarkFromData(benchmark *testing.B) {
112+
ctx := context.Background()
113+
s := Scanner{}
114+
for name, data := range detectors.MustGetBenchmarkData() {
115+
benchmark.Run(name, func(b *testing.B) {
116+
b.ResetTimer()
117+
for n := 0; n < b.N; n++ {
118+
_, err := s.FromData(ctx, false, data)
119+
if err != nil {
120+
b.Fatal(err)
121+
}
122+
}
123+
})
124+
}
125+
}

‎pkg/detectors/sentryorgtoken/sentryorgtoken_test.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package sentryorgtoken
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/google/go-cmp/cmp"
9+
10+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
11+
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
12+
)
13+
14+
var (
15+
validPattern = `
16+
sentry_token := sntrys_eyJFAKEiOjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI
17+
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", sentry_token))
18+
`
19+
invalidPattern = "sntrys_eyJFAKE-OjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI"
20+
token = "sntrys_eyJFAKEiOjE3NDIzNjM1NTIuNTAzMzA5LCJ1cmwiOiJodHRwczovL3NlbnRyeS5pbyIsInJlZ2lvbl91cmwiOiJodHRwczovL3VzLnNlbnRyeS5pbfakem9yZyI6InRydWZmbGUtc2VjdXJpdHktamQifQ==_+zqSnKjs87cicc3FAK08vmZs5cWx9C5EARKHFtW5lqI"
21+
)
22+
23+
func TestSentryToken_Pattern(t *testing.T) {
24+
d := Scanner{}
25+
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
26+
tests := []struct {
27+
name string
28+
input string
29+
want []string
30+
}{
31+
{
32+
name: "valid pattern - with keyword sentry org token",
33+
input: validPattern,
34+
want: []string{token},
35+
},
36+
{
37+
name: "valid pattern - ignore duplicate",
38+
input: fmt.Sprintf("token = '%s' | '%s'", validPattern, validPattern),
39+
want: []string{token},
40+
},
41+
{
42+
name: "invalid pattern",
43+
input: invalidPattern,
44+
want: []string{},
45+
},
46+
}
47+
48+
for _, test := range tests {
49+
t.Run(test.name, func(t *testing.T) {
50+
matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
51+
if len(matchedDetectors) == 0 {
52+
t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
53+
return
54+
}
55+
56+
results, err := d.FromData(context.Background(), false, []byte(test.input))
57+
if err != nil {
58+
t.Errorf("error = %v", err)
59+
return
60+
}
61+
62+
if len(results) != len(test.want) {
63+
if len(results) == 0 {
64+
t.Errorf("did not receive result")
65+
} else {
66+
t.Errorf("expected %d results, only received %d", len(test.want), len(results))
67+
}
68+
return
69+
}
70+
71+
actual := make(map[string]struct{}, len(results))
72+
for _, r := range results {
73+
if len(r.RawV2) > 0 {
74+
actual[string(r.RawV2)] = struct{}{}
75+
} else {
76+
actual[string(r.Raw)] = struct{}{}
77+
}
78+
}
79+
expected := make(map[string]struct{}, len(test.want))
80+
for _, v := range test.want {
81+
expected[v] = struct{}{}
82+
}
83+
84+
if diff := cmp.Diff(expected, actual); diff != "" {
85+
t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
86+
}
87+
})
88+
}
89+
}

‎pkg/detectors/sentrytoken/v1/sentrytoken.go

+2-12
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
8181
func VerifyToken(ctx context.Context, client *http.Client, token string) (map[string]string, bool, error) {
8282
// api docs: https://docs.sentry.io/api/organizations/
8383
// this api will return 200 for user auth tokens with scope of org:<>
84-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/organizations/", nil)
84+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/auth/validate/", nil)
8585
if err != nil {
8686
return nil, false, err
8787
}
@@ -98,17 +98,7 @@ func VerifyToken(ctx context.Context, client *http.Client, token string) (map[st
9898

9999
switch resp.StatusCode {
100100
case http.StatusOK:
101-
var organizations []Organization
102-
if err = json.NewDecoder(resp.Body).Decode(&organizations); err != nil {
103-
return nil, false, err
104-
}
105-
106-
var extraData = make(map[string]string)
107-
for _, org := range organizations {
108-
extraData[fmt.Sprintf("orginzation_%s", org.ID)] = org.Name
109-
}
110-
111-
return extraData, true, nil
101+
return nil, true, nil
112102
case http.StatusForbidden:
113103
var APIResp interface{}
114104
if err = json.NewDecoder(resp.Body).Decode(&APIResp); err != nil {

‎pkg/detectors/sentrytoken/v1/sentrytoken_integration_test.go

-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ func TestSentryToken_FromChunk(t *testing.T) {
5252
{
5353
DetectorType: detectorspb.DetectorType_SentryToken,
5454
Verified: true,
55-
ExtraData: map[string]string{"orginzation_4508567357947904": "Truffle Security"},
5655
},
5756
},
5857
wantErr: false,

0 commit comments

Comments
 (0)