Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update SendGrid detector #2833

Merged
merged 1 commit into from
May 14, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 65 additions & 38 deletions pkg/detectors/sendgrid/sendgrid.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ package sendgrid

import (
"context"
"encoding/json"
"fmt"
regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"

regexp "github.com/wasilibs/go-re2"

"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"

Expand All @@ -23,7 +26,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()

keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
keyPat = regexp.MustCompile(`\bSG\.[\w\-]{20,24}\.[\w\-]{39,50}\b`)
)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -36,53 +39,27 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)

matches := keyPat.FindAllStringSubmatch(dataStr, -1)

for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
uniqueMatches := make(map[string]struct{})
for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
uniqueMatches[match[0]] = struct{}{}
}
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is to avoid verifying the same matches found in same chunk multiple times right?
@ahrav if this is helpful shouldn't we have this in all detectors then? Maybe another [chore] task for you 🥲

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a bit of a bandaid solution to #2262. I mentioned this as well in #2812 (comment).

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea, we're aware of the task and are considering moving the caching logic into the engine to streamline processes by centralizing the logic, which would eliminate the need to replicate it across all detectors. For now, implementing the caching within each individual detector is an acceptable temporary solution.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO it's probably another use case for SQLite, similar to #2696.


for token := range uniqueMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_SendGrid,
Raw: []byte(resMatch),
}
s1.ExtraData = map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/sendgrid/",
Raw: []byte(token),
}

if verify {
// there are a few endpoints we can check, but templates seems the least sensitive.
// 403 will be issued if the scope is wrong but the key is correct
baseURL := "https://api.sendgrid.com/v3/templates"

client := s.client
if client == nil {
client = defaultClient
}

// test `read_user` scope
req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
res.Body.Close() // The request body is unused.

// 200 means good key and has `templates` scope
// 403 means good key but not the right scope
// 401 is bad key
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
s1.Verified = true
} else if res.StatusCode != http.StatusUnauthorized {
err = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
s1.SetVerificationError(err, resMatch)
}
}
verified, extraData, verificationErr := verifyToken(ctx, client, token)
s1.Verified = verified
s1.ExtraData = extraData
s1.SetVerificationError(verificationErr)
}

results = append(results, s1)
Expand All @@ -91,6 +68,56 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return
}

func verifyToken(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
// Check the scopes assigned to the api key.
// https://docs.sendgrid.com/api-reference/api-key-permissions/retrieve-a-list-of-scopes-for-which-this-user-has-access
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.sendgrid.com/v3/scopes", nil)
if err != nil {
return false, nil, err
}

req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
req.Header.Set("Content-Type", "application/json")
res, err := client.Do(req)
if err != nil {
return false, nil, err
}
defer func() {
_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
}()

switch res.StatusCode {
case http.StatusOK:
extraData := map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/sendgrid/",
}

var scopesRes scopesResponse
if err := json.NewDecoder(res.Body).Decode(&scopesRes); err != nil {
return false, nil, err
}

if len(scopesRes.Scopes) > 0 {
extraData["scopes"] = strings.Join(scopesRes.Scopes, ",")
}

return true, extraData, nil
case http.StatusUnauthorized:
// 401 means the key is definitively invalid.
return false, nil, nil
case http.StatusForbidden:
// 403 means good key but not the right scope
return true, nil, nil
default:
return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
}

type scopesResponse struct {
Scopes []string `json:"scopes"`
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_SendGrid
}
Loading