Skip to content

Commit 4f82486

Browse files
TBX3DTBX3D
committed
feat(js): add provider-prefixed and structural secret rules
extends the secret bank with eleven providers it did not cover: pypi, openai (legacy and project/service-account), square, mailgun, discord (bot tokens and incoming webhooks), new relic, cloudinary, jwts and scheme://user:pass@host connection strings. the last two have no fixed prefix, so shape alone is not proof: add a validate hook that runs after the entropy and digit gates. jwt decodes the header segment and requires alg and typ, which drops arbitrary dot-separated base64url blobs. connection strings reject a userinfo password drawn from the usual documentation placeholders. scoped to what the bank does not already cover: gitlab, anthropic, npm, sendgrid, slack webhooks, stripe and github fine-grained pats all landed upstream in the meantime, so the rules that duplicated them are dropped rather than shipped a second time under a different name.
1 parent a38ba0a commit 4f82486

2 files changed

Lines changed: 297 additions & 1 deletion

File tree

‎internal/scan/js/secrets.go‎

Lines changed: 126 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
package js
1414

1515
import (
16+
"encoding/base64"
1617
"math"
1718
"regexp"
1819
"strings"
@@ -39,12 +40,14 @@ const (
3940

4041
// secretRules is the credential regex bank. the matching group (or the whole
4142
// match when there's no group) is what gets reported; minEntropy gates the
42-
// generic high-entropy rules so we don't flag every short literal.
43+
// generic high-entropy rules so we don't flag every short literal. validate
44+
// runs after the gates for rules where shape alone isn't proof.
4345
var secretRules = []struct {
4446
name string
4547
re *regexp.Regexp
4648
minEntropy float64
4749
requireDigit bool
50+
validate func(string) bool
4851
}{
4952
{
5053
// aws access key ids are fixed-shape and unmistakable.
@@ -142,6 +145,76 @@ var secretRules = []struct {
142145
re: regexp.MustCompile(`\b(hooks\.slack\.com/services/T[0-9A-Za-z_]+/B[0-9A-Za-z_]+/[0-9A-Za-z]{24})\b`),
143146
minEntropy: noEntropyGate,
144147
},
148+
{
149+
// pypi tokens all share the pypi-AgEIcHlwaS5vcmc prefix, the base64
150+
// encoding of a fixed macaroon header, so it's effectively unforgeable.
151+
name: "pypi api token",
152+
re: regexp.MustCompile(`\b(pypi-AgEIcHlwaS5vcmc[0-9A-Za-z_-]{50,})\b`),
153+
minEntropy: noEntropyGate,
154+
},
155+
{
156+
// legacy openai secret keys embed a fixed T3BlbkFJ marker (base64 for
157+
// "OpenAI") between two random halves.
158+
name: "openai api key",
159+
re: regexp.MustCompile(`\b(sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})\b`),
160+
minEntropy: noEntropyGate,
161+
},
162+
{
163+
// current-generation project and service-account keys.
164+
name: "openai project api key",
165+
re: regexp.MustCompile(`\b(sk-(?:proj|svcacct)-[A-Za-z0-9_-]{20,})\b`),
166+
minEntropy: noEntropyGate,
167+
},
168+
{
169+
name: "square access token",
170+
re: regexp.MustCompile(`\b(sq0atp-[0-9A-Za-z_-]{22}|sq0csp-[0-9A-Za-z_-]{43})\b`),
171+
minEntropy: noEntropyGate,
172+
},
173+
{
174+
// mailgun api keys, key- then a 32-char hex blob.
175+
name: "mailgun api key",
176+
re: regexp.MustCompile(`\b(key-[0-9a-f]{32})\b`),
177+
minEntropy: noEntropyGate,
178+
},
179+
{
180+
// discord bot tokens: base64 user id, a timestamp segment, then an HMAC.
181+
name: "discord bot token",
182+
re: regexp.MustCompile(`\b([MNOP][A-Za-z0-9_-]{23}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,38})\b`),
183+
minEntropy: noEntropyGate,
184+
},
185+
{
186+
// discord incoming-webhook urls embed the secret in the path.
187+
name: "discord webhook url",
188+
re: regexp.MustCompile(`\b(discord(?:app)?\.com/api/webhooks/[0-9]{17,20}/[A-Za-z0-9_-]{60,68})`),
189+
minEntropy: noEntropyGate,
190+
},
191+
{
192+
name: "new relic license key",
193+
re: regexp.MustCompile(`\b(NRAK-[A-Z0-9]{27})\b`),
194+
minEntropy: noEntropyGate,
195+
},
196+
{
197+
// cloudinary connection urls carry the api key and secret in the userinfo.
198+
name: "cloudinary url",
199+
re: regexp.MustCompile(`\b(cloudinary://[0-9]{10,20}:[A-Za-z0-9_-]{20,}@[A-Za-z0-9_-]+)`),
200+
minEntropy: noEntropyGate,
201+
},
202+
{
203+
// jwts have no fixed prefix; validate decodes the header to rule out
204+
// arbitrary dotted base64url blobs.
205+
name: "jwt",
206+
re: regexp.MustCompile(`\b(eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})\b`),
207+
minEntropy: noEntropyGate,
208+
validate: isStructuredJWT,
209+
},
210+
{
211+
// validate drops the countless doc/template examples that use a
212+
// placeholder password.
213+
name: "database connection string",
214+
re: regexp.MustCompile(`\b((?:mongodb(?:\+srv)?|postgres(?:ql)?|mysql|redis|rediss|amqp|amqps)://[^:\s"'` + "`" + `/@]+:[^@\s"'` + "`" + `/]+@[^\s"'` + "`" + `]+)`),
215+
minEntropy: noEntropyGate,
216+
validate: hasRealConnStringPassword,
217+
},
145218
{
146219
// generic apikey/secret/token = "<value>" assignments; the value is in
147220
// group 2 and only reported if it looks random (entropy gate) and carries
@@ -185,6 +258,11 @@ func ScanSecrets(content, srcURL string) []SecretMatch {
185258
continue
186259
}
187260

261+
// structural validation for rules whose shape alone isn't proof.
262+
if rule.validate != nil && !rule.validate(value) {
263+
continue
264+
}
265+
188266
// dedupe per source so a key referenced twice is one finding.
189267
key := rule.name + "\x00" + value
190268
if _, ok := seen[key]; ok {
@@ -217,6 +295,53 @@ func hasDigit(s string) bool {
217295
return false
218296
}
219297

298+
// alg is mandatory per RFC 7519; requiring both fields keeps arbitrary
299+
// dot-separated base64url blobs from being mistaken for a jwt.
300+
const (
301+
jwtAlgField = `"alg"`
302+
jwtTypField = `"typ"`
303+
)
304+
305+
// connStringPasswordRe pulls the userinfo password out of a scheme://user:pass@host
306+
// connection string, for filtering placeholder credentials post-match.
307+
var connStringPasswordRe = regexp.MustCompile(`://[^:@/\s]*:([^@/\s]+)@`)
308+
309+
// placeholderPasswords are stand-ins that show up constantly in docs, sample
310+
// configs and .env.example files; matching one means the string isn't a real
311+
// leaked credential.
312+
var placeholderPasswords = map[string]struct{}{
313+
"password": {}, "pass": {}, "passwd": {}, "xxxx": {}, "xxxxx": {},
314+
"changeme": {}, "yourpassword": {}, "example": {}, "test": {},
315+
"123456": {}, "secret": {}, "admin": {}, "root": {}, "pwd": {},
316+
}
317+
318+
// isStructuredJWT confirms the header segment decodes to jwt-shaped json.
319+
func isStructuredJWT(token string) bool {
320+
parts := strings.Split(token, ".")
321+
if len(parts) != 3 {
322+
return false
323+
}
324+
325+
header, err := base64.RawURLEncoding.DecodeString(parts[0])
326+
if err != nil {
327+
return false
328+
}
329+
330+
h := string(header)
331+
return strings.Contains(h, jwtAlgField) && strings.Contains(h, jwtTypField)
332+
}
333+
334+
// hasRealConnStringPassword rejects known placeholder passwords.
335+
func hasRealConnStringPassword(value string) bool {
336+
m := connStringPasswordRe.FindStringSubmatch(value)
337+
if len(m) < 2 {
338+
return true
339+
}
340+
341+
_, placeholder := placeholderPasswords[strings.ToLower(m[1])]
342+
return !placeholder
343+
}
344+
220345
// shannonEntropy is the per-character shannon entropy (bits) of s, used to tell
221346
// random-looking secrets apart from plain words. empty input is zero entropy.
222347
func shannonEntropy(s string) float64 {

‎internal/scan/js/secrets_provider_test.go‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
/*
2+
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
3+
: :
4+
: █▀ █ █▀▀ · Blazing-fast pentesting suite :
5+
: ▄█ █ █▀ · BSD 3-Clause License :
6+
: :
7+
: (c) 2022-2026 vmfunc, xyzeva, :
8+
: lunchcat alumni & contributors :
9+
: :
10+
·━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━·
11+
*/
12+
13+
package js
14+
15+
import (
16+
"fmt"
17+
"strings"
18+
"testing"
19+
)
20+
21+
// split into fragments so the file itself never carries a contiguous token a
22+
// secret scanner would flag.
23+
var (
24+
provPyPI = "pypi-AgEIcHlwaS5vcmc" + strings.Repeat("1jKlMn0p", 7)
25+
provOpenAILeg = "sk-" + "aB3dEfGh1jKlMn0pQrSt" + "T3BlbkFJ" + "uVwXyZ012345abcdefgh"
26+
provOpenAIProj = "sk-proj-" + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcd"
27+
provSquare = "sq0atp-" + "aB3dEfGh1jKlMn0pQrSt-U"
28+
provMailgun = "key-" + "0123456789abcdef0123456789abcdef"
29+
provDiscordBot = "M" + "TIzNDU2Nzg5MDEyMzQ1Njc4" + "." + "GaBcDe" + "." + "aB3dEfGh1jKlMn0pQrStUvWxYz012345abcdef"
30+
provDiscordHook = "discord.com/api/webhooks/" + "123456789012345678" + "/" + strings.Repeat("aB3dEfGh1j", 6) + "abcdefgh"
31+
provNewRelic = "NRAK-" + "AB3DEFGH1JKLMN0PQRSTUVWXYZZ"
32+
provCloudinary = "cloudinary://" + "123456789012345" + ":" + "aB3dEfGh1jKlMn0pQrStUvWxYz" + "@my-cloud"
33+
provMongoURI = "mongodb+srv://" + "dbadmin" + ":" + "tR7q!zK2vLp9xC" + "@cluster0.example.mongodb.net/prod"
34+
provMongoPlace = "mongodb://" + "user" + ":" + "password" + "@localhost:27017/app"
35+
36+
// a real jwt (rfc 7519 example header/payload), signature is dummy.
37+
provJWT = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" +
38+
".eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIn0" +
39+
".SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"
40+
)
41+
42+
// each provider rule added on top of the existing bank, plus the two cases the
43+
// shape alone cannot decide: a dotted base64url blob that is not a jwt, and a
44+
// connection string whose password is a documentation placeholder.
45+
func TestScanSecretsProviderRules(t *testing.T) {
46+
tests := []struct {
47+
name string
48+
content string
49+
wantRule string // "" means the content must produce no match
50+
}{
51+
{
52+
name: "pypi api token",
53+
content: fmt.Sprintf(`password = %q`, provPyPI),
54+
wantRule: "pypi api token",
55+
},
56+
{
57+
name: "openai legacy api key",
58+
content: fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAILeg),
59+
wantRule: "openai api key",
60+
},
61+
{
62+
name: "openai project api key",
63+
content: fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAIProj),
64+
wantRule: "openai project api key",
65+
},
66+
{
67+
name: "square access token",
68+
content: fmt.Sprintf(`squareToken = %q`, provSquare),
69+
wantRule: "square access token",
70+
},
71+
{
72+
name: "mailgun api key",
73+
content: fmt.Sprintf(`MAILGUN_KEY=%q`, provMailgun),
74+
wantRule: "mailgun api key",
75+
},
76+
{
77+
name: "discord bot token",
78+
content: fmt.Sprintf(`client.login(%q)`, provDiscordBot),
79+
wantRule: "discord bot token",
80+
},
81+
{
82+
name: "discord webhook url",
83+
content: fmt.Sprintf(`fetch("https://%s")`, provDiscordHook),
84+
wantRule: "discord webhook url",
85+
},
86+
{
87+
name: "new relic license key",
88+
content: fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, provNewRelic),
89+
wantRule: "new relic license key",
90+
},
91+
{
92+
name: "cloudinary url",
93+
content: fmt.Sprintf(`CLOUDINARY_URL=%q`, provCloudinary),
94+
wantRule: "cloudinary url",
95+
},
96+
{
97+
name: "jwt with a valid header",
98+
content: fmt.Sprintf(`const token = %q;`, provJWT),
99+
wantRule: "jwt",
100+
},
101+
{
102+
name: "three dotted base64url blobs without a jwt header",
103+
content: `const s = "eyJhYmNkZWZnaGlq.aGVsbG93b3JsZDEy.c2lnbmF0dXJlYmxvYmhlcmU";`,
104+
},
105+
{
106+
name: "connection string with real credentials",
107+
content: fmt.Sprintf(`const uri = %q;`, provMongoURI),
108+
wantRule: "database connection string",
109+
},
110+
{
111+
name: "connection string with a placeholder password",
112+
content: fmt.Sprintf(`// example: %s`, provMongoPlace),
113+
},
114+
}
115+
116+
for _, tt := range tests {
117+
t.Run(tt.name, func(t *testing.T) {
118+
got := ScanSecrets(tt.content, "https://example.test/app.js")
119+
120+
if tt.wantRule == "" {
121+
if len(got) != 0 {
122+
t.Fatalf("got %d matches (%s), want none", len(got), got[0].Rule)
123+
}
124+
return
125+
}
126+
127+
var rules []string
128+
for i := range got {
129+
rules = append(rules, got[i].Rule)
130+
if got[i].Rule == tt.wantRule {
131+
return
132+
}
133+
}
134+
t.Fatalf("rule %q did not fire, got %v", tt.wantRule, rules)
135+
})
136+
}
137+
}
138+
139+
// the rules added here must not overlap each other or the provider-prefixed
140+
// rules already in the bank: one credential in one script is one finding, not
141+
// two. the generic assignment rule is excluded because it claims any quoted
142+
// high-entropy value behind a token/password/secret keyword, so it already
143+
// doubles up with every prefixed rule on main; that is pre-existing and not
144+
// something these rules introduce.
145+
func TestProviderRulesDoNotDuplicateExistingCoverage(t *testing.T) {
146+
content := strings.Join([]string{
147+
fmt.Sprintf(`password = %q`, provPyPI),
148+
fmt.Sprintf(`OPENAI_API_KEY=%q`, provOpenAILeg),
149+
fmt.Sprintf(`squareToken = %q`, provSquare),
150+
fmt.Sprintf(`MAILGUN_KEY=%q`, provMailgun),
151+
fmt.Sprintf(`client.login(%q)`, provDiscordBot),
152+
fmt.Sprintf(`NEW_RELIC_LICENSE_KEY=%q`, provNewRelic),
153+
fmt.Sprintf(`CLOUDINARY_URL=%q`, provCloudinary),
154+
fmt.Sprintf(`const token = %q;`, provJWT),
155+
fmt.Sprintf(`const uri = %q;`, provMongoURI),
156+
}, "\n")
157+
158+
seen := make(map[string]int)
159+
for _, m := range ScanSecrets(content, "https://example.test/app.js") {
160+
if m.Rule == "generic secret assignment" {
161+
continue
162+
}
163+
seen[m.Match]++
164+
}
165+
166+
for value, n := range seen {
167+
if n > 1 {
168+
t.Errorf("value %q reported %d times, want 1", value, n)
169+
}
170+
}
171+
}

0 commit comments

Comments
 (0)