Skip to content

Commit 94ea476

Browse files
rgmzrgmz
committed
feat(azure): skip low-entropy secrets
1 parent 1fc8961 commit 94ea476

File tree

8 files changed

+289
-74
lines changed

8 files changed

+289
-74
lines changed

‎pkg/detectors/azure_entra/common.go

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"io"
66
"net/http"
7+
stdRegexp "regexp" // Faster for small inputs.
78
"strings"
89

910
regexp "github.com/wasilibs/go-re2"
@@ -24,31 +25,27 @@ var (
2425
// https://learn.microsoft.com/en-us/microsoft-365/admin/setup/domains-faq?view=o365-worldwide#why-do-i-have-an--onmicrosoft-com--domain
2526
tenantIdPat = regexp.MustCompile(fmt.Sprintf(
2627
//language=regexp
27-
`(?i)(?:(?:login\.microsoftonline\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s))`,
28+
`(?i)(?:(?:login\.microsoft(?:online)?\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s))`,
2829
uuidStr,
2930
uuidStr,
3031
uuidStr,
3132
))
3233
tenantOnMicrosoftPat = regexp.MustCompile(`([\w-]+\.onmicrosoft\.com)`)
3334

3435
clientIdPat = regexp.MustCompile(fmt.Sprintf(
35-
`(?i)(?:(?:app(?:lication)?|client)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s)`, uuidStr))
36+
`(?i)(?:(?:api|https?)://(%s)/|myapps\.microsoft\.com/signin/(?:[\w-]+/)?(%s)|(?:[\w:=]{0,10}?(?:app(?:lication)?|cl[ie][ei]nt)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s))`, uuidStr, uuidStr, uuidStr))
3637
)
3738

3839
// FindTenantIdMatches returns a list of potential tenant IDs in the provided |data|.
3940
func FindTenantIdMatches(data string) map[string]struct{} {
4041
uniqueMatches := make(map[string]struct{})
4142

4243
for _, match := range tenantIdPat.FindAllStringSubmatch(data, -1) {
43-
var m string
44-
if match[1] != "" {
45-
m = strings.ToLower(match[1])
46-
} else if match[2] != "" {
47-
m = strings.ToLower(match[2])
48-
} else if match[3] != "" {
49-
m = strings.ToLower(match[3])
50-
}
51-
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
44+
m := strings.ToLower(firstNonEmptyMatch(match))
45+
46+
if detectors.StringShannonEntropy(m) < 3 {
47+
continue
48+
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
5249
continue
5350
}
5451
uniqueMatches[m] = struct{}{}
@@ -59,12 +56,22 @@ func FindTenantIdMatches(data string) map[string]struct{} {
5956
return uniqueMatches
6057
}
6158

59+
// language=regexp
60+
const invalidClientPat = `(?i)(?:client[._-]?request[._-]?(?:id)?(?:.|\s){1,10}%s|cid-v1:%s)`
61+
6262
// FindClientIdMatches returns a list of potential client UUIDs in the provided |data|.
6363
func FindClientIdMatches(data string) map[string]struct{} {
6464
uniqueMatches := make(map[string]struct{})
6565
for _, match := range clientIdPat.FindAllStringSubmatch(data, -1) {
66-
m := strings.ToLower(match[1])
67-
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
66+
m := strings.ToLower(firstNonEmptyMatch(match))
67+
68+
fpPat := stdRegexp.MustCompile(fmt.Sprintf(invalidClientPat, m, m))
69+
if detectors.StringShannonEntropy(m) < 3 {
70+
continue
71+
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
72+
continue
73+
} else if fpPat.MatchString(match[0]) {
74+
// Ignore request context UUID. (https://stackoverflow.com/q/59425520)
6875
continue
6976
}
7077
uniqueMatches[m] = struct{}{}
@@ -125,3 +132,17 @@ func queryTenant(ctx context.Context, client *http.Client, tenant string) bool {
125132
return false
126133
}
127134
}
135+
136+
// firstNonEmptyMatch returns the index and value of the first non-empty match.
137+
func firstNonEmptyMatch(matches []string) string {
138+
if len(matches) <= 1 {
139+
return ""
140+
}
141+
// The first index is the entire matched string.
142+
for _, val := range matches[1:] {
143+
if val != "" {
144+
return val
145+
}
146+
}
147+
return ""
148+
}

‎pkg/detectors/azure_entra/common_test.go

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ type testCase struct {
1212
}
1313

1414
func runPatTest(t *testing.T, tests map[string]testCase, matchFunc func(data string) map[string]struct{}) {
15+
t.Helper()
1516
for name, test := range tests {
1617
t.Run(name, func(t *testing.T) {
1718
matches := matchFunc(test.Input)
@@ -98,6 +99,7 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
9899
"974fde14-c3a4-481b-9b03-cfce18213a07": {},
99100
},
100101
},
102+
// https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/b444dd5f8800c298016ddd9da2e6c05b0bf4b02c/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L241-L245
101103
"login.microsoftonline.com": {
102104
Input: ` auth: {
103105
authority: 'https://login.microsoftonline.com/7bb339cb-e94c-4a85-884c-48ebd9bb28c3',
@@ -113,6 +115,12 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
113115
"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
114116
},
115117
},
118+
"login.microsoft.com": {
119+
Input: `# SYSTEM_CONFIGURATION_ISSUER_URI=https://login.microsoft.com/2b820e29-94a2-402f-b666-c88ebcc69eb4/v2.0`,
120+
Expected: map[string]struct{}{
121+
"2b820e29-94a2-402f-b666-c88ebcc69eb4": {},
122+
},
123+
},
116124
"sts.windows.net": {
117125
Input: `{
118126
"aud": "00000003-0000-0000-c000-000000000000",
@@ -263,6 +271,14 @@ $ApplicationId = "1e002bca-c6e2-446e-a29e-a221909fe8aa"`,
263271
"902aeb6d-29c7-4f6e-849d-4b933117e320": {},
264272
},
265273
},
274+
"cleint (typo)": {
275+
Input: ` console.log({
276+
cleintId:
277+
"f3a45cb9-e388-4358-a6ef-08a63f97457c",`,
278+
Expected: map[string]struct{}{
279+
"f3a45cb9-e388-4358-a6ef-08a63f97457c": {},
280+
},
281+
},
266282
"clientid": {
267283
Input: `export const msalConfig = {
268284
auth: {
@@ -306,6 +322,47 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
306322
"21e144ac-532d-49ad-ba15-1c40694ce8b1": {},
307323
},
308324
},
325+
"uri - api://": {
326+
Input: `AUDIENCE=api://51aaa91a-bb09-40cd-9f1f-e8c0936246c6/.default`,
327+
Expected: map[string]struct{}{
328+
"51aaa91a-bb09-40cd-9f1f-e8c0936246c6": {},
329+
},
330+
},
331+
"uri - http://": {
332+
Input: `AUDIENCE=http://ceb233fb-f14c-4330-9c5b-91f7db4970e1/.default`,
333+
Expected: map[string]struct{}{
334+
"ceb233fb-f14c-4330-9c5b-91f7db4970e1": {},
335+
},
336+
},
337+
"uri - https://": {
338+
Input: `AUDIENCE=https://47c3cfeb-b7f4-466a-b690-f7fcc79472a9/.default`,
339+
Expected: map[string]struct{}{
340+
"47c3cfeb-b7f4-466a-b690-f7fcc79472a9": {},
341+
},
342+
},
343+
"uri - myapps.microsoft.com": {
344+
Input: `# Linkcheck configuration
345+
linkcheck_ignore = [
346+
"https://myapps.microsoft.com/signin/01501f0f-c48b-4327-92a2-2324949b0a9c?tenantId=e4cbd4d7-327c-47fc-bcde-1005207021a5",
347+
]`,
348+
Expected: map[string]struct{}{
349+
"01501f0f-c48b-4327-92a2-2324949b0a9c": {},
350+
},
351+
},
352+
"uri - myapps.microsoft.com with name": {
353+
Input: `$LoginURL = 'https://myapps.microsoft.com/signin/App1/c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4?tenantId=74d7c41b-e3b6-4d40-88cf-436fd5fc231a'`,
354+
Expected: map[string]struct{}{
355+
"c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4": {},
356+
},
357+
},
358+
// TODO
359+
// "createdBy": {
360+
// Input: ` "systemData": {
361+
// "createdAt": "2023-08-21T00:30:04.2907836\u002B00:00",
362+
// "createdBy": "117311a5-df69-4fff-a301-6be98c1bd0ab",
363+
// "createdByType": "Application"
364+
// }`,
365+
// },
309366

310367
// Arbitrary test cases
311368
"spacing": {
@@ -320,6 +377,21 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
320377
"f12345c6-7890-1f23-b456-789eb0bb1c23": {},
321378
},
322379
},
380+
381+
// Invalid
382+
"invalid uri": {
383+
Input: `# ldapadd -Y EXTERNAL -H ldapi:/// -f chrootpw.ldif`,
384+
},
385+
"invalid - AppInsights UUID": {
386+
Input: ` "Date": "Mon, 21 Aug 2023 00:29:56 GMT",
387+
"Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d",
388+
"Strict-Transport-Security": "max-age=15724800; includeSubDomains; preload",`,
389+
},
390+
"invalid - client-request-id": {
391+
Input: ` "Accept-Encoding": "gzip, deflate",
392+
"client-request-id": "c9e15037-e93c-4da9-b885-9641a826ed9a",
393+
"Connection": "keep-alive",`,
394+
},
323395
}
324396

325397
runPatTest(t, cases, FindClientIdMatches)

‎pkg/detectors/azure_entra/serviceprincipal/sp.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string
9090
return false, nil, err
9191
}
9292

93+
fmt.Printf("Status = %d, body=%v\n", res.StatusCode, errResp.Error)
94+
9395
switch res.StatusCode {
9496
case http.StatusBadRequest, http.StatusUnauthorized:
9597
// Error codes can be looked up by removing the `AADSTS` prefix.

‎pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go

Lines changed: 27 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,13 @@ var _ interface {
2525
detectors.Versioner
2626
} = (*Scanner)(nil)
2727

28-
var (
29-
defaultClient = common.SaneHttpClient()
30-
// TODO: Azure storage access keys and investigate other types of creds.
31-
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
32-
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
33-
//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
34-
// TODO: Tighten this regex and replace it with above.
35-
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
36-
)
28+
func (s Scanner) Type() detectorspb.DetectorType {
29+
return detectorspb.DetectorType_Azure
30+
}
31+
32+
func (s Scanner) Description() string {
33+
return serviceprincipal.Description
34+
}
3735

3836
func (s Scanner) Version() int {
3937
return 1
@@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string {
4543
return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"}
4644
}
4745

46+
var (
47+
defaultClient = common.SaneHttpClient()
48+
// TODO: Azure storage access keys and investigate other types of creds.
49+
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
50+
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
51+
// clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
52+
// TODO: Tighten this regex and replace it with above.
53+
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]|\z)`)
54+
55+
invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`)
56+
invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`)
57+
)
58+
4859
// FromData will find and optionally verify Azure secrets in a given set of bytes.
4960
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
5061
dataStr := string(data)
@@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
6879
return results, nil
6980
}
7081

71-
func (s Scanner) Type() detectorspb.DetectorType {
72-
return detectorspb.DetectorType_Azure
73-
}
74-
75-
func (s Scanner) Description() string {
76-
return serviceprincipal.Description
77-
}
78-
7982
func findSecretMatches(data string) map[string]struct{} {
8083
uniqueMatches := make(map[string]struct{})
8184
for _, match := range secretPat.FindAllStringSubmatch(data, -1) {
8285
m := match[1]
83-
// Ignore secrets that are handled by the V2 detector.
8486
if v2.SecretPat.MatchString(m) {
87+
// Ignore secrets that are handled by the V2 detector.
88+
continue
89+
} else if detectors.StringShannonEntropy(m) < 3 {
90+
// Ignore low-entropy results.
91+
continue
92+
} else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) {
93+
// Ignore patterns that are known to be false.
8594
continue
8695
}
8796
uniqueMatches[m] = struct{}{}

‎pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,24 @@ type testCase struct {
1313

1414
func Test_FindClientSecretMatches(t *testing.T) {
1515
cases := map[string]testCase{
16+
// secret
17+
`secret`: {
18+
Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`,
19+
Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}},
20+
},
21+
22+
// client secret
1623
"client_secret": {
1724
Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17",
1825
"ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`,
1926
Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}},
2027
},
28+
"client secret at end": {
29+
Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`,
30+
Expected: map[string]struct{}{
31+
"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {},
32+
},
33+
},
2134
"client_secret1": {
2235
Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624";
2336
public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL";
@@ -72,6 +85,15 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
7285
Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
7386
Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
7487
},
88+
"client_secret9": {
89+
Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a
90+
client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`,
91+
Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}},
92+
},
93+
"client_secret10": {
94+
Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`,
95+
Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}},
96+
},
7597
// "client_secret6": {
7698
// Input: ``,
7799
// Expected: map[string]struct{}{"": {}},
@@ -86,18 +108,30 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
86108
},
87109

88110
// False positives
89-
"placeholder_secret": {
111+
"invalid - placeholder_secret": {
90112
Input: `- Log in with a service principal using a client secret:
91113
92114
az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
93-
Expected: nil,
94115
},
95-
// "client_secret3": {
96-
// Input: ``,
97-
// Expected: map[string]struct{}{
98-
// "": {},
99-
// },
100-
// },
116+
117+
"invalid - only alpha characters": {
118+
Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`,
119+
},
120+
"invalid - low entropy": {
121+
Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
122+
},
123+
"invalid - passwordCredentials1": {
124+
Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`,
125+
},
126+
"invalid - passwordCredentials2": {
127+
Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`,
128+
},
129+
"invalid - passwordCredentials3": {
130+
Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`,
131+
},
132+
"invalid - azure vault path": {
133+
Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`,
134+
},
101135
}
102136

103137
for name, test := range cases {

0 commit comments

Comments
 (0)