Skip to content

Commit

Permalink
feat(azure): skip low-entropy secrets
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgmz
rgmz committed Jan 13, 2025
1 parent 44fc4c8 commit a7ac795
Show file tree
Hide file tree
Showing 8 changed files with 289 additions and 74 deletions.
47 changes: 34 additions & 13 deletions pkg/detectors/azure_entra/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"io"
"net/http"
stdRegexp "regexp" // Faster for small inputs.
"strings"

regexp "github.com/wasilibs/go-re2"
Expand All @@ -24,31 +25,27 @@ var (
// https://learn.microsoft.com/en-us/microsoft-365/admin/setup/domains-faq?view=o365-worldwide#why-do-i-have-an--onmicrosoft-com--domain
tenantIdPat = regexp.MustCompile(fmt.Sprintf(
//language=regexp
`(?i)(?:(?:login\.microsoftonline\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s))`,
`(?i)(?:(?:login\.microsoft(?:online)?\.com/|(?:login|sts)\.windows\.net/|(?:t[ae]n[ae]nt(?:[ ._-]?id)?|\btid)(?:.|\s){0,60}?)(%s)|https?://(%s)|X-AnchorMailbox(?:.|\s){0,60}?@(%s))`,
uuidStr,
uuidStr,
uuidStr,
))
tenantOnMicrosoftPat = regexp.MustCompile(`([\w-]+\.onmicrosoft\.com)`)

clientIdPat = regexp.MustCompile(fmt.Sprintf(
`(?i)(?:(?:app(?:lication)?|client)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s)`, uuidStr))
`(?i)(?:(?:api|https?)://(%s)/|myapps\.microsoft\.com/signin/(?:[\w-]+/)?(%s)|(?:[\w:=]{0,10}?(?:app(?:lication)?|cl[ie][ei]nt)(?:[ ._-]?id)?|username| -u)(?:.|\s){0,45}?(%s))`, uuidStr, uuidStr, uuidStr))
)

// FindTenantIdMatches returns a list of potential tenant IDs in the provided |data|.
func FindTenantIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})

for _, match := range tenantIdPat.FindAllStringSubmatch(data, -1) {
var m string
if match[1] != "" {
m = strings.ToLower(match[1])
} else if match[2] != "" {
m = strings.ToLower(match[2])
} else if match[3] != "" {
m = strings.ToLower(match[3])
}
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
}
uniqueMatches[m] = struct{}{}
Expand All @@ -59,12 +56,22 @@ func FindTenantIdMatches(data string) map[string]struct{} {
return uniqueMatches
}

// language=regexp
const invalidClientPat = `(?i)(?:client[._-]?request[._-]?(?:id)?(?:.|\s){1,10}%s|cid-v1:%s)`

// FindClientIdMatches returns a list of potential client UUIDs in the provided |data|.
func FindClientIdMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range clientIdPat.FindAllStringSubmatch(data, -1) {
m := strings.ToLower(match[1])
if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
m := strings.ToLower(firstNonEmptyMatch(match))

fpPat := stdRegexp.MustCompile(fmt.Sprintf(invalidClientPat, m, m))
if detectors.StringShannonEntropy(m) < 3 {
continue
} else if _, ok := detectors.UuidFalsePositives[detectors.FalsePositive(m)]; ok {
continue
} else if fpPat.MatchString(match[0]) {
// Ignore request context UUID. (https://stackoverflow.com/q/59425520)
continue
}
uniqueMatches[m] = struct{}{}
Expand Down Expand Up @@ -125,3 +132,17 @@ func queryTenant(ctx context.Context, client *http.Client, tenant string) bool {
return false
}
}

// firstNonEmptyMatch returns the index and value of the first non-empty match.
func firstNonEmptyMatch(matches []string) string {
if len(matches) <= 1 {
return ""
}
// The first index is the entire matched string.
for _, val := range matches[1:] {
if val != "" {
return val
}
}
return ""
}
72 changes: 72 additions & 0 deletions pkg/detectors/azure_entra/common_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type testCase struct {
}

func runPatTest(t *testing.T, tests map[string]testCase, matchFunc func(data string) map[string]struct{}) {
t.Helper()
for name, test := range tests {
t.Run(name, func(t *testing.T) {
matches := matchFunc(test.Input)
Expand Down Expand Up @@ -98,6 +99,7 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"974fde14-c3a4-481b-9b03-cfce18213a07": {},
},
},
// https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/blob/b444dd5f8800c298016ddd9da2e6c05b0bf4b02c/tests/Microsoft.Identity.Test.Common/TestConstants.cs#L241-L245
"login.microsoftonline.com": {
Input: ` auth: {
authority: 'https://login.microsoftonline.com/7bb339cb-e94c-4a85-884c-48ebd9bb28c3',
Expand All @@ -113,6 +115,12 @@ tenant_id = "57aabdfc-6ce0-4828-94a2-9abe277892ec"`,
"7bb339cb-e94c-4a85-884c-48ebd9bb28c3": {},
},
},
"login.microsoft.com": {
Input: `# SYSTEM_CONFIGURATION_ISSUER_URI=https://login.microsoft.com/2b820e29-94a2-402f-b666-c88ebcc69eb4/v2.0`,
Expected: map[string]struct{}{
"2b820e29-94a2-402f-b666-c88ebcc69eb4": {},
},
},
"sts.windows.net": {
Input: `{
"aud": "00000003-0000-0000-c000-000000000000",
Expand Down Expand Up @@ -263,6 +271,14 @@ $ApplicationId = "1e002bca-c6e2-446e-a29e-a221909fe8aa"`,
"902aeb6d-29c7-4f6e-849d-4b933117e320": {},
},
},
"cleint (typo)": {
Input: ` console.log({
cleintId:
"f3a45cb9-e388-4358-a6ef-08a63f97457c",`,
Expected: map[string]struct{}{
"f3a45cb9-e388-4358-a6ef-08a63f97457c": {},
},
},
"clientid": {
Input: `export const msalConfig = {
auth: {
Expand Down Expand Up @@ -306,6 +322,47 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"21e144ac-532d-49ad-ba15-1c40694ce8b1": {},
},
},
"uri - api://": {
Input: `AUDIENCE=api://51aaa91a-bb09-40cd-9f1f-e8c0936246c6/.default`,
Expected: map[string]struct{}{
"51aaa91a-bb09-40cd-9f1f-e8c0936246c6": {},
},
},
"uri - http://": {
Input: `AUDIENCE=http://ceb233fb-f14c-4330-9c5b-91f7db4970e1/.default`,
Expected: map[string]struct{}{
"ceb233fb-f14c-4330-9c5b-91f7db4970e1": {},
},
},
"uri - https://": {
Input: `AUDIENCE=https://47c3cfeb-b7f4-466a-b690-f7fcc79472a9/.default`,
Expected: map[string]struct{}{
"47c3cfeb-b7f4-466a-b690-f7fcc79472a9": {},
},
},
"uri - myapps.microsoft.com": {
Input: `# Linkcheck configuration
linkcheck_ignore = [
"https://myapps.microsoft.com/signin/01501f0f-c48b-4327-92a2-2324949b0a9c?tenantId=e4cbd4d7-327c-47fc-bcde-1005207021a5",
]`,
Expected: map[string]struct{}{
"01501f0f-c48b-4327-92a2-2324949b0a9c": {},
},
},
"uri - myapps.microsoft.com with name": {
Input: `$LoginURL = 'https://myapps.microsoft.com/signin/App1/c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4?tenantId=74d7c41b-e3b6-4d40-88cf-436fd5fc231a'`,
Expected: map[string]struct{}{
"c370c8f6-0cb5-44b2-a903-c6cbd5ff6ce4": {},
},
},
// TODO
// "createdBy": {
// Input: ` "systemData": {
// "createdAt": "2023-08-21T00:30:04.2907836\u002B00:00",
// "createdBy": "117311a5-df69-4fff-a301-6be98c1bd0ab",
// "createdByType": "Application"
// }`,
// },

// Arbitrary test cases
"spacing": {
Expand All @@ -320,6 +377,21 @@ subscription_id = "47ab1364-000d-4a53-838d-1537b1e3b49f"
"f12345c6-7890-1f23-b456-789eb0bb1c23": {},
},
},

// Invalid
"invalid uri": {
Input: `# ldapadd -Y EXTERNAL -H ldapi:/// -f chrootpw.ldif`,
},
"invalid - AppInsights UUID": {
Input: ` "Date": "Mon, 21 Aug 2023 00:29:56 GMT",
"Request-Context": "appId=cid-v1:2d2e8e63-272e-4b3c-8598-4ee570a0e70d",
"Strict-Transport-Security": "max-age=15724800; includeSubDomains; preload",`,
},
"invalid - client-request-id": {
Input: ` "Accept-Encoding": "gzip, deflate",
"client-request-id": "c9e15037-e93c-4da9-b885-9641a826ed9a",
"Connection": "keep-alive",`,
},
}

runPatTest(t, cases, FindClientIdMatches)
Expand Down
2 changes: 2 additions & 0 deletions pkg/detectors/azure_entra/serviceprincipal/sp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ func VerifyCredentials(ctx context.Context, client *http.Client, tenantId string
return false, nil, err
}

fmt.Printf("Status = %d, body=%v\n", res.StatusCode, errResp.Error)

switch res.StatusCode {
case http.StatusBadRequest, http.StatusUnauthorized:
// Error codes can be looked up by removing the `AADSTS` prefix.
Expand Down
45 changes: 27 additions & 18 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,13 @@ var _ interface {
detectors.Versioner
} = (*Scanner)(nil)

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
//clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]`)
)
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func (s Scanner) Version() int {
return 1
Expand All @@ -45,6 +43,19 @@ func (s Scanner) Keywords() []string {
return []string{"azure", "az", "entra", "msal", "login.microsoftonline.com", ".onmicrosoft.com"}
}

var (
defaultClient = common.SaneHttpClient()
// TODO: Azure storage access keys and investigate other types of creds.
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#second-case-access-token-request-with-a-certificate
// https://learn.microsoft.com/en-us/entra/identity-platform/v2-oauth2-client-creds-grant-flow#third-case-access-token-request-with-a-federated-credential
// clientSecretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}?([\w~@[\]:.?*/+=-]{31,34}`)
// TODO: Tighten this regex and replace it with above.
secretPat = regexp.MustCompile(`(?i)(?:secret|password| -p[ =]).{0,80}[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]([A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]{31,34})(?:[^A-Za-z0-9!#$%&()*+,\-./:;<=>?@[\\\]^_{|}~]|\z)`)

invalidMatchPat = regexp.MustCompile(`^passwordCredentials":`)
invalidSecretPat = regexp.MustCompile(`^[a-zA-Z]+$`)
)

// FromData will find and optionally verify Azure secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
Expand All @@ -68,20 +79,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}

func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Azure
}

func (s Scanner) Description() string {
return serviceprincipal.Description
}

func findSecretMatches(data string) map[string]struct{} {
uniqueMatches := make(map[string]struct{})
for _, match := range secretPat.FindAllStringSubmatch(data, -1) {
m := match[1]
// Ignore secrets that are handled by the V2 detector.
if v2.SecretPat.MatchString(m) {
// Ignore secrets that are handled by the V2 detector.
continue
} else if detectors.StringShannonEntropy(m) < 3 {
// Ignore low-entropy results.
continue
} else if invalidSecretPat.MatchString(m) || invalidMatchPat.MatchString(match[0]) {
// Ignore patterns that are known to be false.
continue
}
uniqueMatches[m] = struct{}{}
Expand Down
50 changes: 42 additions & 8 deletions pkg/detectors/azure_entra/serviceprincipal/v1/spv1_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,24 @@ type testCase struct {

func Test_FindClientSecretMatches(t *testing.T) {
cases := map[string]testCase{
// secret
`secret`: {
Input: `"secret": "ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~",`,
Expected: map[string]struct{}{"ljjK-62Q5bJbm43xU5At-NdeWDrhIO_28~": {}},
},

// client secret
"client_secret": {
Input: ` "TenantId": "3d7e0652-b03d-4ed2-bf86-f1299cecde17",
"ClientSecret": "gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9",`,
Expected: map[string]struct{}{"gHduiL_j6t4b6DG?Qr-G6M@IOS?mX3B9": {}},
},
"client secret at end": {
Input: `secret: UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==`,
Expected: map[string]struct{}{
"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==": {},
},
},
"client_secret1": {
Input: ` public static string clientId = "413ff05b-6d54-41a7-9271-9f964bc10624";
public static string clientSecret = "k72~odcN_6TbVh5D~19_1Qkj~87trteArL";
Expand Down Expand Up @@ -72,6 +85,15 @@ configs = {"fs.azure.account.auth.type": "OAuth"`,
Input: ` "AZUREAD-AKS-APPID-SECRET": "8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-",`,
Expected: map[string]struct{}{"8w__IGsaY.6g6jUxb1.pPGK262._pgX.q-": {}},
},
"client_secret9": {
Input: ` client-id: 49abd816-45d1-479a-b49a-80bcf6d7213a
client-secret: 7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO`,
Expected: map[string]struct{}{"7.18gt1b2wO-t.~Cf.mlZCyHC7r_micnuO": {}},
},
"client_secret10": {
Input: ` "aadClientSecret": "6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo",`,
Expected: map[string]struct{}{"6p3t93TJzPgsNtQISqWc.-@?GCz9-ZWo": {}},
},
// "client_secret6": {
// Input: ``,
// Expected: map[string]struct{}{"": {}},
Expand All @@ -86,18 +108,30 @@ $Credential = New-Object -TypeName System.Management.Automation.PSCredential -Ar
},

// False positives
"placeholder_secret": {
"invalid - placeholder_secret": {
Input: `- Log in with a service principal using a client secret:
az login --service-principal --username {{http://azure-cli-service-principal}} --password {{secret}} --tenant {{someone.onmicrosoft.com}}`,
Expected: nil,
},
// "client_secret3": {
// Input: ``,
// Expected: map[string]struct{}{
// "": {},
// },
// },

"invalid - only alpha characters": {
Input: `"passwordCredentials":[],"preferredTokenSigningKeyThumbprint":null,"publisherName":"Microsoft"`,
},
"invalid - low entropy": {
Input: `clientSecret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx`,
},
"invalid - passwordCredentials1": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"UAByAGkAbQBhAHIAeQAgAEsAZQB5AA==","endDate":"2019-07-16T23:01:19.028Z","keyId":`,
},
"invalid - passwordCredentials2": {
Input: `"passwordCredentials":[{"customKeyIdentifier":"TQB5ACAARgBpAHIAcwB0ACAASwBlAHkA"`,
},
"invalid - passwordCredentials3": {
Input: `,"passwordCredentials":[{"customKeyIdentifier":"awBlAHkAZgBvAHIAaQBtAHAAYQBsAGEA",`,
},
"invalid - azure vault path": {
Input: ` public const string MsalArlingtonOBOKeyVaultUri = "https://msidlabs.vault.azure.net:443/secrets/ARLMSIDLAB1-IDLASBS-App-CC-Secret";`,
},
}

for name, test := range cases {
Expand Down
Loading

0 comments on commit a7ac795

Please sign in to comment.