Skip to content

Commit 57363b3

Browse files
babenekbabenek
authored
Sentry Auth Token patterns (#693)
* Sentry Organization Auth Token * Duplicate rules name check * Sentry User Auth Token * assertRaisesRegex * ssws && ntlm authentication schemes in keyword * sample added * test added
1 parent 0361bde commit 57363b3

File tree

11 files changed

+376
-7
lines changed

11 files changed

+376
-7
lines changed

‎credsweeper/common/keyword_pattern.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ class KeywordPattern:
2626
string_prefix = r"(((b|r|br|rb|u|f|rf|fr|l|@)(?=(\\*[`'\"])))?"
2727
left_quote = r"(?P<value_leftquote>((?P<esq>\\{1,8})?([`'\"]|&(quot|apos);)){1,4}))?"
2828
# Authentication scheme ( oauth | basic | bearer | apikey ) precedes to credential
29-
auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey)\s)?"
29+
auth_keywords = r"(\s?(oauth|bot|basic|bearer|apikey|accesskey|ssws|ntlm)\s)?"
3030
value = r"(?P<value>" \
3131
r"(?(value_leftquote)" \
3232
r"(" \

‎credsweeper/rules/config.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1407,6 +1407,37 @@
14071407
- code
14081408
- doc
14091409

1410+
- name: Sentry Organization Auth Token
1411+
severity: high
1412+
confidence: strong
1413+
type: pattern
1414+
values:
1415+
- (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntrys_eyJ[0-9A-Za-z_-]{80,8000}=*([0-9A-Za-z_-]{32,256})?)(?![0-9A-Za-z_-])
1416+
min_line_len: 37
1417+
filter_type:
1418+
- ValuePatternCheck(5)
1419+
- ValueEntropyBase64Check
1420+
required_substrings:
1421+
- sntrys_eyJ
1422+
target:
1423+
- code
1424+
- doc
1425+
1426+
- name: Sentry User Auth Token
1427+
severity: high
1428+
confidence: strong
1429+
type: pattern
1430+
values:
1431+
- (?:(?<![0-9A-Za-z_-])|\\[0abfnrtv]|(%|\\x)[0-9A-Fa-f]{2}|\\[0-7]{3}|\\[Uu]([0-9A-Fa-f]{4}){1,2}|\x1B\[[0-9;]{0,80}m)(?P<value>sntryu_[0-9a-f]{64})(?![0-9A-Za-z_-])
1432+
min_line_len: 37
1433+
filter_type:
1434+
- ValuePatternCheck(5)
1435+
required_substrings:
1436+
- sntryu_
1437+
target:
1438+
- code
1439+
- doc
1440+
14101441
- name: Discord Bot Token
14111442
severity: high
14121443
confidence: strong

‎credsweeper/scanner/scanner.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
6969
rule_path = APP_PATH / "rules" / "config.yaml"
7070
rule_templates = Util.yaml_load(rule_path)
7171
if rule_templates and isinstance(rule_templates, list):
72+
rule_names = set()
7273
for rule_template in rule_templates:
7374
try:
7475
rule = Rule(self.config, rule_template)
@@ -77,6 +78,10 @@ def _set_rules_scanners(self, rule_path: Union[None, str, Path]) -> None:
7778
raise exc
7879
if not self._is_available(rule):
7980
continue
81+
if rule.rule_name in rule_names:
82+
raise RuntimeError(f"Duplicated rule name {rule.rule_name}")
83+
else:
84+
rule_names.add(rule.rule_name)
8085
if 0 < rule.min_line_len:
8186
if rule.rule_type == RuleType.KEYWORD:
8287
self.min_keyword_len = min(self.min_keyword_len, rule.min_line_len)
@@ -141,7 +146,7 @@ def scan(self, provider: ContentProvider) -> List[Candidate]:
141146
# "cache" - YAPF and pycharm formatters ...
142147
matched_keyword = \
143148
target_line_stripped_len >= self.min_keyword_len and ( #
144-
'=' in target_line_stripped or ':' in target_line_stripped) #
149+
'=' in target_line_stripped or ':' in target_line_stripped) #
145150
matched_pem_key = \
146151
target_line_stripped_len >= self.min_pem_key_len \
147152
and PEM_BEGIN_PATTERN in target_line_stripped and "PRIVATE" in target_line_stripped

‎tests/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
from pathlib import Path
22

33
# total number of files in test samples
4-
SAMPLES_FILES_COUNT = 148
4+
SAMPLES_FILES_COUNT = 149
55

66
# the lowest value of ML threshold is used to display possible lowest values
77
NEGLIGIBLE_ML_THRESHOLD = 0.0001
88

99
# credentials count after scan with negligible ML threshold
10-
SAMPLES_CRED_COUNT = 472
10+
SAMPLES_CRED_COUNT = 475
1111
SAMPLES_CRED_LINE_COUNT = SAMPLES_CRED_COUNT + 19
1212

1313
# Number of filtered credentials with ML
@@ -17,7 +17,7 @@
1717
SAMPLES_POST_CRED_COUNT = SAMPLES_CRED_COUNT - ML_FILTERED
1818

1919
# with option --doc
20-
SAMPLES_IN_DOC = 654
20+
SAMPLES_IN_DOC = 656
2121

2222
# archived credentials that are not found without --depth
2323
SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 89

‎tests/data/depth_3.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,31 @@
342342
}
343343
]
344344
},
345+
{
346+
"rule": "Auth",
347+
"severity": "medium",
348+
"confidence": "moderate",
349+
"ml_probability": 1.0,
350+
"line_data_list": [
351+
{
352+
"line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
353+
"line_num": 4,
354+
"path": "./tests/samples/auth.hs",
355+
"info": "FILE|RAW",
356+
"value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
357+
"value_start": 20,
358+
"value_end": 59,
359+
"variable": "Authorization",
360+
"variable_start": 0,
361+
"variable_end": 13,
362+
"entropy_validation": {
363+
"iterator": "BASE64STDPAD_CHARS",
364+
"entropy": 4.002348372264613,
365+
"valid": false
366+
}
367+
}
368+
]
369+
},
345370
{
346371
"rule": "Auth",
347372
"severity": "medium",
@@ -10056,6 +10081,56 @@
1005610081
}
1005710082
]
1005810083
},
10084+
{
10085+
"rule": "Sentry Organization Auth Token",
10086+
"severity": "high",
10087+
"confidence": "strong",
10088+
"ml_probability": null,
10089+
"line_data_list": [
10090+
{
10091+
"line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
10092+
"line_num": 1,
10093+
"path": "./tests/samples/sentry",
10094+
"info": "FILE|RAW",
10095+
"value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
10096+
"value_start": 0,
10097+
"value_end": 179,
10098+
"variable": null,
10099+
"variable_start": -2,
10100+
"variable_end": -2,
10101+
"entropy_validation": {
10102+
"iterator": "BASE64STDPAD_CHARS",
10103+
"entropy": 5.424476232986917,
10104+
"valid": true
10105+
}
10106+
}
10107+
]
10108+
},
10109+
{
10110+
"rule": "Sentry User Auth Token",
10111+
"severity": "high",
10112+
"confidence": "strong",
10113+
"ml_probability": null,
10114+
"line_data_list": [
10115+
{
10116+
"line": "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",
10117+
"line_num": 2,
10118+
"path": "./tests/samples/sentry",
10119+
"info": "FILE|RAW",
10120+
"value": "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",
10121+
"value_start": 0,
10122+
"value_end": 71,
10123+
"variable": null,
10124+
"variable_start": -2,
10125+
"variable_end": -2,
10126+
"entropy_validation": {
10127+
"iterator": "BASE36_CHARS",
10128+
"entropy": 4.0572205343720595,
10129+
"valid": true
10130+
}
10131+
}
10132+
]
10133+
},
1005910134
{
1006010135
"rule": "Shopify Token",
1006110136
"severity": "high",

‎tests/data/doc.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16413,6 +16413,56 @@
1641316413
}
1641416414
]
1641516415
},
16416+
{
16417+
"rule": "Sentry Organization Auth Token",
16418+
"severity": "high",
16419+
"confidence": "strong",
16420+
"ml_probability": null,
16421+
"line_data_list": [
16422+
{
16423+
"line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
16424+
"line_num": 1,
16425+
"path": "./tests/samples/sentry",
16426+
"info": "FILE|RAW",
16427+
"value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
16428+
"value_start": 0,
16429+
"value_end": 179,
16430+
"variable": null,
16431+
"variable_start": -2,
16432+
"variable_end": -2,
16433+
"entropy_validation": {
16434+
"iterator": "BASE64STDPAD_CHARS",
16435+
"entropy": 5.424476232986917,
16436+
"valid": true
16437+
}
16438+
}
16439+
]
16440+
},
16441+
{
16442+
"rule": "Sentry User Auth Token",
16443+
"severity": "high",
16444+
"confidence": "strong",
16445+
"ml_probability": null,
16446+
"line_data_list": [
16447+
{
16448+
"line": "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",
16449+
"line_num": 2,
16450+
"path": "./tests/samples/sentry",
16451+
"info": "FILE|RAW",
16452+
"value": "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",
16453+
"value_start": 0,
16454+
"value_end": 71,
16455+
"variable": null,
16456+
"variable_start": -2,
16457+
"variable_end": -2,
16458+
"entropy_validation": {
16459+
"iterator": "BASE36_CHARS",
16460+
"entropy": 4.0572205343720595,
16461+
"valid": true
16462+
}
16463+
}
16464+
]
16465+
},
1641616466
{
1641716467
"rule": "Shopify Token",
1641816468
"severity": "high",

‎tests/data/ml_threshold.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,31 @@
249249
}
250250
]
251251
},
252+
{
253+
"rule": "Auth",
254+
"severity": "medium",
255+
"confidence": "moderate",
256+
"ml_probability": 1.0,
257+
"line_data_list": [
258+
{
259+
"line": "Authorization: NTLM TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
260+
"line_num": 4,
261+
"path": "./tests/samples/auth.hs",
262+
"info": "",
263+
"value": "TlRMTUAAABABoITVNIAAZI1AQBuOh4cSAQ8B1A=",
264+
"value_start": 20,
265+
"value_end": 59,
266+
"variable": "Authorization",
267+
"variable_start": 0,
268+
"variable_end": 13,
269+
"entropy_validation": {
270+
"iterator": "BASE64STDPAD_CHARS",
271+
"entropy": 4.002348372264613,
272+
"valid": false
273+
}
274+
}
275+
]
276+
},
252277
{
253278
"rule": "Auth",
254279
"severity": "medium",
@@ -10197,6 +10222,56 @@
1019710222
}
1019810223
]
1019910224
},
10225+
{
10226+
"rule": "Sentry Organization Auth Token",
10227+
"severity": "high",
10228+
"confidence": "strong",
10229+
"ml_probability": null,
10230+
"line_data_list": [
10231+
{
10232+
"line": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
10233+
"line_num": 1,
10234+
"path": "./tests/samples/sentry",
10235+
"info": "",
10236+
"value": "sntrys_eyJpYXQiOjE3NDEyNjQzNTYuMDAwMCwidXJsIjoiaHR0cHM6Ly9zZW50cnkuaW8iLCJyZWdpb25fdXJsIjoiaHR0cHM6Ly91YS5zZW50cnkuaW8iLCJvcmciOiIifQ==v8D-whr2cUQK91Civi4yNoLRjC3MDZH5I2aMcs_j5GDv",
10237+
"value_start": 0,
10238+
"value_end": 179,
10239+
"variable": null,
10240+
"variable_start": -2,
10241+
"variable_end": -2,
10242+
"entropy_validation": {
10243+
"iterator": "BASE64STDPAD_CHARS",
10244+
"entropy": 5.424476232986917,
10245+
"valid": true
10246+
}
10247+
}
10248+
]
10249+
},
10250+
{
10251+
"rule": "Sentry User Auth Token",
10252+
"severity": "high",
10253+
"confidence": "strong",
10254+
"ml_probability": null,
10255+
"line_data_list": [
10256+
{
10257+
"line": "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",
10258+
"line_num": 2,
10259+
"path": "./tests/samples/sentry",
10260+
"info": "",
10261+
"value": "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",
10262+
"value_start": 0,
10263+
"value_end": 71,
10264+
"variable": null,
10265+
"variable_start": -2,
10266+
"variable_end": -2,
10267+
"entropy_validation": {
10268+
"iterator": "BASE36_CHARS",
10269+
"entropy": 4.0572205343720595,
10270+
"valid": true
10271+
}
10272+
}
10273+
]
10274+
},
1020010275
{
1020110276
"rule": "Shopify Token",
1020210277
"severity": "high",

0 commit comments

Comments
 (0)