Skip to content

Commit bc58c21

Browse files
babenekbabenek
committed
New repo,obfuscation extend,packages update
1 parent 49e136a commit bc58c21

File tree

6 files changed

+111
-24
lines changed

6 files changed

+111
-24
lines changed

‎.ci/benchmark.txt‎

Lines changed: 19 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
META MD5 8cf8469b772217eaa1dd9fc8c0a912de
2-
DATA MD5 5ff49df044c012cb13bab05ee637e708
3-
DATA: 16995334 interested lines. MARKUP: 63736 items
1+
META MD5 346719990276f1c7ab597e7ea85f5b21
2+
DATA MD5 d9a50a89fa4ce3c5bf3cdc5d1958ee2a
3+
DATA: 16999171 interested lines. MARKUP: 63795 items
44
FileType FileNumber ValidLines Positives Negatives
55
--------------- ------------ ------------ ----------- -----------
6-
684 567150 136 695
6+
685 567300 136 695
77
.04 2 149 4
88
.1 2 641 2 10
99
.admx 1 26 1
@@ -62,7 +62,7 @@ FileType FileNumber ValidLines Positives Negatives
6262
.gd 1 37 1
6363
.gml 3 3075 16
6464
.gni 3 5017 17
65-
.go 1242 706630 1484 6137
65+
.go 1251 709843 1494 6177
6666
.golden 5 1168 1 50
6767
.gradle 50 4295 8 142
6868
.graphql 8 454 2 13
@@ -157,7 +157,7 @@ FileType FileNumber ValidLines Positives Negatives
157157
.pug 2 193 2
158158
.purs 1 69 4
159159
.pxd 1 150 2 4
160-
.py 876 292413 755 3881
160+
.py 878 292875 756 3897
161161
.pyi 4 1361 10
162162
.pyp 1 167 1
163163
.python 1 213
@@ -221,21 +221,21 @@ FileType FileNumber ValidLines Positives Negatives
221221
.xcscheme 1 109 1
222222
.xib 11 503 164
223223
.xsl 1 311 1
224-
.yaml 169 31946 207 393
224+
.yaml 171 31958 207 395
225225
.yml 560 56585 1897 1386
226226
.zsh 6 872 11
227227
.zsh-theme 1 97 1
228-
TOTAL: 11361 16995334 17130 53678
228+
TOTAL: 11375 16999171 17141 53736
229229
credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0
230230
Rules Positives Negatives Reported TP FP TN FN FPR FNR ACC PRC RCL F1
231231
------------------------------ ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ----
232-
API 243 4001 0 0 4001 243 0.000000 1.000000 0.942743 0.000000
232+
API 243 4009 0 0 4009 243 0.000000 1.000000 0.942850 0.000000
233233
AWS Client ID 213 33 0 0 33 213 0.000000 1.000000 0.134146 0.000000
234234
AWS Multi 34 66 0 0 66 34 0.000000 1.000000 0.660000 0.000000
235235
AWS S3 Bucket 92 0 0 0 0 92 1.000000 0.000000 0.000000
236236
Akamai Credentials 6 2 0 0 2 6 0.000000 1.000000 0.250000 0.000000
237237
Anthropic API Key 1 0 0 0 0 1 1.000000 0.000000 0.000000
238-
Auth 1165 3610 0 0 3610 1165 0.000000 1.000000 0.756021 0.000000
238+
Auth 1166 3616 0 0 3616 1166 0.000000 1.000000 0.756169 0.000000
239239
Azure Access Token 24 0 0 0 0 24 1.000000 0.000000 0.000000
240240
BASE64 Private Key 22 4 0 0 4 22 0.000000 1.000000 0.153846 0.000000
241241
BASE64 encoded PEM Private Key 12 0 0 0 0 12 1.000000 0.000000 0.000000
@@ -246,41 +246,42 @@ CMD Password 33 137 0 0
246246
CMD Secret 1 18 0 0 18 1 0.000000 1.000000 0.947368 0.000000
247247
CMD Token 6 2 0 0 2 6 0.000000 1.000000 0.250000 0.000000
248248
CURL User Password 7 2 0 0 2 7 0.000000 1.000000 0.222222 0.000000
249-
Credential 96 598 0 0 598 96 0.000000 1.000000 0.861671 0.000000
249+
Credential 96 602 0 0 602 96 0.000000 1.000000 0.862464 0.000000
250250
Docker Swarm Token 2 0 0 0 0 2 1.000000 0.000000 0.000000
251251
Dropbox App secret 74 145 0 0 145 74 0.000000 1.000000 0.662100 0.000000
252252
Facebook Access Token 0 1 0 0 1 0 0.000000 1.000000
253253
Firebase Domain 39 0 0 0 0 39 1.000000 0.000000 0.000000
254+
Github Classic Token 1 0 0 0 0 1 1.000000 0.000000 0.000000
254255
Google API Key 13 0 0 0 0 13 1.000000 0.000000 0.000000
255256
Google Multi 11 0 0 0 0 11 1.000000 0.000000 0.000000
256257
Google OAuth Access Token 3 0 0 0 0 3 1.000000 0.000000 0.000000
257258
Google OAuth Refresh Token 1 2 0 0 2 1 0.000000 1.000000 0.666667 0.000000
258259
Grafana Access Policy Token 0 2 0 0 2 0 0.000000 1.000000
259260
Grafana Provisioned API Key 7 16 0 0 16 7 0.000000 1.000000 0.695652 0.000000
260261
Grafana Service Account Token 3 0 0 0 0 3 1.000000 0.000000 0.000000
261-
JSON Web Token 174 61 0 0 61 174 0.000000 1.000000 0.259574 0.000000
262+
JSON Web Token 175 61 0 0 61 175 0.000000 1.000000 0.258475 0.000000
262263
JWK 80 3 0 0 3 80 0.000000 1.000000 0.036145 0.000000
263264
Jira / Confluence PAT token 0 4 0 0 4 0 0.000000 1.000000
264-
Key 4225 20798 0 0 20798 4225 0.000000 1.000000 0.831155 0.000000
265+
Key 4225 20804 0 0 20804 4225 0.000000 1.000000 0.831196 0.000000
265266
MailGun API Key 8 0 0 0 0 8 1.000000 0.000000 0.000000
266267
NKEY Seed 60 0 0 0 0 60 1.000000 0.000000 0.000000
267268
NTLM Token 4 0 0 0 0 4 1.000000 0.000000 0.000000
268269
Nonce 131 109 0 0 109 131 0.000000 1.000000 0.454167 0.000000
269270
OTP / 2FA Secret 64 3 0 0 3 64 0.000000 1.000000 0.044776 0.000000
270271
Other 0 20 0 0 20 0 0.000000 1.000000
271-
PEM Private Key 1154 72 0 0 72 1154 0.000000 1.000000 0.058728 0.000000
272-
Password 2595 11364 0 0 11364 2595 0.000000 1.000000 0.814098 0.000000
272+
PEM Private Key 1157 72 0 0 72 1157 0.000000 1.000000 0.058584 0.000000
273+
Password 2595 11366 0 0 11366 2595 0.000000 1.000000 0.814125 0.000000
273274
Perplexity API Key 2 0 0 0 0 2 1.000000 0.000000 0.000000
274275
Postman Credentials 2 0 0 0 0 2 1.000000 0.000000 0.000000
275276
SQL Password 44 14 0 0 14 44 0.000000 1.000000 0.241379 0.000000
276277
Salesforce Credentials 6 0 0 0 0 6 1.000000 0.000000 0.000000
277278
Salt 90 130 0 0 130 90 0.000000 1.000000 0.590909 0.000000
278-
Secret 1525 2476 0 0 2476 1525 0.000000 1.000000 0.618845 0.000000
279+
Secret 1525 2492 0 0 2492 1525 0.000000 1.000000 0.620363 0.000000
279280
Slack Token 15 1 0 0 1 15 0.000000 1.000000 0.062500 0.000000
280281
Stripe Credentials 2 0 0 0 0 2 1.000000 0.000000 0.000000
281282
Tencent WeChat API App ID 47 0 0 0 0 47 1.000000 0.000000 0.000000
282-
Token 1139 5269 0 0 5269 1139 0.000000 1.000000 0.822253 0.000000
283+
Token 1144 5285 0 0 5285 1144 0.000000 1.000000 0.822056 0.000000
283284
Twilio Credentials 30 39 0 0 39 30 0.000000 1.000000 0.565217 0.000000
284285
URL Credentials 225 401 0 0 401 225 0.000000 1.000000 0.640575 0.000000
285286
UUID 2517 3716 0 0 3716 2517 0.000000 1.000000 0.596182 0.000000
286-
17130 53678 0 0 0 53678 17130 0.000000 1.000000 0.758078 0.000000
287+
17141 53736 0 0 0 53736 17141 0.000000 1.000000 0.758158 0.000000

‎meta/10e03406.csv‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,ValueStart,ValueEnd,CryptographyKey,PredefinedPattern,Category
2+
11536686,0cd5033e,GitHub,10e03406,data/10e03406/test/pkg/0cd5033e.go,107,107,F,56,120,,,Token
3+
11536687,4278973e,GitHub,10e03406,data/10e03406/test/pkg/4278973e.go,21,47,T,15,29,,,PEM Private Key
4+
11536688,4278973e,GitHub,10e03406,data/10e03406/test/pkg/4278973e.go,57,83,T,20,29,,,PEM Private Key
5+
11536689,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,149,149,T,17,71,,,Token
6+
11536690,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,618,618,T,22,76,,,Token
7+
11536691,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,655,655,T,16,70,,,Token
8+
11536692,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,1179,1179,T,22,76,,,Token
9+
11536693,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,32,32,F,25,128,,HASH,Key:Token
10+
11536694,d483ffb9,GitHub,10e03406,data/10e03406/test/pkg/cluster/d483ffb9.go,54,54,T,175,199,,,Auth
11+
11536695,dfb6fb1d,GitHub,10e03406,data/10e03406/test/pkg/dfb6fb1d.go,708,708,T,18,58,,,Token:Github Classic Token
12+
11536696,fd2654ce,GitHub,10e03406,data/10e03406/test/pkg/fd2654ce.go,46,46,T,70,299,,,JSON Web Token
13+
11536697,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,93,93,F,40,104,,,API
14+
11536698,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,110,110,F,38,102,,,Credential
15+
11536699,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,147,147,F,40,104,,,API
16+
11536700,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,164,164,F,38,102,,,Credential
17+
11536701,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,204,204,F,40,104,,,API
18+
11536702,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,221,221,F,38,102,,,Credential
19+
11536703,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,252,252,F,40,104,,,API
20+
11536704,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,269,269,F,38,102,,,Credential
21+
11536705,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,41,41,F,31,35,,,Secret
22+
11536706,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,43,43,F,23,27,,,Secret
23+
11536707,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,52,52,F,31,35,,,Secret
24+
11536708,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,53,53,F,31,35,,,Secret
25+
11536709,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,25,51,T,9,29,,,PEM Private Key
26+
11536710,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,76,76,F,31,35,,,Secret
27+
11536711,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,78,78,F,23,27,,,Secret
28+
11536712,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,86,86,F,31,35,,,Secret
29+
11536713,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,87,87,F,31,35,,,Secret
30+
11536714,3f7b7565,GitHub,10e03406,data/10e03406/test/secret/resource/3f7b7565.yaml,3,3,F,12,20,,,Password
31+
11536715,92807aff,GitHub,10e03406,data/10e03406/test/secret/resource/92807aff.yaml,3,3,F,12,20,,,Password
32+
11536716,1e4ad539,GitHub,10e03406,data/10e03406/test/util/pkg/1e4ad539.go,23,23,F,20,123,,,Key:Token
33+
11536717,0cd5033e,GitHub,10e03406,data/10e03406/test/pkg/0cd5033e.go,199,199,F,56,120,,,Token
34+
11536718,0cd5033e,GitHub,10e03406,data/10e03406/test/pkg/0cd5033e.go,239,239,F,56,120,,,Token
35+
11536719,4278973e,GitHub,10e03406,data/10e03406/test/pkg/4278973e.go,113,113,F,28,423,,,Key
36+
11536720,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,697,697,F,46,51,,,Token
37+
11536721,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,700,700,F,46,64,,,Token
38+
11536722,7caa428b,GitHub,10e03406,data/10e03406/test/pkg/7caa428b.go,951,951,F,38,45,,,Token
39+
11536723,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,33,33,F,25,83,,,Key:Token
40+
11536724,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,75,75,F,78,95,,,Auth:Token
41+
11536725,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,253,253,F,78,95,,,Auth:Token
42+
11536726,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,460,460,F,78,95,,,Auth:Token
43+
11536727,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,681,681,F,78,95,,,Auth:Token
44+
11536728,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,990,990,F,30,41,,,Auth:Token
45+
11536729,6341416d,GitHub,10e03406,data/10e03406/test/pkg/cluster/6341416d.go,1129,1129,F,30,41,,,Auth:Token
46+
11536730,d483ffb9,GitHub,10e03406,data/10e03406/test/pkg/cluster/d483ffb9.go,27,27,F,42,47,,,Secret
47+
11536731,dfb6fb1d,GitHub,10e03406,data/10e03406/test/pkg/dfb6fb1d.go,641,641,F,57,67,,,API
48+
11536732,dfb6fb1d,GitHub,10e03406,data/10e03406/test/pkg/dfb6fb1d.go,650,650,F,57,67,,,API
49+
11536733,dfb6fb1d,GitHub,10e03406,data/10e03406/test/pkg/dfb6fb1d.go,666,666,F,57,67,,,API
50+
11536734,dfb6fb1d,GitHub,10e03406,data/10e03406/test/pkg/dfb6fb1d.go,679,679,F,59,69,,,API
51+
11536735,51945883,GitHub,10e03406,data/10e03406/test/pkg/sys/template/51945883.go,40,40,F,42,47,,,Secret
52+
11536736,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,39,39,F,27,33,,,Secret
53+
11536737,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,50,50,F,27,33,,,Secret
54+
11536738,8644694b,GitHub,10e03406,data/10e03406/test/secret/8644694b.py,231,231,F,43,47,,,Key
55+
11536739,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,74,74,F,27,33,,,Secret
56+
11536740,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,84,84,F,27,33,,,Secret
57+
11536741,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,248,248,F,43,47,,,Key
58+
11536742,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,291,291,F,19,36,,,Secret
59+
11536743,fd8cb0ea,GitHub,10e03406,data/10e03406/test/secret/fd8cb0ea.py,327,327,F,19,36,,,Secret
60+
11536744,1e4ad539,GitHub,10e03406,data/10e03406/test/util/pkg/1e4ad539.go,24,24,F,22,33,,,Token

‎obfuscate_creds.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
from argparse import Namespace, ArgumentParser
99
from typing import List
1010

11+
import base62
12+
1113
from constants import PRIVATE_KEY_CATEGORY, LABEL_TRUE, MULTI_PATTERN_RULES
1214
from meta_row import read_meta, MetaRow
1315

@@ -133,6 +135,16 @@ def obfuscate_glsa(value):
133135
return obfuscated_value
134136

135137

138+
def obfuscate_crc32_base62(value):
139+
token = generate_value(value[4:-6])
140+
data = token.encode('ascii', errors="strict")
141+
crc32sum = binascii.crc32(data)
142+
crc32data = crc32sum.to_bytes(length=4, byteorder="big")
143+
crc32sign = base62.encodebytes(crc32data)
144+
obfuscated_value = value[:4] + token + crc32sign
145+
return obfuscated_value
146+
147+
136148
def get_obfuscated_value(value, meta_row: MetaRow):
137149
if "Info" == meta_row.PredefinedPattern:
138150
# not a credential - does not require obfuscation
@@ -209,6 +221,8 @@ def get_obfuscated_value(value, meta_row: MetaRow):
209221
else:
210222
# impossible, but linter fix
211223
obfuscated_value = generate_value(value)
224+
elif any(value.startswith(x) for x in ["npm_", "ghp_", "gho_", "ghu_", "ghs_", "ghr_"]):
225+
obfuscated_value = obfuscate_crc32_base62(value)
212226
else:
213227
# the whole value is obfuscated
214228
obfuscated_value = generate_value(value)

‎requirements.txt‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
# Python 3.10.18
2-
# pip 25.1.1
1+
# Python 3.10.19
2+
# pip 25.3
33

4-
GitPython==3.1.44
5-
virtualenv==20.31.2
4+
GitPython==3.1.46
5+
virtualenv==20.35.4
66

7-
setuptools==80.9.0
7+
#setuptools==80.9.0
88
tabulate==0.9.0
9+
pybase62==1.0.0
910

1011
#credentialdigger==4.9.5
1112

0 commit comments

Comments
 (0)