Skip to content

Commit 1b638d1

Browse files
copybara-githubcopybara-github
committed
Merge pull request google#2208 from doyensec:US-SSN
PiperOrigin-RevId: 934925417
2 parents ca1be91 + b13eb37 commit 1b638d1

6 files changed

Lines changed: 587 additions & 55 deletions

File tree

‎docs/supported_inventory_types.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ See the docs on [how to add a new Extractor](/docs/new_extractor.md).
215215
| HTTP Bearer | `secrets/httpbearer` |
216216
| HTTP CSRF Token | `secrets/csrftoken` |
217217

218+
### Sensitive information
219+
| Type | Extractor Plugin |
220+
| ------------------------------------------- | ------------------------------------ |
221+
| US Social Security Number | `sensitiveinformation/ssn` |
222+
218223
### Container inventory
219224

220225
| Type | Extractor Plugin |

‎extractor/filesystem/list/list.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ import (
184184
"github.com/google/osv-scalibr/veles/secrets/tinkkeyset"
185185
"github.com/google/osv-scalibr/veles/secrets/urlcreds"
186186
"github.com/google/osv-scalibr/veles/secrets/vapid"
187+
"github.com/google/osv-scalibr/veles/sensitiveinformation/ssn"
187188

188189
cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto"
189190
)
@@ -454,10 +455,15 @@ var (
454455
{http.NewCSRFTokenDetector(), "secrets/csrftoken", 0},
455456
})
456457

458+
SensitiveInformationDetectors = initMapFromVelesPlugins([]velesPlugin{
459+
{ssn.NewDetector(), "sensitiveinformation/ssn", 0},
460+
})
461+
457462
// Secrets contains both secret extractors and detectors.
458463
Secrets = concat(
459464
SecretDetectors,
460465
SecretExtractors,
466+
SensitiveInformationDetectors,
461467
)
462468

463469
// Misc artifact extractors.

‎veles/sensitiveinformation/common/simpleregex/simpleregex.go‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -31,21 +31,21 @@ import (
3131
// It implements veles.Detector.
3232
type Detector struct {
3333
// The maximum length of the sensitive information.
34-
maxLen uint32
34+
MaxLen uint32
3535
// Matches on the sensitive information.
36-
re *regexp.Regexp
36+
Re *regexp.Regexp
3737

3838
// Context window size for keywords search before the match.
39-
contextWindowBefore uint32
39+
ContextWindowBefore uint32
4040
// Context window size for keywords search after the match.
41-
contextWindowAfter uint32
41+
ContextWindowAfter uint32
4242

4343
// KeywordsRe is the regexp of the Keywords. All keywords are case insensitive.
44-
keywordsRe *regexp.Regexp
44+
KeywordsRe *regexp.Regexp
4545

4646
// Returns a sensitiveinformation.SensitiveInformation from a regexp match
4747
// result.
48-
fromMatch func([]byte) (sensitiveinformation.SensitiveInformation, bool)
48+
FromMatch func(blob []byte, keywordMatch bool) (sensitiveinformation.SensitiveInformation, bool)
4949
}
5050

5151
// KeywordsRe returns a regexp of the keywords. All keywords are case insensitive.
@@ -58,25 +58,22 @@ func KeywordsRe(keywords []string) *regexp.Regexp {
5858

5959
// MaxSecretLen returns the maximum length of the search window.
6060
func (d Detector) MaxSecretLen() uint32 {
61-
return d.maxLen + d.contextWindowBefore + d.contextWindowAfter
61+
return d.MaxLen + d.ContextWindowBefore + d.ContextWindowAfter
6262
}
6363

6464
// Detect finds candidate tokens that match Detector.Re and returns them
6565
// alongside their starting positions.
6666
func (d Detector) Detect(data []byte) (secrets []veles.Secret, positions []int) {
67-
for _, m := range d.re.FindAllIndex(data, -1) {
67+
for _, m := range d.Re.FindAllIndex(data, -1) {
6868
l, r := m[0], m[1]
69-
lowerBound := max(0, l-int(d.contextWindowBefore))
70-
upperBound := min(len(data), r+int(d.contextWindowAfter))
71-
// If keywordsRe is set, check if the keywords are present in the context window before or after
72-
// the match.
73-
if d.keywordsRe != nil &&
74-
!d.keywordsRe.Match(data[lowerBound:l]) &&
75-
!d.keywordsRe.Match(data[r:upperBound]) {
76-
continue
77-
}
69+
lowerBound := max(0, l-int(d.ContextWindowBefore))
70+
upperBound := min(len(data), r+int(d.ContextWindowAfter))
71+
// If KeywordsRe is set, check if the keywords are present in the context
72+
// window before or after the match.
73+
contextMatch := d.KeywordsRe != nil &&
74+
(d.KeywordsRe.Match(data[lowerBound:l]) || d.KeywordsRe.Match(data[r:upperBound]))
7875

79-
if match, ok := d.fromMatch(data[l:r]); ok {
76+
if match, ok := d.FromMatch(data[l:r], contextMatch); ok {
8077
secrets = append(secrets, match)
8178
positions = append(positions, l)
8279
}

‎veles/sensitiveinformation/common/simpleregex/simpleregex_test.go‎

Lines changed: 122 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ func TestDetect_truePositives(t *testing.T) {
4343
in []byte
4444
want []veles.Secret
4545
wantPos []int
46-
fromMatch func([]byte) (sensitiveinformation.SensitiveInformation, bool)
46+
wantFlags []bool
47+
fromMatch func([]byte, bool) (sensitiveinformation.SensitiveInformation, bool)
4748
}{
4849
{
4950
name: "match only",
@@ -123,7 +124,7 @@ BAZ
123124
fakeSensitiveInformation([]byte("BAR")),
124125
},
125126
wantPos: []int{4},
126-
fromMatch: func(b []byte) (sensitiveinformation.SensitiveInformation, bool) {
127+
fromMatch: func(b []byte, contextMatch bool) (sensitiveinformation.SensitiveInformation, bool) {
127128
if string(b) == "FOO" {
128129
return sensitiveinformation.SensitiveInformation{}, false
129130
}
@@ -149,9 +150,14 @@ BAZ
149150
before: 4,
150151
keywords: []string{"abc", "def"},
151152
want: []veles.Secret{
153+
fakeSensitiveInformation([]byte("FOO")),
152154
fakeSensitiveInformation([]byte("BAR")),
153155
},
154-
wantPos: []int{8},
156+
wantPos: []int{0, 8},
157+
wantFlags: []bool{
158+
false,
159+
true,
160+
},
155161
}, {
156162
name: "keywords after",
157163
regexp: "[A-Z]{3}",
@@ -161,8 +167,15 @@ BAZ
161167
keywords: []string{"abc", "def"},
162168
want: []veles.Secret{
163169
fakeSensitiveInformation([]byte("FOO")),
170+
fakeSensitiveInformation([]byte("DEF")),
171+
fakeSensitiveInformation([]byte("BAR")),
172+
},
173+
wantPos: []int{0, 4, 8},
174+
wantFlags: []bool{
175+
true,
176+
false,
177+
false,
164178
},
165-
wantPos: []int{0},
166179
}, {
167180
name: "keywords before and after",
168181
regexp: "[A-Z]{3}",
@@ -172,10 +185,20 @@ BAZ
172185
after: 4,
173186
keywords: []string{"abc", "def"},
174187
want: []veles.Secret{
188+
fakeSensitiveInformation([]byte("ABC")),
175189
fakeSensitiveInformation([]byte("FOO")),
190+
fakeSensitiveInformation([]byte("DEF")),
176191
fakeSensitiveInformation([]byte("BAR")),
192+
fakeSensitiveInformation([]byte("ABC")),
193+
},
194+
wantPos: []int{0, 4, 8, 12, 16},
195+
wantFlags: []bool{
196+
false,
197+
true,
198+
false,
199+
true,
200+
false,
177201
},
178-
wantPos: []int{4, 12},
179202
}, {
180203
name: "keywords matches regex",
181204
regexp: "[A-Z]{3}",
@@ -184,26 +207,37 @@ BAZ
184207
before: 4,
185208
keywords: []string{"abc", "def"},
186209
want: []veles.Secret{
210+
fakeSensitiveInformation([]byte("ABC")),
187211
fakeSensitiveInformation([]byte("DEF")),
188212
fakeSensitiveInformation([]byte("ABC")),
189213
},
190-
wantPos: []int{4, 8},
214+
wantPos: []int{0, 4, 8},
215+
wantFlags: []bool{
216+
false,
217+
true,
218+
true,
219+
},
191220
},
192221
}
193222
for _, tc := range cases {
194223
t.Run(tc.name, func(t *testing.T) {
195-
if tc.fromMatch == nil {
196-
tc.fromMatch = func(b []byte) (sensitiveinformation.SensitiveInformation, bool) {
224+
fromMatch := tc.fromMatch
225+
if fromMatch == nil {
226+
fromMatch = func(b []byte, _ bool) (sensitiveinformation.SensitiveInformation, bool) {
197227
return fakeSensitiveInformation(b), true
198228
}
199229
}
230+
var gotFlags []bool
200231
d := Detector{
201-
maxLen: tc.maxLen,
202-
re: regexp.MustCompile(tc.regexp),
203-
contextWindowBefore: tc.before,
204-
contextWindowAfter: tc.after,
205-
keywordsRe: KeywordsRe(tc.keywords),
206-
fromMatch: tc.fromMatch,
232+
MaxLen: tc.maxLen,
233+
Re: regexp.MustCompile(tc.regexp),
234+
ContextWindowBefore: tc.before,
235+
ContextWindowAfter: tc.after,
236+
KeywordsRe: KeywordsRe(tc.keywords),
237+
FromMatch: func(b []byte, contextMatch bool) (sensitiveinformation.SensitiveInformation, bool) {
238+
gotFlags = append(gotFlags, contextMatch)
239+
return fromMatch(b, contextMatch)
240+
},
207241
}
208242
got, gotPos := d.Detect(tc.in)
209243
if diff := cmp.Diff(tc.want, got, cmpopts.EquateEmpty()); diff != "" {
@@ -212,6 +246,73 @@ BAZ
212246
if diff := cmp.Diff(tc.wantPos, gotPos, cmpopts.EquateEmpty()); diff != "" {
213247
t.Errorf("Detect() diff (-want +got):\n%s", diff)
214248
}
249+
wantFlags := tc.wantFlags
250+
if wantFlags == nil {
251+
wantFlags = make([]bool, len(gotFlags))
252+
}
253+
if diff := cmp.Diff(wantFlags, gotFlags, cmpopts.EquateEmpty()); diff != "" {
254+
t.Errorf("FromMatch() keywordMatch diff (-want +got):\n%s", diff)
255+
}
256+
})
257+
}
258+
}
259+
260+
func TestDetect_passesKeywordContextMatchToFromMatch(t *testing.T) {
261+
type callbackCall struct {
262+
Match string
263+
ContextMatch bool
264+
}
265+
266+
cases := []struct {
267+
name string
268+
keywords []string
269+
in []byte
270+
before uint32
271+
after uint32
272+
want []callbackCall
273+
}{
274+
{
275+
name: "keyword_match_and_no_keyword_match",
276+
keywords: []string{"abc", "def"},
277+
in: []byte("abc FOO x BAR yyy BAZ def"),
278+
before: 4,
279+
after: 4,
280+
want: []callbackCall{
281+
{Match: "FOO", ContextMatch: true},
282+
{Match: "BAR", ContextMatch: false},
283+
{Match: "BAZ", ContextMatch: true},
284+
},
285+
},
286+
{
287+
name: "no_keywords_regexp",
288+
keywords: nil,
289+
in: []byte("abc FOO def"),
290+
before: 4,
291+
after: 4,
292+
want: []callbackCall{
293+
{Match: "FOO", ContextMatch: false},
294+
},
295+
},
296+
}
297+
298+
for _, tc := range cases {
299+
t.Run(tc.name, func(t *testing.T) {
300+
var got []callbackCall
301+
d := Detector{
302+
MaxLen: 3,
303+
Re: regexp.MustCompile("[A-Z]{3}"),
304+
ContextWindowBefore: tc.before,
305+
ContextWindowAfter: tc.after,
306+
KeywordsRe: KeywordsRe(tc.keywords),
307+
FromMatch: func(b []byte, contextMatch bool) (sensitiveinformation.SensitiveInformation, bool) {
308+
got = append(got, callbackCall{Match: string(b), ContextMatch: contextMatch})
309+
return fakeSensitiveInformation(b), true
310+
},
311+
}
312+
d.Detect(tc.in)
313+
if diff := cmp.Diff(tc.want, got); diff != "" {
314+
t.Errorf("FromMatch calls diff (-want +got):\n%s", diff)
315+
}
215316
})
216317
}
217318
}
@@ -234,35 +335,19 @@ func TestDetect_trueNegatives(t *testing.T) {
234335
regexp: "FOO",
235336
maxLen: 3,
236337
in: []byte("BAR"),
237-
}, {
238-
name: "no keyword match",
239-
regexp: "FOO",
240-
keywords: []string{"abc", "def"},
241-
before: 4,
242-
after: 4,
243-
maxLen: 3,
244-
in: []byte("FOO"),
245-
}, {
246-
name: "keyword ahead of context window",
247-
regexp: "FOO",
248-
keywords: []string{"abc", "def"},
249-
before: 3,
250-
after: 3,
251-
maxLen: 3,
252-
in: []byte("abc FOO def"),
253338
}}
254339
for _, tc := range cases {
255340
t.Run(tc.name, func(t *testing.T) {
256-
fromMatch := func(b []byte) (sensitiveinformation.SensitiveInformation, bool) {
341+
fromMatch := func(b []byte, contextMatch bool) (sensitiveinformation.SensitiveInformation, bool) {
257342
return fakeSensitiveInformation(b), true
258343
}
259344
d := Detector{
260-
maxLen: tc.maxLen,
261-
re: regexp.MustCompile(tc.regexp),
262-
contextWindowBefore: tc.before,
263-
contextWindowAfter: tc.after,
264-
keywordsRe: KeywordsRe(tc.keywords),
265-
fromMatch: fromMatch,
345+
MaxLen: tc.maxLen,
346+
Re: regexp.MustCompile(tc.regexp),
347+
ContextWindowBefore: tc.before,
348+
ContextWindowAfter: tc.after,
349+
KeywordsRe: KeywordsRe(tc.keywords),
350+
FromMatch: fromMatch,
266351
}
267352
got, gotPos := d.Detect(tc.in)
268353
if diff := cmp.Diff(tc.want, got, cmpopts.EquateEmpty()); diff != "" {

0 commit comments

Comments
 (0)