Skip to content

fix: obfuscate regexes #2063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
muruganishekar wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: obfuscate regexes #2063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions pkg/helpers/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,11 +370,11 @@ func ObfuscateSensitiveDataFromArray(data []string) []string {

//nolint:gochecknoglobals
var obfuscateRegexes = []*regexp.Regexp{
// Match if contains pass|token|cert|auth|key|secret|salt|cred|pw
// Match if contains pass|token|cert|auth|key|secret|salt|cred|pw (should not have @ )
// and capturing if found the group after one of the separators: ' ', ':', '=' and '"'.
regexp.MustCompile(`(?i)(?:pass|token|cert|auth|key|secret|salt|cred|pw)(?:[^\s:="]*)(?:[\s:="]*)([^\s:="]+)?`),
// Match password in url http://user:pass@localhost
regexp.MustCompile(`(?i)(?:\:\/\/\w+)(?:[\s:="]*)([a-zA-Z0-9]+)(?:[\@])`),
regexp.MustCompile(`(?i)(?:pass|token|cert|auth|key|secret|salt|cred|pw)(?:[^\s:=@"]*)(?:[\s:="]*)([^\s:=@"]+)?`),
// Match password in url http://user:pass@localhost (should not have @ in password/user as it is breaking url identification in json strings)
regexp.MustCompile(`(?i)(?:\:\/\/\w+)(?:[\s:="]*)([a-zA-Z0-9!#$%^&*()_+\-=\[\]{}|;:'",.<>?\/~]+)(?:[@])`),
}

// ObfuscateSensitiveData is used to detect sensitive data like tokens/passwords etc and
Expand All @@ -385,18 +385,16 @@ var obfuscateRegexes = []*regexp.Regexp{
// /usr/bin/custom_cmd -pwd 1234 -arg2 abc => /usr/bin/custom_cmd -pwd * -arg2 abc
func ObfuscateSensitiveData(value string) (matched, isField bool, result string) {
result = value

for _, obfuscateRegex := range obfuscateRegexes {

matches := obfuscateRegex.FindAllStringSubmatchIndex(result, -1)

var transforms bytes.Buffer

lastEndIndex := 0

for _, indexes := range matches {
// Expect array of 4:
// start-end indexes of the full match
// For array of 4 it's start-end indexes of the full match
// start-end indexes of the group 1 (data that should be obfuscated)
if len(indexes) != 4 {
break
Expand Down
75 changes: 62 additions & 13 deletions pkg/helpers/helpers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,25 +294,70 @@ func TestObfuscateSensitiveData_MatchButNothingToObfuscate(t *testing.T) {
assert.True(t, isField)
assert.Equal(t, data, result)
}

func TestObfuscateSensitiveData_CommandLineWithArgs(t *testing.T) {
data := "/usr/bin/custom_cmd -pwd 1234 -arg2 abc"
expected := "/usr/bin/custom_cmd -pwd <HIDDEN> -arg2 abc"
matched, isField, actual := ObfuscateSensitiveData(data)
tests := []struct {
name string
data string
expected string
isField bool
}{
{
name: "Custom command with password and mongodb uri",
data: "/usr/bin/custom_cmd -pwd 1234 --mongodb.uri mongodb://admin:testrw@host:27017/admin -arg2 abc",
expected: "/usr/bin/custom_cmd -pwd <HIDDEN> --mongodb.uri mongodb://admin:<HIDDEN>@host:27017/admin -arg2 abc",
isField: false,
},
{
name: "Prometheus exporter with mongodb uri and sensitive word in password",
data: "/usr/local/prometheus-exporters/bin/mongodb3-exporter --mongodb.uri mongodb://admin:testpass@localhost:27017/admin --no-mongodb.direct-connect --collector.dbstats --collector.collstats",
expected: "/usr/local/prometheus-exporters/bin/mongodb3-exporter --mongodb.uri mongodb://admin:<HIDDEN>@localhost:27017/admin --no-mongodb.direct-connect --collector.dbstats --collector.collstats",
isField: true,
},
}

assert.True(t, matched)
assert.False(t, isField)
assert.Equal(t, expected, actual)
//nolint:varnamelen
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
matched, isField, actual := ObfuscateSensitiveData(tc.data)
assert.True(t, matched)
assert.Equal(t, tc.isField, isField)
assert.Equal(t, tc.expected, actual)
})
}
}

func TestObfuscateSensitiveData_ConfigProtocolOutput(t *testing.T) {
data := `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://root:supercomplex@localhost:17017","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"passw0rd","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`
expected := `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://root:<HIDDEN>@localhost:17017","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"<HIDDEN>","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`
matched, isField, actual := ObfuscateSensitiveData(data)
tests := []struct {
name string
data string
expected string
isField bool
}{
{
name: "Config protocol output with password and mongodb uri",
data: `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://root:supercomplex@localhost:17017","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"passw0rd","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`, //nolint
expected: `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://root:<HIDDEN>@localhost:17017","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"<HIDDEN>","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`, //nolint
isField: false,
},
{
name: "Config protocol output with special characters password and mongodb uri",
data: `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://user:r#wsq@localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://newrelic:rW#ord@host:27017/admin","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"passw0rd","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`, //nolint
expected: `{"config_protocol_version":"1","action":"register_config","config_name":"cfg-nri-ibmmq","config":{"variables":{},"integrations":[{"name":"nri-prometheus","config":{"standalone":false,"verbose":"1","transformations":[],"integration_metadata":{"version":"0.3.0","name":"nri-ibmmq""targets":["urls":["http://user:<HIDDEN>@localhost:9157"]}]}},{"name":"ibmmq-exporter","timeout":0, "exec":[ "/usr/local/prometheus-exporters/bin/ibmmq-exporter","--mongodb.uri","mongodb://newrelic:<HIDDEN>@host:27017/admin","--ibmmq.connName","localhost(1414)","--ibmmq.queueManager","QM1","--ibmmq.channel","DEV.ADMIN.SVRCONN","--ibmmq.userid","admin","--ibmmq.httpListenPort","9157","--ibmmq.monitoredQueues","!SYSTEM.*,*","--ibmmq.monitoredChannels","*","--ibmmq.httpMetricPath","/metrics","--ibmmq.useStatus"],"env":{"IBMMQ_CONNECTION_PASSWORD":"<HIDDEN>","LD_LIBRARY_PATH":"/opt/mqm/lib64:/usr/lib64","HOME":"/tmp"}}]}}`, //nolint
isField: false,
},
}

assert.True(t, matched)
assert.False(t, isField)
assert.Equal(t, expected, actual)
//nolint:varnamelen
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
matched, isField, actual := ObfuscateSensitiveData(tc.data)
assert.True(t, matched)
assert.Equal(t, tc.isField, isField)
assert.Equal(t, tc.expected, actual)
})
}
}

func TestObfuscateSensitiveData_EnvironmentVariable(t *testing.T) {
Expand Down Expand Up @@ -417,7 +462,9 @@ func TestObfuscateSensitiveData_ObfuscateSensitiveDataFromArray(t *testing.T) {
"obfuscare_next_pass",
"12345",
"NRIA_KEY=1234",
"NewrelicMongoDB=pa$$word@",
"final",
"mongodb_uri mongodb://newrelictest:NewrelicMongoDB@localhost:27017/admin",
}

expected := []string{
Expand All @@ -427,7 +474,9 @@ func TestObfuscateSensitiveData_ObfuscateSensitiveDataFromArray(t *testing.T) {
"obfuscare_next_pass",
"<HIDDEN>",
"NRIA_KEY=<HIDDEN>",
"NewrelicMongoDB=pa$$word@",
"final",
"mongodb_uri mongodb://newrelictest:<HIDDEN>@localhost:27017/admin",
}

actual := ObfuscateSensitiveDataFromArray(data)
Expand Down
Loading