Skip to content

Commit 881e1b7

Browse files
flysenflysen
authored
Additional regex pattern for Datadog API/APP tokens.
Authored-by: Fredrik Lysén <[email protected]>
1 parent 5840c9b commit 881e1b7

File tree

6 files changed

+217
-1
lines changed

6 files changed

+217
-1
lines changed

‎src/main/java/com/tsystems/sbs/DefaultRegexpPairs.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,22 @@ public final class DefaultRegexpPairs {
2626
new RegexpPair("(aws_[a-zA-Z_]+=)(\\S+)", "$1********")
2727
));
2828

29+
private final static List<RegexpPair> DEFAULT_REGEXES_DD
30+
= Collections.<RegexpPair>unmodifiableList(
31+
Arrays.<RegexpPair>asList(
32+
new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{32})\\b)", "$1********"), // Datadog RegExp MASKING API KEY
33+
new RegexpPair("((?i)(\\bdatadog|dd|dogapi\\b).*)(\\b([a-zA-Z-0-9]{40})\\b)", "$1********") // Datadog RegExp MASKING APP KEY
34+
));
35+
2936
public static List<RegexpPair> getDefaultRegexes() {
3037
return DEFAULT_REGEXES;
3138
}
3239

3340
public static List<RegexpPair> getDefaultRegexesAWS() {
3441
return DEFAULT_REGEXES_AWS;
3542
}
43+
44+
public static List<RegexpPair> getDefaultRegexesDD() {
45+
return DEFAULT_REGEXES_DD;
46+
}
3647
}

‎src/main/java/com/tsystems/sbs/LogFileFilterConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public static LogFileFilterConfig get() {
4646
*/
4747
private boolean enabledDefaultRegexp;
4848
private boolean enabledDefaultRegexpAWS;
49+
private boolean enabledDefaultRegexpDD;
4950

5051
/**
5152
* Represents the custom regexp pairs specified by the user in the global settings.
@@ -75,6 +76,10 @@ public boolean isEnabledDefaultRegexpAWS() {
7576
return enabledDefaultRegexpAWS;
7677
}
7778

79+
public boolean isEnabledDefaultRegexpDD() {
80+
return enabledDefaultRegexpDD;
81+
}
82+
7883
@DataBoundSetter
7984
public void setEnabledDefaultRegexp(boolean enabledDefaultRegexp) {
8085
this.enabledDefaultRegexp = enabledDefaultRegexp;
@@ -87,6 +92,12 @@ public void setEnabledDefaultRegexpAWS(boolean enabledDefaultRegexpAWS) {
8792
save();
8893
}
8994

95+
@DataBoundSetter
96+
public void setEnabledDefaultRegexpDD(boolean enabledDefaultRegexpDD) {
97+
this.enabledDefaultRegexpDD = enabledDefaultRegexpDD;
98+
save();
99+
}
100+
90101
public List<RegexpPair> getRegexpPairs() {
91102
return regexpPairs;
92103
}

‎src/main/java/com/tsystems/sbs/LogFileFilterOutputStream.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ public class LogFileFilterOutputStream extends LineTransformationOutputStream {
3131
private final boolean isEnabledGlobally;
3232
private final boolean isEnabledDefaultRegexp;
3333
private final boolean isEnabledDefaultRegexpAWS;
34+
private final boolean isEnabledDefaultRegexpDD;
3435
private final List<RegexpPair> defaultRegexpPairs;
3536
private final List<RegexpPair> customRegexpPairs;
3637
private final String jobName;
@@ -51,6 +52,7 @@ public LogFileFilterOutputStream(OutputStream out, Charset charset, String jobNa
5152
isEnabledGlobally = config.isEnabledGlobally();
5253
isEnabledDefaultRegexp = config.isEnabledDefaultRegexp();
5354
isEnabledDefaultRegexpAWS = config.isEnabledDefaultRegexpAWS();
55+
isEnabledDefaultRegexpDD = config.isEnabledDefaultRegexpDD();
5456

5557
if (isEnabledGlobally) {
5658
// Load regexes
@@ -62,6 +64,10 @@ public LogFileFilterOutputStream(OutputStream out, Charset charset, String jobNa
6264
if (isEnabledDefaultRegexpAWS) {
6365
defaultRegexpPairs.addAll(DefaultRegexpPairs.getDefaultRegexesAWS());
6466
}
67+
if (isEnabledDefaultRegexpDD) {
68+
defaultRegexpPairs.addAll(DefaultRegexpPairs.getDefaultRegexesDD());
69+
}
70+
6571
// Log defaultRegexpPairs
6672
for (RegexpPair pair : defaultRegexpPairs) {
6773
LOGGER.log(Level.INFO, pair.toString());

‎src/main/resources/com/tsystems/sbs/LogFileFilterConfig/config.jelly‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@
2828
<f:entry title="Enable Amazon AWS regexp" field="enabledDefaultRegexpAWS">
2929
<f:checkbox checked="${it.enabledDefaultRegexpAWS}" />
3030
</f:entry>
31-
31+
32+
<f:entry title="Enable Datadog API/APP regexp" field="enabledDefaultRegexpDD">
33+
<f:checkbox checked="${it.enabledDefaultRegexpDD}" />
34+
</f:entry>
35+
3236
<!-- Defines the regexp patterns to filter the console logs -->
3337
<f:entry title="Custom regexp pairs" field="regexpPairs">
3438
<f:repeatable name="Regexp Pairs" field="regexpPairs">

‎src/main/resources/com/tsystems/sbs/LogFileFilterConfig/help-enabledDefaultRegexpDD.html‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<!--THIS FILE IS AUTOGENERATED FROM THE global.properties FILE-->
2+
<div>
3+
4+
<style>
5+
.log-file-filter-plugin th, .log-file-filter-plugin td{
6+
border: solid 1px black;
7+
border-collapse: collapse;
8+
padding: 10px
9+
}
10+
</style>
11+
12+
<p>These are the default regular expressions and their respective replacements. These expressions are fixed and come with the plugin.</p>
13+
<table class="log-file-filter-plugin">
14+
<!-- Table header -->
15+
<tr><th>Description</th><th>Regexp</th><th>Replacement</th><th>Sample</th></tr>
16+
17+
<!-- Table rows (regexes) -->
18+
<tr>
19+
<td>Masks Datadog APP secrets</td>
20+
<td>((?i)(\bdatadog|dd|dogapi\b).*)(\b([a-zA-Z-0-9]{40})\b)</td>
21+
<td>$1********</td>
22+
<td>
23+
<ul>
24+
<li>"datadog key = 3c0c3965368a6b10f7640dbda46abfd2 secret= 3c0c3965368a6b10f7640dbda46abfdca981c2d3" -> <b>datadog key = ******** secret= ********</b></li>
25+
</ul>
26+
</td>
27+
</tr>
28+
<tr>
29+
<td>Masks Datadog API secrets</td>
30+
<td>((?i)(\bdatadog|dd|dogapi\b).*)(\b([a-zA-Z-0-9]{32})\b)</td>
31+
<td>$1********</td>
32+
<td>
33+
<ul>
34+
<li>dAtAdOg token = "3c0c3965368a6b10f7640dbda46abfdc"; -> <b>dAtAdOg token = "********";</b></li>
35+
<li>curl -X GET "https://api.datadoghq.eu/api/v1/validate" -H "Accept: application/json" -H "DD-API-KEY: characteristicallycharacteristic" -> <b>curl -X GET "https://api.datadoghq.eu/api/v1/validate" -H "Accept: application/json" -H "DD-API-KEY: ********"</b></li>
36+
</ul>
37+
</td>
38+
</tr>
39+
</table>
40+
</div>

‎src/test/java/com/tsystems/sbs/DefaultRegexpPairsDDTest.java‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package com.tsystems.sbs;
2+
3+
import org.junit.Test;
4+
5+
import java.util.List;
6+
import java.util.regex.Matcher;
7+
import java.util.regex.Pattern;
8+
9+
import static org.hamcrest.MatcherAssert.assertThat;
10+
import static org.hamcrest.Matchers.greaterThan;
11+
import static org.junit.Assert.assertEquals;
12+
13+
14+
public class DefaultRegexpPairsDDTest {
15+
private List<RegexpPair> getDefaultRegexpPairs() {
16+
return DefaultRegexpPairs.getDefaultRegexesDD();
17+
}
18+
@Test
19+
public void testDefaultPairsList() {
20+
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();
21+
assertThat(defaultRegexpPairs.size(), greaterThan(0));
22+
23+
}
24+
25+
@Test
26+
public void testDefaultPairsApi() {
27+
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();
28+
29+
// Define the input string 32 characters
30+
String input = "curl -X GET \"https://api.datadoghq.eu/api/v1/validate\" -H \"Accept: application/json\" -H \"DD-API-KEY: characteristicallycharacteristic\"";
31+
String expected = "curl -X GET \"https://api.datadoghq.eu/api/v1/validate\" -H \"Accept: application/json\" -H \"DD-API-KEY: ********\"";
32+
33+
34+
StringBuilder replacedInput = new StringBuilder(input);
35+
36+
for (RegexpPair pair : defaultRegexpPairs) {
37+
String pattern = pair.getRegexp();
38+
String replacement = pair.getReplacement();
39+
40+
Pattern regexPattern = Pattern.compile(pattern);
41+
Matcher matcher = regexPattern.matcher(replacedInput);
42+
43+
while (matcher.find()) {
44+
String matchedPattern = matcher.group();
45+
String replacedString = replacement;
46+
47+
// Replace all occurrences of $n with the matched groups
48+
for (int i = 1; i <= matcher.groupCount(); i++) {
49+
String group = matcher.group(i);
50+
replacedString = replacedString.replace("$" + i, group);
51+
}
52+
53+
replacedInput.replace(matcher.start(), matcher.end(), replacedString);
54+
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
55+
}
56+
}
57+
58+
String replacedInputString = replacedInput.toString();
59+
System.out.println("Replaced input result: " + replacedInputString);
60+
61+
// Test the behavior
62+
assertEquals(expected, replacedInputString);
63+
}
64+
65+
@Test
66+
public void testDefaultPairsKey() {
67+
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();
68+
69+
// Define the input string 32 characters
70+
String input = "datadog key = 3c0c3965368a6b10f7640dbda46abfd2 secret= 3c0c3965368a6b10f7640dbda46abfdca981c2d3";
71+
String expected = "datadog key = ******** secret= ********";
72+
73+
74+
StringBuilder replacedInput = new StringBuilder(input);
75+
76+
for (RegexpPair pair : defaultRegexpPairs) {
77+
String pattern = pair.getRegexp();
78+
String replacement = pair.getReplacement();
79+
80+
Pattern regexPattern = Pattern.compile(pattern);
81+
Matcher matcher = regexPattern.matcher(replacedInput);
82+
83+
while (matcher.find()) {
84+
String matchedPattern = matcher.group();
85+
String replacedString = replacement;
86+
87+
// Replace all occurrences of $n with the matched groups
88+
for (int i = 1; i <= matcher.groupCount(); i++) {
89+
String group = matcher.group(i);
90+
replacedString = replacedString.replace("$" + i, group);
91+
}
92+
93+
replacedInput.replace(matcher.start(), matcher.end(), replacedString);
94+
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
95+
}
96+
}
97+
98+
String replacedInputString = replacedInput.toString();
99+
System.out.println("Replaced input result: " + replacedInputString);
100+
101+
// Test the behavior
102+
assertEquals(expected, replacedInputString);
103+
}
104+
@Test
105+
public void testDefaultPairsToken() {
106+
List<RegexpPair> defaultRegexpPairs = getDefaultRegexpPairs();
107+
108+
// Define the input string 32 characters
109+
String input = "dAtAdOg token = \"3c0c3965368a6b10f7640dbda46abfdc\";";
110+
String expected = "dAtAdOg token = \"********\";";
111+
112+
113+
StringBuilder replacedInput = new StringBuilder(input);
114+
115+
for (RegexpPair pair : defaultRegexpPairs) {
116+
String pattern = pair.getRegexp();
117+
String replacement = pair.getReplacement();
118+
119+
Pattern regexPattern = Pattern.compile(pattern);
120+
Matcher matcher = regexPattern.matcher(replacedInput);
121+
122+
while (matcher.find()) {
123+
String matchedPattern = matcher.group();
124+
String replacedString = replacement;
125+
126+
// Replace all occurrences of $n with the matched groups
127+
for (int i = 1; i <= matcher.groupCount(); i++) {
128+
String group = matcher.group(i);
129+
replacedString = replacedString.replace("$" + i, group);
130+
}
131+
132+
replacedInput.replace(matcher.start(), matcher.end(), replacedString);
133+
matcher.region(matcher.start() + replacedString.length(), replacedInput.length());
134+
}
135+
}
136+
137+
String replacedInputString = replacedInput.toString();
138+
System.out.println("Replaced input result: " + replacedInputString);
139+
140+
// Test the behavior
141+
assertEquals(expected, replacedInputString);
142+
}
143+
}
144+

0 commit comments

Comments
 (0)