Skip to content

Commit 998226c

Browse files
abhishekbhatia1710abhishekbhatia1710
authored
[Security Solution][Entity Analytics][Risk Scoring] Handle special characters in ESQL query for risk scoring (elastic#247060)
## Summary Fixes `json.parse()` failures when ES|QL risk score calculation query's output contain special characters (quotes, backslashes, newlines, etc.) by encoding field values with Base64 in queries. Fixes: https://github.com/elastic/sdh-security-team/issues/1529 ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) - [ ] Review the [backport guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing) and apply applicable `backport:*` labels.
1 parent a295fd0 commit 998226c

4 files changed

Lines changed: 225 additions & 5 deletions

File tree

‎x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/__snapshots__/calculate_esql_risk_scores.test.ts.snap‎

Lines changed: 3 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.test.ts‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,5 +100,189 @@ describe('Calculate risk scores with ESQL', () => {
100100

101101
expect(bucket).toEqual(expected);
102102
});
103+
104+
/* The below tests are a result of https://github.com/elastic/sdh-security-team/issues/1529 */
105+
106+
describe('Rule name and category special characters', () => {
107+
it('decodes Base64 encoded rule_name and category', () => {
108+
// Simulate ESQL TO_BASE64 output
109+
const ruleNameWithQuotes = 'Test "Quoted" Alert';
110+
const categoryWithBackslash = 'signal\\test';
111+
const ruleNameB64 = Buffer.from(ruleNameWithQuotes, 'utf-8').toString('base64');
112+
const categoryB64 = Buffer.from(categoryWithBackslash, 'utf-8').toString('base64');
113+
114+
const inputs = [
115+
`{ "risk_score": "75", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "${categoryB64}", "id": "test_id_1" }`,
116+
];
117+
const alertCount = 1;
118+
const riskScore = 75;
119+
const entityValue = 'hostname';
120+
121+
const esqlResultRow = [alertCount, riskScore, inputs, entityValue];
122+
123+
const bucket = buildRiskScoreBucket(
124+
EntityType.host,
125+
'.alerts-security.alerts-default'
126+
)(esqlResultRow as FieldValue[]);
127+
128+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(
129+
ruleNameWithQuotes
130+
);
131+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].category).toBe(
132+
categoryWithBackslash
133+
);
134+
});
135+
136+
it('handles rule names with double quotes', () => {
137+
const ruleName = 'Alert: "Suspicious Activity" Detected';
138+
const ruleNameB64 = Buffer.from(ruleName, 'utf-8').toString('base64');
139+
140+
const inputs = [
141+
`{ "risk_score": "80", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "c2lnbmFs", "id": "test_id_1" }`,
142+
];
143+
144+
const esqlResultRow = [1, 80, inputs, 'hostname'];
145+
const bucket = buildRiskScoreBucket(
146+
EntityType.host,
147+
'.alerts-security.alerts-default'
148+
)(esqlResultRow as FieldValue[]);
149+
150+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(ruleName);
151+
});
152+
153+
it('handles rule names with backslashes', () => {
154+
const ruleName = 'C:\\Windows\\System32\\malware.exe';
155+
const ruleNameB64 = Buffer.from(ruleName, 'utf-8').toString('base64');
156+
157+
const inputs = [
158+
`{ "risk_score": "90", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "c2lnbmFs", "id": "test_id_1" }`,
159+
];
160+
161+
const esqlResultRow = [1, 90, inputs, 'hostname'];
162+
const bucket = buildRiskScoreBucket(
163+
EntityType.host,
164+
'.alerts-security.alerts-default'
165+
)(esqlResultRow as FieldValue[]);
166+
167+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(ruleName);
168+
});
169+
170+
it('handles rule names with newlines and tabs', () => {
171+
const ruleName = 'Multi\nLine\tRule';
172+
const ruleNameB64 = Buffer.from(ruleName, 'utf-8').toString('base64');
173+
174+
const inputs = [
175+
`{ "risk_score": "85", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "c2lnbmFs", "id": "test_id_1" }`,
176+
];
177+
178+
const esqlResultRow = [1, 85, inputs, 'hostname'];
179+
const bucket = buildRiskScoreBucket(
180+
EntityType.host,
181+
'.alerts-security.alerts-default'
182+
)(esqlResultRow as FieldValue[]);
183+
184+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(ruleName);
185+
});
186+
187+
it('handles rule names with mixed special characters', () => {
188+
const ruleName = 'Alert: "Path\\To\\File"\nWith Newline\tAnd Tab';
189+
const category = 'Category with "quotes" and \\backslashes\\';
190+
const ruleNameB64 = Buffer.from(ruleName, 'utf-8').toString('base64');
191+
const categoryB64 = Buffer.from(category, 'utf-8').toString('base64');
192+
193+
const inputs = [
194+
`{ "risk_score": "95", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "${categoryB64}", "id": "test_id_1" }`,
195+
];
196+
197+
const esqlResultRow = [1, 95, inputs, 'hostname'];
198+
const bucket = buildRiskScoreBucket(
199+
EntityType.host,
200+
'.alerts-security.alerts-default'
201+
)(esqlResultRow as FieldValue[]);
202+
203+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(ruleName);
204+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].category).toBe(category);
205+
});
206+
207+
it('handles Unicode characters', () => {
208+
const ruleName = 'Alert: 你好世界 🔥 Émojis';
209+
const ruleNameB64 = Buffer.from(ruleName, 'utf-8').toString('base64');
210+
211+
const inputs = [
212+
`{ "risk_score": "70", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "c2lnbmFs", "id": "test_id_1" }`,
213+
];
214+
215+
const esqlResultRow = [1, 70, inputs, 'hostname'];
216+
const bucket = buildRiskScoreBucket(
217+
EntityType.host,
218+
'.alerts-security.alerts-default'
219+
)(esqlResultRow as FieldValue[]);
220+
221+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(ruleName);
222+
});
223+
});
224+
225+
describe('Backward compatibility', () => {
226+
it('handles old format without Base64 encoding (rule_name without _b64 suffix)', () => {
227+
const inputs = [
228+
'{ "risk_score": "50", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name": "Old Format Rule", "category": "signal", "id": "test_id_1" }',
229+
];
230+
const alertCount = 1;
231+
const riskScore = 50;
232+
const entityValue = 'hostname';
233+
234+
const esqlResultRow = [alertCount, riskScore, inputs, entityValue];
235+
236+
const bucket = buildRiskScoreBucket(
237+
EntityType.host,
238+
'.alerts-security.alerts-default'
239+
)(esqlResultRow as FieldValue[]);
240+
241+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(
242+
'Old Format Rule'
243+
);
244+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].category).toBe('signal');
245+
});
246+
247+
it('prefers Base64 encoded fields over plain fields when both exist', () => {
248+
const correctRuleName = 'Rule Name like this would make life so much easier';
249+
const ruleNameB64 = Buffer.from(correctRuleName, 'utf-8').toString('base64');
250+
251+
const inputs = [
252+
`{ "risk_score": "60", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name": "Wrong Name", "rule_name_b64": "${ruleNameB64}", "category": "wrong", "category_b64": "Y29ycmVjdA==", "id": "test_id_1" }`,
253+
];
254+
255+
const esqlResultRow = [1, 60, inputs, 'hostname'];
256+
const bucket = buildRiskScoreBucket(
257+
EntityType.host,
258+
'.alerts-security.alerts-default'
259+
)(esqlResultRow as FieldValue[]);
260+
261+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(correctRuleName);
262+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].category).toBe('correct');
263+
});
264+
});
265+
266+
describe('Multiple inputs with mixed formats', () => {
267+
it('handles array of inputs with both Base64 and plain text', () => {
268+
const ruleNameB64 = Buffer.from('Test "Quoted" Alert', 'utf-8').toString('base64');
269+
const inputs = [
270+
`{ "risk_score": "75", "time": "2021-08-23T18:00:05.000Z", "index": ".alerts-security.alerts-default", "rule_name_b64": "${ruleNameB64}", "category_b64": "c2lnbmFs", "id": "test_id_1" }`,
271+
'{ "risk_score": "50", "time": "2021-08-22T18:00:04.000Z", "index": ".alerts-security.alerts-default", "rule_name": "Plain Rule", "category": "signal", "id": "test_id_2" }',
272+
];
273+
274+
const esqlResultRow = [2, 125, inputs, 'hostname'];
275+
const bucket = buildRiskScoreBucket(
276+
EntityType.host,
277+
'.alerts-security.alerts-default'
278+
)(esqlResultRow as FieldValue[]);
279+
280+
expect(bucket.top_inputs.risk_details.value.risk_inputs).toHaveLength(2);
281+
expect(bucket.top_inputs.risk_details.value.risk_inputs[0].rule_name).toBe(
282+
'Test "Quoted" Alert'
283+
);
284+
expect(bucket.top_inputs.risk_details.value.risk_inputs[1].rule_name).toBe('Plain Rule');
285+
});
286+
});
103287
});
104288
});

‎x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/risk_score/calculate_esql_risk_scores.ts‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* 2.0.
66
*/
77

8-
import { isEmpty } from 'lodash';
8+
import { isEmpty, omit } from 'lodash';
99
import type { FieldValue, QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
1010
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
1111
import { fromKueryExpression, toElasticsearchQuery } from '@kbn/es-query';
@@ -366,7 +366,9 @@ export const getESQL = (
366366
kibana.alert.uuid as alert_id,
367367
event.kind as category,
368368
@timestamp as time
369-
| EVAL input = CONCAT(""" {"risk_score": """", risk_score::keyword, """", "time": """", time::keyword, """", "index": """", _index, """", "rule_name": """", rule_name, """\", "category": """", category, """\", "id": \"""", alert_id, """\" } """)
369+
| EVAL rule_name_b64 = TO_BASE64(rule_name),
370+
category_b64 = TO_BASE64(category)
371+
| EVAL input = CONCAT(""" {"risk_score": """", risk_score::keyword, """", "time": """", time::keyword, """", "index": """", _index, """", "rule_name_b64": """", rule_name_b64, """\", "category_b64": """", category_b64, """\", "id": \"""", alert_id, """\" } """)
370372
| STATS
371373
alert_count = count(risk_score),
372374
scores = MV_PSERIES_WEIGHTED_SUM(TOP(risk_score, ${sampleSize}, "desc"), ${RIEMANN_ZETA_S_VALUE}),
@@ -390,12 +392,43 @@ export const buildRiskScoreBucket =
390392
];
391393

392394
const inputs = (Array.isArray(_inputs) ? _inputs : [_inputs]).map((input, i) => {
393-
const parsedRiskInputData = JSON.parse(input);
395+
let parsedRiskInputData = JSON.parse('{}');
396+
let ruleName: string | undefined;
397+
let category: string | undefined;
398+
399+
try {
400+
// Parse JSON and decode Base64 encoded fields to handle special characters (quotes, backslashes, newlines, etc.)
401+
parsedRiskInputData = JSON.parse(input);
402+
403+
ruleName = parsedRiskInputData.rule_name_b64
404+
? Buffer.from(parsedRiskInputData.rule_name_b64, 'base64').toString('utf-8')
405+
: parsedRiskInputData.rule_name; // Fallback for backward compatibility
406+
category = parsedRiskInputData.category_b64
407+
? Buffer.from(parsedRiskInputData.category_b64, 'base64').toString('utf-8')
408+
: parsedRiskInputData.category; // Fallback for backward compatibility
409+
} catch {
410+
// Attempt to use fallback values if parsedRiskInputData was parsed but decoding failed
411+
if (parsedRiskInputData && Object.keys(parsedRiskInputData).length > 0) {
412+
ruleName = parsedRiskInputData.rule_name;
413+
category = parsedRiskInputData.category;
414+
}
415+
}
416+
394417
const value = parseFloat(parsedRiskInputData.risk_score);
395418
const currentScore = value / Math.pow(i + 1, RIEMANN_ZETA_S_VALUE);
396-
const { risk_score: _, ...otherFields } = parsedRiskInputData;
419+
const otherFields = omit(parsedRiskInputData, [
420+
'risk_score',
421+
'rule_name',
422+
'rule_name_b64',
423+
'category',
424+
'category_b64',
425+
]);
426+
397427
return {
428+
id: parsedRiskInputData.id,
398429
...otherFields,
430+
rule_name: ruleName,
431+
category,
399432
score: value,
400433
contribution: currentScore / RIEMANN_ZETA_VALUE,
401434
index,

‎x-pack/solutions/security/plugins/security_solution/server/lib/entity_analytics/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export interface SearchHitRiskInput {
4848
id: string;
4949
index: string;
5050
rule_name?: string;
51+
category?: string;
5152
time?: string;
5253
score?: number;
5354
contribution?: number;

0 commit comments

Comments
 (0)