Skip to content

Commit 0e7f25b

Browse files
authored
[Security Solution][Endpoint] Better error handling for ES errors (elastic#266524)
## Summary - Improve error handling of errors, specifically those thrown by ES/SO clients - Logic in `wrapErrorIfNeeded()` will now check to see if the error being wrapped is from ES and if so, it will attempt to generate a better error message as well as capture additional debug data
1 parent 4c31d96 commit 0e7f25b

4 files changed

Lines changed: 262 additions & 6 deletions

File tree

x-pack/solutions/security/plugins/security_solution/common/endpoint/errors.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
* for the error.
1111
*/
1212
export class EndpointError<MetaType = unknown> extends Error {
13+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
14+
public debug: any = undefined;
15+
1316
constructor(message: string, public readonly meta?: MetaType) {
1417
super(message);
1518
// For debugging - capture name of subclasses

x-pack/solutions/security/plugins/security_solution/server/endpoint/routes/error_handler.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
import type { IKibanaResponse, KibanaResponseFactory, Logger } from '@kbn/core/server';
99
import { FleetFileNotFound } from '@kbn/fleet-plugin/server/errors';
10+
import { stringify } from '../utils/stringify';
1011
import { CustomHttpRequestError } from '../../utils/custom_http_request_error';
1112
import { EndpointAuthorizationError, EndpointHttpError, NotFoundError } from '../errors';
1213
import { EndpointHostUnEnrolledError, EndpointHostNotFoundError } from '../services/metadata';
@@ -27,9 +28,9 @@ export const errorHandler = <E extends Error>(
2728
};
2829

2930
if (shouldLogToDebug()) {
30-
logger.debug(error.message);
31+
logger.debug(() => stringify(error, 20), { error });
3132
} else {
32-
logger.error(error);
33+
logger.error(stringify(error, 20), { error });
3334
}
3435

3536
if (error instanceof CustomHttpRequestError || error instanceof EndpointHttpError) {
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { AgentNotFoundError } from '@kbn/fleet-plugin/server';
9+
import {
10+
AgentPolicyNotFoundError,
11+
PackagePolicyNotFoundError,
12+
} from '@kbn/fleet-plugin/server/errors';
13+
import { errors } from '@elastic/elasticsearch';
14+
import { EndpointError } from '../../../common/endpoint/errors';
15+
import { NotFoundError } from '../errors';
16+
import { catchAndWrapError, wrapErrorIfNeeded } from './wrap_errors';
17+
18+
describe('wrapErrorIfNeeded', () => {
19+
it('returns the same instance when already an EndpointError', () => {
20+
const original = new EndpointError('already wrapped');
21+
expect(wrapErrorIfNeeded(original)).toBe(original);
22+
});
23+
24+
it('wraps a plain Error in EndpointError', () => {
25+
const original = new Error('plain error');
26+
const result = wrapErrorIfNeeded(original);
27+
28+
expect(result).toBeInstanceOf(EndpointError);
29+
expect(result.message).toBe('plain error');
30+
expect(result.meta).toBe(original);
31+
});
32+
33+
it('applies messagePrefix to a plain Error', () => {
34+
const original = new Error('something went wrong');
35+
const result = wrapErrorIfNeeded(original, 'context');
36+
37+
expect(result).toBeInstanceOf(EndpointError);
38+
expect(result.message).toBe('context: something went wrong');
39+
});
40+
41+
describe('Fleet Not Found errors', () => {
42+
it('wraps AgentNotFoundError in NotFoundError', () => {
43+
const original = new AgentNotFoundError('agent not found');
44+
const result = wrapErrorIfNeeded(original);
45+
46+
expect(result).toBeInstanceOf(NotFoundError);
47+
expect(result.message).toBe('agent not found');
48+
expect(result.meta).toBe(original);
49+
});
50+
51+
it('wraps AgentPolicyNotFoundError in NotFoundError', () => {
52+
const original = new AgentPolicyNotFoundError('policy not found');
53+
const result = wrapErrorIfNeeded(original);
54+
55+
expect(result).toBeInstanceOf(NotFoundError);
56+
expect(result.message).toBe('policy not found');
57+
});
58+
59+
it('wraps PackagePolicyNotFoundError in NotFoundError', () => {
60+
const original = new PackagePolicyNotFoundError('package policy not found');
61+
const result = wrapErrorIfNeeded(original);
62+
63+
expect(result).toBeInstanceOf(NotFoundError);
64+
expect(result.message).toBe('package policy not found');
65+
});
66+
67+
it('applies messagePrefix to Fleet Not Found errors', () => {
68+
const original = new AgentNotFoundError('agent-123');
69+
const result = wrapErrorIfNeeded(original, 'fetch agent');
70+
71+
expect(result).toBeInstanceOf(NotFoundError);
72+
expect(result.message).toBe('fetch agent: agent-123');
73+
});
74+
});
75+
76+
describe('Elasticsearch errors', () => {
77+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
78+
const buildResponseError = (body: any, statusCode = 404) =>
79+
new errors.ResponseError({
80+
body,
81+
statusCode,
82+
headers: {},
83+
warnings: null,
84+
// @ts-expect-error
85+
meta: {
86+
body,
87+
statusCode,
88+
headers: {},
89+
context: {},
90+
request: {
91+
params: { method: 'GET', path: '/_search', querystring: {}, body: undefined },
92+
options: {},
93+
id: 'test-request',
94+
},
95+
attempts: 1,
96+
aborted: false,
97+
} as unknown as errors.ResponseError['meta'],
98+
});
99+
100+
it('wraps ElasticsearchClientError in EndpointError', () => {
101+
const esError = buildResponseError({ error: { type: 'index_not_found_exception' } });
102+
const result = wrapErrorIfNeeded(esError);
103+
104+
expect(result).toBeInstanceOf(EndpointError);
105+
});
106+
107+
it('builds a descriptive message from ES response reason fields', () => {
108+
const esError = buildResponseError({
109+
error: {
110+
type: 'index_not_found_exception',
111+
reason: 'no such index [.fleet-agents]',
112+
index: '.fleet-agents',
113+
},
114+
});
115+
const result = wrapErrorIfNeeded(esError);
116+
117+
expect(result.message).toContain('no such index [.fleet-agents]');
118+
});
119+
120+
it('applies messagePrefix to ES errors', () => {
121+
const esError = buildResponseError({
122+
error: { type: 'search_phase_execution_exception', reason: 'shard failed' },
123+
});
124+
const result = wrapErrorIfNeeded(esError, 'search failed');
125+
126+
expect(result.message).toMatch(/^search failed:/);
127+
});
128+
129+
it('populates debug.es_request with request parameters', () => {
130+
const esError = buildResponseError({ error: { reason: 'not found' } });
131+
const result = wrapErrorIfNeeded(esError);
132+
133+
expect(result.debug).toBeDefined();
134+
expect(result.debug.es_request).toMatchObject({
135+
method: 'GET',
136+
path: '/_search',
137+
});
138+
});
139+
});
140+
});
141+
142+
describe('catchAndWrapError', () => {
143+
it('rejects with an EndpointError wrapping the original error', async () => {
144+
const original = new Error('async failure');
145+
await expect(Promise.reject(original).catch(catchAndWrapError)).rejects.toBeInstanceOf(
146+
EndpointError
147+
);
148+
});
149+
150+
it('rejects with the same EndpointError when already one', async () => {
151+
const original = new EndpointError('already wrapped');
152+
await expect(Promise.reject(original).catch(catchAndWrapError)).rejects.toBe(original);
153+
});
154+
155+
describe('withMessage', () => {
156+
it('rejects with an EndpointError whose message includes the custom prefix', async () => {
157+
const original = new Error('downstream failure');
158+
await expect(
159+
Promise.reject(original).catch(catchAndWrapError.withMessage('custom prefix'))
160+
).rejects.toMatchObject({
161+
message: 'custom prefix: downstream failure',
162+
});
163+
});
164+
165+
it('wraps Fleet Not Found errors in NotFoundError with prefix', async () => {
166+
const original = new AgentNotFoundError('not found');
167+
await expect(
168+
Promise.reject(original).catch(catchAndWrapError.withMessage('get agent'))
169+
).rejects.toBeInstanceOf(NotFoundError);
170+
});
171+
});
172+
});

x-pack/solutions/security/plugins/security_solution/server/endpoint/utils/wrap_errors.ts

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,22 @@
55
* 2.0.
66
*/
77

8+
/* eslint-disable @typescript-eslint/no-explicit-any */
9+
810
import { AgentNotFoundError } from '@kbn/fleet-plugin/server';
911
import {
1012
AgentPolicyNotFoundError,
1113
PackagePolicyNotFoundError,
1214
} from '@kbn/fleet-plugin/server/errors';
13-
import { NotFoundError } from '../errors';
15+
import { errors, type DiagnosticResult } from '@elastic/elasticsearch';
16+
import { isPlainObject } from 'lodash';
1417
import { EndpointError } from '../../../common/endpoint/errors';
18+
import { NotFoundError } from '../errors';
1519

1620
/**
1721
* Will wrap the given Error with `EndpointError`, which will help getting a good picture of where in
18-
* our code the error originated (better stack trace).
22+
* our code the error originated (better stack trace). It will also process some known error types
23+
* and build a more descriptive error message and add additional `debug` details to the error object.
1924
*/
2025
export const wrapErrorIfNeeded = <E extends EndpointError = EndpointError>(
2126
error: Error,
@@ -25,7 +30,79 @@ export const wrapErrorIfNeeded = <E extends EndpointError = EndpointError>(
2530
return error as E;
2631
}
2732

28-
const message = `${messagePrefix ? `${messagePrefix}: ` : ''}${error.message}`;
33+
let debug: EndpointError['debug'];
34+
let message = `${messagePrefix ? `${messagePrefix}: ` : ''}${error.message}`;
35+
36+
try {
37+
// Process known error Types and retrieve additional data not normally output to logs
38+
if (error instanceof errors.ElasticsearchClientError) {
39+
const esError = error as { meta?: DiagnosticResult; body?: any };
40+
41+
debug = {
42+
es_request: {
43+
method: esError.meta?.meta?.request?.params?.method,
44+
path: esError.meta?.meta?.request?.params?.path,
45+
querystring: esError.meta?.meta?.request?.params?.querystring,
46+
body: esError.meta?.meta?.request?.params?.body,
47+
},
48+
es_response: {
49+
body: esError.body,
50+
},
51+
};
52+
53+
// Since this is an elasticsearch client error, lets build a better error message
54+
// that is based on the Elasticsearch error response body
55+
56+
const queue: any[] = [debug.es_response.body];
57+
let newMessage = '';
58+
59+
// The most common Elasticsearch error response structure seems to be something like:
60+
// {
61+
// error?: {
62+
// type: string; // e.g., 'index_not_found_exception'
63+
// reason: string; // Human-readable message
64+
// caused_by?: {
65+
// type?: string;
66+
// reason?: string;
67+
// caused_by?: { ... } // Recursive chain
68+
// };
69+
// root_cause?: Array<{ // Array of root causes
70+
// type?: string;
71+
// reason?: string;
72+
// }>;
73+
// };
74+
// status?: number; // HTTP status code
75+
// }
76+
// So we'll loop through all this data and grab the string values for 'reason'
77+
while (queue.length > 0) {
78+
const record = queue.shift();
79+
80+
if (Array.isArray(record)) {
81+
queue.push(...record);
82+
} else if (isPlainObject(record)) {
83+
Object.entries(record).forEach(([key, value]) => {
84+
if (isPlainObject(value) || Array.isArray(value)) {
85+
queue.push(value);
86+
} else if (key === 'reason') {
87+
newMessage += (newMessage.length > 0 ? ' > ' : '') + value;
88+
89+
if (record.index) {
90+
newMessage += ` (index: ${record.index})`;
91+
}
92+
}
93+
});
94+
}
95+
}
96+
97+
if (newMessage.length > 0) {
98+
message = `${
99+
messagePrefix ? `${messagePrefix}: ` : ''
100+
}Elasticsearch error encountered: ${newMessage}`;
101+
}
102+
}
103+
} catch (_) {
104+
/* best effort - failures are ignored */
105+
}
29106

30107
// Check for known "Not Found" errors and wrap them with our own `NotFoundError`, which will enable
31108
// the correct HTTP status code to be used if it is thrown during processing of an API route
@@ -37,7 +114,10 @@ export const wrapErrorIfNeeded = <E extends EndpointError = EndpointError>(
37114
return new NotFoundError(message, error) as E;
38115
}
39116

40-
return new EndpointError(message, error) as E;
117+
const err = new EndpointError(message, error) as E;
118+
err.debug = debug;
119+
120+
return err;
41121
};
42122

43123
interface CatchAndWrapError {

0 commit comments

Comments
 (0)