Skip to content

Commit 521a5ff

Browse files
tonyghianikibanamachineruflin
authored
[SigEvents] Add description to KI (Knowledge Indicators) queries (elastic#257457)
## 📓 Summary This PR adds a `description` field to significant event queries so that each query carries a natural-language explanation of what it detects and why it matters. The LLM now generates descriptions alongside queries during the AI generation flow, and users can view and edit descriptions through the manual creation form, the generated event preview, and the discovery query details flyout. Existing queries without a description are migrated on read via the storage adapter's `migrateSource` callback, which back-fills an empty string. ## 🧪 Testing - Create a significant event manually via the stream detail flyout and verify the description field appears, can be filled in, and is persisted after save. - Trigger AI generation and confirm generated queries include descriptions visible in the expanded preview; edit a description and verify the change is saved. - Open an existing query (created before this change) in the discovery details flyout and confirm it shows an empty description placeholder without errors; edit and save a description from there. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit ## Release Notes * **New Features** * Added description field to queries throughout the system, allowing users to provide contextual information about what each query detects and why it matters. * Enabled editing and viewing of query descriptions in UI components including forms, flyouts, and detail panels. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Nicolas Ruflin <ruflin@elastic.co>
1 parent 08d5b6e commit 521a5ff

27 files changed

Lines changed: 259 additions & 38 deletions

File tree

‎oas_docs/output/kibana.serverless.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62175,6 +62175,8 @@ paths:
6217562175
items:
6217662176
type: object
6217762177
properties:
62178+
description:
62179+
type: string
6217862180
esql:
6217962181
type: object
6218062182
properties:
@@ -62199,6 +62201,7 @@ paths:
6219962201
required:
6220062202
- id
6220162203
- title
62204+
- description
6220262205
- esql
6220362206
type: array
6220462207
rules:
@@ -62267,6 +62270,8 @@ paths:
6226762270
items:
6226862271
type: object
6226962272
properties:
62273+
description:
62274+
type: string
6227062275
esql:
6227162276
type: object
6227262277
properties:
@@ -62291,6 +62296,7 @@ paths:
6229162296
required:
6229262297
- id
6229362298
- title
62299+
- description
6229462300
- esql
6229562301
type: array
6229662302
rules:
@@ -62366,6 +62372,8 @@ paths:
6236662372
items:
6236762373
type: object
6236862374
properties:
62375+
description:
62376+
type: string
6236962377
esql:
6237062378
type: object
6237162379
properties:
@@ -62390,6 +62398,7 @@ paths:
6239062398
required:
6239162399
- id
6239262400
- title
62401+
- description
6239362402
- esql
6239462403
type: array
6239562404
rules:
@@ -62458,6 +62467,8 @@ paths:
6245862467
items:
6245962468
type: object
6246062469
properties:
62470+
description:
62471+
type: string
6246162472
esql:
6246262473
type: object
6246362474
properties:
@@ -62482,6 +62493,7 @@ paths:
6248262493
required:
6248362494
- id
6248462495
- title
62496+
- description
6248562497
- esql
6248662498
type: array
6248762499
rules:
@@ -62557,6 +62569,8 @@ paths:
6255762569
items:
6255862570
type: object
6255962571
properties:
62572+
description:
62573+
type: string
6256062574
esql:
6256162575
type: object
6256262576
properties:
@@ -62581,6 +62595,7 @@ paths:
6258162595
required:
6258262596
- id
6258362597
- title
62598+
- description
6258462599
- esql
6258562600
type: array
6258662601
rules:
@@ -63497,6 +63512,8 @@ paths:
6349763512
index:
6349863513
type: object
6349963514
properties:
63515+
description:
63516+
type: string
6350063517
esql:
6350163518
type: object
6350263519
properties:
@@ -63521,6 +63538,7 @@ paths:
6352163538
required:
6352263539
- id
6352363540
- title
63541+
- description
6352463542
- esql
6352563543
required:
6352663544
- index
@@ -63628,6 +63646,9 @@ paths:
6362863646
additionalProperties: false
6362963647
type: object
6363063648
properties:
63649+
description:
63650+
default: ''
63651+
type: string
6363163652
esql:
6363263653
additionalProperties: false
6363363654
type: object

‎oas_docs/output/kibana.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66633,6 +66633,8 @@ paths:
6663366633
items:
6663466634
type: object
6663566635
properties:
66636+
description:
66637+
type: string
6663666638
esql:
6663766639
type: object
6663866640
properties:
@@ -66657,6 +66659,7 @@ paths:
6665766659
required:
6665866660
- id
6665966661
- title
66662+
- description
6666066663
- esql
6666166664
type: array
6666266665
rules:
@@ -66725,6 +66728,8 @@ paths:
6672566728
items:
6672666729
type: object
6672766730
properties:
66731+
description:
66732+
type: string
6672866733
esql:
6672966734
type: object
6673066735
properties:
@@ -66749,6 +66754,7 @@ paths:
6674966754
required:
6675066755
- id
6675166756
- title
66757+
- description
6675266758
- esql
6675366759
type: array
6675466760
rules:
@@ -66824,6 +66830,8 @@ paths:
6682466830
items:
6682566831
type: object
6682666832
properties:
66833+
description:
66834+
type: string
6682766835
esql:
6682866836
type: object
6682966837
properties:
@@ -66848,6 +66856,7 @@ paths:
6684866856
required:
6684966857
- id
6685066858
- title
66859+
- description
6685166860
- esql
6685266861
type: array
6685366862
rules:
@@ -66916,6 +66925,8 @@ paths:
6691666925
items:
6691766926
type: object
6691866927
properties:
66928+
description:
66929+
type: string
6691966930
esql:
6692066931
type: object
6692166932
properties:
@@ -66940,6 +66951,7 @@ paths:
6694066951
required:
6694166952
- id
6694266953
- title
66954+
- description
6694366955
- esql
6694466956
type: array
6694566957
rules:
@@ -67015,6 +67027,8 @@ paths:
6701567027
items:
6701667028
type: object
6701767029
properties:
67030+
description:
67031+
type: string
6701867032
esql:
6701967033
type: object
6702067034
properties:
@@ -67039,6 +67053,7 @@ paths:
6703967053
required:
6704067054
- id
6704167055
- title
67056+
- description
6704267057
- esql
6704367058
type: array
6704467059
rules:
@@ -67955,6 +67970,8 @@ paths:
6795567970
index:
6795667971
type: object
6795767972
properties:
67973+
description:
67974+
type: string
6795867975
esql:
6795967976
type: object
6796067977
properties:
@@ -67979,6 +67996,7 @@ paths:
6797967996
required:
6798067997
- id
6798167998
- title
67999+
- description
6798268000
- esql
6798368001
required:
6798468002
- index
@@ -68086,6 +68104,9 @@ paths:
6808668104
additionalProperties: false
6808768105
type: object
6808868106
properties:
68107+
description:
68108+
default: ''
68109+
type: string
6808968110
esql:
6809068111
additionalProperties: false
6809168112
type: object

‎x-pack/platform/packages/shared/kbn-streams-ai/src/significant_events/generate_significant_events.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import {
3030
interface Query {
3131
esql: string;
3232
title: string;
33+
description: string;
3334
category: SignificantEventType;
3435
severity_score: number;
3536
evidence?: string[];

‎x-pack/platform/packages/shared/kbn-streams-ai/src/significant_events/prompt.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ export function createGenerateSignificantEventsPrompt({ systemPrompt }: { system
8383
title: {
8484
type: 'string',
8585
},
86+
description: {
87+
type: 'string',
88+
description:
89+
'A semantically searchable description explaining what the query detects and why it matters. Should be 1-2 sentences that help users find this query when searching by concept or intent.',
90+
},
8691
category: {
8792
type: 'string',
8893
enum: [
@@ -105,7 +110,7 @@ export function createGenerateSignificantEventsPrompt({ systemPrompt }: { system
105110
},
106111
},
107112
},
108-
required: ['esql', 'title', 'category', 'severity_score'],
113+
required: ['esql', 'title', 'description', 'category', 'severity_score'],
109114
},
110115
},
111116
},

‎x-pack/platform/packages/shared/kbn-streams-ai/src/significant_events/system_prompt.text‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Your primary goal is to analyze the provided context about a user's log dataset
1414
* **Coverage:** You generate queries for multiple relevant categories (`operational`, `error`, etc.) where the data provides justification.
1515
* **Best Practices:** You correctly prioritize the `body.text` field over `message` when available and avoid querying on unmapped fields.
1616
* **Feature-Aware:** You leverage extracted features to create more targeted queries. This includes technology-specific error patterns, entity-scoped queries when multiple components are present, and dependency-aware monitoring for inter-component communication paths.
17+
* **Descriptive:** Each query includes a `description` — a 1-2 sentence natural-language explanation of what the query detects and why it matters. Descriptions should be semantically rich so users can find queries by searching for concepts (e.g., "memory pressure", "authentication failure") rather than exact titles.
1718
* **Conciseness:** The final, user-facing output is a brief, natural-language summary (2-3 sentences) of the queries you've created.
1819

1920
---
@@ -114,7 +115,7 @@ Schema features indicate the **log schema family** (ecs, otel, or custom) detect
114115
| Tool | Function | Notes |
115116
| :--- | :--- | :--- |
116117
| `get_stream_features` | Fetches stream features for this stream. | **MUST be called first before any `add_queries` call.** Call without filters to get all features at once. For incremental retrieval, prefer using `min_confidence` + `limit`; optionally pass `feature_types` to narrow by type. Results include computed features (dataset_analysis, log_samples, log_patterns, error_logs) and inferred features. Supported values: `{{{available_feature_types}}}`. |
117-
| `add_queries` | Submits one or more ES|QL queries for the user. | Payload is a list of objects, each with `title`, `esql`, `category`, `severity_score`, and optional `evidence`. |
118+
| `add_queries` | Submits one or more ES|QL queries for the user. | Payload is a list of objects, each with `title`, `description`, `esql`, `category`, `severity_score`, and optional `evidence`. |
118119
| `reason()` | **Begin a Reasoning Monologue** | Outputs your private thoughts. Must use sentinel tags (`<<<BEGIN_INTERNAL>>>`...`<<<END_INTERNAL>>>`). |
119120
| `complete()` | Declare readiness to answer | Ends the loop and triggers the **Definitive Output**. |
120121

@@ -152,15 +153,15 @@ PLAN> Describe your next action in natural language. If you are ready to answer,
152153

153154
### Example 1: Calling `add_queries` with a single query (with evidence)
154155

155-
`>>> ACTION: add_queries(queries=[{"title": "View all errors", "esql": "FROM logs | WHERE error.message IS NOT NULL", "category": "error", "severity_score": 60, "evidence": ["error.message: \"Connection timeout after 30s\""]}])`
156+
`>>> ACTION: add_queries(queries=[{"title": "View all errors", "description": "Detects all log entries with an error message, useful for identifying connection timeouts and other failures.", "esql": "FROM logs | WHERE error.message IS NOT NULL", "category": "error", "severity_score": 60, "evidence": ["error.message: \"Connection timeout after 30s\""]}])`
156157

157158
### Example 2: Calling `add_queries` with multiple queries
158159

159160
```
160161
>>> ACTION: add_queries(queries=[
161-
{"title": "Application startup", "esql": "FROM logs | WHERE MATCH_PHRASE(message, \"Started Application\")", "category": "operational", "severity_score": 25, "evidence": ["message: \"Started Application in 2.3 seconds\""]},
162-
{"title": "Failed login attempts", "esql": "FROM logs | WHERE MATCH_PHRASE(body.text, \"Failed password for\")", "category": "security", "severity_score": 75, "evidence": ["body.text: \"Failed password for invalid user guest from 10.0.0.1\""]},
163-
{"title": "Out of memory errors", "esql": "FROM logs | WHERE body.text:\"OutOfMemoryError\"", "category": "error", "severity_score": 85}
162+
{"title": "Application startup", "description": "Tracks Spring Boot application startup events to monitor deployment health and boot timing.", "esql": "FROM logs | WHERE MATCH_PHRASE(message, \"Started Application\")", "category": "operational", "severity_score": 25, "evidence": ["message: \"Started Application in 2.3 seconds\""]},
163+
{"title": "Failed login attempts", "description": "Identifies failed SSH or system login attempts that may indicate brute-force attacks or unauthorized access.", "esql": "FROM logs | WHERE MATCH_PHRASE(body.text, \"Failed password for\")", "category": "security", "severity_score": 75, "evidence": ["body.text: \"Failed password for invalid user guest from 10.0.0.1\""]},
164+
{"title": "Out of memory errors", "description": "Detects Java OutOfMemoryError exceptions that signal JVM heap exhaustion and potential application crashes.", "esql": "FROM logs | WHERE body.text:\"OutOfMemoryError\"", "category": "error", "severity_score": 85}
164165
])
165166
```
166167

‎x-pack/platform/packages/shared/kbn-streams-ai/src/significant_events/task_description.text‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,6 @@ Use ES|QL to generate the queries. Refer to the ES|QL Query Reference at the end
162162

163163
Available tools:
164164
- `get_stream_features`: Fetch stream features. For incremental retrieval, prefer `min_confidence` and `limit` (for example top 5 by confidence), and optionally pass `feature_types` to narrow by type. Returned features can include computed features (`dataset_analysis`, `log_samples`, `log_patterns`, `error_logs`) when they match the request. Supported `feature_types` values are injected at runtime in `available_feature_types`: `{{{available_feature_types}}}`.
165-
- `add_queries`: Add one or more query suggestions (`title`, `esql`, `category`, `severity_score`, optional `evidence`).
165+
- `add_queries`: Add one or more query suggestions (`title`, `description`, `esql`, `category`, `severity_score`, optional `evidence`).
166166

167167
Every call to `add_queries` returns success/failure details for each query, including syntax errors. You can and should add multiple queries in one go. Only call `add_queries` again if you wish to retry invalid queries or add additional high-value queries. Once you're satisfied with the result, call `complete()`.

‎x-pack/platform/packages/shared/kbn-streams-schema/src/api/significant_events/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ interface GeneratedSignificantEventQuery {
5959
esql: EsqlQuery;
6060
severity_score: number;
6161
evidence?: string[];
62+
description: string;
6263
}
6364

6465
type SignificantEventsGenerateResponse = Observable<

‎x-pack/platform/packages/shared/kbn-streams-schema/src/queries/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export const esqlQuerySchema: z.Schema<EsqlQuery> = z.object({
2222
interface StreamQueryBase {
2323
id: string;
2424
title: string;
25+
description: string;
2526
}
2627

2728
export interface StreamQuery extends StreamQueryBase {
@@ -34,6 +35,7 @@ export interface StreamQuery extends StreamQueryBase {
3435
const streamQueryBaseSchema = z.object({
3536
id: NonEmptyString,
3637
title: NonEmptyString,
38+
description: z.string(),
3739
}) satisfies z.Schema<StreamQueryBase>;
3840

3941
export const streamQuerySchema: z.Schema<StreamQuery> = streamQueryBaseSchema.extend({
@@ -54,6 +56,7 @@ export const upsertStreamQueryRequestSchema = z.object({
5456
esql: esqlQuerySchema,
5557
severity_score: z.number().optional(),
5658
evidence: z.array(z.string()).optional(),
59+
description: z.string().default(''),
5760
});
5861

5962
export interface QueriesGetResponse {

‎x-pack/platform/plugins/shared/streams/server/lib/content/stream/export.test.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const streams = [
2222
{
2323
id: 'logs-query',
2424
title: 'logs-query',
25+
description: '',
2526
esql: { query: 'FROM logs | WHERE level == "error"' },
2627
},
2728
],
@@ -37,6 +38,7 @@ const streams = [
3738
{
3839
id: 'hello-query',
3940
title: 'hello-query',
41+
description: '',
4042
esql: { query: 'FROM logs | WHERE greeting == "hello"' },
4143
},
4244
],
@@ -64,6 +66,7 @@ describe('content pack export', () => {
6466
{
6567
id: 'logs-query',
6668
title: 'logs-query',
69+
description: '',
6770
esql: { query: 'FROM logs | WHERE level == "error"' },
6871
},
6972
],
@@ -79,6 +82,7 @@ describe('content pack export', () => {
7982
{
8083
id: 'hello-query',
8184
title: 'hello-query',
85+
description: '',
8286
esql: { query: 'FROM logs | WHERE greeting == "hello"' },
8387
},
8488
],
@@ -120,6 +124,7 @@ describe('content pack export', () => {
120124
{
121125
id: 'hello-query',
122126
title: 'hello-query',
127+
description: '',
123128
esql: { query: 'FROM logs | WHERE greeting == "hello"' },
124129
},
125130
],

0 commit comments

Comments
 (0)