[SigEvents][Discovery] sig events UI refactor (elastic#271329)

## Summary

Refactors the Significant Events discovery UI adapting to the recent
workflow changes.

**UI changes:**

### Detections
<img width="1727" height="618" alt="image"
src="https://github.com/user-attachments/assets/f3784f21-7656-485b-bf2b-2fef1c151d4d"
/>
<img width="1727" height="743" alt="image"
src="https://github.com/user-attachments/assets/422eff6b-b53a-4d18-a40d-2286bd56e826"
/>


### Discoveries
<img width="1725" height="624" alt="image"
src="https://github.com/user-attachments/assets/f764ea5a-15c4-4638-87fe-5d36ec0ce236"
/>
<img width="1726" height="738" alt="image"
src="https://github.com/user-attachments/assets/e86de8d7-cb68-43f9-8288-c35e2887f583"
/>

### Verdicts (TO BE REMOVED SOON)
<img width="1728" height="472" alt="image"
src="https://github.com/user-attachments/assets/bf46a2b6-c4f2-4812-907f-eecd3ff1cac5"
/>

>[!NOTE]
> Sigevents tab is being worked on here
elastic#271380


### Checklist

- [x] Any text added follows EUI's writing guidelines, uses sentence
case text and includes i18n support
- [ ] Documentation was added for features that require explanation or
tutorials
- [ ] Unit or functional tests were updated or added to match the most
common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the docker list
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] Flaky Test Runner was used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the guidelines
- [ ] Review the backport guidelines and apply applicable `backport:*`
labels.

🤖 Co-authored with AI assistance.

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

Author: 3 people

x-pack/platform/packages/shared/kbn-streams-schema/src/sig_events/detections/index.ts

@@ -14,7 +14,8 @@ const detectionEvidenceSchema = z.object({
 
 export const detectionSchema = z.object({
   '@timestamp': z.iso.datetime(),
-  silent: z.boolean(),
+  detected_at: z.iso.datetime().optional(),
+  kind: z.enum(['detection', 'quiet', 'handled']),
   processed: z.boolean(),
   detection_id: z.string().optional(),
   rule_uuid: z.string(),
@@ -24,7 +25,7 @@ export const detectionSchema = z.object({
   alert_index: z.string().optional(),
   workflow_execution_id: z.string().optional(),
   resolution_lookback_minutes: z.number().optional(),
-  peak_30m_alert_count: z.number().optional(),
+  peak_alert_count: z.number().optional(),
   detection_evidence: detectionEvidenceSchema.optional(),
   alert_samples: z.array(z.record(z.string(), z.unknown())).optional(),
   rules_activity: z.array(z.record(z.string(), z.unknown())).optional(),

x-pack/platform/packages/shared/kbn-streams-schema/src/sig_events/discoveries/index.ts

@@ -12,44 +12,51 @@ import {
   causeKiSchema,
   evidenceSchema,
 } from '../common_schemas';
+import { MAX_STREAM_NAME_LENGTH } from '../../helpers/stream_name_validation';
+
+const MAX_ID_LENGTH = 256;
+const MAX_RULE_NAME_LENGTH = 256;
+const MAX_TITLE_LENGTH = 512;
+const MAX_TEXT_LENGTH = 10_000;
 
 const discoveryDetectionSchema = z.object({
-  detection_id: z.string().optional(),
-  rule_name: z.string().optional(),
-  rule_uuid: z.string().optional(),
-  stream_name: z.string().optional(),
-  change_point_type: z.string().optional(),
+  detection_id: z.string().max(MAX_ID_LENGTH).optional(),
+  rule_name: z.string().max(MAX_RULE_NAME_LENGTH).optional(),
+  rule_uuid: z.string().max(MAX_ID_LENGTH).optional(),
+  stream_name: z.string().max(MAX_STREAM_NAME_LENGTH).optional(),
+  change_point_type: z.string().max(MAX_ID_LENGTH).optional(),
   event_count: z.number().optional(),
+  alert_count: z.number().optional(),
   detected_at: z.string().optional(),
 });
 
 export const discoverySchema = z.object({
   '@timestamp': z.iso.datetime(),
-  kind: z.string(),
-  discovery_id: z.string(),
-  discovery_slug: z.string(),
-  rule_names: z.array(z.string()),
-  stream_names: z.array(z.string()),
-  title: z.string(),
-  summary: z.string(),
-  root_cause: z.string(),
+  kind: z.enum(['finding', 'clearance']),
+  discovery_id: z.string().max(MAX_ID_LENGTH),
+  discovery_slug: z.string().max(MAX_ID_LENGTH),
+  discovered_at: z.iso.datetime().optional(),
+  rule_names: z.array(z.string().max(MAX_RULE_NAME_LENGTH)),
+  stream_names: z.array(z.string().max(MAX_STREAM_NAME_LENGTH)),
+  title: z.string().max(MAX_TITLE_LENGTH),
+  summary: z.string().max(MAX_TEXT_LENGTH),
+  root_cause: z.string().max(MAX_TEXT_LENGTH),
   criticality: z.number(),
   confidence: z.number(),
-  impact: z.string(),
+  impact: z.string().max(MAX_TEXT_LENGTH),
   detections: z.array(discoveryDetectionSchema),
   dependency_edges: z.array(dependencyEdgeSchema).optional(),
   infra_components: z.array(infraComponentSchema).optional(),
   cause_kis: z.array(causeKiSchema).optional(),
   evidences: z.array(evidenceSchema).optional(),
-  closes: z.string().optional(),
-  grouped_into: z.string().optional(),
-  grouped_discovery_ids: z.array(z.string()).optional(),
-  grouping_rationale: z.string().optional(),
-  previous_discovery_id: z.string().optional(),
-  change_point_occurrence: z.string().optional(),
-  workflow_execution_id: z.string().optional(),
-  conversation_id: z.string().optional(),
-  closed_by_execution_id: z.string().optional(),
+  closes_discovery_id: z.string().max(MAX_ID_LENGTH).optional(),
+  grouped_discovery_ids: z.array(z.string().max(MAX_ID_LENGTH)).optional(),
+  grouping_rationale: z.string().max(MAX_TEXT_LENGTH).optional(),
+  previous_discovery_id: z.string().max(MAX_ID_LENGTH).optional(),
+  change_point_occurrence: z.string().max(MAX_ID_LENGTH).optional(),
+  workflow_execution_id: z.string().max(MAX_ID_LENGTH).optional(),
+  conversation_id: z.string().max(MAX_ID_LENGTH).optional(),
+  closed_by_execution_id: z.string().max(MAX_ID_LENGTH).optional(),
 });
 
 export type Discovery = z.infer<typeof discoverySchema>;

x-pack/platform/packages/shared/kbn-streams-schema/src/sig_events/verdicts/index.ts

@@ -12,20 +12,23 @@ import {
   causeKiSchema,
   evidenceSchema,
 } from '../common_schemas';
+import { MAX_STREAM_NAME_LENGTH } from '../../helpers/stream_name_validation';
+
+const MAX_RULE_NAME_LENGTH = 256;
 
 export const verdictSchema = z.object({
   '@timestamp': z.iso.datetime(),
-  verdict: z.string(),
+  verdict: z.enum(['promoted', 'demoted', 'acknowledged']),
   verdict_id: z.string().optional(),
   discovery_id: z.string(),
   discovery_slug: z.string(),
-  rule_names: z.array(z.string()),
-  stream_names: z.array(z.string()),
+  rule_names: z.array(z.string().max(MAX_RULE_NAME_LENGTH)).optional(),
+  stream_names: z.array(z.string().max(MAX_STREAM_NAME_LENGTH)).optional(),
   title: z.string(),
   summary: z.string(),
   root_cause: z.string(),
   criticality: z.number(),
-  confidence: z.number(),
+  confidence: z.number().optional(),
   impact: z.string().optional(),
   recommended_action: z.string().optional(),
   recommendations: z.array(z.string()).optional(),

x-pack/platform/plugins/shared/streams/server/lib/sig_events/detections/data_stream.ts

@@ -16,14 +16,15 @@ export const detectionsMappings = {
   dynamic: false,
   properties: {
     '@timestamp': mappings.date({ format: 'strict_date_optional_time' }),
-    silent: mappings.boolean(),
-    processed: mappings.boolean(),
+    detected_at: mappings.date({ format: 'strict_date_optional_time' }),
+    kind: mappings.keyword(),
     detection_id: mappings.keyword(),
     rule_uuid: mappings.keyword(),
     rule_name: mappings.keyword(),
-    peak_30m_alert_count: mappings.long(),
+    peak_alert_count: mappings.long(),
     detection_evidence: mappings.object({
       properties: {
+        change_point_type: mappings.keyword(),
         p_value: { type: 'double' as const },
       },
     }),
@@ -38,7 +39,7 @@ export const detectionsDataStream: DataStreamDefinition<
   StoredDetection
 > = {
   name: DETECTIONS_DATA_STREAM,
-  version: 3,
+  version: 4,
   hidden: true,
   template: {
     priority: 500,

x-pack/platform/plugins/shared/streams/server/lib/sig_events/detections/detection_client.ts

@@ -16,10 +16,10 @@ import {
 } from '../query_utils';
 import {
   andWhere,
-  inFilter,
   runLatestSourceEsqlQuery,
   runPaginatedLatestSourceEsqlQuery,
-  runFindByIdEsqlQuery,
+  queryEsql,
+  esqlToObjects,
 } from '../latest_source_query';
 import {
   DETECTIONS_DATA_STREAM,
@@ -34,6 +34,10 @@ export type DetectionDataStreamClient = IDataStreamClient<
   StoredDetection
 >;
 
+// _source holds all stored fields; `processed` is derived at query time, not stored.
+// Using Omit<Detection, 'processed'> avoids the string | string[] widening from GetFieldsOf.
+type RawDetection = Omit<Detection, 'processed'>;
+
 export interface DetectionsSearchOptions extends CommonSearchOptions {
   rule_uuid?: string[];
   rule_name?: string;
@@ -44,6 +48,10 @@ export interface DetectionsPaginatedSearchOptions extends PaginatedSearchOptions
   rule_name?: string;
 }
 
+const KIND_HANDLED = 'handled' satisfies Detection['kind'];
+const KIND_QUIET = 'quiet' satisfies Detection['kind'];
+const PROCESSED_IDS_CHUNK_SIZE = 250;
+
 export class DetectionClient {
   constructor(
     private readonly clients: {
@@ -60,9 +68,15 @@ export class DetectionClient {
     });
   }
 
-  private buildWhere(options: DetectionsSearchOptions): ESQLAstExpression | undefined {
-    let where: ESQLAstExpression | undefined;
-    where = inFilter({ where, field: 'rule_uuid', values: options.rule_uuid });
+  // Exclude kind:handled from the main query — handled docs are pipeline stamps,
+  // not anomaly state. processed is derived separately via getProcessedIds.
+  private buildWhere(options: DetectionsSearchOptions): ESQLAstExpression {
+    let where: ESQLAstExpression = esql.exp`${esql.col('kind')} != ${esql.str(KIND_HANDLED)}`;
+
+    const ruleUuidLiterals = options.rule_uuid?.map((ruleUuid) => esql.str(ruleUuid));
+    if (ruleUuidLiterals?.length) {
+      where = andWhere(where, esql.exp`${esql.col('rule_uuid')} IN (${ruleUuidLiterals})`);
+    }
 
     if (options.rule_name) {
       where = andWhere(where, esql.exp`${esql.col('rule_name')} == ${esql.str(options.rule_name)}`);
@@ -71,37 +85,81 @@ export class DetectionClient {
     return where;
   }
 
+  // Returns detection_ids where the latest kind:handled timestamp is on or after the latest
+  // kind:detection OR kind:quiet timestamp, or where only handled docs exist.
+  // Chunked to avoid oversized ES|QL IN clauses when many detections are on screen.
+  private async getProcessedIds(detectionIds: string[]): Promise<Set<string>> {
+    if (!detectionIds.length) return new Set();
+
+    const processed = new Set<string>();
+    for (let i = 0; i < detectionIds.length; i += PROCESSED_IDS_CHUNK_SIZE) {
+      const batch = detectionIds.slice(i, i + PROCESSED_IDS_CHUNK_SIZE);
+      const idLiterals = batch.map((id) => esql.str(id));
+      const kindState = [esql.str('detection'), esql.str(KIND_QUIET)];
+      const allKinds = [...kindState, esql.str(KIND_HANDLED)];
+      const query = esql`FROM ${DETECTIONS_DATA_STREAM}
+        | WHERE kibana.space_ids == ${esql.str(this.clients.space)} OR kibana.space_ids IS NULL
+        | WHERE kind IN (${allKinds})
+        | WHERE detection_id IN (${idLiterals})
+        | STATS max_state_ts = MAX(CASE(kind IN (${kindState}), @timestamp, null)),
+                max_handled_ts = MAX(CASE(kind == ${esql.str(KIND_HANDLED)}, @timestamp, null))
+          BY detection_id
+        | WHERE max_handled_ts >= max_state_ts OR max_state_ts IS NULL
+        | KEEP detection_id`;
+
+      const response = await queryEsql({ esClient: this.clients.esClient, query });
+      const rows = esqlToObjects<{ detection_id?: string }>(response);
+      for (const r of rows) {
+        if (r.detection_id) processed.add(r.detection_id);
+      }
+    }
+    return processed;
+  }
+
+  private toDetection(raw: RawDetection, processed: boolean): Detection {
+    return { ...raw, processed };
+  }
+
   async findLatest(options: DetectionsSearchOptions = {}): Promise<{ hits: Detection[] }> {
-    return runLatestSourceEsqlQuery<Detection>({
+    const result = await runLatestSourceEsqlQuery<RawDetection>({
       esClient: this.clients.esClient,
       space: this.clients.space,
       options,
       index: DETECTIONS_DATA_STREAM,
       where: this.buildWhere(options),
       groupBy: FIELD_DETECTION_ID,
     });
+    const processedIds = await this.getProcessedIds(
+      result.hits.map((h) => h.detection_id).filter((id): id is string => Boolean(id))
+    );
+    return {
+      hits: result.hits.map((raw) =>
+        this.toDetection(raw, processedIds.has(raw.detection_id ?? ''))
+      ),
+    };
   }
 
   async findLatestPaginated(
     options: DetectionsPaginatedSearchOptions = {}
   ): Promise<PaginatedResponse<Detection>> {
-    return runPaginatedLatestSourceEsqlQuery<Detection>({
+    const result = await runPaginatedLatestSourceEsqlQuery<RawDetection>({
       esClient: this.clients.esClient,
       space: this.clients.space,
       options,
       index: DETECTIONS_DATA_STREAM,
       where: this.buildWhere(options),
       groupBy: FIELD_DETECTION_ID,
+      sort: [['@timestamp', 'DESC']],
     });
-  }
 
-  async findById(detectionId: string): Promise<{ hits: Detection[] }> {
-    return runFindByIdEsqlQuery<Detection>({
-      esClient: this.clients.esClient,
-      space: this.clients.space,
-      index: DETECTIONS_DATA_STREAM,
-      idField: FIELD_DETECTION_ID,
-      idValue: detectionId,
-    });
+    const processedIds = await this.getProcessedIds(
+      result.hits.map((h) => h.detection_id).filter((id): id is string => Boolean(id))
+    );
+    return {
+      ...result,
+      hits: result.hits.map((raw) =>
+        this.toDetection(raw, processedIds.has(raw.detection_id ?? ''))
+      ),
+    };
   }
 }

x-pack/platform/plugins/shared/streams/server/lib/sig_events/discoveries/data_stream.ts

@@ -19,10 +19,15 @@ export const discoveriesMappings = {
     kind: mappings.keyword(),
     discovery_id: mappings.keyword(),
     discovery_slug: mappings.keyword(),
-    closes: mappings.keyword(),
-    grouped_into: mappings.keyword(),
+    closes_discovery_id: mappings.keyword(),
+    grouped_discovery_ids: mappings.keyword(),
     criticality: mappings.integer(),
     closed_by_execution_id: mappings.keyword(),
+    detections: mappings.object({
+      properties: {
+        rule_uuid: { type: 'keyword' as const },
+      },
+    }),
   },
 } satisfies MappingsDefinition;
 
@@ -34,7 +39,7 @@ export const discoveriesDataStream: DataStreamDefinition<
   StoredDiscovery
 > = {
   name: DISCOVERIES_DATA_STREAM,
-  version: 2,
+  version: 3,
   hidden: true,
   template: {
     priority: 500,