refactor(streams): rename verdict to status in sig events UI labels

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: crespocarlos

x-pack/platform/packages/shared/kbn-streams-schema/src/sig_events/discoveries/index.ts

@@ -13,12 +13,19 @@ import {
   evidenceSchema,
 } from '../common_schemas';
 
+import { MAX_STREAM_NAME_LENGTH } from '../../helpers/stream_name_validation';
+
+const MAX_ID_LENGTH = 256;
+const MAX_RULE_NAME_LENGTH = 256;
+const MAX_TITLE_LENGTH = 512;
+const MAX_TEXT_LENGTH = 10_000;
+
 const discoveryDetectionSchema = z.object({
-  detection_id: z.string().optional(),
-  rule_name: z.string().optional(),
-  rule_uuid: z.string().optional(),
-  stream_name: z.string().optional(),
-  change_point_type: z.string().optional(),
+  detection_id: z.string().max(MAX_ID_LENGTH).optional(),
+  rule_name: z.string().max(MAX_RULE_NAME_LENGTH).optional(),
+  rule_uuid: z.string().max(MAX_ID_LENGTH).optional(),
+  stream_name: z.string().max(MAX_STREAM_NAME_LENGTH).optional(),
+  change_point_type: z.string().max(MAX_ID_LENGTH).optional(),
   event_count: z.number().optional(),
   alert_count: z.number().optional(),
   detected_at: z.string().optional(),
@@ -27,31 +34,30 @@ const discoveryDetectionSchema = z.object({
 export const discoverySchema = z.object({
   '@timestamp': z.iso.datetime(),
   kind: z.enum(['finding', 'clearance']),
-  discovery_id: z.string(),
-  discovery_slug: z.string(),
+  discovery_id: z.string().max(MAX_ID_LENGTH),
+  discovery_slug: z.string().max(MAX_ID_LENGTH),
   discovered_at: z.iso.datetime().optional(),
-  rule_names: z.array(z.string()),
-  stream_names: z.array(z.string()),
-  title: z.string(),
-  summary: z.string(),
-  root_cause: z.string(),
+  rule_names: z.array(z.string().max(MAX_RULE_NAME_LENGTH)),
+  stream_names: z.array(z.string().max(MAX_STREAM_NAME_LENGTH)),
+  title: z.string().max(MAX_TITLE_LENGTH),
+  summary: z.string().max(MAX_TEXT_LENGTH),
+  root_cause: z.string().max(MAX_TEXT_LENGTH),
   criticality: z.number(),
   confidence: z.number(),
-  impact: z.string(),
+  impact: z.string().max(MAX_TEXT_LENGTH),
   detections: z.array(discoveryDetectionSchema),
   dependency_edges: z.array(dependencyEdgeSchema).optional(),
   infra_components: z.array(infraComponentSchema).optional(),
   cause_kis: z.array(causeKiSchema).optional(),
   evidences: z.array(evidenceSchema).optional(),
-  closes: z.string().optional(),
-  grouped_into: z.string().optional(),
-  grouped_discovery_ids: z.array(z.string()).optional(),
-  grouping_rationale: z.string().optional(),
-  previous_discovery_id: z.string().optional(),
-  change_point_occurrence: z.string().optional(),
-  workflow_execution_id: z.string().optional(),
-  conversation_id: z.string().optional(),
-  closed_by_execution_id: z.string().optional(),
+  closes_discovery_id: z.string().max(MAX_ID_LENGTH).optional(),
+  grouped_discovery_ids: z.array(z.string().max(MAX_ID_LENGTH)).optional(),
+  grouping_rationale: z.string().max(MAX_TEXT_LENGTH).optional(),
+  previous_discovery_id: z.string().max(MAX_ID_LENGTH).optional(),
+  change_point_occurrence: z.string().max(MAX_ID_LENGTH).optional(),
+  workflow_execution_id: z.string().max(MAX_ID_LENGTH).optional(),
+  conversation_id: z.string().max(MAX_ID_LENGTH).optional(),
+  closed_by_execution_id: z.string().max(MAX_ID_LENGTH).optional(),
 });
 
 export type Discovery = z.infer<typeof discoverySchema>;

x-pack/platform/packages/shared/kbn-streams-schema/src/sig_events/events/index.ts

@@ -20,6 +20,7 @@ export const sigEventSchema = z.object({
   discovery_id: z.string(),
   discovery_slug: z.string(),
   previous_event_id: z.string().optional(),
+  // TODO: rename to status once the data stream field is renamed
   verdict: z.string(),
   workflow_execution_id: z.string(),
   rule_names: z.array(z.string()),
@@ -37,13 +38,10 @@ export const sigEventSchema = z.object({
   cause_kis: z.array(causeKiSchema).optional(),
   evidences: z.array(evidenceSchema).optional(),
   grouped_into: z.string().optional(),
-  grouped_discovery_ids: z.array(z.string()).optional(),
+  // TODO: rename once the data stream fields are renamed
   // Audit fields merged from verdict docs
   verdict_summary: z.string().optional(),
   assessment_note: z.string().optional(),
-  verdict_source: z.string().optional(),
-  original_verdict: z.string().optional(),
-  conversation_id: z.string().optional(),
 });
 
 export type SigEvent = z.infer<typeof sigEventSchema>;

x-pack/platform/plugins/shared/streams/server/lib/sig_events/detections/detection_client.ts

@@ -7,14 +7,15 @@
 
 import type { IDataStreamClient } from '@kbn/data-streams';
 import { esql } from '@elastic/esql';
+import type { ESQLAstExpression } from '@elastic/esql/types';
 import type { ElasticsearchClient } from '@kbn/core/server';
 import {
   type CommonSearchOptions,
   type PaginatedSearchOptions,
   type PaginatedResponse,
 } from '../query_utils';
 import {
-  type LatestSourceWhereCondition,
+  andWhere,
   runLatestSourceEsqlQuery,
   runPaginatedLatestSourceEsqlQuery,
   runFindByIdEsqlQuery,
@@ -27,6 +28,7 @@ import {
   type StoredDetection,
   type detectionsMappings,
 } from './data_stream';
+import { FIELD_DETECTION_ID } from '../field_names';
 
 export type DetectionDataStreamClient = IDataStreamClient<
   typeof detectionsMappings,
@@ -47,14 +49,6 @@ export interface DetectionsPaginatedSearchOptions extends PaginatedSearchOptions
   rule_name?: string;
 }
 
-const andWhere = (
-  current: LatestSourceWhereCondition | undefined,
-  next: LatestSourceWhereCondition
-): LatestSourceWhereCondition => {
-  return current ? esql.exp`${current} AND ${next}` : next;
-};
-
-const GROUP_BY_FIELD = 'detection_id';
 const KIND_HANDLED = 'handled' satisfies Detection['kind'];
 const KIND_QUIET = 'quiet' satisfies Detection['kind'];
 const PROCESSED_IDS_CHUNK_SIZE = 250;
@@ -77,10 +71,8 @@ export class DetectionClient {
 
   // Exclude kind:handled from the main query — handled docs are pipeline stamps,
   // not anomaly state. processed is derived separately via getProcessedIds.
-  private buildWhere(options: DetectionsSearchOptions): LatestSourceWhereCondition {
-    let where: LatestSourceWhereCondition = esql.exp`${esql.col('kind')} != ${esql.str(
-      KIND_HANDLED
-    )}`;
+  private buildWhere(options: DetectionsSearchOptions): ESQLAstExpression {
+    let where: ESQLAstExpression = esql.exp`${esql.col('kind')} != ${esql.str(KIND_HANDLED)}`;
 
     const ruleUuidLiterals = options.rule_uuid?.map((ruleUuid) => esql.str(ruleUuid));
     if (ruleUuidLiterals?.length) {
@@ -136,7 +128,7 @@ export class DetectionClient {
       options,
       index: DETECTIONS_DATA_STREAM,
       where: this.buildWhere(options),
-      groupBy: GROUP_BY_FIELD,
+      groupBy: FIELD_DETECTION_ID,
     });
     const processedIds = await this.getProcessedIds(
       result.hits.map((h) => h.detection_id).filter((id): id is string => Boolean(id))
@@ -157,8 +149,8 @@ export class DetectionClient {
       options,
       index: DETECTIONS_DATA_STREAM,
       where: this.buildWhere(options),
-      groupBy: GROUP_BY_FIELD,
-      sort: [['detected_at', 'DESC']],
+      groupBy: FIELD_DETECTION_ID,
+      sort: [['@timestamp', 'DESC']],
     });
     const processedIds = await this.getProcessedIds(
       result.hits.map((h) => h.detection_id).filter((id): id is string => Boolean(id))
@@ -176,7 +168,7 @@ export class DetectionClient {
       esClient: this.clients.esClient,
       space: this.clients.space,
       index: DETECTIONS_DATA_STREAM,
-      idField: GROUP_BY_FIELD,
+      idField: FIELD_DETECTION_ID,
       idValue: detectionId,
     });
     // History returns all doc kinds including kind:handled.

x-pack/platform/plugins/shared/streams/server/lib/sig_events/discoveries/discovery_client.ts

@@ -19,21 +19,23 @@ import {
   runFindByIdEsqlQuery,
   queryEsql,
   esqlToObjects,
+  runFindByIdsEsqlQuery,
 } from '../latest_source_query';
 import {
   DISCOVERIES_DATA_STREAM,
   type Discovery,
   type StoredDiscovery,
   type discoveriesMappings,
 } from './data_stream';
+import { FIELD_DISCOVERY_ID, FIELD_DISCOVERY_SLUG } from '../field_names';
+
+const CLEARED_IDS_CHUNK_SIZE = 250;
 
 export type DiscoveryDataStreamClient = IDataStreamClient<
   typeof discoveriesMappings,
   StoredDiscovery
 >;
 
-const GROUP_BY_FIELD = 'discovery_slug';
-
 export class DiscoveryClient {
   constructor(
     private readonly clients: {
@@ -56,7 +58,7 @@ export class DiscoveryClient {
       space: this.clients.space,
       options,
       index: DISCOVERIES_DATA_STREAM,
-      groupBy: GROUP_BY_FIELD,
+      groupBy: FIELD_DISCOVERY_ID,
     });
   }
 
@@ -68,53 +70,96 @@ export class DiscoveryClient {
       space: this.clients.space,
       options,
       index: DISCOVERIES_DATA_STREAM,
-      groupBy: GROUP_BY_FIELD,
+      groupBy: FIELD_DISCOVERY_ID,
+      where: esql.exp`${esql.col('kind')} == ${esql.str('finding')}`,
     });
 
-    if (result.hits.length === 0) return result;
+    if (!result.hits.length) return result;
+
+    const clearedIds = await this.getClearedIds(
+      result.hits.map((h) => h.discovery_id).filter((id): id is string => Boolean(id))
+    );
 
-    const discoveredAtMap = await this.getDiscoveredAtMap(options);
     return {
       ...result,
       hits: result.hits.map((h) => ({
         ...h,
-        discovered_at: discoveredAtMap.get(h.discovery_slug) ?? h['@timestamp'],
+        kind: clearedIds.has(h.discovery_id ?? '') ? ('clearance' as const) : h.kind,
       })),
     };
   }
 
-  // Returns MIN(@timestamp) of kind:finding documents per discovery_slug for the given time range.
-  // A finding always precedes any clearance for the same slug, so MIN(@timestamp) = first investigation time.
-  private async getDiscoveredAtMap(options: CommonSearchOptions): Promise<Map<string, string>> {
-    try {
-      let query = esql.from([DISCOVERIES_DATA_STREAM]).where`${esql.col('kibana.space_ids')} == ${
-        this.clients.space
-      } OR ${esql.col('kibana.space_ids')} IS NULL`;
+  // Returns the set of finding IDs that have been cleared.
+  // Mirrors getProcessedIds in detection_client: a finding is cleared only when the latest
+  // clearance doc timestamp is on or after the latest finding doc timestamp, so re-opened
+  // findings (no newer clearance) are not reported as cleared.
+  // Chunked at CLEARED_IDS_CHUNK_SIZE to match the getProcessedIds IN-clause guard.
+  private async getClearedIds(findingIds: string[]): Promise<Set<string>> {
+    if (!findingIds.length) return new Set();
 
-      if (options.from !== undefined) {
-        query = query.where`@timestamp >= TO_DATETIME(${esql.str(options.from)})`;
-      }
-      if (options.to !== undefined) {
-        query = query.where`@timestamp <= TO_DATETIME(${esql.str(options.to)})`;
+    const cleared = new Set<string>();
+    for (let i = 0; i < findingIds.length; i += CLEARED_IDS_CHUNK_SIZE) {
+      const batch = findingIds.slice(i, i + CLEARED_IDS_CHUNK_SIZE);
+      const idLiterals = batch.map((id) => esql.str(id));
+      const kindFinding = esql.str('finding');
+      const kindClearance = esql.str('clearance');
+      // Use EVAL to normalize both doc types to the same "finding ID" for grouping:
+      // finding docs: unified_id = discovery_id
+      // clearance docs: unified_id = closes_discovery_id (references the original finding)
+      const query = esql`FROM ${DISCOVERIES_DATA_STREAM}
+        | WHERE ${esql.col('kibana.space_ids')} == ${esql.str(this.clients.space)} OR ${esql.col(
+        'kibana.space_ids'
+      )} IS NULL
+        | WHERE ${esql.col('kind')} IN (${[kindFinding, kindClearance]})
+        | WHERE ${esql.col('discovery_id')} IN (${idLiterals}) OR ${esql.col(
+        'closes_discovery_id'
+      )} IN (${idLiterals})
+        | EVAL unified_id = CASE(${esql.col('kind')} == ${kindFinding}, ${esql.col(
+        'discovery_id'
+      )}, ${esql.col('closes_discovery_id')})
+        | STATS max_finding_ts = MAX(CASE(${esql.col('kind')} == ${kindFinding}, @timestamp, null)),
+                max_clearance_ts = MAX(CASE(${esql.col(
+                  'kind'
+                )} == ${kindClearance}, @timestamp, null))
+          BY unified_id
+        | WHERE max_clearance_ts >= max_finding_ts OR max_finding_ts IS NULL
+        | WHERE unified_id IS NOT NULL
+        | KEEP unified_id`;
+      const response = await queryEsql({ esClient: this.clients.esClient, query });
+      const rows = esqlToObjects<{ unified_id: string }>(response);
+      for (const r of rows) {
+        if (r.unified_id) cleared.add(r.unified_id);
       }
+    }
+    return cleared;
+  }
 
-      query = query.where`${esql.col('kind')} == ${esql.str('finding')}`;
-      query = query.pipe`STATS discovered_at = MIN(@timestamp) BY ${esql.col('discovery_slug')}`;
+  async findById(discoveryId: string): Promise<{ hits: Discovery[] }> {
+    return runFindByIdEsqlQuery<Discovery>({
+      esClient: this.clients.esClient,
+      space: this.clients.space,
+      index: DISCOVERIES_DATA_STREAM,
+      idField: FIELD_DISCOVERY_ID,
+      idValue: discoveryId,
+    });
+  }
 
-      const response = await queryEsql({ esClient: this.clients.esClient, query });
-      const rows = esqlToObjects<{ discovery_slug: string; discovered_at: string }>(response);
-      return new Map(rows.map((r) => [r.discovery_slug, r.discovered_at]));
-    } catch (error) {
-      return new Map();
-    }
+  async findByIds(discoveryIds: string[]): Promise<{ hits: Discovery[] }> {
+    return runFindByIdsEsqlQuery<Discovery>({
+      esClient: this.clients.esClient,
+      space: this.clients.space,
+      index: DISCOVERIES_DATA_STREAM,
+      idField: FIELD_DISCOVERY_ID,
+      idValues: discoveryIds,
+    });
   }
 
   async findBySlug(slug: string): Promise<{ hits: Discovery[] }> {
     return runFindByIdEsqlQuery<Discovery>({
       esClient: this.clients.esClient,
       space: this.clients.space,
       index: DISCOVERIES_DATA_STREAM,
-      idField: 'discovery_slug',
+      idField: FIELD_DISCOVERY_SLUG,
       idValue: slug,
     });
   }

x-pack/platform/plugins/shared/streams/server/lib/sig_events/events/data_stream.ts

@@ -25,10 +25,6 @@ export const eventsMappings = {
     verdict: mappings.keyword(),
     verdict_summary: mappings.text(),
     assessment_note: mappings.text(),
-    verdict_source: mappings.keyword(),
-    original_verdict: mappings.keyword(),
-    conversation_id: mappings.keyword(),
-    grouped_discovery_ids: mappings.keyword(),
     impact: mappings.keyword(),
     criticality: mappings.integer(),
     confidence: mappings.integer(),
@@ -37,9 +33,6 @@ export const eventsMappings = {
     root_cause: mappings.text(),
     recommended_action: mappings.text(),
     recommendations: mappings.text(),
-    workflow_execution_id: mappings.keyword(),
-    created_at: mappings.date({ format: 'strict_date_optional_time' }),
-    grouped_into: mappings.keyword(),
   },
 } satisfies MappingsDefinition;
 

x-pack/platform/plugins/shared/streams/server/lib/sig_events/events/event_client.ts

@@ -28,6 +28,7 @@ import {
   type eventsMappings,
 } from './data_stream';
 import { FIELD_EVENT_ID, FIELD_DISCOVERY_SLUG } from '../field_names';
+import { enrichFromEvidences } from '../utils';
 
 export type EventDataStreamClient = IDataStreamClient<typeof eventsMappings, StoredEvent>;
 
@@ -39,19 +40,6 @@ export interface EventsFilterOptions {
 
 export interface EventsPaginatedSearchOptions extends PaginatedSearchOptions, EventsFilterOptions {}
 
-function enrichFromEvidences(e: SigEvent): SigEvent {
-  const evidences = e.evidences ?? [];
-  const streamNames = e.stream_names?.length
-    ? e.stream_names
-    : [...new Set(evidences.map((ev) => ev.stream_name).filter((s): s is string => !!s))];
-  const ruleNames = e.rule_names?.length
-    ? e.rule_names
-    : [...new Set(evidences.map((ev) => ev.rule_name).filter((s): s is string => !!s))];
-
-  if (streamNames === e.stream_names && ruleNames === e.rule_names) return e;
-  return { ...e, stream_names: streamNames, rule_names: ruleNames };
-}
-
 export class EventClient {
   constructor(
     private readonly clients: {

x-pack/platform/plugins/shared/streams/server/routes/internal/sig_events/events/route.ts

@@ -143,7 +143,7 @@ const eventsLifecycleRoute = createServerRoute({
     access: 'internal',
     summary: 'Get event lifecycle',
     description:
-      'Get the full lifecycle chain for a significant event: detections, discoveries, verdicts, and event versions.',
+      'Get the full lifecycle chain for a significant event: detections, discoveries, and event versions.',
   },
   security: {
     authz: {