[Security Solution][Detection Engine] Fix enrichment tests for Entity Store V2 on MKI

x-pack/solutions/security/test/security_solution_api_integration/config/ess/config.base.ts

@@ -110,7 +110,6 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s
             'previewTelemetryUrlEnabled',
             'endpointExceptionsMovedUnderManagement',
             'ruleChangesHistoryEnabled',
-            'disable:entityAnalyticsEntityStoreV2',
           ])}`,
           '--xpack.alerting.ruleChangeTracking.enabled=true',
           `--plugin-path=${path.resolve(

x-pack/solutions/security/test/security_solution_api_integration/config/serverless/config.base.ts

@@ -49,7 +49,6 @@ export function createTestConfig(options: CreateTestConfigOptions) {
           `--xpack.securitySolution.enableExperimental=${JSON.stringify([
             'endpointExceptionsMovedUnderManagement',
             'ruleChangesHistoryEnabled',
-            'disable:entityAnalyticsEntityStoreV2',
           ])}`,
           '--xpack.alerting.ruleChangeTracking.enabled=true',
           ...(options.kbnTestServerArgs || []),

x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/entity_store_v2_enrichment_setup.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { FtrProviderContext } from '../../../../ftr_provider_context';
+
+const PUBLIC_API_HEADERS: ReadonlyArray<[string, string]> = [
+  ['kbn-xsrf', 'true'],
+  ['x-elastic-internal-origin', 'Kibana'],
+  ['elastic-api-version', '2023-10-31'],
+];
+
+const withHeaders = <T extends { set: (key: string, value: string) => T }>(req: T): T =>
+  PUBLIC_API_HEADERS.reduce((acc, [k, v]) => acc.set(k, v), req);
+
+export interface HostEntityConfig {
+  host: Record<string, unknown>;
+  entity?: Record<string, unknown>;
+  asset?: Record<string, unknown>;
+}
+
+export interface UserEntityConfig {
+  user: Record<string, unknown>;
+  entity?: Record<string, unknown>;
+  asset?: Record<string, unknown>;
+}
+
+export interface EnrichmentSetupConfig {
+  hosts?: HostEntityConfig[];
+  users?: UserEntityConfig[];
+}
+
+/**
+ * Installs Entity Store V2 engines and seeds entities for alert enrichment tests.
+ *
+ * Inspired by the approach started in https://github.com/elastic/kibana/pull/270939
+ * by @denar50.
+ */
+export const EntityStoreV2EnrichmentSetup = (getService: FtrProviderContext['getService']) => {
+  const supertest = getService('supertest');
+  const retry = getService('retry');
+  const log = getService('log');
+
+  const setup = async (enrichmentConfig: EnrichmentSetupConfig): Promise<void> => {
+    const entityTypes: string[] = [];
+    if (enrichmentConfig.hosts?.length) entityTypes.push('host');
+    if (enrichmentConfig.users?.length) entityTypes.push('user');
+
+    if (entityTypes.length === 0) return;
+
+    const installRes = await withHeaders(supertest.post('/api/security/entity_store/install')).send(
+      { entityTypes }
+    );
+
+    if (installRes.status !== 200 && installRes.status !== 201) {
+      throw new Error(
+        `Entity Store V2 install failed (status ${installRes.status}): ${JSON.stringify(
+          installRes.body
+        )}`
+      );
+    }
+
+    await retry.waitForWithTimeout('Entity Store V2 to be ready', 60_000, async () => {
+      const res = await withHeaders(supertest.get('/api/security/entity_store/status'));
+      if (res.body.status === 'error') {
+        throw new Error(`Entity Store V2 install errored: ${JSON.stringify(res.body)}`);
+      }
+      return res.body.status === 'running' || res.body.status === 'stopped';
+    });
+
+    for (const hostConfig of enrichmentConfig.hosts ?? []) {
+      const createRes = await withHeaders(
+        supertest.post('/api/security/entity_store/entities/host')
+      ).send(hostConfig);
+      if (createRes.status !== 200) {
+        throw new Error(
+          `Create host entity failed (status ${createRes.status}): ${JSON.stringify(
+            createRes.body
+          )}`
+        );
+      }
+    }
+
+    for (const userConfig of enrichmentConfig.users ?? []) {
+      const createRes = await withHeaders(
+        supertest.post('/api/security/entity_store/entities/user')
+      ).send(userConfig);
+      if (createRes.status !== 200) {
+        throw new Error(
+          `Create user entity failed (status ${createRes.status}): ${JSON.stringify(
+            createRes.body
+          )}`
+        );
+      }
+    }
+  };
+
+  const teardown = async (): Promise<void> => {
+    try {
+      await withHeaders(supertest.post('/api/security/entity_store/uninstall')).send({
+        entityTypes: ['user', 'host', 'service'],
+      });
+    } catch (err) {
+      log.debug(`Entity Store V2 uninstall skipped: ${err instanceof Error ? err.message : err}`);
+    }
+  };
+
+  return { setup, teardown };
+};

x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql.ts

@@ -63,6 +63,7 @@ import {
 import type { FtrProviderContext } from '../../../../../../ftr_provider_context';
 import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder';
 import { getMetricsRequest, getMetricsWithRetry } from '../../utils';
+import { EntityStoreV2EnrichmentSetup } from '../../entity_store_v2_enrichment_setup';
 
 /**
  * Specific AGENT_ID to use for some of the tests. If the archiver changes and you see errors
@@ -71,12 +72,19 @@ import { getMetricsRequest, getMetricsWithRetry } from '../../utils';
 const AGENT_ID = 'a1d7b39c-f898-4dbe-a761-efb61939302d';
 const specificQueryForTests = `configuration where agent.id=="${AGENT_ID}"`;
 
+// host.id of the auditbeat record matched by `specificQueryForTests`.
+// EUID priority is host.id, so the entity EUID is `host:${ENRICHMENT_HOST_ID}`.
+const ENRICHMENT_HOST_ID = '8cc95778cce5407c809480e8e32ad76b';
+const ENRICHMENT_HOST_NAME = 'suricata-zeek-sensor-toronto';
+const ENRICHMENT_HOST_EUID = `host:${ENRICHMENT_HOST_ID}`;
+
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
   const es = getService('es');
   const log = getService('log');
   const retry = getService('retry');
+  const entityStoreV2 = EntityStoreV2EnrichmentSetup(getService);
 
   // TODO: add a new service for loading archiver files similar to "getService('es')"
   const config = getService('config');
@@ -784,14 +792,25 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('with host risk index', () => {
       before(async () => {
-        await esArchiver.load('x-pack/solutions/security/test/fixtures/es_archives/entity/risks');
+        await entityStoreV2.setup({
+          hosts: [
+            {
+              host: { name: ENRICHMENT_HOST_NAME, id: [ENRICHMENT_HOST_ID] },
+              entity: {
+                id: ENRICHMENT_HOST_EUID,
+                type: 'host',
+                risk: { calculated_level: 'Critical', calculated_score_norm: 96 },
+              },
+            },
+          ],
+        });
       });
 
       after(async () => {
-        await esArchiver.unload('x-pack/solutions/security/test/fixtures/es_archives/entity/risks');
+        await entityStoreV2.teardown();
       });
 
-      it('@skipInServerlessMKI should be enriched with host risk score', async () => {
+      it('should be enriched with host risk score', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
           query: specificQueryForTests,
@@ -810,18 +829,22 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('with asset criticality', () => {
       before(async () => {
-        await esArchiver.load(
-          'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-        );
+        await entityStoreV2.setup({
+          hosts: [
+            {
+              host: { name: ENRICHMENT_HOST_NAME, id: [ENRICHMENT_HOST_ID] },
+              entity: { id: ENRICHMENT_HOST_EUID, type: 'host' },
+              asset: { criticality: 'high_impact' },
+            },
+          ],
+        });
       });
 
       after(async () => {
-        await esArchiver.unload(
-          'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-        );
+        await entityStoreV2.teardown();
       });
 
-      it('@skipInServerlessMKI should be enriched alert with criticality_level', async () => {
+      it('should be enriched alert with criticality_level', async () => {
         const rule: EqlRuleCreateProps = {
           ...getEqlRuleForAlertTesting(['auditbeat-*']),
           query: specificQueryForTests,

x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/eql/trial_license_complete_tier/eql_alert_suppression.ts

@@ -47,6 +47,7 @@ import {
 } from '../../../../utils';
 import type { FtrProviderContext } from '../../../../../../ftr_provider_context';
 import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/utils';
+import { EntityStoreV2EnrichmentSetup } from '../../entity_store_v2_enrichment_setup';
 
 const getQuery = (id: string) => `any where id == "${id}"`;
 const getSequenceQuery = (id: string) =>
@@ -56,6 +57,7 @@ export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
+  const entityStoreV2 = EntityStoreV2EnrichmentSetup(getService);
 
   const es = getService('es');
   const log = getService('log');
@@ -1719,22 +1721,39 @@ export default ({ getService }: FtrProviderContext) => {
 
       describe('alert enrichment', () => {
         before(async () => {
-          await esArchiver.load('x-pack/solutions/security/test/fixtures/es_archives/entity/risks');
-          await esArchiver.load(
-            'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-          );
+          // Dynamic docs in this describe only have host.name / user.name (no host.id),
+          // so the EUID is name-based.
+          await entityStoreV2.setup({
+            hosts: [
+              {
+                host: { name: 'suricata-zeek-sensor-toronto' },
+                entity: {
+                  id: 'host:suricata-zeek-sensor-toronto',
+                  type: 'host',
+                  risk: { calculated_level: 'Critical', calculated_score_norm: 96 },
+                },
+              },
+              {
+                host: { name: 'zeek-newyork-sha-aa8df15' },
+                entity: { id: 'host:zeek-newyork-sha-aa8df15', type: 'host' },
+                asset: { criticality: 'medium_impact' },
+              },
+            ],
+            users: [
+              {
+                user: { name: 'root' },
+                entity: { type: 'user' },
+                asset: { criticality: 'extreme_impact' },
+              },
+            ],
+          });
         });
 
         after(async () => {
-          await esArchiver.unload(
-            'x-pack/solutions/security/test/fixtures/es_archives/entity/risks'
-          );
-          await esArchiver.unload(
-            'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-          );
+          await entityStoreV2.teardown();
         });
 
-        it('@skipInServerlessMKI suppressed alerts are enriched with host risk score', async () => {
+        it('suppressed alerts are enriched with host risk score', async () => {
           const eventId = uuidv4();
           await indexGeneratedSourceDocuments({
             docsCount: 1,
@@ -1763,7 +1782,7 @@ export default ({ getService }: FtrProviderContext) => {
           expect(previewAlerts[0]?._source?.host?.risk?.calculated_score_norm).toBe(96);
         });
 
-        it('@skipInServerlessMKI suppressed alerts are enriched with criticality_level', async () => {
+        it('suppressed alerts are enriched with criticality_level', async () => {
           const id = uuidv4();
           const timestamp = '2020-10-28T06:45:00.000Z';
 

x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/esql/trial_license_complete_tier/esql.ts

@@ -50,13 +50,15 @@ import {
 import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/utils';
 import type { FtrProviderContext } from '../../../../../../ftr_provider_context';
 import { EsArchivePathBuilder } from '../../../../../../es_archive_path_builder';
+import { EntityStoreV2EnrichmentSetup } from '../../entity_store_v2_enrichment_setup';
 
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
   const es = getService('es');
   const log = getService('log');
   const utils = getService('securitySolutionUtils');
+  const entityStoreV2 = EntityStoreV2EnrichmentSetup(getService);
 
   const { indexEnhancedDocuments, indexListOfDocuments, indexGeneratedDocuments } =
     dataGeneratorFactory({
@@ -1973,14 +1975,25 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('alerts enrichment', () => {
       before(async () => {
-        await esArchiver.load('x-pack/solutions/security/test/fixtures/es_archives/entity/risks');
+        await entityStoreV2.setup({
+          hosts: [
+            {
+              host: { name: 'host-0' },
+              entity: {
+                id: 'host:host-0',
+                type: 'host',
+                risk: { calculated_level: 'Low', calculated_score_norm: 1 },
+              },
+            },
+          ],
+        });
       });
 
       after(async () => {
-        await esArchiver.unload('x-pack/solutions/security/test/fixtures/es_archives/entity/risks');
+        await entityStoreV2.teardown();
       });
 
-      it('@skipInServerlessMKI should be enriched with host risk score', async () => {
+      it('should be enriched with host risk score', async () => {
         const id = uuidv4();
         const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
         const doc1 = { host: { name: 'host-0' } };
@@ -2011,18 +2024,22 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('with asset criticality', () => {
       before(async () => {
-        await esArchiver.load(
-          'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-        );
+        await entityStoreV2.setup({
+          hosts: [
+            {
+              host: { name: 'host-0' },
+              entity: { id: 'host:host-0', type: 'host' },
+              asset: { criticality: 'extreme_impact' },
+            },
+          ],
+        });
       });
 
       after(async () => {
-        await esArchiver.unload(
-          'x-pack/solutions/security/test/fixtures/es_archives/asset_criticality'
-        );
+        await entityStoreV2.teardown();
       });
 
-      it('@skipInServerlessMKI should be enriched alert with criticality_level', async () => {
+      it('should be enriched alert with criticality_level', async () => {
         const id = uuidv4();
         const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
         const doc1 = { host: { name: 'host-0' } };