[Security Solution][Detection Engine] EQL sequence document merging: treat dot and nested notation the same (elastic#254830)

## Summary
Fixes elastic#163756

EQL sequence rules merge shared field-value pairs from all events into
the "shell" alert. Previously, if a field appeared in **dot notation**
(e.g. `'user.email': 'me@example.com'`) in some events and **nested
notation** (e.g. `user: { email: 'me@example.com' }`) in others, the
merge logic did not treat them as the same path, so the field was
omitted from the shell alert. Now, this situation will result in the
nested notation field being used. Like before this change, if both
fields have the same notation, that notation is preserved in the alert
too.

Author: dhurley14

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.test.ts

@@ -298,6 +298,69 @@ describe('buildAlert', () => {
         };
         expect(intersection).toEqual(expected);
       });
+
+      test('should treat dot and nested notation the same', () => {
+        const a = {
+          'user.email': 'marshall@elastic.co',
+        };
+        const b = {
+          user: {
+            email: 'marshall@elastic.co',
+          },
+        };
+        const intersection = objectPairIntersection(a, b);
+        const expected = {
+          user: {
+            email: 'marshall@elastic.co',
+          },
+        };
+        expect(intersection).toEqual(expected);
+      });
+
+      test('should return undefined when first argument is undefined', () => {
+        expect(objectPairIntersection(undefined, { a: 1 })).toEqual(undefined);
+      });
+
+      test('should return undefined when second argument is undefined', () => {
+        expect(objectPairIntersection({ a: 1 }, undefined)).toEqual(undefined);
+      });
+
+      test('should return undefined for two empty objects', () => {
+        expect(objectPairIntersection({}, {})).toEqual(undefined);
+      });
+
+      test('should return undefined when one object is empty', () => {
+        expect(objectPairIntersection({}, { a: 1 })).toEqual(undefined);
+        expect(objectPairIntersection({ a: 1 }, {})).toEqual(undefined);
+      });
+
+      test('should treat deep dot notation and nested notation the same', () => {
+        const a = { 'a.b.c': 42 };
+        const b = { a: { b: { c: 42 } } };
+        const intersection = objectPairIntersection(a, b);
+        expect(intersection).toEqual({ a: { b: { c: 42 } } });
+      });
+
+      test('should merge when both use dot notation for the same path and keep the dot notation', () => {
+        const a = { 'user.email': 'a@test.co' };
+        const b = { 'user.email': 'a@test.co' };
+        const intersection = objectPairIntersection(a, b);
+        expect(intersection).toEqual({ 'user.email': 'a@test.co' });
+      });
+
+      test('should merge when use dot notation and nested notation are arrays', () => {
+        const a = { 'user.emails': ['a@test.co'] };
+        const b = { user: { emails: ['a@test.co'] } };
+        const intersection = objectPairIntersection(a, b);
+        expect(intersection).toEqual({ user: { emails: ['a@test.co'] } });
+      });
+
+      test('should use last-wins when the same path appears via both notations in one object', () => {
+        const a = { 'user.email': 'dot@test.co', user: { email: 'nested@test.co' } };
+        const b = { user: { email: 'nested@test.co' } };
+        const intersection = objectPairIntersection(a, b);
+        expect(intersection).toEqual({ user: { email: 'nested@test.co' } });
+      });
     });
 
     test('should treat numbers and strings as unequal', () => {
@@ -638,5 +701,16 @@ describe('buildAlert', () => {
       };
       expect(intersection).toEqual(expected);
     });
+
+    test('should merge objects when some use dot notation and others use nested', () => {
+      const a = { 'user.id': 'u1', event: { action: 'login' } };
+      const b = { user: { id: 'u1' }, event: { action: 'login' } };
+      const c = { 'user.id': 'u1', 'event.action': 'login' };
+      const intersection = objectArrayIntersection([a, b, c]);
+      expect(intersection).toEqual({
+        user: { id: 'u1' },
+        event: { action: 'login' },
+      });
+    });
   });
 });

x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/rule_types/eql/build_alert_group_from_sequence.ts

@@ -14,6 +14,7 @@ import type {
 } from '@kbn/rule-data-utils';
 import { ALERT_URL, ALERT_UUID } from '@kbn/rule-data-utils';
 import { intersection as lodashIntersection, isArray } from 'lodash';
+import { setWith } from '@kbn/safer-lodash-set';
 
 import { getAlertDetailsUrl } from '../../../../../common/utils/alert_detail_path';
 import { DEFAULT_ALERTS_INDEX } from '../../../../../common/constants';
@@ -246,11 +247,139 @@ export const objectArrayIntersection = (objects: object[]) => {
   }
 };
 
+const isPlainObject = (v: unknown): v is Record<string, unknown> =>
+  typeof v === 'object' && v !== null && !Array.isArray(v);
+
+/**
+ * Flattens an object to path-value pairs and records which paths came from
+ * dot-notation keys. A path is "from dot" when it was produced by expanding
+ * a single key that contained '.'.
+ */
+const flattenToPathValuesWithNotation = (
+  obj: object
+): { pathValues: [string[], unknown][]; dotPaths: Set<string> } => {
+  const pathValues: [string[], unknown][] = [];
+  const dotPaths = new Set<string>();
+  const stack: [object, string[]][] = [[obj, []]];
+  while (stack.length > 0) {
+    const [o, prefix] = stack.pop() as [object, string[]];
+    for (const [k, v] of Object.entries(o)) {
+      const path = prefix.concat(k.includes('.') ? k.split('.') : [k]);
+      const keyWasDot = k.includes('.');
+      if (isPlainObject(v)) {
+        stack.push([v, path]);
+      } else {
+        pathValues.push([path, v]);
+        if (keyWasDot) {
+          dotPaths.add(path.join('.'));
+        }
+      }
+    }
+  }
+  return { pathValues, dotPaths };
+};
+
+/**
+ * Flattens an object to path-value pairs (paths as string arrays).
+ * Dot-notation keys are expanded so 'user.email' and user: { email } yield the same path.
+ */
+const flattenToPathValues = (obj: object): [string[], unknown][] =>
+  flattenToPathValuesWithNotation(obj).pathValues;
+
+/**
+ * Builds a nested object from path-value pairs.
+ */
+const unflatten = (pathValues: [string[], unknown][]): Record<string, unknown> => {
+  const result: Record<string, unknown> = {};
+  for (const [path, value] of pathValues) {
+    setWith(result, path, value);
+  }
+  return result;
+};
+
+/** Result of intersecting two values: either a literal value to set, or a nested pair to process later. */
+type IntersectionResult =
+  | { kind: 'value'; value: unknown }
+  | { kind: 'nested'; a: Record<string, unknown>; b: Record<string, unknown> };
+
+/**
+ * Computes the intersection of two values (primitives, arrays, or nested objects).
+ * For nested objects, returns a descriptor so the caller can push work onto the stack.
+ */
+const intersectValues = (aVal: unknown, bVal: unknown): IntersectionResult | undefined => {
+  if (isPlainObject(aVal) && isPlainObject(bVal)) {
+    return { kind: 'nested', a: aVal, b: bVal };
+  }
+  if (aVal === bVal) {
+    return { kind: 'value', value: aVal };
+  }
+  if (isArray(aVal) || isArray(bVal)) {
+    const arrA = isArray(aVal) ? aVal : [aVal];
+    const arrB = isArray(bVal) ? bVal : [bVal];
+    return { kind: 'value', value: lodashIntersection(arrA, arrB) };
+  }
+  return undefined;
+};
+
+const isEmptyOrAllUndefined = (o: Record<string, unknown>): boolean =>
+  Object.keys(o).length === 0 || Object.values(o).every((v) => v === undefined);
+
+/** Removes nested objects that are empty or have only undefined values (iterative). */
+const pruneEmptyNestedObjects = (obj: Record<string, unknown>): void => {
+  type PruneItem = [Record<string, unknown> | null, string | null, Record<string, unknown>];
+  const pruneStack: PruneItem[] = [[null, null, obj]];
+  const toPrune: PruneItem[] = [];
+  while (pruneStack.length > 0) {
+    const item = pruneStack.pop() as PruneItem;
+    toPrune.push(item);
+    const [, , o] = item;
+    for (const [k, v] of Object.entries(o)) {
+      if (isPlainObject(v)) {
+        pruneStack.push([o, k, v]);
+      }
+    }
+  }
+  for (let i = toPrune.length - 1; i >= 0; i--) {
+    const [parent, key, o] = toPrune[i];
+    if (parent !== null && key !== null && isEmptyOrAllUndefined(o)) {
+      delete parent[key];
+    }
+  }
+};
+
+const hasDefinedValues = (obj: Record<string, unknown>): boolean =>
+  Object.values(obj).some((v) => v !== undefined);
+
+/**
+ * Converts a normalized (nested) intersection result to output form:
+ * paths that were dot notation in both inputs stay as dot keys; others stay nested.
+ */
+const applyNotationPreference = (
+  intersectionNested: Record<string, unknown>,
+  aDotPaths: Set<string>,
+  bDotPaths: Set<string>
+): Record<string, unknown> => {
+  const pathValues = flattenToPathValues(intersectionNested);
+  const result: Record<string, unknown> = {};
+  for (const [path, value] of pathValues) {
+    const pathStr = path.join('.');
+    if (aDotPaths.has(pathStr) && bDotPaths.has(pathStr)) {
+      result[pathStr] = value;
+    } else {
+      setWith(result, path, value);
+    }
+  }
+  return result;
+};
+
 /**
- * Finds the intersection of two objects by recursively
- * finding the "intersection" of each of of their common keys'
+ * Finds the intersection of two objects iteratively by
+ * finding the "intersection" of each of their common keys'
  * values. If an intersection cannot be found between a key's
  * values, the value will be undefined in the returned object.
+ * Dot-notation keys (e.g. 'user.email') and nested notation
+ * (e.g. user: { email }) are treated as the same; the result
+ * is always in nested form.
  *
  * @param a object
  * @param b object
@@ -260,40 +389,40 @@ export const objectPairIntersection = (a: object | undefined, b: object | undefi
   if (a === undefined || b === undefined) {
     return undefined;
   }
-  const intersection: Record<string, unknown> = {};
-  Object.entries(a).forEach(([key, aVal]) => {
-    if (key in b) {
-      const bVal = (b as Record<string, unknown>)[key];
-      if (
-        typeof aVal === 'object' &&
-        !(aVal instanceof Array) &&
-        aVal !== null &&
-        typeof bVal === 'object' &&
-        !(bVal instanceof Array) &&
-        bVal !== null
-      ) {
-        intersection[key] = objectPairIntersection(aVal, bVal);
-      } else if (aVal === bVal) {
-        intersection[key] = aVal;
-      } else if (isArray(aVal) && isArray(bVal)) {
-        intersection[key] = lodashIntersection(aVal, bVal);
-      } else if (isArray(aVal) && !isArray(bVal)) {
-        intersection[key] = lodashIntersection(aVal, [bVal]);
-      } else if (!isArray(aVal) && isArray(bVal)) {
-        intersection[key] = lodashIntersection([aVal], bVal);
+  const { pathValues: aPathValues, dotPaths: aDotPaths } = flattenToPathValuesWithNotation(a);
+  const { pathValues: bPathValues, dotPaths: bDotPaths } = flattenToPathValuesWithNotation(b);
+  const aNorm = unflatten(aPathValues);
+  const bNorm = unflatten(bPathValues);
+  const intersectionNested: Record<string, unknown> = {};
+  const stack: [Record<string, unknown>, Record<string, unknown>, Record<string, unknown>][] = [
+    [intersectionNested, aNorm, bNorm],
+  ];
+  while (stack.length > 0) {
+    const [target, aObj, bObj] = stack.pop() as [
+      Record<string, unknown>,
+      Record<string, unknown>,
+      Record<string, unknown>
+    ];
+    for (const [key, aVal] of Object.entries(aObj)) {
+      if (key in bObj) {
+        const result = intersectValues(aVal, bObj[key]);
+        if (result === undefined) {
+          // eslint-disable-next-line no-continue
+          continue;
+        }
+        if (result.kind === 'nested') {
+          const nextTarget: Record<string, unknown> = {};
+          target[key] = nextTarget;
+          stack.push([nextTarget, result.a, result.b]);
+        } else {
+          target[key] = result.value;
+        }
       }
     }
-  });
-  // Count up the number of entries that are NOT undefined in the intersection
-  // If there are no keys OR all entries are undefined, return undefined
-  if (
-    Object.values(intersection).reduce(
-      (acc: number, value) => (value !== undefined ? acc + 1 : acc),
-      0
-    ) === 0
-  ) {
+  }
+  pruneEmptyNestedObjects(intersectionNested);
+  if (!hasDefinedValues(intersectionNested)) {
     return undefined;
-  } else {
-    return intersection;
   }
+  return applyNotationPreference(intersectionNested, aDotPaths, bDotPaths);
 };