[Discover][Traces] Add APM operation IDs to traces log fetching (#268452)

## Summary

Relates to elastic/observability-dev#5348

This PR instruments two log-fetching code paths in the trace document
viewer with APM execution context and explicit error capture, enabling
RED metrics (rate, errors, duration) visibility in APM for those
operations. Without operation IDs, all Kibana search requests appear
identically in APM's search/ese spans with no way to attribute latency
or throughput to a specific feature.

### fetchLogDocumentById

`fetchLogDocumentById` is called when a user opens a log document from
the expanded waterfall flyout. Because this function owns the
`data.search.search` call directly, we can instrument it fully:

- An `executionContext` with `operation_id: fetch-log-by-id` is passed
to the search call. The Kibana HTTP layer forwards this via the
`x-kbn-context` header, and the server-side APM agent attaches it as a
`kibana_meta_operation_id` label on the resulting span.
- The search call is wrapped in a try-catch that calls
`apm.captureError` with the same label, so failures appear in APM with
the correct operation context.
- `toasts.addDanger` is replaced with `toasts.add({ color: 'danger', ...
})` to avoid double error capture: `addDanger` internally calls
`apm.captureError`, which would produce a duplicate error event
alongside ours.

### Logs embeddable (TraceContextLogEvents)

The correlated logs section in the trace document overview renders via
`LazySavedSearchComponent`, which delegates all fetching to
`SearchEmbeddable` internally. Because we do not own the fetch call,
explicit error capture is not feasible: `SearchEmbeddable` handles
errors internally and surfaces them as inline UI state, so failures
never propagate as rejections we can catch or as failed HTTP spans in
APM.

For latency and throughput, we thread `executionContext` with
`operation_id: fetch-trace-context-logs` from `TraceContextLogEvents`
down through `LogEventsComponent` and `LazySavedSearchComponent` to
`SearchEmbeddable`'s `parentApi`. The embeddable reads execution context
from `parentApi` and includes it in every search request, tagging those
spans with `kibana_meta_operation_id: fetch-trace-context-logs` on the
server.

This required adding `executionContext` as a typed prop to
`LogEventsProps` and `SavedSearchComponentProps`, and declaring
`@kbn/core-execution-context-common` in `kbn_references` for both
affected packages.

### How to test

Verify the logs embeddable span label:

- Start Kibana with a synthtrace scenario that includes log documents
correlated to a trace (e.g. `node scripts/synthtrace
logs_traces_hosts`).
- Open a trace document in the doc viewer and expand the Logs section.
- In APM, filter search/ese spans by `labels.kibana_meta_operation_id:
fetch-trace-context-logs` and confirm requests from the logs section
appear with that label.

Verify the fetch log span label and error capture:

- From the expanded waterfall, click a log item to open the log document
flyout.
- In APM, filter by `labels.kibana_meta_operation_id: fetch-log-by-id`
and confirm the fetch span appears.
- To test error capture: temporarily break the fetch (e.g. pass an
invalid index) and confirm a single APM error appears with
`labels.kibana_meta_operation_id: fetch-log-by-id`, not two.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

Author: 2 people

src/platform/packages/shared/kbn-saved-search-component/moon.yml

@@ -17,6 +17,7 @@ project:
   owner: '@elastic/obs-exploration-team'
   sourceRoot: src/platform/packages/shared/kbn-saved-search-component
 dependsOn:
+  - '@kbn/core-execution-context-common'
   - '@kbn/embeddable-plugin'
   - '@kbn/shared-ux-utility'
   - '@kbn/discover-utils'

src/platform/packages/shared/kbn-saved-search-component/src/components/saved_search.tsx

@@ -171,13 +171,15 @@ const SavedSearchComponentTable: React.FC<
   const embeddableApi = useRef<SearchEmbeddableApi | undefined>(undefined);
   const [isEmbeddableApiAvailable, setIsEmbeddableApiAvailable] = useState(false);
 
+  const { executionContext } = props;
   const parentApi = useMemo(() => {
     return {
+      ...(executionContext ? { executionContext } : {}),
       getSerializedStateForChild: () => {
         return initialSerializedState;
       },
     };
-  }, [initialSerializedState]);
+  }, [initialSerializedState, executionContext]);
 
   useEffect(
     function syncIndex() {

src/platform/packages/shared/kbn-saved-search-component/src/types.ts

@@ -16,6 +16,7 @@ import type { CSSProperties } from 'react';
 import type { SortOrder } from '@kbn/saved-search-plugin/public';
 import type { DiscoverGridSettings } from '@kbn/saved-search-plugin/common';
 import type { DataGridDensity } from '@kbn/unified-data-table';
+import type { KibanaExecutionContext } from '@kbn/core-execution-context-common';
 
 export interface SavedSearchComponentDependencies {
   embeddable: EmbeddableStart;
@@ -56,4 +57,5 @@ export interface SavedSearchComponentProps extends SavedSearchTableConfig {
   height?: CSSProperties['height'];
   displayOptions?: NonPersistedDisplayOptions;
   onTableConfigChange?: (config: SavedSearchTableConfig) => void;
+  executionContext?: KibanaExecutionContext;
 }

src/platform/packages/shared/kbn-saved-search-component/tsconfig.json

@@ -17,6 +17,7 @@
     "target/**/*"
   ],
   "kbn_references": [
+    "@kbn/core-execution-context-common",
     "@kbn/embeddable-plugin",
     "@kbn/shared-ux-utility",
     "@kbn/discover-utils",

src/platform/plugins/shared/discover_shared/moon.yml

@@ -26,6 +26,7 @@ dependsOn:
   - '@kbn/data-views-plugin'
   - '@kbn/slo-schema'
   - '@kbn/unified-doc-viewer'
+  - '@kbn/core-execution-context-common'
 tags:
   - plugin
   - prod

src/platform/plugins/shared/discover_shared/public/services/discover_features/types.ts

@@ -13,6 +13,7 @@ import type { FunctionComponent } from 'react';
 import type React from 'react';
 import type { DataGridCellValueElementProps } from '@kbn/unified-data-table';
 import type { Query, TimeRange } from '@kbn/es-query';
+import type { KibanaExecutionContext } from '@kbn/core-execution-context-common';
 import type {
   ErrorsByTraceId,
   FocusedTraceWaterfallProps,
@@ -112,6 +113,7 @@ export interface ObservabilityLogEventsFeature {
       enableDocumentViewer: false;
       enableFilters: false;
     };
+    executionContext?: KibanaExecutionContext;
   }) => JSX.Element;
 }
 

src/platform/plugins/shared/discover_shared/tsconfig.json

@@ -15,5 +15,6 @@
     "@kbn/data-views-plugin",
     "@kbn/slo-schema",
     "@kbn/unified-doc-viewer",
+    "@kbn/core-execution-context-common",
   ]
 }

src/platform/plugins/shared/unified_doc_viewer/public/components/observability/traces/components/full_screen_waterfall/hooks/use_fetch_log/index.ts

@@ -45,7 +45,9 @@ export function useFetchLog({ id, index }: UseFetchLogParams) {
   useEffect(() => {
     if (error) {
       const errorMessage = error instanceof Error ? error.message : String(error);
-      core.notifications.toasts.addDanger({
+      core.notifications.toasts.add({
+        color: 'danger',
+        iconType: 'error',
         title: i18n.translate('unifiedDocViewer.fullScreenWaterfall.logDocument.error', {
           defaultMessage: 'An error occurred while fetching the log document',
         }),

src/platform/plugins/shared/unified_doc_viewer/public/components/observability/traces/components/full_screen_waterfall/hooks/use_fetch_log/use_fetch_log.test.ts

@@ -25,7 +25,7 @@ const mockFetchLogDocumentById = jest.fn<
   >,
   any
 >();
-const mockAddDanger = jest.fn();
+const mockAdd = jest.fn();
 
 const mockGetById: jest.Mock<
   | {
@@ -55,7 +55,7 @@ const mockGetById: jest.Mock<
   core: {
     notifications: {
       toasts: {
-        addDanger: mockAddDanger,
+        add: mockAdd,
       },
     },
   },
@@ -70,7 +70,6 @@ describe('useFetchLog', () => {
     mockGetById.mockReturnValue({
       fetchLogDocumentById: mockFetchLogDocumentById,
     });
-    mockAddDanger.mockClear();
   });
 
   it('should return undefined when feature is not registered', async () => {
@@ -209,7 +208,9 @@ describe('useFetchLog', () => {
     await waitFor(() => !result.current.loading);
 
     await waitFor(() => {
-      expect(mockAddDanger).toHaveBeenCalledWith({
+      expect(mockAdd).toHaveBeenCalledWith({
+        color: 'danger',
+        iconType: 'error',
         title: 'An error occurred while fetching the log document',
         text: errorMessage,
       });
@@ -225,7 +226,9 @@ describe('useFetchLog', () => {
     await waitFor(() => !result.current.loading);
 
     await waitFor(() => {
-      expect(mockAddDanger).toHaveBeenCalledWith({
+      expect(mockAdd).toHaveBeenCalledWith({
+        color: 'danger',
+        iconType: 'error',
         title: 'An error occurred while fetching the log document',
         text: errorMessage,
       });

src/platform/plugins/shared/unified_doc_viewer/public/components/observability/traces/components/trace_context_log_events/index.tsx

@@ -18,6 +18,8 @@ import { useDiscoverLinkAndEsqlQuery } from '../../../../../hooks/use_discover_l
 import { useOpenInDiscoverSectionAction } from '../../../../../hooks/use_open_in_discover_section_action';
 import { TRACES_DOC_VIEWER_EBT_ELEMENTS, TRACES_DOC_VIEWER_EBT_DETAILS } from '../../ebt_constants';
 
+const FETCH_TRACE_CONTEXT_LOGS_OPERATION_ID = 'fetch-trace-context-logs';
+
 const logsTitle = i18n.translate('unifiedDocViewer.observability.traces.section.logs.title', {
   defaultMessage: 'Logs',
 });
@@ -97,6 +99,7 @@ export function TraceContextLogEvents({
           nonHighlightingQuery={query}
           timeRange={savedSearchTimeRange}
           index={indexes.logs}
+          executionContext={{ meta: { operation_id: FETCH_TRACE_CONTEXT_LOGS_OPERATION_ID } }}
         />
       </div>
     </ContentFrameworkSection>