[Automatic Import V2] Edit pipeline for data stream (#252399)

This PR allows:
- Allows to view the results of the AI pipeline after processing.
- A table view displaying a inMemoryTable for viewing and filtering.
- A code editor view for looking at the pipeline processors.
- Document pagination for viewing the results for different documents.
- Tests for the component.
- Some translation additions to this component and other tables.

Author: robester0403

packages/kbn-optimizer/limits.yml

@@ -9,7 +9,7 @@ pageLoadAssetSize:
   apm: 38573
   apmSourcesAccess: 2278
   automaticImport: 12162
-  automaticImportVTwo: 12306
+  automaticImportVTwo: 13752
   banners: 4087
   canvas: 15142
   cases: 225900
@@ -50,7 +50,7 @@ pageLoadAssetSize:
   discoverShared: 2322
   elasticAssistant: 338870
   elasticAssistantSharedState: 4881
-  embeddable: 17521
+  embeddable: 17466
   embeddableAlertsTable: 6524
   embeddableEnhanced: 4488
   enterpriseSearch: 40244
@@ -83,7 +83,7 @@ pageLoadAssetSize:
   globalSearchProviders: 4646
   graph: 9924
   grokdebugger: 5484
-  home: 13560
+  home: 13091
   imageEmbeddable: 6727
   indexLifecycleManagement: 29818
   indexManagement: 35522

x-pack/platform/plugins/shared/automatic_import_v2/moon.yml

@@ -56,6 +56,7 @@ dependsOn:
   - '@kbn/kibana-utils-plugin'
   - '@kbn/ingest-pipelines-plugin'
   - '@kbn/utility-types'
+  - '@kbn/code-editor'
 tags:
   - plugin
   - prod

x-pack/platform/plugins/shared/automatic_import_v2/public/common/hooks/use_get_data_stream_results.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { useQuery } from '@kbn/react-query';
+import type { GetDataStreamResultsResponse } from '../lib/api';
+import { getDataStreamResults } from '../lib/api';
+import { useKibana } from './use_kibana';
+
+export interface UseGetDataStreamResultsResult {
+  data: GetDataStreamResultsResponse | undefined;
+  isLoading: boolean;
+  isError: boolean;
+  error: Error | null;
+  refetch: () => void;
+}
+
+/**
+ * Hook to fetch data stream results (ingest pipeline and processed documents) from the automatic_import_v2 backend.
+ */
+export function useGetDataStreamResults(
+  integrationId: string | undefined,
+  dataStreamId: string | undefined
+): UseGetDataStreamResultsResult {
+  const { http } = useKibana().services;
+
+  const { data, isLoading, isError, error, refetch } = useQuery({
+    queryKey: ['dataStreamResults', integrationId, dataStreamId],
+    queryFn: async () => {
+      if (!integrationId || !dataStreamId) {
+        return undefined;
+      }
+      return getDataStreamResults({ http, integrationId, dataStreamId });
+    },
+    enabled: !!integrationId && !!dataStreamId,
+  });
+
+  return {
+    data,
+    isLoading,
+    isError,
+    error: error as Error | null,
+    refetch,
+  };
+}

x-pack/platform/plugins/shared/automatic_import_v2/public/common/index.ts

@@ -13,6 +13,7 @@ export { useLoadConnectors } from './hooks/use_load_connectors';
 export { useValidateIndex } from './hooks/use_validate_index';
 export { useUploadSamples } from './hooks/use_upload_samples';
 export { useDeleteDataStream } from './hooks/use_delete_data_stream';
+export { useGetDataStreamResults } from './hooks/use_get_data_stream_results';
 
 export {
   INDEX_VALIDATION_FAILED,
@@ -32,7 +33,11 @@ export {
   getIntegrationById,
 } from './lib/api';
 
-export type { CreateUpdateIntegrationRequest, UploadSamplesRequest } from './lib/api';
+export type {
+  CreateUpdateIntegrationRequest,
+  UploadSamplesRequest,
+  GetDataStreamResultsResponse,
+} from './lib/api';
 
 export { getLangSmithOptions } from './lib/lang_smith';
 export type { LangSmithOptions } from './lib/lang_smith';

x-pack/platform/plugins/shared/automatic_import_v2/public/common/lib/api.ts

@@ -127,3 +127,29 @@ export const deleteDataStream = async ({
       version: '1',
     }
   );
+
+export interface GetDataStreamResultsRequest {
+  integrationId: string;
+  dataStreamId: string;
+}
+
+export interface GetDataStreamResultsResponse {
+  ingest_pipeline: Record<string, unknown>;
+  results: Array<Record<string, unknown>>;
+}
+
+export const getDataStreamResults = async ({
+  http,
+  abortSignal,
+  integrationId,
+  dataStreamId,
+}: RequestDeps & GetDataStreamResultsRequest): Promise<GetDataStreamResultsResponse> =>
+  http.get<GetDataStreamResultsResponse>(
+    `${AUTOMATIC_IMPORT_INTEGRATIONS_PATH}/${encodeURIComponent(
+      integrationId
+    )}/data_streams/${encodeURIComponent(dataStreamId)}/results`,
+    {
+      version: '1',
+      signal: abortSignal,
+    }
+  );

x-pack/platform/plugins/shared/automatic_import_v2/public/components/integration_management/contexts/types.ts

@@ -5,13 +5,21 @@
  * 2.0.
  */
 
+import type { DataStreamResponse } from '../../../../common';
+
 export interface UIState {
   isCreateDataStreamFlyoutOpen: boolean;
+  isEditPipelineFlyoutOpen: boolean;
+  selectedDataStream: DataStreamResponse | null;
 }
 
 export interface UIStateActions {
   openCreateDataStreamFlyout: () => void;
   closeCreateDataStreamFlyout: () => void;
+  openEditPipelineFlyout: (dataStream: DataStreamResponse) => void;
+  closeEditPipelineFlyout: () => void;
+  selectPipelineTab: (tab: 'table' | 'pipeline') => void;
+  selectedPipelineTab: 'table' | 'pipeline';
 }
 
 export type UIStateContextValue = UIState & UIStateActions;

x-pack/platform/plugins/shared/automatic_import_v2/public/components/integration_management/contexts/ui_state_context.test.tsx

@@ -34,6 +34,12 @@ describe('UIStateContext', () => {
         isCreateDataStreamFlyoutOpen: false,
         openCreateDataStreamFlyout: expect.any(Function),
         closeCreateDataStreamFlyout: expect.any(Function),
+        isEditPipelineFlyoutOpen: false,
+        selectedDataStream: null,
+        openEditPipelineFlyout: expect.any(Function),
+        closeEditPipelineFlyout: expect.any(Function),
+        selectedPipelineTab: 'table',
+        selectPipelineTab: expect.any(Function),
       });
     });
   });

x-pack/platform/plugins/shared/automatic_import_v2/public/components/integration_management/contexts/ui_state_context.tsx

@@ -7,6 +7,7 @@
 
 import React, { createContext, useContext, useState, useCallback, useMemo } from 'react';
 
+import type { DataStreamResponse } from '../../../../common';
 import type { UIStateContextValue } from './types';
 
 const UIStateContext = createContext<UIStateContextValue | undefined>(undefined);
@@ -17,6 +18,8 @@ export interface UIStateProviderProps {
 
 export const UIStateProvider: React.FC<UIStateProviderProps> = ({ children }) => {
   const [isCreateDataStreamFlyoutOpen, setIsCreateDataStreamFlyoutOpen] = useState(false);
+  const [selectedDataStream, setSelectedDataStream] = useState<DataStreamResponse | null>(null);
+  const [selectedPipelineTab, setSelectedPipelineTab] = useState<'table' | 'pipeline'>('table');
 
   const openCreateDataStreamFlyout = useCallback(() => {
     setIsCreateDataStreamFlyoutOpen(true);
@@ -26,13 +29,43 @@ export const UIStateProvider: React.FC<UIStateProviderProps> = ({ children }) =>
     setIsCreateDataStreamFlyoutOpen(false);
   }, []);
 
+  const openEditPipelineFlyout = useCallback((dataStream: DataStreamResponse) => {
+    setSelectedDataStream(dataStream);
+  }, []);
+
+  const closeEditPipelineFlyout = useCallback(() => {
+    setSelectedDataStream(null);
+  }, []);
+
+  const selectPipelineTab = useCallback((tab: 'table' | 'pipeline') => {
+    setSelectedPipelineTab(tab);
+  }, []);
+
+  const isEditPipelineFlyoutOpen = selectedDataStream !== null;
+
   const value = useMemo<UIStateContextValue>(
     () => ({
       isCreateDataStreamFlyoutOpen,
       openCreateDataStreamFlyout,
       closeCreateDataStreamFlyout,
+      isEditPipelineFlyoutOpen,
+      selectedDataStream,
+      openEditPipelineFlyout,
+      closeEditPipelineFlyout,
+      selectedPipelineTab,
+      selectPipelineTab,
     }),
-    [isCreateDataStreamFlyoutOpen, closeCreateDataStreamFlyout, openCreateDataStreamFlyout]
+    [
+      isCreateDataStreamFlyoutOpen,
+      closeCreateDataStreamFlyout,
+      openCreateDataStreamFlyout,
+      isEditPipelineFlyoutOpen,
+      selectedDataStream,
+      openEditPipelineFlyout,
+      closeEditPipelineFlyout,
+      selectedPipelineTab,
+      selectPipelineTab,
+    ]
   );
 
   return <UIStateContext.Provider value={value}>{children}</UIStateContext.Provider>;

x-pack/platform/plugins/shared/automatic_import_v2/public/components/integration_management/management_contents/data_streams/create_data_stream_flyout.tsx

@@ -242,7 +242,7 @@ export const CreateDataStreamFlyout: React.FC<CreateDataStreamFlyoutProps> = ({
     if (!formData) return;
 
     const integrationId = currentIntegrationId ?? generateId();
-    const dataStreamId = integrationId + '-' + generateId();
+    const dataStreamId = generateId();
     const inputTypes: InputType[] = (formData.dataCollectionMethod ?? []).map((method) => ({
       name: method as InputType['name'],
     }));
@@ -416,7 +416,7 @@ export const CreateDataStreamFlyout: React.FC<CreateDataStreamFlyoutProps> = ({
               initialPromptText={i18n.FILE_PICKER_PROMPT}
               onChange={onChangeLogFile}
               display="large"
-              aria-label="Upload log file"
+              aria-label={i18n.ARIA_LABELS.uploadLogFile}
               isLoading={isParsing}
               isInvalid={fileError != null}
               disabled={logsSourceOption !== 'upload'}
@@ -442,7 +442,7 @@ export const CreateDataStreamFlyout: React.FC<CreateDataStreamFlyoutProps> = ({
                   fullWidth
                   isInvalid={!!indexValidationError}
                   error={indexValidationError}
-                  aria-label="Select an index"
+                  aria-label={i18n.ARIA_LABELS.selectIndex}
                 >
                   <EuiComboBox
                     key={field.value ?? ''}

x-pack/platform/plugins/shared/automatic_import_v2/public/components/integration_management/management_contents/data_streams/data_streams.tsx

@@ -19,10 +19,17 @@ import { CreateDataStreamFlyout } from './create_data_stream_flyout';
 import * as i18n from './translations';
 import { useGetIntegrationById } from '../../../../common';
 import { DataStreamsTable } from './data_streams_table/data_steams_table';
+import { EditPipelineFlyout } from './edit_pipeline_flyout';
 
 export const DataStreams = React.memo<{ integrationId?: string }>(() => {
-  const { isCreateDataStreamFlyoutOpen, openCreateDataStreamFlyout, closeCreateDataStreamFlyout } =
-    useUIState();
+  const {
+    isCreateDataStreamFlyoutOpen,
+    openCreateDataStreamFlyout,
+    closeCreateDataStreamFlyout,
+    isEditPipelineFlyoutOpen,
+    selectedDataStream,
+    closeEditPipelineFlyout,
+  } = useUIState();
   const { integrationId } = useParams<{ integrationId?: string }>();
   const { integration } = useGetIntegrationById(integrationId);
 
@@ -84,6 +91,14 @@ export const DataStreams = React.memo<{ integrationId?: string }>(() => {
       {isCreateDataStreamFlyoutOpen && (
         <CreateDataStreamFlyout onClose={closeCreateDataStreamFlyout} />
       )}
+
+      {isEditPipelineFlyoutOpen && selectedDataStream && integrationId && (
+        <EditPipelineFlyout
+          integrationId={integrationId}
+          dataStream={selectedDataStream}
+          onClose={closeEditPipelineFlyout}
+        />
+      )}
     </>
   );
 });