From 9e97e6dfe9ae39a9a8c4d0f4bd8c4921bebe77db Mon Sep 17 00:00:00 2001
From: Ruchi Sharma <ruchi492@gmail.com>
Date: Tue, 16 Sep 2025 19:31:24 +0000
Subject: [PATCH] bulk upload

Signed-off-by: Ruchi Sharma <ruchi492@gmail.com>
---
 config/opensearch_dashboards.yml              |  7 +++-
 .../public/components/data_importer_app.tsx   |  1 +
 .../public/components/preview_table.tsx       |  1 +
 .../server/routes/import_file.ts              | 36 ++++++++++++++++---
 4 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml
index c058f2c74906..5b4275571adf 100644
--- a/config/opensearch_dashboards.yml
+++ b/config/opensearch_dashboards.yml
@@ -387,4 +387,9 @@
 # banner.content: "This is an important announcement for all users. [Learn more](https://opensearch.org) about OpenSearch."
 
 # Set the value to true to enable the keyboard shortcuts
-# opensearchDashboards.keyboardShortcuts.enabled: true
\ No newline at end of file
+# opensearchDashboards.keyboardShortcuts.enabled: true
+
+data_importer.enabled: true
+opensearch.ignoreVersionMismatch: true
+data_source.enabled: true
+workspace.enabled: false
diff --git a/src/plugins/data_importer/public/components/data_importer_app.tsx b/src/plugins/data_importer/public/components/data_importer_app.tsx
index e7803ed29345..4b287785ab1c 100644
--- a/src/plugins/data_importer/public/components/data_importer_app.tsx
+++ b/src/plugins/data_importer/public/components/data_importer_app.tsx
@@ -169,6 +169,7 @@ export const DataImporterPluginApp = ({
             selectedDataSourceId: dataSourceId,
           });
           setIsLoadingPreview(false);
+
           if (response) {
             setFilePreviewData(response);
             notifications.toasts.addSuccess(
diff --git a/src/plugins/data_importer/public/components/preview_table.tsx b/src/plugins/data_importer/public/components/preview_table.tsx
index a94d4bcb74e4..39ddd450fdbc 100644
--- a/src/plugins/data_importer/public/components/preview_table.tsx
+++ b/src/plugins/data_importer/public/components/preview_table.tsx
@@ -35,6 +35,7 @@ export const PreviewComponent = ({
   existingMapping,
 }: PreviewComponentProps) => {
   const [searchQuery, setSearchQuery] = useState('');
+  previewData = previewData.flat();
   const totalRows = previewData?.length;
   const loadedRows = Math.min(visibleRows, totalRows);
 
diff --git a/src/plugins/data_importer/server/routes/import_file.ts b/src/plugins/data_importer/server/routes/import_file.ts
index 36528ee34269..1e44067dcf64 100644
--- a/src/plugins/data_importer/server/routes/import_file.ts
+++ b/src/plugins/data_importer/server/routes/import_file.ts
@@ -107,18 +107,44 @@ export function importFileRoute(
 
       const file = request.body.file as FileStream;
 
+      // Batching logic for large CSV uploads using bulk API
       try {
-        const message = await processor.ingestFile(file, {
-          indexName: request.query.indexName,
-          client,
+        // Parse the file into rows (assume processor.parseFile returns an array of objects)
+        // Use a large previewCount to get all rows
+        const previewCount = Number.MAX_SAFE_INTEGER;
+        const rows = await processor.parseFile(file, previewCount, {
           delimiter: request.query.delimiter,
-          dataSourceId: request.query.dataSource,
         });
+        const flatRows = Array.isArray(rows[0]) ? rows.flat() : rows;
 
+        const BATCH_SIZE = 1000;
+        const failedRows: any[] = [];
+        for (let i = 0; i < flatRows.length; i += BATCH_SIZE) {
+          const batch = flatRows.slice(i, i + BATCH_SIZE);
+          // Build bulk body
+          const bulkBody = [];
+          for (const row of batch) {
+            bulkBody.push({ index: { _index: request.query.indexName } });
+            bulkBody.push(row);
+          }
+          const bulkResponse = await client.bulk({ body: bulkBody });
+          // Collect failed rows
+          if (bulkResponse.body.errors) {
+            bulkResponse.body.items.forEach((item: any, idx: number) => {
+              if (item.index && item.index.error) {
+                failedRows.push(batch[idx]);
+              }
+            });
+          }
+        }
+        const message = {
+          failedRows,
+          totalRows: rows.length,
+        };
         return response.ok({
           body: {
             message,
-            success: message.failedRows.length < 1,
+            success: failedRows.length < 1,
           },
         });
       } catch (e) {