[Explore] Improve performance with large query result (opensearch-project#11390)

* [Explore] Improve performance with large query result

Signed-off-by: Joey Liu <jiyili@amazon.com>

* Changeset file for PR opensearch-project#11390 created/updated

* Fix tests

Signed-off-by: Joey Liu <jiyili@amazon.com>

---------

Signed-off-by: Joey Liu <jiyili@amazon.com>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
Signed-off-by: Mark Boyd <mark.boyd@gsa.gov>

Author: 2 people

changelogs/fragments/11390.yml

@@ -0,0 +1,2 @@
+chore:
+- Improve performance with large query result ([#11390](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/11390))

src/plugins/explore/public/application/utils/state_management/actions/detect_optimal_tab/detect_optimal_tab.ts

@@ -8,42 +8,53 @@ import { RootState } from '../../store';
 import { setActiveTab } from '../../slices';
 import { ExploreServices } from '../../../../../types';
 import { defaultPrepareQueryString } from '../query_actions';
-import { normalizeResultRows } from '../../../../../components/visualizations/utils/normalize_result_rows';
 import { visualizationRegistry } from '../../../../../components/visualizations/visualization_registry';
-import { prepareQueryForLanguage } from '../../../languages';
+import { FIELD_TYPE_MAP } from '../../../../../components/visualizations/constants';
+import { VisColumn, VisFieldType } from '../../../../../components/visualizations/types';
 import { Query } from '../../../../../../../data/common';
 import { QueryExecutionStatus } from '../../types';
 import { EXPLORE_LOGS_TAB_ID, EXPLORE_VISUALIZATION_TAB_ID } from '../../../../../../common';
 
 /**
- * Determine if results can be visualized
+ * Determine if results can be visualized.
+ * Classifies columns from fieldSchema only — does not iterate over row data.
  */
 const canResultsBeVisualized = (results: any): boolean => {
   if (!results?.hits?.hits || !results?.fieldSchema || results.hits.hits.length === 0) {
     return false;
   }
 
-  const rows = results.hits.hits;
-  const fieldSchema = results.fieldSchema;
-  const { numericalColumns, categoricalColumns, dateColumns } = normalizeResultRows(
-    rows,
-    fieldSchema
-  );
-  const matchedRule = visualizationRegistry.findBestMatch(
-    numericalColumns,
-    categoricalColumns,
-    dateColumns
-  );
+  const rowCount = results.hits.hits.length;
+  const numericalColumns: VisColumn[] = [];
+  const categoricalColumns: VisColumn[] = [];
+  const dateColumns: VisColumn[] = [];
 
-  return !!matchedRule;
+  results.fieldSchema.forEach((field: { type?: string; name?: string }, index: number) => {
+    const schema = FIELD_TYPE_MAP[field.type || ''] || VisFieldType.Unknown;
+    const column: VisColumn = {
+      id: index,
+      schema,
+      name: field.name || '',
+      column: `field-${index}`,
+      // Use rowCount as a conservative upper bound — rules checking uniqueValuesCount
+      // thresholds (e.g. >= 7) will pass correctly when there are enough rows.
+      validValuesCount: rowCount,
+      uniqueValuesCount: rowCount,
+    };
+    if (schema === VisFieldType.Numerical) numericalColumns.push(column);
+    else if (schema === VisFieldType.Categorical) categoricalColumns.push(column);
+    else if (schema === VisFieldType.Date) dateColumns.push(column);
+  });
+
+  return !!visualizationRegistry.findBestMatch(numericalColumns, categoricalColumns, dateColumns);
 };
 
 /**
  * Determine the optimal tab based on results
  * Use visualization tab if we can find a visualization type
  * Otherwise, fallback to logs tab
  */
-const determineOptimalTab = (results: any, services: ExploreServices): string => {
+const determineOptimalTab = (results: any, _services: ExploreServices): string => {
   if (canResultsBeVisualized(results)) {
     return EXPLORE_VISUALIZATION_TAB_ID || EXPLORE_LOGS_TAB_ID;
   }

src/plugins/explore/public/application/utils/state_management/store.ts

@@ -65,15 +65,25 @@ export const configurePreloadedStore = (
   return configureStore({
     reducer: rootReducer,
     preloadedState,
-    middleware: (getDefaultMiddleware) =>
-      services
-        ? getDefaultMiddleware()
+    middleware: (getDefaultMiddleware) => {
+      // Exclude results from dev-mode checks: the results slice holds raw OpenSearch hits which
+      // can be very large. ImmutableStateInvariantMiddleware walks the entire state tree on every
+      // dispatch — including stored results — making every user interaction slow when results are
+      // loaded. Results are always fully replaced, never partially mutated, so these checks add
+      // no safety value there.
+      const middlewareOptions = {
+        immutableCheck: { ignoredPaths: ['results'] },
+        serializableCheck: { ignoredPaths: ['results'] },
+      };
+      return services
+        ? getDefaultMiddleware(middlewareOptions)
             .concat(createPersistenceMiddleware(services))
             .concat(createQuerySyncMiddleware(services))
             .concat(createTimefilterSyncMiddleware(services))
             .concat(createDatasetChangeMiddleware(services))
             .concat(createOverallStatusMiddleware())
-        : getDefaultMiddleware(),
+        : getDefaultMiddleware(middlewareOptions);
+    },
   });
 };
 

src/plugins/explore/public/components/data_table/table_cell/source_field_table_cell.test.tsx

@@ -69,7 +69,7 @@ describe('SourceFieldTableCell', () => {
 
     renderInTable(defaultProps);
 
-    expect(mockDataset.formatHit).toHaveBeenCalledWith(mockRow);
+    expect(mockDataset.formatHit).toHaveBeenCalledWith(mockRow, 'text');
   });
 
   it('renders field names and values', () => {

src/plugins/explore/public/components/data_table/table_cell/source_field_table_cell.tsx

@@ -12,7 +12,6 @@
 import './source_field_table_cell.scss';
 
 import React, { Fragment } from 'react';
-import dompurify from 'dompurify';
 import { IndexPattern, DataView as Dataset } from 'src/plugins/data/public';
 import { shortenDottedString } from '../../../helpers/shorten_dotted_string';
 import { OpenSearchSearchHit } from '../../../types/doc_views_types';
@@ -25,14 +24,14 @@ export interface SourceFieldTableCellProps {
   wrapCellText?: boolean;
 }
 
-export const SourceFieldTableCell: React.FC<SourceFieldTableCellProps> = ({
+const SourceFieldTableCellComponent: React.FC<SourceFieldTableCellProps> = ({
   colName,
   dataset,
   row,
   isShortDots,
   wrapCellText,
 }) => {
-  const formattedRow = dataset.formatHit(row);
+  const formattedRow = dataset.formatHit(row, 'text');
   const metaFields = dataset.metaFields || [];
   const rawKeys = Object.keys(formattedRow).filter((key) => !metaFields.includes(key));
   const keys = isShortDots ? rawKeys.map((k) => shortenDottedString(k)) : rawKeys;
@@ -52,14 +51,9 @@ export const SourceFieldTableCell: React.FC<SourceFieldTableCellProps> = ({
               <span className="source__key" data-test-subj="sourceFieldKey">
                 {key}:
               </span>
-              <span
-                className="source__value"
-                data-test-subj="sourceFieldValue"
-                // eslint-disable-next-line react/no-danger
-                dangerouslySetInnerHTML={{
-                  __html: dompurify.sanitize(formattedRow[rawKeys[index]]),
-                }}
-              />
+              <span className="source__value" data-test-subj="sourceFieldValue">
+                {formattedRow[rawKeys[index]]}
+              </span>
               {index !== keys.length - 1 && ' '}
             </Fragment>
           ))}
@@ -68,3 +62,5 @@ export const SourceFieldTableCell: React.FC<SourceFieldTableCellProps> = ({
     </td>
   );
 };
+
+export const SourceFieldTableCell = React.memo(SourceFieldTableCellComponent);