Enh 1056 improve performance of extractor pipeline query (#1066)

nicoalee · web-flow · commit 1375162861d7 · 2025-08-19T18:46:27.000-04:00
* feat: added caching to query dependant on studies in project

* feat: added loader to the table cells for AI columns instead of entire table

* feat: added loader status for participant data in focus mode

* maint: change file name

* feat: changed GET to POST for extracted data requests
diff --git a/compose/neurosynth-frontend/cypress/e2e/workflows/Curation/CurationAIInterface.cy.tsx b/compose/neurosynth-frontend/cypress/e2e/workflows/Curation/CurationAIInterface.cy.tsx
@@ -18,11 +18,11 @@ describe('CurationAIInterface', () => {
             {}
         ).as('updateProject');
 
-        cy.intercept('GET', `**/api/pipeline-study-results/?feature_display=TaskExtractor*`, {
+        cy.intercept('POST', `**/api/pipeline-study-results/?feature_display=TaskExtractor*`, {
             fixture: 'pipeline-results/TaskExtraction',
         }).as('participantDemographicsExtraction');
 
-        cy.intercept('GET', `**/api/pipeline-study-results/?feature_display=ParticipantDemographicsExtractor*`, {
+        cy.intercept('POST', `**/api/pipeline-study-results/?feature_display=ParticipantDemographicsExtractor*`, {
             fixture: 'pipeline-results/ParticipantDemographicsExtraction',
         }).as('taskExtraction');
 
diff --git a/compose/neurosynth-frontend/cypress/e2e/workflows/SleuthImport/DoSleuthImport.cy.tsx b/compose/neurosynth-frontend/cypress/e2e/workflows/SleuthImport/DoSleuthImport.cy.tsx
diff --git a/compose/neurosynth-frontend/src/hooks/extractions/useGetAllExtractedDataForStudies.tsx b/compose/neurosynth-frontend/src/hooks/extractions/useGetAllExtractedDataForStudies.tsx
@@ -102,25 +102,28 @@ export interface IExtractedDataResult {
     }[];
 }
 
-const useGetAllAIExtractedData = () => {
+const useGetAllAIExtractedDataForStudies = (baseStudyIds?: string[]) => {
+    const baseStudyIdsOrEmpty = baseStudyIds ?? [];
     return useQuery<
         [AxiosResponse<IExtractedDataResult>, AxiosResponse<IExtractedDataResult>],
         AxiosError,
         {
             [EAIExtractors.TASKEXTRACTOR]: IExtractedDataResult;
             [EAIExtractors.PARTICIPANTSDEMOGRAPHICSEXTRACTOR]: IExtractedDataResult;
         },
-        [string]
+        string[]
     >(
-        ['extraction'],
+        ['extraction', ...baseStudyIdsOrEmpty],
         async () => {
             const promises = await Promise.all([
-                API.NeurostoreServices.ExtractedDataResultsService.getAllExtractedDataResults([
-                    EAIExtractors.TASKEXTRACTOR,
-                ]),
-                API.NeurostoreServices.ExtractedDataResultsService.getAllExtractedDataResults([
-                    EAIExtractors.PARTICIPANTSDEMOGRAPHICSEXTRACTOR,
-                ]),
+                API.NeurostoreServices.ExtractedDataResultsService.getAllExtractedDataResults(
+                    [EAIExtractors.TASKEXTRACTOR],
+                    baseStudyIdsOrEmpty
+                ),
+                API.NeurostoreServices.ExtractedDataResultsService.getAllExtractedDataResults(
+                    [EAIExtractors.PARTICIPANTSDEMOGRAPHICSEXTRACTOR],
+                    baseStudyIdsOrEmpty
+                ),
             ]);
 
             promises.forEach((res) => {
@@ -136,8 +139,10 @@ const useGetAllAIExtractedData = () => {
                     [EAIExtractors.PARTICIPANTSDEMOGRAPHICSEXTRACTOR]: participantDemographicsExtractionRes.data,
                 };
             },
+            refetchOnMount: false,
+            enabled: baseStudyIdsOrEmpty.length > 0,
         }
     );
 };
 
-export default useGetAllAIExtractedData;
+export default useGetAllAIExtractedDataForStudies;
diff --git a/compose/neurosynth-frontend/src/hooks/useGetExtractionSummary.tsx b/compose/neurosynth-frontend/src/hooks/useGetExtractionSummary.tsx
@@ -1,8 +1,5 @@
 import { EExtractionStatus } from 'pages/Extraction/ExtractionPage';
-import {
-    useProjectExtractionStudysetId,
-    useProjectExtractionStudyStatusList,
-} from 'pages/Project/store/ProjectStore';
+import { useProjectExtractionStudysetId, useProjectExtractionStudyStatusList } from 'pages/Project/store/ProjectStore';
 import { useEffect, useState } from 'react';
 import { IStudyExtractionStatus } from './projects/useGetProjects';
 import useGetStudysetById from './studysets/useGetStudysetById';
@@ -14,10 +11,7 @@ export interface IExtractionSummary {
     total: number;
 }
 
-export const getExtractionSummary = (
-    studysetStudiesArg: Array<string>,
-    studyStatusList: IStudyExtractionStatus[]
-) => {
+export const getExtractionSummary = (studysetStudiesArg: Array<string>, studyStatusList: IStudyExtractionStatus[]) => {
     const studysetStudies = studysetStudiesArg || [];
 
     const summary = {
diff --git a/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCurator.tsx b/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCurator.tsx
@@ -4,7 +4,6 @@ import { GridTableRowsIcon } from '@mui/x-data-grid';
 import { Row, Table } from '@tanstack/react-table';
 import CurationPromoteUncategorizedButton from 'components/Buttons/CurationPromoteUncategorizedButton';
 import { useUserCanEdit } from 'hooks';
-import useGetAllAIExtractedData from 'hooks/extractions/useGetAllExtractedData';
 import { indexToPRISMAMapping } from 'hooks/projects/useGetProjects';
 import {
     useProjectCurationColumns,
@@ -18,7 +17,6 @@ import useCuratorTableState from '../hooks/useCuratorTableState';
 import { ICurationTableStudy } from '../hooks/useCuratorTableState.types';
 import { IGroupListItem } from './CurationBoardAIGroupsList';
 import CurationBoardAIInterfaceCuratorFocus from './CurationBoardAIInterfaceCuratorFocus';
-import CurationBoardAIInterfaceCuratorTableSkeleton from './CurationBoardAIInterfaceCuratorSkeleton';
 import CurationBoardAIInterfaceCuratorTable from './CurationBoardAIInterfaceCuratorTable';
 import CurationDownloadSummaryButton from './CurationDownloadSummaryButton';
 import PrismaDialog from './PrismaDialog';
@@ -38,7 +36,6 @@ const CurationBoardAIInterfaceCurator: React.FC<{
     const navigate = useNavigate();
     const { projectId } = useParams<{ projectId: string | undefined }>();
     const curationColumns = useProjectCurationColumns();
-    const { isLoading } = useGetAllAIExtractedData();
 
     const { column, columnIndex } = useMemo(() => {
         const columnIndex = curationColumns.findIndex((col) => col.id === group.id);
@@ -61,7 +58,7 @@ const CurationBoardAIInterfaceCurator: React.FC<{
         return column.stubStudies.filter((x) => x.exclusionTag === null);
     }, [column]);
 
-    const table = useCuratorTableState(projectId, stubsInColumn, !isLastColumn, prismaPhase !== 'identification');
+    const { table } = useCuratorTableState(projectId, stubsInColumn, !isLastColumn, prismaPhase !== 'identification');
 
     const [prismaIsOpen, setPrismaIsOpen] = useState(false);
     const [UIMode, setUIMode] = useState<'TABLEMODE' | 'FOCUSMODE'>('TABLEMODE');
@@ -71,9 +68,10 @@ const CurationBoardAIInterfaceCurator: React.FC<{
 
     const [selectedStubId, setSelectedStubId] = useState<string>();
 
+    const rows = table.getCoreRowModel().rows;
     const selectedStub: Row<ICurationTableStudy> | undefined = useMemo(
-        () => (table.getCoreRowModel().rows || []).find((stub) => stub.original.id === selectedStubId),
-        [table, selectedStubId]
+        () => (rows || []).find((stub) => stub.original.id === selectedStubId),
+        [rows, selectedStubId]
     );
 
     const handleToggleUIMode = () => {
@@ -116,10 +114,6 @@ const CurationBoardAIInterfaceCurator: React.FC<{
         return <Typography color="error.dark">There was an error loading studies</Typography>;
     }
 
-    if (isLoading) {
-        return <CurationBoardAIInterfaceCuratorTableSkeleton />;
-    }
-
     return (
         <Box sx={{ height: '100%' }}>
             <Box
diff --git a/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorSkeleton.tsx b/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorSkeleton.tsx
diff --git a/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorTable.tsx b/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorTable.tsx
@@ -1,7 +1,7 @@
 import { Box, Chip, Table, TableCell, TableContainer, TableHead, TableRow, Typography } from '@mui/material';
 import { flexRender, RowData } from '@tanstack/react-table';
 import { useGetCurationSummary } from 'hooks';
-import { EAIExtractors } from 'hooks/extractions/useGetAllExtractedData';
+import { EAIExtractors } from 'hooks/extractions/useGetAllExtractedDataForStudies';
 import { indexToPRISMAMapping } from 'hooks/projects/useGetProjects';
 import { useProjectCurationPrismaConfig } from 'pages/Project/store/ProjectStore';
 import { useRef } from 'react';
diff --git a/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorTableCell.tsx b/compose/neurosynth-frontend/src/pages/Curation/components/CurationBoardAIInterfaceCuratorTableCell.tsx
@@ -1,7 +1,8 @@
-import { Typography } from '@mui/material';
+import { Skeleton, Typography } from '@mui/material';
 import { Box } from '@mui/system';
 import { CellContext } from '@tanstack/react-table';
 import React from 'react';
+import { useIsFetching } from 'react-query';
 import {
     ICurationTableColumnType,
     ICurationTableStudy,
@@ -11,8 +12,14 @@ import {
 const CuratorTableCell: React.FC<Partial<CellContext<ICurationTableStudy, ICurationTableColumnType | null>>> = (
     props
 ) => {
+    const isFetchingExtractions = useIsFetching({ queryKey: ['extraction'] }) > 0;
+    const isAI = !!props?.column?.columnDef?.meta?.AIExtractor;
     const cellValue = props.getValue ? props.getValue() : undefined;
 
+    if (isFetchingExtractions && isAI) {
+        return <Skeleton width="100%" height="40px" />;
+    }
+
     if (cellValue === null) {
         // if there is no extraction done for the given study, we get null. This is in contrast
         // to studies that have an extraction, but some of the extracted values are null.
diff --git a/compose/neurosynth-frontend/src/pages/Curation/components/CurationStubAITableSummary.tsx b/compose/neurosynth-frontend/src/pages/Curation/components/CurationStubAITableSummary.tsx
@@ -4,6 +4,7 @@ import {
     AccordionDetails,
     AccordionSummary,
     Box,
+    Skeleton,
     Table,
     TableBody,
     TableCell,
@@ -12,16 +13,18 @@ import {
     Typography,
 } from '@mui/material';
 import AIICon from 'components/AIIcon';
-import { IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedData';
+import { IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedDataForStudies';
 import { useState } from 'react';
 import { useParams } from 'react-router-dom';
 import {
     PARTICIPANTS_DEMOGRAPHICS_EXTRACTOR_CURATOR_COLUMNS,
     TASK_EXTRACTOR_CURATOR_COLUMNS,
 } from '../hooks/useCuratorTableState.consts';
 import { ICurationTableStudy } from '../hooks/useCuratorTableState.types';
+import { useIsFetching } from 'react-query';
 
 const CurationStubAITableSummary: React.FC<{ stub: ICurationTableStudy | undefined }> = ({ stub }) => {
+    const isFetchingExtractions = useIsFetching({ queryKey: ['extraction'] }) > 0;
     const TaskExtractor = stub?.TaskExtractor;
     const { projectId } = useParams<{ projectId: string }>();
     const AIFocusModeSummaryLocalStorageKey = `${projectId}_FOCUS_MODE_AI_SUMMARY_EXPANDED_STATE`;
@@ -43,6 +46,15 @@ const CurationStubAITableSummary: React.FC<{ stub: ICurationTableStudy | undefin
         setExpandedState(newExpandedState);
     };
 
+    if (isFetchingExtractions) {
+        return (
+            <>
+                <Skeleton width="100%" height="50px" sx={{ transform: 'none', marginBottom: '2px' }} />
+                <Skeleton width="100%" height="50px" sx={{ transform: 'none' }} />
+            </>
+        );
+    }
+
     if (!TaskExtractor && !ParticipantDemographicsExtractor) {
         return (
             <Typography color="warning.dark" variant="body2">
diff --git a/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.consts.ts b/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.consts.ts
@@ -1,4 +1,4 @@
-import { EAIExtractors, IBehavioralTask, IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedData';
+import { EAIExtractors, IBehavioralTask, IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedDataForStudies';
 import {
     ICurationBoardAIInterfaceCuratorColumnType,
     ICurationTableStudy,
diff --git a/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.tsx b/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.tsx
@@ -9,11 +9,11 @@ import {
     SortingState,
     useReactTable,
 } from '@tanstack/react-table';
-import useGetAllAIExtractedData, {
+import useGetAllAIExtractedDataForStudies, {
     EAIExtractors,
     IParticipantDemographicExtractor,
     ITaskExtractor,
-} from 'hooks/extractions/useGetAllExtractedData';
+} from 'hooks/extractions/useGetAllExtractedDataForStudies';
 import { useCallback, useEffect, useMemo, useState } from 'react';
 import {
     retrieveCurationTableState,
@@ -38,7 +38,10 @@ const useCuratorTableState = (
     const [sorting, setSorting] = useState<SortingState>([]);
     const [rowSelection, setRowSelection] = useState<RowSelectionState>({});
     const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
-    const { data: extractedData } = useGetAllAIExtractedData();
+    const filteredStubs = useMemo(() => {
+        return (allStubs.filter((stub) => !!stub.neurostoreId).map((stub) => stub!.neurostoreId) as string[]).sort();
+    }, [allStubs]);
+    const { data: extractedData, isLoading } = useGetAllAIExtractedDataForStudies(filteredStubs);
 
     useEffect(() => {
         if (!projectId) return;
@@ -260,7 +263,10 @@ const useCuratorTableState = (
         );
     }, [allowAIColumns, columns, projectId]);
 
-    return table;
+    return {
+        table,
+        isLoading,
+    };
 };
 
 export default useCuratorTableState;
diff --git a/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.types.ts b/compose/neurosynth-frontend/src/pages/Curation/hooks/useCuratorTableState.types.ts
@@ -3,7 +3,7 @@ import {
     EAIExtractors,
     IParticipantDemographicExtractor,
     ITaskExtractor,
-} from 'hooks/extractions/useGetAllExtractedData';
+} from 'hooks/extractions/useGetAllExtractedDataForStudies';
 import { ICurationStubStudy } from '../Curation.types';
 
 export interface IGenericCustomAccessorReturn {
diff --git a/compose/neurosynth-frontend/src/pages/CurationImport/components/CurationImportFinalize.tsx b/compose/neurosynth-frontend/src/pages/CurationImport/components/CurationImportFinalize.tsx
@@ -57,9 +57,9 @@ const CurationImportFinalize: React.FC<{
 
         // // 2. first find duplicates ONLY WITHIN THE IMPORT ITSELF. Label as duplicate.
         let snackbarNotify = false;
-        const duplicatesExist = hasDuplicates(stubs);
-        if (duplicatesExist) snackbarNotify = true;
-        const deduplicatedStubs = duplicatesExist ? automaticallyResolveDuplicates(stubs) : stubs;
+        const duplicatesExistWithinImport = hasDuplicates(stubs);
+        if (duplicatesExistWithinImport) snackbarNotify = true;
+        const deduplicatedStubs = duplicatesExistWithinImport ? automaticallyResolveDuplicates(stubs) : stubs;
 
         // // 3. Label the stubs in the import as duplicates automatically if we find existing stubs IN THE PROJECT
         const allStubsInProject = columns.reduce(
diff --git a/compose/neurosynth-frontend/src/utils/api.ts b/compose/neurosynth-frontend/src/utils/api.ts

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { EAIExtractors, IBehavioralTask, IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedData';`
	`1`	`+import { EAIExtractors, IBehavioralTask, IfMRITask, IGroup } from 'hooks/extractions/useGetAllExtractedDataForStudies';`
`2`	`2`	`import {`
`3`	`3`	`ICurationBoardAIInterfaceCuratorColumnType,`
`4`	`4`	`ICurationTableStudy,`