[FIX] Be more forgiving with white spaces and newlines in Sleuth Files (#1121)

* replace windows (\r\n) /old mac (\r) return carriages with (\n)

* fix: added cypress test case

* be more forgiving on whitespace

* fix extra parens

* remove the only function

* be more restrictive on metadata

* rollback messages

* give error that matches tests

* Revert "give error that matches tests"

This reverts commit cf460d5.

* revert this change

* fix: clean up helper file

---------

Co-authored-by: Nicholas Lee <[email protected]>

Author: jdkent

compose/neurosynth-frontend/cypress/e2e/workflows/SleuthImport/DoSleuthImport.cy.tsx

@@ -29,7 +29,7 @@ describe('DoSleuthImport', () => {
             cy.contains('button', 'next').click();
         });
 
-        describe('should upload invalid sleuth files', () => {
+        describe('should handle invalid sleuth files', () => {
             it('should upload a file and show invalid with no reference', () => {
                 cy.get('input[type="file"]').selectFile(
                     'cypress/fixtures/DoSleuthImport/sleuthFiles/invalidSleuthFileNoReference.txt',
@@ -142,6 +142,24 @@ describe('DoSleuthImport', () => {
                 cy.get('[data-testid="InsertDriveFileIcon"]').should('have.length', 2);
             });
 
+            it('should upload a valid sleuth file with windows line endings', () => {
+                cy.get('input[type="file"]').selectFile(
+                    'cypress/fixtures/DoSleuthImport/sleuthFiles/validSleuthFileWindowsLineEndings.txt',
+                    { force: true }
+                );
+                cy.get('[data-testid="InsertDriveFileIcon"]').should('exist').and('be.visible');
+                cy.contains('button', 'create project').should('be.enabled');
+            });
+
+            it('should upload a valid sleuth file with wonky white space', () => {
+                cy.get('input[type="file"]').selectFile(
+                    'cypress/fixtures/DoSleuthImport/sleuthFiles/validSleuthFileWonkyWhiteSpace.txt',
+                    { force: true }
+                );
+                cy.get('[data-testid="InsertDriveFileIcon"]').should('exist').and('be.visible');
+                cy.contains('button', 'create project').should('be.enabled');
+            });
+
             it('should enable if a valid file is uploaded', () => {
                 cy.get('input[type="file"]').selectFile(
                     'cypress/fixtures/DoSleuthImport/sleuthFiles/validSleuthFileWithDOI.txt',

compose/neurosynth-frontend/cypress/fixtures/DoSleuthImport/sleuthFiles/validSleuthFileWindowsLineEndings.txt

@@ -0,0 +1,21 @@
+// Reference=MNI
+// DOI=10.1016/1234567
+// PubMedId=67123237
+// Smith et al., 2019: Working Memory vs Baseline
+// Subjects=23
+-7.5	-8.5	-9.5
+10	-12	-62
+21	-14	-2
+0	-9	16
+
+
+// DOI=10.217/1234568
+// PubMedId=23782389
+// Roberts et al., 1995: 2 Back vs 1 Back
+// Graeff et al., 2000: 1 Back vs 0 Back
+// Edwards et al., 2017: 2 Back vs 0 Back
+// Subjects=62
+82	12	0
+-27	34	72
+-7	-8	-9
+10	-12	-62

compose/neurosynth-frontend/cypress/fixtures/DoSleuthImport/sleuthFiles/validSleuthFileWonkyWhiteSpace.txt

@@ -0,0 +1,14 @@
+// Reference=MNI
+// DOI=10.9999/example-doi-0001 
+// PubMedID=12345678	
+// Doe et al., 2021: sleep disorder > controls Table 1
+//Subjects=12	
+-12	28	54  
+3	 12 		27	
+
+// DOI=10.8888/example-doi-0002
+// PubMedID=23456789
+// Roe et al., 2022: sleep disorder < controls
+// Subjects = 10
+18 -12	63 
+-42	-57	0	

compose/neurosynth-frontend/src/helpers/requests.ts

@@ -0,0 +1,32 @@
+export const executeHTTPRequestsAsBatches = async <T, Y>(
+    requestList: T[],
+    mapFunc: (request: T) => Promise<Y>,
+    rateLimit: number,
+    delayInMS?: number,
+    progressCallbackFunc?: (progress: number) => void
+) => {
+    const arrayOfRequestArrays = [];
+    for (let i = 0; i < requestList.length; i += rateLimit) {
+        arrayOfRequestArrays.push(requestList.slice(i, i + rateLimit));
+    }
+
+    const batchedResList: Y[] = [];
+    for (const requests of arrayOfRequestArrays) {
+        /**
+         * I have to do the mapping from object to HTTP request here because
+         * the promises are not lazy. The HTTP requests are launched as soon as
+         * the function is called regardless of whether a .then() is added
+         */
+        const batchedRes = await Promise.all(requests.map(mapFunc));
+        batchedResList.push(...batchedRes);
+        if (progressCallbackFunc) {
+            progressCallbackFunc(Math.round((batchedResList.length / requestList.length) * 100));
+        }
+        if (delayInMS) {
+            await new Promise((res) => {
+                setTimeout(() => res(null), delayInMS);
+            });
+        }
+    }
+    return batchedResList;
+};

compose/neurosynth-frontend/src/hooks/metaAnalyses/useGetNeurovaultImages.tsx

@@ -1,5 +1,5 @@
 import axios from 'axios';
-import { executeHTTPRequestsAsBatches } from 'pages/SleuthImport/SleuthImport.helpers';
+import { executeHTTPRequestsAsBatches } from 'helpers/requests';
 import { useQuery } from 'react-query';
 
 export interface INeurovault {