fix: handle duplicate properties in imported files (#1065)

nicoalee · web-flow · commit 8fa20d4f8088 · 2025-08-08T18:36:25.000-04:00
diff --git a/compose/neurosynth-frontend/cypress/e2e/workflows/Curation/ImportStudiesDialog.cy.tsx b/compose/neurosynth-frontend/cypress/e2e/workflows/Curation/ImportStudiesDialog.cy.tsx
@@ -1,4 +1,5 @@
 /// <reference types="cypress" />
+import { expect } from 'chai';
 
 describe('ImportStudiesDialog', () => {
     beforeEach(() => {
@@ -225,10 +226,32 @@ describe('ImportStudiesDialog', () => {
             cy.contains('button', 'next').should('be.visible');
         });
 
-        // TODO : create a test for importing bibtex file
-        // it('should import studies via a file', () => {})
+        it('should upload a .RIS file and import successfully', () => {
+            cy.get('input[role="combobox"]').click();
+            cy.contains('li', 'Scopus').click();
+            cy.get('label[role="button"]').selectFile('cypress/fixtures/standardFiles/ris.ris');
+            cy.contains('button', 'next').should('be.visible').and('not.be.disabled');
+            cy.contains('button', 'next').click();
+        });
 
-        // TODO : create a test for importing RIS file
+        it('should handle the duplicates in an import file', () => {
+            cy.get('input[role="combobox"]').click();
+            cy.contains('li', 'Scopus').click();
+            cy.get('label[role="button"]').selectFile('cypress/fixtures/standardFiles/duplicates.ris');
+            cy.contains('button', 'next').should('be.visible').and('not.be.disabled');
+            cy.contains('button', 'next').click();
+            cy.contains('Click to view 1 imported studies').click();
+            cy.contains('(2023). Manifold Learning for fMRI time-varying FC').should(
+                // doing this to enforce strict equals
+                ($el: JQuery<HTMLElement> | undefined) => {
+                    if (!$el) throw new Error('Could not find element');
+                    expect($el.text()).to.equal('(2023). Manifold Learning for fMRI time-varying FC');
+                }
+            );
+            cy.contains(/^\bbioRxiv\b/).should('exist'); // \b is a word boundary which means there shouldnt be any other letters/words before and after
+        });
+
+        // TODO : create a test for importing bibtex file
         // it('should import studies via a file', () => {})
     });
 
diff --git a/compose/neurosynth-frontend/cypress/fixtures/standardFiles/duplicates.ris b/compose/neurosynth-frontend/cypress/fixtures/standardFiles/duplicates.ris
@@ -0,0 +1,17 @@
+TY  - JOUR
+T1  - Manifold Learning for fMRI time-varying FC
+T1  - DUPLICATE TITLE!!!
+JF  - bioRxiv
+JF  - DUPLICATE JOURNAL!
+DO  - 10.1101/2023.01.14.523992
+SP  - 2023.01.14.523992
+SP  - DUPLICATE DATE!
+AU  - Gonzalez-Castillo, Javier
+AU  - Smith, Michael
+Y1  - 2023/01/01
+Y1  - DUPLICATE OTHER DATE!
+UR  - http://biorxiv.org/content/early/2023/01/16/2023.01.14.523992.abstract
+UR  - DUPLICATE LINK!
+N2  - Whole-brain functional connectivity (FC) measured with functional MRI (fMRI) evolve over time in meaningful ways at temporal scales going from years (e.g., development) to seconds (e.g., within-scan time-varying FC (tvFC)). Yet, our ability to explore tvFC is severely constrained by its large dimensionality (several thousands). To overcome this difficulty, researchers seek to generate low dimensional representations (e.g., 2D and 3D scatter plots) expected to retain its most informative aspects (e.g., relationships to behavior, disease progression). Limited prior empirical work suggests that manifold learning techniques (MLTs)—namely those seeking to infer a low dimensional non-linear surface (i.e., the manifold) where most of the data lies—are good candidates for accomplishing this task. Here we explore this possibility in detail. First, we discuss why one should expect tvFC data to lie on a low dimensional manifold. Second, we estimate what is the intrinsic dimension (i.e., minimum number of latent dimensions; ID) of tvFC data manifolds. Third, we describe the inner workings of three state-of-the-art MLTs: Laplacian Eigenmaps (LE), T-distributed Stochastic Neighbor Embedding (T-SNE), and Uniform Manifold Approximation and Projection (UMAP). For each method, we empirically evaluate its ability to generate neuro-biologically meaningful representations of tvFC data, as well as their robustness against hyper-parameter selection. Our results show that tvFC data has an ID that ranges between 4 and 26, and that ID varies significantly between rest and task states. We also show how all three methods can effectively capture subject identity and task being performed: UMAP and T-SNE can capture these two levels of detail concurrently, but LE could only capture one at a time. We observed substantial variability in embedding quality across MLTs, and within-MLT as a function of hyper-parameter selection. To help alleviate this issue, we provide heuristics that can inform future studies. Finally, we also demonstrate the importance of feature normalization when combining data across subjects and the role that temporal autocorrelation plays in the application of MLTs to tvFC data. Overall, we conclude that while MLTs can be useful to generate summary views of labeled tvFC data, their application to unlabeled data such as resting-state remains challenging.Competing Interest StatementThe authors have declared no competing interest.
+N2  - DUPLICATE ABSTRACT!
+ER  - 
diff --git a/compose/neurosynth-frontend/cypress/fixtures/standardFiles/ris.ris b/compose/neurosynth-frontend/cypress/fixtures/standardFiles/ris.ris
@@ -0,0 +1,56 @@
+TY  - JOUR
+T1  - Distinct neurochemical influences on fMRI response polarity in the striatum
+JF  - bioRxiv
+DO  - 10.1101/2023.02.20.529283
+SP  - 2023.02.20.529283
+AU  - Cerri, Domenic H.
+AU  - Albaugh, Daniel L.
+AU  - Walton, Lindsay R.
+AU  - Katz, Brittany
+AU  - Wang, Tzu-Wen
+AU  - Chao, Tzu-Hao Harry
+AU  - Zhang, Weiting
+AU  - Nonneman, Randal J.
+AU  - Jiang, Jing
+AU  - Lee, Sung-Ho
+AU  - Etkin, Amit
+AU  - Hall, Catherine N.
+AU  - Stuber, Garret D.
+AU  - Shih, Yen-Yu Ian
+Y1  - 2023/01/01
+UR  - http://biorxiv.org/content/early/2023/02/21/2023.02.20.529283.abstract
+N2  - The striatum is the primary input nucleus of the basal ganglia, widely studied for its complex roles in health and disease. Functional magnetic resonance imaging (fMRI) studies are essential for discerning striatal function, however the relationship between neuronal and hemodynamic activity, critical for interpreting fMRI signals, has not been rigorously examined in striatum. We find that optogenetic stimulation of striatal neurons or afferents evokes negative striatal fMRI responses in rats that can occur despite broad increases in local neuronal activity. Intra-striatal pharmacological manipulations suggest that opioidergic, but not dopaminergic transmission contributes to negative striatal fMRI signals (the latter instead associated with positive signals). Striatal neuronal activity peaks are also associated with negative hemodynamic signals in behaving rats. Negative fMRI responses are observed in human striatum under conditions of anticipated neuronal activity increases. Our results prompt consideration of local cellular and neurochemical environments along with neuronal activity in fMRI signal interpretation.Competing Interest StatementThe authors have declared no competing interest.
+ER  - 
+
+TY  - JOUR
+T1  - A systematically optimized awake mouse fMRI paradigm
+JF  - bioRxiv
+DO  - 10.1101/2022.11.16.516376
+SP  - 2022.11.16.516376
+AU  - Xu, Wenjing
+AU  - Pei, Mengchao
+AU  - Zhang, Kaiwei
+AU  - Tong, Chuanjun
+AU  - Bo, Binshi
+AU  - Feng, Jianfeng
+AU  - Zhang, Xiao-Yong
+AU  - Liang, Zhifeng
+Y1  - 2022/01/01
+UR  - http://biorxiv.org/content/early/2022/11/16/2022.11.16.516376.abstract
+N2  - Functional magnetic resonance imaging (fMRI) has been increasingly utilized in mice. Due to the non-negligible effects of anesthetics on mouse fMRI, it is becoming more common to perform fMRI in the awake mice. However, high stress level and head motion in awake mouse fMRI remain to be fully addressed, which limits its practical applications. Therefore, here we presented a systematically optimized awake mouse fMRI paradigm as a practical and open-source solution. First, we designed a soundproof habituation chamber in which multiple mice can be habituated simultaneously and independently. Then, combining corticosterone, body weight and behavioral measurements, we systematically evaluated the potential factors that may contribute to animals’ stress level for awake imaging. Among many factors, we found that the restraining setup allowing forelimbs freely moving and head tilted at 30-degree was optimal for minimizing stress level. Importantly, we implemented multiband simultaneous multi-slice imaging to enable ultrafast fMRI acquisition in awake mice. Compared to conventional single-band EPI, faster acquisition enabled by multiband imaging were more robust to head motion and yielded higher statistical power. Thus, more robust resting-state functional connectivity was detected using multiband acquisition in awake mouse fMRI, compared to conventional single-band acquisition. In conclusion, we presented an awake mouse fMRI paradigm that is highly optimized in both awake mice habituation and fMRI acquisition, and such paradigm minimized animals’ stress level and provided more resistance to head motion and higher statistical power.Competing Interest StatementThe authors have declared no competing interest.
+ER  - 
+
+TY  - JOUR
+T1  - Real-time feedback reduces participant motion during task-based fMRI
+JF  - bioRxiv
+DO  - 10.1101/2023.01.12.523791
+SP  - 2023.01.12.523791
+AU  - Rogers, Chad S.
+AU  - Jones, Michael S.
+AU  - McConkey, Sarah
+AU  - McLaughlin, Drew J.
+AU  - Peelle, Jonathan E.
+Y1  - 2023/01/01
+UR  - http://biorxiv.org/content/early/2023/01/24/2023.01.12.523791.abstract
+N2  - The potential negative impact of head movement during fMRI has long been appreciated. Although a variety of prospective and retrospective approaches have been developed to help mitigate these effects, reducing head movement in the first place remains the most appealing strategy for optimizing data quality. Real-time interventions, in which participants are provided feedback regarding their scan-to-scan motion, have recently shown promise in reducing motion during resting state fMRI. However, whether feedback might similarly reduce motion during task-based fMRI is an open question. In particular, it is unclear whether participants can effectively monitor motion feedback while attending to task-related demands. Here we assessed whether a combination of real-time and between-run feedback could reduce head motion during task-based fMRI. During an auditory word repetition task, 78 adult participants (aged 18–81) were pseudorandomly assigned to receive feedback, implemented in FIRMM software, or not. We quantified movement using framewise displacement (FD). We found that motion feedback resulted in a statistically significant reduction in participant head motion, with a small-to-moderate effect size (reducing average FD from 0.347 to 0.282). There was some evidence for a change of effect over the course of six runs, but no clear evidence for practice effects based on motion feedback. We conclude that under some circumstances real-time feedback may reduce head motion during task-based fMRI, although its effectiveness may depend on the specific participant population and task demands of a given study.Competing Interest StatementThe authors have declared no competing interest.
+ER  - 
diff --git a/compose/neurosynth-frontend/src/pages/CurationImport/components/CurationImportStandardFormat.tsx b/compose/neurosynth-frontend/src/pages/CurationImport/components/CurationImportStandardFormat.tsx
@@ -14,6 +14,18 @@ import '@citation-js/plugin-enw';
 import '@citation-js/plugin-bibtex';
 import '@citation-js/plugin-ris';
 
+const normalize = (t: unknown): string => {
+    if (typeof t === 'string') return t;
+    if (Array.isArray(t)) return t.find((x) => typeof x === 'string' && x.trim()) || '';
+    return '';
+};
+
+const extractYear = (dateString: string | undefined) => {
+    if (!dateString) return '';
+    const match = dateString.match(/\b\d{4}\b/);
+    return match ? match[0] : '';
+};
+
 enum EValidationReason {
     EMPTY = 'Input is empty',
     INCORRECT = 'Format is incorrect or unsupported',
@@ -24,6 +36,7 @@ interface CSLJSONDateParts {
         | [string | number, number | number, string | number][]
         | [string | number, string | number][]
         | [string | number][];
+    raw?: string | string[];
 }
 
 interface CSLJSON {
@@ -92,27 +105,38 @@ const CurationImportStandardFormat: React.FC<{
             const citeObj = new Cite(uploadState.rawIdText);
             citeObj.format('data', { format: 'object' });
 
-            const formattedArticles: ICurationStubStudy[] = ((citeObj.data as CSLJSON[]) || []).map((article) => ({
-                id: uuidv4(),
-                title: article.title || article['title-short'] || '',
-                authors: (article.author || []).reduce((acc, curr, index, arr) => {
-                    const middleParticle = curr['non-dropping-particle'] || curr['dropping-particle'] || '';
-                    return `${acc}${curr.given} ${middleParticle ? middleParticle + ' ' : ''}${
-                        curr.family
-                    }${index < arr.length - 1 ? ', ' : ''}`;
-                }, ''),
-                keywords: '',
-                pmid: article.PMID || '',
-                pmcid: article.PMCID || '',
-                doi: article.DOI || '',
-                articleYear: `${article?.issued?.['date-parts']?.[0]?.[0] || ''}`,
-                journal: article['container-title'] || '',
-                abstractText: article.abstract || article.annote || '',
-                articleLink: article.URL || (article.PMID ? `https://pubmed.ncbi.nlm.nih.gov/${article.PMID}` : ''),
-                exclusionTag: null,
-                identificationSource: source,
-                tags: [],
-            }));
+            const formattedArticles: ICurationStubStudy[] = ((citeObj.data as CSLJSON[]) || []).map((article) => {
+                const titleRaw = article.title ?? article['title-short'];
+                const articleYear =
+                    article?.issued?.['date-parts']?.[0]?.[0]?.toString() ??
+                    extractYear(normalize(article?.issued?.raw));
+                const articleJournal = normalize(article['container-title']);
+                const articleAbstract = article.abstract ?? article.annote;
+                const articleLink =
+                    article.URL ?? (article.PMID ? `https://pubmed.ncbi.nlm.nih.gov/${normalize(article.PMID)}` : '');
+
+                return {
+                    id: uuidv4(),
+                    title: normalize(titleRaw),
+                    authors: (article.author || []).reduce((acc, curr, index, arr) => {
+                        const middleParticle = curr['non-dropping-particle'] || curr['dropping-particle'] || '';
+                        return `${acc}${curr.given} ${middleParticle ? middleParticle + ' ' : ''}${
+                            curr.family
+                        }${index < arr.length - 1 ? ', ' : ''}`;
+                    }, ''),
+                    keywords: '',
+                    pmid: normalize(article.PMID),
+                    pmcid: normalize(article.PMCID),
+                    doi: normalize(article.DOI),
+                    articleYear: articleYear ?? '',
+                    journal: articleJournal ?? '',
+                    abstractText: normalize(articleAbstract),
+                    articleLink: articleLink,
+                    exclusionTag: null,
+                    identificationSource: source,
+                    tags: [],
+                };
+            });
 
             setUploadState((prev) => ({
                 ...prev,
