Merge commit '06e9b2b062971d30f9' into staging

Author: jdkent

compose/backend/pyproject.toml

@@ -46,7 +46,7 @@ dev = [
     "flake8~=7.3",
     "pip-chill~=1.0",
     "wheel~=0.36",
-    "pytest~=8.4"
+    "pytest~=9.0"
 ]
 
 [tool.setuptools.packages.find]

compose/neurosynth-frontend/package-lock.json

@@ -87,7 +87,7 @@ export const sleuthIngestedStudiesToStubs = (
         // although we know that each individual upload is deduplicated,
         // its possible that there are multiple uploads with the same study. We want to deduplicate
         // studies across all uploads so we only allow a unique pmid or doi
-        if ((pmid && allIdentifiersSet.has(pmid)) || (doi && allIdentifiersSet.has(doi))) return;
+        if ((pmid && allIdentifiersSet.has(pmid)) || (doi && allIdentifiersSet.has(doi))) continue;
 
         if (pmid) allIdentifiersSet.add(pmid);
         if (doi) allIdentifiersSet.add(doi);

compose/neurosynth-frontend/src/pages/CurationImport/helpers/SleuthImport.convert.ts

@@ -16,6 +16,7 @@
 from neurostore.database import db
 from neurostore.models import (
     Analysis,
+    Table,
     AnalysisConditions,
     AnnotationAnalysis,
     Annotation,
@@ -296,9 +297,14 @@ def ingest_neurosynth(max_rows=None):
             )
             analyses = []
             points = []
+            tables = {}
 
             for order, (t_id, df) in enumerate(study_coord_data.groupby("table_id")):
-                a = Analysis(name=str(t_id), study=s, order=order, table_id=str(t_id))
+                table = tables.get(t_id) or Table(
+                    t_id=str(t_id), name=str(t_id), study=s, user_id=s.user_id
+                )
+                tables[t_id] = table
+                a = Analysis(name=str(t_id), study=s, order=order, table=table)
                 analyses.append(a)
                 point_idx = 0
                 for _, p in df.iterrows():
@@ -314,6 +320,7 @@ def ingest_neurosynth(max_rows=None):
                     )
                     points.append(point)
                     point_idx += 1
+            to_commit.extend(tables.values())
             to_commit.extend(points)
             to_commit.extend(analyses)
             studies.append(s)
@@ -413,9 +420,14 @@ def ingest_neuroquery(max_rows=None):
         )
         analyses = []
         points = []
+        tables = {}
 
         for order, (t_id, df) in enumerate(study_coord_data.groupby("table_id")):
-            a = Analysis(name=str(t_id), table_id=str(t_id), order=order, study=s)
+            table = tables.get(t_id) or Table(
+                t_id=str(t_id), name=str(t_id), study=s, user_id=s.user_id
+            )
+            tables[t_id] = table
+            a = Analysis(name=str(t_id), table=table, order=order, study=s)
             analyses.append(a)
             point_idx = 0
             for _, p in df.iterrows():
@@ -432,7 +444,9 @@ def ingest_neuroquery(max_rows=None):
                 points.append(point)
                 point_idx += 1
 
-        db.session.add_all([s] + analyses + points + [base_study])
+        db.session.add_all(
+            [s] + analyses + points + list(tables.values()) + [base_study]
+        )
         # db.session.commit()
 
     # make a neuroquery studyset
@@ -556,20 +570,34 @@ def update_study_info(study, metadata_row, text_row, doi, pmcid, year, level):
     def process_coordinates(id_, s, metadata_row):
         analyses = []
         points = []
+        tables = []
         try:
             study_coord_data = coordinates_df.loc[[id_]]
         except KeyError:
             print(f"pmid: {id_} has no coordinates")
-            return analyses, points
+            return analyses, points, tables
         for order, (t_id, df) in enumerate(study_coord_data.groupby("table_id")):
-            a = (
-                Analysis.query.filter_by(
-                    table_id=str(t_id), study_id=s.id
-                ).one_or_none()
-                or Analysis()
+            table = Table.query.filter_by(
+                t_id=str(t_id), study_id=s.id
+            ).one_or_none() or Table(t_id=str(t_id), study=s, user_id=s.user_id)
+            if table not in tables:
+                tables.append(table)
+            if not table.name:
+                table.name = df["table_label"][0] or str(t_id)
+            if table.caption is None:
+                table.caption = (
+                    df["table_caption"][0]
+                    if not df["table_caption"].isna()[0]
+                    else None
+                )
+            existing_analysis = (
+                Analysis.query.filter_by(table_id=table.id, study_id=s.id).one_or_none()
+                if table.id
+                else None
             )
+            a = existing_analysis or Analysis()
             a.name = df["table_label"][0] or str(t_id)
-            a.table_id = str(t_id)
+            a.table = table
             a.order = a.order or order
             a.description = (
                 df["table_caption"][0] if not df["table_caption"].isna()[0] else None
@@ -594,7 +622,7 @@ def process_coordinates(id_, s, metadata_row):
                 )
                 points.append(point)
                 point_idx += 1
-        return analyses, points
+        return analyses, points, tables
 
     to_commit = []
     all_base_studies = []
@@ -663,9 +691,10 @@ def process_coordinates(id_, s, metadata_row):
             s = all_studies.get(pmid, Study())
             update_study_info(s, metadata_row, text_row, doi, pmcid, year, level)
 
-            analyses, points = process_coordinates(pmid, s, metadata_row)
+            analyses, points, tables = process_coordinates(pmid, s, metadata_row)
             to_commit.extend(points)
             to_commit.extend(analyses)
+            to_commit.extend(tables)
             base_study.versions.append(s)
 
     db.session.add_all(to_commit)

store/backend/neurostore/ingest/__init__.py

@@ -5,6 +5,7 @@
     BaseStudy,
     Study,
     Analysis,
+    Table,
     Condition,
     Point,
     Image,
@@ -26,6 +27,7 @@
     "BaseStudy",
     "Study",
     "Analysis",
+    "Table",
     "Condition",
     "Point",
     "Image",

store/backend/neurostore/models/__init__.py

@@ -473,6 +473,14 @@ class Study(BaseMixin, db.Model):
         passive_deletes=True,
         cascade="all, delete-orphan",
     )
+    tables = relationship(
+        "Table",
+        back_populates="study",
+        passive_deletes=True,
+        cascade="all, delete-orphan",
+        cascade_backrefs=False,
+        lazy="selectin",
+    )
 
     __table_args__ = (
         db.CheckConstraint(level.in_(["group", "meta"])),
@@ -510,18 +518,45 @@ class StudysetStudy(db.Model):
     )
 
 
+class Table(BaseMixin, db.Model):
+    __tablename__ = "tables"
+
+    study_id = db.Column(
+        db.Text, db.ForeignKey("studies.id", ondelete="CASCADE"), index=True
+    )
+    t_id = db.Column(db.Text)
+    name = db.Column(db.Text)
+    footer = db.Column(db.Text)
+    caption = db.Column(db.Text)
+    user_id = db.Column(db.Text, db.ForeignKey("users.external_id"), index=True)
+
+    study = relationship(
+        "Study",
+        back_populates="tables",
+        passive_deletes=True,
+    )
+    user = relationship(
+        "User", backref=backref("tables", cascade_backrefs=False, passive_deletes=True)
+    )
+
+    __table_args__ = (
+        db.UniqueConstraint("study_id", "t_id", name="uq_tables_study_t_id"),
+    )
+
+
 class Analysis(BaseMixin, db.Model):
     __tablename__ = "analyses"
 
     study_id = db.Column(
         db.Text, db.ForeignKey("studies.id", ondelete="CASCADE"), index=True
     )
+    table_id = db.Column(
+        db.Text, db.ForeignKey("tables.id", ondelete="SET NULL"), index=True
+    )
     name = db.Column(db.String)
     description = db.Column(db.String)
     metadata_ = db.Column(JSONB)
     order = db.Column(db.Integer)
-    # used to keep track of neurosynth analyses (in case of neurosynth/ace updates)
-    table_id = db.Column(db.String)
     points = relationship(
         "Point",
         backref=backref("analysis", passive_deletes=True),
@@ -553,6 +588,12 @@ class Analysis(BaseMixin, db.Model):
         cascade="all, delete-orphan",
         cascade_backrefs=False,
     )
+    table = relationship(
+        "Table",
+        backref=backref("analyses", cascade_backrefs=False, passive_deletes=True),
+        foreign_keys=[table_id],
+        passive_deletes=True,
+    )
 
 
 class Condition(BaseMixin, db.Model):

store/backend/neurostore/models/data.py

@@ -5,6 +5,7 @@
     BaseStudiesView,
     StudiesView,
     AnalysesView,
+    TablesView,
     ConditionsView,
     ImagesView,
     PointsView,
@@ -29,6 +30,7 @@
     "BaseStudiesView",
     "StudiesView",
     "AnalysesView",
+    "TablesView",
     "ConditionsView",
     "ImagesView",
     "PointsView",