Merge branch 'master' into FEAT-1193-allow-users-to-edit-exclusion-labels

Author: nicoalee

.github/workflows/deploy-to-staging.yml

@@ -70,8 +70,6 @@ jobs:
 
           # Wait for PostgreSQL to be ready
           until docker compose exec store-pgsql pg_isready; do sleep 5; done
-
-          docker compose exec neurostore flask db migrate
           docker compose exec neurostore flask db upgrade
         '
 
@@ -86,8 +84,6 @@ jobs:
 
           # Wait for PostgreSQL to be ready
           until docker compose exec compose_pgsql pg_isready; do sleep 5; done
-
-          docker compose exec compose flask db migrate
           docker compose exec compose flask db upgrade
         '
 

.github/workflows/workflow.yml

@@ -473,39 +473,29 @@ jobs:
 
         docker compose exec -T \
         compose_pgsql17 \
-        psql -U postgres -c "create database test_db"
+        psql -U postgres -c "CREATE DATABASE test_db"
     -
       name: Create Store Database
       run: |
         cd store
 
         docker compose exec -T \
         store-pgsql17 \
-        psql -U postgres -c "create database test_db"
+        psql -U postgres -c "CREATE DATABASE test_db"
 
         docker compose exec -T \
         store-pgsql17 \
         psql -U postgres -d test_db -c "CREATE EXTENSION IF NOT EXISTS vector;"
     -
-      name: Initialize Compose Database
+      name: Apply Compose migrations
       run: |
         cd compose
-        docker compose exec -T compose \
-        bash -c \
-            "flask db merge heads && \
-             flask db stamp head && \
-             flask db migrate && \
-             flask db upgrade"
+        docker compose exec -T compose flask db upgrade
     -
-      name: Initialize Store Database
+      name: Apply Store migrations
       run: |
         cd store
-        docker compose exec -T neurostore \
-        bash -c \
-            "flask db merge heads && \
-             flask db stamp head && \
-             flask db migrate && \
-             flask db upgrade"
+        docker compose exec -T neurostore flask db upgrade
     - 
       name: Ingest data into Store
       run: |

.gitignore

@@ -71,7 +71,7 @@ instance/
 .webassets-cache
 
 # Postgres Python migrations:
-**/backend/migrations/versions/*
+# **/backend/migrations/versions/*
 
 # Scrapy stuff:
 .scrapy

compose/backend/README.md

@@ -27,39 +27,57 @@ The server should now be running at http://localhost:81
 
 Create the database for compose:
 
-    docker-compose exec compose_pgsql psql -U postgres -c "create database compose"
+    docker-compose exec compose_pgsql17 psql -U postgres -c "create database compose"
 
-Next, migrate and upgrade the database migrations.
+Next, apply the existing migrations (they are the canonical schema definition):
 
-    docker-compose exec compose \
-        bash -c \
-            "flask db merge heads && \
-             flask db stamp head && \
-             flask db migrate && \
-             flask db upgrade"
-
-**Note**: `flask db merge heads` is not strictly necessary
-unless you have multiple schema versions that are not from the same history
-(e.g., multiple files in the `versions` directory).
-However, `flask db merge heads` makes the migration more robust
-when there are multiple versions from different histories.
+    docker-compose exec compose flask db upgrade
 
 
 ## Maintaining docker image and db
 If you make a change to compose, you should be able to simply restart the server.
 
     docker-compose restart compose
 
-If you need to upgrade the db after changing any models:
+If you change any models, generate a new Alembic migration and migrate the database (commit the generated revision file so it becomes the new source of truth):
 
     docker-compose exec compose flask db migrate
     docker-compose exec compose flask db upgrade
 
 
+## Database migrations
+
+The migrations stored in `backend/migrations` are the **only** source of truth for the schema—avoid merging heads, stamping, or manually altering the history. Always move the database forward (or rebuild from scratch) by applying the tracked revisions.
+
+### Applying migrations after pulling a branch
+
+Any time you start the backend or pull the latest changes, bring the database to the expected state with:
+
+```sh
+docker-compose exec compose flask db upgrade
+```
+
+`upgrade` is idempotent, so rerunning it is harmless; it only applies migrations that have not been run yet.
+
+### Resetting the database when switching branches
+
+Because each branch might change the schema independently, recreate the database before starting work on a different branch so that Alembic can replay only the migrations that exist on that branch.
+
+```sh
+docker-compose stop compose
+docker-compose exec compose_pgsql17 psql -U postgres -c "DROP DATABASE IF EXISTS compose;"
+docker-compose exec compose_pgsql17 psql -U postgres -c "CREATE DATABASE compose;"
+docker-compose start compose
+docker-compose exec compose flask db upgrade
+```
+
+If you're using the legacy Postgres container, replace `compose_pgsql17` with `compose_pgsql` in the commands above.
+
+
 ## Running tests
 To run tests, after starting services, create a test database:
 
-    docker-compose exec compose_pgsql psql -U postgres -c "create database test_db"
+    docker-compose exec compose_pgsql17 psql -U postgres -c "create database test_db"
 
 **NOTE**: This command will ask you for the postgres password which is defined
 in the `.env` file.

compose/backend/manage.py

@@ -14,6 +14,9 @@
 from neurosynth_compose.database import db
 from neurosynth_compose import models
 from neurosynth_compose.ingest import neurostore as ingest_nstore
+from neurosynth_compose.scripts.backfill_extraction_metadata import (
+    add_missing_extraction_ids,
+)
 
 
 app.config.from_object(os.environ["APP_SETTINGS"])
@@ -44,3 +47,12 @@ def create_meta_analyses(n_studysets, neurostore_url):
     if n_studysets is not None:
         n_studysets = int(n_studysets)
     ingest_nstore.create_meta_analyses(url=neurostore_url, n_studysets=n_studysets)
+
+
+@app.cli.command("backfill-extraction-metadata")
+def backfill_extraction_metadata():
+    """Add missing extractionMetadata ids to project provenance."""
+    updated, skipped = add_missing_extraction_ids()
+    click.echo(
+        f"Updated {updated} project(s); skipped {skipped} project(s) with no changes."
+    )

compose/backend/migrations/versions/0717f5de725b_.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 0717f5de725b
+Revises: c1fb13263539
+Create Date: 2023-12-12 22:59:12.319399
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = '0717f5de725b'
+down_revision = 'c1fb13263539'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('specification_conditions', sa.Column('user_id', sa.Text(), nullable=True))
+    op.create_foreign_key(None, 'specification_conditions', 'users', ['user_id'], ['external_id'])
+    op.add_column('specifications', sa.Column('database_studyset', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('specifications', 'database_studyset')
+    op.drop_constraint(None, 'specification_conditions', type_='foreignkey')
+    op.drop_column('specification_conditions', 'user_id')
+    # ### end Alembic commands ###

compose/backend/migrations/versions/164fb969f549_.py

@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: 164fb969f549
+Revises: e1c296d89bf1
+Create Date: 2023-05-18 19:15:07.407065
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = '164fb969f549'
+down_revision = 'e1c296d89bf1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('meta_analysis_results', sa.Column('diagnostic_table', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('meta_analysis_results', 'diagnostic_table')
+    # ### end Alembic commands ###

compose/backend/migrations/versions/26aa47ee8e3e_.py

@@ -0,0 +1,45 @@
+"""empty message
+
+Revision ID: 26aa47ee8e3e
+Revises: 76c16b00a2bc
+Create Date: 2022-12-16 18:22:04.536097
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = '26aa47ee8e3e'
+down_revision = '76c16b00a2bc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('projects',
+    sa.Column('id', sa.Text(), nullable=False),
+    sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
+    sa.Column('updated_at', sa.DateTime(timezone=True), nullable=True),
+    sa.Column('name', sa.Text(), nullable=True),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('provenance', sa.JSON(), nullable=True),
+    sa.Column('user_id', sa.Text(), nullable=True),
+    sa.ForeignKeyConstraint(['user_id'], ['users.external_id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('meta_analyses', sa.Column('project_id', sa.Text(), nullable=True))
+    op.add_column('meta_analyses', sa.Column('provenance', sa.JSON(), nullable=True))
+    op.create_foreign_key(None, 'meta_analyses', 'projects', ['project_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'meta_analyses', type_='foreignkey')
+    op.drop_column('meta_analyses', 'provenance')
+    op.drop_column('meta_analyses', 'project_id')
+    op.drop_table('projects')
+    # ### end Alembic commands ###

compose/backend/migrations/versions/28251f5cb6d5_.py

@@ -0,0 +1,39 @@
+"""empty message
+
+Revision ID: 28251f5cb6d5
+Revises: 3d0fc483f06b
+Create Date: 2025-10-13 19:11:58.785877
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = '28251f5cb6d5'
+down_revision = '3d0fc483f06b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('projects', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('studyset_id', sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column('annotation_id', sa.Text(), nullable=True))
+        batch_op.create_foreign_key(None, 'annotations', ['annotation_id'], ['id'])
+        batch_op.create_foreign_key(None, 'studysets', ['studyset_id'], ['id'])
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('projects', schema=None) as batch_op:
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_column('annotation_id')
+        batch_op.drop_column('studyset_id')
+
+    # ### end Alembic commands ###

compose/backend/migrations/versions/3d0fc483f06b_.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 3d0fc483f06b
+Revises: 52eabbbdffe6
+Create Date: 2025-08-21 22:16:48.621497
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = '3d0fc483f06b'
+down_revision = '52eabbbdffe6'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(op.f('ix_projects_draft'), 'projects', ['draft'], unique=False)
+    op.create_index(op.f('ix_projects_public'), 'projects', ['public'], unique=False)
+    op.create_index(op.f('ix_projects_user_id'), 'projects', ['user_id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_projects_user_id'), table_name='projects')
+    op.drop_index(op.f('ix_projects_public'), table_name='projects')
+    op.drop_index(op.f('ix_projects_draft'), table_name='projects')
+    # ### end Alembic commands ###