[ENH] add cluster table metadata (#1045)

* oops mixed changes

* keep track of filename with cluster table

* reference correct code

* run black

* fix flake8 errors

* refactor celery to work with app

* remove the unix socket

Author: jdkent

compose/backend/Dockerfile

@@ -33,4 +33,4 @@ RUN pip install --no-cache-dir .[dev]
 # Now copy the rest of the app
 COPY . /compose/backend/
 
-RUN pip install --no-cache-dir .
+RUN pip install --no-cache-dir -e .

compose/backend/neurosynth_compose/celery_app.py

@@ -0,0 +1,19 @@
+from celery import Celery
+
+
+def make_celery(app):
+    celery = Celery(
+        app.import_name,
+        backend=app.config.get("CELERY_RESULT_BACKEND"),
+        broker=app.config.get("CELERY_BROKER_URL"),
+    )
+    celery.conf.update(app.config)
+    TaskBase = celery.Task
+
+    class ContextTask(TaskBase):
+        def __call__(self, *args, **kwargs):
+            with app.app_context():
+                return TaskBase.__call__(self, *args, **kwargs)
+
+    celery.Task = ContextTask
+    return celery

compose/backend/neurosynth_compose/core.py

@@ -1,8 +1,5 @@
-from flask_celeryext import FlaskCeleryExt
-
 from .__init__ import create_app
+from .celery_app import make_celery
 
 app = create_app()
-
-ext_celery = FlaskCeleryExt(app)
-celery_app = ext_celery.celery
+celery_app = make_celery(app)

compose/backend/neurosynth_compose/models/analysis.py

@@ -230,9 +230,7 @@ class Project(BaseMixin, db.Model):
     name = db.Column(db.Text)
     description = db.Column(db.Text)
     provenance = db.Column(db.JSON)
-    user_id = db.Column(
-        db.Text, db.ForeignKey("users.external_id"), index=True
-    )
+    user_id = db.Column(db.Text, db.ForeignKey("users.external_id"), index=True)
     public = db.Column(db.Boolean, default=True, index=True)
     draft = db.Column(db.Boolean, default=True, index=True)
 

compose/backend/neurosynth_compose/resources/analysis.py

@@ -48,6 +48,10 @@
     NeurostoreStudySchema,
     ProjectSchema,
 )
+from ..tasks.neurostore import create_or_update_neurostore_analysis
+from ..tasks.neurovault import file_upload_neurovault
+from celery import group
+
 from .singular import singularize
 
 
@@ -521,9 +525,6 @@ def post(self):
         return self.__class__._schema().dump(record)
 
     def put(self, id):
-        from .tasks import file_upload_neurovault, create_or_update_neurostore_analysis
-        from celery import group
-
         result = self._model.query.filter_by(id=id).one()
 
         if request.files:

compose/backend/neurosynth_compose/resources/tasks.py

@@ -1,14 +1,3 @@
 """Celery tasks package."""
 
-from .neurovault import file_upload_neurovault
-from .neurostore import create_or_update_neurostore_analysis
-from ..core import celery_app
-
-__all__ = [
-    "file_upload_neurovault",
-    "create_or_update_neurostore_analysis",
-]
-
-# Register tasks
-celery_app.tasks.register(file_upload_neurovault)
-celery_app.tasks.register(create_or_update_neurostore_analysis)
+# Celery tasks are now registered via decorators or autodiscovery.

compose/backend/neurosynth_compose/tasks/__init__.py

@@ -27,10 +27,8 @@ def __call__(self, *args, **kwargs):
             result = self._orig_run(*args, **kwargs)
             bound_logger.info("task_completed", extra={"task_name": self.name})
             return result
-        except Exception as exc:
-            bound_logger.exception(
-                "task_failed", extra={"task_name": self.name, "exc_info": exc}
-            )
+        except Exception:
+            bound_logger.exception("task_failed", extra={"task_name": self.name})
             raise
 
     def on_failure(self, exc, task_id, args, kwargs, einfo):