[REF] record neurovault ingestion (#688)

* record neurovault ingestion

* style

* ignore recording for spelling

Author: jdkent

.codespellrc

@@ -1,5 +1,5 @@
 [codespell]
-skip = .git,*.pdf,*.svg,*.ts,*.tsx,package-lock.json,data-*vocabulary.txt,sample_*.csv,CHANGELOG.rst
+skip = .git,*.pdf,*.svg,*.ts,*.tsx,package-lock.json,data-*vocabulary.txt,sample_*.csv,CHANGELOG.rst,ingest_neurovault.yml
 # regexes
 ignore-regex = r"\(\?i\).*\\\\1
 ignore-words-list = te,fwe,connexion,zoon,covert,rime

store/cassettes/ingest_neurovault.yml

@@ -29,7 +29,7 @@
 from neurostore.models.data import StudysetStudy, _check_type
 
 
-def ingest_neurovault(verbose=False, limit=20, overwrite=False):
+def ingest_neurovault(verbose=False, limit=20, overwrite=False, max_images=None):
     # Store existing studies for quick lookup
     all_studies = {s.doi: s for s in Study.query.filter_by(source="neurovault").all()}
 
@@ -141,7 +141,9 @@ def add_collection(data):
                 [
                     add_collection(c)
                     for c in data["results"]
-                    if c["DOI"] is not None and c["number_of_images"]
+                    if c["DOI"] is not None
+                    and c["number_of_images"] > 0
+                    and (max_images is None or c["number_of_images"] < max_images)
                 ],
             )
         )

store/neurostore/ingest/__init__.py

@@ -28,4 +28,6 @@ wrapt~=1.12
 openapi-spec-validator~=0.3
 scipy~=1.9
 pytest~=7.1
+vcrpy~=6.0
+pytest-recording~=0.13.1
 orjson~=3.8

store/neurostore/requirements.txt

@@ -167,7 +167,10 @@ class BaseDataSchema(BaseSchema):
         attribute="user_id", dump_only=True, metadata={"info_field": True}
     )
     username = fields.String(
-        attribute="user.name", dump_only=True, metadata={"info_field": True}, default=None,
+        attribute="user.name",
+        dump_only=True,
+        metadata={"info_field": True},
+        default=None,
     )
 
 

store/neurostore/schemas/data.py

@@ -19,6 +19,7 @@
 )
 from auth0.v3.authentication import GetToken
 import shortuuid
+import vcr
 
 import logging
 
@@ -336,8 +337,9 @@ def ingest_neurosynth(session):
 
 
 @pytest.fixture(scope="function")
+@vcr.use_cassette("cassettes/ingest_neurovault.yml")
 def ingest_neurovault(session):
-    return ingest.ingest_neurovault(limit=5)
+    return ingest.ingest_neurovault(limit=5, max_images=50)
 
 
 @pytest.fixture(scope="function")