Add gzip JSON support and test (#992)

* Add gzip JSON support and test

* Fix lint issues

* Fix TypeError issues

* Fix Exception issues

* Fix Lint and TypeError issue

* fix lint issues

* Fix lint issues

* Fix Studyset issues

* enhance gzip test

* delete test_gz.py

* refactor: standardise JSON loading with load_json utility

* add import json

* fix lint issues and add import json

* Add import json

Author: lyraluoyu

examples/01_datasets/06_plot_dataset_json.py

@@ -133,12 +133,11 @@
 import json
 import os
 
-from nimare.utils import get_resource_path
+from nimare.utils import get_resource_path, load_json
 
 dset_file = os.path.join(get_resource_path(), "nidm_pain_dset.json")
 
-with open(dset_file, "r") as f_obj:
-    data = json.load(f_obj)
+data = load_json(dset_file)
 
 ###############################################################################
 # Example of accessing coordinates for a study

nimare/dataset.py

@@ -2,7 +2,6 @@
 
 import copy
 import inspect
-import json
 import logging
 import os.path as op
 import warnings
@@ -23,6 +22,7 @@
     _validate_images_df,
     get_masker,
     get_template,
+    load_json,
     mm2vox,
 )
 
@@ -79,12 +79,11 @@ class Dataset(NiMAREBase):
 
     def __init__(self, source, target="mni152_2mm", mask=None):
         if isinstance(source, str):
-            with open(source, "r") as f_obj:
-                data = json.load(f_obj)
+            data = load_json(source)
         elif isinstance(source, dict):
             data = source
         else:
-            raise Exception("`source` needs to be a file path or a dictionary")
+            raise TypeError("`source` needs to be a file path or a dictionary")
 
         # Datasets are organized by study, then experiment
         # To generate unique IDs, we combine study ID with experiment ID

nimare/extract/extract.py

@@ -1,7 +1,6 @@
 """Tools for downloading datasets."""
 
 import itertools
-import json
 import logging
 import os
 import os.path as op
@@ -22,7 +21,7 @@
     _get_dataset_dir,
     _longify,
 )
-from nimare.utils import get_resource_path
+from nimare.utils import get_resource_path, load_json
 
 LGR = logging.getLogger(__name__)
 
@@ -61,8 +60,7 @@ def _find_entities(filename, search_pairs, log=False):
 def _fetch_database(search_pairs, database_url, out_dir, overwrite=False):
     """Fetch generic database."""
     res_dir = get_resource_path()
-    with open(op.join(res_dir, "database_file_manifest.json"), "r") as fo:
-        database_file_manifest = json.load(fo)
+    database_file_manifest = load_json(op.join(res_dir, "database_file_manifest.json"))
 
     out_dir = op.abspath(out_dir)
     os.makedirs(out_dir, exist_ok=True)

nimare/nimads.py

@@ -6,6 +6,7 @@
 import os
 import weakref
 from copy import deepcopy
+from pathlib import Path
 
 import numpy as np
 import pandas as pd
@@ -18,6 +19,7 @@
     _mask_img_to_bool,
     _validate_df,
     _validate_images_df,
+    load_json,
     mm2vox,
 )
 
@@ -28,6 +30,9 @@
 
 def _validate_studyset_source(source):
     """Validate the minimal schema required to construct a Studyset."""
+    if isinstance(source, (str, Path)):
+        return
+
     if not isinstance(source, dict):
         raise InvalidStudysetError("Studyset source must be a dictionary or JSON path")
 
@@ -163,8 +168,7 @@ def __init__(
 
         # load source as json
         if isinstance(source, str):
-            with open(source, "r+") as f:
-                source = json.load(f)
+            source = load_json(source)
 
         _validate_studyset_source(source)
 
@@ -350,8 +354,7 @@ def _coerce_annotation(self, annotation):
         if isinstance(annotation, dict):
             return Annotation(annotation, self)
         if isinstance(annotation, str):
-            with open(annotation, "r+") as f:
-                return Annotation(json.load(f), self)
+            return Annotation(load_json(annotation), self)
         if isinstance(annotation, Annotation):
             return annotation
         raise TypeError(f"Unsupported annotation type: {type(annotation)}")
@@ -863,8 +866,7 @@ def from_table_cache(
     @classmethod
     def from_nimads(cls, filename):
         """Create a Studyset from a NIMADS JSON file."""
-        with open(filename, "r+") as fn:
-            nimads = json.load(fn)
+        nimads = load_json(filename)
 
         return cls(nimads)
 

nimare/tests/conftest.py

@@ -1,7 +1,6 @@
 """Generate fixtures for tests."""
 
 import copy
-import json
 import os
 from shutil import copyfile
 
@@ -15,7 +14,7 @@
 import nimare
 from nimare.generate import create_coordinate_dataset
 from nimare.tests.utils import get_test_data_path
-from nimare.utils import get_resource_path
+from nimare.utils import get_resource_path, load_json
 
 # Only enable the following once in a while for a check for SettingWithCopyWarnings
 # pd.options.mode.chained_assignment = "raise"
@@ -197,8 +196,7 @@ def sample_size_nimads_studyset():
         response = request("GET", url)
         with open(out_file, "wb") as f:
             f.write(response.content)
-    with open(out_file, "r") as f:
-        studyset = json.load(f)
+    studyset = load_json(out_file)
     return studyset
 
 
@@ -211,8 +209,7 @@ def _example_nimads_studyset_data():
         response = request("GET", url)
         with open(out_file, "wb") as f:
             f.write(response.content)
-    with open(out_file, "r") as f:
-        studyset = json.load(f)
+    studyset = load_json(out_file)
     return studyset
 
 
@@ -231,8 +228,7 @@ def example_nimads_annotation():
         response = request("GET", url)
         with open(out_file, "wb") as f:
             f.write(response.content)
-    with open(out_file, "r") as f:
-        annotation = json.load(f)
+    annotation = load_json(out_file)
     return annotation
 
 

nimare/tests/test_dataset.py

@@ -1,7 +1,6 @@
 """Test nimare.dataset (Dataset IO/transformations)."""
 
 import copy
-import json
 import os.path as op
 import pickle
 import warnings
@@ -14,6 +13,7 @@
 from nimare import dataset
 from nimare.nimads import Studyset
 from nimare.tests.utils import get_test_data_path
+from nimare.utils import load_json
 
 # ---------------------------------------------------------------------------
 # Helpers for the parameterized smoke test
@@ -188,8 +188,7 @@ def test_empty_dset():
 def test_posneg_warning():
     """Smoke test for nimare.dataset.Dataset initialization with positive and negative z_stat."""
     db_file = op.join(get_test_data_path(), "neurosynth_dset.json")
-    with open(db_file, "r") as f_obj:
-        data = json.load(f_obj)
+    data = load_json(db_file)
 
     data_pos_zstats = copy.deepcopy(data)
     data_neg_zstats = copy.deepcopy(data)

nimare/tests/test_nimads.py

@@ -12,6 +12,7 @@
 from nimare import nimads
 from nimare.dataset import Dataset
 from nimare.tests.utils import get_test_data_path
+from nimare.utils import load_json
 
 
 def test_load_nimads(example_nimads_studyset, example_nimads_annotation):
@@ -336,8 +337,7 @@ def test_studyset_to_nimads(example_nimads_studyset):
         assert os.path.exists(tmp_path)
 
         # Verify the saved file can be loaded
-        with open(tmp_path, "r") as f:
-            saved_data = json.load(f)
+        saved_data = load_json(tmp_path)
 
         assert saved_data["id"] == studyset.id
         assert saved_data["name"] == studyset.name

nimare/utils.py

@@ -1,6 +1,7 @@
 """Utility functions for NiMARE."""
 
 import datetime
+import gzip
 import inspect
 import json
 import logging
@@ -62,6 +63,16 @@ def get_resource_path():
     return op.abspath(op.join(op.dirname(__file__), "resources") + op.sep)
 
 
+def load_json(source):
+    """Load a JSON file (plain or gzipped) and return parsed content."""
+    source_str = str(source)
+    if source_str.endswith(".gz"):
+        with gzip.open(source, "rt", encoding="utf-8") as f_obj:
+            return json.load(f_obj)
+    with open(source, "r", encoding="utf-8") as f_obj:
+        return json.load(f_obj)
+
+
 def get_template(space="mni152_2mm", mask=None):
     """Load template file.
 
@@ -1315,8 +1326,7 @@ def load_nimads(studyset, annotation=None):
     if isinstance(studyset, dict):
         studyset = Studyset(studyset)
     elif isinstance(studyset, str):
-        with open(studyset, "r") as f:
-            studyset = Studyset(json.load(f))
+        studyset = Studyset(load_json(studyset))
     elif isinstance(studyset, Studyset):
         pass
     else: