Entry points for testing accept configuration files at runtime (Inria-Empenn#242)

* Runner allows for configuration at runtime

* Other scripts allow for configuration at runtime

* Documentation update

* New actions versions

* Custom configuration allowed for testing

* DOC + Adding config argument in narps_open_tester

Author: bclenet

docs/testing.md

@@ -42,6 +42,7 @@ Tests can be launched manually or while using CI (Continuous Integration).
 * To specify a test -for which the name contains 'test_pattern'- inside a test file : `pytest test_file.py -k "test_pattern"`
 * To run a tests with a given mark 'mark' : `pytest -m 'mark'`
 * To create code coverage data : `coverage run -m pytest ./tests` then `coverage report` to see the code coverage result or `coverage xml` to output a .xml report file
+* To use a custom configuration file (other than `narps_open/utils/configuration/test_config.toml` that is used by default): `pytest --narps_open_config=<path_to_the_custom_config_file>`
 
 ## Command line tools
 
@@ -62,6 +63,7 @@ The command line tool `narps_open_correlations` is also available and can be use
 
 ```bash
 narps_open_correlations -t 2T6S -n 60
+narps_open_correlations -t 2T6S -n 60 --config <path_to_a_custom_config_file>
 ```
 
 to get the correlation values for the results of a previously executed pipeline (here team 2T6S, with 60 subjects).
@@ -70,7 +72,7 @@ to get the correlation values for the results of a previously executed pipeline
 
 * `pytest.ini` is a global configuration files for using pytest (see reference [here](https://docs.pytest.org/en/7.1.x/reference/customize.html)). It allows to [register markers](https://docs.pytest.org/en/7.1.x/example/markers.html) that help to better identify tests. Note that `pytest.ini` could be replaced by data inside `pyproject.toml` in the next versions.
 * `tests/conftest.py` defines common functions, parameters, and [helpers](https://pytest-helpers-namespace.readthedocs.io/en/latest/) that are later available to all tests
-* `narps_open/utils/configuration/testing_config.toml` sets the parameters for the `testing` configuration type (see how the [configuration](/docs/configuration.md) module works). This configuration type is automatically used for testing (as defined in `tests/conftest.py`).
+* `narps_open/utils/configuration/testing_config.toml` sets the parameters for the `testing` configuration type (see how the [configuration](/docs/configuration.md) module works). This configuration type is used by default for testing (as defined in `tests/conftest.py`), but a configuration can be passed at runtime using the `narps_open_config` option in the `pytest` command line.
 
 ## Writing tests
 

narps_open/data/task.py

@@ -12,13 +12,16 @@
 class TaskInformation(dict, metaclass=SingletonMeta):
     """ This class allows to access information about the task performed in NARPS """
 
-    task_information_file = join(Configuration()['directories']['dataset'], 'task-MGT_bold.json')
-
-    def __init__(self):
+    def __init__(self, task_file=''):
         super().__init__()
 
         # Load information from the task-MGT_bold.json file
-        with open(self.task_information_file, 'rb') as file:
+        if task_file:
+            task_information_file = task_file # For testing purpose only
+        else:            
+            task_information_file = join(
+                Configuration()['directories']['dataset'], 'task-MGT_bold.json')
+        with open(task_information_file, 'rb') as file:
             self.update(load(file))
 
         # Compute derived information

narps_open/tester.py

@@ -17,15 +17,22 @@ def main():
     parser = ArgumentParser(description='Test the pipelines from NARPS.')
     parser.add_argument('-t', '--team', type=str, required=True,
         help='the team ID', choices=get_implemented_pipelines())
+    parser.add_argument('--config', type=str, required=False,
+        help='custom configuration file to be used')
     arguments = parser.parse_args()
 
-    sys.exit(pytest.main([
+    pytest_arguments = [
         '-s',
         '-q',
         '-x',
         f'tests/pipelines/test_team_{arguments.team}.py',
         '-m',
-        'pipeline_test']))
+        'pipeline_test']
+
+    if 'config' in arguments:
+        pytest_arguments.append(f'--narps_open_config={arguments.config}')
+
+    sys.exit(pytest.main(pytest_arguments))
 
 if __name__ == '__main__':
     main()

narps_open/utils/configuration/docker_testing_config.toml

@@ -0,0 +1,26 @@
+[general]
+title = "Testing configuration for the NARPS open pipelines project, to be used inside the nipype/nipype:py38 docker container"
+config_type = "custom"
+
+[directories]
+dataset = "/work/data/original/ds001734/"
+reproduced_results = "/work/run/reproduced/"
+narps_results = "/work/data/results/"
+test_data = "/work/tests/test_data/"
+test_runs = "/work/run/"
+
+[runner]
+nb_procs = 8 # Maximum number of threads executed by the runner
+nb_trials = 3 # Maximum number of executions to have the pipeline executed completely
+
+[pipelines]
+remove_unused_data = true # set to true to activate remove nodes of pipelines
+
+[results]
+neurovault_naming = true # true if results files are saved using the neurovault naming, false if they use naming of narps
+
+[testing]
+
+[testing.pipelines]
+nb_subjects_per_group = 4 # Compute first level analyses by subgroups of N subjects, to avoid lacking of disk and memory
+correlation_thresholds = [0.30, 0.70, 0.78, 0.85, 0.93] # Correlation between reproduced hypotheses files and results, respectively for [20, 40, 60, 80, 108] subjects.

tests/conftest.py

@@ -24,8 +24,24 @@
 from narps_open.data.results import ResultsCollection
 from narps_open.data.participants import get_participants_subset
 
-# Init configuration, to ensure it is in testing mode
-Configuration(config_type='testing')
+def pytest_addoption(parser):
+    """ Add a configuration option for narps_open to the pytest command line """
+    parser.addoption('--narps_open_config',
+        action='store', help='NARPS Open Pipelines configuration file')
+
+@fixture(scope='session', autouse=True)
+def load_configuration(request):
+    """ Automatically load narps_open configuration from the corresponding
+        pytest command line argument.
+    """
+    config_file = request.config.getoption('--narps_open_config')
+
+    if config_file:
+        # Init configuration with custom file
+        Configuration(config_type='custom').config_file = config_file
+    else:
+        # Ensure it is in testing mode
+        Configuration(config_type='testing')
 
 @fixture
 def temporary_data_dir():

tests/data/test_task.py

@@ -18,10 +18,9 @@
 from narps_open.data import task
 
 @fixture(scope='function', autouse=True)
-def mock_task_data(mocker):
-    """ Patch the json.load method to mock task data """
-    mocker.patch.object(
-        task.TaskInformation, 'task_information_file',
+def load_test_task_data():
+    """ Init the TaskInformation class with task data for testing """
+    task.TaskInformation(
         join(Configuration()['directories']['test_data'], 'data', 'task', 'task-info.json')
         )