Finish tests (#5)

* removed submodule
* update tests

Author: bendhouseart

local_bids_validation/local.py

@@ -1,10 +1,12 @@
 from argparse import ArgumentParser
 from bids_validator import BIDSValidator
-from os import walk, sep, listdir
+from os import walk, sep, listdir, path
 from typing import Union
 from pathlib import Path
 import json
 import subprocess
+import glob
+import sys
 
 
 def collect_data_set(curl_download_file):
@@ -24,8 +26,12 @@ def check_for_bids_validator_js():
 
 def report_number_of_files_bids_validator_js_found(input_folder: Union[str, Path]) -> int:
     if check_for_bids_validator_js:
-        # run the js bids validator on the folder
-        js = subprocess.check_output(["bids-validator", str(input_folder), "--json"])
+        try:
+            # run the js bids validator on the folder
+            js = subprocess.check_output(["bids-validator", str(input_folder), "--json"])
+        except subprocess.CalledProcessError as e:
+            js = e.output
+            
         validator_output = json.loads(js)
         totalFiles = validator_output.get('summary', {}).get('totalFiles')
     else:
@@ -54,7 +60,7 @@ def check_bids_valid(manifest: list) -> dict:
     validity = {}
     validator = BIDSValidator()
     for file in manifest:
-        validity[str(file)] = {'ValidBids': validator.is_bids(str(file)), 'bidsignored': False}
+        validity[str(file)] = {'ValidBIDS': validator.is_bids(str(file)), 'bidsignored': False}
     return validity
 
 
@@ -72,25 +78,160 @@ def collect_bidsignored(input_folder: Union[str, Path]) -> list:
         ignored = []
         print("No .bidsignore file found")
 
+    pop_index = []
+    for entry in ignored:
+        if entry.startswith('#'):
+            pop_index.append(entry)
+        elif entry == '':
+            pop_index.append(entry)
+
+    for index in pop_index:
+        ignored.remove(index)
+
     return ignored
 
 
-def determine_if_file_is_ignored(validity: dict, bidsignore: list) -> dict:
+def expand_bids_ignored(ignored: list, root_path: Union[str, Path]) -> list:
+    """given a bidsignore file, expand the entries to include all files that match the pattern"""
+    expanded_paths = []
+    for entry in ignored:
+        if '*' in entry:
+            matching_paths = glob.glob(str(Path(root_path) / entry))
+            for path in matching_paths:
+                if Path(path).is_dir():
+                    # if the path is a directory, add a trailing slash
+                    for root, dirs, files in walk(path):
+                        for file in files:
+                            expanded_paths.append(str(Path(root) / file))
+                if Path(path).is_file():
+                    expanded_paths.append(path)
+        else:
+            pass
+
+    # create a set from the expanded paths to remove duplicates
+    expanded_paths = set(expanded_paths)
+
+    return list(expanded_paths)
+
+
+def determine_ignored_files(validity: dict, bidsignore: list, print_output=False) -> dict:
     """
     Given a dict from check_bids_valid, determine if the valid file falls under the scope of the .bidsignore file
     """
+    common_path = path.commonpath(bidsignore)
+    all_bids_ignored = expand_bids_ignored(bidsignore, common_path)
+
+    for key in validity.keys():
+        for ignored in all_bids_ignored:
+            if key in ignored:
+                validity[key]['bidsignored'] = True
+
+        # count the number of valid but ignored files
+    valid_and_ignored = []
+    valid_bids_files = []
+    valid_bids_files_not_ignored = []
+    invalid_bids_files_and_ignored = []
+    invalid_and_not_ignored = []
+
     for k, v in validity.items():
-        print(f"{k}: {v}")
+        if v['ValidBIDS'] and v['bidsignored']:
+            valid_and_ignored.append(k)
+        if v['ValidBIDS']:
+            valid_bids_files.append(k)
+        if v['ValidBIDS'] and not v['bidsignored']:
+            valid_bids_files_not_ignored.append(k)
+        if not v['ValidBIDS'] and v['bidsignored']:
+            invalid_bids_files_and_ignored.append(k)
+        if not v['ValidBIDS'] and not v['bidsignored']:
+            invalid_and_not_ignored.append(k)
+
+    output = {
+        'valid_and_ignored': valid_and_ignored,
+        'valid_bids_files': valid_bids_files,
+        'valid_bids_files_not_ignored': valid_bids_files_not_ignored,
+        'invalid_bids_files_and_ignored': invalid_bids_files_and_ignored,
+        'invalid': invalid_and_not_ignored
+    }
+
+    if print_output:
+        for k, v in output.items():
+            print(f"Found {len(v)} files that are {k}")
+            for file in v:
+                print(file)
+            print("\n")
+
+    return output
+
+
+def run_all(bids_dir):
+    """
+    Run all the functions in this module.
+    """
+    manifest = make_manifest(bids_dir)
+    bids_ignored = collect_bidsignored(bids_dir)
+    check_if_valid = check_bids_valid(manifest)
+    valid_bids_files = []
+    invalid_bids_files = []
+    for file, validity in check_if_valid.items():
+        if validity['ValidBIDS']:
+            valid_bids_files.append(file)
+        if not validity['ValidBIDS']:
+            invalid_bids_files.append(file)
+
+    num_bids_validator_found = report_number_of_files_bids_validator_js_found(bids_dir)
+
+    all_bids_ignored = expand_bids_ignored(bids_ignored, bids_dir)
+
+    for key in check_if_valid.keys():
+        for ignored in all_bids_ignored:
+            if key in ignored:
+                check_if_valid[key]['bidsignored'] = True
 
+    file_lists = determine_ignored_files(check_if_valid, all_bids_ignored, print_output=True)
 
+    return file_lists
 
 if __name__ == "__main__":
     parser = ArgumentParser()
-    parser.add_argument("input-folder", help="input folder to check for bids files", type=str)
+    parser.add_argument("input_folder", help="input folder to check for bids files", type=str)
+    parser.add_argument("--show-valid-and-ignored", help="show all valid and ignored files", action="store_true")
+    parser.add_argument("--show-valid-bids-files", help="show all valid bids files", action="store_true")
+    parser.add_argument("--show-valid-bids-files-not-ignored", help="show all valid bids files that are not ignored",
+                        action="store_true")
+    parser.add_argument("--show-invalid-bids-files-and-ignored", help="show all invalid bids files that are ignored",
+                        action="store_true")
+    parser.add_argument("--show-invalid", help="show all invalid files", action="store_true")
+
     args = parser.parse_args()
     manifest = make_manifest(args.input_folder)
     check_if_valid = check_bids_valid(manifest)
+    bidsignore = collect_bidsignored(args.input_folder)
+    if not bidsignore:
+        print("Exiting")
+        sys.exit(1)
+    output = determine_ignored_files(check_if_valid, bidsignore, print_output=False)
+    if not args.show_valid_and_ignored and not args.show_valid_bids_files and not args.show_valid_bids_files_not_ignored and not args.show_invalid_bids_files_and_ignored and not args.show_invalid:
+        output = determine_ignored_files(check_if_valid, bidsignore, print_output=True)
+    elif args.show_valid_and_ignored:
+        print("Found the following valid and ignored files:")
+        for file in output.get('valid_and_ignored'):
+            print(file)
+    elif args.show_valid_bids_files:
+        print("Found the following valid bids files:")
+        for file in output.get('valid_bids_files'):
+            print(file)
+    elif args.show_valid_bids_files_not_ignored:
+        print("Found the following valid bids files that are not ignored:")
+        for file in output.get('valid_bids_files_not_ignored'):
+            print(file)
+    elif args.show_invalid_bids_files_and_ignored:
+        print("Found the following invalid bids files that are ignored:")
+        for file in output.get('invalid_bids_files_and_ignored'):
+            print(file)
+    elif args.show_invalid:
+        print("Found the following invalid and ignored files:")
+        for file in output.get('invalid'):
+            print(file)
+    else:
+        pass
 
-
-
-    print("stop")

poetry.lock

@@ -12,6 +12,9 @@ python = "^3.10"
 bids-validator = "^1.12.0"
 
 
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.4.2"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"