Merge pull request #45 from s-ccs/feature-new_test

Feature new test / slight refactor / new CLI

Author: behinger

.github/workflows/pytest.yml

@@ -0,0 +1,29 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Set up Python
+        run: uv python install
+        
+      - name: Install dependencies with uv
+        run: uv sync --all-extras --dev
+      - name: install local
+        run: uv pip install --editable ./
+
+      - name: Run tests with uv
+        run: uv run pytest

.gitignore

@@ -7,6 +7,8 @@ jsonschema
 .DS_Store
 empty_log_process_temp.py
 
+tests/**/bids/
+tests/test_main_functionality/data/projects/test-project/sub-100
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

docs/configuration.md

@@ -17,8 +17,6 @@ lslautobids gen-dv-config
 ```
 _Currently, the package doesn't allow you to have multiple dataverse configurations. This will be added in future versions and can be easily adapted_
 
-However for testing purposes, we create a separate test configuration file `~/.config/lslautobids/test-autobids_config.yaml` which is used when running the tests.
-
 #### Project Configuration (`gen_project_config.py`)
 This module generates a project-specific configuration file in TOML format. This file is stored in the `projects/<PROJECT_NAME>/<PROJECT_NAME>_config.toml` file and contains:
 - Project metadata: Title, description, license, and authors, etc.

docs/developers_documentation.md

@@ -92,9 +92,7 @@ The command to generate the dataverse configuration file is:
 ```
 lslautobids gen-dv-config
 ```
-_Currently, the package doesn't allow you to have multiple dataverse configurations. This will be added in future versions and can be easily adapted_
-
-However for testing purposes, we create a separate test configuration file `~/.config/lslautobids/test-autobids_config.yaml` which is used when running the tests.
+_Currently, the package doesn't allow you to have multiple dataverse configurations. This will be added in future versions.
 
 #### 2. Project Configuration (`gen_project_config.py`)
 This module generates a project-specific configuration file in TOML format. This file is stored in the `projects/<PROJECT_NAME>/<PROJECT_NAME>_config.toml` file and contains:

docs/testing.md

@@ -14,7 +14,6 @@ Tests will be added continuously as new features are added and existing features
 
 ### Running Tests
 
-To run the tests, navigate to the `tests/` directory and execute:
-`python tests/run_all_tests.py`
+To run the tests, we recommend to use `uv run pytest` (caveat, for some reason sometimes tests fail if they are all run at the same time. you can then run them via `uv run pytest tests/testcase/test_main_functionality` and they will work).
 
 These tests ensure that each component functions as expected and that the overall pipeline works seamlessly. This tests will also be triggered automatically on each push or PR to the main repository using GitHub Actions.

lslautobids/config_globals.py

@@ -13,6 +13,7 @@ def __init__(self):
             "redo_bids_conversion": False,
             "reupload": False,
             "redo_other_pc": False,
+            "push_to_dataverse": True,
         }
 
     def init(self, args):
@@ -55,7 +56,7 @@ def parse_yaml_file(yaml_file):
 
 # Determine config paty based on context
 if "pytest" in sys.modules:
-    config_file = os.path.join(os.path.expanduser("~"), ".config/lslautobids/test-autobids_config.yaml")
+    config_file = "tests/pytest-autobids_config.yaml"
 else:
     config_file = os.path.join(os.path.expanduser("~"), ".config/lslautobids/autobids_config.yaml")
 config = parse_yaml_file(config_file)

lslautobids/convert_to_bids_and_upload.py

@@ -364,7 +364,7 @@ def convert_to_bids(self, xdf_path,subject_id,session_id, run_id, task_id,other,
         # Validate BIDS data
         logger.info("Validating BIDS data...")
         # Validate the BIDS data
-        val = self.validate_bids(bids_root+project_name,subject_id,session_id, logger)
+        val = self.validate_bids(os.path.join(bids_root,project_name),subject_id,session_id, logger)
         return val
 
     def validate_bids(self,bids_path,subject_id,session_id, logger):
@@ -482,26 +482,29 @@ def bids_process_and_upload(processed_files,logger):
     bids.populate_dataset_description_json(project_name, logger)
     logger.info('Generating metadatafiles........')   
     generate_json_file(project_name, logger)
-    logger.info('Generating dataverse dataset........')
 
-    doi, status = create_dataverse(project_name)
 
-    logger.info("Creating and adding files to Dataverse dataset...")
-    create_and_add_files_to_datalad_dataset(bids_root+project_name,status, logger)
+    logger.info("Creating and adding files to Datalad dataset...")
+    create_and_add_files_to_datalad_dataset(os.path.join(bids_root,project_name),logger)
 
-    if status == 0:
-        logger.info('Linking dataverse dataset with datalad')
-        add_sibling_dataverse_in_folder(doi, logger)
-
-    if cli_args.yes:
-        logger.info('Pushing files to dataverse........')
-        push_files_to_dataverse(project_name, logger)
-    else:
-        user_input = get_user_input("Do you want to push the files to Dataverse? ",logger)
+    if cli_args.push_to_dataverse:
+        logger.info('Generating dataverse dataset........')
+        doi, status = create_dataverse(project_name, logger)
+        if status == 0: # run only if a new dataverse was created
+            logger.info('Linking dataverse dataset with datalad')
+            add_sibling_dataverse_in_folder(doi, logger)
+
+        if cli_args.yes:
+            user_input = "y"
+        else:
+            user_input = get_user_input("Do you want to push the files to Dataverse? ",logger)
+        
         if user_input == "y":
             logger.info('Pushing files to dataverse........')
             push_files_to_dataverse(project_name, logger)
         elif user_input == "n":
             logger.info("Program aborted.")
         else:
             logger.error("Invalid Input.")
+    else:
+        logger.info('cli.push_to_dataverse was false, not pushing.')

lslautobids/datalad_create.py

@@ -2,29 +2,33 @@
 import os
 
 
-def create_and_add_files_to_datalad_dataset(dataset_path,flag, logger):
+def create_and_add_files_to_datalad_dataset(dataset_path,logger):
     message = "LSL Auto BIDS: new files found and added"
-    if flag==0:
+    #if flag==0:
+    
+    try:
+        dl.Dataset(dataset_path)
+    except:
         message ="LSL Auto BIDS: new datalad dataset created"
         # Create a new dataset
         logger.info('Creating a new datalad dataset........')
         try:
             dl.create(dataset_path, force=True) # files already exist, so we eforce it
+
+            # make sure only large files are saved
+            with open(os.path.join(dataset_path,".gitattributes"), "a") as f:
+                f.write("* annex.largefiles=largerthan=100kb")
+                f.write("\n*.csv annex.largefiles=nothing")
+                f.write("\n*.log annex.largefiles=nothing")
+                f.write("\n*.tsv annex.largefiles=nothing")
+                f.write("\n*.md annex.largefiles=nothing")
+                f.write("\n*.json annex.largefiles=nothing")
+
         except:
             logger.info("Could not create a new dataset, maybe it exists already?")
 
-    # Commit changes
-    # Change to dataset path
-    os.chdir(dataset_path)
-    if flag==0:
-    	# needed to modify participants.tsv etc. later
-    	with open(os.path.join(dataset_path,".gitattributes"), "a") as f:
-            f.write("* annex.largefiles=largerthan=100kb")
-            f.write("\n*.csv annex.largefiles=nothing")
-            f.write("\n*.log annex.largefiles=nothing")
-            f.write("\n*.tsv annex.largefiles=nothing")
-            f.write("\n*.md annex.largefiles=nothing")
-            f.write("\n*.json annex.largefiles=nothing")
+
+    # commit current files
     logger.info('Committing current changes........')
-    dl.save(path = '.', message=message)
+    dl.save(path = dataset_path, message=message)
 

lslautobids/dataverse_dataset_create.py

@@ -12,7 +12,7 @@
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 
-def create_dataverse(project_name):
+def create_dataverse(project_name, logger):
     """
     Creates a Dataverse dataset and returns the PID and dataset ID.
     

pyproject.toml

@@ -6,5 +6,11 @@ build-backend = "setuptools.build_meta"
 # Ref: https://github.com/codespell-project/codespell#using-a-config-file
 skip = '.git*,*.svg,*.bib'
 check-hidden = true
+
+[dependency-groups]
+dev = [
+    "git-annex>=10.20251114",
+    "pytest>=9.0.1",
+]
 # ignore-regex = ''
 # ignore-words-list = ''