Test uploads (#52)

Possibly, premature merge, but need to add workflow file a step 0

Adding upload test for local version of ezBIDS w/ multi-modality mixed folder of dicoms
merging and pushing so that test can be triggered in actions as it's not presently in
workflows. Small changes to Makefile to ensure running of tests.
* adding local upload test to CI
* add create mixed dicom folder to makefile

Author: bendhouseart

.github/workflows/collect-test-data.yml

@@ -0,0 +1,50 @@
+name: Collect Test Data
+
+on:
+  push:
+    branches: [ '*' ]
+  pull_request:
+    branches: [ '*' ]
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  collect-data:
+    runs-on: ubuntu-latest
+    outputs:
+      cache-hit: ${{ steps.test-data-cache.outputs.cache-hit }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensure we get the submodule info
+
+      - name: Cache test data
+        uses: actions/cache@v4
+        id: test-data-cache
+        with:
+          path: test/test_data
+          key: test-data-${{ hashFiles('.gitmodules') }}
+          restore-keys: |
+            test-data-
+
+      - name: Get test data if not cached
+        if: steps.test-data-cache.outputs.cache-hit != 'true'
+        run: |
+          make get-test-data
+
+      - name: Verify test data
+        run: |
+          if [ ! -d "test/test_data" ]; then
+            echo "Test data directory not found"
+            exit 1
+          fi
+          if [ ! -d "test/test_data/mixed_dicoms" ]; then
+            echo "Mixed DICOM directory not found"
+            exit 1
+          fi
+
+      - name: Upload test data
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-data
+          path: test/test_data 

.github/workflows/test-upload-local.yml

@@ -0,0 +1,97 @@
+name: Test Local Upload
+
+on:
+  workflow_run:
+    workflows: ["Docker Build (Local)"]
+    types:
+      - completed
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    
+    env:
+      COMPOSE_FILE: ./docker-compose.yml
+      REGISTRY_PREFIX: docker.io/library
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensure we get the submodule info
+
+      - name: Set repository name
+        run: echo "REPO_NAME=$(basename $GITHUB_WORKSPACE)" >> $GITHUB_ENV
+
+      - name: Cache test data
+        uses: actions/cache@v4
+        id: test-data-cache
+        with:
+          path: test/test_data
+          key: test-data-${{ hashFiles('.gitmodules') }}
+          restore-keys: |
+            test-data-
+
+      - name: Get test data if not cached
+        if: steps.test-data-cache.outputs.cache-hit != 'true'
+        run: |
+          cd test
+          make gettestdata
+
+      - name: Download images
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-images-local
+          path: .
+
+      - name: Load images
+        run: |
+          docker load < mongo.tar
+          docker load < api.tar
+          docker load < handler.tar
+          docker load < ui.tar
+
+      - name: Tag images
+        run: |
+          docker tag mongo:latest mongo:test
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-api:latest ${REPO_NAME}-api:test
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-handler:latest ${REPO_NAME}-handler:test
+          docker tag ${REGISTRY_PREFIX}/${REPO_NAME}-ui:latest ${REPO_NAME}-ui:test
+
+      - name: Start services
+        run: docker compose -f ${{ env.COMPOSE_FILE }} up -d
+
+      - name: Wait for services to be ready
+        run: |
+          # Wait for MongoDB
+          timeout 60 bash -c 'until docker compose -f ${{ env.COMPOSE_FILE }} exec -T mongo mongosh --eval "db.adminCommand(\"ping\")" > /dev/null 2>&1; do sleep 2; done'
+          
+          # Wait for API
+          timeout 60 bash -c 'until curl -s http://localhost:8082/health > /dev/null; do sleep 2; done'
+          
+          # Wait for UI
+          timeout 60 bash -c 'until curl -s http://localhost:3000 > /dev/null; do sleep 2; done'
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Verify uv installation
+        run: uv --version
+
+      - name: Setup test environment
+        run: |
+          cd test
+          make setup-test
+
+      - name: Run upload test
+        run: |
+          cd test
+          make test-upload
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker compose -f ${{ env.COMPOSE_FILE }} down
+          docker system prune -f 

.gitignore

@@ -42,5 +42,5 @@ nginx/ssl/*
 *__pycache__*
 
 # Ignore test data submodules contents but not the submodule tracking
-# test/test_data/*/
-# !test/test_data/.gitkeep
+test/test_data/*/
+!test/test_data/.gitkeep

Makefile

@@ -1,13 +1,29 @@
-.PHONY: gettestdata clean-testdata
+.PHONY: gettestdata clean-testdata setup-test test-upload clean-test
 
 # Create test_data directory if it doesn't exist
 test/test_data:
 	mkdir -p test/test_data
 
 # Initialize and update submodules for test data
-gettestdata: test/test_data
+get-test-data: test/test_data
 	test/gettestdata
+	test/createmixedfolder
 
 # Clean test data (remove submodules and their directories)
 clean-testdata:
-	test/cleantestdata
+	test/cleantestdata
+
+# Test environment setup
+setup-test:
+	cd test && uv venv .venv
+	cd test && uv pip install -e .
+
+# Run upload test
+test-upload: setup-test
+	cd test && uv run pytest upload_flat_folder_of_dicoms.py -v -s
+
+# Clean test environment
+clean-test:
+	rm -rf test/.venv
+	rm -rf test/__pycache__
+	rm -rf test/.pytest_cache