improve test download script

Author: jbohnslav

.github/workflows/gpu.yml

@@ -39,6 +39,10 @@ jobs:
         pip install pytest pytest-cov
         python setup.py develop
 
+    - name: Setup test data
+      run: |
+        python setup_tests.py
+
     - name: GPU Tests
       run: |
         pytest -v -m "gpu" tests/

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, windows-latest, macos-12]
+        os: [ubuntu-20.04, windows-latest, macos-13]
 
     steps:
     - uses: actions/checkout@v3
@@ -35,7 +35,7 @@ jobs:
         choco install ffmpeg
 
     - name: Install FFMPEG (macOS)
-      if: matrix.os == 'macos-12'
+      if: matrix.os == 'macos-13'
       run: |
         brew install ffmpeg
 
@@ -55,6 +55,10 @@ jobs:
         pip install pytest pytest-cov
         python setup.py develop
 
+    - name: Setup test data
+      run: |
+        python setup_tests.py
+
     - name: Run CPU tests
       run: |
         pytest -v -m "not gpu" tests/

requirements.txt

@@ -17,3 +17,4 @@ pytorch_lightning==1.6.5
 ruff>=0.1.0
 pre-commit>=2.20.0,<3.0.0
 setuptools
+gdown

reset_venv.sh

@@ -26,6 +26,10 @@ uv pip install pytest pytest-cov
 echo "Installing package in editable mode..."
 uv pip install -e .
 
+# Setup test data
+echo "Setting up test data..."
+python setup_tests.py
+
 # Run tests
 echo "Running tests..."
 pytest -v tests/

setup_tests.py

@@ -38,39 +38,49 @@ def download_file(url, destination):
 
 def setup_tests():
     """Sets up the testing environment for DeepEthogram."""
-
-    # Create tests/DATA directory if it doesn't exist
-    tests_dir = Path("tests")
-    data_dir = tests_dir / "DATA"
-    data_dir.mkdir(parents=True, exist_ok=True)
-
-    # Download the test archive
-    zip_path = data_dir / "testing_deepethogram_archive.zip"
-
     try:
-        print("Downloading test data archive...")
-        gdown.download(id="1IFz4ABXppVxyuhYik8j38k9-Fl9kYKHo", output=str(zip_path), quiet=False)
+        # Create tests/DATA directory if it doesn't exist
+        tests_dir = Path("tests")
+        data_dir = tests_dir / "DATA"
+        data_dir.mkdir(parents=True, exist_ok=True)
 
-        print("Extracting archive...")
-        with zipfile.ZipFile(zip_path, "r") as zip_ref:
-            zip_ref.extractall(data_dir)
-
-        # Verify the extraction
+        # Define paths and requirements
         archive_path = data_dir / "testing_deepethogram_archive"
+        zip_path = data_dir / "testing_deepethogram_archive.zip"
         required_items = ["DATA", "models", "project_config.yaml"]
 
+        # Check if test data already exists and is complete
+        if archive_path.exists():
+            missing_items = [item for item in required_items if not (archive_path / item).exists()]
+            if not missing_items:
+                print("Test data already exists and appears complete. Skipping download.")
+                return True
+            print("Test data exists but is incomplete. Re-downloading...")
+
+        # Download and extract if needed
+        if not archive_path.exists() or not all((archive_path / item).exists() for item in required_items):
+            # Download if zip doesn't exist
+            if not zip_path.exists():
+                print("Downloading test data archive...")
+                gdown.download(id="1IFz4ABXppVxyuhYik8j38k9-Fl9kYKHo", output=str(zip_path), quiet=False)
+
+            # Extract archive
+            print("Extracting archive...")
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                zip_ref.extractall(data_dir)
+
+            # Clean up zip file after successful extraction
+            zip_path.unlink()
+
+        # Final verification
         missing_items = [item for item in required_items if not (archive_path / item).exists()]
-
         if missing_items:
             print(f"Warning: The following items are missing: {missing_items}")
             return False
 
         print("Setup completed successfully!")
         print("\nYou can now run the tests using: pytest tests/")
-        print("Note: The zz_commandline test module will take a few minutes to complete.")
-
-        # Clean up the zip file
-        zip_path.unlink()
+        print("Note: The gpu tests will take a few minutes to complete.")
         return True
 
     except Exception as e: