Testing of scripts using Github actions

.github/workflows/code_formatting.yml → .github/workflows/code_formatting.yaml

@@ -14,18 +14,26 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
-      - name: Install dependencies
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: setup.py
+
+      - name: Install requirements
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
+          pip list
+
       - name: ruff check
         run: |
           ruff check .
+
       - name: ruff format
         run: |
           ruff format .

.github/workflows/code_tester.yaml

@@ -0,0 +1,66 @@
+name: Code tester
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+
+jobs:
+  test_scripts:
+    strategy:
+      fail-fast: false
+      matrix:
+        folders: [
+          {
+            "name": "s1_development_environment",
+            "skip_files": "-"
+          },
+          {
+            "name": "s2_organisation_and_version_control",
+            "skip_files": "make_dataset_solution.py|visualize_solution.py"
+          },
+          {
+            "name": "s3_reproducibility",
+            "skip_files": "reproducibility_tester.py"
+          },
+          {
+            "name": "s4_debugging_and_logging",
+            "skip_files": "weights_and_bias_solution.py weights_and_bias_solution2.py weights_and_bias_solution3.py"
+          },
+        ]
+
+    name: ${{ matrix.folders.name }}
+
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: 'pip'
+          cache-dependency-path: setup.py
+
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_exercises.txt
+          pip list
+
+      - name: Extract relevant files
+        run: |
+          skip_files="${{ matrix.folders.skip_files }}"
+          find ${{ matrix.folders.name }} -type f -name "*.py" | grep -vE $skip_files > python_files.txt
+          echo "===== Files to run ====="
+          cat python_files.txt
+
+      - name: Run script
+        run: |
+          while IFS= read -r file; do
+            echo "===== Running $file ====="
+            python "$file"
+          done < python_files.txt

.github/workflows/deploy_docs.yaml

@@ -24,8 +24,11 @@ jobs:
           cache: 'pip'
           cache-dependency-path: setup.py
 
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip list
 
       - name: Deploy docs
         run: mkdocs gh-deploy --force

.github/workflows/repo_scraper.yml → .github/workflows/repo_scraper.yaml

@@ -17,6 +17,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
+          cache: 'pip'
 
       - name: Install dependencies
         run: |

requirements_exercises.txt

@@ -0,0 +1,7 @@
+torch>=2.2
+matplotlib>=3.8.1
+click>=8.1.7
+hydra-core>=1.3.2
+torchvision>=0.17.0
+rich>=13.6.0
+wandb>=0.16.1

s2_organisation_and_version_control/exercise_files/make_dataset_solution.py

@@ -8,8 +8,8 @@ def normalize(images: torch.Tensor) -> torch.Tensor:
 
 
 @click.command()
-@click.option("raw_dir", default="data/raw", help="Path to raw data directory")
-@click.option("processed_dir", default="data/processed", help="Path to processed data directory")
+@click.option("--raw_dir", default="data/raw", help="Path to raw data directory")
+@click.option("--processed_dir", default="data/processed", help="Path to processed data directory")
 def make_data(raw_dir: str, processed_dir: str):
     """Process raw data and save it to processed directory."""
     train_images, train_target = [], []

s3_reproducibility/exercise_files/vae_mnist.py

@@ -16,8 +16,8 @@
 
 # Model Hyperparameters
 dataset_path = "~/datasets"
-cuda = True
-DEVICE = torch.device("cuda" if cuda else "cpu")
+device_name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+DEVICE = torch.device(device_name)
 batch_size = 100
 x_dim = 784
 hidden_dim = 400
@@ -38,7 +38,7 @@
 encoder = Encoder(input_dim=x_dim, hidden_dim=hidden_dim, latent_dim=latent_dim)
 decoder = Decoder(latent_dim=latent_dim, hidden_dim=hidden_dim, output_dim=x_dim)
 
-model = Model(Encoder=encoder, Decoder=decoder).to(DEVICE)
+model = Model(encoder=encoder, decoder=decoder).to(DEVICE)
 
 
 def loss_function(x, x_hat, mean, log_var):

s4_debugging_and_logging/exercise_files/vae_mnist_bugs.py

@@ -13,8 +13,8 @@
 
 # Model Hyperparameters
 dataset_path = "datasets"
-cuda = True
-DEVICE = torch.device("cuda" if cuda else "cpu")
+device_name = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
+DEVICE = torch.device(device_name)
 batch_size = 100
 x_dim = 784
 hidden_dim = 400