Testing updates: soundness and regressions; minor bug fixes

.github/workflows/ci.yml

@@ -12,6 +12,11 @@ on:
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
+    inputs:
+      run_cp:
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   # This workflow contains a single job called "build"
@@ -27,20 +32,39 @@ jobs:
           submodules: 'true'
       # Sets up MATLAB on the GitHub Actions runner
       - name: Setup MATLAB
-        uses: matlab-actions/setup-matlab@v1
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          release: R2024b
+          products: >
+            Computer_Vision_Toolbox
+            Control_System_Toolbox
+            Deep_Learning_Toolbox
+            Image_Processing_Toolbox
+            Optimization_Toolbox
+            Parallel_Computing_Toolbox
+            Statistics_and_Machine_Learning_Toolbox
+            Symbolic_Math_Toolbox
+            System_Identification_Toolbox
+            Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format
+            Deep_Learning_Toolbox_Converter_for_PyTorch_Model_Format
+            Deep_Learning_Toolbox_Converter_for_TensorFlow_models
+            Deep_Learning_Toolbox_Verification_Library
+
       # Sets up python for CP
       - name: Setup python
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
-      - name: Seup CP requirements
+      - name: Setup CP requirements
+        if: github.event.inputs.run_cp == 'true'
         run: |
           python3 -m venv .venv
           source .venv/bin/activate
           pip install --upgrade pip
           pip install -r requirement.txt
+        continue-on-error: true
       # Runs a set of commands using the runners shell
       - name: Run all tests
-        uses: matlab-actions/run-command@v1
+        uses: matlab-actions/run-command@v2
         with:
           command: disp('Running NNV custom testing procedure!'); diary github_actions.txt; disp('creating file for testing'); diary off; cd("code/nnv"); install; results = runtests('tests', 'IncludeSubfolders', true); assertSuccess(results);

.github/workflows/regression-tests.yml

@@ -0,0 +1,133 @@
+# Regression Tests Workflow
+# This workflow runs the NNV test suite with baseline comparison to detect regressions
+# It runs separately from the main CI workflow to provide detailed regression analysis
+
+name: Regression Tests
+
+on:
+  # Run on pull requests to master
+  pull_request:
+    branches: [ "master" ]
+
+  # Allow manual triggering
+  workflow_dispatch:
+    inputs:
+      update_baselines:
+        description: 'Update baselines after tests'
+        required: false
+        default: false
+        type: boolean
+      run_cp:
+        type: boolean
+        required: false
+        default: false
+
+jobs:
+  regression-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'true'
+
+      - name: Setup MATLAB
+        uses: matlab-actions/setup-matlab@v2
+        with:
+          release: R2024b
+          products: >
+            Computer_Vision_Toolbox
+            Control_System_Toolbox
+            Deep_Learning_Toolbox
+            Image_Processing_Toolbox
+            Optimization_Toolbox
+            Parallel_Computing_Toolbox
+            Statistics_and_Machine_Learning_Toolbox
+            Symbolic_Math_Toolbox
+            System_Identification_Toolbox
+            Deep_Learning_Toolbox_Converter_for_ONNX_Model_Format
+            Deep_Learning_Toolbox_Converter_for_PyTorch_Model_Format
+            Deep_Learning_Toolbox_Converter_for_TensorFlow_models
+            Deep_Learning_Toolbox_Verification_Library
+
+      - name: Setup Python for CP
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Python requirements
+        if: github.event.inputs.run_cp == 'true'
+        run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install --upgrade pip
+          pip install -r requirement.txt
+        continue-on-error: true
+
+      - name: Run regression tests
+        uses: matlab-actions/run-command@v2
+        env:
+          NNV_TEST_COMPARE_BASELINES: '1'
+          NNV_TEST_FAIL_ON_REGRESSION: '1'
+        with:
+          command: |
+            disp('=== NNV Regression Test Suite ===');
+            cd("code/nnv");
+            install;
+
+            % Add test utilities to path
+            addpath(fullfile(pwd, 'tests', 'test_utils'));
+
+            % Run tests with regression detection
+            try
+                [test_results, regression_results] = run_tests_with_regression("compare", true, "verbose", true);
+
+                % Check for failures
+                if ~isempty(test_results)
+                    n_failed = sum([test_results.Failed]);
+                    if n_failed > 0
+                        error('Test failures detected: %d tests failed', n_failed);
+                    end
+                end
+
+                % Check for regressions
+                if isfield(regression_results, 'regressions') && ~isempty(regression_results.regressions)
+                    error('Regressions detected: %d baseline mismatches', length(regression_results.regressions));
+                end
+
+                disp('All tests passed, no regressions detected.');
+            catch ME
+                disp(['Error: ' ME.message]);
+                rethrow(ME);
+            end
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: |
+            results/tests/data/
+            results/tests/figures/
+          retention-days: 30
+
+      - name: Update baselines (manual trigger only)
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.update_baselines == 'true'
+        uses: matlab-actions/run-command@v1
+        with:
+          command: |
+            cd("code/nnv");
+            install;
+            addpath(fullfile(pwd, 'tests', 'test_utils'));
+            manage_baselines('save');
+            disp('Baselines updated successfully.');
+
+      - name: Commit updated baselines
+        if: github.event_name == 'workflow_dispatch' && github.event.inputs.update_baselines == 'true'
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+          git add results/tests/baselines/
+          git diff --staged --quiet || git commit -m "Update test baselines [skip ci]"
+          git push

PYTHON_SETUP.md

@@ -0,0 +1,107 @@
+# Python Setup for Conformal Prediction (CP) Verification
+
+NNV's Conformal Prediction (CP) verification feature requires a Python environment with PyTorch and related dependencies. This guide covers setup for Windows, macOS, and Linux.
+
+## Overview
+
+The CP verification methods use Python for neural network training of surrogate models. MATLAB calls Python scripts located in `code/nnv/engine/nn/Prob_reach/`:
+- `Trainer_Linear.py` - Linear surrogate model training
+- `Trainer_ReLU.py` - ReLU surrogate model training
+- `Direction_trainer.py` - Direction computation
+
+## Quick Setup
+
+### Windows
+
+```cmd
+cd <nnv-root-directory>
+python -m venv .venv
+.venv\Scripts\activate
+pip install -r requirement.txt
+```
+
+### macOS / Linux
+
+```bash
+cd <nnv-root-directory>
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirement.txt
+```
+
+## Verification
+
+After setup, verify in MATLAB:
+
+```matlab
+% Check Python environment is detected
+check_nnv_setup
+
+% Or test directly
+python_path = cp_env()
+```
+
+The output should show the path to your Python executable in the `.venv` directory.
+
+## Running CP Verification
+
+Once configured, you can run CP verification:
+
+```matlab
+% Example: CP robustness verification
+cd(fullfile(nnvroot(), 'code', 'nnv', 'examples', 'NN', 'cifar10'))
+CP_verify_robustness
+```
+
+## Dependencies
+
+The `requirement.txt` file includes:
+- `torch` - PyTorch for neural network training
+- `numpy` - Numerical computing
+- `scipy` - Scientific computing (for .mat file I/O)
+
+## GPU Support
+
+By default, CP verification uses CPU. To use GPU acceleration:
+
+1. Install CUDA-enabled PyTorch (see [PyTorch installation guide](https://pytorch.org/get-started/locally/))
+2. Set the training device in MATLAB:
+
+```matlab
+reachOptions.train_device = 'gpu';
+result = verify_robustness_cp(net, IS, reachOptions, target, numClasses);
+```
+
+## Troubleshooting
+
+### Python not found
+
+If you see "Python virtual environment not found":
+1. Ensure you created the venv in the NNV root directory (where `requirement.txt` is located)
+2. Verify the `.venv` folder exists
+3. On Windows, check that `.venv\Scripts\python.exe` exists
+4. On Unix, check that `.venv/bin/python` exists
+
+### Module not found errors
+
+If Python scripts fail with import errors:
+```bash
+# Activate the virtual environment first
+# Windows:
+.venv\Scripts\activate
+# Unix:
+source .venv/bin/activate
+
+# Then reinstall dependencies
+pip install -r requirement.txt
+```
+
+### MATLAB can't find Python
+
+Ensure the virtual environment is in the NNV root directory (same level as `requirement.txt`). The `cp_env()` function uses `nnvroot()` to locate the Python executable.
+
+## Related Documentation
+
+- [NNV README](README.md) - Main installation guide
+- [CP verification example](code/nnv/examples/NN/cifar10/CP_verify_robustness.m)
+- [Prob_reach functions](code/nnv/engine/utils/Prob_reach.m)