update of actions

Author: Carole Sudre

.github/workflows/python-app.yml

@@ -9,7 +9,11 @@ name: Unit Tests
 
 on:
   push:
+<<<<<<< HEAD
     branches: [ main, docs_test ]  # run when anything is pushed to these branches
+=======
+    branches: [ main, docs_tests ]  # run when anything is pushed to these branches
+>>>>>>> 7bef02a4357737d41391ceef32945c656e64c08e
   pull_request:
     branches: [ main ]  # run for the code submitted as a PR to these branches
 
@@ -55,12 +59,14 @@ jobs:
 
     # Using Codecov's action, upload the coverage report for the triggering commit/PR
     - name: Upload coverage
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v4
       with:
         file: ./coverage.xml
         fail_ci_if_error: true
         verbose: true
         version: "v0.1.15"
+        codecov: 
+        token: ${{ secrets.CODECOV_TOKEN }}
 
   build_docs:
     runs-on: ubuntu-latest

MetricsReloaded/metrics/calibration_measures.py

@@ -117,6 +117,7 @@ def class_wise_expectation_calibration_error(self):
             nbins = self.dict_args["bins_ece"]
         else:
             nbins = 10
+        print('number bins is ',nbins)
         step = 1.0 / nbins
         range_values = np.arange(0, 1.00001, step)
         list_values = []
@@ -385,8 +386,10 @@ def top_label_classification_error(self):
         prob_ref_values, prob_ref_counts = np.unique(self.ref, return_counts=True)
         for k in range(nclasses):
             idx = np.where(prob_ref_values == k)
-            if len(idx) == 0:
+            print(k, idx)
+            if np.size(idx) == 0:
                 prob[k] = 0
+                print('nothing in ', k)
             else:
                 prob[k] = prob_ref_counts[idx[0]] / numb_samples
 

pyproject.toml

@@ -1,7 +1,7 @@
 [build-system]
 requires = [
     "setuptools >= 40.0.4",
-    "setuptools < 82.0.0
+    "setuptools < 82.0.0",
     "setuptools_scm >= 2.0.0",
     "wheel >= 0.29.0",
 ]

setup.py

@@ -25,7 +25,7 @@
     description=version_info["__description__"],
     author=version_info["__author__"],
     author_email=version_info["__author_email__"],
-    url="https://github.com/csudre/MetricsReloaded",
+    url="https://github.com/Project-MONAI/MetricsReloaded",
     packages=find_packages(),
     py_modules=[
         os.path.splitext(os.path.basename(path))[0]
@@ -50,7 +50,7 @@
     ],
     project_urls={
         "Documentation": "https://MetricsReloaded.readthedocs.io/",
-        "Issue Tracker": "https://github.com/csudre/MetricsReloaded/issues",
+        "Issue Tracker": "https://github.com/Project-MONAI/MetricsReloaded/issues",
     },
     python_requires=">=3.7",
     install_requires=requirements,

test/test_metrics/test_calibration_metrics.py

@@ -71,6 +71,15 @@ def test_brier_score():
     expected_bs = 0.4
     assert_allclose(expected_bs, value_test, atol=0.01)
 
+def test_root_brier_score():
+    ref_bs = [1, 0]
+    pred_bs = [[0.2,0.8],
+                [0.4,0.6]]
+    ppm = CalibrationMeasures(np.asarray(pred_bs), np.asarray(ref_bs))
+    value_test = ppm.root_brier_score()
+    expected_bs = 0.6325
+    assert_allclose(expected_bs, value_test, atol=0.01)
+
 #To use SN 2.14 p 99 of Metrics Reloaded
 
 def test_top_label_classification_error():
@@ -86,6 +95,22 @@ def test_top_label_classification_error():
     value_test = cm.top_label_classification_error()
     assert_allclose(value_test, expected_tce, atol=0.001)
 
+def test_top_label_classification_error_oneemptyclass():
+    ref_tce = [1, 0, 1, 1]
+    pred_tce = [[0.1, 0.8, 0, 0.1], [0.6, 0.1, 0.6, 0.7], [0.3, 0.1, 0.4, 0.2]]
+    # 0.25 - 0.75 - 0
+    #
+    pred_tce = np.asarray(pred_tce).T
+    ref_tce = np.asarray(ref_tce)
+    expected_prob = [0.75, 0.25, 0.75, 0.75]
+    best_prob = [0.6, 0.8, 0.6, 0.7]
+    pred_class = [1, 0, 1, 1]
+    # sqrt(0.15^2 + 0.55^2 + 0.15^2 + 0.05^2)/4
+    expected_tce = 0.2958
+    cm = CalibrationMeasures(pred_tce, ref_tce)
+    value_test = cm.top_label_classification_error()
+    assert_allclose(value_test, expected_tce, atol=0.001)
+
 
 def test_negative_log_likelihood():
     ref_nll = [1, 0, 2, 1]
@@ -111,32 +136,56 @@ def test_class_wise_expectation_calibration_error():
     pred_cwece = np.asarray(pred_cwece).T
     dict_args = {"bins_ece": 2}
     cm = CalibrationMeasures(pred_cwece, ref_cwece, dict_args=dict_args)
+    cm2 = CalibrationMeasures(pred_cwece, ref_cwece)
     value_test = cm.class_wise_expectation_calibration_error()
+    value_test2 = cm2.class_wise_expectation_calibration_error()
     expected_cwece = 0.150
+    expected_cwece2 = 0.150
     assert_allclose(value_test, expected_cwece, atol=0.001)
+    assert_allclose(value_test2, expected_cwece2, atol=0.001)
 
 
 def test_gamma_ik():
     pred = [[0.1, 0.8, 0, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 1, 0.2]]
     pred = np.asarray(pred).T
     ref = np.asarray([1, 0, 2, 1])
     cm = CalibrationMeasures(pred, ref)
+    cm2 = CalibrationMeasures(pred, ref, dict_args={'bandwidth':0.5})
     value_test = cm.gamma_ik(0, 0)
+    value_test2 = cm2.gamma_ik(0,0)
     expected_gamma = gamma(1.2)
+    expected_gamma2 = gamma(1.2)
     assert_allclose(value_test, expected_gamma, atol=0.001)
+    assert_allclose(value_test2, expected_gamma2, atol=0.001)
 
 
 def test_dirichlet_kernel():
     pred = [[0.1, 0.8, 0, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 1, 0.2]]
     pred = np.asarray(pred).T
     ref = np.asarray([1, 0, 2, 1])
     cm = CalibrationMeasures(pred, ref)
+    cm2 = CalibrationMeasures(pred,ref,dict_args={'bandwidth':0.5})
     numerator = gamma(1.2 + 2.2 + 1.6)
     denominator = gamma(1.2) * gamma(2.2) * gamma(1.6)
     prod = np.power(0.8, 0.2) * np.power(0.1, 1.2) * np.power(0.1, 0.6)
     value_test = cm.dirichlet_kernel(1, 0)
+    value_test2 = cm2.dirichlet_kernel(1,0)
     expected_dir = numerator * prod / denominator
+    expected_dir2 = expected_dir
     assert_allclose(value_test, expected_dir, atol=0.001)
+    assert_allclose(value_test2, expected_dir2, atol=0.001)
+
+
+
+def test_kernel_calculation():
+    pred = [[0.1, 0.8, 0, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 1, 0.2]]
+    #sqrt(0.7^2 + 0.5^2 + 0.2^2)/0.2
+    pred = np.asarray(pred).T
+    ref = np.asarray([1, 0, 2, 1])
+    cm = CalibrationMeasures(pred,ref,dict_args={'bandwidth_kce':0.2})
+    value_test = cm.kernel_calculation(0,1)[0,0]
+    expected_value = 0.01208
+    assert_allclose(value_test, expected_value, atol=0.001)
 
 def test_kernel_calibration_error():
     pred = [[0.1, 0.8, 0, 0.1], [0.6, 0.1, 0, 0.7], [0.3, 0.1, 1, 0.2]]