Albrja/mic 6263/general fuzzy output (#107)

Albrja/mic 6263/general fuzzy output

Add new dataclass and FuzzyChecker method to return class for proportion tests
- *Category*: Feature
- *JIRA issue*: https://jira.ihme.washington.edu/browse/MIC-6263

Changes and notes
-adds new dataclass to store metadata for each proportion test
-adds method on FuzzyChecker to return class and not raise assertions

Author: albrja

src/vivarium_testing_utils/fuzzy_checker.py

@@ -3,6 +3,7 @@
 #################
 from __future__ import annotations
 
+from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
 from typing import Any
@@ -14,6 +15,34 @@
 from scipy.stats._distn_infrastructure import rv_continuous_frozen, rv_discrete_frozen
 
 
+@dataclass
+class TestResult:
+    """Class to store metadata for individual tests run by FuzzyChecker."""
+
+    name: str
+    """Name of the test proportion being calculated."""
+    name_additional: str
+    """Additional name for test, used for when the same proportion is calculated multiple times."""
+    observed_proportion: float
+    """The observed proportion of a specific event happening."""
+    observed_numerator: int
+    """Observed counts of the event happening."""
+    observed_denominator: int
+    """Total counts of opportunities for the event to happen."""
+    target_lower_bound: float
+    """Lower bound of the target proportion range."""
+    target_upper_bound: float
+    """Upper bound of the target proportion range."""
+    bayes_factor: float
+    """Calculated Bayes factor from the test for the observed proportion."""
+    reject_null: bool
+    """Whether the null hypothesis was rejected."""
+    bug_issue_distribution: tuple[float, float]
+    """The bug/issue distribution used in the test."""
+    no_bug_issue_distribution: rv_discrete_frozen
+    """The no-bug/issue distribution used in the test."""
+
+
 class FuzzyChecker:
     """
     This class manages "fuzzy" checks -- that is, checks of values that are
@@ -47,7 +76,7 @@ def fuzzy_checker(output_directory) -> FuzzyChecker:
     """
 
     def __init__(self) -> None:
-        self.proportion_test_diagnostics: list[dict[str, Any]] = []
+        self.proportion_test_diagnostics: list[TestResult] = []
 
     def fuzzy_assert_proportion(
         self,
@@ -114,6 +143,70 @@ def fuzzy_assert_proportion(
             Useful for e.g. specifying the timestep when an assertion happened.
 
         """
+        test_proportion = self.test_proportion(
+            name=name,
+            name_additional=name_additional,
+            target_proportion=target_proportion,
+            observed_numerator=observed_numerator,
+            observed_denominator=observed_denominator,
+            bug_issue_beta_distribution_parameters=bug_issue_beta_distribution_parameters,
+            fail_bayes_factor_cutoff=fail_bayes_factor_cutoff,
+        )
+        self.proportion_test_diagnostics.append(test_proportion)
+
+        if test_proportion.reject_null:
+            if test_proportion.observed_proportion < test_proportion.target_lower_bound:
+                raise AssertionError(
+                    f"{name} value {test_proportion.observed_proportion:g} is significantly less than expected, bayes factor = {test_proportion.bayes_factor:g}"
+                )
+            else:
+                raise AssertionError(
+                    f"{name} value {test_proportion.observed_proportion:g} is significantly greater than expected, bayes factor = {test_proportion.bayes_factor:g}"
+                )
+
+        if (
+            test_proportion.target_lower_bound > 0
+            and self._calculate_bayes_factor(
+                0,
+                test_proportion.bug_issue_distribution,
+                test_proportion.no_bug_issue_distribution,
+            )
+            < fail_bayes_factor_cutoff
+        ):
+            logger.warning(
+                f"Sample size too small to ever find that the simulation's '{name}' value is less than expected."
+            )
+
+        if test_proportion.target_upper_bound < 1 and (
+            self._calculate_bayes_factor(
+                observed_denominator,
+                test_proportion.bug_issue_distribution,
+                test_proportion.no_bug_issue_distribution,
+            )
+            < fail_bayes_factor_cutoff
+        ):
+            logger.warning(
+                f"Sample size too small to ever find that the simulation's '{name}' value is greater than expected."
+            )
+
+        if (
+            fail_bayes_factor_cutoff
+            > test_proportion.bayes_factor
+            > inconclusive_bayes_factor_cutoff
+        ):
+            logger.warning(f"Bayes factor for '{name}' is not conclusive.")
+
+    def test_proportion(
+        self,
+        name: str = "",
+        name_additional: str = "",
+        target_proportion: float | tuple[float, float] = 0.1,
+        observed_numerator: int = 0,
+        observed_denominator: int = 0,
+        bug_issue_beta_distribution_parameters: tuple[float, float] = (0.5, 0.5),
+        fail_bayes_factor_cutoff: float = 100.0,
+    ) -> TestResult:
+        """Convert a dictionary representation of a test result to a TestResult object."""
         if isinstance(target_proportion, tuple):
             target_lower_bound, target_upper_bound = target_proportion
         else:
@@ -124,7 +217,7 @@ def fuzzy_assert_proportion(
         ), f"There cannot be more events ({observed_numerator}) than opportunities for events ({observed_denominator})"
         assert (
             target_upper_bound >= target_lower_bound
-        ), f"The lower bound of the V&V target ({target_lower_bound}) cannot be greater than the upper bound ({target_upper_bound})"
+        ), f"The lower bound of the V&  V target ({target_lower_bound}) cannot be greater than the upper bound ({target_upper_bound})"
 
         bug_issue_alpha, bug_issue_beta = bug_issue_beta_distribution_parameters
         bug_issue_distribution = scipy.stats.betabinom(
@@ -150,54 +243,20 @@ def fuzzy_assert_proportion(
 
         observed_proportion = observed_numerator / observed_denominator
         reject_null = bayes_factor > fail_bayes_factor_cutoff
-        self.proportion_test_diagnostics.append(
-            {
-                "name": name,
-                "name_addl": name_additional,
-                "observed_proportion": observed_proportion,
-                "observed_numerator": observed_numerator,
-                "observed_denominator": observed_denominator,
-                "target_lower_bound": target_lower_bound,
-                "target_upper_bound": target_upper_bound,
-                "bayes_factor": bayes_factor,
-                "reject_null": reject_null,
-            }
+        return TestResult(
+            name=name,
+            name_additional=name_additional,
+            observed_proportion=observed_proportion,
+            observed_numerator=observed_numerator,
+            observed_denominator=observed_denominator,
+            target_lower_bound=target_lower_bound,
+            target_upper_bound=target_upper_bound,
+            bayes_factor=bayes_factor,
+            reject_null=reject_null,
+            bug_issue_distribution=bug_issue_distribution,
+            no_bug_issue_distribution=no_bug_issue_distribution,
         )
 
-        if reject_null:
-            if observed_proportion < target_lower_bound:
-                raise AssertionError(
-                    f"{name} value {observed_proportion:g} is significantly less than expected, bayes factor = {bayes_factor:g}"
-                )
-            else:
-                raise AssertionError(
-                    f"{name} value {observed_proportion:g} is significantly greater than expected, bayes factor = {bayes_factor:g}"
-                )
-
-        if (
-            target_lower_bound > 0
-            and self._calculate_bayes_factor(
-                0, bug_issue_distribution, no_bug_issue_distribution
-            )
-            < fail_bayes_factor_cutoff
-        ):
-            logger.warning(
-                f"Sample size too small to ever find that the simulation's '{name}' value is less than expected."
-            )
-
-        if target_upper_bound < 1 and (
-            self._calculate_bayes_factor(
-                observed_denominator, bug_issue_distribution, no_bug_issue_distribution
-            )
-            < fail_bayes_factor_cutoff
-        ):
-            logger.warning(
-                f"Sample size too small to ever find that the simulation's '{name}' value is greater than expected."
-            )
-
-        if fail_bayes_factor_cutoff > bayes_factor > inconclusive_bayes_factor_cutoff:
-            logger.warning(f"Bayes factor for '{name}' is not conclusive.")
-
     def _calculate_bayes_factor(
         self,
         numerator: int,

tests/test_fuzzy_checker.py

@@ -12,7 +12,7 @@
 if TYPE_CHECKING:
     from py._path.local import LocalPath
 
-from vivarium_testing_utils.fuzzy_checker import FuzzyChecker
+from vivarium_testing_utils.fuzzy_checker import FuzzyChecker, TestResult
 
 OBSERVED_DENOMINATORS = [100_000, 1_000_000, 10_000_000]
 TARGET_PROPORTION = 0.1
@@ -231,7 +231,7 @@ def test_save_diagnostic_output(tmpdir: LocalPath) -> None:
     assert len(tmpdir.listdir()) == 1
 
     output = pd.read_csv(tmpdir.listdir()[0])
-    assert output.shape == (1, 9)
+    assert output.shape == (1, 11)
 
 
 ###########
@@ -248,3 +248,19 @@ def _make_beta_distribution(lower_bound: float, upper_bound: float) -> rv_contin
         a=mean * concentration,
         b=(1 - mean) * concentration,
     )
+
+
+def test_fuzzy_checker_test_proportion_no_assertion_error() -> None:
+    """Tests that FuzzyChecker.test_proportion returns a TestResult without raising an assertion."""
+
+    test_proportion = FuzzyChecker().test_proportion(
+        name="test_proportion_no_assertion_error",
+        name_additional="unit_test",
+        target_proportion=0.9,
+        observed_numerator=10_008,
+        observed_denominator=100_000,
+        bug_issue_beta_distribution_parameters=(0.5, 0.5),
+        fail_bayes_factor_cutoff=3.0,
+    )
+    assert isinstance(test_proportion, TestResult)
+    assert test_proportion.reject_null is True