batch error logs (#484)

Category: feature
JIRA issue: MIC-5503
Add with checks.

Testing
Added multiple assert statements that would fail using with check and verified that they all appeared in the output logs.

Author: hussain-jafari

.readthedocs.yaml

@@ -20,4 +20,4 @@ python:
         - docs
 
 formats:
-  - pdf
+  - pdf

pyproject.toml

@@ -23,5 +23,6 @@ exclude = [
 [[tool.mypy.overrides]]
 module = [
     "scipy.*",
+    "pytest_check",
 ]
 ignore_missing_imports = true

setup.py

@@ -49,6 +49,7 @@
         "tqdm",
         "layered_config_tree>=2.1.0",
         "loguru",
+        "pytest_check",
         # type stubs
         "pandas-stubs",
         "types-PyYAML",

tests/integration/release/test_release.py

@@ -5,6 +5,7 @@
 import pandas as pd
 import pytest
 from _pytest.fixtures import FixtureRequest
+from pytest_check import check
 from vivarium_testing_utils import FuzzyChecker
 
 from pseudopeople.dataset import Dataset

tests/integration/release/test_runner.py

@@ -8,23 +8,22 @@
 @pytest.mark.parametrize(
     "pytest_args",
     [
-        #([]),
+        ([]),
         (["--dataset", "acs"]),
-        #(["--dataset", "cps"]),
+        (["--dataset", "cps"]),
         # (["--dataset", "acs", "--population", "USA"]),
         # (["--dataset", "acs", "--population", "USA", "--state", "RI"]),
-        #(["--dataset", "wic", "--year", "2015"]),
+        (["--dataset", "wic", "--year", "2015"]),
         # (["--dataset", "wic", "--population", "USA", "--state", "RI", "--year", "2015"]),
     ],
-    #ids=["1", "2", "3", "4"],
-    ids = ['1'],
+    ids=["1", "2", "3", "4"],
 )
 def test_release_tests(
     pytest_args: list[str], release_output_dir: Path, request: pytest.FixtureRequest
 ) -> None:
     os.chdir(Path(__file__).parent)  # need this to access cli options from conftest.py
-    base_cmd = ["pytest", "--release", "test_release.py", f"--output-dir={release_output_dir}"]
-    cmd = base_cmd + pytest_args + ["--population", "USA"]
+    base_cmd = ["pytest", "--release", "test_release.py", "--check-max-tb=1000", f"--output-dir={release_output_dir}"]
+    cmd = base_cmd + pytest_args
 
     # log using job id
     job_id = request.node.callspec.id

tests/utilities.py

@@ -8,6 +8,7 @@
 import numpy as np
 import numpy.typing as npt
 import pandas as pd
+from pytest_check import check
 from vivarium_testing_utils import FuzzyChecker
 
 from pseudopeople.configuration import Keys, get_configuration
@@ -34,7 +35,8 @@ def run_column_noising_tests(
 
         # Check that originally missing data remained missing
         originally_missing_idx = check_original.index[check_original[col.name].isna()]
-        assert check_noised.loc[originally_missing_idx, col.name].isna().all()
+        with check:
+            assert check_noised.loc[originally_missing_idx, col.name].isna().all()
 
         # Check for noising where applicable
         to_compare_idx = shared_idx.difference(originally_missing_idx)
@@ -43,7 +45,8 @@ def run_column_noising_tests(
                 check_original.loc[to_compare_idx, col.name].values
                 != check_noised.loc[to_compare_idx, col.name].values
             )
-            assert different_check.any()
+            with check:
+                assert different_check.any()
 
             noise_level = different_check.sum()
 
@@ -64,7 +67,8 @@ def run_column_noising_tests(
                 == check_noised.loc[to_compare_idx, col.name].values
             )
 
-            assert same_check.all()
+            with check:
+                assert same_check.all()
 
 
 def run_omit_row_or_do_not_respond_tests(
@@ -87,15 +91,20 @@ def run_omit_row_or_do_not_respond_tests(
     ]:
         # Census and household surveys have do_not_respond and omit_row.
         # For all other datasets they are mutually exclusive
-        assert len(noise_types) == 2
+        with check:
+            assert len(noise_types) == 2
     else:
-        assert len(noise_types) < 2
+        with check:
+            assert len(noise_types) < 2
     if not noise_types:  # Check that there are no missing indexes
-        assert noised_data.index.symmetric_difference(original_data.index).empty
+        with check:
+            assert noised_data.index.symmetric_difference(original_data.index).empty
     else:  # Check that there are some omissions
         # TODO: assert levels are as expected
-        assert noised_data.index.difference(original_data.index).empty
-        assert not original_data.index.difference(noised_data.index).empty
+        with check:
+            assert noised_data.index.difference(original_data.index).empty
+        with check:
+            assert not original_data.index.difference(noised_data.index).empty
 
 
 def validate_column_noise_level(
@@ -158,7 +167,8 @@ def validate_column_noise_level(
                     [1 - p for p in token_probability]
                 )
             else:
-                assert isinstance(tokens_per_string, pd.Series)
+                with check:
+                    assert isinstance(tokens_per_string, pd.Series)
                 avg_probability_any_token_noised = (
                     1 - (1 - token_probability) ** tokens_per_string
                 ).mean()