update do not respond fuzzy checker (#500)

Category: bugfix
JIRA issue: MIC-5893

Use a range instead of single value when fuzzy checking do not respond, expanding by 3% on both sides as described here.

Testing
Ran test on acs, cps, and census sample data and acs and cps for full scale data.

Author: hussain-jafari

src/pseudopeople/noise_level.py

@@ -31,7 +31,7 @@ def _get_census_omission_noise_levels(
         .astype(str)
         .map(data_values.DO_NOT_RESPOND_ADDITIVE_PROBABILITY_BY_RACE)
     )
-    ages = pd.Series(np.arange(population["age"].max() + 1))
+    ages = pd.Series(np.arange(population["age"].astype(int).max() + 1))
     for sex in ["Female", "Male"]:
         effect_by_age_bin = data_values.DO_NOT_RESPOND_ADDITIVE_PROBABILITY_BY_SEX_AGE[sex]
         # NOTE: calling pd.cut on a large array with an IntervalIndex is slow,
@@ -44,7 +44,7 @@ def _get_census_omission_noise_levels(
         )
         sex_mask = population["sex"] == sex
         probabilities[sex_mask] += (
-            population[sex_mask]["age"].map(effect_by_age).astype(float)
+            population[sex_mask]["age"].astype(int).map(effect_by_age).astype(float)
         )
     probabilities[probabilities < 0.0] = 0.0
     probabilities[probabilities > 1.0] = 1.0

tests/integration/release/test_release.py

@@ -131,7 +131,8 @@ def test_do_not_respond(
         name="test_do_not_respond",
         observed_numerator=len(original_data) - len(noised_data),
         observed_denominator=len(original_data),
-        target_proportion=expected_noise,
+        # 3% uncertainty on either side
+        target_proportion=(expected_noise * 0.97, expected_noise * 1.03),
         name_additional=f"noised_data",
     )
     assert set(noised_data.columns) == set(original_data.columns)