Add the facility active proportion field (#71)

## Nate's Summary
Added the new field, tests, and docstrings. I think I got it into all
the necessary places, but it's been a few months since I worked with
this repo so I may have forgotten something.

## Copilot's Summary
This pull request introduces a new parameter,
`facility_active_proportion`, to the EpiNow2 configuration generation
pipeline. This parameter represents the minimum proportion of days a
facility must be active during the modeling period and is integrated
across multiple functions, constants, and tests to ensure proper
handling and validation.

### Core Feature Addition:
* Introduced `facility_active_proportion` as a new argument to
configuration generation functions (`generate_config`,
`generate_rerun_config`, `generate_backfill_config`, and
`generate_task_configs`). It is validated to ensure it is a float
between 0 and 1, with a default value of 1.0.
[[1]](diffhunk://#diff-d56536f6759432dcf08d9f3961d8344b4e7014d136f4c4450c4069e829e86926R44)
[[2]](diffhunk://#diff-d56536f6759432dcf08d9f3961d8344b4e7014d136f4c4450c4069e829e86926R164)
[[3]](diffhunk://#diff-d56536f6759432dcf08d9f3961d8344b4e7014d136f4c4450c4069e829e86926R348)
[[4]](diffhunk://#diff-ab59143b1ba343d18c0ed85e6e50cacaa76d64fad93c142b8f349b5e61db04f8R348)

### Configuration and Constants Updates:
* Added `facility_active_proportion` to the default configuration
constants and included it in the list of required parameters for task
generation.
[[1]](diffhunk://#diff-1e19d89684d25107f8db5de423069387a48db2ce4ea6bc4421388b450e8c3938R16)
[[2]](diffhunk://#diff-1e19d89684d25107f8db5de423069387a48db2ce4ea6bc4421388b450e8c3938R97)
* Updated the `extract_user_args` function to parse
`facility_active_proportion` from environment variables, with error
handling for invalid formats.
[[1]](diffhunk://#diff-ab59143b1ba343d18c0ed85e6e50cacaa76d64fad93c142b8f349b5e61db04f8R27-R39)
[[2]](diffhunk://#diff-ab59143b1ba343d18c0ed85e6e50cacaa76d64fad93c142b8f349b5e61db04f8R97)

### Validation Enhancements:
* Extended the `validate_args` function to include validation logic for
`facility_active_proportion`, ensuring it is a valid float within the
acceptable range.
[[1]](diffhunk://#diff-ab59143b1ba343d18c0ed85e6e50cacaa76d64fad93c142b8f349b5e61db04f8R204)
[[2]](diffhunk://#diff-ab59143b1ba343d18c0ed85e6e50cacaa76d64fad93c142b8f349b5e61db04f8R278-R287)

### Test Coverage:
* Updated existing test cases and added new assertions to verify that
`facility_active_proportion` is correctly passed, validated, and
included in generated configurations.
[[1]](diffhunk://#diff-d5d888969fd8e06ca26b226838d662a18b517912f4df62b41e87e8f8a2e1d9ffR50)
[[2]](diffhunk://#diff-d5d888969fd8e06ca26b226838d662a18b517912f4df62b41e87e8f8a2e1d9ffR286)
[[3]](diffhunk://#diff-8c80c5198a3e12b29443be1e74e33e55e5bd044c05c5d81c6ac2e5cf1acb6e2eR49-R53)
* Added tests to ensure the default value of
`facility_active_proportion` is correctly applied when not explicitly
provided.
[[1]](diffhunk://#diff-8c80c5198a3e12b29443be1e74e33e55e5bd044c05c5d81c6ac2e5cf1acb6e2eR49-R53)
[[2]](diffhunk://#diff-8c80c5198a3e12b29443be1e74e33e55e5bd044c05c5d81c6ac2e5cf1acb6e2eR83-R87)

---------

Co-authored-by: Adam Howes <adamthowes@gmail.com>

Author: natemcintosh

pipelines/epinow2/generate_config.py

@@ -29,4 +29,5 @@
         output_container=user_args["output_container"],
         task_exclusions=user_args.get("task_exclusions"),
         exclusions=user_args.get("exclusions"),
+        facility_active_proportion=user_args["facility_active_proportion"],
     )

pipelines/epinow2/generate_rerun_config.py

@@ -33,4 +33,5 @@
         as_of_date=user_args["as_of_date"],
         output_container=user_args["output_container"],
         data_exclusions_path=data_exclusions_path,
+        facility_active_proportion=user_args["facility_active_proportion"],
     )

src/cfa_config_generator/utils/epinow2/constants.py

@@ -13,6 +13,7 @@
     "config_version": "1.0",
     "quantile_width": [0.5, 0.95],
     "model": "EpiNow2",
+    "facility_active_proportion": 1.0,
 }
 
 all_states = (
@@ -93,6 +94,7 @@
     "exclusions",
     "quantile_width",
     "model",
+    "facility_active_proportion",
 )
 
 sample_task = {
@@ -139,4 +141,5 @@
     "config_version": "1.0",
     "quantile_width": [0.5, 0.95],
     "model": "EpiNow2",
+    "facility_active_proportion": 1.0,
 }

src/cfa_config_generator/utils/epinow2/driver_functions.py

@@ -41,6 +41,7 @@ def generate_config(
     job_id: str,
     as_of_date: str,
     output_container: str,
+    facility_active_proportion: float,
     task_exclusions: str | None = None,
     exclusions: dict | None = None,
 ):
@@ -77,6 +78,9 @@ def generate_config(
     exclusions: dict | None
         Dictionary with keys 'path' and 'blob_storage_container' for the exclusions file.
         If provided, this will be used to generate the task exclusions string.
+    facility_active_proportion: float
+        Minimum proportion of days a facility must be active during the modeling period.
+        Must be a number between 0 and 1 (inclusive).
 
     Returns
     -------
@@ -108,6 +112,7 @@ def generate_config(
         output_container=output_container,
         task_exclusions=task_exclusions,
         exclusions=exclusions,
+        facility_active_proportion=facility_active_proportion,
     )
 
     # Generate task-specific configs
@@ -156,6 +161,7 @@ def generate_rerun_config(
     job_id: str,
     as_of_date: str,
     output_container: str,
+    facility_active_proportion: float,
     data_exclusions_path: str | None = None,
 ):
     """
@@ -193,6 +199,9 @@ def generate_rerun_config(
         Path to the data exclusion CSV file. If in Blob, use form
         `az://<container-name>/<path>`. Defaults to
         `az://nssp-etl/outliers-v2/<report_date>.csv` if None or empty.
+    facility_active_proportion: float
+        Minimum proportion of days a facility must be active during the modeling period.
+        Must be a number between 0 and 1.
     Returns
     -------
     None
@@ -293,6 +302,7 @@ def generate_rerun_config(
         output_container=output_container,
         task_exclusions=task_excl_str,
         exclusions=excl_field,
+        facility_active_proportion=facility_active_proportion,
     )
 
     # Generate task-specific configs
@@ -335,6 +345,7 @@ def generate_backfill_config(
     backfill_name: str,
     as_of_dates: list[str],
     output_container: str,
+    facility_active_proportion: float,
     task_exclusions: str | None = None,
 ) -> list[str]:
     """
@@ -388,6 +399,9 @@ def generate_backfill_config(
         Blob storage container to store output.
     task_exclusions: str | None
         Comma separated state:disease pair to exclude from model run.
+    facility_active_proportion: float
+        Minimum proportion of days a facility must be active during the modeling period.
+        Must be a number between 0 and 1.
 
     Returns
     -------
@@ -475,6 +489,7 @@ def generate_backfill_config(
             output_container=output_container,
             task_exclusions=task_exclusions,
             exclusions=exclusions_dict.get(rep_date, None),
+            facility_active_proportion=facility_active_proportion,
         )
         logger.info(
             f"Successfully generated config for {job_id} with report date {rep_date.isoformat()}"

src/cfa_config_generator/utils/epinow2/functions.py

@@ -24,6 +24,19 @@ def extract_user_args(as_of_date: str) -> dict[str, Any]:
     state = os.getenv("state") or "all"
     disease = os.getenv("disease") or "all"
 
+    # Handle facility_active_proportion
+    facility_active_proportion_str = os.getenv("facility_active_proportion")
+    try:
+        facility_active_proportion = (
+            float(facility_active_proportion_str)
+            if facility_active_proportion_str
+            else 1.0
+        )
+    except ValueError:
+        raise ValueError(
+            f"Invalid facility_active_proportion format: {facility_active_proportion_str}. Must be a float."
+        )
+
     # Handle report_date
     report_date_str = os.getenv("report_date")
     try:
@@ -81,6 +94,7 @@ def extract_user_args(as_of_date: str) -> dict[str, Any]:
         "job_id": job_id,
         "as_of_date": as_of_date,
         "output_container": output_container,
+        "facility_active_proportion": facility_active_proportion,
     }
 
 
@@ -187,6 +201,7 @@ def validate_args(
     job_id: str,
     as_of_date: str,
     output_container: str,
+    facility_active_proportion: float,
     task_exclusions: str | None = None,
     exclusions: dict | None = None,
 ) -> dict:
@@ -207,6 +222,8 @@ def validate_args(
         task_exclusions: comma separated state:disease pairs to exclude
         exclusions: A dictionary with `path` and `blob_storage_container` keys
         to specify the path to the exclusions file and its container.
+        facility_active_proportion: Minimum proportion of days a facility must be active
+            during the modeling period. Must be a number between 0 and 1.
     Returns:
         A dictionary of sanitized arguments.
     """
@@ -258,6 +275,16 @@ def validate_args(
             f"as_of_date must be a string. Got {type(as_of_date)} instead."
         )
 
+    # Check that the facility_active_proportion is a number between 0 and 1
+    if not isinstance(facility_active_proportion, (int, float)):
+        raise ValueError(
+            f"facility_active_proportion must be a number. Got {type(facility_active_proportion)} instead."
+        )
+    if facility_active_proportion < 0 or facility_active_proportion > 1:
+        raise ValueError(
+            f"facility_active_proportion must be between 0 and 1. Got {facility_active_proportion} instead."
+        )
+
     args_dict["reference_dates"] = reference_dates
     args_dict["report_date"] = report_date
     args_dict["data_path"] = data_path
@@ -267,6 +294,7 @@ def validate_args(
     args_dict["as_of_date"] = as_of_date
     args_dict["exclusions"] = exclusions
     args_dict["output_container"] = output_container
+    args_dict["facility_active_proportion"] = facility_active_proportion
     return args_dict
 
 
@@ -317,6 +345,7 @@ def generate_task_configs(
     production_date: date,
     job_id: str,
     output_container: str,
+    facility_active_proportion: float,
     task_exclusions: dict[str, list[str]] | None = None,
     exclusions: str | None = None,
 ) -> tuple[list[dict], str]:
@@ -336,6 +365,9 @@ def generate_task_configs(
         output_container: Azure container for output
         task_exclusions: dictionary of state:disease pairs to exclude
         exclusions: a path to exclusions csv
+        facility_active_proportion: Minimum proportion of days a facility must be active
+            during the modeling period. Must be a number between 0 and 1.
+
     Returns:
         A list of configuration objects and the job_id.
     """
@@ -347,6 +379,7 @@ def generate_task_configs(
                 **shared_params,
                 "job_id": job_id,
                 "task_id": generate_task_id(state=s, disease=d),
+                "facility_active_proportion": facility_active_proportion,
                 "exclusions": exclusions or {"path": None},
                 "min_reference_date": min(reference_dates).isoformat(),
                 "max_reference_date": max(reference_dates).isoformat(),

tests/test_args.py

@@ -47,6 +47,7 @@ def test_extract_user_args(monkeypatch):
         "job_id": generate_default_job_id(as_of_date=as_of_date),
         "as_of_date": as_of_date,
         "output_container": "test-container",
+        "facility_active_proportion": 1.0,
     }
 
     extracted_args = extract_user_args(as_of_date=as_of_date)
@@ -68,7 +69,6 @@ def test_validate_args_default():
     validated_args = validate_args(
         state="all",
         disease="all",
-        exclusions=None,
         report_date=report_date,
         production_date=production_date,
         reference_dates=[min_reference_date, max_reference_date],
@@ -77,6 +77,8 @@ def test_validate_args_default():
         job_id="test-job-id",
         as_of_date=as_of_date,
         output_container="test-container",
+        facility_active_proportion=1.0,
+        exclusions=None,
     )
     assert validated_args == {
         "state": list(nssp_valid_states),
@@ -90,6 +92,7 @@ def test_validate_args_default():
         "job_id": "test-job-id",
         "as_of_date": as_of_date,
         "output_container": "test-container",
+        "facility_active_proportion": 1.0,
     }
 
 
@@ -108,6 +111,7 @@ def test_invalid_state():
             job_id="test-job-id",
             as_of_date=generate_timestamp(),
             output_container="test-container",
+            facility_active_proportion=1.0,
         )
 
 
@@ -127,6 +131,7 @@ def test_invalid_disease():
             job_id="test-job-id",
             as_of_date=generate_timestamp(),
             output_container="test-container",
+            facility_active_proportion=1.0,
         )
 
 
@@ -152,6 +157,7 @@ def test_invalid_reference_date_logic():
             job_id="test-job-id",
             as_of_date=generate_timestamp(),
             output_container="test-container",
+            facility_active_proportion=1.0,
         )
 
 
@@ -173,8 +179,9 @@ def test_invalid_disease_exclusion():
             production_date=today,
             job_id="test-job-id",
             as_of_date=as_of_date,
-            task_exclusions=task_exclusions,
             output_container="test-container",
+            facility_active_proportion=1.0,
+            task_exclusions=task_exclusions,
         )
 
 
@@ -276,4 +283,5 @@ def test_generate_backfill_bad_lists():
             backfill_name=backfill_name,
             as_of_dates=as_of_dates,
             output_container=output_container,
+            facility_active_proportion=1.0,
         )

tests/test_config_generation.py

@@ -34,6 +34,7 @@ def test_default_config_set():
         job_id="test-job-id",
         as_of_date=as_of_date,
         output_container="test-container",
+        facility_active_proportion=1.0,
         exclusions=None,
         task_exclusions=None,
     )
@@ -45,6 +46,11 @@ def test_default_config_set():
     )
     assert len(task_configs) == total_tasks_expected
 
+    # Test that facility_active_proportion is present with default value
+    for config in task_configs:
+        assert "facility_active_proportion" in config
+        assert config["facility_active_proportion"] == 1.0
+
 
 def test_single_geo_disease_set():
     """Tests that a single geography-disease combination returns a single task."""
@@ -64,6 +70,7 @@ def test_single_geo_disease_set():
         job_id="test-job-id",
         as_of_date=as_of_date,
         output_container="test-container",
+        facility_active_proportion=1.0,
         exclusions=None,
         task_exclusions=None,
     )
@@ -73,6 +80,11 @@ def test_single_geo_disease_set():
     total_tasks_expected = 1
     assert len(task_configs) == total_tasks_expected
 
+    # Test that facility_active_proportion is present with default value
+    config = task_configs[0]
+    assert "facility_active_proportion" in config
+    assert config["facility_active_proportion"] == 1.0
+
 
 @pytest.mark.parametrize(
     "report_dates, time_span, expected_ref_dates",
@@ -133,3 +145,26 @@ def test_gen_ref_date_tuples(report_dates, time_span, expected_ref_dates):
     """
     got = generate_ref_date_tuples(report_dates=report_dates, delta=time_span)
     assert expected_ref_dates == got
+
+
+def test_facility_active_proportion_in_shared_params():
+    """Test that facility_active_proportion is included in shared_params with correct default value."""
+    from cfa_config_generator.utils.epinow2.constants import shared_params
+
+    assert "facility_active_proportion" in shared_params
+    assert shared_params["facility_active_proportion"] == 1.0
+
+
+def test_facility_active_proportion_modifiable():
+    """Test that facility_active_proportion is included in modifiable_params for CLI usage."""
+    from cfa_config_generator.utils.epinow2.constants import modifiable_params
+
+    assert "facility_active_proportion" in modifiable_params
+
+
+def test_sample_task_facility_active_proportion():
+    """Test that sample_task includes facility_active_proportion with correct default value."""
+    from cfa_config_generator.utils.epinow2.constants import sample_task
+
+    assert "facility_active_proportion" in sample_task
+    assert sample_task["facility_active_proportion"] == 1.0